diff --git a/.agents/skills/blog-authoring/SKILL.md b/.agents/skills/blog-authoring/SKILL.md deleted file mode 100644 index 275ee059c8..0000000000 --- a/.agents/skills/blog-authoring/SKILL.md +++ /dev/null @@ -1,95 +0,0 @@ ---- -name: blog-authoring -description: Use when creating, editing, previewing, or publishing blog posts through the local working-copy flow, especially when the task involves `content/blog`, `npm run blog:*` commands, or `/blog-preview/...` URLs. ---- - -# Neon Blog Authoring - -Default editing workflow: - -1. Work in the site repository. -2. Edit the local working copy under `content/blog/`. -3. Run the local site to visually inspect the result. -4. Publish the snapshot into the configured remote blog-content branch with the provided CLI. - -Do not default to editing the remote source-of-truth repository directly unless the user explicitly asks for it. - -## Required checks - -Before changing content, verify the current state: - -```bash -git branch --show-current -npm run blog:status -``` - -If `content/blog` is missing, initialize it: - -```bash -npm run blog:bootstrap -``` - -If the user explicitly wants to refresh the working copy from remote state: - -```bash -npm run blog:sync -- --force -``` - -## Core workflow - -### 1. Edit locally - -Edit only: - -- `content/blog/posts/*.md` -- `content/blog/authors/data.json` -- `content/blog/categories/data.json` - -Then run: - -```bash -npm run dev -``` - -Use the local app to review: - -- `/blog` -- `/blog/` - -### 2. Publish to a branch in `blog` - -Use the built-in CLI: - -```bash -npm run blog:publish-branch -- --branch -``` - -Defaults: - -- if `--branch` is omitted, the current git branch name is used -- the command pushes the current `content/blog` snapshot into the configured remote content repository -- the command prints preview URLs under `/blog-preview/...` - -### 3. Review branch preview - -Open the generated preview URL, for example: - -```text -/blog-preview/?branch=&secret= -``` - -If the preview returns `404`, treat that as “access denied or missing branch/slug”. Do not assume the route is public. - -## Guardrails - -- Prefer `content/blog` in the current repository as the editing surface. -- Do not overwrite existing `content/blog` automatically; only use `blog:sync -- --force` when the user explicitly wants a refresh. -- Do not push to the main source-of-truth remote accidentally. If you must work in a separate source repo directly, inspect `git remote -v` first and confirm which remote is safe to push to. -- When credentials are missing, limit yourself to local editing and local preview; explain exactly which publish/preview actions are blocked. -- Do not touch unrelated local files such as `src/scripts/compare-sites.js`. - -## References - -Load only what you need: - -- Commands and environment expectations: `references/commands.md` diff --git a/.agents/skills/blog-authoring/references/commands.md b/.agents/skills/blog-authoring/references/commands.md deleted file mode 100644 index 9bdd9ba458..0000000000 --- a/.agents/skills/blog-authoring/references/commands.md +++ /dev/null @@ -1,76 +0,0 @@ -# Commands - -## Blog CLI - -Run these from the site repository: - -```bash -npm run blog:bootstrap -npm run blog:sync -- --force -npm run blog:status -npm run blog:publish-branch -- --branch -npm run dev -npm run build -``` - -Behavior: - -- `blog:bootstrap`: materializes `content/blog` only if it is missing -- `blog:sync -- --force`: replaces the local working copy from the branch source or CDN -- `blog:status`: shows current branch, whether a matching branch exists in the configured `blog` repo, whether local content exists, and whether local content differs from remote -- `blog:publish-branch`: publishes the current `content/blog` snapshot into the configured `blog` repo branch and prints preview URLs - -## Environment needed for publish / branch preview - -Expected in env: - -```bash -BLOG_CDN_URL=... -BLOG_REPO_OWNER=... -BLOG_REPO_NAME=... -BLOG_GITHUB_TOKEN=... -BLOG_PREVIEW_SECRET=... -``` - -Notes: - -- `BLOG_GITHUB_TOKEN` is currently used for both branch fetches and `blog:publish-branch`, so it needs write access to the configured remote content repo. -- Without the repo credentials, local editing still works, but publishing and branch-aware remote fetches do not. - -## Useful git checks - -Current branch in `website`: - -```bash -git branch --show-current -``` - -Remotes in a separate source repo: - -```bash -git remote -v -``` - -Use this before pushing directly from a separate source repo. Prefer a personal/fork remote over the source-of-truth remote unless the user explicitly asks otherwise. - -## Preview routes - -Examples: - -```text -/blog-preview/?branch=&secret= -``` - -Expected behavior: - -- valid branch + valid secret -> preview page -- missing/invalid secret -> `404` -- missing branch or missing slug -> `404` - -## Editing targets - -Only edit these blog-content working copy files unless the user explicitly asks otherwise: - -- `content/blog/posts/*.md` -- `content/blog/authors/data.json` -- `content/blog/categories/data.json` diff --git a/.env.example b/.env.example index 3bf05416b5..baacfc7776 100644 --- a/.env.example +++ b/.env.example @@ -33,8 +33,8 @@ NEON_BRANCHING_DEMO_QSTASH_TOKEN= QSTASH_CURRENT_SIGNING_KEY= QSTASH_NEXT_SIGNING_KEY= -BLOG_CDN_URL= +BLOG_PREVIEW_ENABLED=false +BLOG_PREVIEW_SECRET= BLOG_REPO_OWNER= BLOG_REPO_NAME= BLOG_GITHUB_TOKEN= -BLOG_PREVIEW_SECRET= diff --git a/.github/workflows/sync-fork-main.yml b/.github/workflows/sync-fork-main.yml new file mode 100644 index 0000000000..605e37382e --- /dev/null +++ b/.github/workflows/sync-fork-main.yml @@ -0,0 +1,29 @@ +name: Sync Main + +on: + schedule: + - cron: '*/30 * * * *' + workflow_dispatch: + +permissions: + contents: write + +jobs: + sync-main: + if: ${{ vars.ENABLE_FORK_MAIN_SYNC == 'true' }} + runs-on: ubuntu-latest + env: + UPSTREAM_REPOSITORY: ${{ vars.FORK_MAIN_SYNC_UPSTREAM_REPOSITORY || 'neondatabase/website' }} + steps: + - name: Checkout fork + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 + with: + fetch-depth: 0 + + - name: Sync main from upstream + run: | + git remote add upstream "https://github.com/${UPSTREAM_REPOSITORY}.git" + git fetch upstream main + git checkout main + git reset --hard upstream/main + git push origin main --force diff --git a/.gitignore b/.gitignore index 5f3a0e0723..54057d7968 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,3 @@ package-lock.json # Evals directory (test validation for documentation) .evals/ - -# Blog content gets imported automatically -content/blog diff --git a/content/blog/authors/data.json b/content/blog/authors/data.json new file mode 100644 index 0000000000..1e01ec726f --- /dev/null +++ b/content/blog/authors/data.json @@ -0,0 +1,786 @@ +{ + "aaikansh-agrawal": { + "name": "Aaikansh Agrawal", + "role": "Growth Marketer @ DronaHQ", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/aaikansh-agrawal.jpg", + "photoAlt": "Aaikansh Agrawal" + }, + "adi-griever": { + "name": "Adi Griever", + "role": "Back End Golang Developer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/adi-griever.jpg", + "photoAlt": "Adi Griever" + }, + "akshat-agrawal": { + "name": "Akshat Agrawal", + "role": "Product at Codeium", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/akshat-agrawal.jpg", + "photoAlt": "Akshat Agrawal" + }, + "anastasia-lubennikova": { + "name": "Anastasia Lubennikova", + "role": "Software Engineer", + "url": "https://twitter.com/alubennikova", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/anastasia-lubennikova.jpg", + "photoAlt": "Anastasia Lubennikova" + }, + "andre-landgraf": { + "name": "Andre Landgraf", + "role": "Senior Developer Advocate", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/andre-landgraf.jpg", + "photoAlt": "Andre Landgraf" + }, + "andreas-scherbaum": { + "name": "Andreas Scherbaum", + "role": "Postgres Performance Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/andreas-scherbaum.jpg", + "photoAlt": "Andreas Scherbaum" + }, + "andrew-hamilton": { + "name": "Andrew Hamilton", + "role": "Chief Technology Officer at Layer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/andrew-hamilton.jpg", + "photoAlt": "Andrew Hamilton" + }, + "andrew-tate": { + "name": "Andrew Tate", + "role": "Software Developer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/andrew-tate.jpg", + "photoAlt": "Andrew Tate" + }, + "andy-claremont": { + "name": "Andy Claremont", + "role": "Community & Ecosystem @ Glide", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/andy-claremont.jpg", + "photoAlt": "Andy Claremont" + }, + "andy-hattemer": { + "name": "Andy Hattemer", + "role": "Member of Product Staff", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/andy-hattemer.jpg", + "photoAlt": "Andy Hattemer" + }, + "anna-stepanyan": { + "name": "Anna Stepanyan", + "role": "Head of Product", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/anna-stepanyan.jpg", + "photoAlt": "Anna Stepanyan" + }, + "arjun-rajeswaran": { + "name": "Arjun Rajeswaran", + "role": "VP Revenue", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/arjun-rajeswaran.jpg", + "photoAlt": "Arjun Rajeswaran" + }, + "arseni-kravchenko": { + "name": "Arseni Kravchenko", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/arseni-kravchenko.jpg", + "photoAlt": "Arseni Kravchenko" + }, + "atila": { + "name": "Atila Fassina", + "role": "Full Stack Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/atila.jpg", + "photoAlt": "Atila" + }, + "atli-cervantes": { + "name": "Atli Cervantes", + "role": "Head of PLG Sales", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/atli-cervantes.jpg", + "photoAlt": "Atli Cervantes" + }, + "barry-grenon": { + "name": "Barry Grenon", + "role": "Technical Writer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/barry-grenon.jpg", + "photoAlt": "Barry Grenon" + }, + "ben-hagan": { + "name": "Ben Hagan", + "role": "Product Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/ben-hagan.jpg", + "photoAlt": "Ben Hagan" + }, + "bereket-engida": { + "name": "Bereket Engida", + "role": "Creator of Better Auth", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/bereket-engida.jpg", + "photoAlt": "Bereket Engida" + }, + "bojan-serafimov": { + "name": "Bojan Serafimov", + "role": "Software Engineer", + "url": "https://twitter.com/Bojan93112526", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/bojan-serafimov.jpg", + "photoAlt": "Bojan Serafimov" + }, + "brad-van-vugt": { + "name": "Brad Van Vugt", + "role": "Growth Product Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/brad-van-vugt.jpg", + "photoAlt": "Brad Van Vugt" + }, + "brandon-strittmatter": { + "name": "Brandon Strittmatter", + "role": "Co-Founder & CEO at Outerbase", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/brandon-strittmatter.jpg", + "photoAlt": "Brandon Strittmatter" + }, + "brian-holt": { + "name": "Brian Holt", + "role": "Staff Product Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/brian-holt.jpg", + "photoAlt": "Brian Holt" + }, + "brian-morrison": { + "name": "Brian Morrison", + "role": "Software Developer at Clerk", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/brian-morrison.jpg", + "photoAlt": "Brian Morrison" + }, + "bryan-clark": { + "name": "Bryan Clark", + "role": "VP Product", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/bryan-clark.jpg", + "photoAlt": "Bryan Clark" + }, + "busra-demir": { + "name": "Busra Demir", + "role": "Lead Security Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/busra-demir.jpg", + "photoAlt": "Busra Demir" + }, + "carl-thomas": { + "name": "Carl Thomas", + "role": "Head of Design", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/carl-thomas.jpg", + "photoAlt": "Carl Thomas" + }, + "carlota-soto": { + "name": "Carlota Soto", + "role": "Product Marketing Lead", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/carlota-soto.jpg", + "photoAlt": "Carlota Soto" + }, + "charly-poly": { + "name": "Charly Poly", + "role": "Product Marketing at Inngest", + "url": "https://www.linkedin.com/in/charly-poly/", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/charly-poly.jpg", + "photoAlt": "Charly Poly Avatar" + }, + "christian-schwarz-2": { + "name": "Christian Schwarz", + "role": "Systems Software Engineer", + "url": "https://x.com/problame", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/christian-schwarz-2.jpg", + "photoAlt": "Christian Schwarz" + }, + "daniel-price": { + "name": "Daniel Price", + "role": "Technical Writer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/daniel-price.jpg", + "photoAlt": "Daniel Price" + }, + "david-gomes": { + "name": "David Gomes", + "role": "Engineering Team", + "url": "https://davidgomes.com/", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/david-gomes.jpg", + "photoAlt": "David Gomes" + }, + "david-wein": { + "name": "David Wein", + "role": "Performance Engineering, Databricks", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/david-wein.jpg", + "photoAlt": "David Wein" + }, + "deepesh-genani": { + "name": "Deepesh Genani", + "role": "Software Developer (external)", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/deepesh-genani.jpg", + "photoAlt": "Deepesh Genani" + }, + "dian-m-fay": { + "name": "Dian M Fay", + "role": "Senior Data Architect", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/dian-m-fay.jpg", + "photoAlt": "Dian M Fay" + }, + "dom-eccleston": { + "name": "Dom Eccleston", + "role": "Software Engineer @ Unkey", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/dom-eccleston.jpg", + "photoAlt": "Dom Eccleston" + }, + "eduard-dyckman": { + "name": "Eduard Dyckman", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/eduard-dyckman.jpg", + "photoAlt": "Eduard Dyckman Photo" + }, + "em-sharnoff": { + "name": "Em Sharnoff", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/em-sharnoff.jpg", + "photoAlt": "Em Sharnoff" + }, + "eric-goldman": { + "name": "Eric Goldman", + "role": "Co-founder at Sequin", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/eric-goldman.jpg", + "photoAlt": "Eric Goldman" + }, + "erik-grinaker": { + "name": "Erik Grinaker", + "role": "Backend Developer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/erik-grinaker.jpg", + "photoAlt": "Erik Grinaker" + }, + "evan-shortiss": { + "name": "Evan Shortiss", + "role": "Developer Advocate", + "url": "https://twitter.com/evanshortiss", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/evan-shortiss.jpg", + "photoAlt": "Evan Shortiss' Headshot" + }, + "evgenii-kniazev": { + "name": "Evgenii Kniazev", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/evgenii-kniazev.jpg", + "photoAlt": "Evgenii Kniazev" + }, + "evis-drenova": { + "name": "Evis Drenova", + "role": "CEO/Co-founder @ Neosync", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/evis-drenova.jpg", + "photoAlt": "Evis Drenova" + }, + "fedor-dikarev": { + "name": "Fedor Dikarev", + "role": "Member of Technical Staff", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/fedor-dikarev.jpg", + "photoAlt": "Fedor Dikarev" + }, + "felipe-freitag": { + "name": "Felipe Freitag", + "role": "Flashboard Founder", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/felipe-freitag.jpg", + "photoAlt": "Felipe Freitag" + }, + "george-mackerron": { + "name": "George MacKerron", + "role": "Software Developer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/george-mackerron.jpg", + "photoAlt": "George MacKerron" + }, + "guillaume-rivals": { + "name": "Guillaume Rivals", + "role": "Lead Engineer, Growth", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/guillaume-rivals.jpg", + "photoAlt": "Guillaume Rivals" + }, + "gustavo-salomao": { + "name": "Gustavo Salomão", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/gustavo-salomao.jpg", + "photoAlt": "Gustavo Salomão" + }, + "hans-norheim": { + "name": "Hans Norheim", + "role": "Sr. Staff Software Engineer, Databricks", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/hans-norheim.jpg", + "photoAlt": "Hans Norheim" + }, + "hayla-westhead": { + "name": "Hayla Westhead", + "role": "Lead Legal Counsel", + "url": "https://www.linkedin.com/in/hayla-westhead/", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/hayla-westhead.jpg", + "photoAlt": "Hayla Westhead" + }, + "heikki-linnakangas": { + "name": "Heikki Linnakangas", + "role": "Co-Founder at Neon", + "url": "https://twitter.com/HLinnakangas", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/heikki-linnakangas.jpg", + "photoAlt": "Heikki Linnakangas" + }, + "igor-rekun": { + "name": "Igor Rekun", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/igor-rekun.jpg", + "photoAlt": "Igor Rekun" + }, + "ilya-tkachov": { + "name": "Ilya Tkachov", + "role": "Co-founder & CEO wispbit.com", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/ilya-tkachov.jpg", + "photoAlt": "Ilya Tkachov" + }, + "jacob-prall": { + "name": "Jacob Prall", + "role": "Senior Developer Advocate @ Airbyte", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/jacob-prall.jpg", + "photoAlt": "Jacob Prall" + }, + "jeff-christoffersen": { + "name": "Jeffrey Christoffersen", + "role": "Staff Product Designer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/jeff-christoffersen.jpg", + "photoAlt": "Jeff Christoffersen" + }, + "joe-drumgoole": { + "name": "Joe Drumgoole", + "role": "Head of Developer Relations", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/joe-drumgoole.jpg", + "photoAlt": "Joe Drumgoole" + }, + "john-spray": { + "name": "John Spray", + "role": "Engineering Manager Storage", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/john-spray.jpg", + "photoAlt": "John Spray" + }, + "krishna-b-parab": { + "name": "Krishna B. Parab", + "role": "Kiro Product Marketing, AWS", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/krishna-b-parab.jpg", + "photoAlt": "Krishna B. Parab" + }, + "lachezar-petkov": { + "name": "Lachezar Petkov", + "role": "Senior Product Designer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/lachezar-petkov.jpg", + "photoAlt": "Lachezar Petkov" + }, + "lassi-polonen": { + "name": "Lassi Pölönen", + "role": "Member of Technical Staff", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/lassi-polonen.jpg", + "photoAlt": "Lassi Pölönen" + }, + "luca-cittadini": { + "name": "Luca Cittadini", + "role": "Member of Technical Staff", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/luca-cittadini.jpg", + "photoAlt": "Luca Cittadini" + }, + "luis-neves": { + "name": "Luís Neves", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/luis-neves.jpg", + "photoAlt": "Luís Neves" + }, + "luis-tavares": { + "name": "Luis Tavares", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/luis-tavares.jpg", + "photoAlt": "Luis Tavares" + }, + "mahmoud-abdelwahab": { + "name": "Mahmoud Abdelwahab", + "role": "Developer Advocate", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/mahmoud-abdelwahab.jpg", + "photoAlt": "Mahmoud Abdelwahab" + }, + "marc-dupuis": { + "name": "Marc Dupuis", + "role": "Co-founder at Fabi.ai", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/marc-dupuis.jpg", + "photoAlt": "Marc Dupuis" + }, + "marcus-lowe": { + "name": "Marcus Lowe", + "role": "Co-Founder / CEO, Anything", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/marcus-lowe.jpg", + "photoAlt": "Marcus Lowe" + }, + "mario-cadenas": { + "name": "Mario Cadenas", + "role": "Senior Software Engineer", + "url": "https://www.mariocadenas.dev/", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/mario-cadenas.jpg", + "photoAlt": "Mario Cadenas" + }, + "marta-paes": { + "name": "Marta Paes", + "role": "Head of Developer Experience, Materialize", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/marta-paes.jpg", + "photoAlt": "Marta Paes" + }, + "matt-palmer": { + "name": "Matt Palmer", + "role": "Developer Advocate at Replit", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/matt-palmer.jpg", + "photoAlt": "Matt Palmer" + }, + "matt-sherman": { + "name": "Matt Sherman", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/matt-sherman.jpg", + "photoAlt": "Matt Sherman" + }, + "mervin-praison": { + "name": "Mervin Praison", + "role": "AI Engineer", + "url": "https://www.youtube.com/@MervinPraison", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/mervin-praison.jpg", + "photoAlt": "Mervin Praison" + }, + "mihai-bojin": { + "name": "Mihai Bojin", + "role": "Director of Engineering", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/mihai-bojin.jpg", + "photoAlt": "Mihai Bojin" + }, + "mikael-lirbank": { + "name": "Mikael Lirbank", + "role": "Software developer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/mikael-lirbank.jpg", + "photoAlt": "Mikael Lirbank" + }, + "mike-jerome": { + "name": "Mike Jerome", + "role": "Product Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/mike-jerome.jpg", + "photoAlt": "Mike Jerome" + }, + "monica-steinke": { + "name": "Monica Steinke", + "role": "Technical Product Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/monica-steinke.jpg", + "photoAlt": "Monica Steinke" + }, + "nikita-shamgunov": { + "name": "Nikita Shamgunov", + "role": "CEO at Neon", + "url": "https://twitter.com/nikitabase", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/nikita-shamgunov.jpg", + "photoAlt": "Nikita Shamgunov" + }, + "nikolai-rekubratskii": { + "name": "Nikolai Rekubratskii", + "role": "Data Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/nikolai-rekubratskii.jpg", + "photoAlt": "Nikolai Rekubratskii" + }, + "noa-rogoszinski": { + "name": "Noa Rogoszinski", + "role": "DevRel Engineer at Ariga", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/noa-rogoszinski.jpg", + "photoAlt": "Noa Rogoszinski" + }, + "paul-scanlon": { + "name": "Paul Scanlon", + "role": "Technical Product Marketing Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/paul-scanlon.jpg", + "photoAlt": "Paul Scanlon" + }, + "pedro-figueiredo": { + "name": "Pedro Figueiredo", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/pedro-figueiredo.jpg", + "photoAlt": "Pedro Figueiredo" + }, + "peter-bendel": { + "name": "Peter Bendel", + "role": "Postgres Performance Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/peter-bendel.jpg", + "photoAlt": "Peter Bendel" + }, + "phil-sheffield": { + "name": "Phil Sheffield", + "role": "Member of Product Staff", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/phil-sheffield.jpg", + "photoAlt": "Phil Sheffield" + }, + "ran-sheinberg": { + "name": "Ran Sheinberg", + "role": "Co-Founder and CPO of xpander.ai", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/ran-sheinberg.jpg", + "photoAlt": "Ran Sheinberg" + }, + "raouf-chebri": { + "name": "Raouf Chebri", + "role": "Senior Developer Advocate", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/raouf-chebri.jpg", + "photoAlt": "raouf-chebri" + }, + "rishi-raj-jain": { + "name": "Rishi Raj Jain", + "role": "Solutions Engineer", + "url": "https://twitter.com/rishi_raj_jain_", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/rishi-raj-jain.jpg", + "photoAlt": "Rishi Raj Jain" + }, + "rodney-sherwin-shibu": { + "name": "Rodney Sherwin Shibu", + "role": "Member of Technical Staff", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/rodney-sherwin-shibu.jpg", + "photoAlt": "Rodney Sherwin Shibu" + }, + "roman-zaynetdinov": { + "name": "Roman Zaynetdinov", + "role": "Backend Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/roman-zaynetdinov.jpg", + "photoAlt": "Roman Zaynetdinov" + }, + "rory-de-zoete": { + "name": "Rory de Zoete", + "role": "Lead Site Reliability Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/rory-de-zoete.jpg", + "photoAlt": "Rory de Zoete" + }, + "rotem-tamir": { + "name": "Rotem Tamir", + "role": "Сo-founder & CTO at Ariga", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/rotem-tamir.jpg", + "photoAlt": "Rotem Tamir" + }, + "ruslan-talpa": { + "name": "Ruslan Talpa", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/ruslan-talpa.jpg", + "photoAlt": "Ruslan Talpa" + }, + "russ-dias": { + "name": "Russ Dias", + "role": "Full Stack Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/russ-dias.jpg", + "photoAlt": "Russ Dias" + }, + "ryan-vogel": { + "name": "Ryan Vogel", + "role": "Developer Advocate", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/ryan-vogel.jpg", + "photoAlt": "Ryan Vogel" + }, + "sai-srirampur": { + "name": "Sai Srirampur", + "role": "Product at PeerDB/ClickHouse", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/sai-srirampur.jpg", + "photoAlt": "Sai Srirampur" + }, + "sam-aybar": { + "name": "Sam Aybar", + "role": "Developer Advocate, PolyScale.ai", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/sam-aybar.jpg", + "photoAlt": "Sam Aybar" + }, + "sam-harrison": { + "name": "Sam Harrison", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/sam-harrison.jpg", + "photoAlt": "Sam Harrison photo" + }, + "sam-kleinman": { + "name": "Sam Kleinman", + "role": "Backend Engineer", + "url": "https://github.com/tychoish", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/sam-kleinman.jpg", + "photoAlt": "Sam Kleinman" + }, + "savannah-longoria": { + "name": "Savannah Longoria", + "role": "Developer Advocate", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/savannah-longoria.jpg", + "photoAlt": "Savannah Longoria" + }, + "scott-cate": { + "name": "Scott Cate", + "role": "301.Pro Founder", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/scott-cate.jpg", + "photoAlt": "Scott Cate" + }, + "sheldon-niu": { + "name": "Sheldon Niu", + "role": "CEO at AskYourDatabase", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/sheldon-niu.jpg", + "photoAlt": "Sheldon Niu" + }, + "shridhar-deshmukh": { + "name": "Shridhar Deshmukh", + "role": "Software Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/shridhar-deshmukh.jpg", + "photoAlt": "Shridhar Deshmukh" + }, + "stas-kelvich": { + "name": "Stas Kelvich", + "role": "Co-Founder at Neon", + "url": "https://twitter.com/kelvich", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/stas-kelvich.jpg", + "photoAlt": "Stas Kelvich" + }, + "stephen-siegert": { + "name": "Stephen Siegert", + "role": "Developer Advocate", + "url": "https://twitter.com/siegerts", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/stephen-siegert.jpg", + "photoAlt": "Stephen Siegert" + }, + "taraneh-dohmer": { + "name": "Taraneh Dohmer", + "role": "Developer Marketing Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/taraneh-dohmer.jpg", + "photoAlt": "Taraneh Dohmer" + }, + "thorsten-ries": { + "name": "Thorsten Rieß", + "role": "Sr. Software Engineer at traconiq", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/thorsten-ries.jpg", + "photoAlt": "Thorsten Rieß" + }, + "tony-holdstock-brown": { + "name": "Tony Holdstock-Brown", + "role": "CEO of Inngest", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/tony-holdstock-brown.jpg", + "photoAlt": "Tony Holdstock-Brown Photo" + }, + "tristan-partin": { + "name": "Tristan Partin", + "role": "Software Developer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/tristan-partin.jpg", + "photoAlt": "Tristan Partin" + }, + "tyler-rockwood": { + "name": "Tyler Rockwood", + "role": "Software Engineer @ Redpanda", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/tyler-rockwood.jpg", + "photoAlt": "Tyler Rockwood" + }, + "victor-ditadi": { + "name": "Victor Ditadi", + "role": "Senior Software Engineer", + "url": "https://www.victorditadi.com/", + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/victor-ditadi.jpg", + "photoAlt": "Victor Ditadi" + }, + "vlad-lazar": { + "name": "Vlad Lazar", + "role": "Storage Engineering, Databricks", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/vlad-lazar.jpg", + "photoAlt": "Vlad Lazar" + }, + "will-adams": { + "name": "Will Adams", + "role": "Senior Customer Success Manager", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/will-adams.jpg", + "photoAlt": "Will Adams" + }, + "will-chen": { + "name": "Will Chen", + "role": "Dyad Founder", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/will-chen.jpg", + "photoAlt": "Will Chen" + }, + "yan-cui": { + "name": "Yan Cui", + "role": "Independent Consultant", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/yan-cui.jpg", + "photoAlt": "Yan Cui" + }, + "philip-olson": { + "name": "Philip Olson", + "role": "Documentation Engineer", + "url": null, + "photo": "https://cdn.neonapi.io/public/images/pages/blog/authors/philip-olson.jpg", + "photoAlt": "Philip Olson" + } +} diff --git a/content/blog/categories/data.json b/content/blog/categories/data.json new file mode 100644 index 0000000000..3e89bfed41 --- /dev/null +++ b/content/blog/categories/data.json @@ -0,0 +1,42 @@ +[ + { + "slug": "ai", + "name": "AI" + }, + { + "slug": "app-platform", + "name": "App Platform" + }, + { + "slug": "case-study", + "name": "Case Studies" + }, + { + "slug": "community", + "name": "Community" + }, + { + "slug": "company", + "name": "Company" + }, + { + "slug": "engineering", + "name": "Engineering" + }, + { + "slug": "postgres", + "name": "Postgres" + }, + { + "slug": "product", + "name": "Product" + }, + { + "slug": "uncategorized", + "name": "Uncategorized" + }, + { + "slug": "workflows", + "name": "Workflows" + } +] diff --git a/content/blog/posts/1-year-of-autoscaling-postgres-at-neon.md b/content/blog/posts/1-year-of-autoscaling-postgres-at-neon.md new file mode 100644 index 0000000000..2afeadf495 --- /dev/null +++ b/content/blog/posts/1-year-of-autoscaling-postgres-at-neon.md @@ -0,0 +1,135 @@ +--- +title: '1 Year of Autoscaling Postgres: How it’s going, and what’s next' +description: >- + Neon can autoscale your Postgres instance without dropping connections or + interrupting your queries. Today, we want to share how it’s going and give you + a peek at what’s coming next for autoscaling. +excerpt: >- + Neon introduced autoscaling for serverless Postgres to the world over a year + ago, enabling your applications to handle peak demand without incurring peak + infrastructure costs 24/7. Our autoscaling feature performs zero-downtime + vertical scaling of your Postgres instance, provisio... +date: '2024-04-15T15:55:23' +updatedOn: '2024-04-18T13:21:24' +category: engineering +categories: + - engineering +authors: + - em-sharnoff + - evan-shortiss +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/1-year-of-autoscaling-postgres-at-neon/cover.png + alt: null +isFeatured: false +seo: + title: '1 Year of Autoscaling Postgres: How it’s going, and what’s next' + description: >- + Neon autoscales Postgres without dropping connections or interrupting your + queries. Let's talk about what's coming next for autoscaling! + keywords: [] + noindex: false + ogTitle: '1 Year of Autoscaling Postgres: How it’s going, and what’s next' + ogDescription: >- + Neon autoscales Postgres without dropping connections or interrupting your + queries. Let's talk about what's coming next for autoscaling! + image: >- + https://cdn.neonapi.io/public/images/pages/blog/1-year-of-autoscaling-postgres-at-neon/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/1-year-of-autoscaling-postgres-at-neon/image-24-1024x576-728fa3e9.png) + +Neon introduced [autoscaling for serverless Postgres](https://github.com/neondatabase/autoscaling/) to the world [over a year ago](https://neon.tech/blog/scaling-serverless-postgres), enabling your applications to handle peak demand without incurring peak infrastructure costs 24/7. Our [autoscaling feature](https://neon.tech/docs/introduction/autoscaling) performs zero-downtime vertical scaling of your Postgres instance, provisioning extra CPU and memory when your workload needs it and scaling down to reduce costs when possible. + +Almost a third of our customers use autoscaling today. Recrowd, a Neon customer, recently shared how Neon’s [autoscaling provides them with the peace of mind](https://neon.tech/blog/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand#scaling-up-and-down-automatically-meeting-fluctuating-demand-with-neon) that they’re ready to handle fluctuating demand. + +Implementing autoscaling for Postgres is no easy feat though, and we’ve learned many lessons by bringing over 700,000 databases under management this past year and a half. We’d like to take this opportunity to thank everyone building on Neon — your valuable feedback has helped grow and shape our platform. + +As part of our thanks, we’d like to share some of the challenges we’ve overcome and give you a sneak peek into what’s next for Neon’s autoscaling. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/1-year-of-autoscaling-postgres-at-neon/autoscaling-graph-1024x227-d00d6c38.jpg) + +_An hour of autoscaling by a selection of your databases, captured from real production data while writing this post. Each line is a single endpoint, with a height equal to provisioned compute units._ + +## Architecture of Autoscaling: A refresher + +Neon deploys each Postgres instance as a [virtual machine (VM)](https://en.wikipedia.org/wiki/Virtual_machine) in one of our [Kubernetes](https://kubernetes.io/) clusters. If you’re familiar with Kubernetes, you might wonder why we use virtual machines instead of containers — after all, containers are the standard way to run workloads in Kubernetes, and VMs aren’t natively supported. + +In short, we chose VMs because they: + +- Provide strong isolation boundaries +- Support dynamic adjustment of assigned resources, which is necessary for autoscaling without restarting Postgres +- Can support seamless live migration, allowing us to rebalance across nodes + +Since Kubernetes doesn’t natively support VMs, and preexisting VMs-in-Kubernetes tools didn’t meet our requirements, [we created NeonVM](https://github.com/neondatabase/autoscaling/tree/main/neonvm). This provides a [custom resource definition (CRD)](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources) and [controller](https://kubernetes.io/docs/concepts/architecture/controller/) for deploying and managing virtual machines in our Kubernetes clusters, including the ability to dynamically scale the CPU and memory available to them in-place. + +To implement the scaling logic of autoscaling, we use our `autoscaler-agent` — a daemon we deploy on each Kubernetes node to monitor metrics for each Postgres VM and make scaling decisions based on those metrics. The `autoscaler-agent` also works in conjunction with our modified [Kubernetes scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/scheduling-framework/) to prevent unintentionally overcommitting resources, making sure we don’t run out of resources on the host node. + +The `autoscaler-agent` also communicates with the `vm-monitor`, a small program inside the VM. The `vm-monitor` continuously monitors Postgres’ resource usage and will request upscaling on its behalf when there’s an imminent need, e.g., if a query is about to exhaust all available memory. `vm-monitor` is also responsible for adjusting the size of our [Local File Cache](https://neon.tech/docs/reference/glossary#local-file-cache) in Postgres when scaling occurs, to take advantage of added resources. + +For more background information, [read our original Scaling Serverless Postgres article](https://neon.tech/blog/scaling-serverless-postgres). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/1-year-of-autoscaling-postgres-at-neon/diagram-1024x498-844cc38e.jpg) + +## Autoscaling at Scale + +Here’s a quick recap of some challenges we ran into. Many of these engineering challenges warrant an article of their own — and this doesn’t even include the kernel panics or I/O throttling we worked to resolve in our quest for stability! + +Let us know through [Discord](https://neon.tech/discord) or [X](https://x.com/neondatabase) if you’d like to hear more about these. + +### Cgroup Signals and Memory Scaling + +Our initial memory scaling implementation ran Postgres in a [control group (cgroup)](https://docs.kernel.org/admin-guide/cgroup-v2.html) inside the VM, and listened to `memory.high` events to trigger upscaling. In theory, this makes sense — we can get notified almost instantly when Postgres’ memory usage exceeds a defined threshold, scale up before memory runs out, and then increase the threshold. In practice, we [had issues with this approach](https://github.com/neondatabase/neon/issues/5444) because `memory.high` isn’t really meant to be used that way — it’s best interpreted as a soft upper-bound on memory usage, above which the kernel will start forcing the processes in the cgroup into memory reclamation and aggressively throttling them. + +To fix these, we simplified — just polling the Postgres cgroup’s memory usage 100ms and requesting more if it looks like it’ll run out soon. We found this was more predictable – and critically, more stable – than trying to use `memory.high`. And while it can’t always react fast enough, there’s limits elsewhere in the system, too — ACPI memory hotplug isn’t always instant, either. More on that later. + +### Scaling Down to Zero TPS + +Just before Neon’s last offsite in November, we discovered a strange issue during internal pre-release benchmarking: Under certain circumstances, communication appeared to stall between the Postgres instance and its [safekeepers](https://neon.tech/docs/introduction/architecture-overview), for up to a minute. At the same time, the ongoing `pgbench` run would start reporting that it was making no progress, i.e., 0 TPS (transactions per second). + +We quickly identified that the issue only affected autoscaling-enabled endpoints. The issue wasn’t consistently reproducible, and when we did see it, we had a limited window during which we could interrogate the state of the VM. To make matters worse, it sometimes seemed as if looking at the problem caused it to resolve! Occasionally, when we’d run `ps` to see the state of all processes inside the VM, `pgbench` would immediately resume as if nothing had happened. + +We build a custom Linux kernel, so after some debugging with `kallsyms` during the offsite, we found that both Postgres and the kernel’s `acpi_hotplug` worker were stuck in a suspicious 100-millisecond sleep, deep in the same memory allocation code path. Suspecting this could be a kernel bug, we tried testing with a newer kernel version — and after running a comprehensive suite of tests on both versions, we found that the issue was fully resolved by updating the kernel. + +## What’s Next for Autoscaling Postgres at Neon + +We’ve been busy throughout this first year of autoscaling prioritizing its stability, but as we look towards further improvement, we’ve turned our focus on expanding the set of workloads that autoscaling is a great fit for. + +And while in the medium-term, we have some deeper technical changes coming (virtio-mem, free page reporting, and DAMON, oh my!) — we wanted to give you a sneak peek of some improvements coming soon. + +### Smarter Autoscaling using Local File Cache Metrics + +[Neon’s architecture](https://neon.tech/blog/architecture-decisions-in-neon) separates storage and compute. It’s an integral part of what enables us to autoscale and dramatically reduce cold start times for serverless Postgres. Of course, [accessing pages across the network](https://neon.tech/blog/get-page-at-lsn) can result in increased query latency, so Neon’s Postgres has a [Local File Cache (LFC)](https://neon.tech/docs/reference/glossary#local-file-cache) that acts as a resizable extension of [Postgres’ shared buffers](https://www.postgresql.org/docs/16/runtime-config-resource.html#GUC-SHARED-BUFFERS). + +Picking the right size for the cache is crucial — with certain OLTP workloads, we see a stepwise effect based on whether the working set fits into cache, sometimes with a 10x increase in performance from just a marginal increase in LFC size. A corollary to this is that the ideal endpoint size is often just big enough to fit the [working set size](https://en.wikipedia.org/wiki/Working_set_size), but no larger. + +However, determining the right LFC size on the fly is challenging. Cache hit ratio isn’t a reliable indicator of what the size should be, and there’s little benefit to scaling up if the working set wouldn’t fit into cache either way. + +To help larger workloads stay performant, we’re currently working on augmenting our scaling algorithm with metrics from the LFC — specifically, using a [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) estimator for working set size based on the number of unique pages accessed. + +In the meantime, you can read more about [sizing your Postgres database on Neon](https://neon.tech/docs/manage/endpoints#sizing-your-computed-based-on-the-working-set) for optimal LFC usage in our documentation. + +### Accommodating Rapid Memory Allocation with Swap + +With the way autoscaling works today, there’s fundamental limits to how fast we can react to increased memory usage — and as a result, we’ve seen that workloads allocating large blocks of shared memory can sometimes fail, or even get hit by [OOMs](https://en.wikipedia.org/wiki/Out_of_memory). + +We’ve considered a couple of strategies, and in the end landed on the combination of adding swap1 and disabling memory overcommitting2. + +What practical applications does this have? Well, pgvector 0.6 implemented a parallel Hierarchical Navigable Small World (HNSW) index build feature. We wrote about how using this feature [can result in 30x faster index builds](https://neon.tech/blog/pgvector-30x-faster-index-build-for-your-vector-embeddings). As a consequence of supporting parallel index building, pgvector 0.6 switched to allocating all its memory up-front ­ — without these changes, we found it’d fail with inscrutable errors if the dataset was too large. + +From an implementation perspective though, it’s not trivial. To support blazingly fast cold starts, we keep [a pool of pre-created VMs](https://neon.tech/blog/cold-starts-just-got-hot), waiting to be assigned an endpoint to run — unfortunately this means that we don’t know how much swap the VM will need until after it’s created. + +Thankfully, the solution’s not so bad — we can mount an empty disk for swap, and when the VM is assigned an endpoint, `mkswap` with the desired size and `swapon`. All `mkswap` does is write the header page (which itself is only 4KiB on most systems), and `swapon` is only expensive for non-contiguous disk space (like swapfiles), so this ends up quick enough to be included in the hot path. + +We’re in the process of rolling this out over the next couple of weeks — including new observability into swap usage, to help you assess if your endpoint’s scaling limits should be increased. + +## Conclusion + +Building a truly cloud-native serverless Postgres platform is challenging, but we’ve risen to the challenge at Neon. Our engineering team has been hard at work running Postgres at scale and enhancing it with cutting-edge features. Autoscaling – alongside branching, point-in-time restore, and time travel queries – are just the start of the unique features made possible by Neon’s separation of storage and compute. + +Are you using Neon’s autoscaling? We’d love your feedback and to hear about what you’re building with Neon. Follow us on [X](https://twitter.com/neondatabase), join us on [Discord](https://neon.tech/discord), and let us know how we can help you build the next generation of applications. + +--- + +1. Historically, the reason we didn’t have swap is because we were concerned about performance – for a long time we used EBS-backed nodes with relatively little IOPS capacity (because storage is separate from compute), so we were concerned that swap may do more harm than good. But we eventually switched away from EBS to local SSDs, which are tons faster, so this is no longer a concern. +2. This is quite common for database software, and indeed is what’s typically recommended for Postgres. We initially were concerned about potential interactions with memory hotplug, but through testing found it to be more stable than the alternative. diff --git a/content/blog/posts/10x-projects-on-free-plan.md b/content/blog/posts/10x-projects-on-free-plan.md new file mode 100644 index 0000000000..b90827eba1 --- /dev/null +++ b/content/blog/posts/10x-projects-on-free-plan.md @@ -0,0 +1,70 @@ +--- +title: '10x-ing our Free Plan: Everyone Gets Ten Projects' +description: 'You asked for a second project, so we gave you ten.' +excerpt: "Lots of you told us it would be nice to have more projects. Starting today (10/10 \U0001F609), everyone on the Free Plan can now create up to ten projects. You can create up to 10 branches on each Free Plan Project. Account-level limits on compute hours, storage, and bandwidth remain unc..." +date: '2024-10-10T15:10:51' +updatedOn: '2024-10-10T15:10:54' +category: company +categories: + - company +authors: + - nikita-shamgunov +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/10x-projects-on-free-plan/cover.jpg + alt: null +isFeatured: false +seo: + title: '10x-ing our Free Plan: Everyone Gets Ten Projects - Neon' + description: 'You asked for a second project, so we gave you ten.' + keywords: [] + noindex: false + ogTitle: '10x-ing our Free Plan: Everyone Gets Ten Projects - Neon' + ogDescription: "Lots of you told us it would be nice to have more projects. Starting today (10/10 \U0001F609), everyone on the Free Plan can now create up to ten projects. You can create up to 10 branches on each Free Plan Project. Account-level limits on compute hours, storage, and bandwidth remain unchanged. We’re confident that only […]" + image: >- + https://cdn.neonapi.io/public/images/pages/blog/10x-projects-on-free-plan/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/10x-projects-on-free-plan/neon-10-projects-1-1024x576-db211228.jpg) + +Lots of you told us it would be nice to have more projects. Starting today (10/10 😉), **everyone on the Free Plan can now create up to ten projects.** + + +See [Neon Free Plan Documentation](https://neon.tech/docs/introduction/plans#free-plan) for all details. + + +You can create up to 10 branches on each Free Plan Project. Account-level limits on compute hours, storage, and bandwidth remain unchanged. We’re confident that only increasing the Project limit will lead to a nice quality-of-life boost amongst Free Plan users because when we look at resource usage, it is stratified: + +- At any point in time, the majority of Free Projects are using less than 100MB of storage and not actively using compute (thanks to [scale-to-zero](https://neon.tech/blog/why-you-want-a-database-that-scales-to-zero).) +- Free Projects that are maxing out resources are the ones powering an app or business that is taking off – the owner is happy to upgrade to a paid plan for access to larger computes, increased data retention, and support.
+ +So if we can increase the Project limit and make it easier for you to learn a new stack, [try Azure](https://neon.tech/blog/first-azure-region-available-in-neon), launch an AI-powered IDE, ship an MVP, we see it as a win. + +## Engineering Unlock, not Financial Risk + +A couple high-profile shutdowns this year led many developers to question the financial viability of database free tiers. But Neon is built differently in a way that changes the equation. + +Free tier economics can be boiled down to two variables: **Cost** (_of infrastructure_) in, **benefit** (_of paying customers who chose you in part because of the accessibility of the free tier_) out. + +Everyone assumes cost is constant and focuses on increasing the benefit side. When they can’t move it enough they give up and say “Database Free Plans don’t work!” It doesn’t take a genius to figure out: If you can’t change the benefit variable, **change cost.** + +So that’s what we did! Because of separation of storage and compute, and because of scale to zero, the minimum cost to run a single database on Neon is a fraction of what it is on other architectures. + +## Neon: 0.01x-ing Database costs from day one + +This isn’t a new idea, it was written into the [founding manifesto of Neon](https://neon.tech/blog/hello-world): + +
+

We realized that a modern Postgres service can be designed differently in order to be cheaper and more efficient in cloud environments, but it will require some real systems engineering. We call this approach separation of storage and compute. It allows us to architect the service around performance, reliability, manageability, and cost. Cost is particularly important when you design a system for the cloud. Any cloud service has an infrastructure bill that it has to pass on to the end user. If you don’t account for cost at the architecture level running a service can get very expensive. That’s why when you build for the cloud you have to make the cost of running the service an important design consideration on par with manageability, reliability, and performance. One of the immediate implications of designing for cost was to never use EBS volumes and use a combination of local storage and S3 instead. Local storage for hot and S3 for cold data.

+
+ +The impact of a step-function reduction in the entry cost of databases goes well beyond free tiers:
+ +- Companies like [Retool use Neon to operate fleets](https://neon.tech/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases) of hundreds of thousands of separate databases for each of their customers. +- AI Platforms like [Replit Agent use Neon](https://x.com/nikitabase/status/1837138637516931252) to put database provisioning in the hands of AI Agents. + +None of this would be possible in the bygone era of databases running on statically provisioned CPU, RAM and Storage. + +So go ahead–build that app you’ve been thinking about. If it doesn’t take off you can always build another.\* + +\*Up to ten times. diff --git a/content/blog/posts/6-best-practices-for-running-neon-in-production.md b/content/blog/posts/6-best-practices-for-running-neon-in-production.md new file mode 100644 index 0000000000..1cc0e0bd0f --- /dev/null +++ b/content/blog/posts/6-best-practices-for-running-neon-in-production.md @@ -0,0 +1,99 @@ +--- +title: Best Practices for Running Neon in Production +description: Our go-to advice +excerpt: >- + In Neon, we’re always trying to strike a balance between out-of-the-box + configurations that work for most people and the flexibility to accommodate a + wide range of use cases. By tweaking features like autoscaling, branching, and + scale to zero, you can optimize Neon for many diffe... +date: '2025-08-18T17:26:31' +updatedOn: '2025-08-19T18:16:51' +category: workflows +categories: + - workflows +authors: + - russ-dias +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/6-best-practices-for-running-neon-in-production/cover.jpg + alt: null +isFeatured: false +seo: + title: Best Practices for Running Neon in Production - Neon + description: >- + Neon offers configuration flexibility to optimize usage across many + scenarios. This is our core advice for production workloads. + keywords: [] + noindex: false + ogTitle: Best Practices for Running Neon in Production - Neon + ogDescription: >- + Neon offers configuration flexibility to optimize usage across many + scenarios. This is our core advice for production workloads. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/6-best-practices-for-running-neon-in-production/social.jpg +--- + +In Neon, we’re always trying to strike a balance between out-of-the-box configurations that work for most people and the flexibility to accommodate a wide range of use cases. By tweaking features like autoscaling, branching, and scale to zero, you can optimize Neon for many different scenarios – from low-cost development environments to production setups that prioritize performance. + +This flexibility naturally raises configuration questions. Teams that started on Neon with side projects often come to us for guidance as they grow into real companies serving customers. We’ve distilled our core advice into a simple “production checklist” sent straight to your inbox, highlighting the key steps we recommend to avoid common pitfalls. + +
+Post image +
When running Neon in production, you may receive a checklist similar to this in your inbox, reminding you of good practices
+
+ +## Use a High-Performance Production Branch + +When you first create a project in Neon, your main branch is configured to scale to zero when not in use. This is perfect for most side projects: you’re not consuming a ton of compute hours, your database scales to zero when idle (so you’re not billed), and it automatically wakes up when needed. + +But in a production setting where you need to prioritize performance, we recommend [configuring your production branch](https://neon.com/docs/guides/autoscaling-guide#configure-autoscaling-defaults-for-your-project) with a higher compute capacity and disabling scale to zero to avoid cold starts entirely. For example, you might set the autoscaling range to 1 – 4 CUs. This gives you peace of mind that: + +- **Your users won’t be impacted by cold start latency.** Neon’s cold starts are very fast (subsecond), but in a customer-facing environment, you want to avoid any additional latency you can. By disabling scale to zero on your production branch, you avoid this entirely. +- **Your performance will stay consistent even with spikes.** A high enough autoscaling limit acts as a safety net against unexpected traffic. If your workload stays small, you won’t hit the upper bound, but if you need it, your performance won’t degrade, and you won’t have to manually resize anything + + +Scale to zero is still immensely useful to reduce costs in all your other branches, e.g. development, staging, testing, or even production environments that can tolerate an extra 500 ms of latency (for example internal tools). + + +## Create a Read Replica + +As your application grows, you’ll likely start running read-heavy workloads that don’t need to hit your primary database – things like analytics dashboards, background jobs, and reports. [On most platforms, setting up a replica to handle these tasks means spinning up a full additional Postgres instance](https://neon.com/blog/the-problem-with-postgres-replicas). But in Neon, read replicas are much more lightweight. + +A [read replica](https://neon.com/docs/introduction/read-replicas) in Neon is just essentially another branch – one that shares storage with your production branch and runs in its own isolated compute instance. The difference is that it’s read-only. Replicas are easy to scale and destroy independently. You can autoscale them based on workload, or spin them up just for specific tasks and shut them down afterward, like a nightly job or heavy query batch. + +By enabling scale to zero on the read replica, it will only run when you’re actively querying it, and you won’t pay additional storage for it. They’re very economical and perfect for scaling out reads without overloading your main compute. Why this matters in production: + +- **You protect user-facing performance.** Read replicas allow you to keep your main compute focused on writes and latency-sensitive queries by offloading everything else. +- **You protect your production database.** Read replicas are truly read-only, so there’s no risk of accidental updates or schema changes. You can safely give access to a read replica to anyone on your team — even those less experienced with databases. + +## Enable Instant Restores + +[Instant Restores](https://neon.com/blog/pitr-deep-dive) allow you to restore a branch to any point within your restore window instantly, even if the database is large. If you’re running a critical database, we recommend always setting a restore window of at least 24 hours, but 7 days gives you extra peace of mind. + + +A 7-day restore window can save you when a bug from a Thursday release isn’t noticed until the weekend. + + +This is one of the most loved features by our customers, because it’s not just a [safety net for emergencies](https://neon.com/restores-survey) but an awesome tool for everyday operations. Having Instant Restores enabled allows you to: + +- **Recover instantly from mistakes.** You can roll back instantly from bad deploys, accidental data deletions, bad scripts. [This is true even if you have many TBs of data.](https://neon.com/blog/recover-large-postgres-databases) +- **Test risky changes with confidence.** If you’re about to alter a critical table or deploy a complex migration, you can snapshot your production branch, apply the change in a temporary branch, and verify the results. +- **Investigate issues using historical data.** Anytime, you can instantly branch off from a past point and inspect the database exactly as it was. + +## Restrict Access by IP + +Controlling who can connect to your database is a key part of protecting your production environment, especially if your database is exposed over the internet. Neon lets you restrict access to your project by specifying an [IP allow list,](https://neon.com/docs/introduction/ip-allow) a set of trusted IP addresses or ranges that are allowed to connect to your database. Any connection attempt from outside that list will be denied. This gives you a simple but powerful layer of protection, especially when paired with strong authentication. + +Why this is key: + +- **Block unauthorized access to essential databases.** Ensure only your servers, office network, or CI runners can connect and block everything else at the network level. +- **Catch accidental exposure.** Even if your credentials are leaked or misused, IP restrictions can prevent unauthorized database access. +- **Better compliance posture.** Security standards (e.g. SOC 2, HIPAA) might expect network-level access controls for production databases. + +## Conclusion + +When your project starts serving real users critically, there are a few smart steps you can take to get the best performance, reliability, and safety out of your database. This production checklist will act as a reminder of these practices so you can track your progress and take action when you’re ready. + + +New to Neon? [Sign up for a free Neon account](https://console.neon.tech/signup), spin up your first project in seconds (no credit card required), and grow all the way to prod. + diff --git a/content/blog/posts/6-tips-to-optimize-storage-costs-for-your-postgres-databases.md b/content/blog/posts/6-tips-to-optimize-storage-costs-for-your-postgres-databases.md new file mode 100644 index 0000000000..ffe971cf2b --- /dev/null +++ b/content/blog/posts/6-tips-to-optimize-storage-costs-for-your-postgres-databases.md @@ -0,0 +1,123 @@ +--- +title: 6 tips to optimize storage costs for your Postgres databases +description: Simple strategies to lower your storage bill +excerpt: >- + As your data grows, so does your storage bill. Here are 6 quick tips that can + help you streamline your storage usage, keeping your Postgres database running + lean. 1. Remove unused indexes Indexes are crucial for Postgres query + performance, but they also consume storage space. Ove... +date: '2024-03-15T17:52:15' +updatedOn: '2024-03-15T18:39:08' +category: postgres +categories: + - postgres +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/6-tips-to-optimize-storage-costs-for-your-postgres-databases/cover.jpg + alt: null +isFeatured: false +seo: + title: 6 tips to optimize storage costs for your Postgres databases - Neon + description: >- + As your data grows, so does your storage bill. Here are some quick tips that + can help you optimize your database storage usage. + keywords: [] + noindex: false + ogTitle: 6 tips to optimize storage costs for your Postgres databases - Neon + ogDescription: >- + As your data grows, so does your storage bill. Here are some quick tips that + can help you optimize your database storage usage. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/6-tips-to-optimize-storage-costs-for-your-postgres-databases/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/6-tips-to-optimize-storage-costs-for-your-postgres-databases/neon-6-tips-e60bbd5a.jpg) + +As your data grows, so does your storage bill. Here are 6 quick tips that can help you streamline your storage usage, keeping your Postgres database running lean. + +## 1. Remove unused indexes + +Indexes are crucial for Postgres query performance, but they also consume storage space. Over time, as applications evolve, indexes often become obsolete or redundant, yet they continue to occupy valuable storage space and even add overhead to write operations. + +For example, let’s say you have a Postgres table storing orders. Initially, you focused on troubleshooting shipping issues, so you created an index on the shipped column to speed up finding unshipped orders (`WHERE shipped = false`). After a few months, the shipping problems are resolved. Your focus now shifts to analyzing overall sales performance, requiring aggregates across all orders and identifying high-value orders quickly. The unused index on `shipped` still consumes storage and adds overhead to inserts/updates, even though it’s no longer relevant for key queries. + +To identify unused indexes, you can leverage the Postgres `pg_stat_user_indexes` and `pg_index` views. These views provide insights into index usage statistics, including the number of index scans, which can help you pinpoint indexes that are not being used for query optimization. + +## 2. Reduce storage duplicities + +In production settings, it’s common practice not just to maintain a single production database but also to run multiple data copies for staging, development, and testing purposes. This significantly increases the overall storage costs; every additional copy translates directly into more storage space, multiplying the expenses. + +
+Image +
Production Postgres deployments often include multiple data copies (e.g. for dev instances) which increases the overall costs for storage.
+
+ +If you’re in Neon, you can use [database branching](https://neon.tech/docs/introduction/branching) to avoid this. The custom storage architecture of Neon, allows for the creation of database copies via copy-on-write (just like you’re used to with code in Git). This means that database branches share the same underlying data as the main database branch until changes are made, and as a result, you can have numerous development, testing, and staging environments that mirror your production database without paying extra for storage. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/6-tips-to-optimize-storage-costs-for-your-postgres-databases/screenshot-2024-03-15-at-104257percente2percent80percentafam-30a860e5.png) + +## 3. Use efficient data types (if you can) + +Every data type in Postgres has its own storage requirements, so it’s better to avoid selecting a data type that is larger than necessary. For example: + +- Postgres offers a range of numeric types, including `INTEGER`, `NUMERIC`, `REAL`, and `DOUBLE PRECISION`. Each has its use case, but `INTEGER` types are often sufficient for counts and identifiers and use less space than floating-point types. +- If you’re storing small integers, you can use the `SMALLINT` type instead of `INTEGER` or `BIGINT`, as it uses less space. + +## 4. Order your columns from largest to smallest + +In Postgres, the physical order of columns in your table definitions also can impact storage. This is due to the alignment requirements of different data types, where each type is aligned at memory addresses that are multiples of their size. This ensures efficient data retrieval and adherence to memory and storage management protocols but can introduce padding to meet these criteria, leading to unused space within your database. + +To mitigate this, order your table columns from the largest to the smallest data type. By doing so, you minimize the padding necessary for alignment, thus reducing wasted space. For example, consider a Postgres table designed to store user information: + +```sql +CREATE TABLE user_info ( + + user_id SERIAL, -- 4 bytes + + name VARCHAR(100), -- variable size + + profile_pic_url TEXT, -- variable size, potentially large + + registration_date DATE -- 4 bytes + +); +``` + +If we wanted to apply the optimization strategy of ordering columns from largest to smallest, you might redesign the table like this: + +```sql +CREATE TABLE user_info_optimized ( + + profile_pic_url TEXT, -- variable size, potentially large + + name VARCHAR(100), -- variable size + + registration_date DATE, -- 4 bytes + + user_id SERIAL -- 4 bytes + +); +``` + +## 5. Adjust your history retention period in Neon + +Neon automatically retains a [history of changes](https://neon.tech/docs/introduction/point-in-time-restore#history-retention) across all branches within a project, empowering users with the capability to restore data to any point within the defined retention period. Therefore, the size of storage for a Neon project will be calculated as the sum of two variables: + +- Data size (size of all the tables in Postgres databases) +- History size (corresponding to the WAL size up to the configurable history retention period) + +The default retention periods vary by pricing plan (24 hours for Free, 7 days for Launch, and 30 days for Scale). If you’re in Launch or Scale plans, you could consider configuring a shorter retention period, which leads to storage savings. + +But before you jump right on it, consider the potential impact on your Point-In-Time Recovery (PITR) capabilities. In Neon, you can revert changes or recover lost data by restoring a branch to an earlier state within the retention window. You can pinpoint the exact moment needed for restoration, down to the millisecond. But reducing the history retention period narrows this window of opportunity for recovery, so make sure tol balance between the need for storage optimization and the ability to recover from unintended data alterations or loss. + +## 6. Consider fine-tuning autovacuum + +Postgres autovacuum automates the process of reclaiming storage space after rows are updated or deleted (“dead tuples”). If you have a highly transactional workload, you might want to consider a more frequent autovacuum schedule than the configured by default. + +If you suspect bloat is being an issue for you, you can check for long-running transactions that might be impeding autovacuum from properly running. You can use `pg_stat_activity` to identify transactions that are running longer than expected (e.g., over 10 minutes). + +## Running Postgres? Try Neon + +Neon’s approach to storage can transform your Postgres experience, making it more efficient, agile, and cost-effective. [Request an Enterprise trial to get a 30-day unrestricted access to the platform.](https://neon.tech/enterprise#request-trial) diff --git a/content/blog/posts/a-conversation-on-building-smarter-ai-agents-with-neon-and-wordware.md b/content/blog/posts/a-conversation-on-building-smarter-ai-agents-with-neon-and-wordware.md new file mode 100644 index 0000000000..05f6d1e4cf --- /dev/null +++ b/content/blog/posts/a-conversation-on-building-smarter-ai-agents-with-neon-and-wordware.md @@ -0,0 +1,93 @@ +--- +title: A Conversation on Building Smarter AI Agents with Neon and Wordware +description: 'Fast iteration is key, from the IDE all the way down to the infra' +excerpt: >- + A few weeks ago, we participated in a panel talking about AI Agents with + Wordware. For those of you who prefer to scroll through it, here’s a summary + of our conversation—plus a Wordware demo. The interlocutors in this + conversation were Raouf Chebri, Sr Developer Advocate at Neon... +date: '2024-11-14T18:32:14' +updatedOn: '2024-11-14T18:32:17' +category: community +categories: + - community + - ai +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/a-conversation-on-building-smarter-ai-agents-with-neon-and-wordware/cover.jpg + alt: null +isFeatured: false +seo: + title: A Conversation on Building Smarter AI Agents with Neon and Wordware - Neon + description: >- + A few weeks ago, we participated in a panel talking about AI Agents with + Wordware. Here's a TL;DR of our conversation. + keywords: [] + noindex: false + ogTitle: A Conversation on Building Smarter AI Agents with Neon and Wordware - Neon + ogDescription: >- + A few weeks ago, we participated in a panel talking about AI Agents with + Wordware. Here's a TL;DR of our conversation. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/a-conversation-on-building-smarter-ai-agents-with-neon-and-wordware/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/a-conversation-on-building-smarter-ai-agents-with-neon-and-wordware/neon-ai-agent-2-1-1024x576-ad362636.jpg) + +A few weeks ago, we participated in a [panel](https://www.youtube.com/watch?v=DSbf_t_VOuM) talking about [AI Agents](https://neon.tech/use-cases/ai-agents) with [Wordware](https://www.wordware.ai/). For those of you who prefer to scroll through it, here’s a summary of our conversation—plus a Wordware demo. + + +The team at Wordware (YC S24) is building their platform on Neon. You can read more about why they picked Neon and how they’re using it [here](https://neon.tech/blog/building-ai-agents-just-got-faster-with-wordware-and-neon). If you're also YC, ask us about our [deals](https://neon.tech/yc-startups). + + +--- + +_The interlocutors in this conversation were [Raouf Chebri](https://www.linkedin.com/in/raoufchebri/), Sr Developer Advocate at Neon (asking the questions), and [Robert Chandler](https://www.linkedin.com/in/robertjhchandler/), Co-founder and CTO of Wordware._ + + + +### What are the challenges teams are facing when building AI agents? + +A huge challenge we see often among teams building [AI agents](https://www.wordware.ai/blog/best-ai-agent-frameworks-for-developing-autonomous-systems) is managing the slow feedback loop. This is especially common when engineers aren’t domain experts. Without a clear understanding of what “good” output looks like in your agent, engineers have to rely on someone else for feedback which slows down development significantly. Constant iteration is key for AI agents. It’s essential to put the person who knows what “good” means—the domain expert—more in the driver’s seat so teams can achieve a much faster iteration cycle. + +This challenge is what inspired us to build Wordware. Wordware allows engineers and product teams to collaborate seamlessly within a prompt-first environment that’s both intuitive and powerful. This way, teams can experiment, adapt, and refine agents in real time, drastically improving development efficiency and agent quality. + +My co-founder and I met over 10 years ago while studying machine learning at Cambridge, so we’ve been around AI for a while. Before Wordware, we each went on different paths within industry—he developed early memory-augmenting tech using GPT-2, I worked on self-driving cars—but in both experiences, we saw how crucial it was to tighten feedback loops to succeed. + +### What sets Wordware apart from other agent frameworks out there? + +Something different about Wordware is that it’s designed from first principles for prompt engineering, blending the flexibility of natural language with the structure of programming. Wordware has a prompt-first experience that allows both engineers and non-technical team members (e.g. the domain experts) to collaborate via an intuitive, web-based IDE. This allows the team to iterate on prompts directly, move much faster, and build more effective agents as a result. + +Another key differentiator of Wordware is its modularity. Users can create specialized, reusable components that make up narrow agents, each designed for specific tasks, and then connect these components to build more comprehensive solutions. Wordware also supports multiple LLM—teams can optimize their agents based on the strengths of each model without reworking their entire system. + +### This adaptability with different LLMs is interesting. + +Yeah, we built Wordware to be completely model-agnostic so users can leverage [the best LLMs](https://www.wordware.ai/blog/which-llm-is-the-best-a-guide-to-the-top-large-language-models) for their specific needs without any fuss. Sometimes, you want a model that’s fast and cost-effective—something like Llama or Mistral is perfect for that. Other times, you need deeper reasoning or creativity, and that’s where GPT-4 or Claude shine. + +What’s cool about this approach is that it’s as easy as switching tabs. You’re not rewriting code or changing SDKs; you’re just picking the model that best fits your task. For example, GPT models are sharp for logic and precision, but Claude has this subtle, more tasteful style for writing or summarizing content. So if you’re crafting a blog post you might go with Claude, and for intense reasoning or math-heavy tasks, you’d choose GPT. + + + +You can even combine these models within a single workflow. Need GPT to handle some reasoning and then Claude to polish the language? No problem, Wordware lets you chain these capabilities together seamlessly, so you get the best of all worlds in one place. It’s all about letting the models play to their strengths and building the most effective AI workflows without being locked into a single option. + +### I heard that Wordware had a viral moment recently… + +We launched [an agent that analyzed Twitter feeds and offered personality insights](https://twitter.wordware.ai/), including a roast feature that people absolutely loved. + + + +Overnight it became a hit, and we were not expecting it. We saw a big traffic spike that turned out to be a real stress test for our infrastructure. That’s when [Neon](https://neon.tech/home)’s serverless architecture and [autoscaling](https://neon.tech/docs/introduction/autoscaling) truly proved useful. + +### Tell us more about how Neon supported you! + +Because we’re using Neon, we didn’t have to scramble to manually scale up or worry about provisioning additional resources—it all happened automatically. Neon provisions new databases really fast, and these databases autoscale up very quickly also, which was critical given how much our load was increasing. + +This whole experience made clear the importance of having dynamic autoscaling in your database, especially for applications where traffic can fluctuate wildly. We also love Neon’s [branching](https://neon.tech/docs/introduction/branching), which lets us create isolated, on-demand environments that mirror production data without the overhead of duplicating it. This makes it easy for us to quickly test and iterate new features and schema changes, shipping faster and with fewer mistakes. + +--- + +_To learn more about Neon and how it powers startups like Wordware, explore our_ [case studies](https://neon.tech/case-studies)_. Neon has a_ [Free Plan](https://console.neon.tech/signup) _ – you can get started right away, no credit card required and no questions asked._ diff --git a/content/blog/posts/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte.md b/content/blog/posts/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte.md new file mode 100644 index 0000000000..a62fc63b6a --- /dev/null +++ b/content/blog/posts/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte.md @@ -0,0 +1,152 @@ +--- +title: A Guide to Logical Replication and CDC in PostgreSQL with Airbyte +description: Synchronize your PostgreSQL and Airbyte data with Logical Replication +excerpt: >- + PostgreSQL is a database that needs no introduction. Started as an open-source + project out of UC Berkeley, it has evolved through decades of careful + stewardship to become one of the world’s most relied on and beloved relational + database management systems (RDBMS). There will inev... +date: '2024-01-11T16:04:46' +updatedOn: '2024-03-01T13:59:14' +category: community +categories: + - community +authors: + - jacob-prall +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte/cover.jpg + alt: null +isFeatured: false +seo: + title: A Guide to Logical Replication and CDC in PostgreSQL with Airbyte - Neon + description: Synchronize your PostgreSQL and Airbyte data with Logical Replication + keywords: [] + noindex: false + ogTitle: A Guide to Logical Replication and CDC in PostgreSQL with Airbyte - Neon + ogDescription: >- + PostgreSQL is a database that needs no introduction. Started as an + open-source project out of UC Berkeley, it has evolved through decades of + careful stewardship to become one of the world’s most relied on and beloved + relational database management systems (RDBMS). There will inevitably come a + time when the data captured and stored in your […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte/neon-logical-replication-1-1024x576-2d7412cb.jpg) + +PostgreSQL is a database that needs no introduction. Started as an [open-source project](https://dsf.berkeley.edu/papers/ERL-M85-95.pdf) out of UC Berkeley, it has evolved through decades of careful stewardship to become one of the world’s most relied on and [beloved relational database management systems](https://db-engines.com/en/blog_post/106#:~:text=PostgreSQL%20is%20the%20database%20management,DBMS%20of%20the%20Year%202023.) (RDBMS). + +There will inevitably come a time when the data captured and stored in your PostgreSQL database needs to exist somewhere else. Perhaps you need to replicate data between Postgres servers, to ensure your system survives a critical failure or heavy spikes in traffic. Alternatively, you may need to move your data to an environment better suited for data analysis and activation. + +In this blog, we’ll look at how to satisfy both of these requirements using logical replication. We’ll review the internal mechanisms that enable replication in PostgreSQL, compare different types of replication, and finally, provide a guide to efficiently replicate your data between Postgres instances and other data stores. + +Let’s jump right in! + +**What is Database Replication?** + +Database replication is the process of copying and maintaining database objects (in our case today, tables), in multiple locations. This can happen synchronously or asynchronously, at the byte, block, or logical level, and is crucial for high availability, load balancing, and data activation. + +Replication in PostgreSQL is handled by several processes and features. We’ll discuss how these components work below. + +**PostgreSQL Replication Mechanisms** + +Let’s look at a couple of the key internal components in PostgreSQL that make replication possible. + +The first component to know is the **Write-Ahead Log (WAL).** The idea behind the WAL is that changes to data files (eg. tables, indexes) must be written to disk _only_ after those changes have been logged. The WAL, then, is that log – an append-only ledger that records every change in the database. + +Using a WAL comes with a few benefits. It ensures PostgreSQL can recover from a crash, even if the crash occurs in the middle of a transaction. It also allows for point-in-time recovery. It even assists in PostgreSQL’s implementation of **Multiversion Concurrency Control (MVCC)** – the WAL keeps a version history of data changes, enabling multiple transactions to occur concurrently, accessing data in the order it was changed based on the query’s start time. + +WAL records are created every time a table’s contents are modified. They are first written to the **WAL buffer** (whose size is determined by the \`wal_buffers\` settings). By writing to memory first, PostgreSQL can optimize and reduce the number of disk I/O operations. When the buffer is full, the data is flushed to disk as a **WAL segment**. + +Each WAL record entry describes a change at the byte or block level in the database. The insert position of the record is described by its Log Sequence Number (LSN), a byte offset that increases with each new record. WAL files are stored in the pg_wal directory, and are a maximum of 16 MB by default (though this is configurable). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/a-guide-to-logical-replication-and-cdc-in-postgresql-with-airbyte/56-1024x649-51009158.jpg) + +_How incoming data and WAL records move through PostgreSQL_ + +Through the process of **logical decoding**, WAL files can be turned into a readable format that represents high-level database operations like INSERT, UPDATE, and DELETE. So, for example, the record might originally say “byte A in file B was changed to C”, but through the process of logical decoding, it can be read as “row R in table T was updated to value V”. These logical change records can be “published” by a database instance to “subscribers”. After WAL records are flushed to the publisher’s disk, the **WAL Sender Process** streams the committed WAL segment data to the subscribed standby servers. This, in a (rather large) nutshell, is how **CDC replication** works in PostgreSQL. + +A PostgreSQL database can only have so many subscribers. The relationship between publisher and subscriber is mediated by PostgreSQL’s replication slots. **Replication slots** are a persistent data structure used to track the progress of replication across subscribers, and to ensure that WAL data needed for replication is not prematurely removed (or recycled). They work by storing the LSN of the WAL record most recently received by each subscriber. When a replication slot is active, PostgreSQL will retain all relevant WAL segments that are necessary for the subscriber to stay in sync with the publisher – even if they are older than the retention policy would normally allow. This guarantees that the subscriber can always catch up, and prevents data loss due to missing WAL segments. + +There are two kinds of replication slots in PostgreSQL – physical (for physical replication) and logical (for logical replication). Let’s take a moment to distinguish between the two types of replication, before we cover how to implement CDC between Postgres servers and other data stores. + +**Physical vs Logical Replication** + +Replication in PostgreSQL is either physical or logical. **Physical replication** is at the byte-level. The exact binary data from the master server’s disk are copied to the replica (including, but not limited to, the WAL). Physical replication is typically used when setting up a master-replica architecture. Physical replication in Postgres has native support for streaming, making it useful for setups that ensure high availability via standby servers. + +**Logical replication**, on the other hand, is at the transaction level. Rather than copying bytes off the disk, the logical change records (detailing INSERTs, UPDATEs, DELETEs) are copied over. Because it’s at the logic level, filters can be applied to only replicate specific tables, rows or columns, making it much more flexible than physical replication. Logical replication is ideal for syncing your transactional data to a data lake or data warehouse. Both approaches are suitable for read-replica load-balancing, though logical replication introduces **replication lag.** + +It’s important to note that when using logical replication, the database schema and DDL commands are not replicated. Schema changes must be kept in sync manually. When the schema between producer and subscriber are out of sync, the replication process will error – this can be avoided by applying any additive schema changes to the subscriber first. + +We’ll first look at how to replicate PostgreSQL data between two or more PostgreSQL servers, otherwise known as **master-replica logical replication**. Then, we’ll cover how to replicate PostgreSQL data to OLAP environments for further processing and analysis. + +**Replicating Data Between PostgreSQL Servers** + +Load-balancing read requests to replicas is a common approach to reduce the load on your primary database. In this guide, we’ll look at how to implement the master-replica (or **primary-standby**) pattern using logical replication. In this case, the primary is the publishing PostgreSQL database, while the standbys are subscribers. This approach is only suitable for workloads without hard real-time requirements, as there will always be some degree of replication lag when implementing logical replication. + +To set up logical replication following the primary-standby pattern, start by configuring your primary database. + +1. Edit your `postgresql.conf` file to enable logical replication by setting the `wal_level` to `logical`, and adjust `max_replication_slots` and `max_wal_senders` to accommodate the number of replicas you need. +2. Create a user role with replication privileges with the following command: + +```sql +CREATE ROLE replicator WITH REPLICATION LOGIN PASSWORD 'password'; +``` + +3. Edit the `pg_hba.conf` file to allow the replication role to connect form the replicas’ IP addresses. +4. Create a Publication on the primary using the following command: + +CREATE PUBLICATION my_publication FOR TABLE table1, table2; + +Next, on each standby server, create a subscription with the following command: + +```sql +CREATE SUBSCRIPTION my_subscription +CONNECTION 'host=master_ip dbname=db_name user=replicator password=password' +PUBLICATION my_publication; +``` + +Before creating your subscription, you may want to initialize the standby’s database with a snapshot of the primary’s data. This can be achieved with `pg_dump` and `pg_restore`, Postgres utilities for backup and restoration. + +The above approach is well-suited for PostgreSQL-to-PostgreSQL replication. But what if you need your data in a centralized environment more suitable for analysis? + +Several tools exist to help you extract and replicate your data out of PostgreSQL and into analytical stores through logical replication. In the next section, we’ll look at how to do so with Airbyte, the open-source standard for data movement. + +**Replicating Data Between PostgreSQL and External Data Stores with Airbyte** + +Airbyte works by providing two connectors – a source connector and a destination connector. These connectors can then create a connection, which is your EL pipeline. Many connectors in the Airbyte catalog are community built and maintained. The PostgreSQL source connector is [certified](https://docs.airbyte.com/integrations/connector-support-levels), meaning the Airbyte team maintains the connector, and provides a production-readiness guarantee. You can expect excellent performance as well – check out our [replication benchmark against Fivetran](https://airbyte.com/blog/postgres-replication-performance-benchmark-airbyte-vs-fivetran) to learn more. + +To begin replicating your PostgreSQL database, start with steps 1 and 2 from above. Create a replication slot on your Postgres database to track changes you want to sync. Then, create publication and replication identities for each Postgres table you want to replicate. **Publication identities** specify the set of tables (and, optionally, specific rows within those tables) whose changes you want to publish. **Replication identities** are configurations on the replica side that determine how the replicated data should be handled or applied. + +From there, it’s as simple as walking through Airbyte’s UI to set up your PostgreSQL source connector, and a destination connector (for example, [BigQuery](https://docs.airbyte.com/integrations/destinations/bigquery), [Snowflake](https://docs.airbyte.com/integrations/destinations/snowflake), or [Redshift](https://docs.airbyte.com/integrations/destinations/redshift)). For a more detailed walkthrough, check out our [documentation](https://docs.airbyte.com/integrations/sources/postgres). + +**Replicating Data Between Neon and External Data Stores with Airbyte** + +Effectively managing and scaling PostgreSQL deployments in the cloud can be expensive and impractical for smaller teams. Several companies now offer Postgres database maintenance with cloud-native scalability. [Neon](https://neon.tech/) is one such platform for serverless PostgreSQL. It comes with features you’d expect from a managed cloud solution, like autoscaling and the separation of compute and storage, but also supports advanced features like [database branching](https://neon.tech/branching). + +Neon recently added support for [logical replication](https://neon.tech/blog/change-data-capture-with-serverless-postgres), and is fully-compatible with Airbyte’s CDC solution. Pairing a cloud database like Neon with Airbyte’s own managed offering, Airbyte Cloud, can deliver a scalable, reliable, and low-cost solution for your OLTP and replication needs. + +To get started, check out Neon’s [documentation](https://neon.tech/docs/guides/logical-replication-airbyte) on connecting to Airbyte. + +**Final Thoughts** + +Before signing off, let’s take a quick look at some WAL configurations you’ll want to keep in mind when configuring your logical replication setup. + +- **`wal_compression`** is a setting that can minimize the impact of WAL accumulation between Airbyte syncs. As mentioned, once a replication slot is filled, WAL records are kept around until they are successfully published to the subscriber. If there is significant time between syncs and you are storage-conscientious, setting a wal_compression policy will save you on space at the cost of some extra CPU. +- **`max_wal_size`** sets the amount of disk usage the WAL is allocated between checkpoints. The default value is 1 GB. To ensure a seamless replication experience, set the max_wal_size large enough for the WAL to be easily stored between syncs. +- **`min_wal_size`** sets the limit at which WAL files will be removed, rather than [recycled](https://postgresqlco.nf/doc/en/param/min_wal_size/), between checkpoints. The default value is 80 MB. Adjusting this value to align with your workload can optimize disk space usage and improve replication performance. + +PostgreSQL is a time- and battle-tested workhorse. Its robust community, dedicated contributors, and flexible feature-set make it an excellent choice for a wide range of use cases. + +Today, we’ve examined how PostgreSQL implements logical replication, as well as how Airbyte can be used in your CDC replication setup. We hope you found this guide informative. If you liked this content, share it with a friend, or reach out to us on LinkedIn. We’d love to hear from you! + +Until next time. + +**Relevant Resources** + +- [https://www.postgresql.org/docs/current/different-replication-solutions.html](https://www.postgresql.org/docs/current/different-replication-solutions.html) +- [https://www.postgresql.org/docs/current/wal-internals.html](https://www.postgresql.org/docs/current/wal-internals.html) +- [https://www.postgresql.org/docs/current/logical-replication.html](https://www.postgresql.org/docs/current/logical-replication.html) +- [https://www.postgresql.org/docs/16/runtime-config-replication.html](https://www.postgresql.org/docs/16/runtime-config-replication.html) diff --git a/content/blog/posts/a-postgrest-compatible-data-api-now-on-neon.md b/content/blog/posts/a-postgrest-compatible-data-api-now-on-neon.md new file mode 100644 index 0000000000..7f20053bbe --- /dev/null +++ b/content/blog/posts/a-postgrest-compatible-data-api-now-on-neon.md @@ -0,0 +1,113 @@ +--- +title: 'A PostgREST-Compatible Data API, Now on Neon' +description: >- + Query Postgres over HTTPS with JWT + RLS, with per-branch endpoints. Built for + developers and agents +excerpt: >- + The Neon Data API is now live, a PostgREST-compatible way to query Neon + directly over HTTPS. If you’ve used PostgREST before (or you have a platform + built on it), you’ll feel right at home: the protocol for the Neon Data API is + identical, and migration is as simple as pointing yo... +date: '2025-10-02T15:34:34' +updatedOn: '2026-01-02T17:40:31' +category: app-platform +categories: + - app-platform + - product +authors: + - ruslan-talpa + - luis-tavares + - luis-neves +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/a-postgrest-compatible-data-api-now-on-neon/cover.jpg + alt: null +isFeatured: true +seo: + title: 'A PostgREST-Compatible Data API, Now on Neon - Neon' + description: >- + The Neon Data API is now live, a re-implementation of PostgREST in Rust + built directly into Neon’s proxy fleet. + keywords: [] + noindex: false + ogTitle: 'A PostgREST-Compatible Data API, Now on Neon - Neon' + ogDescription: >- + The Neon Data API is now live, a re-implementation of PostgREST in Rust + built directly into Neon’s proxy fleet. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/a-postgrest-compatible-data-api-now-on-neon/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/a-postgrest-compatible-data-api-now-on-neon/neon-postgrest-2-1024x576-51b0e6e4.jpg) + +The [Neon Data API](https://neon.com/docs/data-api/get-started) is now live, a PostgREST-compatible way to query Neon directly over HTTPS. If you’ve used PostgREST before (or you have a platform built on it), you’ll feel right at home: the protocol for the Neon Data API is identical, and migration is as simple as pointing your client to a new endpoint and updating your auth configuration. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/a-postgrest-compatible-data-api-now-on-neon/image-10-1024x435-51855e36.png) + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/a-postgrest-compatible-data-api-now-on-neon/image-11-1024x419-c6ead81b.png) + +Under the hood, the Data API is a re-implementation of PostgREST in Rust built directly into Neon’s proxy fleet. We built it this way to make it an integrated part of the Neon platform so that it is seamless, reliable and performant for customers. This Data API is CPU/memory efficient, multi-tenant, and designed to scale across thousands of databases without extra moving parts. + +The Data API is another building block in the Neon toolbox. If what you need is to stand up a full backend quickly (say you’re building an AI agent) **Neon now gives you [auth](https://neon.com/docs/neon-auth/overview) + a REST API + a Postgres database in 1 minute.** All of it comes with autoscaling, branching, and scale-to-zero built in. + + + +## PostgREST is great, but sometimes, you need more scale + +[PostgREST](https://docs.postgrest.org/en/v13/) is one of the most beloved open-source projects in the Postgres ecosystem, and we’re happy to be a sponsor. It turns your database into a REST API automatically – tables, views, and functions become endpoints, standard HTTP verbs (GET, POST, PATCH, DELETE) map directly to SQL operations, and access control flows through Postgres RLS. You can go straight from schema to API. + +This model has powered thousands of apps and even entire platforms. It’s elegant, it’s simple, and it’s battle-tested. + +But if you’re running a platform like Neon, [where we serve many agent platforms that spin up tens of thousands of new databases every day](https://neon.com/use-cases/ai-agents), PostgREST starts to show its limits. Each PostgREST instance is compute-heavy and generally runs on a per-project basis, which works fine for individual apps but starts getting harder to operate efficiently for large multi-tenant environments that need tens of thousands of lightweight APIs at once. + +## How Neon’s Data API works + +This is the challenge we set out to solve with the latest implementation of Neon’s Data API. After starting the beta in PostgREST and hitting some friction, we decided to move forward with a solution that’s much more integrated into the rest of our platform, while keeping everything PostgREST-compatible to make life easier for developers. This updated version of the Data API is a Rust-based re-implementation of PostgREST built to run as a shared service across the Neon fleet. + +This gives us some important benefits: + +- **Lighter footprint.** This new Rust implementation is leaner vs PostgREST, consuming less CPU and memory. We can scale to far more databases without wasting resources or raising prices. +- **Multi-tenant by design.** Traditional PostgREST deployments run one process per project. At Neon scale (tens of thousands of active databases at any given time) this becomes fragmented and very compute-heavy. Neon’s Data API now runs as a shared service, much simpler to manage. We’ve also implemented safeguards that help us deliver consistent performance for everyone without having to keep up with PostgREST limits for thousands of separate processes. +- **Every branch is an API.** This implementation also allows us to offer dedicated Data API endpoints for each Neon branch, something that supports the most common Neon workflow – where developers and agents are [using branches](https://neon.com/branching) to build previews, [versioning / checkpoints](https://neon.com/blog/checkpoints-for-agents-with-neon-snapshots), dev environments, and so on. +- **Scale-to-zero.** In Neon databases, compute scales down to zero when the database is not in use – something crucial to save costs at scale, not only for us but for our customers managing very large Postgres fleets. However, we keep the REST service always available so your apps can connect anytime, even if you only pay for compute when it’s active. +- **Auth.** As we saw earlier, the Data API works out-of-the-box with [Neon Auth](https://neon.com/docs/neon-auth/overview), making it simple to build your backend around Neon. But you’re not locked in: you can also bring your own auth provider by pointing the Data API to any JWKS (Auth0, Clerk, Keycloak, etc.). + + +If you’re interested in trying out Neon and have a full backend to migrate, we’ve put together this [complete migration guide](https://neon.com/guides/complete-supabase-migration) for you to follow. Battle-tested by many customers. + + +## A kit for full-stack agent platforms + +AI agents are expected to work across diverse environments and spin infrastructure up and down on the fly. They need to query data and store results with almost no setup, often in places where developers can’t pre-configure drivers, TCP connections, or credentials. + +We’re working to give agent platforms the tools they need to build a backend quickly, with elements that follow the Neon principles of speed of provisioning, flexibility, and no-hands serverless management. So far, you can deploy + +- Neon Auth for authentication and user management, issuing JWTs that tie directly into Postgres RLS – or bring your own auth provider by pointing the Data API to any JWKS. +- Neon Data API for a PostgREST-compatible REST layer +- Serverless Postgres underneath it all, with autoscaling, branching, scale-to-zero, and instant restore + +The Data API is a great latest addition to the stack, as it removes the friction of connecting agents to Postgres: + +- The Data API speaks HTTPS, just like calling OpenAI or Stripe +- Agents can use fetch() or curl, no Postgres client libraries needed +- JWT + RLS ensure each agent has only the access it needs +- Every Neon branch comes with its own endpoint +- While database compute idles down when not in use and wakes up instantly when traffic comes back, the REST service itself is always available, so requests never fail + +## Try it + +The Data API is out in beta. [Check out our docs](https://neon.com/docs/data-api/get-started) for instructions and [create a free account](https://console.neon.tech/signup) if you don’t have one already. If you have any questions, reach out to us in our [Discord](https://discord.gg/92vNTzKDGp). + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + + +## A note for Neon RLS users + +Neon RLS (previously called Neon Authorize) was a project we launched in 2024 to make it easier to work with Postgres RLS by letting you query the database over HTTP with JWT + RLS. + +Going forward, we recommend users start using either the Neon Data API or Postgres RLS directly instead of Neon RLS. The Data API offers a PostgREST-compatible REST surface, while direct Postgres RLS can be used via the serverless driver with JWT self-verification ([docs here](https://neon.com/docs/serverless/serverless-driver#using-transactions-with-jwt-self-verification)). + +If you are currently using Neon RLS, we recommend you begin planning a migration. Depending on your setup, this may be very simple, but some setups require changes to your queries. [Here’s a SQL-to-PostgREST translation helper](https://neon.com/docs/data-api/sql-to-rest) to help you convert existing SQL into PostgREST syntax. + +_Finally, note that Neon RLS is not the same as Postgres Row-Level Security (RLS). Postgres RLS is a core database feature and it will remain fully supported on Neon._ diff --git a/content/blog/posts/add-an-interface-to-your-neon-database-via-outerbase.md b/content/blog/posts/add-an-interface-to-your-neon-database-via-outerbase.md new file mode 100644 index 0000000000..030bb83d6a --- /dev/null +++ b/content/blog/posts/add-an-interface-to-your-neon-database-via-outerbase.md @@ -0,0 +1,76 @@ +--- +title: Add an interface to your Neon database via Outerbase +description: >- + Run queries in natural language, visualize data, and collaborate with your + team +excerpt: >- + We are beyond excited to announce that the Outerbase integration for Neon is + now publicly available. This integration enables you to instantly connect your + Neon Postgres database to Outerbase’s Data Studio and invite your team to + view, edit, query, and visualize your data with yo... +date: '2024-06-07T15:02:35' +updatedOn: '2024-06-07T15:02:38' +category: community +categories: + - community +authors: + - brandon-strittmatter +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/cover.jpg + alt: null +isFeatured: false +seo: + title: Add an interface to your Neon database via Outerbase - Neon + description: >- + You can integrate Neon and Outerbase to run queries in natural language, + build dashboards to visualize data, and collaborate with your team. + keywords: [] + noindex: false + ogTitle: Add an interface to your Neon database via Outerbase - Neon + ogDescription: >- + You can integrate Neon and Outerbase to run queries in natural language, + build dashboards to visualize data, and collaborate with your team. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/neon-outerbase-1-1024x576-396c4a3c.jpg) + +We are beyond excited to announce that **the Outerbase integration for Neon is now publicly available**. + +This integration enables you to instantly connect your Neon Postgres database to Outerbase’s Data Studio and invite your team to view, edit, query, and visualize your data with you. To learn how to connect Outerbase to your Neon project, [check out this guide](https://neon.tech/docs/guides/outerbase). + +## What is Outerbase? + +[Outerbase](https://www.outerbase.com/)’s mission is to democratize data access, helping everyone get access to the data they need independently of their role or background. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/outerbase-a5a60881.gif) + +With [Data Studio](https://www.outerbase.com/products/data-studio/), users can easily edit and query data from various databases, including Neon. Its intuitive interface supports collaborative data exploration, allowing teams to invite teammates to view, edit, query, and visualize data together. + +Outerbase’s AI integration takes this a step further by enabling users to perform complex SQL tasks using natural language, making data manipulation accessible even to those without advanced SQL knowledge. + +Outerbase also includes data visualization tools that help create concise and beautiful charts and dashboards, making it easier to present and interpret data. + +## Explore Neon collaboratively via Outerbase Data Studio + +![Image](https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/ad4nxdyz2as-q-xys-2kssf191aedek4n5wdug-1namy-oksebbq2va-ocykzd1segjg4ffhzbx679o0dm7zupohnjaauxywxfq7tmp6vatwneli6bvsolihucmyoaelzhwd97zdl5mpqj3s5l6qzbd99eo5-fc8c8869.png) + +Outerbase simplifies data exploration for everyone, whether you are a solo-founder or part of a large organization. Once you have [connected your Neon Postgres database through the Outerbase integration](https://neon.tech/docs/guides/outerbase), you can [invite your teammates](https://docs.outerbase.com/workspaces/members) and start querying your data.

With Outerbase’s Data Studio, you can edit data, update schemas, create tables, and more. You manage multiple Neon databases from a single interface. Its built-in filters, sorting, and queries make data management intuitive and efficient no matter your skill level. Outerbase also allows you and your team to write and save SQL queries, ensuring everyone has access to the same data views. + +## Ask data questions in natural language + +![Image](https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/ad4nxdpb5raayyvarzcmglscygiyg3vvpkdlggzuuzlfr0exhj4sndduyapsxqvkamd5tbihsru49mtfa7fwfs7uw37z6lxfc1rycsrqwvgh2ulih9j5kizajosgwrkf3n6gbg4ykc6iice4gnyqoa1iffqfl-bf155169.png) + +Outerbase doesn’t just make data exploration easy—it makes querying easy too. Using [Outerbase AI](https://www.outerbase.com/products/ai), powered by state-of-the-art AI models and their EZQL™ framework, you can generate SQL queries with natural language. Just ask a question, and let the AI do all the heavy lifting.

It can also help you create new schemas and data to help you kick start your next project. AI can also generate data visualizations to present your data clearly. + +## Data visualization and dashboarding + +![Image](https://cdn.neonapi.io/public/images/pages/blog/add-an-interface-to-your-neon-database-via-outerbase/ad4nxdwb41yir9d8wq25q6oy05xm1ptisz2i5oqql78tnfh1-2yadmpe1ucmaqwiie6gqz8-fn6l6qdexffnld79gnttpbsyqibwfd4zbijzgrj0xtxrygctgkgi-os6wvx1lj2kehq7rvfyaouugqbanixi-10bc09e0.png) + +You can also use Outerbase to create beautiful charts and share them with your team. It supports all the popular visualization types. You can also use SQL variables to filter time ranges, categories, and so much more. + +## Start exploring + +We are so excited for you to try this integration. If you’re a Neon user, you can [sign up to Outerbase](https://app.outerbase.com/signup/) for free and get started. We would also love your feedback: share it in the [Neon Discord](https://discord.gg/92vNTzKDGp) or email us at [support@outerbase.com](mailto:support@outerbase.com) diff --git a/content/blog/posts/add-mcp.md b/content/blog/posts/add-mcp.md new file mode 100644 index 0000000000..5821dbf3b1 --- /dev/null +++ b/content/blog/posts/add-mcp.md @@ -0,0 +1,146 @@ +--- +title: "add-mcp: Install MCP Servers Across Coding Agents and Editors" +description: "A CLI inspired by add-skill, built for the growing MCP ecosystem" +excerpt: >- + A few weeks ago, Vercel released add-skill (now npx skills), a CLI for + installing agent skills across different coding agents and editors like Claude + Code, Cursor, and VS Code. It solves a very real problem: each tool looks for + agent skills in a different place, which makes setup... +date: "2026-02-10T19:47:14" +updatedOn: "2026-02-10T19:56:02" +category: product +categories: + - product + - ai +authors: + - andre-landgraf +cover: + image: "https://cdn.neonapi.io/public/images/pages/blog/add-mcp/cover.jpg" + alt: null +isFeatured: true +seo: + title: "add-mcp: Install MCP Servers Across Coding Agents and Editors - Neon" + description: >- + add-mcp lets you install MCP servers across Claude, Cursor, VS Code, and + more with a single command. Think npx skills, but for MCP. + keywords: [] + noindex: false + ogTitle: "add-mcp: Install MCP Servers Across Coding Agents and Editors - Neon" + ogDescription: >- + add-mcp lets you install MCP servers across Claude, Cursor, VS Code, and + more with a single command. Think npx skills, but for MCP. + image: "https://cdn.neonapi.io/public/images/pages/blog/add-mcp/cover.jpg" +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/add-mcp/neon-add-mcp-2-2-1024x576-f1b6b9ea.jpg) + +A few weeks ago, Vercel released [add-skill](https://vercel.com/changelog/introducing-skills-the-open-agent-skills-ecosystem) (now `npx skills`), a CLI for installing agent skills across different coding agents and editors like Claude Code, Cursor, and VS Code. + +It solves a very real problem: each tool looks for agent skills in a different place, which makes setup repetitive and documentation painful to maintain. The situation that made `npx skills` necessary is slowly improving, with more tools beginning to standardize on the `/.agents/` folder, now supported by Cursor, Codex, and others. + +But while agent skills seem to be converging, **MCP server configuration is still very much fragmented.** Every editor and coding agent uses its own config files, paths, and formats, and it doesn’t look like that unification is coming anytime soon. + +So when we saw add-skill, the immediate reaction was, _“this is great. Why don’t we have this for MCP servers?”_ + +## add-mcp: one command to install an MCP server everywhere you work + +As we realized that something like `add-skill` didn’t exist for MCP servers, we decided to build it ourselves. + +We cloned `add-skill` and started vibe coding a first version that matched the experience. Next, we reached out to Vercel to get access to the `add-mcp` npm package name (shoutout to [Andrew](https://x.com/andrewqu) for his support!). And now, after a few debugging sessions, iterations, and DX adjustments, we’re bringing you the v1 of `add-mcp`: + +[https://github.com/neondatabase/add-mcp](https://github.com/neondatabase/add-mcp) + + + +**`add-mcp` is a small CLI that installs an MCP server across all your coding agents and editors with a single command.** Instead of manually configuring MCPs separately for Claude Code, Cursor, VS Code, Codex, and others, add-mcp detects which agents you’re using and writes the correct configuration files for you, either at the project level or globally. + +## How to use add-mcp + +Any time you want to start using a new MCP server in your project, you run `add-mcp` once and it takes care of configuring all your tools. + +For example, to install a remote MCP server for the current project, you run: + +```bash +npx add-mcp https://mcp.context7.com/mcp +``` + +Or for the next-devtools MCP server (local MCP server): + +```bash +npx add-mcp next-devtools-mcp@latest +``` + +What this command will trigger: + +1. `add-mcp` looks at your project directory +2. detects which coding agents and editors you’re using +3. installs the MCP server into the appropriate configuration files for each of them +4. from that point on, the MCP server is available everywhere you work on that project + +When you want to use another MCP server, you repeat the same process. + + + +## Works across Claude, Cursor, VS Code, and other MCP-enabled tools + +At launch, `add-mcp` supports the following installation targets: + +```bash +% bunx add-mcp@latest list-agents + + █████╗ ██████╗ ██████╗ ███╗ ███╗ ██████╗██████╗ +██╔══██╗██╔══██╗██╔══██╗ ████╗ ████║██╔════╝██╔══██╗ +███████║██║ ██║██║ ██║█████╗██╔████╔██║██║ ██████╔╝ +██╔══██║██║ ██║██║ ██║╚════╝██║╚██╔╝██║██║ ██╔═══╝ +██║ ██║██████╔╝██████╔╝ ██║ ╚═╝ ██║╚██████╗██║ +╚═╝ ╚═╝╚═════╝ ╚═════╝ ╚═╝ ╚═╝ ╚═════╝╚═╝ + +Supported agents: + + Argument MCP Client Aliases Local Global + -------------- -------------- -------------- ----- ------ + claude-code Claude Code ✓ ✓ + claude-desktop Claude Desktop - ✓ + codex Codex ✓ ✓ + cursor Cursor ✓ ✓ + gemini-cli Gemini CLI gemini ✓ ✓ + goose Goose - ✓ + opencode OpenCode ✓ ✓ + vscode VS Code github-copilot ✓ ✓ + zed Zed ✓ ✓ +``` + +By default, `add-mcp` detects which of these agents are already configured in your project and installs the MCP server only for those tools. If you want to target specific agents explicitly, you can do that as well: + +```bash +npx add-mcp https://mcp.context7.com/mcp -a cursor -a claude-code +``` + +If you’re setting things up for a team, or you want an MCP server to be available across all projects on your machine, you can install it globally instead of per project. For example, to install the Neon MCP server, globally, for Cursor only, and without any interactive prompts: + +```bash +npx add-mcp https://mcp.neon.tech/mcp -g -y -a cursor +``` + +And also, if you want to skip all prompts but still rely on automatic agent detection, you can use the `-y` flag on its own: + +```bash +npx add-mcp https://mcp.context7.com/mcp -y +``` + +For a full list of options and configuration details, [check out the repo’s README.](https://github.com/neondatabase/add-mcp) + +## If you maintain MCP docs or build agents… + +`add-mcp` is meant to be a shared building block for the MCP ecosystem: + +- **Documentation maintainers: consider recommending add-mcp as the default way to install your MCP server.** This lets you replace editor-specific setup sections with a single command users can copy, and keeps your docs future-proof as new tools and agents are added. +- **Builders of agents and editors: consider adding native support for your tool by** [opening a PR to the add-mcp repo.](https://github.com/neondatabase/add-mcp) This reduces setup friction for users and ensures your MCP server works consistently across the growing ecosystem. + +## Start using add-mcp + +If you’re already using MCP servers, add-mcp should make your life immediately easier. [Try it!](https://github.com/neondatabase/add-mcp) + +We’d also love feedback and contributions, especially as more agents and MCP servers come online. + +Happy coding! diff --git a/content/blog/posts/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd.md b/content/blog/posts/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd.md new file mode 100644 index 0000000000..0b9687d35b --- /dev/null +++ b/content/blog/posts/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd.md @@ -0,0 +1,159 @@ +--- +title: "Adopting Neon branching in CI/CD pipelines: a practical story by Shepherd" +description: How an insurtech startup uses branching to save developer time and money +excerpt: >- + “Branching saves us both money and developer time. We no longer have to set up + an actual testing database instance and make sure the data is always synced + with production. We now spin up an ephemeral branch when we need to and then + tear it down via the create/delete Github Action... +date: "2024-07-11T16:14:05" +updatedOn: "2024-10-31T16:07:07" +category: community +categories: + - community + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Adopting Neon branching in CI/CD pipelines: a practical story by Shepherd - Neon + description: >- + Learn how adopting database branching for dev and testing workflows can save + you money and developer time. + keywords: [] + noindex: false + ogTitle: >- + Adopting Neon branching in CI/CD pipelines: a practical story by Shepherd - Neon + ogDescription: >- + Learn how adopting database branching for dev and testing workflows can save + you money and developer time. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd/neon-shepherd-1024x576-ede6acc2.jpg) + +
+

“Branching saves us both money and developer time. We no longer have to set up an actual testing database instance and make sure the data is always synced with production. We now spin up an ephemeral branch when we need to and then tear it down via the create/delete Github Actions”

+Angelina Quach, Software Engineer at Shepherd +
+ +[Shepherd](https://shepherdinsurance.com/) is a startup building an all-in-one commercial insurance platform aimed at high-hazard industries. They specialize in underwriting and pricing complex insurance risks efficiently, minimizing administrative waste. Their approach combines fast underwriting, AI-powered risk reduction, and integration with leading safety technologies, offering significant premium savings and enhanced risk management for clients. + +## The backend journey: from Excel to SQLite to Postgres + +**👉 Dig deeper: Shepherd explained their migration story in detail** [here](https://shepherdinsurance.com/blog/the-great-database-migration). + +Motivated by the growth [following their Series A round](https://www.globenewswire.com/en/news-release/2024/02/07/2825155/0/en/Shepherd-Raises-13-5-Million-in-Series-A-Funding-to-Help-Insure-the-10-Trillion-Commercial-Construction-Industry.html), earlier this year Shepherd decided to migrate the database for their pricing engine (called Alchemist) from SQLite to Postgres, looking for a more robust system that could support their growth. + +Alchemist is a crucial component of Shepherd’s product, responsible for underwriting and pricing complex insurance risks. It integrates complex actuarial models, factors, and Shepherd’s proprietary algorithms to deliver accurate and efficient pricing decisions. When Sheperd first started, Alchemist ran on Excel and JSON; then it moved to SQLite. As the business scaled, the limitations of the SQLite setup asked for a move to Postgres. + +## Adopting Neon branching + +When evaluating different Postgres options, Shepherd was attracted to Neon mostly because of its [branching paradigm](https://neon.tech/blog/what-you-get-when-you-think-of-postgres-storage-as-a-transaction-journal). Neon implements database branching at the storage level, enabling users to create database branches with data and schema via copy-on-write. + +The team at Shepherd immediately recognized the **benefits of this model**, especially when combined with Neon’s serverless architecture with auto-scaling and scale to zero: + +- **Less wasted time and money.** The branching model would enable Shepherd to immediately create temporary branches for testing, vs manually setting up clusters and making sure data is kept in sync. On top of this, Neon branches scale to zero when idle: no more paying 24/7 for a database that’s only being used for a few hours a day. + +
+

“On a conventional database, we’d have to set up a database cluster or instance and ensure it always stays synced with production data for accurate testing. We’d also have to scale instances appropriately, avoiding unnecessary expenses when instances are idle. As a startup, we are especially conscious of cost, but doing this manually is very time-consuming and requires engineering bandwidth. Neon eliminates these issues with branching”

+Angelina Quach, Software Engineer at Shepherd +
+ +- **Improved workflows.** Since Neon branches can be managed via Github Actions, Shepherd could directly integrate them into their existing CI/CD pipelines. +- **Automatic scalability.** Since Neon branches autoscale according to load, there’ll be no need to manually resize them up and down to make sure resources are used optimally. +- **Less risk.** This setup also provides inherent guardrails against data-breaking changes. A junior dev in the team could quickly spin up a branch with the peace of mind that they cannot interfere with the production environment. + +## Adding Neon branches to CI/CD pipelines: how Shepherd does it + +Shepherd implemented [Neon branching](https://neon.tech/flow) to streamline their development and deployment processes, integrating it deeply with GitHub actions. Here’s an overview of their setup: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd/screenshot-2024-07-10-at-10844percente2percent80percentafpm-1024x429-77ab9922.png) + +**👉** **TL;DR workflow overview:** + +- GitHub is used to manage all SQL and code files. +- Shepherd creates short-lived testing branches for every PR. Data and schema changes are first tested in these short-lived branches, which are created and deleted automatically via Github Actions as PRs get opened and merged. +- Render is used to then deploy migration scripts against the staging (stg db) and production branches (prod db). +- Migrations are tracked via a separate database table, ensuring that scripts are applied in the correct order and not duplicated. + +### Branch hierarchy + +Shepherd uses different database branches for each step of the pipeline: + +- **Development testing branches**. Each developer creates a “development” testing branch for their own testing purposes. These branches are derived from the production branch. +- **Temporary test branches (short-lived)**. When a PR is opened, a temporary test branch is created from the main branch using Neon’s GitHub actions. Once the test passes and the PR is closed, this temporary branch gets deleted automatically. +- **Staging branch**: Once changes have been tested in the temporary test branches, they are deployed to the staging branch via Render scripts. +- **Production branch**: Once changes are confirmed stable in the staging branch, they are promoted to the production branch. + +💡 _Shepherd is also exploring a “hot swapping” solution to alternate traffic between staging and production databases, ensuring zero downtime during deployments. More info coming soon._ + +### Workflow + +- **Temporary branch creation for PRs**. When a pull request (PR) is opened on GitHub, Shepherd uses Neon’s GitHub actions to create a temporary testing branch derived from the production branch. The [`create branch` action](https://github.com/marketplace/actions/neon-database-create-branch-action) looks similar to this: + +```bash +name: Create Neon Branch + +on: + pull_request: + types: [opened, synchronize] + +jobs: + create-branch: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Create Neon Branch + run: | + curl -X POST https://console.neon.tech/api/v1/projects//branches \ + -H "Authorization: Bearer ${{ secrets.NEON_API_KEY }}" \ + -d '{"name": "${{ github.head_ref }}"}' +``` + +- **Deploying schema and data changes**. Any schema or data changes associated with the PR are first deployed to the temporary branch. +- **Integration and performance testing**. The temporary branch is used for running a series of integration and performance tests. +- **Branch cleanup**. Once the tests pass and the PR is approved, [Neon’s GitHub actions then trigger the deletion of the temporary branch](https://github.com/neondatabase/delete-branch-action): + +1. + +```bash +name: Delete Neon Branch + +on: + pull_request: + types: [closed] + +jobs: + delete-branch: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + + - name: Delete Neon Branch + run: | + curl -X DELETE https://console.neon.tech/api/v1/projects//branches/${{ github.head_ref }} \ + -H "Authorization: Bearer ${{ secrets.NEON_API_KEY }}" +``` + +- **Deploying to staging and production**. Once the changes have been tested in the temporary branches, they are then deployed to the staging branch, where the team performs additional checks and manual verifications. After confirming stability and correctness, the changes deployed to the production branch. + +### Automated deployment of database changes via Render + +Shepherd uses Render to deploy database changes through automated migration scripts. Once changes are tested in the temporary branches, Render executes pre-deploy commands, applying SQL changes against the Neon staging database (`stg db`). These migration scripts are tracked using a dedicated table, ensuring each script is logged with a timestamp to prevent duplication and make sure they’ve been applied in order. After successful verification in staging, the same pre-deploy commands apply the changes to the Neon production database (`prod db`). + +## Start small: create your first branch today + +If you’ve never experimented with database branches yet, the best way to get a feel for it is via the [Neon Free plan](https://console.neon.tech/signup). Create an account in seconds, load some data, and create your first branch. + +A huge thank you to [Shepherd](https://shepherdinsurance.com/) for sharing their story and [the work they’re doing](https://shepherdinsurance.com/case-studies)! Can’t wait to see how far you get 🚀 diff --git a/content/blog/posts/agent-skills-in-2026.md b/content/blog/posts/agent-skills-in-2026.md new file mode 100644 index 0000000000..3237239aa9 --- /dev/null +++ b/content/blog/posts/agent-skills-in-2026.md @@ -0,0 +1,144 @@ +--- +title: Agent Skills in 2026 +description: Reusable knowledge and workflows for coding agents +excerpt: >- + 2026 is starting off hot, and skills are suddenly everywhere. The Agent Skills + spec is now supported across all major coding agents and increasingly adopted + by developer tools, including Neon. But let’s start from the beginning. The + origins of agent skills The concept of agent sk... +date: "2026-01-22T18:13:30" +updatedOn: "2026-01-22T18:50:37" +category: ai +categories: + - ai +authors: + - andre-landgraf +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/agent-skills-in-2026/cover.png + alt: null +isFeatured: true +seo: + title: Agent Skills in 2026 - Neon + description: >- + Agent Skills are becoming the standard for AI coding agents. Learn how they + work, why they matter, and how Neon is adopting them. + keywords: [] + noindex: false + ogTitle: Agent Skills in 2026 - Neon + ogDescription: >- + Agent Skills are becoming the standard for AI coding agents. Learn how they + work, why they matter, and how Neon is adopting them. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/agent-skills-in-2026/social.png +--- + +
+ +
+ +2026 is starting off hot, and **skills** are suddenly everywhere. The [Agent Skills](https://agentskills.io/specification) spec is now supported across all major coding agents and [increasingly adopted by developer tools](https://vercel.com/changelog/introducing-skills-the-open-agent-skills-ecosystem), [including Neon](https://github.com/neondatabase/agent-skills). + +But let’s start from the beginning. + +## The origins of agent skills + +The concept of agent skills didn’t arrive fully formed: + +- [Anthropic published the skills spec](https://github.com/anthropics/skills), and initially [only Claude Code supported it](https://code.claude.com/docs/en/skills). +- [Cursor had skill support behind a feature flag](https://cursor.com/docs/context/skills) for a few weeks but distribution was an issue. +- Claude Code has plugins for distribution, but how do you share skills for Cursor users? +- This problem was solved early this year, when [Vercel published a large React best-practices skill](https://vercel.com/changelog/introducing-skills-the-open-agent-skills-ecosystem) and simultaneously introduced the `npx skills` CLI, which finally solved the distribution problem. + + + +- Now skills can be shared by just publishing them in a public GitHub repository, and every major harness (Claude Code, Cursor, VS Code, OpenCode, Antigravity) can be selected and configured by npx skills. + +## **What agent skills are (and why they matter)** + +With the ecosystem in place, the shape of agent skills has become clear: + +- Agent skills package instructions, scripts, and references into a folder that an agent can load when needed. +- The core of every skill is the `SKILL.md` file, which acts as the entry point and gives the agent a high-level map of what the skill can do. +- When a task matches, the agent pulls in more detailed guidance on demand. This design keeps context small and helps the agent follow a structured workflow. + +What’s special about skills (compared to most previous rule or prompt-based setups) is that **the spec assumes skills are always known to the agent.** That promise is ultimately up to agent developers to uphold, but if it holds, it gives skill authors a reliable way to steer agent behavior. + +This is a meaningful step forward from having to reference MCP resources or invoke slash commands in every prompt just to remind the agent to play by the rules (_pun intended_). + +## Our own early work + +If you were already coding with AI, the need for something like agent skills (and a way to distribute them) was hard to miss. We felt that pain firsthand at Neon, so before the Agent Skills spec existed, we were already experimenting with a similar idea which we called [“AI rules”.](https://neon.com/blog/ai-rules-bring-neon-context-into-your-editor) + +We published [Neon AI Rules](https://neon.com/docs/ai/ai-rules) as a way to give coding agents reliable, structured guidance when working with Neon, starting with agent rules, examples, and a Claude Code plugin for local development. In hindsight, AI Rules surfaced the same underlying need the Agent Skills spec formalizes today. + +Since then, we’ve transitioned this work to Agent Skills, reshaping it based on what we’ve learned about authoring skills and adopting the folder structure introduced by Vercel. It’s all skills now, a more portable and accepted standard. + +## Introducing neondatabase/agent-skills + +So we’re now launching our own skill repository, which lives at [neondatabase/agent-skills](https://github.com/neondatabase/agent-skills). You can install it with: + +```bash +npx skills add neondatabase/agent-skills +``` + +This installs the **`using-neon` skill**, which includes guidelines and best practices for using Neon. + +
+ +Post image + +
https://github.com/neondatabase/agent-skills
+
+ +### **Why a single Neon skill**? + +You may be wondering why we shipped only one Neon skill instead of one per feature. The reason is simple: **the SKILL.md file is always loaded into the agent’s context** (or is always considered available by the harness). That means we want to keep this file small, stable, and long-lived. + +Neon has a large surface area. If we create a top-level skill for every part of Neon, we would either overload the context or force users to install many skills. Both are unnecessary. It’s cleaner to ship one skill that gives a clear overview of Neon and then links into deeper detail. + +## Progressive discovery: how the using-neon skill is structured + +Modern coding agents are very good at **progressive discovery**, the same pattern you see in UI navigation when moving from `menu → submenu → detail`. Instead of giving the agent everything up front, we let it discover information step by step. + +The Neon skill follows this pattern: + +1. `SKILL.md` gives a tight overview of what the skill covers. +2. It points to files in a `reference/ directory`. +3. Those reference files point to specific Neon docs pages. +4. The agent is prompted to `curl` the right docs page when it needs details.
+ +This keeps the top-level skill small. It also makes navigation predictable: the agent follows the references like a decision tree until it has the information it needs. + +## The docs stay the source of truth + +Another reason we rely on progressive discovery is that **skills can go stale**. Once a user installs a skill, it lives on disk until they reinstall it with `npx skills add neondatabase/agent-skills`. + +That works, but most users won’t update their skills often. If we packed too much static content into the skill, it would drift away from the current state of the Neon API. So instead, we keep the source of truth in one place: [the Neon documentation.](https://neon.com/docs/introduction) + +We’ve optimized our docs for AI: + +- We publish an `LLMs.txt` file with a list of all documentation. This serves as the index for agents. +- Our docs accept `Accept: text/markdown`, so a simple curl returns clean Markdown. +- We put a lot of thought and our docs in general for a better developer experience which also serves agents well. + +The skill’s job is not to copy the docs; its job is to help the agent find and retrieve the right docs page at the right time. This avoids staleness and keeps the skill lightweight. + +## How the using-neon skill works with our MCP server + +Agent skills and MCPs actually solve different problems, so they work well together: + +- The `using-neon` skill provides best practices and instructions for reusable workflows, and decision logic. +- The [Neon MCP](https://github.com/neondatabase/mcp-server-neon) exposes real capabilities as authenticated, structured tools.
+ +In other words: the Neon skill teaches the agent _how_ to work with Neon (how to set up a new project, what connection method to pick based on the project architecture, how to set up different ORMs, how branching workflows looks like). And the MCP server lets the agent _act_ (create branches, inspect projects, list databases, query a database, and so on). + +It’s helpful to think of **skills as the reasoning layer**, and **MCP as the capabilities layer**. Both matter, but for different reasons. + +## Add using-neon to your project + +Thanks to `npx skills`, installing agent skills across coding agents is now straightforward. Check the repository out on GitHub here: [https://github.com/neondatabase/agent-skills](https://github.com/neondatabase/agent-skills), and install our `using-neon` skill using `npx skill add`: + +```bash +npx skills add neondatabase/agent-skills +``` diff --git a/content/blog/posts/agents-grew-up-so-did-our-docs.md b/content/blog/posts/agents-grew-up-so-did-our-docs.md new file mode 100644 index 0000000000..1b567c240c --- /dev/null +++ b/content/blog/posts/agents-grew-up-so-did-our-docs.md @@ -0,0 +1,160 @@ +--- +title: 'Agents grew up, so did our docs' +description: >- + Lessons from making Neon's docs agent-readable: MDX-to-Markdown pipelines, + content negotiation, llms.txt structure, and a scan of 250+ doc sites. +excerpt: >- + A year ago, if you asked an agent about Neon, you got whatever it + half-remembered from training. Now it goes looking and reads what it finds. + Our docs were written for humans who scroll, not machines that fetch. We’ve + been fixing this in pieces, not all at once. This post is what... +date: '2026-04-23T13:47:50' +updatedOn: '2026-04-24T15:54:40' +category: engineering +categories: + - engineering +authors: + - philip-olson +cover: + image: https://cdn.neonapi.io/public/images/pages/blog/agents-grew-up-so-did-our-docs/cover.jpg + alt: 'Agents grew up, so did our docs' +isFeatured: false +seo: + title: 'Agents grew up, so did our docs - Neon' + description: >- + Lessons from making Neon's docs agent-readable: MDX-to-Markdown pipelines, + content negotiation, llms.txt structure, and a scan of 250+ doc sites. + keywords: [] + noindex: false + ogTitle: 'Agents grew up, so did our docs - Neon' + ogDescription: >- + A year ago, if you asked an agent about Neon, you got whatever it + half-remembered from training. Now it goes looking and reads what it finds. + Our docs were written for humans who scroll, not machines that fetch. We’ve + been fixing this in pieces, not all at once. This post is what worked, what + didn’t, […] + image: https://cdn.neonapi.io/public/images/pages/blog/agents-grew-up-so-did-our-docs/cover.jpg +--- + +![Agents grew up, so did our docs](https://cdn.neonapi.io/public/images/pages/blog/agents-grew-up-so-did-our-docs/neon-cover-agents-docs-1-1024x538-5a7db1a4.jpg) + +A year ago, if you asked an agent about Neon, you got whatever it half-remembered from training. Now it goes looking and reads what it finds. Our docs were written for humans who scroll, not machines that fetch. + +We’ve been fixing this in pieces, not all at once. This post is what worked, what didn’t, and what we’re still figuring out. Maybe it saves you a few curl commands. + +## The setup + +Agents can read HTML fine. Crawlers have been at it for decades and modern agents handle it well. We just think we can do better. Our pages are built from dozens of rendered React components (``, ``, ``, ``), which expand into nested `
`s, class names, and event handlers in the final HTML. The actual docs are buried in there somewhere. + +You might think: just serve your source MDX from GitHub. We once did, and it works. MDX is Markdown with React components mixed in. Our MDX uses 30+ custom ones, and some, like ``, inline text from separate files at render time. An agent reading the raw MDX just sees the tag. + +You will correctly say: convert them. We do now, after plenty of yak shaving. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/agents-grew-up-so-did-our-docs/before-after-1024x467-2c74d041.jpg) + +## Phase 1: hand-maintained text files + +Our first approach: ask Claude for “one of them cool [llms.txt](https://llmstxt.org/) things that all the kids are talking about.” It produced a `public/llms/` directory, one `.txt` file per doc page, and an enormous `llms.txt` index listing them all. Keeping them current was a handful of Python scripts, run by hand, no CI. + +It worked. The thinking at the time was “feed the models” not “serve the agents” (the spec itself leans that way). Live fetching was new and rare. Predictably, the files drifted from the source, went missing, went stale weeks at a time. The implementation was an afterthought because the use case still felt like one. + +**The lesson:** if keeping two copies in sync is a manual job, they will drift. Clearer now than it was at the time. + +## Phase 2: teach the site to recognize agents + +What if the site detected agent requests and served something cleaner than HTML? We built middleware that checks the `User-Agent` (ChatGPT, Claude, Cursor, Copilot, and others) and the `Accept` header. When either matches, we serve Markdown instead. + +What we actually served was raw MDX from GitHub’s API with a `text/markdown` content type. Technically Markdown-ish, practically Markdown with a pile of React components. We hit GitHub rate limits within hours, switched to pre-built local files, still MDX. Detection was solved, content was not. + +## Phase 3: converting MDX to Markdown + +I (okay, Claude) wrote a Node.js post-build script that converts MDX to Markdown and writes it to `public/md/`, which we serve via URL rewrites. + +For example, `` becomes labeled code blocks. `` tags inline the referenced text directly. About 30 components handled, all from one file. + +The processor builds ~1,400 files in a few seconds. Doc authors edit MDX as usual. No manual sync, no drift, no thought. + +### Context matters too + +Clean Markdown isn’t enough. Agents need to know where they are and what to read next. So we wrap each page with a breadcrumb at the top and related docs at the bottom: + +```markdown +> This page location: Connect to Neon > Connection pooling +> Full Neon documentation index: https://neon.com/docs/llms.txt + +... + +## Related docs (Connect to Neon) +- [Connect to Neon](https://neon.com/docs/connect/connect-intro) +- [Choosing your connection method](https://neon.com/docs/connect/choose-connection) + +... +``` + +Without it, an agent fetches one page and doesn’t know what else is nearby. + +## What other sites are doing + +Nikita (Neon’s fearless leader) has a habit of pointing people back to first principles. It’s why we tend to build small tools instead of guessing, even when the tool’s whole point is to see how others are doing it. Ours, a scanner, probes doc sites and measures how they serve content to agents: same URL as HTML, with `.md` appended, `Accept: text/markdown`, discovery headers, plus variations. Findings across over 250 sites, mostly tech docs such as Vercel, Stripe, Mintlify, Sentry, and Google: + +- **53% serve Markdown by appending `.md` to the URL.** +- **41% honor content negotiation** via `Accept: text/markdown`. The ones that do also tend to have `llms.txt`, discovery headers, and structured indexes. They’ve thought about agents. About **30%** also accept `text/plain`. +- **`llms.txt` is common but placement varies.** **93%** of polled sites have one, and **58%** also publish `llms-full.txt` with concatenated doc content. The standard says place llms.txt in root. In practice, sites put it at `/docs/llms.txt`, at the root, or both. Some have different content at each path, and some use sub-indexes (child `llms.txt` files within `llms.txt`). +- **404 handling is mostly not content-type aware.** Only **9%** return Markdown for a 404 when Markdown was requested. The rest return HTML, and a handful return empty responses, even when the agent clearly asked for Markdown via `.md` or `Accept: text/markdown`. Of those **9%**, most sites return 200 instead of 404 (we chose 404). +- **Discovery hints are rarely used, and the conventions aren’t settled.** Only **9%** include a `` tag in the HTML head, a convention that emerged organically (ours did). The `X-LLMs-Txt` and `Link: rel="llms-txt"` headers Mintlify proposed have adoption almost entirely driven by Mintlify itself. +- **Headers are mixed and the impact is unclear.** Only **3%** set `Vary: Accept` on HTML **(6%** on Markdown). **27%** set `noindex` on Markdown. We’re still figuring out which of these actually help versus which are habit. + +Doc-specific platforms like Mintlify, GitBook, and Fern score near **100%** on most of these, because agent readiness is the point. Open-source frameworks are further behind and could use agent advocates. Tooling exists in the community but often sits unmaintained. + +## A few more lessons + +**404s should be helpful and aware, not empty.** Our 404s match the request: HTML for browsers, Markdown for agents, the latter returning links to the full index, the complete docs bundle, and the API reference. Idea stolen from a [Vercel tweet](https://x.com/studio_hungry/status/2035125161741963638) and implemented immediately. + +**Discovery has to be automatic, and responding to agents has to be too.** Agents don’t know to look for `llms.txt` or that appending `.md` works. Set discovery headers on every HTML response so they find out, and honor `Accept: text/markdown` when they do ask. Like children, they often ignore the reminders, but we do our best as parents. + +**The index needs structure, not just a list.** Our first `llms.txt` was a flat list of over 1,000 URLs. Way too much to parse before deciding what to read. We now restructure it with sections and descriptions, sub-indexes for large areas, a “Common Queries” section at the top (pricing, connection methods and troubleshooting, API reference), and collapsed routes for large but useful content (changelog, Postgres tutorials). The primary index is now ~200 entries with sub-indexes for the rest. + +**Agents use HTTP clients, not browsers.** Looking at User-Agent strings, we saw `axios`, `got`, `node-fetch` as often as named agents. Claude Code uses `axios`, Cursor uses `got`. The agent identity is in the tool, not always the header. We added those patterns to the detection list. A false positive (Markdown to a human) is harmless; a false negative (HTML to an agent) defeats the purpose. A real question: is changing content based on who’s asking a form of cloaking? + +## What the system looks like now + +Four layers: + +- **Build time.** The MDX processor converts source docs to Markdown. The index generator builds `llms.txt`, sub-indexes, and `llms-full.txt` (all docs concatenated). +- **URL rewrites.** Appending `.md` to any doc URL serves its Markdown version from `public/md/`. Non-doc pages will follow. +- **Middleware.** Detects agents via User-Agent and `Accept` headers. Serves Markdown transparently. Adds discovery headers to HTML responses. +- **Content.** Every doc page gets navigation context. The index is hierarchical. 404s are helpful and content-type aware. + +## What we’d do differently + +**One URL, two ways to ask for Markdown.** We built a parallel `/llms/` namespace first. Eventually we moved to serving Markdown from the canonical URL via a `.md` suffix or an `Accept: text/markdown` header. That should have been the starting point. + +**Invest in analytics earlier.** We added agent traffic tracking late. Having it from the start would have shown which pages agents request, which ones they 404 on, and how they navigate. That data would have shaped our system sooner. + +**Design the index first.** The flat file list was an afterthought. Structuring it with sections, descriptions, and sub-indexes earlier would have made it more useful. + +**Build the scanner first.** Studying other sites first would have saved us from reinventing patterns and surfaced cracks we didn’t think of until later. + +None of this was planned from the start. It came together one small change at a time. + +## What’s next + +Humans reach docs through agents, not just browsers. That’s the new audience and it doesn’t execute JavaScript or follow visual navigation. Agents want plain text, structured metadata, and machine-readable discovery. The tools aren’t exotic: a remark pipeline, some middleware, a few HTTP headers, a config file. The hard part is recognizing that and choosing to serve them. + +An agent can implement most of this for you. What it can’t do is write good content without review. + +Community tooling is catching up. The [afdocs](https://github.com/agent-ecosystem/afdocs) scorecard flagged a coverage issue in our `llms.txt` that we were briefly convinced wasn’t our problem, but it was. The associated [agent doc spec](https://github.com/agent-ecosystem/agent-docs-spec) is also growing, turning ad-hoc conventions into something documented. The tools are new, the category is new, and everyone is figuring it out together. + +On our list: + +- **Focus on accuracy.** Continue testing whether an agent can complete tasks using a given doc page, similar to agent skills testing. Goal: fewer mistake-then-fix cycles. +- **Offer interfaces built for agents.** Like search APIs, and ways for them to send feedback when we get something wrong. Markdown is a human format agents happen to parse well, and we can do better than that. +- **Think more about agent skills.** There’s something wrong with committing `.claude` folders into every repo. Treating them like `devDependencies` feels saner, and we’re watching how this evolves. +- **Continue integrating tools like `afdocs`.** Discuss with maintainers and submit PRs to include more (optional) checks, such as 404 handling and headers. +- But most importantly, what every doc site has tried to do since the dawn of time: **write good, reliable content.** Treat docs like code, like tests, like the source of truth. + +None of this is magic. Just small, honest work that only matters if the content is worth reading. + +## Thanks + +Thanks to Neon and Databricks for letting engineers experiment (and for the tokens), and to my docs-team colleagues Dan and Barry for keeping the real docs moving while I poked at this. diff --git a/content/blog/posts/agentuity-a-cloud-where-agents-can-actually-build.md b/content/blog/posts/agentuity-a-cloud-where-agents-can-actually-build.md new file mode 100644 index 0000000000..df8289cda8 --- /dev/null +++ b/content/blog/posts/agentuity-a-cloud-where-agents-can-actually-build.md @@ -0,0 +1,132 @@ +--- +title: 'Agentuity: A Cloud Where Agents Can Actually Build' +description: Agents need infrastructure made for them vs outdated primitives +excerpt: >- + “Agents don’t want heavyweight infrastructure that lives forever, they want + primitives they can spin up, use, and discard as part of their work. Neon fits + that model perfectly: it behaves the way agents actually think about state” + (Rick Blalock, Co-founder at Agentuity) Existing... +date: '2026-02-02T17:10:46' +updatedOn: '2026-02-02T17:40:12' +category: ai +categories: + - ai + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/agentuity-a-cloud-where-agents-can-actually-build/cover.jpg + alt: null +isFeatured: true +seo: + title: 'Agentuity: A Cloud Where Agents Can Actually Build - Neon' + description: >- + Agentuity is a new developer cloud built specifically for AI agents, + matching how they run, communicate, and manage state in production. + keywords: [] + noindex: false + ogTitle: 'Agentuity: A Cloud Where Agents Can Actually Build - Neon' + ogDescription: >- + Agentuity is a new developer cloud built specifically for AI agents, + matching how they run, communicate, and manage state in production. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/agentuity-a-cloud-where-agents-can-actually-build/social.png +--- + +
+ +
+ +
+

“Agents don’t want heavyweight infrastructure that lives forever, they want primitives they can spin up, use, and discard as part of their work. Neon fits that model perfectly: it behaves the way agents actually think about state”



(Rick Blalock, Co-founder at Agentuity)

+
+ +Existing cloud infrastructure is built around the software model developers have used for years: you deploy an app, expose an API, and your infrastructure exists to serve those requests as efficiently as possible. + +When AI agents are the ones building though, things work differently. + +## The three infra roadblocks agents keep hitting + +Agents don’t exclusively fit execution models built around short-lived requests. They reason, iterate, pause, resume – they run for seconds or minutes or hours at a time – they manage context, spawn other agents, adapt their behavior as they go… As soon as human teams try using agents to build anything beyond a simple demo, they start running into the friction points you’d expect from infrastructure that was never designed for autonomous AI. + +### Agents don’t fit request–response infrastructure + +Most cloud platforms are optimized for short execution windows. When an agent hits a hard timeout (for example, 30 seconds), execution is cut off and developers are forced to rethink their architecture. Any agent system built around long-running reasoning loops, background tasks, or stateful workflows doesn’t map cleanly to the serverless tooling offered by most commercial clouds today. + +### Multi-agent systems are hard to wire securely + +The moment you introduce more than one agent, things get even more finicky. Agents need to communicate with each other, sometimes within the same system, sometimes across environments, and they need secure networking, authentication, and coordination. With current cloud tooling, this usually means stitching together networking rules, service discovery, and security layers by hand. + +### Developers end up over-architecting instead of letting agents build + +Developers are used to thinking in terms of apps, services, and APIs, while agents operate on tasks. What agents actually need to thrive are primitives they can use directly: a database to manage state, storage for intermediate results, a sandbox to execute code, and tools they can spin up, use, and discard. Instead, they inherit frameworks and infrastructure designed around an older model, and end up being constrained by the assumptions and preconceptions of the human overseeing the process. + +## A cloud designed for agentic systems from first principles + +Instead of forcing agents into app-centric infrastructure, we need a cloud where agents can run, coordinate, and manage state naturally without fighting the underlying platform. This is what [Agentuity](https://agentuity.com/) is building: [a new developer cloud designed specifically for agentic software.](https://agentuity.com/blog/agentuity-v1-is-here) + + + +Instead of adapting existing infrastructure to support agents, this new platform treats agents as the organizing unit of the system and designs its primitives around how agents actually run, communicate, and manage state: + +### Long-running execution without artificial limits + +Agents in Agentuity can run for minutes or hours, pause and resume work, and wait on external signals without being cut off. There’s no need to break workflows into awkward chains of callbacks or fall back to VMs just to keep agents alive. Execution lifecycles are designed to match how agents actually reason and operate, not how HTTP requests behave. + +### Agent-native communication and coordination + +Secure agent-to-agent communication is a first-class capability of the platform. Agents can hand off work, collaborate, and share context without developers having to manually configure networking, service discovery, or authentication. + +### Direct access to core infrastructure primitives + +Agentuity exposes infrastructure as primitives agents can use directly. Compute, storage, and databases are available as tools that agents can create, interact with, and clean up as part of a task’s natural flow. Agents can manage state, persist intermediate results, and execute code without relying on long-lived, manually managed infrastructure. + +
+ +
Explore more at agentuity.com
+
+ +### A familiar developer experience on top + +From the outside, Agentuity still looks like a standard developer platform. Developers work with code, SDKs, APIs, and a web console. The difference is in what those tools are optimized for – tasks and autonomy rather than apps and endpoints. The familiar surface area makes it easy to adopt, while the underlying model is purpose-built for agentic systems. + +
+ +
On Agentuity by Ben Davis
+
+ +## Neon as an agent-native database primitive + +
+

“Neon turns a database into something an agent can actually use. Spin it up, load data, reason over it, shut it down when the task is done. That’s exactly how agents want to work” (Rick Blalock, Co-founder at Agentuity)

+
+ +Once agents become the organizing unit of the system, databases stop looking like long-lived infrastructure and start behaving like tools. [Neon](https://neon.com/)’s model aligns with how agents think about state: databases are fast to create, affordable to operate, and flexible enough to be ephemeral or long-lived depending on the workload. + +In Agentuity, databases are treated as task-level tools, not permanent systems: + +### Databases as working memory for agents + +- Agents can create Postgres databases on demand as part of their execution flow +- Databases can exist for minutes, hours, or indefinitely, depending on what the agent needs +- State is isolated per task or per agent, avoiding shared, long-lived infrastructure +- Cleanup happens automatically when the task completes + +An example: + +- An agent working on research starts with a large CSV file containing raw data +- Instead of trying to hold everything in memory or repeatedly re-process the file, the agent spins up a Neon database, loads the CSV, and uses SQL to query, segment, and enrich the data as it reasons through the problem +- Once the task is complete, the agent shuts the database down, without long-lived resources left behind + +### Provisioning databases programmatically via Neon’s API + +Neon’s mature [API-first model](https://neon.com/blog/provision-postgres-neon-api) is also critical at the platform level. Agentuity exposes databases as one of its built-in primitives, meaning they are provisioned programmatically on behalf of users and agents, without requiring credentials or manual setup of any kind. Provisioning and teardown are fast enough in Neon to be part of normal agent workflows. + +## The future is now + +Agentic software is pushing past the limits of app-centric infrastructure. Platforms like Agentuity are rebuilding the cloud from first principles so agents can run, coordinate, and manage state the way they naturally work. [Try it out for free.](https://app-v1.agentuity.com/?_gl=1*1todeia*_gcl_au*MTA0ODM2NDMxNy4xNzY5NjI3NDQ4*_ga*MTA3NDk4ODE0OS4xNzY5NjI3NDQ4*_ga_DW10XSJ0H2*czE3Njk2Mjc0NDckbzEkZzEkdDE3Njk2Mjg0MTIkajQ4JGwwJGgw) + + +Agentuity is using our [Agent Plan](https://neon.com/use-cases/ai-agents). If you’re building a full-stack agent or agentic tooling and need Postgres databases, apply to get credits, higher resource limits, and direct support as you scale. + diff --git a/content/blog/posts/ai-rules-bring-neon-context-into-your-editor.md b/content/blog/posts/ai-rules-bring-neon-context-into-your-editor.md new file mode 100644 index 0000000000..ab30806b51 --- /dev/null +++ b/content/blog/posts/ai-rules-bring-neon-context-into-your-editor.md @@ -0,0 +1,121 @@ +--- +title: 'AI Rules: Bring Neon Context into Your Editor' +description: Make AI tools like Cursor Neon-aware +excerpt: >- + Using AI coding assistants like Cursor to build your full-stack apps makes for + a powerful workflow, but they don’t always know the specifics of every tool – + including Neon – and can do a little help. To give your editor Neon-specific + context and help it generate better code from... +date: '2025-09-02T18:00:24' +updatedOn: '2025-10-17T16:59:13' +category: ai +categories: + - ai + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ai-rules-bring-neon-context-into-your-editor/cover.jpg + alt: null +isFeatured: true +seo: + title: 'AI Rules: Bring Neon Context into Your Editor - Neon' + description: >- + To give your editor Neon-specific context, we’ve created a set of rules you + can add to your AI tool to teach it the right patterns for Neon. + keywords: [] + noindex: false + ogTitle: 'AI Rules: Bring Neon Context into Your Editor - Neon' + ogDescription: >- + To give your editor Neon-specific context, we’ve created a set of rules you + can add to your AI tool to teach it the right patterns for Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ai-rules-bring-neon-context-into-your-editor/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ai-rules-bring-neon-context-into-your-editor/neon-ai-rules-1-1024x576-35afafdb.jpg) + +Using AI coding assistants like Cursor to build your full-stack apps makes for a powerful workflow, but they don’t always know the specifics of every tool – including Neon – and can do a little help. To give your editor Neon-specific context and help it generate better code from the start, we’ve created a set of [rules](https://neon.com/docs/ai/ai-rules) you can add to your AI tool to teach it the right patterns for Neon. + +## What Are AI Rules? + +If you haven’t used AI rules before, they are small `.mdc` files that guide your coding assistant on how to handle specific contexts in your codebase. Each rule defines which file types it applies to, e.g.`*.tsx` or `schema.sql` and includes a brief description that explains when and why to use the rule. When the AI editor processes those files, it automatically adds the relevant rules as context, helping the assistant generate more accurate and helpful results. + +For Neon in particular, these rules capture best practices and recommended patterns that are specific to our stack. Instead of generic Postgres or ORM advice, you get guidance that’s tailored to Neon. + +## Available Neon Rules + +We’ll be adding more rules overtime, but we started with three rules that cover some common areas where we observed AI assistants struggling often: + +### Neon Auth + +Authentication can be tricky to get right. In Neon particularly, you’re combining [Stack Auth](https://stack-auth.com/) on the frontend with [Neon Auth](https://neon.com/docs/neon-auth/overview) in the database, so some context really helps. The `neon-auth.mdc` rule teaches your AI assistant the right setup – including how to protect pages and how to safely join against the `neon_auth.users_sync` table, which is critical for your auth to work. + +So you see an example of what the rule contains, here’s an excerpt from `neon-auth.mdc`: + +```bash +## Page protection +- In Server Components, use `stackServerApp.getUser({ or: "redirect" })`. +- In Client Components, use `useUser({ or: "redirect" })`. + +## Database usage +- Join against `neon_auth.users_sync` with LEFT JOIN. +- Always filter out rows where `deleted_at IS NOT NULL`. +- Never create FKs to `neon_auth.users_sync`; never insert directly. +``` + +Apart from this, `neon-auth.mdc` also includes guidelines on + +- Initial setup of Stack Auth in your Next.js app +- Using pre-built components like `` and `` +- Protecting routes and middleware patterns + +By including Neon Auth rules in your project, simple prompts like “add auth to the project” are much more likely to work seamlessly, allowing the AI assistant to automatically apply the correct authentication patterns and configurations for Neon. + +[Read all the code in our docs.](https://neon.com/docs/ai/ai-rules-neon-auth) + +### Neon Serverless Driver + +Another area in which the AI could do some help is our serverless driver [(600k+ weekly downloads](https://www.npmjs.com/package/@neondatabase/serverless)). The `neon-serverless.mdc` rule will help your AI assistant suggest the right patterns when connecting to Neon from serverless environments, preventing mistakes you really want to avoid (e.g. hardcoding credentials). + +The rule includes things like this (excerpt from `neon-serverless.mdc`): + +```bash +## Connections +- Always use `import { neon } from '@neondatabase/serverless'`. +- Never hardcode DATABASE_URL credentials. +- Use template literals with the `sql` tag for parameters (avoid string concat). +- Open and close pools inside request handlers, not at module scope. +``` + +Other things to look for when handling serverless connections are: + +- Using transactions correctly in serverless functions +- Handling WebSocket support in older Node.js runtimes +- Error handling for query failures and idle clients + +This is all covered by `neon-serverless.mdc. ` [Check out the complete rule in our docs.](https://neon.com/docs/ai/ai-rules-neon-serverless) + +### Neon + Drizzle ORM + +Using Neon with Drizzle ORM is a favorite combo for many developers. To make sure everything works great, the `neon-drizzle.mdc` rule teaches your AI assistant how to set up Drizzle correctly with Neon – here’s what it covers (maybe it teaches you something as well): + +- **Initial setup.** The rule prevents AI from defaulting to the Node Postgres adapter, which isn’t optimized for serverless. The rule ensures the AI selects the optimal Neon serverless driver, avoiding the default Node Postgres adapter, which isn’t ideal for serverless environments. +- **Schema design.** It encourages to use powerful Postgres-native types fully supported by Neon when schemas are being set up, like jsonb, enums (pgEnum), and arrays. +- +- **Query optimization.** This tells the AI to prevent inefficient one-by-one queries that slow down serverless apps, using batch inserts and use prepared statements where possible. +- **Branching.** Neon [branches](https://neon.com/branching) work wonderfully with Drizzle – this rule makes sure coding assistant knows how to properly handle Neon’s branching model, using branches when creating different environments vs hardcoding a single DB. +- **Error handling.** This section is set to capture some Neon-specific errors, like pool timeouts, in a way that’s easier to diagnose and fix. + +## Get Started with AI Rules + +Adding these rules to your editor is simple. You have two options: + +1. **Copy the files into your project.** With Cursor, save the [rules](https://docs.cursor.com/context/rules-for-ai#project-rules-recommended) to .cursor/rules/neon-serverless.mdc and they’ll be automatically applied. If you’re using another editor, check their docs for the right method for loading rules manually. +2. **Or clone them from the repo.** You can also grab the rules directly [from our repo.](https://github.com/neondatabase-labs/ai-rules) Once they’re in your project, your editor will use them automatically, and you can also reference them explicitly in prompts when you want to guide the AI. +3. **Get them through Neon’s** [MCP Server](https://neon.com/docs/ai/neon-mcp-server). If you are already leveraging Neon’s MCP server for your development, simply prompt it with “Give me Neon specific AI rules,” and the server will provide the same rules we have in our repo. + +--- + +_Neon is a serverless Postgres database designed to help you build reliable and scalable applications faster – from prototypes all the way to production – with minimal manual configuration._ [Start building for free with our free plan.](https://console.neon.tech/signup) diff --git a/content/blog/posts/ai-workflows-for-docs-devin.md b/content/blog/posts/ai-workflows-for-docs-devin.md new file mode 100644 index 0000000000..1380c7297a --- /dev/null +++ b/content/blog/posts/ai-workflows-for-docs-devin.md @@ -0,0 +1,113 @@ +--- +title: 'AI Workflows for Docs: Putting Devin to Work' +description: >- + How we’re using Devin to help with changelogs, PR reviews, and large-scale + edits across our documentation +excerpt: >- + At Neon, our docs team does a little bit of everything. We work on technical + documentation, sometimes UI copy, changelogs, reviews, and the occasional + regex-heavy cleanup across hundreds of pages. It’s a lot of small, steady work + – often, exactly the kind of work you wish an AI a... +date: '2025-06-23T17:10:34' +updatedOn: '2025-08-19T18:14:55' +category: ai +categories: + - ai +authors: + - daniel-price + - barry-grenon +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ai-workflows-for-docs-devin/cover.jpg + alt: null +isFeatured: false +seo: + title: 'AI Workflows for Docs: Putting Devin to Work - Neon' + description: >- + We’ve been experimenting with Devin to lighten the load on our docs team, + automating some of the more repetitive tasks. + keywords: [] + noindex: false + ogTitle: 'AI Workflows for Docs: Putting Devin to Work - Neon' + ogDescription: >- + We’ve been experimenting with Devin to lighten the load on our docs team, + automating some of the more repetitive tasks. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ai-workflows-for-docs-devin/social.jpg +--- + +At [Neon](https://neon.com/), our docs team does a little bit of everything. We work on technical documentation, sometimes UI copy, changelogs, reviews, and the occasional regex-heavy cleanup across hundreds of pages. It’s a lot of small, steady work – often, exactly the kind of work you wish an AI agent could just take off your plate. + +That’s what got us curious about [Devin](https://devin.ai/). Unlike other tools we tried, Devin doesn’t just suggest code or text inline; you give it instructions, and it opens a full pull request on your behalf. It’s designed to work across large codebases, operate autonomously, and take care of actual workflows. + +That sounded promising for documentation work, which often involves well-defined, repetitive changes or coordination across multiple files. We didn’t expect Devin to write our docs for us, but we wondered if it could become a kind of assistant. We’ve been trying it out: here’s what’s working so far. + +## Docs Automation with Devin: What’s Working So Far + +### Simple tasks, automated fast + +Some of Devin’s biggest wins in our experience come from saving us time on small, repetitive work. For example: we use a Devin Playbook every week to set up our [changelog file.](https://neon.com/docs/changelog) It’s not a complicated task, but it used to be a manual one; now we just say, “Set up a changelog for this Friday,” and Devin gets it started. We’ve standardized the process using a [playbook](https://docs.devin.ai/product-guides/creating-playbooks). + +We also use Devin often for addressing GitHub Issues and PRs. We can point Devin to an issue, tell it to review the code and find the relevant parts of the docs to update, and it’ll create or revise a PR with the changes. That’s helped us work through a backlog of older docs issues without needing to manually audit every one. + +### Writing docs with context awareness + +One of our favorite moments with Devin was when we asked it to document a small new feature. We expected a basic write-up, but when we opened the PR, it included a screenshot (!) + +At first, we thought it was a placeholder. But Devin had actually found a relevant image elsewhere in the repo (a .png that matched the UI being described) and inserted it appropriately. There was nothing in our prompt that mentioned the image, and nothing in the surrounding text that made it an obvious fit. + +Our guess is that Devin used OCR or some kind of visual recognition to infer the match. Either way, it was a genuinely helpful addition, and a great little glimpse of how powerful context-aware agents will become. + +### Multi-file edits without the regex headache + +One of the most annoying aspects of documentation work is when a feature name or term changes. You then need to find and update dozens of references across the docs, without accidentally breaking context or formatting. Traditionally, this meant writing cautious regex passes and still doing lots of cleanup. + +Naturally, we tried Devin to see how it did here. When we renamed a feature from “history retention” and “point-in-time recovery” to the simpler term [Instant Restore](https://neon.com/docs/introduction/branch-restore), we asked Devin to help. The results were mixed. Devin _did_ find nearly every place in the docset where the old terms were used, something that would have taken a lot of manual effort – that was a win. But its edits leaned heavily on direct find/replace, which led to some awkward constructions like “instant instant restore.” It took another 20+ manual commits to polish the changes. + +Even if not perfect, having Devin surface all the right places to edit was a huge time-saver. This is exactly the kind of clerical work we’d love to hand off to an AI agent, and Devin’s close. + +### Explaining code when we need it + +As a docs team, we often have to understand implementation details in the codebase. Devin has been genuinely helpful here. + +Practical example: recently, our Support team asked for details on the usage alerts customers see when approaching plan limits. After some recent changes, we weren’t sure how those alerts were triggered behind the scenes – and instead of digging through the code ourselves or asking an engineer, we asked Devin. It traced the logic and explained how the alerts worked, saving us a chunk of time and Slack messages. + +When we don’t have full context (or just need a second brain to walk us through something) we’ve found that Devin is a great tool for quick explanations. [Cursor](https://neon.com/blog/how-were-using-cursor-at-neon) still tends to be our go-to when we want to work through something interactively, but Devin is very useful when the ask is simple and well-scoped. + +### UI copy + +The Docs team also contributes to the Neon Console. especially small things like copy tweaks, tooltip fixes, and style updates. But rather than spin up the full dev environment to make tiny UI changes, we’ve started asking Devin to do them for us. + +In one case, [we spotted a tooltip style issue and sent Devin to fix it.](https://github.com/neondatabase/cloud/pull/29396) In another, we needed to revise a problematic UI component and didn’t want to wait for engineering to prioritize it – [Devin handled the change.](https://github.com/neondatabase/cloud/issues/27073) This kind of UI work is great for Devin, where the scope is small but the implementation is still real. + +## What We’re Still Missing + +After experimenting so much with Devin, it’s also worth sharing our perspective on those areas where it still falls short. + +### Black-box behavior + +One of the trickiest things about working with Devin is that it often feels like a black box. You give it a goal, walk away, and later a pull request appears – sometimes with good results, sometimes not. + +In one case, we asked Devin to help us ingest an external Markdown file into our docs site at runtime. The idea was to avoid having to copy-paste content from another repo and instead have a single source of truth that remained discoverable within the docs. We used ChatGPT to help plan out the feature and fed that plan into Devin, but Devin just couldn’t figure it out. The frustrating part wasn’t just that it failed, it was that it failed silently. We didn’t get real-time feedback or error signals, and it didn’t ask for clarification. It just kept going until we happened to check in. + +That kind of thing happens often, and it makes working with Devin feel a bit like a gamble. You’re either pleasantly surprised or confused by something Devin misunderstood hours ago. + +### Complex, fuzzy tasks are still too much for Devin + +Devin does well when the problem is clearly scoped, but once things get open-ended, it still struggles. + +We ran into this while trying to build a DNS resolution checker for the docs, something users could use to troubleshoot regional database connectivity issues. The request was a bit under-defined, and there was some real complexity involved. This is an example of something that Devin couldn’t handle. + +To be fair, the task was probably a stretch for any AI assistant, but it reinforced something we’ve seen a few times: Devin is best when it’s executing a clean, defined task. Once there’s ambiguity or edge cases, it’s much harder to trust the outcome. + +### Lack of interactivity + +Lastly, compared to tools like Cursor, Devin feels a little _lonely_. With Cursor, we can work alongside the LLM, see what it’s doing, and steer it in real time. That fits how a lot of us prefer to work – jumping in, exploring, refining on the fly. + +Devin’s model, by contrast, is more like giving an assistant a task and hoping for the best. We’d love to see Devin adopt a more interactive mode, where it pauses to ask for help when stuck or lets us check in mid-task before pushing a PR. Just an idea! + +## Our Conclusion + +Devin isn’t a replacement for writing or judgment, and it’s not always predictable – but it’s already helping us move faster on the kinds of tasks that slow our team down. We’re still figuring out where it fits best in our workflow, and where it needs more guardrails or guidance. But when the task is clear and the scope is right, it’s a genuinely helpful assistant. + +We’ll keep sending Devin on errands, and we’ll keep sharing what we learn 🙂 diff --git a/content/blog/posts/announcing-branch-reset.md b/content/blog/posts/announcing-branch-reset.md new file mode 100644 index 0000000000..02073f56c9 --- /dev/null +++ b/content/blog/posts/announcing-branch-reset.md @@ -0,0 +1,123 @@ +--- +title: Announcing Branch Reset +description: Pull the latest data and schema from a parent branch +excerpt: >- + One of the benefits of Neon’s serverless architecture, which sets it apart + from other Postgres providers, is its database branching capabilities. Neon’s + database branching allows you to create an instant copy-on-write clone of your + data (a child branch) that you can modify withou... +date: '2023-12-14T12:35:03' +updatedOn: '2024-03-27T11:26:53' +category: workflows +categories: + - workflows +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-branch-reset/cover.jpg + alt: null +isFeatured: false +seo: + title: Announcing Branch Reset - Neon + description: Pull the latest data and schema from a parent branch + keywords: [] + noindex: false + ogTitle: Announcing Branch Reset - Neon + ogDescription: >- + One of the benefits of Neon’s serverless architecture, which sets it apart + from other Postgres providers, is its database branching capabilities. + Neon’s database branching allows you to create an instant copy-on-write + clone of your data (a child branch) that you can modify without compromising + your main data (parent branch). You can use it to manage […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-branch-reset/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/announcing-branch-reset/neon-database-branch-reset1-1024x576-22acba36.jpg) + +One of the benefits of Neon’s serverless architecture, which sets it apart from other Postgres providers, is its database branching capabilities. + +Neon’s [database branching](https://neon.tech/docs/introduction/branching#what-is-a-branch) allows you to create an instant copy-on-write clone of your data (a child branch) that you can modify without compromising your main data (parent branch). You can use it to manage your database environments, for development purposes, or incorporate it into your CI/CD pipeline using the [Neon API](https://neon.tech/docs/reference/api-reference) or [CLI](https://neon.tech/docs/reference/cli-install). + +Today, we are announcing the release of our newest feature for your developer workflows: branch reset. + +## Streamline your development workflows with Branch Reset + +Imagine you’re working on a new feature in a development branch of your database. While you’re busy coding, one of your teammates merges a pull request that includes critical schema changes. This is where Neon’s branch reset feature comes into play. + +Branch reset functions much like a \`git reset –hard parent\` in traditional Git workflows. It allows you to seamlessly update your development branch with the latest schema and data from the main branch. This ensures that your work remains compatible with the most recent changes made by your team. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/announcing-branch-reset/image-4-1024x600-d909f0fd.png) + +Here’s how it works in practice. + +1. Suppose you have a project with the id=raspy-water-84552850. You can find the id associated with your project using the `neonclt projects list` command or in your project’s settings. You can create a new branch using the following command: + +```bash +neonctl branches create --project-id=raspy-water-84552850 +``` + +Output: + +```bash +┌──────────────────────────┬──────────────────────────┬─────────┬──────────────────────┬──────────────────────┐ +│ Id │ Name │ Primary │ Created At │ Updated At │ +├──────────────────────────┼──────────────────────────┼─────────┼──────────────────────┼──────────────────────┤ +│ br-divine-glade-19837478 │ br-divine-glade-19837478 │ false │ 2023-12-13T16:14:56Z │ 2023-12-13T16:14:56Z │ +└──────────────────────────┴──────────────────────────┴─────────┴──────────────────────┴──────────────────────┘ +endpoints +┌─────────────────────────┬──────────────────────┐ +│ Id │ Created At │ +├─────────────────────────┼──────────────────────┤ +│ ep-winter-hall-10626581 │ 2023-12-13T16:14:56Z │ +└─────────────────────────┴──────────────────────┘ +connection_uris +┌────────────────────────────────────────────────────────────────────────────────────────────┐ +│ Connection Uri │ +├────────────────────────────────────────────────────────────────────────────────────────────┤ +│ postgres://:@ep-winter-hall-10626581.us-east-2.aws.neon.tech/neondb │ +└────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +Notice that output contains the branch id `br-divine-glade-19837478`. You can always find your branch-id using `neonctl branches list` command like so: + +```bash +neonctl branches list --project-id=raspy-water-84552850 +``` + +Output: + +```bash +┌────────────────────────────┬──────────────────────────┬─────────┬──────────────────────┬──────────────────────┐ +│ Id │ Name │ Primary │ Created At │ Updated At │ +├────────────────────────────┼──────────────────────────┼─────────┼──────────────────────┼──────────────────────┤ +│ br-divine-glade-19837478 │ br-divine-glade-19837478 │ false │ 2023-12-13T16:14:56Z │ 2023-12-14T11:16:04Z │ +├────────────────────────────┼──────────────────────────┼─────────┼──────────────────────┼──────────────────────┤ +``` + +2. You can then instantly align your branch with the latest version of the main branch using the following command in the CLI: `neonctl branches reset --parent`. For example: + +```bash +neonctl branches reset br-divine-glade-19837478 --parent --project-id=raspy-water-84552850 +``` + +Output: + +```bash +┌──────────────────────────┬──────────────────────────┬─────────┬──────────────────────┬──────────────────────┐ +│ Id │ Name │ Primary │ Created At │ Last Reset At │ +├──────────────────────────┼──────────────────────────┼─────────┼──────────────────────┼──────────────────────┤ +│ br-divine-glade-19837478 │ br-divine-glade-19837478 │ false │ 2023-12-13T16:14:56Z │ 2023-12-14T11:16:03Z │ +└──────────────────────────┴──────────────────────────┴─────────┴──────────────────────┴──────────────────────┘ +``` + +The reset process fetches the latest data and schema from the parent branch, integrating them into your current branch. + +## Limitations + +The reset process fetches the latest data and schema from the parent branch, integrating them into your current branch. However, it’s essential to be aware of certain limitations. + +The reset operation is a complete overwrite, meaning any local changes in your branch will be lost. Also, while the reset is in progress, database connections will be temporarily interrupted, though they will automatically re-establish once the reset is completed. + +By leveraging the branch reset feature, you can ensure that your development efforts are always in sync with your team’s progress, leading to a more efficient and collaborative development process.
We encourage you to log in or [create a Neon account](https://console.neon.tech/login) to explore the branch reset feature and see how it can transform your development process. [Read more about branch reset in the docs](https://neon.tech/docs/manage/branches#reset-a-branch-from-parent) and share your feedback. Join our [Discord](https://neon.tech/discord) and tell us what you think. diff --git a/content/blog/posts/announcing-monitoring-and-organizations.md b/content/blog/posts/announcing-monitoring-and-organizations.md new file mode 100644 index 0000000000..7f8d742ecc --- /dev/null +++ b/content/blog/posts/announcing-monitoring-and-organizations.md @@ -0,0 +1,111 @@ +--- +title: Announcing Monitoring and Organizations +description: Announcing enhanced monitoring and organizations on Neon. +excerpt: >- + Neon’s serverless Postgres improves developer velocity and allows + organizations to ship faster. When using a database for your app, monitoring + its performance is crucial. You can do that using Postgres extensions such as + pg_stat_statements. However, if you’d rather do it from the... +date: '2024-04-16T16:10:15' +updatedOn: '2024-04-17T09:33:35' +category: community +categories: + - community +authors: + - evan-shortiss +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/cover.jpg + alt: Neon Monitoring Blogpost Cover +isFeatured: false +seo: + title: Announcing Monitoring and Organizations - Neon + description: Announcing enhanced monitoring and organizations on Neon. + keywords: [] + noindex: false + ogTitle: Announcing Monitoring and Organizations - Neon + ogDescription: >- + Neon’s serverless Postgres improves developer velocity and allows + organizations to ship faster. When using a database for your app, monitoring + its performance is crucial. You can do that using Postgres extensions such + as pg_stat_statements. However, if you’d rather do it from the UI, we’re + happy to announce the release of the monitoring page on the […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/social.jpg +--- + +![Neon Monitoring Blogpost Cover](https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/cover.jpg) + +Neon’s serverless Postgres improves developer velocity and allows organizations to ship faster. + +When using a database for your app, monitoring its performance is crucial. You can do that using Postgres extensions such as pg_stat_statements. However, if you’d rather do it from the UI, we’re happy to announce the release of the monitoring page on the Neon console. + +We also understand that modern application development requires collaboration and teamwork, so we added Organizations to the Console, which is currently in Private Preview. Customers on the Launch and Scale plans can request access by contacting [customer-success@neon.tech](mailto:customer-success@neon.tech). + +Let’s take a quick tour of these new features. + +## Monitoring + +Using Neon means using Postgres. When you interact with Neon, you aren’t doing so using a wrapper – you’re interacting directly with a Postgres database. + +This provides you with a great deal of power and flexibility. It also means you can exhaust connections, cause deadlocks, and max out your allocated CPU and RAM. That’s why we’ve shipped a comprehensive [monitoring dashboard](https://neon.tech/docs/introduction/monitoring-page) in the Neon console. + +The usual suspects, CPU and RAM, are present, but it also includes: + +- Connections count +- Buffer cache hit rate +- Database size +- Deadlocks +- Rows + +Understanding your application’s baseline performance and general performance trends is essential. Database branching alongside this knowledge enables teams to ship with greater confidence since they can use branches to test and compare their changes against their baseline. As the saying goes, [“Measure twice, cut once”](https://en.wiktionary.org/wiki/measure_twice_and_cut_once). + +## Take Monitoring for a Spin with pgbench + +Take a test scenario created using `pgbench` as an example. First, prepare a database using `pgbench`. + +```bash +pgbench -i -s 100 $DATABASE_URL +``` + +This results in a database that’s approximately 1.5 GiB in size, assuming it was empty prior. You can confirm the size using the new metrics **dashboard database** size field. + +Now you can test the performance of this database and generate metrics using the following `pgbench` command. Try various values for the number of concurrent clients (`-c`) and see how the progress report changes every 30 seconds. + +```bash +pgbench -c 95 -j 4 -P 30 -t 10000 $PG_URL +``` + +Using the default 0.25 CU [compute size](https://neon.tech/docs/manage/endpoints#compute-size-and-autoscaling-configuration) results in a lower number of transactions per second (TPS) versus 3 CU. You can see this clearly in the screenshot below. The cache hit rate increases dramatically after autoscaling is enabled and up to 12 GB of RAM becomes available to Postgres. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/monitoring-cache-hit-1024x579-398978bf.jpg) + +Monitoring can help you identify potential performance issues, but the metrics can also help you understand your storage usage. The **Rows** metric tracks the inserts, updates, and deletes you’ve made. More changes mean a larger [history](https://neon.tech/docs/reference/glossary#history), which means [increased storage usage](https://neon.tech/docs/introduction/usage-metrics#storage-details) if you have a large history retention window configured. + +## Organizations + +Each project in Neon provides an isolated database timeline, the ability to create read-write endpoints and read-replicas, and branches for development and testing. Projects can be [shared with other Neon users](https://neon.tech/docs/manage/projects#share-a-project) to facilitate collaboration. + +Today, we’re announcing our Organizations feature in Private Preview. An organization represents a collection of projects, and members of the organization have access to those projects. + +Take the following screenshot, for example. It looks like a regular Neon account that contains a single project, right? + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/orgs-personal-1024x575-448401e0.jpg) + +Clicking the dropdown in the navigation bar reveals that this user is part of an organization, **Evan’s org**. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/orgs-selector-1024x580-557fb2e1.jpg) + +Selecting the organization reveals that the user can access the two projects within the organization. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/announcing-monitoring-and-organizations/org-two-projects-1024x577-85fb62f0.jpg) + +Projects within an organization can be shared just like personal projects, so you can invite individuals to a project, even if it’s within an organization. If you’re keen to try out organizations, then you should get in touch with [customer-success@neon.tech](mailto:customer-success@neon.tech). + +## Conclusion + +Monitoring will provide you with better insights into your Neon usage and help you better understand your application’s traffic and load patterns. + +Our new organizations feature is the next step for us in supporting our customers who are managing a fleet of Postgres databases and teams on Neon. + +Remember, contact our customer success team if you’d like early access to organizations! Join us in [Discord](https://neon.tech/discord), follow us on [X](https://x.com/neondatabase), and let us know what observability tools you’d like us to integrate with so you can scale your applications to millions of users. diff --git a/content/blog/posts/announcing-neon-snapshots-a-smoother-path-to-recovery.md b/content/blog/posts/announcing-neon-snapshots-a-smoother-path-to-recovery.md new file mode 100644 index 0000000000..49121d3528 --- /dev/null +++ b/content/blog/posts/announcing-neon-snapshots-a-smoother-path-to-recovery.md @@ -0,0 +1,90 @@ +--- +title: 'Announcing Neon Snapshots: A Smoother Path to Recovery' +description: >- + Bounce back from setbacks with read-only snapshots designed for disaster + recovery, safe testing, and stress-free rollbacks. +excerpt: >- + When working with databases, there are moments when you’d love to freeze time + — before making a big change, running a migration, or simply bookmarking a + stable state. With snapshots in Neon, that’s now possible. Snapshots provide + an easy way to capture a backup of your database a... +date: '2025-04-23T15:50:07' +updatedOn: '2025-08-14T09:26:35' +category: product +categories: + - product + - engineering +authors: + - bryan-clark +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/cover.jpg + alt: 'Announcing Neon Snapshots: A Smoother Path to Recovery' +isFeatured: false +seo: + title: 'Announcing Neon Snapshots: A Smoother Path to Recovery - Neon' + description: >- + Bounce back from setbacks with read-only snapshots designed for disaster + recovery, safe testing, and stress-free rollbacks. + keywords: [] + noindex: false + ogTitle: 'Announcing Neon Snapshots: A Smoother Path to Recovery - Neon' + ogDescription: >- + When working with databases, there are moments when you’d love to freeze + time — before making a big change, running a migration, or simply + bookmarking a stable state. With snapshots in Neon, that’s now possible. + Snapshots provide an easy way to capture a backup of your database at a + specific point in time and are […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/social.jpg +--- + +![Announcing Neon Snapshots: A Smoother Path to Recovery](https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/neon-snapshot-cover-1-1024x576-80d09390.jpg) + +When working with databases, there are moments when you’d love to freeze time — before making a big change, running a migration, or simply bookmarking a stable state. With snapshots in Neon, that’s now possible. Snapshots provide an easy way to capture a backup of your database at a specific point in time and are now available to all users in our [Early Access Program](https://console.neon.tech/app/settings/early-access), and we’re eager to hear your feedback. Give them a try by joining the program today! + +## What are snapshots? + +Snapshots are point-in-time copies of the database state. They’re designed to preserve the exact data and schema at the moment they were created, providing a lighter alternative to traditional backup solutions. Unlike traditional backups which are typically full copies created using tools like `pg_dump` or physical backup utilities, snapshots are created instantly with minimal performance impact. + +Snapshots in Neon can be conveniently restored into a new branch without altering the original branch, making them ideal for testing, debugging, or rolling back to a known good state. They complement traditional backup strategies but offer an easier, more flexible recovery option when you need to act fast! + +You can create snapshots directly from the Neon console using the **Create snapshot** button. Support for configuring custom retention periods and creating snapshots on schedule is coming later this year, along with programmatic control via our API and CLI, so stay tuned for the updates. + +![Screenshot of Neon console with Create snapshot button highlighted](https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/snapshots-launch-snapshot-create-1024x640-6f0d58af.jpg) + +### Restoring a snapshot + +In this initial release, a restored snapshot creates a new branch (e.g., `main_from_snapshot_2025-04-14`). You can then add a [compute](https://neon.tech/docs/reference/glossary#compute-endpoint) to the branch and interact with it like any other branch in Neon, by either connecting with your preferred Postgres client or using Neon’s built-in [SQL editor](https://neon.tech/docs/get-started-with-neon/query-with-neon-sql-editor). Future releases will make it possible to switch all client connections to the newly restored branch, similar to how Point-In-Time Restore (PITR) works today. + +![Screenshot of Neon console with Restore button highlighted](https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/snapshots-launch-snapshot-restore-1024x640-4cd9d8bc.jpg) + +A compute can be added to a restored branch at any time, allowing you to explore the schema and data in a safe, isolated environment. This makes it easy to inspect historical states, run ad-hoc queries and validate assumptions. + +![Screenshot of Neon console displaying Autoscaling settings](https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/snapshots-launch-snapshot-compute-1024x640-c1ba947d.jpg) + +## Point-In-Time Restore (PITR) or snapshots + +Both [Point-In-Time Restore](https://neon.tech/docs/manage/backups#instant-restore) (PITR) and snapshots allow you to instantly recover or preserve the state of your database, but they serve different needs and work in different ways. + +### Point-In-Time Restore (PITR) + +PITR lets you revert an entire branch to a previous state using a timestamp or Log Sequence Number (LSN). PITR creates a new branch and immediately switches over all client connections to it while preserving the original branch under a new name (e.g., `main_old_2025-04-14`). It is especially useful for maintaining continuity in applications, as the compute is moved to the new branch, keeping the same connection string. + +### Snapshots + +Snapshots capture read-only copies of a branch’s state for backup or historical reference. They can be restored into a new branch (e.g., `main_from_snapshot_2025-04-14`), without altering the original branch. Unlike PITR, snapshots are perfect for isolating data at a specific point in time for testing or creating a rollback point, and in a future release, they’ll have the added benefit of automatic expiration for cleanup. + +### When to use PITR vs snapshots + +PITR is ideal for reverting entire branches to a precise past state within the retention period. It offers transaction-level accuracy within a configurable time window, making it well-suited for quickly recovering from issues while preventing disruption. + +Snapshots, by contrast, can be retained for longer and more cost-effectively, making them a reliable option for longer-term backups, testing, and archival purposes. Additionally, their ability to capture database states over extended retention periods provides a valuable tool for ensuring compliance with data retention and audit requirements, as well as offering a safety net before performing potentially risky operations. + +![Diagram displaying retention period for PITR vs Snapshot](https://cdn.neonapi.io/public/images/pages/blog/announcing-neon-snapshots-a-smoother-path-to-recovery/snapshots-launch-snapshot-diagram-1024x292-7c895404.jpg) + +## Start snapping today + +Snapshots are a powerful addition to your backup workflow—helping you move faster with confidence by capturing safe, reliable, restorable points in time. Whether you’re testing, experimenting, or just want peace of mind before a big change, Snapshots make it simple. They also play a key role in supporting compliance with data protection regulations, ensuring you can reliably preserve and restore data when needed. We’re excited to see how you’ll use them and would love your feedback as we continue to refine the experience.
+ +Ready to take snapshots for a spin? Join our [Early Access Program](https://console.neon.tech/app/settings/early-access) today and start exploring a smoother path to recovery. diff --git a/content/blog/posts/announcing-neons-remote-mcp-server.md b/content/blog/posts/announcing-neons-remote-mcp-server.md new file mode 100644 index 0000000000..be0333368a --- /dev/null +++ b/content/blog/posts/announcing-neons-remote-mcp-server.md @@ -0,0 +1,82 @@ +--- +title: Announcing Neon’s Remote MCP Server +description: 'Bringing MCP Servers to the cloud: Powering AI workflows with Neon' +excerpt: >- + At Neon, we like to be on top of things, especially when it comes to AI. We + first announced our MCP server back on December 3rd, 2024. That was a long + time before MCP really took off, and we’ve been iterating on our MCP server’s + capabilities ever since. You can read more about ou... +date: '2025-04-04T13:08:34' +updatedOn: '2025-05-01T00:15:55' +category: ai +categories: + - ai +authors: + - david-gomes + - shridhar-deshmukh +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-neons-remote-mcp-server/cover.jpg + alt: null +isFeatured: true +seo: + title: Announcing Neon’s Remote MCP Server - Neon + description: Bringing MCP Servers to the cloud + keywords: [] + noindex: false + ogTitle: Announcing Neon’s Remote MCP Server - Neon + ogDescription: Bringing MCP Servers to the cloud + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-neons-remote-mcp-server/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/announcing-neons-remote-mcp-server/neon-remote-mcp-server-cover-v2-5971f229.jpg) + +At Neon, we like to be on top of things, especially when it comes to AI. We first announced our MCP server [back on December 3rd, 2024](https://neon.tech/blog/let-claude-manage-your-neon-databases-our-mcp-server-is-here). That was a long time before MCP really took off, and we’ve been iterating on our MCP server’s capabilities ever since. You can read more about our MCP server [in our docs](https://neon.tech/docs/ai/neon-mcp-server). + +As such, we’ve been closely following the [Model Context Protocol specification](https://spec.modelcontextprotocol.io/specification/2025-03-26/) and waiting anxiously for the protocol to be ready for **remote hosting.** Today, we’re pleased to announce that we’ve got a hosted version of our MCP server ready for our users to use. + +A remote MCP Server greatly simplifies setting it up through any client such as Cursor or Windsurf, without having to create API keys in our service. Furthermore, as we add new features to our MCP server, our users will automatically get them without having to upgrade their local setup. + +![Workflow diagram of Neon's hosted MCP server](https://cdn.neonapi.io/public/images/pages/blog/announcing-neons-remote-mcp-server/neon-hosted-mcp-server-06ca39e9.jpg) + +## How to use Neon’s remote MCP Server + +Because the MCP specification for OAuth is still very new, we’re launching this under a preview state. It’s likely that we have to make some changes to the setup and things might break in unexpected ways during the first few weeks. Nevertheless, the following instructions should be simple to **try today**. + +1. Go to your MCP Client’s settings and register a new MCP Server +2. As an example, if you’re using Cursor, add the following to the “MCP Servers” configuration in the “Cursor Settings”: + +```bash +"Neon": { + "command": "npx", + "args": ["-y", "mcp-remote", "https://mcp.neon.tech/sse"] +} +``` + +That’s it, our hosted MCP Server is running at [https://mcp.neon.tech](https://mcp.neon.tech). + +Here’s a video of everything from start to finish using Windsurf: + + + +
If you’re looking for instructions for different clients, here they are: + +- [Claude Desktop](https://neon.tech/guides/neon-mcp-server) +- [Cursor](https://neon.tech/guides/cursor-mcp-neon) +- [Cline](https://neon.tech/guides/cline-mcp-neon) +- [Windsurf](https://neon.tech/guides/windsurf-mcp-neon) + +Note that these instructions might still refer to the self-managed version of our MCP Server, but in time they will all be updated to have both methods. + +## How is it built? + +Well, everything is open-source of course. Soon, we’ll be writing more about how we implemented this. For now, you can refer to the source code that’s [available on GitHub here](https://github.com/neondatabase/mcp-server-neon). The pull request with the bulk of the work can be [found here](https://github.com/neondatabase-labs/mcp-server-neon/pull/36). Notice that for now, we’re using the [geelen/mcp-remote](https://github.com/geelen/mcp-remote) Node package to make this all work (this will almost definitely change in the near future). + +The MCP Server communicates with clients via SSE (Server-sent events), and it can be deployed on many different cloud providers. + +## What’s next? + +Our remote MCP Server is just the beginning. As the MCP specification evolves, we’re committed to refining and expanding our offering to provide a reliable experience for developers. By bringing MCP to the cloud, we’re making AI workflows more accessible, scalable, and future-proof. We can’t wait to see what you build with it. Try it out today, and let us know your feedback on [Discord](https://discord.com/invite/92vNTzKDGp)—we’re listening. diff --git a/content/blog/posts/announcing-pg-tiktoken-a-postgres-extension-for-fast-bpe-tokenization.md b/content/blog/posts/announcing-pg-tiktoken-a-postgres-extension-for-fast-bpe-tokenization.md new file mode 100644 index 0000000000..2f2a8ee930 --- /dev/null +++ b/content/blog/posts/announcing-pg-tiktoken-a-postgres-extension-for-fast-bpe-tokenization.md @@ -0,0 +1,194 @@ +--- +title: 'Announcing pg_tiktoken: A Postgres Extension for Fast BPE Tokenization' +description: Analyze and Process Text Data in Neon with pg_tiktoken's Fast BPE Tokenization +excerpt: >- + We’re excited to announce the release of the pg_tiktoken extension on Neon. + This new Postgres extension provides fast and efficient tokenization using the + BPE (Byte Pair Encoding) algorithm. pg_tiktoken is a wrapper around OpenAI’s + tokenizer, known for its speed and performance i... +date: '2023-03-14T17:04:34' +updatedOn: '2023-06-01T11:11:48' +category: engineering +categories: + - engineering +authors: + - stas-kelvich +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-pg-tiktoken-a-postgres-extension-for-fast-bpe-tokenization/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Announcing pg_tiktoken: A Postgres Extension for Fast BPE Tokenization - + Neon + description: >- + Analyze and Process Text Data in Neon with pg_tiktoken's Fast BPE + Tokenization + keywords: [] + noindex: false + ogTitle: >- + Announcing pg_tiktoken: A Postgres Extension for Fast BPE Tokenization - + Neon + ogDescription: >- + We’re excited to announce the release of the pg_tiktoken extension on Neon. + This new Postgres extension provides fast and efficient tokenization using + the BPE (Byte Pair Encoding) algorithm. pg_tiktoken is a wrapper around + OpenAI’s tokenizer, known for its speed and performance in handling natural + language processing tasks. pg_tiktoken solves the problem of tokenizing text + data […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-pg-tiktoken-a-postgres-extension-for-fast-bpe-tokenization/social.png +--- + +We’re excited to announce the release of the `pg_tiktoken` extension on Neon. This new Postgres extension provides fast and efficient tokenization using the BPE (Byte Pair Encoding) algorithm. `pg_tiktoken` is a wrapper around [OpenAI’s tokenizer](https://github.com/openai/tiktoken), known for its speed and performance in handling natural language processing tasks. + +`pg_tiktoken` solves the problem of tokenizing text data within a Postgres database. The `tiktoken_encode` function allows you to tokenize text inputs and returns a tokenized output, making it easier to analyze and process text data for various applications. The `tiktoken_count` function enables users to return the number of tokens in a text, which is useful for checking text length limits, like those imposed by OpenAI’s language models. + +## What are text tokens + +Language models process text in chunks known as tokens. In English, a token can range from a single character to a complete word such as “a” or “apple.” In certain languages, tokens can even be shorter than one character or longer than one word. + +For instance, the sentence “Neon is Serverless Postgres” is divided into seven tokens: [“Ne”, “on”, ” is”, ” Server”, “less”, ” Post”, “gres”]. + +## Get started with `pg_tiktoken` + +With `pg_tiktoken`, you can tokenize text data within your Postgres database, making it easier to analyze and process the data for various applications. The extension supports various text inputs, including multiple languages and special characters, and is optimized for speed and efficiency. + +To start with `pg_tiktoken`, you need to install the extension: + +```sql +CREATE EXTENSION pg_tiktoken +``` + +Once the extension is installed, you can use the pg_tiktoken function within your queries to tokenize text data within your database. The function takes a text input and returns a tokenized output: + +```sql +SELECT tiktoken_encode('text-davinci-003', 'The universe is a vast and captivating mystery, waiting to be explored and understood.'); + +tiktoken_encode + +--------------- + +{464,6881,318,257,5909,290,3144,39438,10715,11,4953,284,307,18782,290,7247,13} +``` + +This will tokenize the input text using the BPE algorithm and return the tokenized output. + +You can also return the number of tokens in a text using the `tiktoken_count` function: + +```sql +SELECT tiktoken_count('text-davinci-003', 'The universe is a vast and captivating mystery, waiting to be explored and understood.'); + +tiktoken_count + +--------------- + +17 +``` + +### Supported models + +`tiktoken_count` and `tiktoken_encode` functions accept both encoding and OpenAI model names as the first argument: + +```sql +tiktoken_count(,) +``` + +Here is the list of supported models: + +| **Encoding** **name** | **OpenAI** **models** | +| ----------------------- | ------------------------------------------------------------------------- | +| `cl100k_base` | ChatGPT models, `text-embedding-ada-002` | +| `p50k_base` | Code models, `text-davinci-002`, `text-davinci-003` | +| `p50k_edit` | Use for edit models like `text-davinci-edit-001`, `code-davinci-edit-001` | +| `r50k_base` (or `gpt2`) | GPT-3 models like davinci | + +## Use `pg_tiktoken` with the ChatGPT model + +You can persist the message history in Postgres and query the database to get messages that fit within OpenAI’s model limits. + +Here is an example of the “message” table: + +```sql +CREATE TABLE message ( + role VARCHAR(50) NOT NULL, -- equals to 'system', 'user' or 'assistant' + content TEXT NOT NULL, + created TIMESTAMP NOT NULL DEFAULT NOW(), + n_tokens INTEGER -- number of content tokens +); +``` + +The `gpt-3.5-turbo` model requires the following parameters: + +```json +{ + "model": "gpt-3.5-turbo", + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Who won the world series in 2020?"}, + {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."} + ] +} +``` + +The “messages” parameter consists of an array of message objects. Each object within the array should contain two key pieces of information: the role of the message sender (either “system,” “user,” or “assistant”) and the actual message content. Conversations can be brief, consisting of just one message, or they may continue for multiple pages as long as the sum of the message tokens is within the 4096 limit. + +You can use the following query to insert role, content, and the number of tokens into the database: + +```sql +INSERT INTO message (role, content, n_tokens) +VALUES ('user', 'Hello, how are you?', tiktoken_count('Hello, how are you?')); +``` + +## Manage your text tokens + +When you have a conversation with more tokens than a model can handle (like more than 4096 tokens for `gpt-3.5-turbo`), you’ll need to shorten your text to fit the limit. But be careful. If you remove a message from the conversation, the model will not know about it anymore. + +Also, you might receive incomplete replies if your conversation is too long. For instance, if your `gpt-3.5-turbo` conversation is 4090 tokens long, you’ll only get a reply of 6 tokens. + +The query below allows you to get messages up to your desired token limits: + +```sql +WITH cte AS ( + SELECT role, content, created, n_tokens, + SUM(tokens) OVER (ORDER BY created DESC) AS cumulative_sum + FROM message +) + +SELECT role, content, created, n_tokens, cumulative_sum +FROM cte +WHERE cumulative_sum <= ; +``` + +`MAX_HISTORY_TOKENS` represents the conversation history you want to keep for the chat completion and follow the following formula: + +MAX_HISTORY_TOKENS = MODEL_MAX_TOKENS – NUM_SYSTEM_TOKENS – NUM_COMPLETION_TOKENS + +Let’s use the example seen above and assume that the desired completion length is 100 tokens (`NUM_COMPLETION_TOKENS=90`). + +MAX_HISTORY_TOKENS = 4096 – 6 – 90 = 4000 + +```json +{ + "model": "gpt-3.5-turbo", // MODEL_MAX_TOKENS = 4096 + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, // NUM_SYSTEM_TOKENS = 6 + {"role": "user", "content": "Who won the world series in 2020?"}, + {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."}, + {"role": ...} + . + . + . + {"role": "user", "content": "Great! Have a great day."} // MAX_HISTORY_TOKENS = 4000 + ] +} +``` + +## Conclusion + +In summary, the `pg_tiktoken` extension provides fast and efficient tokenization using the BPE algorithm within Postgres, making it easier to analyze and process text data for various applications. We explored how to use the `tiktoken_count` function to query the database and retrieve messages that fit within OpenAI’s model limits and how to avoid failing API calls. + +What is your use case for the `pg_tiktoken` extension and AI? We would love to hear your feedback and learn about features you would like to see added to the `pg_tiktoken` in future updates. + +We look forward to seeing the natural language processing applications you will build with `pg_tiktoken`! diff --git a/content/blog/posts/announcing-point-in-time-restore.md b/content/blog/posts/announcing-point-in-time-restore.md new file mode 100644 index 0000000000..919fd5c310 --- /dev/null +++ b/content/blog/posts/announcing-point-in-time-restore.md @@ -0,0 +1,112 @@ +--- +title: Announcing Point-in-Time Restore +description: >- + Restore your database to a previous point in time using Neon's Restore with + Time Travel Assist +excerpt: >- + Neon’s unique architecture separates storage and compute for Postgres. This + enables us to provide features such as serverless and autoscaling for your + Postgres instances. Another benefit of Neon’s architecture is that it enables + us to retain the history of changes, including data... +date: '2024-02-20T20:19:47' +updatedOn: '2024-03-27T11:53:05' +category: community +categories: + - community +authors: + - evan-shortiss +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-point-in-time-restore/cover.jpg + alt: null +isFeatured: false +seo: + title: Announcing Point-in-Time Restore - Neon + description: >- + Restore your database to a previous point in time using Neon's Restore with + Time Travel Assist + keywords: [] + noindex: false + ogTitle: Announcing Point-in-Time Restore - Neon + ogDescription: >- + Restore your database to a previous point in time using Neon's Restore with + Time Travel Assist + image: >- + https://cdn.neonapi.io/public/images/pages/blog/announcing-point-in-time-restore/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/announcing-point-in-time-restore/neon-backup-restore-1-1024x576-f839da8d.jpg) + +Neon’s [unique architecture](https://neon.tech/blog/architecture-decisions-in-neon) separates storage and compute for Postgres. This enables us to provide features such as serverless and autoscaling for your Postgres instances. Another benefit of Neon’s architecture is that it enables us to retain the history of changes, including data [Data Definition (DDL)](https://www.postgresql.org/docs/current/ddl.html) changes, for all database branches. This enables Neon to provide instantaneous [point-in-time restore (PITR)](https://en.wikipedia.org/wiki/Point-in-time_recovery) operations. + +Point-In-Time Restore allows you to restore your database to its previous state if an operation you perform has unintended consequences. This feature gives you peace of mind and confidence in shipping your product with minimal risk of unintended consequences. + +We’re excited to announce that our powerful Branch Restore feature is now available in the [Neon Console](https://console.neon.tech/). Branch Restore enables you to [restore a branch from history](https://neon.tech/docs/guides/branch-restore#how-to-use-branch-restore) with [time-travel assistance](https://neon.tech/docs/guides/branch-restore#time-travel-assist).  This streamlines the previously [manual process](https://neon.tech/blog/point-in-time-recovery#restore-the-branch-to-a-previous-state-while-keeping-the-same-compute-endpoint) of performing a PITR operation. + +## Restore a Branch from History + +Neon projects support a configurable [retention value](https://neon.tech/docs/introduction/point-in-time-restore#history-retention) that allows up to 30 days of history to be retained. Increasing the retention value will increase your project’s storage usage but provide greater flexibility when performing point-in-time restore operations. + +The new Branch Restore functionality can restore your database branch to a specific [log sequence number (LSN)](https://www.postgresql.org/docs/current/datatype-pg-lsn.html) from the [write-ahead log (WAL)](https://www.postgresql.org/docs/current/wal-internals.html) within your project’s configured history retention window. If you don’t have a specific LSN, you can select a timestamp instead, and Neon will determine the LSN associated with your chosen timestamp behind the scenes.   + +To test the Restore feature, open up the [Neon Console](https://console.neon.tech/) and create a new project. Select the **SQL Editor** and run the following queries to create a table named `playing_with_neon` and seed it with 20 rows of data: + +```sql +CREATE TABLE playing_with_neon(id SERIAL PRIMARY KEY, name TEXT NOT NULL, value REAL); +INSERT INTO playing_with_neon(name, value) SELECT LEFT(md5(i::TEXT), 10), random() FROM generate_series(1, 20) s(i); +``` + +Take note of the time. Wait a minute or two, then insert 10 more rows of data into the database: + +```sql +INSERT INTO playing_with_neon(name, value) SELECT LEFT(md5(i::TEXT), 10), random() FROM generate_series(1, 10) s(i); +``` + +Confirm that the table contains 30 rows using the `SELECT count(*) from playing_with_neon;` query. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/announcing-point-in-time-restore/screenshot-2024-02-20-at-120645-1024x514-08cf0a4a.png) + +Navigate from the **SQL Editor** to the **Restore** screen in the Neon Console and: + +- Confirm that the **Branch to restore** is set to **main** (or the branch you’re using to test).  +- Set the **Timestamp** value to the time you took note of earlier. +- Click the **Next** button. +- Confirm you want to proceed by clicking **Restore** in the popup that appears. + +In the screenshot above, you can see that the second INSERT operation occurred at 12:06 PM. Setting the **Timestamp** field to 12:05 PM is sufficient to restore the database branch to the point before the second INSERT operation. You can specify second and millisecond accuracy if necessary. Before clicking **Next**, you can also use the **Time travel assist** feature to verify the data at your chosen timestamp. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/announcing-point-in-time-restore/screenshot-2024-02-20-at-120936-1024x516-44d99d29.png) + +The restore operation typically takes just a couple of seconds to complete. During this period, applications connected to the compute endpoint experience a brief loss of connectivity to the database. However, they can reconnect once the compute has restarted, similar to a cold start. + +## Time Travel Assist + +Validating you have chosen the correct LSN or timestamp is critical to confidently using the Restore functionality. You can use Time Travel Assist to issue a query against your database as it was at the timestamp or LSN and branch you selected in the Neon console’s Restore section before performing the restore operation. + +
Let’s try it out using the same project and `playing_with_neon table` as before. Select the **SQL Editor** and run the following query against the `main` branch: + +```sql +DELETE from playing_with_neon; +``` + +Uh oh! The DELETE query without a WHERE clause deleted everything from the database. In a real-world scenario, you could have a situation like this occur when you push new code to an application that contains a bug. Your application logs might not reveal precisely when the delete operation was performed. In this case, you can use Time Travel Assistant to narrow down when the data loss occurred. + +Select the **Restore** section in the Neon console, select the main branch from the dropdown, and enter the current time in the timestamp field. Enter  `SELECT count(*) from playing_with_neon;` click the **Query at timestamp** button in the query editor. Zero rows should be returned since you’re effectively querying the current version of the `main` branch that has suffered data loss. + +Set the timestamp further in the past until the query returns the expected non-zero number of rows. At this point you’ve found the timestamp required to perform the Restore operation, so go ahead and click **Next** then proceed with the restoration. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/announcing-point-in-time-restore/screenshot-2024-02-20-at-120839-1024x517-25cc5084.png) + +Return to the **SQL Editor** and issue a `SELECT * from playing_with_neon;` query against the main branch. A non-zero number of rows will be returned. This is because you successfully restored the branch to a point before the erroneous `DELETE` query was performed. + +## Neon’s Point-in-Time Restore vs. Roll-your-own Restore + +Postgres is a mature enterprise-ready database and includes tools such as [pg_dump](https://www.postgresql.org/docs/current/app-pgdump.html), [pg_dumpall](https://www.postgresql.org/docs/current/app-pg-dumpall.html), [pg_restore](https://www.postgresql.org/docs/current/app-pgrestore.html), [pg_basebackup](https://www.postgresql.org/docs/current/app-pgbasebackup.html), and WAL archival capabilities out-of-the-box. Outside of the core tooling, projects such as pgBackRest, WAL-G, and Barman exist to provide complete backup and restore solutions for Postgres that can create redundant backups by sending archives to object storage and simplifying PITR. If you’re running Postgres in a virtual or cloud environment, creating a snapshot of the mounted disk volume where your data is stored is possible. + +Of the core tools, pg_dump is the most straightforward to use. It creates a logical backup that can be used to restore your schema(s) and data to your existing Postgres version or migrate to a newer version. The downside of pg_dump is that it doesn’t support incremental or point-in-time restore since it creates a full copy of your database’s data and schema exactly as they were at the time pg_dump was run. If someone accidentally runs an erroneous DELETE you could restore data using a recent pg_dump, but you’d have lost any data created between when the pg_dump was performed and when the delete was performed. + +A backup and restore strategy that only utilizes pg_dump will result in a Recovery Point Objective (RPO) that could lead to unacceptable levels of data loss. Organizations running production workloads on Postgres must minimize their RPO and Recovery Time Objective (RTO) by defining a strategy that uses pg_basebackup combined with WAL archival or tools such as Barman and pgBackRest. + +## Conclusion + +Neon’s PITR restore capability offers up to 30 days of retention with LSN-level granularity and takes just seconds to perform. This means your RPO and RTO can be minimized without implementing and maintaining complex backup infrastructure. Depending on your organization’s data retention strategy, you may still want to create occasional [backups of your Neon data using pg_dump](https://neon.tech/docs/import/import-from-postgres#export-data-with-pgdump). If you’re looking for a Postgres database, [sign up and try Neon](https://neon.tech/blog/python-django-and-neons-serverless-postgres#:~:text=sign%20up%20and%20try%20Neon) for free. Join us in our [Discord server](https://neon.tech/discord) to share your experiences, suggestions, and challenges. diff --git a/content/blog/posts/anything-the-new-ai-agent-for-building-mobile-and-web-apps.md b/content/blog/posts/anything-the-new-ai-agent-for-building-mobile-and-web-apps.md new file mode 100644 index 0000000000..7e26bfd22a --- /dev/null +++ b/content/blog/posts/anything-the-new-ai-agent-for-building-mobile-and-web-apps.md @@ -0,0 +1,111 @@ +--- +title: 'Anything: The New AI Agent for Building Mobile and Web Apps' +description: >- + Powered by Neon, Anything turns prompts into full-stack apps with backend, + auth, and database built in +excerpt: >- + The former Create.xyz has now relaunched as Anything, a new platform agent + that builds and ships full-stack applications from a single prompt with + support for mobile and web, production-ready designs, and built-in + infrastructure, with Neon powering the backend with serverless Pos... +date: '2025-08-07T17:49:19' +updatedOn: '2025-09-10T01:00:21' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/anything-the-new-ai-agent-for-building-mobile-and-web-apps/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Anything: The New AI Agent for Building Mobile and Web Apps - Neon' + description: >- + Powered by Neon, Anything turns prompts into full-stack apps with backend, + auth, and database built in. + keywords: [] + noindex: false + ogTitle: 'Anything: The New AI Agent for Building Mobile and Web Apps - Neon' + ogDescription: >- + Powered by Neon, Anything turns prompts into full-stack apps with backend, + auth, and database built in. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/anything-the-new-ai-agent-for-building-mobile-and-web-apps/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/anything-the-new-ai-agent-for-building-mobile-and-web-apps/neon-anything-1024x576-90c4dd21.jpg) + +[The former Create.xyz](https://neon.com/blog/from-idea-to-full-stack-app-in-one-conversation-with-create) has now relaunched as [Anything](https://createanything.com), a new platform agent that builds and ships full-stack applications from a single prompt with support for mobile and web, production-ready designs, and built-in infrastructure, with Neon powering the backend with serverless Postgres, built-in auth, and seamless developer workflows. + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + + +Anything is live now and available at [createanything.com](https://createanything.com). Let’s take a look at what’s new. + + + +## Build Anything: What You Get From A Prompt + +The team behind Anything rebuilt the agent experience from the ground up with a more powerful agent, native mobile support, and an improved developer experience. + +### Mobile + web app support + +![Image](https://cdn.neonapi.io/public/images/pages/blog/anything-the-new-ai-agent-for-building-mobile-and-web-apps/ad4nxfed-cxxpxjtyvjs0cwp8bognhjoxzmknlqwi59vin-mvez5yvig1gpklsnq4hdzu61v58udwfig0cpajryp8q5wr8yqmiqglzg22evrdtfyzxtvnytesiva3vb7dpk9e6euzw-ecc3c337.png) + +Anything now creates production-ready apps for iOS, Android, and the web from a single prompt. You can: + +- Preview mobile apps on your phone in minutes +- Submit directly to the App Store with one click +- Share a unified backend across mobile and web + +### All-in-one infrastructure (no setup needed) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/anything-the-new-ai-agent-for-building-mobile-and-web-apps/ad4nxeqi49ve1enmvlequ7kjjv8bmqt331cpjkjq2sq5rwcmcnnuwcnor2h92g3bnaentxuclpadayepw6wwg748nbkmmbvcftheb-u5jakxynpft31ls7ad3gby-v1id24o6fjeghg-3dd9b210.png) + +Anything builds everything your app requires to go live. Everything is wired together automatically, so you can focus on building vs stitching services together. + +- Backend (auto-generated from your prompt) +- Postgres +- Authentication +- Payments +- File storage +- Logs +- Built-in AI models +- 100+ integrations + +### Smarter agent, faster feedback + + + +The new version of the agent It’s designed to help you ship faster, without getting stuck debugging every step. It comes with: + +- Improved reasoning across complex prompts +- In-stream error checking and automatic recovery +- Fast updates and live previews + +### Beautiful, custom designs + +![Image](https://cdn.neonapi.io/public/images/pages/blog/anything-the-new-ai-agent-for-building-mobile-and-web-apps/ad4nxfrj1zvk2qi7lbeom6lpfkvgcwqbsups5gfsevmjyus8qsuixp3hte2dawfyq5-uhaqxigb3w1c6sifzo0k2gp71n4p4dsxcd19cntoa820nvcojwdjmkprgehopsvjm9liftq-6e2fb873.png) + +The team at Anything has put extra effort to build an agent that creates designs that feel human-made. It’s trained on thousands of real-world examples and tailors UIs to your prompt yo output clean, elegant layouts. + +## Codegen Platforms Are Building on Neon. Find Out Why. + +Every app built with Anything runs on [Neon](https://neon.com/). It provides the Postgres backend, authentication, and storage that Anything apps rely on – all with a serverless, scale-to-zero architecture that’s perfect for agent-driven development. + +
+

“Neon’s speed of provisioning and serverless scale-to-zero is critical for us. We can serve users iterating on quick ideas efficiently while also supporting them as they scale, without making them think about database setup.” (Dhruv Amin, Co-founder at Anything)

+
+ +Codegen platforms from Anything to Replit are choosing Neon to power their backends. If you’re building agents, Neon gives you the Postgres infrastructure to move fast, scale automatically, and simplify your stack. [Keep reading about it](https://neon.com/use-cases/ai-agents) and see why it’s the default database for codegen. + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + diff --git a/content/blog/posts/api-cf-drizzle-neon.md b/content/blog/posts/api-cf-drizzle-neon.md new file mode 100644 index 0000000000..21d5de1f1c --- /dev/null +++ b/content/blog/posts/api-cf-drizzle-neon.md @@ -0,0 +1,398 @@ +--- +title: 'Build a serverless API using Cloudflare Workers, Drizzle ORM, and Neon' +description: >- + Learn how you can use Cloudflare Workers, Drizzle ORM, and Neon to build a + serverless API +excerpt: >- + In this guide, you will learn how to build a serverless API using Cloudflare + Workers, Hono, Drizzle ORM, and Neon. What are Cloudflare Workers? Cloudflare + Workers enable you to build and deploy serverless code instantly across the + globe without worrying about managing and scaling... +date: '2023-05-30T00:08:49' +updatedOn: '2024-01-12T15:47:20' +category: community +categories: + - community +authors: + - mahmoud-abdelwahab +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Build a serverless API using Cloudflare Workers, Drizzle ORM, and Neon - + Neon + description: >- + In this guide, you will learn how to build and deploy a serverless API using + Cloudflare Workers, Drizzle ORM, and Neon. + keywords: [] + noindex: false + ogTitle: >- + Build a serverless API using Cloudflare Workers, Drizzle ORM, and Neon - + Neon + ogDescription: >- + In this guide, you will learn how to build and deploy a serverless API using + Cloudflare Workers, Drizzle ORM, and Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-5-1024x576-b2e79a47.png) + +In this guide, you will learn how to build a serverless API using [Cloudflare Workers](https://workers.cloudflare.com/), [Hono](https://hono.dev/), [Drizzle ORM](https://orm.drizzle.team/), and Neon. + +## What are Cloudflare Workers? + +Cloudflare Workers enable you to build and deploy serverless code instantly across the globe without worrying about managing and scaling infrastructure. + +## What is Neon? + +Neon is fully managed serverless Postgres. This means you do not have to pick a size for your database upfront, and it can automatically scale up based on your workload and down to zero when not in use. + +
+

Note: Neon’s architecture separates storage and compute. This makes a Neon Postgres instance stateless, which makes it possible to automatically scale compute resources up or down based on demand. To learn more, check out Neon’s architecture.

+
+ +## What is Drizzle ORM? + +[Drizzle ORM](https://orm.drizzle.team/docs/quick-start) is a lightweight TypeScript ORM. It is compatible with Cloudflare Workers and comes with [drizzle-kit](https://orm.drizzle.team/kit-docs/overview), a CLI companion for generating SQL migrations automatically. + +## Prerequisites + +This is a beginner-friendly guide. However, it assumes basic knowledge of JavaScript or TypeScript (preferred). You also need to have Node.js installed on your machine. + +To successfully complete this guide, you will need: + +- [A Cloudflare account](https://dash.cloudflare.com/sign-up) +- [A Neon account](https://console.neon.tech/) + +You can find the [final code on GitHub](https://github.com/neondatabase/cloudflare-drizzle-neon). + +## Set up the project using create-cloudflare-cli + +To get started, run the following command in the directory of your choice: + +```bash +npm create cloudflare +``` + +This command runs [create-cloudflare-cli](https://developers.cloudflare.com/pages/get-started/c3/) (also known as C3), which is a command-line tool designed to help you set up and deploy Workers. You will be prompted to install the create-cloudflare package, and you will be presented with a setup wizard. + +First, give your project a name (or leave it blank so it gets automatically generated). Next, select the following options: + +- What type of application do you want to create? `"Hello World" worker` +- Do you want to use TypeScript? `Yes` +- Do you want to deploy your application? `No` + +### A look at the project’s folder structure + +After the project’s dependencies are installed, open the project in your text editor of choice. The two main files are: + +- `src/index.ts`: this file contains a basic worker. +- `wrangler.toml`: a configuration file to customize the development and publishing setup for a Worker. + +When you open the `src/index.ts` file, you will see the following code: + +```typescript +/** + * Welcome to Cloudflare Workers! This is your first worker. +* + * - Run `npm run dev` in your terminal to start a development server + * - Open a browser tab at https://localhost:8787/ to see your worker in action + * - Run `npm run deploy` to publish your worker +* + * Learn more at https://developers.cloudflare.com/workers/ + */ + +export interface Env { + // Example binding to KV. Learn more at https://developers.cloudflare.com/workers/runtime-apis/kv/ + // MY_KV_NAMESPACE: KVNamespace; + // + // Example binding to Durable Object. Learn more at https://developers.cloudflare.com/workers/runtime-apis/durable-objects/ + // MY_DURABLE_OBJECT: DurableObjectNamespace; + // + // Example binding to R2. Learn more at https://developers.cloudflare.com/workers/runtime-apis/r2/ + // MY_BUCKET: R2Bucket; + // + // Example binding to a Service. Learn more at https://developers.cloudflare.com/workers/runtime-apis/service-bindings/ + // MY_SERVICE: Fetcher; + // + // Example binding to a Queue. Learn more at https://developers.cloudflare.com/queues/javascript-apis/ + // MY_QUEUE: Queue; +} + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { + return new Response('Hello World!'); + }, +}; +``` + +The worker performs a default export of an async `fetch` function. The following three parameters are always passed into it: + +- `request`: represents an HTTP request and is part of the Fetch API. +- `env`: represents the bindings (in the Workers platform, environment variables, secrets, and KV namespaces are known as bindings) assigned to the Worker. +- `ctx`: represents the context your function runs in. + +Whenever this worker is called, it returns a `Response` object with the string ’Hello World!’. + +To enable auto-completion for your project’s environment variables, they are added to the `Env` interface. + +To ensure everything is set up correctly, you can run npm run start. This command starts a local development server using [wrangler](https://developers.cloudflare.com/workers/wrangler/), a command-line tool for building with Cloudflare developer products. If you open your browser and navigate to https://localhost:8787, you will see ’Hello World!’. + +While you can handle different HTTP methods by checking the request’s method, using a framework will make it easier to build the API. To do that, we will use [Hono](https://hono.dev/), a web framework that is compatible with Cloudflare Workers. + +## Set up Hono.js + +To get started, run the following command in your project to add Hono as a dependency: + +```bash +npm install hono +``` + +Next, go to your `src/index.ts` file and replace the existing code with the code provided below: + +```typescript +// src/index.ts +import { Hono } from 'hono'; + +export type Env = { + DATABASE_URL: string; +}; + +const app = new Hono<{ Bindings: Env }>(); + +app.get('/', (c) => { + return c.json({ + message: 'Hello World!', + }); +}); + +export default app; +``` + +You are first importing `Hono` and creating a new instance. You then pass the `Env` type as a generic to your Hono instance. This makes it possible to enable autocompletion for environment variables. You then create an API endpoint located at `/` and return a JSON object with the message ”Hello World!”. Finally, you do a default export for the app variable. + +## Create a Neon project + +Go ahead and [create an account](https://console.neon.tech) if you do not have one already. Next, create a new project. Choose `15` as the Postgres version, pick the region closest to where you want to deploy your app and pick a size for your compute endpoint (you can change this later). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-1024x576-3d28c3f2.png) + +After you create the project, you get a connection string that you can use to connect to your database. In the root of your project, create a .`dev.vars` file and add the connection string as an environment variable. It should be formatted like a `dotenv` file, such as `KEY=VALUE`. + +```bash +# .dev.vars + +DATABASE_URL="postgres://daniel:@ep-mute-rain-952417.us-east-2.aws.neon.tech:5432/neondb" +``` + +## Add Drizzle ORM to your project + +To add Drizzle to your project, run the following commands + +```bash +npm i drizzle-orm @neondatabase/serverless +npm i -D drizzle-kit postgres dotenv tsx +``` + +The first command installs `drizzle-orm` along with `@neondatabase/serverless`. This enables you to connect to Neon from serverless environments. + +You are then installing `drizzle-kit` for generating migrations, [`postgres.js`](https://www.npmjs.com/package/postgres) to establish a connection when running migrations, [`dotenv`](https://www.npmjs.com/package/dotenv) for loading environment variables, and [`tsx`](https://www.npmjs.com/package/tsx) for executing TypeScript files. + +### Define the schema using TypeScript + +In your `src` directory, create a new `db/schema.ts` file. This file will contain the database schema definition in TypeScript. Add the following code to the file you just created: + +```typescript +// db/schema.ts +import { pgTable, serial, text, doublePrecision } from 'drizzle-orm/pg-core'; + +export const products = pgTable('products', { + id: serial('id').primaryKey(), + name: text('name'), + description: text('description'), + price: doublePrecision('price'), +}); +``` + +You are defining a table called products, which has four columns: + +- `id`: this is the table’s primary key, and it is of type `serial`, which is an auto-incrementing 4-bytes integer. +- `name`: which has a variable-length(unlimited) character string. +- `description`: which has a variable-length(unlimited) character string. +- `price`: which is a double-precision floating-point number + +### Generate database migrations + +In the project’s root directory, create a `drizzle.config.ts` file and add the following code to it: + +```javascript +//drizzle.config.ts +import type { Config } from 'drizzle-kit'; + +export default { + schema: './src/db/schema.ts', + out: './drizzle', +} satisfies Config; +``` + +In this config file, you specify the location of your schema as well as the output directory, which will contain the generated migrations. In our case, the output directory is called `drizzle` and will be located in the project’s root directory. + +The next step is to generate the database migrations. To do that, modify your `package.json` file and add a new ”`db:generate`” command in the scripts object: + +```javascript +// package.json +... +"scripts": { + ... + "db:generate": "drizzle-kit generate:pg" +}, +... +``` + +If you run the command `npm run db:generate`, you will see a newly generated SQL migration file in the `/drizzle` directory. The final step is to apply the migration to the database. + +### Apply migrations to the database + +In your project’s root directory, create a `migrate.ts` file and add the following code to it: + +```typescript +// migrate.ts +import { config } from 'dotenv'; +import { migrate } from 'drizzle-orm/postgres-js/migrator'; +import postgres from 'postgres'; +import { drizzle } from 'drizzle-orm/postgres-js'; + +config({ path: '.dev.vars' }); + +const databaseUrl = drizzle(postgres(`${process.env.DATABASE_URL}`, +{ ssl: 'require', max: 1 })); + +const main = async () => { + try { + await migrate(databaseUrl, { migrationsFolder: 'drizzle' }); + console.log('Migration complete'); + } catch (error) { + console.log(error); + } + process.exit(0); +}; +main(); +``` + +This TypeScript file will be responsible for running migrations. First, you are importing the `DATABASE_URL` environment variable from the `.dev.vars` file. You are setting the max number of connections to 1 to [ensure that queries are being executed in order and over the same connection](https://github.com/porsager/postgres#unsafe_transaction). + +You then have a `main()` function that will call a `migrate()` function that is imported from the `drizzle-orm` package. This function takes a database connection string and an object where you specify the location of the migrations folder. + +Finally, to be able to execute this `migrate.ts` file, modify your `package.json` file and add a new ”db:migrate” script in the scripts object: + +```javascript +// package.json +... +"scripts": { + ... + "db:migrate": "tsx migrate.ts", +}, +... +``` + +This command runs the `migrate.ts` file using the `tsx` package you installed previously. You can test it by running the following command, which applies the migration to your database: + +```bash +npm run db:migrate +``` + +You can check that the tables have been created successfully by going to the “Tables” page in the Neon console. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-2-1024x576-84300f72.png) + +### Add data using Neon’s SQL editor + +Right now, the `products` table you created is empty. In the Neon console, go to the SQL editor and run the following SQL query to add data to the `products` table. + +```sql +INSERT INTO products (name, price, description) VALUES + ('Product A', 10.99, 'This is the description for Product A.'), + ('Product B', 5.99, 'This is the description for Product B.'), + ('Product C', 15.99, 'This is the description for Product C.'), + ('Product D', 8.99, 'This is the description for Product D.'), + ('Product E', 20.99, 'This is the description for Product E.'); +``` + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-1-1024x576-712894ea.png) + +The next step is to connect to the database from the worker. + +### Connect to Neon from the worker + +Navigate to your `src/index.ts` and add the following code: + +```typescript +import { drizzle } from 'drizzle-orm/neon-http'; +import { neon } from '@neondatabase/serverless'; +import { products } from './db/schema'; +import { Hono } from 'hono'; + +export type Env = { + DATABASE_URL: string; +}; + +const app = new Hono<{ Bindings: Env }>(); + +app.get('/', async (c) => { + try { + const sql = neon(c.env.DATABASE_URL); + + const db = drizzle(sql); + + const result = await db.select().from(products); + + return c.json({ + result, + }); + } catch (error) { + console.log(error); + return c.json( + { + error, + }, + 400 + ); + } +}); + +export default app; +``` + +You are creating a new `Pool()` instance, passing in the database connection string, and passing the instance to the `drizzle()` function to enable sending queries. + +Now, if you start your development server and go to `https://localhost:8787`, you will be able to see data being returned as JSON. + +## Deploy the worker using wrangler + +To deploy your app, you must first log into your Cloudflare account. To do that, run `npx` `wrangler login`. You will be redirected to Cloudflare, where you can connect the CLI to your account. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-29-1024x573-bd4d1e5e.png) + +Once logged in, you can run `npx wrangler deploy`, which deploys your worker. If you try to visit the deployed version, you will run into an error because you have not included the `DATABASE_URL` environment variable. To do that, you will leverage the Neon integration on Cloudflare. + +## Use the Neon integration on Cloudflare to simplify credential management + +Log into the Cloudflare dashboard, select “Workers & Pages” from the sidebar, and then “Overview”. + +Next, choose the Worker you deployed, go to the “Settings” tab, choose “Integrations”, and select “Neon”. After accepting the terms, you will be redirected to an OAuth consent screen where you can authorize Cloudflare. Finally, select your project, branch, database, and role to finish setting up the integration. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-4-1024x576-ec83e03e.png) + +Adding the integration automatically redeploys your worker. So now, when you visit the deployed worker, you will be able to see data returned from the database as JSON. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/api-cf-drizzle-neon/image-3-1024x576-654524be.png) + +## Conclusion + +In this guide, you learned about Cloudflare workers, Hono, Drizzle ORM, Neon, and how you can use them together to create a serverless API. + +If you have any questions or run into issues, please reach out to us in the [Neon Discord community](https://neon.tech/discord). diff --git a/content/blog/posts/api-track-schema-changes.md b/content/blog/posts/api-track-schema-changes.md new file mode 100644 index 0000000000..f685187f43 --- /dev/null +++ b/content/blog/posts/api-track-schema-changes.md @@ -0,0 +1,116 @@ +--- +title: An API to Track Database Schema Changes +description: >- + Use the compare_schema endpoint to integrate schema checks into agentic + systems and deployment pipelines +excerpt: >- + We keep expanding our Schema Diff feature, this time adding an API endpoint: + compare_schema. You can use it in all your Neon projects, including the Free + Plan. Schema Diff allows you to easily compare schemas between Neon databases. + It was first made available via the Neon Consol... +date: '2025-01-09T17:08:23' +updatedOn: '2025-01-21T15:37:49' +category: postgres +categories: + - postgres + - workflows +authors: + - luis-tavares +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/api-track-schema-changes/cover.jpg + alt: null +isFeatured: true +seo: + title: An API to Track Database Schema Changes - Neon + description: >- + The compare_schema API tracks Neon database schemas programmatically, + supporting migrations in CI/CD pipelines and agentic systems. + keywords: [] + noindex: false + ogTitle: An API to Track Database Schema Changes - Neon + ogDescription: >- + The compare_schema API tracks Neon database schemas programmatically, + supporting migrations in CI/CD pipelines and agentic systems. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/api-track-schema-changes/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/api-track-schema-changes/neon-api-1-1024x576-78a92fa9.jpg) + +**We keep expanding our Schema Diff feature, this time adding an** [API endpoint](https://neon.tech/docs/guides/schema-diff#using-the-neon-api)**: `compare_schema`. You can use it in all your Neon projects, including the** [Free Plan](https://console.neon.tech/signup). + +[Schema Diff](https://neon.tech/docs/guides/schema-diff) allows you to easily compare schemas between Neon databases. It was first made available via the Neon Console and CLI, and we recently expanded it by launching the [Schema Diff GitHub Action](https://neon.tech/blog/track-schema-changes-automatically-in-your-pull-requests). This action integrates schema diffs directly into pull requests; whenever a pull request is created or updated, the GitHub Action runs a schema comparison, posting a comment in the PR summarizing the differences. + +After shipping the GitHub Action, we wanted to extend these automation capabilities to other CI/CD pipelines via an API. Also, we saw another use case demanding an API for checking schema diffs: **Agentic systems**. + +## Managing database migrations in agentic systems + +[Agentic systems](https://www.anthropic.com/research/building-effective-agents) are autonomous AI entities powered by LLMs that can dynamically determine their own actions to achieve complex objectives, for example deploying software infrastructure to build end-to-end apps. Unlike predefined AI workflows (which follow static sequences), agentic systems can dynamically adapt to the task at hand. + +If we want these systems to deploy functional applications, persistent data is a requirement. Agentic systems need databases to enable continuity and memory for their apps: for example, [Replit Agent](https://docs.replit.com/replitai/agent) knows how to [deploy and manage Postgres databases](https://neon.tech/blog/how-to-add-a-postgres-database-to-your-replit-agent-project) as required by each project. + +This means agentic systems also have to handle schema evolution. When the structure of stored data needs to change (e.g. due to new features), the agent must assess the current Postgres schema, determine necessary updates, and apply them. + +The `compare_schema` API facilitates this workflow by enabling programmatic schema comparisons. For example, Replit Agent can use the API to: + +- Dynamically assess schema differences when implementing new features +- Automate the generation and application of migration scripts during deployment +- Validate that database updates align with the intended schema + +## Automating workflows: from agents to developers + +What’s useful for agents is useful for developers. Agentic systems benefit from tools that enable autonomy, same for developers looking to streamline their pipelines. + +Database migrations are one of the trickiest things to automate. They require comparing the current database schema with a target schema, identifying discrepancies, writing migration scripts to align the two, and testing the migrations to ensure they work. The `compare_schema` API facilitates the automation of these steps, especially when combined with [Neon branches](https://neon.tech/docs/introduction/branching). Teams can set up a system in which: + +1. Schema changes are applied and tested in a Neon branch. Branches include a copy of production data but don’t interfere with the production database (they have their own compute) +2. The API is used to track schema discrepancies +3. Once everything is tested, a script executes the migration in production + +This method reduces the risk of errors (since new schemas are being tested in real data) while supporting a smoother deployment process. + +## Taking a look at the API + +The [API](https://neon.tech/docs/guides/schema-diff#using-the-neon-api) accepts two Neon branch IDs and returns a schema diff, highlighting additions, modifications, and deletions in database objects. It supports these parameters: + +- **`project_id`**: the ID of your Neon project +- **`branch_id`**: the ID of the target branch to compare—the branch with the modified schema +- **`db_name`**: the name of the database +- **`base_branch_id`** (optional): the ID of the base branch for comparison, if empty, infers, by default, the parent branch of branch_id +- **`lsn`** (optional): the [LSN](https://neon.tech/blog/get-page-at-lsn) on the target branch +- **`timestamp`** (optional): the point in time on the target branch +- **`base_lsn`** (optional): the LSN for the base branch +- **`base_timestamp`** (optional): the point in time on the base branch + +The point-in-time parameters (LSN and timestamp) are mutually exclusive. You cannot define them simultaneously, as they could represent two different times in the history of the branch. + +This example cURL command compares the schema of a target branch (`br-rough-boat-a54bs9yb`) to a base branch (`br-royal-star-a54kykl2`): + +```bash +curl --request GET \ + --url 'https://console.neon.tech/api/v2/projects/wispy-butterfly-25042691/branches/br-rough-boat-a54bs9yb/compare_schema?base_branch_id=br-royal-star-a54kykl2&db_name=neondb' \ + --header 'accept: application/json' \ + --header 'authorization: Bearer $NEON_API_KEY' | jq -r '.diff' +``` + +The output would have like structure: + +``` +--- a/neondb ++++ b/neondb +@@ -27,7 +27,8 @@ + CREATE TABLE public.playing_with_neon ( + id integer NOT NULL, + name text NOT NULL, +- value real ++ value real, ++ created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP + ); +``` + +The (-) lines indicate schema elements removed from the base branch, and the (+) lines indicate schema elements added in the target branch. + +## Getting started + +Review our [API reference](https://api-docs.neon.tech/reference/getprojectbranchschemacomparison) for detailed instructions. If you don’t have a Neon account yet, you can create one for free [here](https://console.neon.tech/signup). Neon has a [Free Plan](https://neon.tech/pricing) that includes 10 projects, with 10 branches per project. diff --git a/content/blog/posts/app-build-can-now-build-python-data-apps.md b/content/blog/posts/app-build-can-now-build-python-data-apps.md new file mode 100644 index 0000000000..bce9bfd13b --- /dev/null +++ b/content/blog/posts/app-build-can-now-build-python-data-apps.md @@ -0,0 +1,79 @@ +--- +title: app.build Can Now Build Python Data Apps +description: Generate data apps and dashboards in seconds +excerpt: >- + When we started working on app.build, we knew in the longer run we wanted to + build a generic agent that could build apps with different “coding stacks”. + However, for our first release and initial announcement, we could only build + apps written with a single fixed stack: In the rec... +date: '2025-07-18T15:08:01' +updatedOn: '2025-10-02T00:17:33' +category: product +categories: + - product + - ai +authors: + - arseni-kravchenko + - david-gomes + - pedro-figueiredo +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/app-build-can-now-build-python-data-apps/cover.jpg + alt: null +isFeatured: false +seo: + title: app.build Can Now Build Python Data Apps - Neon + description: >- + app.build now supports Python, enabling you to build data applications and + ML dashboards with a modern Python stack. + keywords: [] + noindex: false + ogTitle: app.build Can Now Build Python Data Apps - Neon + ogDescription: >- + app.build now supports Python, enabling you to build data applications and + ML dashboards with a modern Python stack. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/app-build-can-now-build-python-data-apps/social.png +--- + + +Since publishing this post, we’ve shifted focus. The managed version of app.build has been discontinued, but the source code is available - if you’re also building an agent, you can still explore the app.build [agent](https://github.com/appdotbuild/agent) and [platform](https://github.com/appdotbuild/platform) code for reference and implementation examples. We’re also applying this learnings and code to our next project. + + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/app-build-can-now-build-python-data-apps/ad4nxfqab9g0mxpa8aksmys7qf6gynm0m-exeqooxvvvmvumrydokemtrmrcxasw1vf-l6wvqagiazsqa4dlxyh54k8rrrfzyhjvkisrddbjj-wcikpxcmwvrakltolqdi-75de0oiw-265fb0d1.png) + +When we started working on [app.build](https://app.build), we knew in the longer run we wanted to build a generic agent that could build apps with different “coding stacks”. However, for our first release and initial announcement, we could only build apps written with a single fixed stack: + +- React + Vite + Tailwind CSS + Radix UI for frontend; +- tRPC + Fastify for backend; +- Postgres + DrizzleORM for data persistence. + +In the [recent blog post on design decisions](https://www.app.build/blog/design-decisions), we elaborated why we introduced those restrictions in the first place. + +But our original implementation was designed to be extendable. For that reason, we’ve been eager to add new supported stacks to app.build. Today we’re announcing we got a new one – data apps based on a modern Python stack with the [NiceGUI](https://nicegui.io/) framework. + +NiceGUI has many built-in features for data visualization, making it a solid default choice for data apps – e.g. dashboards or ML demos. NiceGUI apps we build are also more flexible compared to the tRPC apps we supported before. While not as visually polished initially, they are more functional – our agent adapts to user requirements and adds specific Python libraries if needed. + +Unlike some alternative frameworks like Streamlit, NiceGUI apps are more production-ready – they’re basically FastAPI under the hood, so there are no problems with concurrent usage, state management, and testing. And, of course, app.build’s NiceGUI apps use PostgreSQL for persistent storage! You can find the base template [here on GitHub](https://github.com/appdotbuild/agent/tree/e9a4503ab3dc61ebb6357daa004c71b8cb4def58/agent/nicegui_agent/template). + +Unlike tRPC apps, this new NiceGUI stack is more suitable for internal enterprise apps – such as dashboards, tech demos and other glue. For now, users can leverage from the rich Python ecosystem to add new libraries. We aim to add more advanced integrations supported in this stack similar to how we treat PostgreSQL as first class addition, starting with Databricks Unity Catalog. However, those integrations will stay optional, so app.build stays friendly to any customizations for your specific needs. + +Here’s an example of an app that was generated with Python+NiceGUI: + + + +Another example of the [app created by app.build](https://github.com/appdotbuilder/stock-tracker) on Python template leveraging real Yahoo Finance API to fetch up-to-date data has been deployed to [https://app-0e0689a4-d17d-465f-a5d4-f4f80a0a4595.build.myneon.app/](https://app-0e0689a4-d17d-465f-a5d4-f4f80a0a4595.build.myneon.app/) as interactive demo. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/app-build-can-now-build-python-data-apps/ad4nxegjbe27zf6pgssf8zjiaep3sn1b9kiosdoo4rbvrwksngof9pjtxwhxtesgbxicoob0lrasem1mgx6ffbwamu-njq2ajypav1zcqn7denlbadnvc3q9ifzq6wu-64in-3lyxq-c6eb741d.png) + +To create your own NiceGUI app like that, just run `npx @app.build/cli --template=python`. + +Our approach to generating the apps requires proper scaffolding – we start with a polished default template and a set of validation checks to establish the feedback loop for the agent. Despite it taking way more time than just vibe-coding an app in any stack one can imagine, those efforts pay off very soon – once the stable scaffolding is ready, those apps’ quality becomes acceptable for typical usage scenarios. + +We aim to support a limited but well-curated set of stacks for various needs. A little sneak peek: we’re working on the third stack to add to the collection. Stay tuned for the new announcements and technical details posts! + +--- + +_This blog post was originally published in the [app.build blog](https://www.app.build/blog/appbuild-can-now-build-python-data-apps)._ diff --git a/content/blog/posts/app-build-open-source-ai-agent.md b/content/blog/posts/app-build-open-source-ai-agent.md new file mode 100644 index 0000000000..2847b0d719 --- /dev/null +++ b/content/blog/posts/app-build-open-source-ai-agent.md @@ -0,0 +1,98 @@ +--- +title: 'app.build: An Open-Source AI Agent That Builds Full-Stack Apps' +description: >- + A reference implementation for any codegen product looking to build on top of + Neon +excerpt: >- + Code generation has been one of the most interesting use cases for LLMs. While + the best models can generate decent code for isolated problems, there is a big + gap between these code snippets and fully functional applications. Agent-based + solutions are better suited to create apps.... +date: '2025-06-04T19:40:32' +updatedOn: '2025-10-01T16:46:35' +category: ai +categories: + - ai + - company +authors: + - david-gomes +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/app-build-open-source-ai-agent/neon-appbuild.jpg + alt: null +isFeatured: true +seo: + title: 'app.build: An Open-Source AI Agent That Builds Full-Stack Apps - Neon' + description: >- + A reference implementation built from everything we’ve learned helping agent + platforms scale. Use it, fork it, remix it. + keywords: [] + noindex: false + ogTitle: 'app.build: An Open-Source AI Agent That Builds Full-Stack Apps - Neon' + ogDescription: >- + A reference implementation built from everything we’ve learned helping agent + platforms scale. Use it, fork it, remix it. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/app-build-open-source-ai-agent/neon-appbuild-1.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/app-build-open-source-ai-agent/neon-appbuild.jpg) + + +Since publishing this post, we’ve shifted focus. The managed version of app.build has been discontinued, but the source code is available - if you’re also building an agent, you can still explore the app.build [agent](https://github.com/appdotbuild/agent) and [platform](https://github.com/appdotbuild/platform) code for reference and implementation examples. We’re also applying this learnings and code to our next project. + + +Code generation has been one of the most interesting use cases for LLMs. While the best models can generate decent code for isolated problems, there is a big gap between these code snippets and fully functional applications. Agent-based solutions are better suited to create apps. To close the gap, an agent needs to iterate on the code, test the result and make decisions based on the feedback – either human comments or technical (like logs, test results, linters output etc.). In the last few months, a lot of AI app builders have come to market to tackle this problem, but the gap is not closed yet. + +And today, we’re launching [app.build](https://app.build) – an **open-source agent that can build and deploy full-stack applications, with end-to-end tests and automated deployments**. We want to build a community around this project, since we know a lot of developers are interested in hacking in this space, particularly by bringing their own models and doing most things locally. + +## How can I try it then? + +``` +npx @app.build/cli +``` + +**This very simple command is all you need to get started.** + +Notice that, by default, it will ask you to sign in with GitHub. Any application that you generate will have its own repository (in your GitHub account), and will be deployed to the Internet with a real backend and a real database. This is our “managed service” experience. We’re also working on a smoother way for you to run everything locally (which will include being able to choose which models you want for which tasks, and even self-hosted models). + +## Can I run this locally? + +Yes, everything runs locally. Our [README.md](https://readme.md/) contains instructions for how to run this locally, but we have yet to create a full guide for running both the CLI and the agent locally with more detail. + +We also want to write down instructions on how you can bring your own models (including self-hosted ones). + +## What features does app.build offer? + +Our CLI allows both creating new apps from scratch, and iterating on previously created apps (add new features or other types of changes). **It is extremely barebones for now!** + +But our platform gives you: + +- An authentication provider (it defaults to Neon Auth) +- A hosted database (it defaults to Neon Postgres) +- Your app’s frontend and backend get deployed immediately (in Neon and Koyeb’s infrastructure) +- Hosted repository on your own GitHub account + +1. + +## How does the agent work? + +We’ve taken a lot of steps towards having an agent architecture that is focused on generating really high-quality applications that actually work as intended. This is why our agent writes end-to-end tests and actually runs them to succession as part of the generation pipeline. + +Furthermore, we have a very solid base template for every app (for now, just 1 template, which is for web apps), and we made sure that the agent is extremely well-versed on all the technologies that are used in this template (Fastify, Drizzle, React, Vite, etc.). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/app-build-open-source-ai-agent/appbuild-1-1024x928-81e63aed.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/app-build-open-source-ai-agent/appbuild-2-1024x686-0d9ff52f.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/app-build-open-source-ai-agent/appbuild-3-1024x819-9d3a69a5.png) + +Our agent follows the “divide-and-conquer” principle — app creation is decomposed into multiple tasks, and each can be solved independently. Furthermore, those tasks themselves may have their own subtasks, so we don’t rely on LLMs generating large chunks of code at once. Each particular task passes a series of checks (e.g. Does it compile? Does it pass ESLint? Does it pass tests?), thus ensuring the final app works. + +In future blog posts and talks, we’ll definitely dive deeper into the inner workings of the agent. + +## How can I get involved? + +For now, we’re hacking over at [github.com/appdotbuild](https://github.com/appdotbuild). You can see all the code, all the commits and all of our planned tasks. We hope to create a Discord channel, and a proper public-facing roadmap soon! + +We’re also currently working on more blog posts and videos explaining our code generation pipeline, and other architectural decisions we made while building the agent. diff --git a/content/blog/posts/app-build-supports-open-source-models-locally.md b/content/blog/posts/app-build-supports-open-source-models-locally.md new file mode 100644 index 0000000000..8195831aba --- /dev/null +++ b/content/blog/posts/app-build-supports-open-source-models-locally.md @@ -0,0 +1,116 @@ +--- +title: 'Generate Apps Locally for Free: App.build Now Supports Open Source Models' +description: 'Run it locally with Ollama, LMStudio, and OpenRouter' +excerpt: >- + App.build now supports open weights LLMs via Ollama, LMStudio, and OpenRouter + – enabling you to generate complete applications end-to-end without cloud API + dependency or associated costs. Why Run App.build Locally? Zero API costs + Cloud LLM APIs can become expensive fast during ex... +date: '2025-08-11T19:41:53' +updatedOn: '2025-10-02T00:23:37' +category: ai +categories: + - ai +authors: + - arseni-kravchenko + - evgenii-kniazev +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/app-build-supports-open-source-models-locally/cover.jpg + alt: null +isFeatured: true +seo: + title: >- + Generate Apps Locally for Free: App.build Now Supports Open Source Models - + Neon + description: >- + App.build, our open-source architecture reference for building agents, now + supports open weights LLMs via Ollama, LMStudio, and OpenRouter. + keywords: [] + noindex: false + ogTitle: >- + Generate Apps Locally for Free: App.build Now Supports Open Source Models - + Neon + ogDescription: >- + App.build, our open-source architecture reference for building agents, now + supports open weights LLMs via Ollama, LMStudio, and OpenRouter. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/app-build-supports-open-source-models-locally/social.png +--- + + +Since publishing this post, we’ve shifted focus. The managed version of app.build has been discontinued, but the source code is available - if you’re also building an agent, you can still explore the app.build [agent](https://github.com/appdotbuild/agent) and [platform](https://github.com/appdotbuild/platform) code for reference and implementation examples. We’re also applying this learnings and code to our next project. + + +App.build now supports open weights LLMs via Ollama, LMStudio, and OpenRouter – enabling you to generate complete applications end-to-end without cloud API dependency or associated costs. + +## Why Run App.build Locally? + +### Zero API costs + +Cloud LLM APIs can become expensive fast during extended coding sessions. Even our small dev team can burn through hundreds of dollars in API costs during a single intensive testing session. That is acceptable for a company and less so for hobbyists, and we want more enthusiasts to hack around our framework. + +### No rate limits or outages + +API-based tools frequently hit rate limits during intensive development sessions, especially when major providers struggle with the load. Getting 5xx errors from Anthropic isn’t uncommon. + +### Data privacy and control + +Running locally means your code, ideas, and proprietary information never leave your machine – addressing GDPR, HIPAA, and IP protection concerns without relying on cloud provider policies. + +### Because it’s cool! + +Using open weights models, running everything locally, contributing to the open source ecosystem – [sometimes the philosophical reasons matter too](https://www.smbc-comics.com/comic/2010-12-09). 🙂 We’re having a lot of fun working on app.build, and want to align it with our personal preferences. + +## The Open Weights Models Situation + +### Hardware available for pet projects + +Consumer GPUs like RTX 3090/4090 with 24GB VRAM can run capable 8B models at full precision or heavily quantized 30B models (e.g. Qwen3-30B3A). Apple M4 Macbooks with up to 48GB unified memory offer excellent local inference performance with even more flexibility. + +E.g. MoE (mixture of experts) models like aforementioned Qwen3-30B3A or recent GPT-OSS-20B by OpenAI can be inferred with very reasonable speed on such laptops (~60-70 tokens per second in 4-bit quantized versions, which is [comparable](https://artificialanalysis.ai/models#speed) to some popular commercial APIs). Among the other upcoming hardware for local LLM inference we should mention [Nvidia DGX Spark](https://www.nvidia.com/en-us/products/workstations/dgx-spark/), [Asus Ascent GX10](https://www.asus.com/networking-iot-servers/desktop-ai-supercomputer/ultra-small-ai-supercomputers/asus-ascent-gx10/); there are also rumors on next gen of Mac Studios being a great choice for local LLMs. + +
+Image +
app.build works in your homelab too 🙂 ours pictured
+
+ +### OpenRouter: best of two worlds + +Not everyone is a hardware enthusiast and wants to fill their apartment with GPUs. It should not be a blocker for experiments! + +[OpenRouter](https://openrouter.ai/) is a service allowing users to query multiple LLMs via unified API, including both closed models and open weights ones served by various infra providers, including [Cerebras](https://www.cerebras.ai/) and [Groq](https://groq.com/) – companies with unique serving tech allowing for hyperfast inference. Unlike local inference, OpenRouter can’t guarantee absolute privacy, though they claim not to store users’ prompts unless stated explicitly for some models. + +Using OpenRouter allows you to try open-source models that are barely available for typical home setup and require 100s of GB VRAM. It comes with a very reasonable pricing: e.g. [Kimi K2](https://moonshotai.github.io/Kimi-K2/) – open-source model comparable to Claude Sonnet can be inferred for $0.6/$2.50 for 1M input/output tokens (compare it with $3/$15 for Sonnet). The model has 1T parameters, which is an absolute blocker for most self-serving adepts. + +### What’s the situation with available models? + +We don’t have a holistic benchmark to evaluate the whole model variety (and are working on closing this gap). However, our early evaluation shows large open source models being approximately on par with closed source alternatives. Honorable mentions: Qwen3-Coder, Kimi K2. + +Smaller open source models do not really perform well enough at the moment. Building an app in a single shot with a model served on your local laptop is a very probabilistic gamble: you may get lucky, but more likely the model gets confused and fails to converge. Some of the failures are associated with malformed tool calling that is likely to be addressed by the inference software. However, overall performance on agentic tasks – that are crucial for app.build needs – is still lagging behind closed models. + +We’re certain the next generation of open source models runnable on consumer hardware will get closer to production-ready experience. For now, we consider them applicable for less autonomous use cases rather than generating full apps in a single run. + +
+Image +
A screenshot of the app generated using open weights models
+
+ +## Getting Started + +We added new environmental variable for the agent configuration forcing it to use non-default models: + +```bash +LLM_BEST_CODING_MODEL=openrouter:qwen/qwen3-coder +LLM_UNIVERSAL_MODEL=openrouter:z-ai/glm-4.5-air uv run generate "make me another todo app but make it stylized to Roman Empire because I think about it too often" +``` + +For local inference use `ollama:vendor/model` and `lmstudio:host` variables (`lmstudio:` with no host works too – we like reasonable defaults). + +## The Bottom Line + +Local LLM support gives you the freedom to experiment, prototype, and build without vendor lock-in, cost anxiety, or data sharing concerns. Combined with the rapid improvement of open source models, local app generation is not quite a viable alternative to cloud APIs, but getting there quickly. + +--- + +[Try app.build locally](https://github.com/appdotbuild/agent/) _or use our_ [managed service](https://app.build/) _for free. If you’re interested in building something similar, [check out the code (](https://github.com/appdotbuild/agent) and contribute!)._ diff --git a/content/blog/posts/architecture-decisions-in-neon.md b/content/blog/posts/architecture-decisions-in-neon.md new file mode 100644 index 0000000000..51b2704841 --- /dev/null +++ b/content/blog/posts/architecture-decisions-in-neon.md @@ -0,0 +1,125 @@ +--- +title: Architecture decisions in Neon +description: Decisions we made on Neon architecture +excerpt: >- + The idea behind Neon is to create a new serverless Postgres service with a + modern cloud-native architecture. When building for the cloud it usually is a + good idea to separate storage and compute. For operational databases such + design was first introduced by AWS Aurora 1, followed... +date: '2022-07-08T09:29:00' +updatedOn: '2026-03-13T15:54:09' +category: engineering +categories: + - engineering +authors: + - heikki-linnakangas +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/architecture-decisions-in-neon/cover.jpg + alt: Architecture decisions in Neon +isFeatured: false +seo: + title: Architecture decisions in Neon - Neon + description: Decisions we made on Neon architecture + keywords: [] + noindex: false + ogTitle: Architecture decisions in Neon - Neon + ogDescription: >- + The idea behind Neon is to create a new serverless Postgres service with a + modern cloud-native architecture. When building for the cloud it usually is + a good idea to separate storage and compute. For operational databases such + design was first introduced by AWS Aurora 1, followed by many others 2 3, + however none of the implementations were […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/architecture-decisions-in-neon/cover.jpg +--- + +![Architecture decisions in Neon](https://cdn.neonapi.io/public/images/pages/blog/architecture-decisions-in-neon/architecture-decisions-in-neon-1-1024x538-a53a6eb3.jpg) + +The idea behind Neon is to create a new serverless Postgres service with a modern cloud-native architecture. When building for the cloud it usually is a good idea to separate storage and compute. For operational databases such design was first introduced by AWS Aurora 1, followed by many others 2 3, however none of the implementations were open source and native to Postgres. + +We wanted to make Neon the best platform to run Postgres on. As we started to figure out the details we needed to understand what exactly the architecture should look like for an OLTP cloud database. We also knew that we couldn’t deviate from Postgres. People choose Postgres for many reasons. It’s open source, feature-rich, and has a large ecosystem of extensions and tools. But increasingly, it’s simply the default choice. There are a lot of databases out there with different strengths and weaknesses, but unless you have a particular reason to pick something else, you should just go with Postgres. Therefore, we don’t want to compete with Postgres itself or maintain a fork. We understood that Neon would only work in the market if it doesn’t fork Postgres and gives users 100% compatibility with their apps written for Postgres. + +So before we wrote a single line of code, we had some big upfront decisions to make on the architecture. + +## Separating storage and compute + +The core idea of an Aurora-like architecture of separation of storage and compute is to replace the regular filesystem and local disk with a smart storage layer. + +Separating compute and storage allows you to do things that are difficult or impossible to do otherwise: + +- Run multiple compute instances without having multiple copies of the data. +- Perform a fast startup and shutdown of compute instances. +- Provide instant recovery for your database. +- Simplify operations, like backups and archiving, to be handled by the storage layer without affecting the application. +- Scale CPU and I/O resources independently. + +The first major decision for us was if we should just use a SAN or an off-the-shelf distributed file system. You can certainly get some of these benefits from smart filesystems or SANs, and there are a lot of tools out there to manage these in a traditional installation. But having a smart storage system that knows more about the database system and the underlying infrastructure of the cloud provider makes the overall system simpler, and gives a better developer experience. We were aware of Delphix – a company that is built on the premise of providing dev and test environments for database products using zfs. If we took a similar approach due to the fact that we don’t control the filesystem tier, it would be hard to efficiently integrate it with the cloud and result in a clunky and expensive solution. We could still sell it to large enterprises, but we knew we could do better. **So the first decision was made: no SANs, no third party filesystems. Let’s build our own storage from the first principles.** + +## Storage Interface + +We started to think about what the interface should be between compute and storage. Since we have many Postgres hackers on the team, we already knew how it works in vanilla Postgres. Postgres generates WAL (write ahead log) for all data modifications, and the WAL is written to disk. Each WAL record references one or more pages, and an operation and some payload to apply to them. In essence, each WAL record is a diff against the previous version of the page. As Postgres processes a WAL record it applies the operation encoded in the WAL record to the page cache, which will eventually write the page to disk. If a crash occurs before this happens, the page is reconstructed using the old version of the page and the WAL. + +Postgres architecture gave us a hint of how to integrate our cloud native storage. We can make Postgres stream WAL to Neon storage over the network and similarly read pages from Neon storage using RPC calls. If we did that, Postgres changes would be minimal and we can even hope to push them upstream. + +It was clear that we will need to have a consensus algorithm for persisting the WAL – the database is the log, it has to be incredibly robust. And it was also clear that we need to organize pages so that we can quickly return them when requested by Postgres. What was not clear was if we should have _two_ services: one for WAL and one for serving pages or _one_ that combines all of it. Aurora has one, SQL Server, which came later, has two. There was a decision to make. + +## Separating Page servers and the WAL service + +One early decision was to separate the WAL and page service. The WAL service consists of multiple WAL _safekeeper_ nodes that receive the WAL from Postgres, and run a consensus algorithm. The consensus algorithm ensures durability, even if one of the safekeeper nodes is down. It also ensures that only one Postgres instance is acting as the primary at any given time, avoiding split-brain problems. Pageservers store committed WAL and can reconstruct a page at any given point of WAL on request from the compute layer. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/architecture-decisions-in-neon/neon-architecture-2-f9689a73.png) + +Separating the WAL service has several advantages: + +- The WAL service and the page servers can be developed independently and in parallel. +- It is easier to reason about and verify the correctness of the consensus algorithm when it is a separate component. +- We can use hardware optimized for different purposes efficiently; the I/O pattern and workload of the safekeepers is very different from the page servers – one is append-only, and the other one is both read, write, and update. + +## Relationship between compute and pageservers + +Does one compute only talk to one pageserver or should we spread out pages from one database across multiple pageservers? Also, does one pageserver only contain data for one or many databases? The latter question is a simple one. We need to build multi-tenancy to support a large number of small databases efficiently. So one pageserver can contain pages from many databases. + +The former question is a trade-off between simplicity and availability. If we spread database pages across many pageservers, and especially if we cache the same page on multiple page servers, we can provide better availability in case a page server goes down. To get started, we implemented a simple solution with one pageserver, but will add a pageserver “sharding” feature later to support high availability and very large databases. + +## Treat historical data the same as recent data + +The most straightforward model for the page servers would be to replay the WAL as it is received, to keep an up-to-date copy of the database – just like a Postgres replica. However, replicas connected to the storage system can lag behind the primary, and need to see older versions of pages. So at least you need some kind of a buffer to hold old page versions, in case a read replica requests them. **But for how long?** There’s no limitation on how far behind a read replica can lag. + +Then we started to think about WAL archiving, backups and Point-in-Time Recovery (PITR). How are those things going to work in Neon? Do we need to build them as separate features or can we do better? Could the storage handle all of those? + +PITR is a standard feature in most serious OLTP installations. The canonical use case for PITR is that you accidentally drop a table, and want to restore the database to the state just before that. You don’t do PITR often, but you want to have the capability. To allow PITR, you need to retain all old page versions in some form, as far back as you want to allow PITR. Traditionally, that’s done by taking daily or weekly backups and archiving all the WAL. + +You don’t do PITR often, because it has traditionally been a very expensive operation. You start from the last backup and replay all the archived WAL to get to the desired point in time. This can take hours. And if you pick the wrong point to recover to, you have to start all over again. + +What if PITR was a quick and computationally cheap operation? If you don’t know the exact point to recover to, that’s OK; you can do PITR as many times as you need to. You could use it for many things that are not feasible otherwise. For example, if you want to run an ad hoc analytical query against an OLTP database, you could do that against a PITR copy instead, without affecting the primary. + +If you have a storage system that keeps all the old page versions, such operations become cheap. You can query against an older point in time just the same as the latest version. + +We decided to embrace the idea of keeping old page versions, and build the storage system so that it can do that efficiently. It replaces the traditional backups and the WAL archive, and makes all of the history instantly accessible. The immediate question is “But at what cost”? If you have a PITR horizon of several weeks or months, that can be a lot of old data, even for a small database. We needed a way to store the old and cold data efficiently and the solution was to move cold and old data to cloud object storage such as S3. + +## Leverage cloud object storage + +The most efficient I/O pattern is to write incoming data sequentially and avoid random updates of old data. In Neon, incoming WAL is processed as it arrives, and indexed and buffered in memory. When the buffer fills up, it is written to a new file. Files are never modified in place. In the background, old data is reorganized by merging and deleting old files, to keep read latency in check and to garbage collect old page versions that are no longer needed for PITR. This design was inspired by Log-Structured Merge-trees. + +This system, based on immutable files, has a few important benefits. Firstly, it makes compression easy. You can compress one file at a time, without having to worry about updating parts of the file later. Secondly, it makes it easy to scale the storage, and swap in and out parts of the database as needed. You can move a file to cold storage, and fetch it back if it’s needed again. + +Neon utilizes cloud object storage to make the storage cost-efficient and robust. By relying on object storage, we don’t necessarily need multiple copies of data in the page servers, and we can utilize fast but less reliable local SSDs. Neon offloads cold parts of the database to object storage, and can bring it back online when needed. In a sense, the page servers are just a cache of what’s stored in the object storage, to allow fast random access to it. Object storage provides for the long-term durability of the data, and allows easy sharding and scaling of the storage system. + +## One year later + +We built a modern, cloud-native architecture that separates storage from compute to provide an excellent Postgres experience. Different decisions would have made some things easier and others harder, but so far, we haven’t regretted any of these choices. The immutable file format made it straightforward to support branching, for example, and we have been able to develop the page server and safekeeper parts fairly independently, just like we thought. You can [get early access](https://neon.tech/early-access/) to our service and experience the benefits directly. + +--- + +1\. A. Verbitski et al., “Amazon Aurora,” Proceedings of the 2017 ACM International Conference on Management of Data. ACM, May 09, 2017 [Online]. Available: [https://dx.doi.org/10.1145/3035918.3056101](https://dx.doi.org/10.1145/3035918.3056101) [↩](https://neon.tech/blog/architecture-decisions-in-neon/#fnref-1) + +2\. P. Antonopoulos et al., “Socrates,” Proceedings of the 2019 International Conference on Management of Data. ACM, Jun. 25, 2019 [Online]. Available: [https://www.microsoft.com/en-us/research/uploads/prod/2019/05/socrates.pdf](https://www.microsoft.com/en-us/research/uploads/prod/2019/05/socrates.pdf) [↩](https://neon.tech/blog/architecture-decisions-in-neon/#fnref-2) + +3\. W. Cao et al., “PolarDB Serverless,” Proceedings of the 2021 International Conference on Management of Data. ACM, Jun. 09, 2021 [Online]. Available: [https://dx.doi.org/10.1145/3448016.3457560](https://dx.doi.org/10.1145/3448016.3457560) [↩](https://neon.tech/blog/architecture-decisions-in-neon/#fnref-3) + +## 📚 Keep reading + +- **[A deep dive into our storage engine:](https://neon.tech/blog/get-page-at-lsn)** Neon’s custom-built storage is the core of the platform—get details into how its built. +- **[How we scale an open-source, multi-tenant storage engine for Postgres written in Rust](https://neon.tech/blog/how-we-scale-an-open-source-multi-tenant-storage-engine-for-postgres-written-rust):** keep learning about how we implemented sharding in our storage to allow Neon to host larger datasets and I/O. +- **[1 year of autoscaling Postgres:](https://neon.tech/blog/1-year-of-autoscaling-postgres-at-neon)** a review of our autoscaling design. Neon can autoscale your Postgres instance without dropping connections or interrupting your queries, avoiding the need for overprovisioning or resizing manually. diff --git a/content/blog/posts/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you.md b/content/blog/posts/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you.md new file mode 100644 index 0000000000..3ef96a613b --- /dev/null +++ b/content/blog/posts/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you.md @@ -0,0 +1,118 @@ +--- +title: "Atoms is Out, a Multi-Agent AI Team that Builds Full-Stack Apps for You" +description: Every Atoms app runs on its own Neon project +excerpt: >- + “We chose Neon as our backend because of scale, cost-efficiency, and superior + developer and user experience, all critical for Atoms’ multi-tenant + architecture. Each user app needs its own dedicated database, and Neon lets us + spin up Postgres instances dynamically while only payin... +date: "2026-01-13T17:48:26" +updatedOn: "2026-01-13T17:58:03" +category: ai +categories: + - ai + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you/cover.png + alt: null +isFeatured: true +seo: + title: >- + Atoms is Out, a Multi-Agent AI Team that Builds Full-Stack Apps for You - + Neon + description: >- + Atoms just launched a multi-agent AI platform that builds full-stack apps, + powered by Neon’s backend and the Agent Plan. + keywords: [] + noindex: false + ogTitle: >- + Atoms is Out, a Multi-Agent AI Team that Builds Full-Stack Apps for You - + Neon + ogDescription: >- + Atoms just launched a multi-agent AI platform that builds full-stack apps, + powered by Neon’s backend and the Agent Plan. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you/social.png +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you/screenshot-2026-01-13-at-93258-am-1024x568-2c18adc2.png) + +
+

“We chose Neon as our backend because of scale, cost-efficiency, and superior developer and user experience, all critical for Atoms’ multi-tenant architecture. Each user app needs its own dedicated database, and Neon lets us spin up Postgres instances dynamically while only paying for them when they’re actually in use.” (Atoms Engineering Team)

+
+ +[Atoms](https://atoms.dev/) just launched, a new multi-agent platform that uses a team of AI engineers to build full-stack applications. They’re part of the [Neon Agent Plan](https://neon.com/use-cases/ai-agents), taking advantage of higher resource limits and special pricing to power their projected scale of millions of projects. + +## Meet Atoms: A Full-Stack AI Team Working For You + +Atoms isn’t conceived as a code generator, but more as a platform for building complete products. + +Instead of relying on a single general-purpose agent, Atoms brings together a team of specialized AI agents that collaborate like a real product organization, each focused on a specific area of product development: researching, planning, architecting, and shipping. They’re trained to work together from your prompt and deliver a production-ready software product. + +
+ +
+ +You can think of Atoms’ core agents as your Full-Stack AI Team: + +- [Iris](https://atoms.dev/blog/introduce-iris-deep-research-agent) is the DeepResearch agent. She conducts structured market and trend analysis. +- Emma is the Product Manager agent. She turns Iris’s insights into product requirements and user stories. +- Bob is the Architect agent, who defines the technical blueprint. +- Alex, the Engineer agent, writes and tests the code. + +Together, all these agents take your idea and take it all the way to a live product. They can build full SaaS tools, membership sites, content platforms, marketplaces, and even online course systems, complete with authentication, payments, CMS capabilities, and more. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you/image-6-1024x640-a53a5f7f.png) + +A big part of this experience is the abstraction of the entire process of building and operating both the front end and the backend. To deliver on that promise, Atoms integrates the full build process: front-end generation, backend automation, and deployment. Long-running tasks are supported by Race Mode, which runs multiple build variants in parallel and automatically tests for quality, and by the Data Analyst Agent, which turns uploaded data into visual insights through conversational analysis. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you/image-7-1024x640-60f46969.png) + +## Why Atoms is Built on Neon + +Every product generated by Atoms’ multi-agent team comes with a fully functional backend, with all the elements created automatically. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/atoms-is-out-a-multi-agent-ai-team-that-builds-full-stack-apps-for-you/image-8-1024x640-581a2d4c.png) + +To make this possible at scale, Atoms needed infrastructure that could provision many thousands of Postgres databases instantly, scale to zero when idle, and guarantee isolation between projects. + +[Neon](https://neon.com/) was the perfect match. + +### Dedicated backend per app + +Every new app created with Atoms receives its own [dedicated Neon instance](https://neon.com/use-cases/database-per-tenant). This architecture relies on Neon’s API-driven [instant provisioning:](https://neon.new/) databases are ready in seconds without any manual setup or credentials. For Atoms’ users, the backend simply exists when the front end is ready, making full-stack generation feel effortless. + +### Full-stack automation support + +Neon doesn’t just let Atoms create a database but enables complete backend automation. Backends include [authentication](https://neon.com/docs/neon-auth/overview), schema generation, data pipelines, and payment integration, all built through Neon’s programmable API. This integration turns what would normally be days of manual backend configuration into an invisible, fully automated workflow. + +### Total isolation and security + +Because each Neon instance is physically isolated, projects built on Atoms never interfere with one another. This separation ensures consistent performance and data security across millions of independently running environments, something that was crucial for the Deepwisdom team. + +### Ready for AI-scale + +None of this would be possible without Neon’s [serverless compute layer](https://neon.com/docs/introduction/serverless) and [scale-to-zero](https://neon.com/docs/introduction/scale-to-zero), which make Atoms’ multi-tenant model both cost-efficient and operationally simple even at massive scale. When an app goes idle, Neon automatically suspends its compute, resuming instantly when activity returns and [autoscaling](https://neon.com/docs/introduction/autoscaling) as required for active workloads. This lets Atoms deploy thousands of concurrent apps while aligning infrastructure cost perfectly with real usage. + +## A Special Plan for Agent Builders + +
+

“The Neon Agent Plan expanded quotas and credits directly accelerated our product testing and development cycle. We were able to run larger, more realistic experiments much earlier in the process, which gave us confidence to move faster on new ideas” (Atoms Engineering Team)

+
+ +As one of the teams in the [Neon Agent Plan](https://neon.com/use-cases/ai-agents), Atoms gained access to a program designed specifically to accelerate time to market for agentic platforms. This includes not only special pricing and higher resource limits but also onboarding guidance and configuration advice based on the experience we’ve gained from working with many other agent teams. + +
+

“The onboarding support from the Neon team was so valuable. They took the time to understand our unique challenges and advice us on a smoother path from prototype to production, with a strong sense of partnership” (Atoms Engineering Team)

+
+ +If you’re building your own AI agent or codegen platform, apply for the [Neon Agent Plan](https://neon.com/use-cases/ai-agents). Our team will reach out to design a path that helps you launch faster and grow with confidence. + +--- + +_A big congratulations to the team on the launch of Atoms. Don’t forget to try it_ [here](https://atoms.dev/)_._ diff --git a/content/blog/posts/auth-setup-with-neon-keycloak-and-koyeb.md b/content/blog/posts/auth-setup-with-neon-keycloak-and-koyeb.md new file mode 100644 index 0000000000..657d97ca08 --- /dev/null +++ b/content/blog/posts/auth-setup-with-neon-keycloak-and-koyeb.md @@ -0,0 +1,151 @@ +--- +title: 'Auth setup with Neon, Keycloak and Koyeb' +description: 'Learn how to use Keycloak with Neon, and how to deploy Keycloak on Koyeb.' +excerpt: >- + Keycloak is an open-source identity and access management solution that + centralizes authentication and authorization management. It provides features + such as single sign-on (SSO), two-factor authentication, social login, and + user federation with LDAP or Active Directory user fede... +date: '2023-12-15T19:15:41' +updatedOn: '2024-03-01T14:02:26' +category: community +categories: + - community +authors: + - evan-shortiss +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/cover.jpg + alt: Keycloak with Neon and Koyeb +isFeatured: false +seo: + title: 'Auth setup with Neon, Keycloak and Koyeb - Neon' + description: 'Learn how to use Keycloak with Neon, and how to deploy Keycloak on Koyeb.' + keywords: [] + noindex: false + ogTitle: 'Auth setup with Neon, Keycloak and Koyeb - Neon' + ogDescription: >- + Keycloak is an open-source identity and access management solution that + centralizes authentication and authorization management. It provides + features such as single sign-on (SSO), two-factor authentication, social + login, and user federation with LDAP or Active Directory user federation. + Various identity protocols such as Open ID Connect, SAML 2.0, and OAuth 2.0 + are supported, which makes integrating […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/cover.jpg +--- + +![Keycloak with Neon and Koyeb](https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/neon-keycloak1-1-1024x576-6059864e.jpg) + +[Keycloak](https://www.keycloak.org/) is an open-source identity and access management solution that centralizes authentication and authorization management. It provides features such as single sign-on (SSO), two-factor authentication, social login, and user federation with LDAP or Active Directory user federation. Various identity protocols such as Open ID Connect, SAML 2.0, and OAuth 2.0 are supported, which makes integrating Keycloak with new and existing applications easy. + +
Whether you’re a novice getting started with Keycloak or a veteran user, you’ll need a database to support your Keycloak deployment in production. Since Keycloak has first-class support for Postgres, this guide will outline a minimal setup to use Keycloak with Neon’s Postgres. All configurations and code in this guide can be found in the [neondatabase/keycloak-example](https://github.com/neondatabase/keycloak-example) repository on GitHub. + +## Create a Neon project and Keycloak Database + +Each Neon project includes a single Postgres instance that contains a `neondb` database by default. Still, it’s best to create a dedicated `keycloak` database to store Keycloak’s data. + +To create a project, visit the [Neon console](https://console.neon.tech/app/projects). If it’s your first time visiting the console you should be prompted to create a new project as shown. If you already have a project you’d like to use, select it from the [projects list](https://console.neon.tech/app/projects). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/create-project-1024x595-1e65d318.png) + +Create the Keycloak database by visiting the [SQL Editor](https://neon.tech/docs/get-started-with-neon/query-with-neon-sql-editor) section of your project, and running the following query: + +```sql +CREATE DATABASE keycloak; +``` + +You can verify that the `keycloak` database was created by viewing the **Databases** section of the Neon console or running the `SELECT datname FROM pg_database;` query in the **SQL Editor**. + +## Configure a Keycloak Database User + +Instead of accessing your new `keycloak` database using the default user with the [`neon_superuser`](https://neon.tech/docs/manage/roles#the-neonsuperuser-role) role, creating a new user named `keycloak_admin` is best. This new user will have administrative privileges limited to the `keycloak` database. + +Use the Neon SQL Editor to input the commands to create the user and assign permissions: + +1. Visit the **SQL Editor** in your project. +2. Select the `keycloak` database using the dropdown menu in the top-right. + +Issue the following command to create a new user named `keycloak_admin`, making sure to replace the password with a value that has 60 bits of entropy, [per our documentation](https://neon.tech/docs/manage/roles#manage-roles-with-sql): + +```sql +/* Create a strong password per: https://neon.tech/docs/manage/roles */ +CREATE USER keycloak_admin WITH PASSWORD 'r3plac3_th1s'; + +/* Assign permissions to the `keycloak_admin` user */ +GRANT ALL ON SCHEMA public TO keycloak_admin; +``` + +## Deploy Keycloak on Koyeb + +Keycloak can be deployed on a machine with a JDK installed, or from a container image using Docker or Podman. Setting up infrastructure manually can be quite a chore, so instead I’ll show you how to deploy Keycloak on [Koyeb](https://koyeb.com) with valid SSL certificates. If they support deploying container images, you can replace Koyeb with your preferred hosting provider. + +To get started, sign up for an account at [koyeb.com](https://koyeb.com). Next, click [this link](https://app.koyeb.com/apps/deploy?type=docker&image=quay.io%2Fkeycloak%2Fkeycloak%3A23.0.1&name=keycloak&env%5BKC_DB_USERNAME%5D=&env%5BKC_DB_PASSWORD%5D=&env%5BKC_DB_URL%5D=jdbc%3Apostgresql%3A%2F%2FNEON_HOSTNAME%2Fkeycloak%3Fsslmode%3Drequire&env%5BKC_HOSTNAME%5D=&env%5BKC_HTTP_ENABLED%5D=true&env%5BKC_PROXY%5D=edge&env%5BKC_DB%5D=postgres&ports=8080%3Bhttp%3B%2F&tag=23.0.1&docker.image.tag=23.0.1&image-tag=23.0.1&command=start&env%5BKEYCLOAK_ADMIN%5D=admin&env%5BKEYCLOAK_ADMIN_PASSWORD%5D=) to start deploying Keycloak on Koyeb from a template with some preconfigured values. + +Set the following parameters for the deployment: + +1. Instance type: Eco is adequate for testing. +2. Region: Choose the region closest to your Neon Postges database’s region. +3. Configure the following environment variables: + 1. `KC_DB_URL`: Replace the `NEON_HOSTNAME` with your Neon database’s hostname from the **Connection Details** on Neon’s console, e.g `ep-broken-hill-12345.us-east-2.aws.neon.tech`. + 2. `KC_DB_USERNAME`: `keycloak_admin`. + 3. `KC_DB_PASSWORD`: The password you set for `keycloak_admin`. + 4. `KC_HOSTNAME`: This must match the hostname you define when deploying the service on Koyeb. The hostname is listed at the bottom of the deploy page and is a combination of the App name and your organization name, e.g `keycloak-your-org.koyeb.app`. + 5. `KEYCLOAK_ADMIN_PASSWORD`: This will be the password you use to log in as the `admin` user and perform administrative actions in Keycloak + +Click the **Deploy** button and you’ll be directed to a screen where you can monitor the status of your deployment. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/koyeb-deploy-1024x578-f5f4ef84.png) + +Scroll down to the bottom of this page and wait for the logs to print a line stating that Keycloak has started. It will resemble the following example: + +``` +Keycloak 23.0.1 on JVM (powered by Quarkus 3.2.9.Final) started in 33.009s. Listening on: https://0.0.0.0:8080 +``` + +## Create a Keycloak Realm and Client + +Keycloak uses the concept of [realms](https://www.keycloak.org/docs/latest/server_admin/#core-concepts-and-terms) to create a self-contained space where you manage users, roles, and authentication settings for one or more client applications. The `master` realm is created by default and is used to manage other realms. A realm contains one or more clients representing applications, e.g., a Next.js application deployed on a specific domain. + +Return to your [Koyeb Apps list](https://app.koyeb.com/) and click on the hostname underneath your Keycloak service to visit the Keycloak welcome page. Click the link to access the **Administration Console** and log in using the username and password you set using `KEYCLOAK_ADMIN` and `KEYCLOAK_ADMIN_PASSWORD` when first deploying Keycloak. + +The `master` realm will be selected by default. Use the dropdown in the top left corner to click the **Create realm** button. Upload the example [quickstart.json](https://github.com/neondatabase/keycloak-example/blob/main/realms/quickstart.json) realm file from the [neondatabase/keycloak-example](https://github.com/neondatabase/keycloak-example) repository to create a realm. + +Once the realm is created you can see a sample user named `alice`, and a client named `nextjs-application` within the **Users** and **Clients** sections. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/kc-users-1024x573-d1a821ac.png) + +By querying some tables, you can confirm that Keycloak is using your Neon Postgres database. For example: + +1. Visit the **SQL Editor** in your project on the [Neon console](https://console.neon.tech/). +2. Run the query `SELECT * from user_entity;`. + +The user named `alice` defined in the sample realm JSON should be listed in the output. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/user-alice-in-output-1024x553-5fcd469d.png) + +## Connecting a Next.js Application to Keycloak + +The repository associated with this blog post at [neondatabase/keycloak-example](https://github.com/neondatabase/keycloak-example) includes a Next.js application that uses [NextAuth.js](https://next-auth.js.org/) to integrate with Keycloak to perform user authentication. + +Clone the repository, install the dependencies, and create a `.env.local` file: + +```bash +git clone git@github.com:neondatabase/keycloak-example.git + +cd keycloak-example/next-auth-example + +cp .env.local.example .env.local + +npm i +``` + +Edit the `.env.local` file so that the hostname of the `KEYCLOAK_ISSUER` points to your Keycloak instance, and generate an `AUTH_SECRET` using the instructions in the file. Start the application using the `npm run dev` command, and visit [https://localhost:3000](https://localhost:3000) to interact with it. + +Click the **Sign In** button in the top right corner to initiate the OAuth-based login flow using Keycloak. You can sign in using the username `alice` and password `alice`. Once logged in you can visit the different pages in the application and view the user’s session data. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/auth-setup-with-neon-keycloak-and-koyeb/next-auth-logged-in-1-1024x573-2f11c6d0.png) + +## Conclusion and Next Steps + +Now you know how to use Neon as your Postgres storage provider for Keycloak. Consider crafting an [optimized Keycloak container](https://www.keycloak.org/server/containers#_creating_a_customized_and_optimized_container_image) for production deployment to decrease startup time, and refer to the [Keycloak database configuration guide](https://www.keycloak.org/server/db) for information on connection pooling and other database configuration properties. You can also experiment with Neon’s branching and [point-in-time restore](https://neon.tech/docs/guides/branching-pitr) capability to discover how to restore Keycloak data to an earlier version if needed. diff --git a/content/blog/posts/auth-that-just-works-in-vercel-previews.md b/content/blog/posts/auth-that-just-works-in-vercel-previews.md new file mode 100644 index 0000000000..b4ffb8e58d --- /dev/null +++ b/content/blog/posts/auth-that-just-works-in-vercel-previews.md @@ -0,0 +1,134 @@ +--- +title: Auth That Just Works in Vercel Previews +description: Our Vercel integration now automatically provisions auth for every preview +excerpt: >- + If you’re integrating your Vercel preview deployments with Neon, you’re + already getting an isolated Postgres branch per preview. We just extended the + same automation to authentication. When you use Neon through the Vercel + integration and enable Auth, every Vercel deployment gets... +date: "2026-01-19T17:43:45" +updatedOn: "2026-01-19T18:27:32" +category: product +categories: + - product + - workflows +authors: + - shridhar-deshmukh +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/auth-that-just-works-in-vercel-previews/cover.jpg + alt: null +isFeatured: true +seo: + title: Auth That Just Works in Vercel Previews - Neon + description: Our Vercel integration now automatically provisions auth for every preview + keywords: [] + noindex: false + ogTitle: Auth That Just Works in Vercel Previews - Neon + ogDescription: >- + If you’re integrating your Vercel preview deployments with Neon, you’re + already getting an isolated Postgres branch per preview. We just extended + the same automation to authentication. When you use Neon through the Vercel + integration and enable Auth, every Vercel deployment gets its own isolated + authentication endpoint, with URLs injected automatically and trusted + domains already configured […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/auth-that-just-works-in-vercel-previews/social.jpg +--- + +**If you’re** [integrating your Vercel preview deployments with Neon](https://neon.com/docs/guides/vercel-overview)**, you’re already getting an isolated Postgres branch per preview. We just extended the same automation to authentication**. When you use Neon through the Vercel integration and enable [Auth](https://neon.com/docs/auth/overview), every Vercel deployment gets its own isolated authentication endpoint, with URLs injected automatically and trusted domains already configured for you. Just check a box, push a branch, and auth works: + +Post image +## A quick refresher on the Neon <> Vercel integrations + +[Neon](https://neon.com/) supports two different ways to integrate with Vercel, depending on how you prefer to manage ownership and billing: + +- [Vercel-managed integration (Vercel Marketplace):](https://neon.com/docs/guides/vercel-managed-integration) You add Neon directly to your Vercel project via the Vercel Marketplace. Vercel manages the connection, and Neon resources are provisioned automatically as part of your Vercel workflow. +- [Neon-managed integration (Connected Accounts):](https://neon.com/docs/guides/neon-managed-vercel-integration) You connect an existing Neon project to Vercel from the Neon side. This is often used by teams that want to keep billing and project management in Neon, while still enabling features like branch-per-preview deployments. + +Both integrations support the same core workflow: + +- In Vercel, every commit creates a deployment, with preview deployments designed to be safe to test in isolation. +- When you connect a Vercel project to Neon via either integration and enable branching for previews, + - Every Vercel preview deployment gets its own Neon database branch + - This branch is a copy of the production database, but isolated. This gives you a realistic preview to work against without manual migrations, schema changes, or data mocks + - These branches are very affordable: they incur no additional storage charges, and you only pay for compute while they’re running + - When a preview deployment is removed, the corresponding database branch is cleaned up automatically. + +The Auth update we’re discussing in this blog post applies to both integrations, but more about that in a minute. + +## What is Neon Auth? + +Before we dive into how the Neon <> Vercel integration now works with auth, it’s worth a quick step back. You might already be using Neon purely as a database through Vercel, and not be familiar with Neon Auth yet. + +[Neon Auth](https://neon.com/docs/auth/overview) is a fully managed authentication layer built directly into Neon. It’s [powered by Better Auth under the hood](https://neon.com/blog/neon-auth-branchable-identity-in-your-database), but deeply integrated into Neon’s platform so authentication behaves like a first-class part of your database, not a separate service you have to provision, sync, and maintain. + +
+ +All authentication data lives directly in your Neon database. Users, sessions, roles, organizations, and configuration are stored in the `neon_auth.*` schema, which makes Auth fully compatible with Neon’s branching model. When you enable Neon Auth on a project, every Neon branch gets its own auth endpoint: + +- Each branch has its own Auth URL +- Auth data (users, sessions, OAuth config, JWKS) is scoped to that branch +- Branching your database also branches your entire authentication state
+ +**So when you create a Neon branch, you’re not just cloning schema and data now – you’re also cloning identity.** On its own, [this already makes it much easier to test real authentication flows safely](https://neon.com/blog/stop-mocking-auth-its-breaking-your-tests) – but when Neon Auth is wired into Vercel previews, that model becomes automatic, which is where the integration update comes in next. + +## What’s new: Neon Auth now follows your Vercel deployments + +**With this update, the Neon <> Vercel integration links database branching and auth provisioning into a single, automatic workflow.** When you enable the Auth toggle while connecting Neon to your Vercel project, authentication becomes part of the deployment lifecycle, just like previews and databases already are: + +
+ +### How it works + +
Post image
+ +When your Vercel project is connected to Neon with Auth enabled, + +1. Neon provisions Auth on your production branch +2. On the next deployment, + 1. `NEON_AUTH_BASE_URL` is injected automatically into your Vercel environment variables + 2. Your production deployment URL is added as a trusted origin + 3. Any custom production domains are also added as trusted origins +3. When you push a commit and Vercel creates a preview deployment, the full flow is triggered: + 1. Vercel creates the preview deployment + 2. Neon creates a matching database branch + 3. Neon Auth provisions a dedicated auth instance for that branch + 4. `NEON_AUTH_BASE_URL` is injected into that preview deployment + 5. The preview URL plus any matching custom preview domains are added as trusted origins +4. Cleanup follows the same rules: when a preview deployment is removed, + 1. Vercel deletes the deployment + 2. Neon deletes the corresponding database branch + 3. The associated Auth instance is removed automatically + +### Using it in your code + +Nothing changes in your application code. The integration injects the correct environment variables for each deployment automatically. + +```typescript +import { createAuthClient } from "better-auth/client"; + +const auth = createAuthClient({ + baseURL: process.env.NEON_AUTH_BASE_URL, +}); + +// Real authentication, scoped to the current deployment +await auth.signIn.email({ email, password }); +``` + +### What this unlocks: testing auth changes safely + +This integration between previews, databases, and auth enables workflows that are usually quite awkward. For example, imagine you’re adding a new OAuth provider or changing part of your login flow – you can now: + +1. Push a commit to a feature branch +2. Vercel builds a preview deployment +3. You open the preview URL, authentication already works +4. In the Neon Console, you configure the new OAuth provider for the preview branch +5. You can test your new flow end to end: initiation, callback, token exchange, session creation. +6. When you’re happy with the changes, + 1. Merge the code + 2. And apply the same auth configuration to the production branch + +## Try it now + +To try it, [connect your Vercel project to Neon](https://neon.com/docs/guides/vercel-overview), enable Auth, and start building. Preview deployments will come up with fully working authentication automatically. [Here’s a complete guide for you to follow](https://neon.com/guides/vercel-neon-auth-branching), and if you have any questions, reach out to us on [Discord](https://discord.gg/92vNTzKDGp). diff --git a/content/blog/posts/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth.md b/content/blog/posts/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth.md new file mode 100644 index 0000000000..cf7c805b49 --- /dev/null +++ b/content/blog/posts/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth.md @@ -0,0 +1,675 @@ +--- +title: Authenticating users in Astro using Neon Postgres and Lucia Auth +description: >- + Step-by-step guide to building user authentication in Astro application with + Lucia Auth and Postgres powered by Neon +excerpt: >- + This guide covers the step-by-step process of building user authentication + APIs and HTML pages in Astro application with Lucia Auth and Postgres, powered + by Neon. User authentication provides a way to manage user identities and + access control in your application. Upon completing... +date: '2024-04-12T19:47:52' +updatedOn: '2024-04-12T19:56:47' +category: community +categories: + - community +authors: + - rishi-raj-jain +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth/cover.png + alt: null +isFeatured: false +seo: + title: Authenticating users in Astro using Neon Postgres and Lucia Auth - Neon + description: >- + Step-by-step guide to building user authentication in Astro application with + Lucia Auth and Postgres powered by Neon + keywords: [] + noindex: false + ogTitle: Authenticating users in Astro using Neon Postgres and Lucia Auth - Neon + ogDescription: >- + This guide covers the step-by-step process of building user authentication + APIs and HTML pages in Astro application with Lucia Auth and Postgres, + powered by Neon. User authentication provides a way to manage user + identities and access control in your application. Upon completing the + guide, you would have an understanding of how to perform user authentication + […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth/image-16-1024x576-08142efa.png) + +This guide covers the step-by-step process of building user authentication APIs and HTML pages in Astro application with Lucia Auth and Postgres, powered by Neon. User authentication provides a way to manage user identities and access control in your application. Upon completing the guide, you would have an understanding of how to perform user authentication using Lucia Auth and protect a page from unauthorized access. + +## Prerequisites + +To follow along this guide, you will need the following: + +- [Node.js 18](https://nodejs.org/en) or later +- A [Neon](https://console.neon.tech/signup) account +- A [Vercel](https://vercel.com/dashboard) account + +## Steps + +- [Provisioning a Serverless Postgres powered by Neon](#provisioning-a-serverless-postgres-powered-by-neon) +- [Create a new Astro application](#create-a-new-astro-application) +- [Add Tailwind CSS to the application](#add-tailwind-css-to-the-application) +- [Enabling Server Side Rendering in Astro with Vercel](#enabling-server-side-rendering-in-astro-with-vercel) +- [Setting up a Postgres Database Connection and Schema](#setting-up-a-postgres-database-connection-and-schema) +- [Setup Lucia Auth with Neon Postgres](#setup-lucia-auth-with-neon-postgres) +- [Define the Astro application routes](#define-the-astro-application-routes) +- [Build the User Authentication Routes](#build-the-user-authentication-routes) +- [Deploy To Vercel](#deploy-to-vercel) + +## Provisioning a Serverless Postgres powered by Neon + +Using Serverless Postgres database powered by Neon helps you scale down to zero. With Neon, you only have to pay for what you use. + +To get started, go to the [Neon console](https://console.neon.tech/app/projects) and enter the name of your choice as the project name. You can pick a region near where you will deploy your Astro application. By default, version 16 of Postgres is used. Finally, click on **Create Project** to create the Postgres database named `neondb` (by default). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth/image-12-1024x555-864791b2.png) + +You will then be presented with a dialog that provides a connecting string of your database. Click on **Pooled connection** on the top right of the dialog and the connecting string automatically updates in the box below it. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/authenticating-users-in-astro-using-neon-postgres-and-lucia-auth/image-13-1024x556-631daa2d.png) + +All Neon connection strings have the following format: + +```bash +postgres://:@.neon.tech:/?sslmode=require +``` + +- `user` is the database user. +- `password` is the database user’s password. +- `endpoint_hostname` is the host with neon.tech as the [TLD](https://www.cloudflare.com/en-gb/learning/dns/top-level-domain/). +- `port` is the Neon port number. The default port number is 5432. +- `dbname` is the name of the database. “neondb” is the default database created with each Neon project. +- `?sslmode=require` an optional query parameter that enforces the [SSL](https://www.cloudflare.com/en-gb/learning/ssl/what-is-ssl/) mode while connecting to the Postgres instance for better security. + +Save this connecting string somewhere safe to be used as the `POSTGRES_URL` further in the guide. Proceed further in this guide to create a Astro application. + +## Create a new Astro application + +Let’s get started by creating a new Astro project. Open your terminal and run the following command: + +```bash +npm create astro@latest authenticating-users-neon-lucia +``` + +`npm create astro` is the recommended way to scaffold an Astro project quickly. + +When prompted, choose: + +- `Empty` when prompted on how to start the new project. +- `Yes` when prompted if plan to write Typescript. +- `Strict` when prompted how strict Typescript should be. +- `Yes` when prompted to install dependencies. +- `Yes` when prompted to initialize a git repository. + +Once that’s done, you can move into the project directory and start the app: + +```bash +cd authenticating-users-neon-lucia +npm run dev +``` + +The app should be running on [localhost:4321](https://localhost:4321/). + +Next, in your first terminal window, run the command below to install the necessary libraries and packages for building the application: + +```bash +npm install pg +npm install -D dotenv tsx @types/pg +``` + +The above command installs the packages passed to the install command, with the -D flag specifying the libraries intended for development purposes only. + +The libraries installed include: + +- `pg`: A PostgreSQL client for Node.js. + +The development-specific libraries include: + +- `@types/pg`: Type definitions for pg. +- `tsx`: To execute and rebuild TypeScript efficiently. +- `dotenv`: A library for handling environment variables. + +Then, make the following additions in your `astro.config.mjs` file to populate the environment variables and make them accessible via `process.env` object as well: + +```javascript +// File: astro.config.mjs + ++ import dotenv from 'dotenv'; +import { defineConfig } from 'astro/config'; + ++ dotenv.config(); + +// https://astro.build/config +export default defineConfig({}); +``` + +Then, make the following additions in your `tsconfig.json` file to make relative imports within the project easier: + +```javascript +{ + "extends": "astro/tsconfigs/base", ++ "compilerOptions": { ++ "baseUrl": ".", ++ "paths": { ++ "@/*": ["src/*"] ++ } ++ } +} +``` + +## Add Tailwind CSS to the application + +For styling the app, you will be using Tailwind CSS. Install and set up Tailwind at the root of our project’s directory by running: + +```bash +npx astro add tailwind +``` + +When prompted, choose: + +- `Yes` when prompted to install the Tailwind dependencies. +- `Yes` when prompted to generate a minimal `tailwind.config.mjs` file. +- `Yes` when prompted to make changes to Astro configuration file. + +With choices as above, the command finishes integrating TailwindCSS into your Astro project. It installed the following dependency: + +- `tailwindcss`: TailwindCSS as a package to scan your project files to generate corresponding styles. +- `@astrojs/tailwind`: The adapter that brings Tailwind’s utility CSS classes to every `.astro` file and framework component in your project. + +## Enabling Server Side Rendering in Astro with Vercel + +To authenticate users using server-side APIs, you’re going to enable server-side rendering in your Astro application. Execute the following command in your terminal: + +```bash +npx astro add vercel +``` + +When prompted, choose: + +- `Yes` when prompted to install the Vercel dependencies. +- `Yes` when promted to make changes to Astro configuration file. + +With choices as above, the command finishes integrating Vercel adapter into your Astro project. It installed the following dependency: + +- `@astrojs/vercel`: The adapter that allows you to deploy server-side rendered Astro application to Vercel. + +## Setting up a Postgres Database Connection and Schema + +In this section, you’ll learn how to configure a secure connection to the Postgres database, create a client to interact with it, and populate the tables in the database. + +### Set up the database connection + +Create an `.env` file in the root directory of your project with the following enviroment variable to initiate the setup of a database connection: + +```bash +# Neon Postgres Pooled Connection URL +POSTGRES_URL="postgres://:@.neon.tech:/?sslmode=require" +``` + +The file, `.env` should be kept secret and not included in Git history. Ensure that `.env` is added to the `.gitignore` file in your project. + +### Create the database client + +First, create a `postgres` directory in the `src` directory by running the following command: + +```bash +mkdir src/postgres +``` + +Then, to create a client that interacts with your serverless postgres, create a `setup.ts` file inside the `src/postgres` directory with the following code: + +```bash +// File: src/postgres/setup.ts + +// Load the environment variables +import 'dotenv/config' + +// Load the postgres module +import pg from 'pg' + +// Create a connection string to the postgres instance powered by Neon +const connectionString: string = `${process.env.POSTGRES_URL}` + +// Create a in-memory pool so that it's cached for multiple calls +export default new pg.Pool({ connectionString }) +``` + +The code imports the `dotenv` configuration, making sure that all the environment variables in the `.env` file are present in the runtime. Then, the code imports the `pg` library, retrieves the database URL from the environment variables, and uses it to create a new pool instance, which is subsequently exported. + +### Create the database schema + +In the `postgres` directory, create a file named `schema.ts` with the following code which will allow you to create and populate the user and sessions database tables for authentication. + +```javascript +// File: src/postgres/schema.ts + +import pool from './setup' + +async function createSchema() { + // Create the user table if it does not exist + await pool.query('CREATE TABLE IF NOT EXISTS auth_user ( id TEXT PRIMARY KEY, username TEXT UNIQUE, hashed_password TEXT );') + // Create the user_session table if it does not exist + await pool.query('CREATE TABLE IF NOT EXISTS user_session ( id TEXT PRIMARY KEY, expires_at TIMESTAMPTZ NOT NULL, user_id TEXT NOT NULL REFERENCES auth_user(id) );') + console.log('Finished setting up the database.') + // Drain the pool of all active clients, disconnect them, and shut down any internal timers in the pool. + await pool.end() +} + +createSchema() +``` + +The code above defines how data will be stored, organized and managed in the database. Using the `pool` database instance, it executes an SQL query to create a `auth_user` table within the database if it does not already exist. This table comprises of three columns: + +- An `id` column for storing random identifiers for each user in the table. +- A `username` column for storing unique identifiers for each user in the table. +- A `hashed_password` column for storing Argon2id hashed passwords for each user in the table. + +A subsequent SQL query creates a `user_session` table within the database if it does not already exist. This table comprises three columns: + +- An `id` column for storing random identifiers for each session in the table. +- An `expires_at` column for storing the timestamp with time zone for each session in the table. +- An `user_id` column for the associated `id` of each associated user for each session in the table. + +After executing the two SQL queries, a message is printed to the console if there’s an error during the execution. + +Finally, to execute the code in the schema file, make the following addition in the `scripts` of your `package.json` file: + +```bash +{ + // ... + "scripts": { + // ... ++ "db:schema": "tsx src/postgres/schema.ts" + // ... + } + // ... +} +``` + +### Test the database setup locally + +To execute the code within `schema.ts` to set up the database, run the following command in your terminal window: + +```bash +npm run db:schema +``` + +If the command is executed successfully, you will see no logs in your terminal window except `Finished setting up the database.`, marking the completion of the schema setup in your Postgres Database powered by Neon. + +## Setup Lucia Auth with Neon Postgres + +To start authenticating users and managing their sessions, install [Lucia](https://lucia-auth.com/) and [Oslo](https://oslo.js.org/), for various auth utilities by executing the following command in your terminal: + +```bash +npm install lucia oslo @lucia-auth/adapter-postgresql @neondatabase/serverless +``` + +The above command installs the packages passed to the install command. The libraries installed include: + +- `lucia`: An open source auth library that abstracts away the complexity of handling sessions. +- `oslo`: A collection of auth-related utilities. +- `@lucia-auth/adapter-postgresql`: PostgreSQL adapter for Lucia. +- `@neondatabase/serverless`: Neon’s PostgreSQL driver for JavaScript and TypeScript. + +Then, create a `lucia` directory in the `src` directory by running the following command: + +```bash +mkdir src/lucia +``` + +Then, create a file `index.ts` inside the `lucia` directory with the following code: + +```javascript +// File: src/lucia/index.ts + +import { Lucia } from 'lucia' +import { neon } from '@neondatabase/serverless' +import { NeonHTTPAdapter } from '@lucia-auth/adapter-postgresql' + +const sql = neon(import.meta.env.POSTGRES_URL) + +const adapter = new NeonHTTPAdapter(sql, { + user: 'auth_user', + session: 'user_session', +}) + +export const lucia = new Lucia(adapter, { + sessionCookie: { + attributes: { + secure: import.meta.env.PROD, + }, + }, + getUserAttributes: (attributes) => { + return { + id: attributes.id, + username: attributes.username, + } + }, +}) + +interface DatabaseUserAttributes { + id: string + username: string +} + +declare module 'lucia' { + interface Register { + Lucia: typeof lucia + DatabaseUserAttributes: DatabaseUserAttributes + } +} +``` + +The code above does the following: + +- Imports the Lucia class, Neon HTTP serverless driver and Lucia’s PostgreSQL adapter. +- Creates a one-shot SQL query function compatible with Neon. +- Creates a Lucia adapter with the tables `auth_user` and `user_session`. +- Creates a Lucia instance that uses cookies to maintain user sessions. The `auth_session` cookie set by Lucia, is set to be `secure` if your application is running in production mode (detected by [PROD](https://vitejs.dev/guide/env-and-mode#:~:text=config%20option.-,import.meta.env.PROD,-%3A%20%7Bboolean%7D%20whether%20the) Vite environment variable). Using `getUserAttributes` property, the Lucia instance is informed of the attributes that need to be fetched whenever a user information is requested. +- Defines types related to the user information and the Lucia instance. + +To fetch and validate the current user session, and to verify if the users’ credentials are valid while they’re signing up, create a file `user.ts` with the following code: + +```javascript +// File: src/lucia/user.ts + +import { lucia } from '.' +import type { User } from 'lucia' +import type { AstroCookies } from 'astro' + +export function getSessionID(cookies: AstroCookies): string | null { + const auth_session = cookies.get('auth_session') + if (!auth_session) return null + return lucia.readSessionCookie(`auth_session=${auth_session.value}`) +} + +export async function getUser(cookies: AstroCookies): Promise { + const { user } = await lucia.validateSession(getSessionID(cookies)) + return user +} + +export function validateCredentials(username, password): string | null { + // username must be between 4 ~ 31 characters, and only consists of lowercase letters, 0-9, -, and _ + // keep in mind some database (e.g. mysql) are case insensitive + if (typeof username !== 'string' || username.length < 3 || username.length > 31 || !/^[a-z0-9_-]+$/.test(username)) { + return 'Invalid username' + } + if (typeof password !== 'string' || password.length < 6 || password.length > 255) { + return 'Invalid password' + } + return null +} +``` + +The code above begins with the importing the lucia instance we created earlier and the types of user’s information and cookies (an Astro internal utility for cookies). Then, it exports the three function as follows: + +- `getSessionID`: it accepts all the cookies received in a request, decode the `auth_session` cookie from it, and calls `readSessionCookie` function by lucia. It returns the `session_id` associated with the particular session. +- `getUser`: it calls the `validateSession` function by lucia that returns the user and session information associated with the given cookies. The user information returned by this function contains only the attributes defined in the `getUserAttributes` we created earlier. +- `validateCredentials`: it accepts in a username and password and returns a particular message if either of them didn’t meet the validity requirements. + +## Define the Astro application routes + +With Astro, creating a `.astro` or `.(js|ts)` file in the `src/pages` directory maps it to a route in your application. The name of the file created maps to the route’s URL pathname (with the exception of `index.(astro|ts|js)`, which is the index route). + +The structure below is what our `pages` folder will look like at the end of this section: + +``` +├── signin.astro +├── signup.astro +├── protected.astro +├── api/ +├──── sign/ +└────── in.ts +└────── up.ts +└────── out.ts +``` + +- `protected.astro` will serve responses with dynamically created HTML to incoming requests at localhost:4321/protected. +- `signin.astro` will serve responses with statically generated HTML to incoming requests at localhost:4321/signin. +- `signup.astro` will serve responses with statically generated HTML to incoming requests at localhost:4321/signup. +- `api/sign/in.ts` will serve responses as an API Endpoint to incoming requests at localhost:4321/api/sign/in. +- `api/sign/up.ts` will serve responses as an API Endpoint to incoming requests at localhost:4321/api/sign/up. +- `api/sign/out.ts` will serve responses as an API Endpoint to incoming requests at localhost:4321/api/sign/out. + +## Build the User Authentication Routes + +For minimal user authentication, a user of your application should be able to sign up, sign in and sign out to your application at a given time. In this section, you’ll build the frontend pages for sign in and sign up and the API routes (in, up and out) that’ll process the user authentication logic for the same. + +### Build the Sign Up HTML and API Route + +Create a file `signup.astro` in the `src/pages` directory with the following Astro code to serve incoming requests to `/signup`: + +```javascript +--- +// File: src/pages/signup.astro + +export const prerender = true +--- + + + + + + + +

Sign Up

+
+ + + + + +
+ + +``` + +The above code does the following: + +- Exports a `prerender` flag set to (boolean) `true` indicating the page to be statically generated at the build time. +- Serves an containing a form for users to enter their username and password in. +- Passes the form data to `/api/sign/up` API Endpoint when user submits their information. + +Let’s create the endpoint for users to sign up with. Create a file `api/sign/up.ts` inside the `src/pages` directory with the following code: + +```javascript +// File: src/pages/api/sign/up.ts + +import pool from '@/postgres/setup' +import { generateId } from 'lucia' +import { lucia } from '@/lucia/index' +import type { APIContext } from 'astro' +import { Argon2id } from 'oslo/password' +import { validateCredentials } from '@/lucia/user' + +export async function POST(context: APIContext): Promise { + const formData = await context.request.formData() + const username = formData.get('username') as string + const password = formData.get('password') as string + const message = validateCredentials(username, password) + if (message) return new Response(message, { status: 400 }) + const user_id = generateId(15) + const hashed_password = await new Argon2id().hash(password) + await pool.query({ + text: 'INSERT INTO auth_user (id, username, hashed_password) VALUES ($1, $2, $3)', + values: [user_id, username, hashed_password], + }) + const session = await lucia.createSession(user_id, {}) + const sessionCookie = lucia.createSessionCookie(session.id) + context.cookies.set(sessionCookie.name, sessionCookie.value, sessionCookie.attributes) + return context.redirect('/protected') +} +``` + +The above code does the following for incoming requests to `/api/sign/up`: + +- Imports the Postgres pool, the initialized lucia instance, `generateId`, `Argon2id` and `validateCredentials` functions. +- Exports a `POST` function indiciating that the API route would only process the incoming POST requests. +- Parses the form data in the request. +- Extracts username and password from the form data. Validates them using the `validateCredentials` function created earlier. +- Creates a hashed password using Argon2id. +- Attempts to create the user in `auth_user` table using the pg client. As `username` is a unique field in the database, any conflicts will result in an error response from the endpoint. +- Creates a user session using `createSession` helper by Lucia. +- Sets the cookie pertaining to the user session and redirect to the `/protected` page. + +### Build the Sign In HTML and API Route + +Create a file `signin.astro` in the `src/pages` directory with the following Astro code to serve incoming requests to `/signin`: + +```javascript +--- +// File: src/pages/signin.astro + +export const prerender = true +--- + + + + + + + +

Sign In

+
+ + + + + +
+ + +``` + +The above code does the following: + +- Exports a `prerender` flag set to (boolean) `true` indicating the page to be statically generated at the build time. +- Serves an containing a form for users to enter their username and password in. +- Passes the form data to `/api/sign/in` API Endpoint when user submits their information. + +Let’s create the endpoint for users to sign in with. Create a file `api/sign/in.ts` inside the `src/pages` directory with the following code: + +```javascript +// File: src/pages/api/sign/in.ts + +import pool from '@/postgres/setup' +import { lucia } from '@/lucia/index' +import type { APIContext } from 'astro' +import { Argon2id } from 'oslo/password' +import { validateCredentials } from '@/lucia/user' + +export async function POST(context: APIContext): Promise { + const formData = await context.request.formData() + const username = formData.get('username') as string + const password = formData.get('password') as string + const message = validateCredentials(username, password) + if (message) return new Response(message, { status: 400 }) + const existingUser = await pool.query({ + text: 'SELECT id as user_id, username, hashed_password FROM auth_user WHERE username = $1', + values: [username.toLowerCase()], + }) + if (!existingUser || existingUser.rowCount < 1) return new Response('Incorrect email or password', { status: 400 }) + const validPassword = await new Argon2id().verify(existingUser.rows[0].hashed_password, password) + if (!validPassword) return new Response('Incorrect email or password', { status: 400 }) + const session = await lucia.createSession(existingUser.rows[0].user_id, {}) + const sessionCookie = lucia.createSessionCookie(session.id) + context.cookies.set(sessionCookie.name, sessionCookie.value, sessionCookie.attributes) + return context.redirect('/protected') +} +``` + +The above code does the following for incoming requests to `/api/sign/in`: + +- Imports the Postgres pool, the initialized lucia instance, `Argon2id` and `validateCredentials` functions. +- Exports a `POST` function indiciating that the API route would only process the incoming POST requests. +- Parses the form data in the request. +- Extracts username and password from the form data. Validates them using the `validateCredentials` function created earlier. +- Checks for an existing user with the given username. If not found, sends a response with 400 Bad Request status. +- Otherwise, verifies the hashed password in the Neon Postgres with the password user entered. If they don’t match, sends a response with 400 Bad Request status. +- Otherwise, creates a user session using `createSession` helper by Lucia. +- Sets the cookie pertaining to the user session and redirect to the `/protected` page. + +### Build the Sign Out API Route + +Let’s create the endpoint for users to sign out with. Create a file `api/sign/out.ts` inside the `src/pages` directory with the following code: + +```javascript +// File: src/pages/api/sign/out.ts + +import { lucia } from '@/lucia/index' +import type { APIContext } from 'astro' +import { getSessionID } from '@/lucia/user' + +export async function GET({ cookies, redirect }: APIContext): Promise { + await lucia.invalidateSession(getSessionID(cookies)) + const sessionCookie = lucia.createBlankSessionCookie() + cookies.set(sessionCookie.name, sessionCookie.value, sessionCookie.attributes) + return redirect('/protected') +} +``` + +The above code does the following for incoming requests to `/api/sign/out`: + +- Imports the initialized lucia instance, and `getSessionID` function. +- Exports a `GET` function indiciating that the API route would only process the incoming GET requests. +- Invalidates the current user session using the `invalidateSession` helper by Lucia. +- Creates an empty user session cookie using `createBlankSessionCookie` helper by Lucia. +- Sets the blank user session cookie. +- Redirects to the `/protected` page. + +### Build the Protected HTML Page + +Create a file `protected.astro` in the `src/pages` directory with the following Astro code to serve incoming requests to `/protected`: + +```javascript +--- +// File: src/pages/protected.astro + +import { getUser } from '@/lucia/user' + +const user = await getUser(Astro.cookies) + +if (!user) return new Response(null, { status: 403 }) +--- + + + + + + + + Protected Content for {user.id} {user.username} + + +``` + +The code above does the following to the incoming requests at `/protected`: + +- Imports the `getUser` function (created earlier) +- Using `Astro.cookies`, verifies if there is an authenticated user with the current request. If not found, returns a 403 Forbidden status. +- Otherwise, it serves the protected HTML. + +Great! Now you’re able to authenticate users and protected specific rouets in your application using API Endpoints in Astro and Lucia Auth. + +## Deploy to Vercel + +The code is now ready to deploy to Vercel. Use the following steps to deploy: + +- Start by creating a GitHub repository containing your app’s code. +- Then, navigate to the Vercel Dashboard and create a **New Project**. +- Link the new project to the GitHub repository you just created. +- In **Settings**, update the **Environment Variables** to match those in your local `.env` file. +- Deploy! + +## Summary & Final Thoughts + +In this guide, you learned how to authenticate users in your Astro application using Lucia Auth and Serverless Postgres Database powered by Neon. Further, you learned how to create protected routes that are forbidden for un-authenticated users. + +For more, join us on our [Discord server](https://neon.tech/discord) to share your experiences, suggestions, and challenges. diff --git a/content/blog/posts/autogenerating-mcp-servers-openai-schemas.md b/content/blog/posts/autogenerating-mcp-servers-openai-schemas.md new file mode 100644 index 0000000000..826a6ff59b --- /dev/null +++ b/content/blog/posts/autogenerating-mcp-servers-openai-schemas.md @@ -0,0 +1,79 @@ +--- +title: 'Auto-generating MCP Servers from OpenAPI Schemas: Yay or Nay?' +description: 'OpenAPI to MCP: Shortcut or Short-Sighted?' +excerpt: >- + Should we be generating Model Context Protocol (MCP) servers directly from + existing API specs? MCP is designed to let AI agents like those in Claude, + Cursor, and Windsurf interact with tools and APIs. So, can we just treat our + existing REST APIs as the interface for LLMs and enti... +date: '2025-05-01T19:07:18' +updatedOn: '2025-10-17T17:03:50' +category: engineering +categories: + - engineering + - ai +authors: + - david-gomes +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/autogenerating-mcp-servers-openai-schemas/cover.jpg + alt: null +isFeatured: true +seo: + title: 'Auto-generating MCP Servers from OpenAPI Schemas: Yay or Nay? - Neon' + description: >- + Should we be generating Model Context Protocol (MCP) servers directly from + existing API specs? Here's our experience. + keywords: [] + noindex: false + ogTitle: 'Auto-generating MCP Servers from OpenAPI Schemas: Yay or Nay? - Neon' + ogDescription: >- + Should we be generating Model Context Protocol (MCP) servers directly from + existing API specs? Here's our experience. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/autogenerating-mcp-servers-openai-schemas/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/autogenerating-mcp-servers-openai-schemas/neon-mcp-1-1024x576-a199dba0.jpg) + +Should we be generating Model Context Protocol (MCP) servers directly from existing API specs? MCP is designed to let AI agents like those in Claude, Cursor, and Windsurf interact with tools and APIs. So, can we just treat our existing REST APIs as the interface for LLMs and entirely codegen an MCP server? + +## Generating MCP from REST + +The idea is certainly appealing. You’ve already invested time in building and documenting your REST API with an OpenAPI schema, why not transform that into an MCP server? + +The thinking goes: if you create a 1-to-1 mapping, surely the MCP server will be comprehensive? Several projects have come out to help bridge the gap between APIs and LLMs. Tools like _openapisearch,_ for instance, allow agents to explore and understand existing APIs based on their OpenAPI specs. You could ask, “How do I create a repository using the GitHub API?”, and it would first find out which OpenAPI identifier it needs, get a summary of it in simple natural language, then find and return the appropriate endpoint allowing you to sift through the entire API using conversational queries. + +There are also services like Tadata which offer a more direct approach, and discover all FastAPI endpoints then expose them automatically as MCP tools. + +## The Pitfalls + + + +Cramer’s core argument hits on several fundamental mismatches. + +First, LLMs struggle when faced with a large amount of choices, which is quite common when the typical enterprise REST API might expose dozens or hundreds of distinct operations via its OpenAPI spec. The GitHub API, for example, contains over 600! Simply dumping all of these into an MCP server as individual tools creates an overwhelming decision space for the LLM. + +Even with a small set of _slightly_ similar tools, models often fail to select the right one without very specific instructions. At best, this is a very frustrating user experience where the MCP calls continuously fail or don’t work as expected. At worst, this can be potentially catastrophic. When you’re dealing with critical infrastructure like a database, getting the wrong tool call is no longer just an inconvenience. + +Next, there’s the structure of the interaction itself. LLMs aren’t particularly great at navigating complex JSON payloads or figuring out which combination of optional parameters in an API call (like a generic _create_resource_ endpoint) is appropriate for a given task. It’s not interpretable for an LLM trying to understand the outcome and decide the next step, because it was never meant to be. You’re essentially asking the LLM to deal with formats and ambiguity it’s not well-suited for. + +Finally, and perhaps most importantly, there’s a fundamental mismatch in the goal of a REST API and an MCP Server. REST API’s are typically built around resource-centric, low-level operations: create a user, fetch a record, update a field. MCP Servers operate on the level of tasks and workflows: “Onboard this new customer,” “Migrate the database schema,” “Summarize recent activity”. Simply mapping every granular API endpoint to an MCP tool forces the LLM to act like a traditional programmatic client, and this isn’t leveraging the LLM’s strengths. + +Effective MCP design means thinking about the _workflows_ you want the agent to perform and potentially creating higher-level tools that encapsulate multiple API calls, rather than just exposing the raw building blocks of your entire API spec. + +## A Hybrid Approach + +So, if a direct 1:1 approach is problematic, does that mean we ditch the idea entirely and resign ourselves to writing every MCP server tool from scratch? + +Probably not. Manually making every tool definition, input/output schema, and handler function, especially when you already have a well-defined API, is probably a huge time sink. This is where a more hybrid approach comes into play, where the codegen is not the final product, but the starting point. + +I’d recommend looking at the tools for generating an MCP server from OpenAPI specs, then begin aggressively pruning and removing the vast majority of the generated tools, keeping only low-level operations that represent genuinely useful, distinct capabilities that an LLM might perform. Then, rewrite the descriptions of the remaining tools such that instead of just stating _what_ the tool does (like mirroring an API endpoint summary), focus on _when_ and _why_ an LLM should use it. Provide clear examples, outline expected workflows, and give guidance on how to handle common scenarios or parameters. + +Finally, and most importantly, create new higher level tools that abstract away common multi-step workflows, calling several underlying API endpoints under the hood. (Be careful about making your MCP server stateful though!) + +## Wrapping Up + +For [our MCP server](https://github.com/neondatabase-labs/mcp-server-neon), we decided to use our [TypeScript API SDK](https://www.npmjs.com/package/@neondatabase/api-client) in order to make it really easy to replicate a lot of our API endpoints. However, we implemented only tools that we believe make sense for LLMs to use via Cursor/Windsurf, and then we built higher-level workflows that join together multiple endpoints to complete tasks. In fact, one of these higher-level workflows (running and testing database migrations in a streamlined, opinionated manner), was [the highlight of the release of our MCP Server back in December 2024](https://neon.tech/blog/let-claude-manage-your-neon-databases-our-mcp-server-is-here). + +Finally, our MCP server is open-source and we continue to make lots of improvements to it. If you want to get involved, [check it out on GitHub](https://github.com/neondatabase-labs/mcp-server-neon)! diff --git a/content/blog/posts/automate-partial-data-dumps-with-postgresql-and-github-actions.md b/content/blog/posts/automate-partial-data-dumps-with-postgresql-and-github-actions.md new file mode 100644 index 0000000000..cb10109f4f --- /dev/null +++ b/content/blog/posts/automate-partial-data-dumps-with-postgresql-and-github-actions.md @@ -0,0 +1,231 @@ +--- +title: 'Neon Twin: Automate Partial Data Dumps with Postgres and GitHub Actions' +description: >- + Create a Neon Twin to develop safely with production data—without migrating + your database. +excerpt: >- + Your production database is tried and tested and likely bulletproof. Merely + mentioning migrating to a new provider would likely set off multiple alarms, + so don’t, use a Neon Twin instead. In this post, I’ll show you how to create a + partial copy of your production database so you... +date: '2025-02-10T16:31:46' +updatedOn: '2025-02-17T18:50:28' +category: workflows +categories: + - workflows +authors: + - paul-scanlon +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/automate-partial-data-dumps-with-postgresql-and-github-actions/cover.jpg + alt: Automate Partial Data Dumps with PostgreSQL and GitHub Actions +isFeatured: false +seo: + title: >- + Neon Twin: Automate Partial Data Dumps with Postgres and GitHub Actions - + Neon + description: >- + Create a Neon Twin to develop safely with production data—without migrating + your database. + keywords: [] + noindex: false + ogTitle: >- + Neon Twin: Automate Partial Data Dumps with Postgres and GitHub Actions - + Neon + ogDescription: >- + Create a Neon Twin to develop safely with production data—without migrating + your database. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/automate-partial-data-dumps-with-postgresql-and-github-actions/social.jpg +--- + +![Automate Partial Data Dumps with PostgreSQL and GitHub Actions](https://cdn.neonapi.io/public/images/pages/blog/automate-partial-data-dumps-with-postgresql-and-github-actions/neon-partial-data-dumps-cover-e2229704.jpg) + +Your production database is tried and tested and likely bulletproof. Merely mentioning migrating to a new provider would likely set off multiple alarms, so don’t, use a Neon Twin instead. + +In this post, I’ll show you how to create a partial copy of your production database so you and your team can start developing on Neon while leaving production where it is. + +## What is a Neon Twin? + +A [Neon Twin](https://neon.tech/docs/guides/neon-twin-intro) is a copy of your production database, free from the cumbersome, hard-to-manage, and expensive workflows. With a Neon Twin, you can instantly spin up a new branch, develop features, or track down bugs, all from the safety of an isolated environment using production data. Any changes made to a Twin would be migrated back to your production database using your existing workflow or pipeline. + +## How to create a Neon Twin + +In the code below, I’ll show you how to create a scheduled, recurring GitHub Action that performs a partial data dump of a production (or staging) Postgres database and restore it to a Neon Serverless Postgres instance. + +### Creating a GitHub Action + +At the root of your project, create a new directory named `.github`. Within this directory, create another directory named `workflows`, and within this directory, create a new file named `create-neon-twin.yml` e.g. + +``` +.github + |-- workflows + |-- create-neon-twin.yml +``` + +Now add the following code. + +```yaml +name: Create Neon Twin + +on: + schedule: + - cron: '0 0 * * *' # Runs at midnight ET (us-east-1) + workflow_dispatch: + +env: + PROD_DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }} # Production or primary database + DEV_DATABASE_URL: ${{ secrets.DEV_DATABASE_URL }} # Development database + PG_VERSION: '17' + +jobs: + dump-and-restore: + runs-on: ubuntu-latest + + steps: + - name: Install PostgreSQL + run: | + sudo apt update + yes '' | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh + sudo apt install -y postgresql-${{ env.PG_VERSION }} + + - name: Set PostgreSQL binary path + run: echo "POSTGRES=/usr/lib/postgresql/${{ env.PG_VERSION }}/bin" >> $GITHUB_ENV + + - name: Dump schema + run: | + $POSTGRES/pg_dump -Fc --schema-only -f "${{ github.workspace }}/all-schema.bak" "${{ env.PROD_DATABASE_URL }}" + + - name: Dump data + run: | + $POSTGRES/psql "${{ env.PROD_DATABASE_URL }}" -c "\copy (SELECT * FROM users ORDER BY user_id DESC LIMIT 50) TO '${{ github.workspace }}/users-subset.csv' WITH CSV HEADER" + + - name: Drop tables and schema + run: | + $POSTGRES/psql "${{ env.DEV_DATABASE_URL }}" -c "DROP SCHEMA IF EXISTS public CASCADE;" + $POSTGRES/psql "${{ env.DEV_DATABASE_URL }}" -c "CREATE SCHEMA public;" + + - name: Restore schema + run: | + $POSTGRES/pg_restore --clean --no-owner --no-acl --if-exists --schema-only -d "${{ env.DEV_DATABASE_URL }}" "${{ github.workspace }}/all-schema.bak" + + - name: Restore data + run: | + $POSTGRES/psql "${{ env.DEV_DATABASE_URL }}" -c "\copy public.users FROM '${{ github.workspace }}/users-subset.csv' WITH CSV HEADER" +``` + +### The GitHub Action Explained + +At the top of the file is the `name` of the Action (Create Neon Twin), which will be displayed in the GitHub UI. + +Below is the `cron` schedule. It is written using [POSIX cron syntax](https://pubs.opengroup.org/onlinepubs/9699919799/utilities/crontab.html#tag_20_25_07) and is set to run each night at midnight (Eastern Time).
Below that is `workflow_dispatch`, this allows you to trigger the workflow manually from the GitHub UI (useful when developing/testing this Action) + +#### Environment Variables + +There are three environment variables, the first two will be securely stored as GitHub secrets: + +1. `PROD_DATABASE_URL`: The Postgres connection string for your production or staging database. +2. `DEV_DATABASE_URL`: The Postgres connection string for your Neon database. If you don’t have a Neon account, [sign up here](https://console.neon.tech/signup) and follow our [getting started guide](https://neon.tech/docs/get-started-with-neon/signing-up). +3. `PG_VERSION`: The Postgres version is to be installed in the Action environment. It should be compatible with both your production and Neon Twin databases. + +You’ll also need to add the values from steps 1 and 2 as GitHub repository secrets—I’ll cover that later. + +#### Install PostgreSQL + +This job installs the specified Postgres version using Apt (Advanced Packaging Tool) on the Ubuntu environment in GitHub Actions. While there are other ways to install Postgres available in the [GitHub Marketplace](https://github.com/marketplace?query=Postgres), I personally recommend using Apt for added security.
_GitHub Action runners come with a_ [built-in version of Postgres](https://github.com/actions/runner-images?tab=readme-ov-file#package-managers-usage)_, which would remove the need to manually install the package, but as of now, you can’t control which version of Postgres is installed. If you’d like to follow the progress, I’ve opened an issue here:_ [Update/Add Postgres Version option #11531](https://github.com/actions/runner-images/issues/11531) + +#### Set PostgreSQL binary path + +This step stores the installed Postgres binary path as a GitHub Action environment variable named `POSTGRES`. You’ll notice that it’s referenced in all the following steps, using, e.g., `$POSTGRES/psql`. + +#### Dump schema + +This step uses `pg_dump` to export all the schema from the production database into a file named `all-schema.bak`, which is temporarily stored in the GitHub Actions workspace. + +| Flag | Meaning | +| ------------ | -------------------------------------------------------------------------------- | +| -Fc | Dumps the database in a custom format. | +| –schema-only | Dumps only the schema (table structures, indexes, constraints) without any data. | +| -f | Specifies the output file where the schema dump will be stored. | + +#### Dump data + +This step uses `psql` to query the production database. In my example, I’m querying the `users` table and limiting the response to `50`. The result of this query is saved in a file named `users-subset.csv`, which is also temporarily stored in the GitHub Actions workspace. + +#### Drop tables and schema + +This step uses `psql` to ensure a fully clean database state before restoring data. It’ll prevents conflicts with existing tables, sequences, and constraints and avoids errors where `pg_restore` tries to create objects that already exist. + +| Step | Effect | +| ------------------------------------- | -------------------------------------------------------- | +| DROP SCHEMA IF EXISTS public CASCADE; | Removes the public schema and everything inside it. | +| CREATE SCHEMA public; | Recreates the public schema for a clean restore process. | + +#### Restore schema + +This step uses `pg_restore` to restore the database schema from the `all-schema.bak` backup file. + +| Flag | Meaning | +| ------------ | -------------------------------------------------------------------------------------------------------------- | +| –clean | Drops existing database objects before recreating them, ensuring a clean restore. | +| –no-owner | Ignores ownership information in the dump file, so restored objects are owned by the user running the restore. | +| –no-acl | Excludes access control (GRANT/REVOKE) statements from the restore, preventing permission changes. | +| –if-exits | Ensures that DROP commands (used with –clean) only execute if the object exists, preventing errors. | +| –schema-only | Restores only the schema (table structures, indexes, constraints) without inserting any data. | +| -d | Specifies the target database to restore into. | + +#### Restore data + +And lastly, this step uses `psql` to restore the data to the `public.users` table from the `users-subset.csv` file. + +## Dump from multiple tables + +You can dump and restore as many or as few rows/tables as needed to create an accurate copy of the production database. The closer it mirrors production, the better for testing and development. Just ensure that any foreign key constraints remain intact within the selected data. + +For example, my test database includes two additional tables: **products** and **transactions**. The transactions table references both `product_id` from the products table and `user_id` from the users table. + +If I want to dump a subset of transactions, I must also include a corresponding subset of products while ensuring that all `product_id` and `user_id` values referenced in transactions are present in the dumped data. This guarantees that the restored dataset maintains valid foreign key relationships.
Following on from the above example, i’ve modified two of the steps; **Dump data** and **Restore data**. + +### Modified Dump data + +In this modified step, I first select 50 products and store them in a subset. When extracting data from the `transactions` table, I ensure that only transactions referencing product_ids from this `product` table subset are included. Similarly, when dumping user data, I filter for user_ids that exist in the transactions subset, ensuring that only users associated with the selected products and transactions are included. + +```yaml + - name: Dump data + run: | + $POSTGRES/psql "${{ env.PROD_DATABASE_URL }}" -c "\copy (SELECT * FROM products ORDER BY product_id DESC LIMIT 50) TO '${{ github.workspace }}/products-subset.csv' WITH CSV HEADER" + + $POSTGRES/psql "${{ env.PROD_DATABASE_URL }}" -c "\copy (SELECT * FROM transactions WHERE product_id IN (SELECT product_id FROM products ORDER BY product_id DESC LIMIT 50)) TO '${{ github.workspace }}/transactions-subset.csv' WITH CSV HEADER" + + $POSTGRES/psql "${{ env.PROD_DATABASE_URL }}" -c "\copy (SELECT * FROM users WHERE user_id IN (SELECT user_id FROM transactions WHERE product_id IN (SELECT product_id FROM products ORDER BY product_id DESC LIMIT 50))) TO '${{ github.workspace }}/users-subset.csv' WITH CSV HEADER" +``` + +### Modified Restore data + +Now that i’m confident the correct data has been extracted into the CSV files, I can now restore it while maintaining the correct order. Users are restored first, ensuring that `user_id` values exist before inserting transactions. Products follow next, so `product_id` references are established. Finally, transactions are restored last, ensuring all foreign key dependencies are met. + +```yaml + - name: Restore data + run: | + $POSTGRES/psql "${{ env.DEV_DATABASE_URL }}" -c "\copy public.users FROM '${{ github.workspace }}/users-subset.csv' WITH CSV HEADER" + $POSTGRES/psql "${{ env.DEV_DATABASE_URL }}" -c "\copy public.products FROM '${{ github.workspace }}/products-subset.csv' WITH CSV HEADER" + $POSTGRES/psql "${{ env.DEV_DATABASE_URL }}" -c "\copy public.transactions FROM '${{ github.workspace }}/transactions-subset.csv' WITH CSV HEADER" +``` + +## Adding connection strings as GitHub Secrets + +There are two environment variables used in the Action: one for the production database and the other for the Neon Twin. Both must be added as Repository Secrets in GitHub to give the Action access to them.
+ +In the GitHub UI, navigate to **Settings** > **Secrets and variables** > **Actions**, then add both variables as **Repository secrets**. + +![Adding connection strings as GitHub Secrets](https://cdn.neonapi.io/public/images/pages/blog/automate-partial-data-dumps-with-postgresql-and-github-actions/post-images-githb-secrets-2304c8b9.jpg) + +## Issues with production data + +Dumping production data into development environments can raise potential concerns, especially regarding Personally Identifiable Information (PII). + +However, sensitive information is often confined to specific tables and columns. Partial data dumps can potentially avoid these issues and ensure that only safe-to-use data is included. That said, data privacy remains an important consideration. [We’ve been actively working on a new solution](https://neon.tech/docs/introduction/roadmap#what-were-working-on-now) that enables the anonymization of sensitive data whenever a branch is created, addressing these privacy concerns more effectively. + +## Conclusion + +With a [Neon Twin](https://neon.tech/docs/guides/neon-twin-intro), you can create isolated environments that mirror staging or production for safe development and testing. And our plans to help you anonymize sensitive data will further secure your workflow, making your testing more effective. Happy coding!! diff --git a/content/blog/posts/automating-neon-branch-creation-with-githooks.md b/content/blog/posts/automating-neon-branch-creation-with-githooks.md new file mode 100644 index 0000000000..83698c5ab7 --- /dev/null +++ b/content/blog/posts/automating-neon-branch-creation-with-githooks.md @@ -0,0 +1,315 @@ +--- +title: Automating Neon branch creation with Githooks +description: Instant development databases with post-checkout hook +excerpt: >- + In this blog post, we’ll walk through creating a Githook script that automates + the creation of Neon branches every time a new Git branch is created. This is + possible because Neon provides an API to manage your projects, branches, and + most other operations supported by the Neon Co... +date: "2023-05-25T21:33:03" +updatedOn: "2023-12-20T21:01:25" +category: community +categories: + - community + - workflows +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/automating-neon-branch-creation-with-githooks/cover.jpg + alt: null +isFeatured: false +seo: + title: Automating Neon branch creation with Githooks - Neon + description: Instant development databases with post-checkout hook + keywords: [] + noindex: false + ogTitle: Automating Neon branch creation with Githooks - Neon + ogDescription: >- + In this blog post, we’ll walk through creating a Githook script that + automates the creation of Neon branches every time a new Git branch is + created. This is possible because Neon provides an API to manage your + projects, branches, and most other operations supported by the Neon Console. + You can learn more about the Neon […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/automating-neon-branch-creation-with-githooks/social.png +--- + +
+ +
+ +In this blog post, we’ll walk through creating a [Githook](https://git-scm.com/docs/githooks) script that automates the creation of [Neon branches](https://neon.tech/docs/introduction/branching) every time a new Git branch is created. This is possible because Neon provides an API to manage your projects, branches, and most other operations supported by the Neon Console. You can [learn more about the Neon API here](https://api-docs.neon.tech/reference/getting-started-with-neon-api). + +We’ll start with a simple Githook example and incrementally build up to our final solution. You can find the repository along with the instructions on [GitHub](https://github.com/raoufchebri/create-neon-branch-with-githooks). + +## What are Neon branches? + +Similarly to Git branches, Neon branches are isolated copies of your database that are helpful for experimenting with new features without compromising your database. You can instantly create Neon branches using the Neon console or the API. [Learn more about database branching in the docs](https://neon.tech/docs/introduction/branching). + +## What are Githooks? + +Githooks are custom scripts that run on certain events in the Git lifecycle. You can use them to customize Git’s internal behavior and trigger customizable actions at key points in your development workflow. In our case, we’ll use a `post-checkout` hook, which runs after the `git checkout` command. + +## The challenges of local development + +Traditionally, local development with Postgres can be somewhat cumbersome. You either need to have Postgres installed on your local machine or use Docker, both of which can be resource-intensive and may slow down your computer. In addition, setting up and maintaining your local Postgres database to mirror your production environment requires writing and managing seed scripts. This means you spend significant time managing your database setup rather than focusing on coding. + +Having a separate Neon branch for each of your Git branches can improve your local development experience. With this setup, you don’t need to worry about managing a local Postgres instance or creating seed scripts. You also avoid the risk of inconsistencies between your local and production database schemas, as your Neon branches are exact copies of your main database. The Githook we describe in this article automates the process of creating these Neon branches, saving you even more time and effort. + +## Starting simple: the basic Githook + +Let’s start by creating a simple Git post-checkout hook that prints a message after a `git checkout` operation. Githooks should be placed in the .git/hooks directory of your Git repository. Let’s create a new project folder and the post-checkout file: + +```bash +mkdir neon-githook +cd neon-githook +git init +touch .git/hooks/post-checkout +``` + +In the `post-checkout` file, add the following code: + +```bash +#!/bin/sh +echo "You just checked out a git branch!" +``` + +To make this hook run, we have to make the script executable: + +```bash +chmod +x .git/hooks/post-checkout +``` + +Now, every time you run `git checkout`, you’ll see the message: “You just checked out a git branch!” + +## Checking if a Git branch is new + +Next, we’ll modify the script to create a Neon branch whenever a new Git branch is created. + +First, we need to determine if we just checked out a new Git branch or an existing one. The `git checkout` command passes three arguments to the post-checkout hook: + +`$1` – the ref of the previous HEAD + +`$2` – the ref of the new HEAD (which may or may not have changed) + +`$3` – a flag indicating whether the checkout was a branch checkout (changing branches, flag=1) or a file checkout (retrieving a file from the index, flag=0) + +We can determine if the command is a Git branch checkout by checking the third argument passed to the `post-checkout` hook. If it’s 1, we have checked out a new Git branch. + +```bash +#!/bin/sh +if [ $3 == 1 ]; then + echo "You just checked out a new git branch!" +fi +``` + +The issue here is if we switch back to a previously checked-out Git branch, the script considers it a new Git branch. To avoid this, we count the number of times the Git branch has been checked out. If it’s 1, it means that the Git branch has been checked out for the first time. + +## Adding More Functionality: Creating a Neon Branch + +We want to create a new Neon branch for every new Git branch in our local development environment and paste the new connection string to our environment variables. + +To that end, we need to add code that sends a POST request to the Neon API to create a new database branch. For that, you need to [create a project](https://console.neon.tech/) and [generate an API key](https://neon.tech/docs/manage/api-keys#create-an-api-key) on the Neon Console. To generate an API key: + +1. Log in to the Neon Console. +2. Click your account in the bottom left corner of the Neon Console, and select Account. +3. Select Developer Settings and click Generate new API Key. +4. Enter a name for the API key. +5. Click Create and copy the generated key. + +
+ +
+ +The `PROJECT_ID` for a Neon project is found on the Settings page in the Neon Console. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/automating-neon-branch-creation-with-githooks/image-28-1024x573-e5c88362.png) + +Use the commands below to add the `PROJECT_ID` and `API_KEY` to your environment variables: + +```bash +export NEON_PROJECT_ID= +export NEON_API_KEY= +``` + +Now, we can add the following the `curl` command to the `post-checkout` file to create a branch: + +```bash +#!/bin/sh +if [ $3 == 1 ]; then + BRANCH_NAME=$(git symbolic-ref --short HEAD) + NUM_CHECKOUTS=`git reflog --date=local | grep -o ${BRANCH_NAME} | wc -l` + if [ ${NUM_CHECKOUTS} -eq 1 ]; then + # Add your own PROJECT_ID and API_KEY + PROJECT_ID=$NEON_PROJECT_ID + API_KEY=$NEON_API_KEY + # Execute the curl command and save the output + RESPONSE=$(curl --silent --request POST \ + --url https://console.neon.tech/api/v2/projects/$PROJECT_ID/branches \ + --header 'accept: application/json' \ + --header "authorization: Bearer $API_KEY" \ + --header 'content-type: application/json' \ + --data ' + { + "endpoints": [ + { + "type": "read_write" + } + ], + "branch": { + "name": "'"$BRANCH_NAME"'" + } + } + ') + + # Extract the connection_uri from the response + CONNECTION_URI=$(echo $RESPONSE | jq -r '.connection_uris [0].connection_uri') + fi +fi +``` + +## Adding User Interaction: Prompting for Confirmation + +Before we automatically create a Neon branch, we might want to ask the user if they want to create one. We can do this with the `sh` read command: + +```bash +#!/bin/sh +if [ $3 == 1 ]; then + BRANCH_NAME=$(git symbolic-ref --short HEAD) + read -p "Do you want to create a Neon branch for $BRANCH_NAME? (y/n): " answer > .env + else + echo "Invalid connection string." + fi +fi +``` + +## Checking if a Neon Branch Already Exists + +Before making a request to create a Neon branch, we can first check if it already exists. To do this, we can send a GET request to the Neon API, parse the response, and look for our Neon branch in the returned list of branches. + +Here’s how we add this to our script: + +```bash +# ... existing code ... + +# Get the list of branches + +BRANCHES_JSON=$(curl --silent --request GET \ + --url https://console.neon.tech/api/v2/projects/$PROJECT_ID/branches \ + --header 'accept: application/json' \ + --header "authorization: Bearer $API_KEY") +# Check if the branch already exists +if echo "$BRANCHES_JSON" | jq -e --arg BRANCH_NAME "$BRANCH_NAME" '.branches [] | select(.name == $BRANCH_NAME)' >/dev/null; then + echo "Neon branch $BRANCH_NAME already exists." + exit +fi +``` + +We then use `jq` to parse the response and search for a Neon branch with the same name as the one we want to create. If the branch name is found, we print a message and terminate the script. This small change makes the script more robust and user-friendly. + +This is what the final code looks like: + +```bash +#!/bin/bash +if [ $3 == 0 ]; then exit; fi +# Check if we just checked out a new branch +if [ $3 == 1 ]; then + BRANCH_NAME=$(git symbolic-ref --short HEAD) + NUM_CHECKOUTS=`git reflog --date=local | grep -o ${BRANCH_NAME} | wc -l` + if [ ${NUM_CHECKOUTS} -eq 1 ]; then + # Add your own PROJECT_ID and API_KEY + PROJECT_ID=$PROJECT_ID + API_KEY=$API_KEY + + # Get the list of branches + BRANCHES_JSON=$(curl --silent --request GET \ + --url https://console.neon.tech/api/v2/projects/$PROJECT_ID/branches \ + --header 'accept: application/json' \ + --header "authorization: Bearer $API_KEY") + + # Check if the branch already exists + if echo "$BRANCHES_JSON" | jq -e --arg BRANCH_NAME "$BRANCH_NAME" '.branches [] | select(.name == $BRANCH_NAME)' >/dev/null; then + + echo "Neon branch $BRANCH_NAME already exists." + exit + fi + + # Prompt the user to ask if they want to create a Neon branch + read -p "Do you want to create a Neon branch for $BRANCH_NAME? (y/n): " answer > .env + else + # If it doesn't, print an error message or handle this case as needed + echo "Invalid connection string." + fi + else + echo "Skipping Neon branch creation" + fi + fi +fi +``` + +## Conclusion + +This blog post showed how to implement a Git post-checkout hook that automates the creation of Neon branches every time you create a new Git branch. We started from a simple script and gradually added more features, handling user interactions and API responses, and ensuring it only runs for newly created Git branches. + +Luckily, you won’t have to implement this code in your environments for much longer. We are currently working on a CLI that helps you manage your projects and includes other capabilities such as creating branches using Git commands. So, make sure to stay tuned.
[Get started with Neon and it’s API](https://console.neon.tech/) today for free, and let us know how you use Githooks in your workflows. diff --git a/content/blog/posts/autoscaling-in-action-postgres-load-testing-with-pgbench.md b/content/blog/posts/autoscaling-in-action-postgres-load-testing-with-pgbench.md new file mode 100644 index 0000000000..b450eeb880 --- /dev/null +++ b/content/blog/posts/autoscaling-in-action-postgres-load-testing-with-pgbench.md @@ -0,0 +1,176 @@ +--- +title: 'Autoscaling in Action: Postgres Load Testing with pgbench' +description: 'Develop, Test, Ship, Autoscale' +excerpt: >- + In this article, I’ll show Neon autoscaling in action by running a load test + using one of Postgres’ most popular benchmarking tool, pgbench. The test + simulates 30 clients running a heavy query. While 30 doesn’t sound like a lot, + the query involves a mathematical function with hig... +date: '2024-02-23T09:25:39' +updatedOn: '2024-02-28T15:51:10' +category: postgres +categories: + - postgres + - community +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/autoscaling-in-action-postgres-load-testing-with-pgbench/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Autoscaling in Action: Postgres Load Testing with pgbench - Neon' + description: 'Develop, Test, Ship, Autoscale' + keywords: [] + noindex: false + ogTitle: 'Autoscaling in Action: Postgres Load Testing with pgbench - Neon' + ogDescription: >- + In this article, I’ll show Neon autoscaling in action by running a load test + using one of Postgres’ most popular benchmarking tool, pgbench. The test + simulates 30 clients running a heavy query. While 30 doesn’t sound like a + lot, the query involves a mathematical function with high computational + overhead, which signals to the autoscaler-agent that […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/autoscaling-in-action-postgres-load-testing-with-pgbench/social.jpg +--- + +In this article, I’ll show Neon autoscaling in action by running a load test using one of Postgres’ most popular benchmarking tool, `pgbench`. The test simulates 30 clients running a heavy query. + +While 30 doesn’t sound like a lot, the query involves a mathematical function with high computational overhead, which signals to the autoscaler-agent that it needs to allocate more resources to the VM. + +We will not cover how autoscaling works, but for those interested in knowing the details, you can read more about [how we implemented autoscaling in Neon](https://neon.tech/blog/scaling-serverless-postgres). + +For this load test, you will need: + +1. [A Neon account](https://console.neon.tech) +2. [pgbench](https://wiki.postgresql.org/wiki/Homebrew) + +## The load test + +Ensuring your production database can perform under varying loads is crucial. That’s why we implemented autoscaling to Neon, a feature that dynamically adjusts resources allocated to a database in real-time, based on its current workload. + +However, the effectiveness and efficiency of autoscaling are often taken for granted without thorough testing. To showcase autoscaling in action, we turn to Postgres and `pgbench`. + +`pgbench` is a benchmarking tool included with Postgres, designed to evaluate the performance of a Postgres server. The tool simulates client load on the server and runs tests to measure how the server handles concurrent data requests. + +`pgbench` is executed from the command line, and its usage can vary widely depending on the specific tests or benchmarks being run. Here is the command we will use in our test: + +```bash +pgbench -f test.sql -c 30 -T 120 -P 1 +``` + +In this example, `pgbench` executes the query in `test.sql`. The parameter `-c 30` specifies 30 client connections, and `-T 120` runs the test for 120 seconds against your database.`-P 1` specifies that pgbench should report the progress of the test every 1 second. The progress report typically includes the number of transactions completed so far and the number of transactions per second. + +30 clients don’t seem like enough do stress a database. Well, it depends on the query you’re executing, which we’ll see next. + +## Query execution plan + +Here is the query we’ll use for our load test: + +```sql +SELECT log(factorial(32000)) / log(factorial(20000)); +``` + +Mathematically, this query essentially compares the growth rates of the factorials of 32,000 and 20,000 by examining the ratio of their logarithms. + +Remember factorials? The factorial of a number n (denoted as n!) is the product of all positive integers less than or equal to n. For example, the factorial of 5 (5!) is 5 \* 4 \* 3 \* 2 \* 1 = 120. Factorials grow very rapidly with increasing numbers. + +To give you a sense of scale, the factorial of just 20 is already a 19-digit number: 20!=2,432,902,008,176,640,000 + +The natural logarithmic function (log), on the other hand, is the power to which _e_ (Euler’s number = 2.71828) must be raised to obtain the value x. + +In other words, this operation should take a long time to process. How long? Let’s examine the query execution plan using EXPLAIN ANALYZE: + +```sql +EXPLAIN ANALYZE SELECT log(factorial(32000)) / log(factorial(20000)); +``` + +Output: + +```bash +QUERY PLAN + +------------------------------------------------------------------------------------- + + Result (cost=0.00..0.01 rows=1 width=32) (actual time=0.000..0.001 rows=1 loops=1) + + Planning Time: 1921.630 ms + + Execution Time: 0.005 ms + +(3 rows) +``` + +This query was executed on ¼ vCPU.`EXPLAIN ANALYZE` includes the planner’s estimates and real execution metrics. Execution Time appears to be quite fast. However, Planning Time (the time taken by the Postgres query planner to generate the execution plan) takes almost 2 seconds and suggests that preparing to run this mathematical function involves significant computational overhead. + +Combine 30 of those, and we should stress Postgres enough to trigger autoscaling. + +## Enabling autoscaling + +Autoscaling is the process of automatically increasing or decreasing the CPU and memory allocated to a database based on its current load. It dynamically adjusts the compute resources allocated to a Neon compute instance in response to the current load, eliminating the need for manual intervention. [Learn more about autoscaling in the docs](https://neon.tech/docs/introduction/autoscaling). + +You can enable autoscaling by defining the minimum and maximum compute units (CU) you’d like to allocate to your Postgres instance. This way, you remain in control of your resource consumption. For example, 1 CU allocates 1vCPU and 4GB of RAM to your instance. + +You can set your instance size when you create a new project or by navigating to the Branches page on your Neon Console, clicking on the database branch, and setting the CU range. + + + +I will set the range for this load test from ¼ to 7 CUs. + +# Executing & monitoring the load test + +Let’s run our load test now and observe its effect on our Postgres instance. We recently added graphs to monitor the resources allocated to your Postgres instance and its usage, which will come in handy later. After enabling autoscaling, follow these steps to execute the load test: + +1\. Create your project folder and test.sql file: + +```bash +mkdir pgbench-load-test + +cd pgbench-load-test + +echo "SELECT log(factorial(32000)) / log(factorial(20000));" > test.sql +``` + +2\. Execute the load test by running the following command: + +```bash +pgbench -f test.sql -c 8 -T 120 -P 1 +``` + +you can create a [Neon project](https://console.neon.tech) if you don’t have a connection string. + +3\. Navigate to the autoscaling graph to monitor usage: + + + +You should observe a rapid change in CPU and memory allocated. The result should look similar to the graph below. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/autoscaling-in-action-postgres-load-testing-with-pgbench/screenshot-2024-02-23-at-100445-982x1024-b40450d8.png) + +The performance summary returned by `pgbench` should look like this: + +```bash +latency average = 6000.891 ms +latency stddev = 2768.066 ms +initial connection time = 3712.770 ms +tps = 4.978907 (without initial connection time) +``` + +On average, each operation took slightly over 6 seconds to complete. A standard deviation of 2768.066 ms means that the latencies of individual operations varied quite a bit around the average latency. A higher standard deviation indicates more variability in how long each operation took to complete. + +Establishing this connection took approximately 3.7 seconds before any operations could be performed. A TPS of around 4.98 means that, on average, the database was able to complete nearly five transactions every second during the test, after excluding the initial connection time. + +## Conclusion + +`pgbench` is a simple yet powerful tool to test your database and simulate multiple clients running heavy SQL queries. We also saw how to examine the query execution plan with `EXPLAIN ANALYZE`, which provides insights to optimize your SQL queries. + +If you’re running an application that can be subject to varying workloads, autoscaling offers you the confidence that your database will always under the stress of real-world demands. + +Thanks for reading. If you are curious about autoscaling, [give Neon a try](https://console.neon.tech) and join our [Discord](https://neon.tech/discord). We look forward to seeing you there and hearing your feedback. + +Happy scaling! diff --git a/content/blog/posts/autoscaling-report-2025.md b/content/blog/posts/autoscaling-report-2025.md new file mode 100644 index 0000000000..230bbfd22c --- /dev/null +++ b/content/blog/posts/autoscaling-report-2025.md @@ -0,0 +1,30 @@ +--- +title: Neon Compute Autoscaling Report +description: A deep-dive into the numbers behind Neon Autoscaling. +excerpt: View the Autoscaling Report here. +date: '2026-02-12T19:57:38' +updatedOn: '2026-02-26T17:50:09' +category: product +categories: + - product +authors: + - andy-hattemer +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/autoscaling-report-2025/cover.png + alt: null +isFeatured: true +seo: + title: Neon Compute Autoscaling Report - Neon + description: A deep-dive into the numbers behind Neon Autoscaling. + keywords: [] + noindex: false + ogTitle: Neon Compute Autoscaling Report - Neon + ogDescription: View the Autoscaling Report here. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/autoscaling-report-2025/cover.png +--- + +[View the Autoscaling Report here.](https://neon.com/autoscaling-report) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/autoscaling-report-2025/image-c18df87b.png) diff --git a/content/blog/posts/avoid-mitm-attacks-with-psql-postgres-16.md b/content/blog/posts/avoid-mitm-attacks-with-psql-postgres-16.md new file mode 100644 index 0000000000..d27d8a35e3 --- /dev/null +++ b/content/blog/posts/avoid-mitm-attacks-with-psql-postgres-16.md @@ -0,0 +1,183 @@ +--- +title: MITM attacks are easier to avoid with psql (Postgres) 16 +description: >- + Learn how the psql client in Postgres 16 makes it simpler than ever to connect + a secure interactive session to your Neon database +excerpt: >- + The psql client accepts a new connection string option in Postgres 16: + ?sslrootcert=system. This new option makes it simpler than ever to connect a + secure interactive session to your Neon database: You can use psql version 16 + with this new option even if your Neon database is sti... +date: '2023-10-05T15:45:41' +updatedOn: '2024-03-01T16:01:34' +category: community +categories: + - community +authors: + - george-mackerron +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/avoid-mitm-attacks-with-psql-postgres-16/cover.jpg + alt: null +isFeatured: false +seo: + title: MITM attacks are easier to avoid with psql (Postgres) 16 - Neon + description: >- + Learn how the psql client in Postgres 16 makes it simpler than ever to + connect a secure interactive session to your Neon database + keywords: [] + noindex: false + ogTitle: MITM attacks are easier to avoid with psql (Postgres) 16 - Neon + ogDescription: >- + Learn how the psql client in Postgres 16 makes it simpler than ever to + connect a secure interactive session to your Neon database + image: >- + https://cdn.neonapi.io/public/images/pages/blog/avoid-mitm-attacks-with-psql-postgres-16/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/avoid-mitm-attacks-with-psql-postgres-16/neon-mitm-attacks-2-1024x576-a502e26b.jpg) + +The psql client accepts a new connection string option in Postgres 16: `?sslrootcert=system`. This new option makes it simpler than ever to connect a secure interactive session to your Neon database: + +```bash +psql "postgres://user:pass@endpoint.neon.tech/main?sslrootcert=system" +``` + +You can use psql version 16 with this new option even if your Neon database is still on Postgres 14 or 15. + +Read on to discover why this is important, especially if you’ve previously connected to Postgres using `?sslmode=require`, which isn’t secure. + +## Securing connections in general + +What does it mean for a connection to a server to be secure? Two things are required: + +1. Encryption, which makes your connection safe from eavesdropping and potential takeover. +2. Authentication, which ensures the server you connect to is the one you meant to connect to. + +When you access a website over https — which mostly, these days, [you do](https://letsencrypt.org/stats/#percent-pageloads) — you get both of these things. + +For example, when you loaded this blog post from [https://neon.tech](https://neon.tech), your browser encrypted the request with TLS. It checked that the certificate presented by the server was current; that it said ‘neon.tech’ on it; and that it was signed by a certificate authority (CA) it trusts. + +Your browser trusts the CA because the CA has promised only to provide a signed certificate that says ‘neon.tech’ on it to someone who can prove that they control ‘neon.tech’. + +
+

I know this because I made a page where you can see it all happen all in real-time.

+
+ +This security model — public key infrastructure with a large set of widely trusted CAs — [isn’t perfect](https://www.redhat.com/sysadmin/pki-protection). But we might regard it as the bare minimum level of security we’d want for the data in our databases, much of which will be more valuable and more sensitive than a blog post. + +## Securing connections to Postgres + +Unfortunately, even this minimum level of security is not what you get by default when you use psql to connect to Postgres. + +psql, reflecting the libpq library it’s built on, provides [six different sslmode options](https://www.postgresql.org/docs/16/libpq-ssl.html#LIBPQ-SSL-PROTECTION). These are: `disable`, `allow`, `prefer`, `require`, `verify-ca` and `verify-full`. The default is `sslmode=prefer`, which doesn’t guarantee either encryption or authentication. + +
+

To quote the Postgres docs, sslmode=prefer means: “I don’t care about encryption, but I wish to pay the overhead of encryption if the server supports it”. To quote the same docs again: “this makes no sense from a security point of view”.

+
+ +Commonly specified in preference to `sslmode=prefer` is `sslmode=require`. It is, for example, [what Heroku has been recommending](https://devcenter.heroku.com/articles/connecting-to-heroku-postgres-databases-from-outside-of-heroku#enable-ssl) for years. And if you’re not an expert in Postgres security, it certainly sounds plausible that `sslmode=require` might do the right thing. + +When you use `sslmode=require` you do, technically, get encryption: psql encrypts your connection to the server with TLS. And as part of this process, the server presents psql with a certificate. + +But this certificate does not get you authentication. Because, perhaps a little surprisingly, `sslmode=require` instructs psql not to care _either_ whether the certificate has the server’s name on it _or_ whether it was signed by any particular CA. Just about any certificate, self-signed by anyone, with any server’s name on it, will do. + +This is a bad problem, because a connection that’s not authenticated is vulnerable to a pretty straightforward meddler-in-the-middle (MITM) attack that gives full access to your database. + +
+

Some years ago I was involved in a startup whose main API and database were hosted with a well-known platform-as-a-service (PaaS). The security issues with sslmode=require are nasty enough that they caused me to migrate us off this PaaS very shortly after confirming, a little incredulously, that sslmode=require was the best security this PaaS could provide.

+
+ +### What’s all the fuss about authentication? + +The problem is this. When you connect to your database, psql issues a DNS query to turn the server’s domain name into the server’s IP address. But a bad actor can use [DNS spoofing or cache poisoning](https://www.cloudflare.com/en-gb/learning/dns/dns-cache-poisoning/) to have some other IP address returned by the DNS query. This other IP address, of course, belongs to a machine running the bad actor’s software. + +On receiving a spoofed DNS response containing the bad actor’s IP address, psql goes ahead and connects to the bad actor’s machine. Next, it negotiates TLS encryption, but without checking the certificate the bad actor’s server presents (as discussed, `sslmode=require` means the client doesn’t authenticate the server). Then it sends a startup message, and waits for the bad actor’s server to ask it to authenticate as a client. + +Postgres has [several methods](https://www.postgresql.org/docs/current/auth-password.html) for authenticating a client using a password. But you can be pretty sure the bad actor’s machine will simply ask psql to hand over its password in cleartext, and psql will cheerfully oblige. + +The bad actor now has free access to your database. They might begin by running, say, `pg_dumpall`, or `DROP DATABASE main`. + +
+

If the bad actor did run DROP DATABASE main, you might be happy to hear about Neon’s point-in-time restore functionality, but that’s not really the point here.

+
+ +Perhaps you’re now saying: “But _a-ha_! _My_ Postgres server uses [SCRAM-SHA-256](https://www.postgresql.org/docs/current/auth-password.html#AUTH-PASSWORD) to authenticate me as a user. This is a challenge-response scheme, so my password never gets sent over the network. Therefore the worst that can happen is that the bad actor gets to forward on and observe this particular Postgres session”. + +Well, it’s bad news, I’m afraid. Neon uses SCRAM-SHA-256 too, but SCRAM-SHA-256 will only help you if you remembered to specify `require_auth=scram-sha-256` for your connection. Since the `require_auth` option is also new as of Postgres 16, and not yet widely trailed, that seems pretty unlikely. + +If you didn’t specify a `require_auth` method, then psql lets the server call the shots. So it doesn’t matter that _your_ database wouldn’t have asked for a cleartext password. If the bad actor’s machine asks for a cleartext password, that’s what psql will give it, and BOOM: game over. + +Given the right circumstances, DNS-spoofing MITM attacks of this sort really aren’t very difficult. They’re easily [within reach]() of a competent command-line user on the same local network. + +> To check that this claim stands up, I tried mounting a DNS spoofing attack on myself. I’m not a networking or security expert, and it took me less than an hour — including the necessary research into network tools like [bettercap](https://www.bettercap.org/) — to make a domain of my choice resolve to an IP address of my choice on a PC elsewhere on my home network. +> +> Spurred on by this success, I decided to try the actual MITM attack too. I spent about an hour writing a short Ruby script using the `openssl` gem. This poses as a TLS-enabled Postgres server in order to request the client’s cleartext password. I spent roughly a further hour making the script then connect to the real server and proxy all subsequent communication, so that the MITM victim sees nothing out of the ordinary. The final script is under 100 lines of code. +> +> I generated a self-signed certificate, ran the script, and arranged for the target machine to resolve `*.eu-central-1.aws.neon.tech` to the IP address of the machine the script was running on. +> +> On the spoofing target machine, connecting to and querying my database with `sslmode=require` appeared to work perfectly normally. But, over on the attacker machine, bettercap had logged the following: +> +> ```bash +> george@attacker postgres-dns-spoof % ./pg-poser.rb +> listening ... +> got SSLRequest message +> TLS connection established with client +> name via SNI: ep-long-grass-595339.eu-central-1.aws.neon.tech +> got startup message +> user: george +> database: main +> application_name: psql +> client_encoding: WIN1252 +> requested cleartext password authentication +> authentication message received +> password: MQZIQzlnIe11 +> sending SSLRequest to server +> [...] +> ``` +> +> The log continues by displaying every byte transmitted between client and server. All in all, this was worryingly straightforward, and only a short morning’s work. + +### Doing it right + +The solution to all this nastiness, as you probably guessed, is to ensure that your psql connection uses both encryption **and** authentication. + +_Before version 16,_ that meant (a) providing psql with one or more trusted CA certificates for it to verify against using the `sslrootcert` option (or a file at `~/.postgresql/root.crt`), and (b) telling it to actually check it, using `sslmode=verify-full`. + +To make that work with Neon from a Mac looks something like this: + +```bash +psql "postgres://user:pass@endpoint.neon.tech/main?sslmode=verify-full&sslrootcert=/etc/ssl/cert.pem" +``` + +On Linux distributions, you alter the sslrootcert location: [see our docs](https://neon.tech/docs/connect/connect-securely). On Windows, you download [a root cert bundle](https://curl.se/docs/caextract.html) and point `sslrootcert` to that. + +If you’re using a psql version below 16, that’s how you should connect. It’s fine, but it’s pretty verbose and not so easy to remember (where are those SSL certs again?). + +The new `sslrootcert=system` option in psql 16 provides a nice, easy shortcut for this behavior. Rather than you having to provide psql with trusted CA certificates, `sslrootcert=system` instructs psql _both_ to use the trusted CA certificates built into your OS as root certs _and_ to do proper authentication via `sslmode=verify-full`. + +As a reminder, on all platforms, it looks like this: + +```bash +psql "postgres://user:pass@endpoint.neon.tech/main?sslrootcert=system" +``` + +As expected, this breaks my proof-of-concept MITM attack. Using psql with `sslmode=verify-full` from my DNS-spoofing target machine gets me: + +`psql: error: connection to server at "ep-long-grass-595339.eu-central-1.aws.neon.tech" (192.168.1.119), port 5432 failed: SSL error: certificate verify failed` + +
+

Note: at the time of writing there appears to be a problem with the EnterpriseDB Postgres 16 builds for Windows, such that using sslrootcert=system always results in the message SSL error: unregistered scheme. I’ve reported this: let’s hope it’s fixed soon.

+
+ +To insure yourself against errors or omissions, it’s probably also a good idea to change the insecure libpq defaults by setting the following values in `.bash_profile`, `.zprofile`, or wherever you define your environment variables: + +```bash +export PGSSLROOTCERT=system # or a root cert file for psql <16 +export PGSSLMODE=verify-full # in case psql <16 is used +export PGREQUIREAUTH=scram-sha-256 +``` + +Then the only way you’ll get a connection that’s vulnerable to MITM attacks is if you specifically ask for one. + +Lastly, if you’re connecting to Postgres using other clients or libraries — e.g. from a web server or API — make sure you know how to specify at least an equivalent level of security to `sslrootcert=system`. For example, in JavaScript, I’m pleased to report that the [node-postgres TLS implementation](https://node-postgres.com/features/ssl) defaults to `sslrootcert=system` behavior unless you explicitly specify the option `rejectUnauthorized=false`. diff --git a/content/blog/posts/aws-cni-lessons-from-a-production-outage.md b/content/blog/posts/aws-cni-lessons-from-a-production-outage.md new file mode 100644 index 0000000000..4876f8b356 --- /dev/null +++ b/content/blog/posts/aws-cni-lessons-from-a-production-outage.md @@ -0,0 +1,332 @@ +--- +title: AWS CNI lessons from a Production Outage +date: '2025-06-06T20:57:44' +updatedOn: '2025-06-07T08:49:27' +category: engineering +categories: + - engineering +authors: + - em-sharnoff + - mihai-bojin +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxckdcmuel8sipntjphfv8kyflswpz1nv0xufnyewbs3vgrwxrp3w-r1krk4cv-atdhlpxosudl3lcwv3lxogswgm5fhbaq3kghwncse-oawl8-7twbk1eufxhzhe47geb7yrg-c3a9c973.png + alt: null +isFeatured: false +seo: + title: AWS CNI lessons from a Production Outage - Neon + description: >- + This post covers what we learned about AWS CNI through our post-mortem and + root-cause investigation into our outages on 2025-05-16 and 2025-05-19. + keywords: [] + noindex: false + ogTitle: AWS CNI lessons from a Production Outage - Neon + ogDescription: >- + This post covers what we learned about AWS CNI through our post-mortem and + root-cause investigation into our outages on 2025-05-16 and 2025-05-19. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxckdcmuel8sipntjphfv8kyflswpz1nv0xufnyewbs3vgrwxrp3w-r1krk4cv-atdhlpxosudl3lcwv3lxogswgm5fhbaq3kghwncse-oawl8-7twbk1eufxhzhe47geb7yrg-c3a9c973.png +--- + +_This post is the last in a series discussing the Neon outages on 2025-05-16 and 2025-05-19 in our AWS us-east-1 region. In this post, we cover the IP allocation failures that persisted through the majority of the disruption. For further details, read our_ [top-level Post-Mortem here](https://neon.com/blog/postmortem-delayed-start-compute-operations)_._ + +## Summary + +Neon separates Storage and Compute to provide [Serverless Postgres](https://neon.com/blog/scaling-serverless-postgres). Our Compute Instances run in lightweight Virtual Machines in Kubernetes, each Compute running in its own Pod. + +On 2025-05-16, the [Neon Control Plane](https://neon.com/blog/control-planes-for-database-per-user-in-neon)’s periodic job responsible for terminating idle Computes started failing, eventually resulting in our VPC subnets running out of IP addresses in two of three availability zones. Configuration changes to AWS CNI to free up IP addresses, while beneficial in the immediate term, later prevented returning to a healthy state. A post-incident follow-up to revert this temporary state on 2025-05-19, resulted in similar issues. + +During this investigation, we have learned a lot about the behaviour of the AWS CNI plugin, how it interacts with our highly-dynamic environment, and have [filed an improvement PR](https://github.com/aws/amazon-vpc-cni-k8s/pull/3300). + +This article covers how the incident happened and details about what we learned about AWS CNI through our post-mortem and root-cause investigation. + +### Glossary of terms + +- **AWS CNI**: refers to the [AWS VPC CNI plugin](https://github.com/aws/amazon-vpc-cni-k8s/). A more in-depth description of AWS CNI is provided below. +- **ipamd**: part of AWS CNI, refers to the L-IPAM [daemon](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go) +- **AWS ENI**: AWS Elastic Network Interface; ENIs are allocated to EC2 instances and are associated with a subnet +- **AWS VPC**: logically isolated [virtual network](https://docs.aws.amazon.com/vpc/latest/userguide/what-is-amazon-vpc.html) provided by AWS +- **AWS VPC subnet (or _subnet_)**: represents a range of IP addresses in a VPC +- **Allocated IPs**: AWS subnet IPs allocated to ENIs +- **Assigned IPs**: IP addresses assigned to Kubernetes Pods (most Pods in our clusters are Neon Computes) +- **Total IPs**: total IP addresses available for allocation in a subnet (or subnets) + +## 2025-05-16: Running out of IP addresses + +Neon operates Kubernetes clusters in 11 cloud regions. Our _us-east-1_ cluster in AWS typically operates a daily peak of 6,000 running databases (which we call Computes), with an incoming rate of 500 new Pods started every minute and a similar rate of terminating idle databases. + +When the incident started, the job responsible for shutting down idle databases failed (we have described this in more detail in a separate [post](https://neon.com/blog/delayed-start-compute-operations-triggering-event)). As terminations were not processed, but creations continued, the number of running Computes quickly rose past our cluster’s typical operating conditions, reaching ~8k active Computes in the space of a few minutes. + +At ~8k active computes, our AWS VPC subnets ran out of IPv4 addresses. This was unexpected, as we test our clusters for up to 10k Computes, and our subnets were sized to a total of 12k IP addresses! + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxckdcmuel8sipntjphfv8kyflswpz1nv0xufnyewbs3vgrwxrp3w-r1krk4cv-atdhlpxosudl3lcwv3lxogswgm5fhbaq3kghwncse-oawl8-7twbk1eufxhzhe47geb7yrg-c3a9c973.png) + +### A summary of the conditions that led to IP allocation unavailability + +- With its default settings, AWS CNI reserves at least 1-2 _extra_ ENIs worth of IP addresses on each node +- Our nodes can utilize up to 49 IPv4 addresses per ENI +- Our AWS _us-east-1_ region only had 12k total IP addresses instead of the 24k we have in other regions. +- During the incident, we had ~4k extra IPs allocated on nodes that didn’t have enough CPU or memory available for new compute Pods to be scheduled. +- As a result, we became unable to start new computes while only 8k of 12k IPs were assigned to compute Pods — at the time, this was confusing and unexpected. + +### Aside: Why only 12k IP addresses? + +As one of our first regions, we hadn’t originally planned to run the cluster at this scale. Our load testing had indicated that our clusters could work with vertical scaling up to 10k Computes, but that after that, we would need to scale out horizontally. + +Even though each of our three subnets was configured with a /20 CIDR block (half the size of our other clusters), we assumed we would always have sufficient available IPs due to the identified upper bound of 10k active Computes. + +The rate of growth of our service in recent months has been faster than anticipated, so we’ve been working in parallel on deeper architectural changes to support horizontal scaling. We will post articles describing the new architecture after we launch it. + +### Background: What is AWS CNI? How does it work? + +Explaining the behaviour we saw, requires some understanding of how AWS CNI works. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxfsm4sj69hmintf3oqdvghpxwmegnk4t1kq2jn3jo5v6wzvvutjwc9vteqfmnrdrgg4c-6o6qcmljoc4dwdp3b7kppyrh41kfgxld13be-hydanx7qwdeblz85kvpvyiaaba6ysvw-d3d14cdd.png) + +The Kubernetes [Container Networking Interface](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) (CNI) is the standard interface used for configuring Pod networking in Kubernetes. CNI _plugins_ are called by the container runtime to set up (“_add_”) and tear down (“_del_”) networking for each Pod. “AWS CNI” is how we refer to the [AWS VPC CNI plugin](https://github.com/aws/amazon-vpc-cni-k8s/). At the time of this incident we were using AWS CNI v1.18.6. + +Each Pod needs an IP address for networking within the cluster, and AWS CNI’s job is mostly assigning IP addresses to Pods, pulling from the appropriate VPC subnet. Internally, the CNI plugin itself makes RPC calls to _ipamd_ — the host daemon on each node, responsible for allocating IPs from the subnet onto the ENIs attached to the EC2 instance and handing those out to Pods. + +To isolate Pod starts from AWS API calls (and vice versa), _ipamd_ keeps a pool of IP addresses – more than is strictly necessary for the number of Pods on the node. The pool is resized every few seconds by a [separate reconcile loop](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L658-L665), outside the context of any individual CNI request. + +AWS CNI has several configuration options to influence how it manages its pool of IP addresses. We include details about our choice of options below. + +### A quick recap + +Our AWS _us-east-1_ cluster typically operates with 5-6k active Computes. We run our Compute Pods on [m6id.metal](https://aws.amazon.com/ec2/instance-types/m6i/) AWS instances, with 49 IP addresses per ENI (plus one IP address assigned to the network interface itself). In theory, these instances can support up to [737 pods each](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/misc/eni-max-pods.txt#L499) (or more, with [prefix delegation](https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html)) — in practice, we tend to run 100-400 pods per node. + +It’s worth mentioning that not all databases are equal — the number of running compute Pods on any Kubernetes node is dynamic and depends on the size of the scheduled workloads. For example, a 128 CPU node can run 128 pods with 1 CPU each, 4 pods with 32 CPUs each, or any combination in between. + +During Friday’s incident, our Control Plane became [unable to terminate](https://neon.com/blog/delayed-start-compute-operations-triggering-event) idle databases. This resulted in the number of active Compute Pods quickly rising from ~5k to ~8.1k. As new Pods exhausted all schedulable CPU and memory across the cluster, [our cluster-autoscaler](https://github.com/neondatabase/autoscaling/tree/244f0f0725f1d523315bf04e2228600221dd9465/cluster-autoscaler) added more nodes. + +At this point, we had old nodes without CPU or memory capacity, but with many additional allocated IPs that could never be assigned to Pods due to these scheduling constraints. This issue was not clear to us at the time. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxfsx9lu6vh8zwnzxpskcqtocx8prjnzpbqeivvxnd7jcq4p9nmbbfmtvpwqfnva3tqghydvpujhtf2fdsz6maxyjlak9elay1blxsyjdtsqztptiswoukus9fhgmr39m20cbw-6628a39d.png) + +As more nodes were added, we started observing IP allocation errors when new Pods were scheduled but were unable to start. + +### Why did we run out of IP addresses? + +Prior to the incident on Friday, we were using the default AWS CNI configuration (`WARM_ENI_TARGET=1` and `WARM_IP_TARGET` unset, more on these later). + +Prior to the incident, each of the cluster’s 3 subnets had 3.7-3.9k **allocated** IP addresses (stored in _ipamd_’s IP pools), with only 1.6-2.3k IP addresses **assigned** to Pods (~50% utilization). Each of our three subnets were configured with a /20 CIDR block. This meant we had up to (4096 – 5) × 3 = 12273 **total** IP addresses ([5 IPs in each VPC subnet are reserved](https://docs.aws.amazon.com/vpc/latest/userguide/subnet-sizing.html#subnet-sizing-ipv4)). + +
+ +
(11:09 UTC on 2025-05-16)
+
+ +During the incident, with the sudden increase in running Pods, the cluster had **assigned**~8.8k total IP addresses (71%). However, across our three subnets, _99% of all IPs_ were **allocated**, totaling ~12,200 out of 12,273. + +Because only 8.8k IP addresses were assigned, we expected that the already allocated IP addresses would be assignable to Pods, but the result was different and unexpected: IPs allocated to old nodes were, in practice, unusable. These were allocated to nodes already at CPU/memory capacity and were also not being released by AWS CNI. + +Because of this detail, it appeared that the subnets had sufficient unassigned IPs available to be used for new Pods. + +
+ +
(15:00 UTC on 2025-05-16)
+
+ +In practice, as new nodes were added, they became unable to obtain sufficient IPs to match available CPU/memory capacity. + +### Why were there so many IPs allocated to nodes with no spare resources? + +Overallocation of IPs has to do with AWS CNI’s behavior under the default settings, which has `WARM_ENI_TARGET=1` and `WARM_IP_TARGET/MINIMUM_IP_TARGET` unset: + +- Whenever _ipamd_ sees that the number of “available” IP addresses (allocated minus assigned) is [less than WARM_ENI_TARGET × (IPs per ENI)](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L2179), it will attempt to allocate more. +- Allocating more IPs – if none of the existing ENIs have room – will attempt to allocate an ENI’s worth of IP addresses [[1](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L863-L865), [2](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L2119-L2120), [3](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L1831-L1832)], specifically by the code block below: + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxdxkgeupdu3k1pfeuiksp7zl3dkxwc81-7ktlqlsqjgrvw9pswc4c-0ew9ompptly14bi7iiizbtbll4s28mjtl1bljl7yaj7cwc24phz2d4dvytzataxhkjjq0rylbkm-xihw-09783b91.png) + +- Releasing IPs is desired when the number of “available” IP addresses is [more than (WARM_ENI_TARGET + 1) × (IPs per ENI)](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L1276) — e.g., if an ENI’s worth of IPs could be removed without falling below the target. +- However, releasing IPs can only happen [when there’s an **ENI with no assigned IPs**](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/datastore/data_store.go#L918-L921), because this configuration limits removal to happen only at the ENI level. +- Critically, IP assignment to Pods [randomly picks among all ENIs](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/datastore/data_store.go#L680) (because ENIs are [stored in a hashmap](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/datastore/data_store.go#L237-L238), and in Go, [hashmap iteration order is random](https://github.com/golang/go/blob/go1.24.3/src/runtime/map_noswiss.go#L912)!) + +The target for available IPs means that _ipamd_ must allocate 50-100 IP addresses above what’s needed by Pods (50 IPs per ENI for the _m6id.metal_ instance type). Because we have a stable rate of incoming Pods in our cluster, the random distribution of Pods onto ENIs keeps all ENIs in use, once added _we never free IP addresses back to the subnet_. + +We were surprised to find this, so [we have opened a PR to AWS CNI to improve ipamd’s behavior](https://github.com/aws/amazon-vpc-cni-k8s/pull/3300) under these circumstances. + +As an example of just how severe this can be, consider a node that very occasionally peaks up to 400 active Pods, but normally has enough large workloads that it only has the CPU / memory capacity to support 200 active Pods. We might see a sequence of events such as the following: + +1. A burst of Pod starts causes the node to have 400 active pods, with no available IPs left +2. _ipamd_ sees that the number of “available” IPs is [low](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L674-L679), and [allocates more](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L790) (with a new ENI) from the VPC subnet, aiming to always maintain at least 50 available (extra) IPs +3. As older Pods on the node are removed, their IP addresses are **kept in cooldown for 30s** before they can be reused – so IP addresses on the new ENIs must be used for new Pod starts during this period +4. After the load subsides, the random distribution of Pods onto ENIs results in a high probability of having at least one Pod per ENI, causing all 50 IPs per ENI to remain allocated to the node (remember: the **entire** ENI must be unused to remove any of the IPs.) +5. As a result: This node is left with 200 running pods but 450 allocated IPs! + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxfm2cccvl5u1qwj8mqihxzaxxuhztxoxwogxriue4slsfcucwcu6uwa8y6vn-vkms5atfswxdidvzhfko-suaub9djicyv97ejyrqnmadqxojlobtsaawhy2z0c-bo4g8ra4xg-edb91527.png) + +This happened enough in practice that almost all the IP addresses in our VPC subnets were either assigned to Pods (~8.8k) or allocated to nodes with no remaining CPU/Memory capacity (~3.4k, including ENI device IPs). Once we ran out of IPs in the subnet, we were unable to start new compute Pods, which in turn meant that idle databases couldn’t be woken up. + +That still leaves ~100 IP addresses in our subnets unaccounted for – we’re not certain why they weren’t allocated (and we are following up with AWS Support to understand why this was the case). However, an extra 100 IPs likely wouldn’t have helped much, since our cluster needed an additional ~3.4k IPs. + +## 2025-05-16: WARM_IP_TARGET=1, Releasing IPs, IPs still not assignable + +After our VPC subnets ran out of IP addresses, we looked to simultaneously unblock further compute Pod creation and fix our control plane’s [periodic job](https://neon.com/blog/delayed-start-compute-operations-triggering-event) so that old computes _were_ terminated. + +To unblock compute Pod creation, we set [WARM_IP_TARGET=1](https://github.com/aws/amazon-vpc-cni-k8s/?tab=readme-ov-file#warm_ip_target). This had the immediate intended and expected effect – freeing allocated IP addresses from nodes that couldn’t use them, and allowing more Pods to start. + +Once our control plane started successfully terminating idle Computes, we observed a significant drop in the rate of successful Pod starts. As we later found out, `WARM_IP_TARGET=1` unexpectedly _prevents_ new Pod starts for 30s after each Pod deletion. + +### Background: What does WARM_IP_TARGET do? + +Above, we described how `WARM_ENI_TARGET=` works: _ipamd_ ensures that there are at least T _ENIs_ worth of extra IP addresses allocated to the node, only freeing them when an entire ENI is unused. + +In contrast, when `WARM_IP_TARGET=` is set, _ipamd_ attempts to maintain _exactly_ N extra IP addresses on the node. More IP addresses are allocated when fewer than N IPs are available Extra IP addresses are freed if there are more than N available. + +If both `WARM_IP_TARGET` and `WARM_ENI_TARGET` are set, `WARM_IP_TARGET` takes precedence. + +### Why set WARM_IP_TARGET=1? + +During the incident, we observed that our VPC subnets were out of IP addresses with only 72% of those IPs actually assigned to Pods. We inferred that those unused IPs must have been allocated to nodes with no room to start new Pods and looked for a quick way to free them up. + +Setting `WARM_IP_TARGET` appeared to be the most straightforward option. + +At the time, we were operating AWS CNI with the default configuration options. During the incident, we misread the documentation, mistakenly believing that the default value for `WARM_IP_TARGET` was 5 when unset, leading us to decide to “reduce” it to one. + +However, when this parameter is unset, AWS CNI actually bases its logic on `WARM_ENI_TARGET`, which has substantially different behaviour from using `WARM_IP_TARGET`. Unbeknownst to us, this had much larger implications than just reducing the value, many of which we didn’t understand until much later. + +### What happened with WARM_IP_TARGET=1? + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxfmxovy3mcjxhcopp1jdqvzpo1vt725ukl3b9akwltsr-jvf6mpaplmdtgki1o33va4hhyiunfnbeze0omhb6hybt7qwp1wc8zcentcwg5vangavk8hjlg4qbppjiogkwo-5fh-8b0bffbe.png) + +The immediate effects _were_ as we expected: Thousands of IP addresses were returned to the VPC subnets from nodes that couldn’t use them, and subsequently allocated by nodes with CPU / memory capacity to start Pods. New Pods started on these nodes, and when we eventually hit ~10k concurrent Pods, our rate of starts slowed again due to limits we’d previously identified in load testing (e.g. kube_proxy sync latency). + +Soon after, we stabilized our Control Plane’s failing job and ~7k idle computes were terminated. However, the rate of successful Pod starts remained far below the pre-incident baseline. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxe0fgvzfgyr9k-mvgy7q0m75-n80wylnb8sjegwg0joj5rzsiz7jl8khwyxyuwdb40nfnc3cmjddkildvwlwkessoul-yjztny6khss-l2zf122ocoz2zdag8rlbmdkscpni4c2a-1c69e81c.png) + +Investigation at the time showed that most of our new Pods were failing to start due to IP assignment issues — even though VPC subnet metrics confirmed that there were thousands of **unallocated** IP addresses that should have been available. + +At the time, we couldn’t figure out why IP allocation was failing. The AWS CNI documentation mentioned that [WARM_IP_TARGET can trigger rate-limiting on EC2 API requests](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/docs/eni-and-ip-target.md?plain=1#L48-L50), however, this is not a problem we expected with only 1-2 Pod starts **per node, per minute**. Shouldn’t _ipamd_ be able to request more than that? + +We spent much of the time following this incident digging through [AWS CNI’s source code](https://github.com/aws/amazon-vpc-cni-k8s/) to understand its behaviour, cross-referencing with metrics and logs we’d captured from the time of the incident. + +### Why did WARM_IP_TARGET=1 prevent Pods from starting? + +Broadly, AWS CNI has two methods of operation, depending on whether WARM_IP_TARGET and/or MINIMUM_IP_TARGET are specified (internally referred to as the [“warm target” being “defined”](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L1685-L1688)). + +We described the default above – if there _isn’t_ a warm target, ipamd relies on `WARM_ENI_TARGET`’s value to determine how many IPs to allocate. But with `WARM_IP_TARGET` set, ipamd has the following behavior: + +- When the number of unused IP addresses is [less than the warm target](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L2165-L2168), more IPs are allocated from the VPC subnet until there are `WARM_IP_TARGET` available IPs [[1](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L1829), [2](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L921), [3](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L930-L931)] (automatically adding ENIs as needed). +- If the number of unused IP addresses is [more than the warm target](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L2189-L2192), unassigned IPs are returned to the VPC subnet (subject to [rate limiting](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/ipamd.go#L694-L699)). +- As with other configurations, IP addresses go into “cooldown” for [30 seconds](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/datastore/data_store.go#L224-L230) after being unassigned, during which they [cannot be reused](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/datastore/data_store.go#L1321). +- Crucially, and perhaps unexpectedly, **IP addresses in cooldown** [count towards the warm target](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/pkg/ipamd/datastore/data_store.go#L787-L789). + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxeyuswvd48341kg0eimyfrdz2jhxrxo7omsbjjaeacnjz9do26fgz4rtonmyk15nqemg97n7r4scq4dat8m-iqlh-m3cyksgag697ukapp4g0ffkjvcntwhdehoqbdjnczcdbm-66227fda.png) + +This combination of factors means that **setting `WARM_IP_TARGET=1` _can prevent all Pod starts on a node for 30 seconds after each Pod removal_ **, because _ipamd_ will ensure that there’s exactly one “available” IP address (even if that IP address can’t be assigned due to the 30-second cooldown period). + +That’s why we only saw this problem _after_ our control plane started terminating idle computes. When no Pods are removed, _ipamd_ can sustain a high rate of Pod starts, in spite of the small warm IP target. Deleting Pods, however, can simultaneously prevent assigning existing allocated IPs while _also_ preventing _ipamd_ from allocating more (because `WARM_IP_TARGET=1` is guaranteed to be satisfied while any IP address is in cooldown). + +To make matters worse, our control plane retries compute creation if it doesn’t succeed within the timeout window (currently 2 minutes). Combined with our preexisting long Pod startup times, these retries exacerbated the problem as many of the successful Pod starts were deleted shortly before the rest of setup could continue – each time resetting the 30 second countdown to being able to start more Pods. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxdprcahf5w-t2kxprrpxlcy-ktvpeh5bstij3knzfwdqu1hyupbuxutdbd4vxvub-d1w8bixlcedjlsjh-gggeuefo1clb9xn9izlbvtre2gilv3afjzm29hxyh3vrvvboxyip-9790ca82.png) + +### Back to the incident: What did we do at the time? + +At the time, we didn’t understand why our rate of successful Pod starts was so low. + +We thought that it was _theoretically_ possible that there were still unused IP addresses somewhere, or maybe `WARM_IP_TARGET=1` was misbehaving. So in a last-ditch effort to free up any other allocated IP addresses, we reduced `WARM_IP_TARGET` even further, **to zero**, also setting `MINIMUM_IP_TARGET=0` and `WARM_ENI_TARGET=0`. This helped! + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxf236eosv-selgkriwz1ihjmbrs5dc943rq0asgh7ctr12imyuwuiet4-vs9mukqoq3li5b7rawv8nzfo80hxl3yuntezhmzjmj6rcdf6tdkniqt72c9pewoqjhljqhervx9-sg-23da9533.png) + +Unbeknownst to us at the time, setting `WARM_IP_TARGET` to zero is _equivalent to disabling it_, resulting in _ipamd_ using the `WARM_ENI_TARGET` logic. + +There were **two key side effects** of this configuration: + +1. Similarly to before the incident, _ipamd_ effectively stopped returning IP addresses to the subnet +2. New IP allocations from the subnet attempted to reserve as many IPs as could fit on the ENI (instead of one IP at a time) + +Together, these resulted in enough IP addresses being allocated to the nodes, allowing the cluster to stabilize. The average number of allocated IP addresses on each node increased from ~75 per node to ~250, and our rate of successful Pod starts returned to normal. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxei8hmsya4n-ciwutsggpzkyku7d04kfzkpxouyct5l15-r52-mnvme9rs1sn9wg4zpcbvtrfhroefwbtrh2p5aew-h3awbofeiochynmrqkdwyddo3ctbnnuyyvufaxlxrg1yq-cc755912.png) + +We continued to monitor the cluster over the weekend and observed a stable state until the following Monday. + +## 2025-05-19: AWS CNI config change goes wrong, reverting doesn’t help + +**The following Monday**, as an incident follow-up, we decided to revert the final change to our AWS CNI configuration. We believed that the state of our _us-east-1_ cluster after Friday’s incident was not stable, and thought that switching back to `WARM_IP_TARGET=1` would help. + +At the time, we were concerned that AWS CNI’s behavior with `WARM_IP_TARGET=0` was unspecified, and believed that `WARM_IP_TARGET=1` would be more stable. + +Rolling out `WARM_IP_TARGET=1` triggered the same behavior where deleting Pods interfered with our ability to start new ones. Upon observing the same conditions, we then undid that change back to `WARM_IP_TARGET=0`. However, IP assignment errors continued. + +We compensated for high error rates by increasing the size of our [pre-created compute pools](https://neon.com/blog/cold-starts-just-got-hot). The IP assignment errors continued for hours afterwards, until a 86-second window where our control plane didn’t stop any Pods, allowing more IPs to be allocated and resolving the errors. + +### Why set WARM_IP_TARGET=1 again? + +As is often the case, we knew much less then than we do now. + +We were becoming less certain about the behavior of `WARM_IP_TARGET=0`. [In one place](https://github.com/aws/amazon-vpc-cni-k8s/blob/v1.18.6/README.md?plain=1#L301-L303), the documentation said that zero was equivalent to “_not setting the variable_”, but that the default was “_None_”, leaving us uncertain about the actual behavior with that configuration. If zero were the default, that would have been the same configuration that originally caused us to run out of IP addresses. + +We also suspected that Friday’s IP assignment issues may have been resolved by coincidence and not by setting `WARM_IP_TARGET=0`. For example, we saw temporary improvements every time we restarted the `aws-node` DaemonSet (which reinitializes _ipamd_) — the symptoms could have been resolved by the final restart. + +Remembering that setting `WARM_IP_TARGET=1` had initially helped on Friday, we believed that it was likely to be more stable than the unknown situation we found ourselves in. + +This was a mistake at the time. Further, reverting back to `WARM_IP_TARGET=0` was not sufficient to recover from the resulting degraded state. + +### What happened with WARM_IP_TARGET=1 this time? + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxdoxknbh7spojjuccfhbadtbv9nwznfxb7jzsrqifhnn4nv3fuqhgcqiqwy3vjpze8cyzsu8rwe2yi16quywvtfqvj1o2sdyuzlki3hyadndxe7hvpj3djsajyujpphxfiatg-3e5cb270.png) + +IP assignment immediately started failing, and with it, our rate of successful Pod starts dropped to the same level as it was with `WARM_IP_TARGET=1` on Friday. + +This was unexpected. At the time, we thought Friday’s IP assignment errors were due to the cluster being left in a bad state after our VPC subnets ran out of IP addresses. Here, the errors started from a stable state — clearly inconsistent with our understanding. + +Aiming to avoid further outages, we wanted to be sure of any additional configuration changes. We took some time to examine _ipamd_ logs and eventually determined that there were likely specific issues with `WARM_IP_TARGET=1`. + +We reverted back to `WARM_IP_TARGET=0`, but continued to see IP assignment errors. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxdycmgpejwlzxjbnfl7lucloaacuvb7gozbop3wkudndoq7ya7wwrxjsaomspvx20n41ymfx8cdsio9ybkwy-twptkk2n08kylqbmivvigyaudthpsp2srgad4dcxwnormtxuw-c27ccf49.png) + +### Why didn’t reverting fix the issue? + +It was very unexpected that issues persisted after reverting WARM_IP_TARGET back to 0. This was the healthy state through the weekend, so why didn’t it work now!? + +The rate of errors _had_ decreased enough for more Pods to get through, but the overall success rate was far below expectations: + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxdqzaru4tn9p327yrzhnhixwpia8erresdvmixvh6mjimggbaxufh8p3ilepcteigj0aql4yuhvf5bzu0n04gwqfuqugrmcwnuznrysmmemu-aixbqipo6gyfpsq1gg6nz762q-4cb88c32.png) + +In our investigation over the following weeks, we attained a deeper understanding of the AWS CNI codebase. + +When `WARM_IP_TARGET=0` and `MINIMUM_IP_TARGET=0`, AWS CNI uses the behavior for `WARM_ENI_TARGET` — even though we had `WARM_ENI_TARGET=0` as well. + +Under these conditions, ipamd will only allocate more IP addresses to the node [if there are **no available IP addresses**](https://github.com/aws/amazon-vpc-cni-k8s/blob/0d02624a2d97e65ebde0977f608cb129184f9cb5/pkg/ipamd/ipamd.go#L2179). Together with our finding that IP addresses in cooldown are counted towards the number of available addresses. + +This means that these settings only allow allocating more IP addresses if: + +1. All of the IP addresses on the node are assigned to Pods; **and** +2. No Pods on the node were removed in the last 30 seconds + +Setting `WARM_IP_TARGET=1` released many of the IP addresses on our nodes. Setting it back to zero while we continued to have high Pod churn meant that _ipamd_ never saw the necessary conditions to reallocate those IP addresses. This only happened because we had also set `WARM_ENI_TARGET=0`. + +### What did we do to work around errors in the meantime? + +Internally, our control plane maintains a pool of “warm” Compute pods, so that [Pod starts are not on the hot path](https://neon.com/blog/cold-starts-just-got-hot) for waking an idle database. + +We were only seeing ~10% of Pods failing to start, so we were able to compensate for failures by increasing the size of the pool. + +This mitigated the user impact as errors continued behind the scenes. + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxchxzeycqqjnshtpy6p0ohxffjajplesxgiofhlnwiagdyox8f6frngm-5nul5xjswbvwjxt05yzlopjnvegkb6xadgxcmhd4p-hohp38te62rp1pp6ujvxfuvbipv-phxabf-96638155.png) + +### What eventually caused the errors to stop? + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxeseajhnylpn2lqmo0ur3rbflhlizqotdc6ujvi7jn5vqdkky-ciaz5oqlt3jvcrm8lomgvyfftfkxho-pcbp6bs6zdi0g-aqswbq6ktfvzdcjwrxw55c6n5mgij-ewnncbgg-b8e1c626.png) + +Many hours after we’d mitigated user impact, we saw the IP assignment errors suddenly drop to near-zero. + +We weren’t sure why at the time, but in the course of our deeper investigation, we found that this recovery was ironically due to the same issue that initially triggered Friday’s incident: Our control plane stopped shutting down idle computes for 86 seconds, due to an expensive query to the backing Postgres database. (We wrote more about this query and our changes to improve its execution plan in [this related blog post](https://neon.com/blog/delayed-start-compute-operations-triggering-event).) + +![](https://cdn.neonapi.io/public/images/pages/blog/aws-cni-lessons-from-a-production-outage/ad4nxcy4anh3xrndfndjlyf7hvcpmeedzk4ncdnazuztgtdrz8expod3au61-fhknqj53fxtvjhglhieegjnj1b68sr9bsmgt9rxj2vfndkjvummreklqdx59uhjncgevqxbklbapg-80a74bd5.png) + +This brief gap with no Pod deletions meant there were no IP addresses in cooldown, so further compute starts allowed _ipamd_’s conditions for allocating more IP addresses to be satisfied. And indeed, there were simultaneous allocations across the cluster, which in turn reduced the Pod start failure rate. + +## Final thoughts + +This incident resulted in significant downtime for our customers and we were determined to understand the conditions that led to it, so we can prevent it – and incidents like it – from happening again. + +Throughout this investigation our team learned a lot about AWS CNI internals, and we’ve even submitted a pull request to help improve the behavior for others. + +In keeping with our philosophy of learning from incidents, we decided to make the investigation public. We hope that other teams can benefit from what we’ve learned, helping us all move towards a more reliable future. diff --git a/content/blog/posts/aws-privatelink-for-neon-databases.md b/content/blog/posts/aws-privatelink-for-neon-databases.md new file mode 100644 index 0000000000..74e9b2f6f2 --- /dev/null +++ b/content/blog/posts/aws-privatelink-for-neon-databases.md @@ -0,0 +1,71 @@ +--- +title: AWS PrivateLink for Neon Databases +description: Keep your Postgres connections within AWS’s private network +excerpt: >- + We just shipped Neon Private Networking, a feature that lets you connect to + your Neon database through AWS PrivateLink with zero exposure to the public + internet. If your infra is in AWS, this feature makes it much easier to meet + your security and compliance requirements while enj... +date: '2025-03-05T00:02:46' +updatedOn: '2025-08-14T09:27:43' +category: product +categories: + - product + - company +authors: + - anna-stepanyan +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/aws-privatelink-for-neon-databases/cover.jpg + alt: null +isFeatured: true +seo: + title: AWS PrivateLink for Neon Databases - Neon + description: >- + Private Networking is now available on Neon. Connect to your Neon database + through AWS PrivateLink with zero exposure to the public internet. + keywords: [] + noindex: false + ogTitle: AWS PrivateLink for Neon Databases - Neon + ogDescription: >- + Private Networking is now available on Neon. Connect to your Neon database + through AWS PrivateLink with zero exposure to the public internet. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/aws-privatelink-for-neon-databases/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/aws-privatelink-for-neon-databases/neon-aws-privatelink-for-neon-databases-1-1024x576-2721599f.jpg) + +**We just shipped** [Neon Private Networking](https://neon.tech/docs/guides/neon-private-networking), **a feature that lets you connect to your Neon database through [AWS PrivateLink](https://aws.amazon.com/privatelink/) with zero exposure to the public internet.** If your infra is in AWS, this feature makes it much easier to meet your security and compliance requirements while enjoying Neon’s developer experience. Private Networking is available in our [Business and Enterprise](https://neon.tech/pricing) plans. + +## Securing Connectivity Between AWS and Neon + +If your infrastructure runs on AWS, you’re probably familiar with keeping services inside a VPC for security. Companies using Amazon RDS or Aurora typically deploy those databases in private subnets, ensuring that database traffic never leaves AWS’s internal network. + +However, using a managed cloud database service outside your AWS account (like [Neon](https://neon.tech/home)) would mean your application had to connect over the public internet—and even with encryption, sending database queries over the internet can raise security flags. Compliance regulations might also demand guarantees that their data flows are contained within controlled networks, or internal policies might prohibit direct internet access for critical systems. + +## How Neon Private Networking works + +Neon Private Networking solves this problem by integrating Neon with AWS PrivateLink to provide a secure, private endpoint for your database. Instead of connecting to Neon over the internet, your application connects to an endpoint within your AWS environment that bridges directly to Neon: + +1. Neon provides an AWS PrivateLink endpoint service in the same AWS region as your database. +2. You then create a VPC endpoint in your AWS VPC, and link it to Neon +3. Your application then routes all database queries through this private endpoint, where Neon’s isolated proxy forwards traffic securely to your database. + +The entire setup is self-serve and easy to configure, and there’s no code changes required (your database connection string stays the same). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/aws-privatelink-for-neon-databases/ad4nxeumgeqhxu5eim2vxsifjzba28njwvf1nvs8vqrfa-yvglkw8zzyezdarztprq01f0derbhlkc4eyq2y3ijdthd9j-drmtph43nlb8zszt6boof28yabjn9f8lpec8wn7jyu4w-5e10441c.png) + +## How to set it up + +Getting started with Neon Private Networking is straightforward. You’ll need to create a VPC endpoint in your AWS account and link it to Neon’s AWS PrivateLink service. Once set up, your application will automatically route database queries through the private connection—no code changes needed. + +For step-by-step instructions, [check out our Private Networking setup guide.](https://neon.tech/docs/guides/neon-private-networking) + +## Wrap up + +Neon’s Private Networking is available now for all customers on our Business and Enterprise plans. If you have questions, [contact us](https://neon.tech/contact-sales), and we’ll be happy to help. + +--- + +_If you’re new to Neon, [sign up today](https://console.neon.tech/signup) and see how easy it is to run Postgres with built-in security, scalability, and a developer-friendly experience._
diff --git a/content/blog/posts/aws-workshop-neon-dev-rds.md b/content/blog/posts/aws-workshop-neon-dev-rds.md new file mode 100644 index 0000000000..77b28879f7 --- /dev/null +++ b/content/blog/posts/aws-workshop-neon-dev-rds.md @@ -0,0 +1,120 @@ +--- +title: Learn How to Use Neon with AWS RDS to Boost Development Velocity in Postgres +description: AWS Workshop is out +excerpt: >- + AWS workshops are a fantastic free resource created by the AWS team to help + you tackle real-world challenges using AWS infrastructure. As an AWS Partner, + we’ll be launching an AWS workshop to teach you how to use Neon as a + development environment while keeping production workload... +date: "2025-01-29T18:01:36" +updatedOn: "2025-03-16T19:49:28" +category: workflows +categories: + - workflows +authors: + - savannah-longoria +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/aws-workshop-neon-dev-rds/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Learn How to Use Neon with AWS RDS to Boost Development Velocity in Postgres + - Neon + description: >- + In this AWS Workshop, learn to use Neon as a dev environment for AWS RDS, + and improving your workflows without complex production migrations. + keywords: [] + noindex: false + ogTitle: >- + Learn How to Use Neon with AWS RDS to Boost Development Velocity in Postgres + - Neon + ogDescription: >- + In this AWS Workshop, learn to use Neon as a dev environment for AWS RDS, + and improving your workflows without complex production migrations. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/aws-workshop-neon-dev-rds/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/aws-workshop-neon-dev-rds/neon-aws-workshops-1024x576-5cc6169e.jpg) + +[AWS workshops](https://workshops.aws/) are a fantastic free resource created by the AWS team to help you tackle real-world challenges using AWS infrastructure. As an AWS Partner, we’ll be launching an AWS workshop to teach you **how to use Neon as a development environment while keeping production workloads on AWS RDS:** + +
+ +
+ +You can follow the guided workshop here: [https://neon.awsworkshop.io/](https://neon.awsworkshop.io/) + +_Tip: Make sure to login to your AWS account to be able to see the content!_ + +## Why use Neon as a development environment for AWS RDS + +
+

“Neon’s branching paradigm has been great for us. It lets us create isolated environments without having to move huge amounts of data around. This has lightened the load on our ops team, now it’s effortless to spin up entire environments.” (Jonathan Reyes, Principal Engineer at Dispatch)

+
+ +Teams love using [Neon](https://neon.tech/use-cases/dev-test) as a development environment because it reduces operational overhead, boosting productivity. Since it’s 100% Postgres, it integrates perfectly with AWS RDS, giving teams a superior development experience for their development and testing pipelines without forcing them into hairy production migrations. On top of it, using Neon can also help you reduce your monthly bill. + +If you’re an AWS RDS user but have never heard of Neon, here’s how it can help you: + +### Ephemeral environments provisioned instantly + +Neon supports **database branching with data and schema**, which is a game-changer for deploying environments. Instead of manually maintaining development and testing instances and struggling to keep them up to date with production, you can spin up ephemeral database environments in seconds—complete with an up-to-date copy of schema and data. + +### No more issues caused by lack of data consistency + +This agility allows you to **automate environment creation** via CI/CD pipelines. For example, you can: + +- Create ephemeral environments for every pull request. +- Run tests in isolated environments. +- Discard environments as soon as tasks are complete. + +Need to sync an environment with the latest production state? It’s as simple as making one API call in Neon. + +### Easy team collaboration without production risks + +Neon serves as the perfect development sandbox for teams, with each process or developer having their own independent environment. **Each environment is a dedicated database branch in Neon**, complete with its own compute resources, and hundreds of branches can be active in parallel. This setup allows teams to safely experiment, run migrations, or test changes without the risk of disrupting production or interfering with one another’s work. + +### Scale-to-zero saves costs + +Neon environments automatically **scale-to-zero when idle**, pausing to eliminate unnecessary compute costs. When you need them again, they wake up with a fast cold start of under 1 second. + +## What you’ll learn in the AWS RDS and Neon workshop + +
+

“We are using the Neon Twin workflow. We just install the GitHub action and it takes care of the rest. Developers may not know how to dump and restore well, but they know how to run a GitHub Action. It’s amazing” (Alex Co, Head of Platform Engineering at Mindvalley)

+
+ +![Image](https://cdn.neonapi.io/public/images/pages/blog/aws-workshop-neon-dev-rds/ad4nxf6bpj5umoaunoxg5g8kb6vmbxybdgt9gwozcby7tqadm2z7r3dbymetcjozk-owjj-hcxkiipqqdjmecpvx9imgmhcovkidmfoerkbywfv2i28acek3thqwiih2qvdyztk6-a17ac5a4.png) + +You may be wondering how it’s possible to implement this workflow without migrating from AWS RDS. That’s exactly what we’ll teach you how to do in the workshop. + +We’ll cover all of this and more: + +#### Introduction to Neon as a complement to RDS + +For those of you unfamiliar with Neon, we’ll kick off the workshop by introducing you to its key features, focusing on how it enhances development workflows. You’ll also learn how to configure your first Neon project for integration with RDS. + +#### How to set up a synchronized copy of your RDS database in Neon (a Neon Twin) + +A Neon Twin is an isolated, up-to-date replica of your RDS database, designed specifically for development and testing in Neon. You’ll learn how to automate the creation of this synchronized copy using tools like pg_dump and pg_restore. + +#### Automate synchronization of environments with production + +To ensure your Neon Twin stays current with your production RDS database, we’ll teach you how to automate synchronization workflows. Using GitHub Actions, you’ll configure nightly backups to keep your development environments consistently in sync, without manual effort. + +#### Set up environments as database branches + +Once your Neon Twin is ready, you’ll learn how to use Neon’s database branching to create multiple isolated environments. This allows for parallel development, enabling team members or CI/CD pipelines to work independently without impacting the main database or each other’s branches. + +#### Best practices + +We’ll wrap up the workshop by sharing best practices for setting up efficient workflows. You’ll learn tricks for parallel development and how to keep your environments responsive and cost-effective. + +## Get started + +The workshop is live here: [https://neon.awsworkshop.io/](https://neon.awsworkshop.io/) + +We’re planning to expand it with many more scenarios. If there’s something you’d like to see covered, [tell us.](https://neon.tech/contact-sales) diff --git a/content/blog/posts/azure-native-integration-ga.md b/content/blog/posts/azure-native-integration-ga.md new file mode 100644 index 0000000000..732b55072f --- /dev/null +++ b/content/blog/posts/azure-native-integration-ga.md @@ -0,0 +1,124 @@ +--- +title: Neon’s Microsoft Azure Native Integration is Generally Available +description: >- + Deploy and manage Neon Serverless Postgres as a deeply integrated Azure-native + service +excerpt: >- + Neon’s Microsoft Azure Native Integration has reached General Availability, + providing developers with a powerful, serverless Postgres solution on Azure. + Neon integrates with your existing Azure workflows, including unified billing, + Microsoft single-sign on (SSO), and full MACC el... +date: '2025-05-07T18:01:03' +updatedOn: '2025-08-14T09:25:40' +category: product +categories: + - product + - company +authors: + - monica-steinke +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/azure-native-integration-ga/cover.jpg + alt: null +isFeatured: true +seo: + title: Neon’s Microsoft Azure Native Integration is Generally Available - Neon + description: >- + Neon’s Microsoft Azure Native Integration has reached General Availability: + developers have a powerful, serverless Postgres solution on Azure. + keywords: [] + noindex: false + ogTitle: Neon’s Microsoft Azure Native Integration is Generally Available - Neon + ogDescription: >- + Neon’s Microsoft Azure Native Integration has reached General Availability: + developers have a powerful, serverless Postgres solution on Azure. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/azure-native-integration-ga/social.jpg +--- + +**Neon’s Microsoft Azure Native Integration has reached General Availability,** providing developers with a powerful, serverless Postgres solution on Azure. Neon integrates with your existing Azure workflows, including unified billing, [Microsoft single-sign on (SSO)](https://learn.microsoft.com/en-us/azure/partner-solutions/neon/manage), and full [MACC eligibility](https://neon.tech/docs/introduction/billing-azure-marketplace#microsoft-azure-consumption-commitment-macc). Whether you’re building new apps or scaling global workloads, Neon brings the power of serverless Postgres to your Azure environment with zero friction. + + + + +Neon is working with Microsoft to bring serverless Postgres to the Azure ecosystem. [Review our original announcement](https://neon.tech/blog/neon-is-coming-to-azure) for a refresher. + + +## What Is the Neon Azure Native Integration? + +[Neon’s Azure Native Integration](https://learn.microsoft.com/en-us/azure/partner-solutions/neon/overview) lets you use Neon like any other native Azure service, fully embedded within your Azure workflows, identity systems, and billing structure. + +You can provision Neon Serverless Postgres directly from the [Azure portal](https://portal.azure.com/), where it appears alongside Microsoft’s own database offerings in the Databases section, complete with a partner badge: + +![Post image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXf24hOiRAfyuyTBMGLJERRXPX2MkfsBEx_AR-0lLgaFgwDX2xT1P5srwboUAy3i4MQYUOKC9ELgxFuc92624S84f8ASXS1JjNm8SnJDG7UQs7yK3Y_n3bih2fbFEjd3BBfV-POxRg?key=Lvjl-lrKGNJJhzj3YImvjvTa) + +Once provisioned, you can manage Neon projects, branches, and connection strings using the Azure Command-Line Interface (CLI) and SDK. + +This deep integration also simplifies security and procurement: + +- Authenticate using Microsoft Entra ID (formerly Azure Active Directory) +- Consolidate Neon usage into your Azure bill—no separate account or payment required + +## What’s New with General Availability? + +While in preview, developers were able to create a Neon organization through Azure; now, with general availability, they can also manage Neon projects, branches, and connection strings directly through Azure. + +![Post image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXf8ZnjusHUNvMorjd__Pa_x0enV7AR_HYLKHnSqBKhYoNFszWZ9pOeH--mMhfeEtDQjrv1jO4pB1_POjFKryihVEMsMs5vALwnCVbZ7lcCAVW6zHpKpH5e48SdSHuCcb6qAGYlcvw?key=Lvjl-lrKGNJJhzj3YImvjvTa) + +This update makes it easier to operationalize Neon across your team, enabling tighter CI/CD integration, more granular environment management, and a faster path from provisioning to production. + +And while we’re excited about the improvements available today, we’re already working on our next enhancement, designed to further simplify identity and access management, making your workflow even more streamlined within the Azure ecosystem. + +## Who Benefits From the Neon Azure Native Integration? + +### Enterprises with Azure MACC + +If your organization has a [Microsoft Azure Consumption Commitment (MACC)](https://learn.microsoft.com/en-us/marketplace/microsoft-commercial-marketplace-benefits#azure-consumption-commitment), Neon is now an eligible partner. Any Neon usage through the Azure Marketplace contributes toward your committed Azure spend, with no additional procurement steps or vendor onboarding required. This makes it simple for enterprises to adopt serverless Postgres within their existing Azure contracts and governance structures. + +#### **Why Neon** for Enterprise + +[Enterprises choose Neon](https://neon.tech/enterprise) due to its efficient scalability, strong recovery guarantees, and developer workflows that align with modern software development practices: + +- **Efficient scalability**: [Neon’s compute layer scales automatically based on workload](https://neon.tech/docs/introduction/autoscaling), including CPU, memory, and active connections. There’s no need to overprovision resources or worry about connection limits. The benefits are smoother performance during peak traffic, lower infrastructure costs, and reduced operational overhead for engineering teams. +- **Reliable recovery options**: Neon also enables [instant restores](https://neon.tech/docs/introduction/branch-restore), even for multi-terabyte databases. This significantly reduces downtime in the event of failures, providing a [strong safety net for production environments](https://neon.tech/blog/the-true-cost-of-slow-postgres-restores). Every database on Neon also includes [high availability by default,](https://neon.tech/docs/introduction/high-availability) with no need to manage replicas or monitor replication lag. +- **Smoother development workflows**: Neon’s [database branching](https://neon.tech/docs/introduction/branching) makes it easy to create isolated, production-like environments for testing, CI/CD, and feature development. Teams can spin up branches in seconds, iterate quickly, and [launch with confidence](https://neon.tech/blog/how-mindvalley-minimizes-time-to-launch-with-neon-branches). And with scale-to-zero, idle dev and staging environments incur no cost, keeping budgets lean across the SDLC. + +All of this is backed by Neon’s commitment to [enterprise-grade security and compliance.](https://neon.tech/security) We follow industry best practices for encryption, access control, and data protection. + + +If you’re just discovering Neon, [schedule a meeting with our team](https://neon.tech/contact-sales). Our Solutions Engineers are ready to review your requirements and help you evaluate whether Neon is the right fit. + + +### AI Startups scaling on Azure + +If you’re a startup, especially if part of the [Microsoft for Startups program](https://www.microsoft.com/startups) and embracing the AI wave, the Neon Azure Native Integration is also a perfect fit. Neon’s serverless Postgres architecture delivers the speed, flexibility, and automation that modern AI applications demand. Here’s how AI startups are building on Neon: + +### AI agents powered by Neon + +[Replit Agent and Create.xyz are using Neon as a foundation for agentic app creation](https://x.com/nikitabase/status/1837138637516931252). These AI agents automatically spin up Neon databases in response to user input, whether it’s a new AI workspace, a micro SaaS, or a proof-of-concept app. + +Neon’s instant provisioning, fully managed storage, and [powerful API](https://neon.tech/blog/provision-postgres-neon-api) enable agents to deploy Postgres completely autonomously. No manual setup, no disk provisioning, no credential management. Neon handles it all in the background, so the agent can stay focused on the user experience. + +### RAG pipelines with pgvector + +Neon is also an ideal vector store for Retrieval-Augmented Generation (RAG) workflows. Startups are building pipelines that generate embeddings with Azure OpenAI Service and store them in Neon using the pgvector extension, unlocking fast, SQL-native semantic search inside a fully managed Postgres environment. + +There’s no need for a separate vector database or added infrastructure. With Neon, you get native vector search, seamless scaling, and the simplicity of standard Postgres tooling to insert, index, and retrieve vectors efficiently. + +And if you’re building with Microsoft-native tools, [Neon integrates directly with Semantic Kernel](https://github.com/microsoft/semantic-kernel/tree/main/samples/apps/NeonMemory). Using the Postgres memory connector, you can orchestrate a full RAG pipeline: + +- Generate embeddings with Azure OpenAI Service +- Store and query them in Neon with pgvector +- Retrieve relevant results in milliseconds, all with SQL + +## A Big Milestone, But We’re Just Getting Started + +This release is the result of close collaboration between Neon and Microsoft engineering teams. With a roadmap and ongoing co-investment, we’re working to make Neon truly native to the Microsoft developer ecosystem. + +Whether you’re an individual developer or part of an enterprise engineering team, log in to the Azure Marketplace and [try Neon’s Native Integration today.](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/neon1722366567200.neon_serverless_postgres_azure_prod?tab=overview) + + +If you’re attending [Microsoft Build 2025](https://build.microsoft.com/en-US/home), come find us at booth 518. We’d love to meet you and hear your feedback on how we can build the best serverless Postgres experience on Azure! + diff --git a/content/blog/posts/behind-modem-dev-product-velocity.md b/content/blog/posts/behind-modem-dev-product-velocity.md new file mode 100644 index 0000000000..ea1b65d1be --- /dev/null +++ b/content/blog/posts/behind-modem-dev-product-velocity.md @@ -0,0 +1,133 @@ +--- +title: Behind Modem.dev’s Product Velocity +description: >- + Their agent centralizes scattered user feedback and automates follow-ups. + Powered by Neon branching +excerpt: >- + “Neon branching fundamentally accelerates our developer experience. It’s a + huge reason we’re able to ship faster without worrying about breaking things” + (Ben Vinegar, Co-founder at Modem) If you’re a PM or even an engineer at a + small startup, this will hit home: This is the pain... +date: '2026-01-29T17:31:27' +updatedOn: '2026-02-02T17:13:07' +category: ai +categories: + - ai + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/behind-modem-dev-product-velocity/cover.jpg + alt: null +isFeatured: false +seo: + title: Behind Modem.dev’s Product Velocity - Neon + description: >- + Modem is building an auto-triage, PM agent that centralizes scattered user + feedback and automates follow-ups. Powered by Neon branching. + keywords: [] + noindex: false + ogTitle: Behind Modem.dev’s Product Velocity - Neon + ogDescription: >- + Modem is building an auto-triage, PM agent that centralizes scattered user + feedback and automates follow-ups. Powered by Neon branching. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/behind-modem-dev-product-velocity/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/behind-modem-dev-product-velocity/neon-modem-devs-1-1024x576-0f7c9eb0.jpg) + +
+

“Neon branching fundamentally accelerates our developer experience. It’s a huge reason we’re able to ship faster without worrying about breaking things”



(Ben Vinegar, Co-founder at Modem)

+
+ +If you’re a PM or even an engineer at a small startup, this will hit home: + +- You’re getting user feedback, but it’s is everywhere and nowhere at the same time +- Bug reports get buried in obscure Slack channels +- Feature requests show up in customer conversations but never quite turn into an action item +- Things slip through because of the pain of keeping track of it all, and when it’s time to consolidate users’ feedback and requests to inform prioritization, there’s no clear place to look + +This is the pain that [Modem](https://modem.dev/) is set to solve. + +
+ +
Check them out at modem.dev
+
+ +## From Noisy Conversations to Clear Product Signals + +The team at Modem is building an auto-triage, AI PM – an agent that listens across all places where work happens, identifies feedback and feature requests, and turns them into organized, actionable feedback. Over time, the agent builds an internal knowledge base: when you need to know what your users are saying, you can simply ask. + + + +The idea is to help teams with: + +- **Centralizing feedback.** Modem connects to tools like Slack, Discord, GitHub, and everywhere where your discussions are happening, pulling in real user conversations and digesting the conclusions into an easy-to-consume format. +- **Clustering raw conversations into product-level topics.** The agent groups related messages into distinct topics and categorizes them (e.g. bugs, feature requests, positive signals). +- **Preserving context.** Each topic retains links back to the original conversations, making it easier to understand why something matters for prioritization. +- **Automating follow-ups and next steps.** Modem can also trigger direct follow-ups with users, create tickets, and kick off workflows. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/behind-modem-dev-product-velocity/full-ui-example-aaaf0e44.svg) + + + +## Turning a Postgres Database into a Development Environment + +When Modem first set up its database, the goal was simple: set up hosted Postgres and keep moving. They were looking for a database that stayed out of the way and thought [Neon](https://neon.com/) fit the bill. Neon did deliver on a smooth experience, but the Modem team also found something much more powerful – a development environment built around [branching](https://neon.com/branching). + +Neon’s branching quickly became a core part of how Modem ships software. Rather than treating the database as a scary, monolithic resource, the team started treating it the same way they treat code – something they can safely branch, test, diff, and discard. + +### Branch-per-PR: de-risking database migrations + +The first place Modem applied branching was database migrations. This is one of the scariest parts of shipping quickly, especially when you’re heavily relying on AI coding to accelerate development speed (and who isn’t these days). + +To avoid surprises, Modem put the following system in place: + +- Every PR that includes a migration [triggers the creation of its own Neon branch](https://neon.com/docs/guides/neon-github-integration) +- The migration is applied to that branch, not to production +- Neon computes and surfaces a [schema diff,](https://neon.com/docs/guides/schema-diff) showing exactly what the migration would change [via a comment on the PR](https://github.com/marketplace/actions/neon-schema-diff-github-action) +- The team reviews it to confirm the migration applies cleanly + +### Re-checking migrations at merge time + +At merge time, Modem runs the same process again. Because the main branch may have advanced (with new commits or migrations landing in parallel) Modem spins up afresh branch from the latest state, applies the migration, and validates it will work on the current production database. Only if that succeeds does the migration get applied to production. + +
+

“Conflicting migrations have pretty much disappeared from our list of daily worries” 



(Ben Vinegar, Co-founder at Modem)

+
+ +### Branch-per-preview + +[Vercel previews](https://vercel.com/docs/deployments/environments) became essential for Modem’s evaluation and QA, especially once the missing piece (the database) was added to the picture. + +Every Vercel preview deployment now runs against its own Neon database branch. This ensures that migrations introduced in a PR are visible in the preview environment. Previews stay realistic, while production remains completely untouched. All these branches are ephemeral by the way, with [expiration times](https://neon.com/docs/guides/branch-expiration) and [automatic cleanups](https://neon.com/blog/big-dx-improvements-for-neon-users-on-vercel#automatic-branch-cleanup) set up. + +## Tips From the Trenches + +### Combining branching with coding agents + +Like most teams these days, [Modem engineers constantly experiment with agentic coding](https://www.youtube.com/watch?v=xoynR-hWNZY). Many of their PRs are actually opened by agents working through Claude Code or internal workflows triggered from Slack and Linear, so preview environments are among the primary way changes get evaluated. + +To make sure this works smoothly, rather than relying on Neon’s [one-click integrations](https://neon.com/docs/guides/integrations), Modem favors explicit, scriptable workflows built on the [Neon CLI](https://neon.com/docs/reference/neon-cli) and automation so that agents can reason about infrastructure, with branch creation, migration application, and cleanup are all defined in code. + +### Navigating short-lived tokens + +When branching becomes a first-class part of your workflow, some interesting edge cases come up, especially around integrations that rely on short-lived or refresh tokens (like Slack or Linear). + +In a preview environment, it’s possible to refresh a token and persist it to a database branch that will later be deleted. If that refreshed token becomes the “latest” token from the integration’s point of view, production can suddenly find itself holding an invalid one. + +If you’ve also encountered this issue, Modem solved it by tagging refresh tokens with the database branch that produced them. Token refreshes only occur if the current database matcheswhich makes it impossible for one environment to invalidate another by accident. + +## Start Building Faster (and Safer) + +Modem is relying on Neon branching to move quickly without breaking things. If this sounds good for your team’s velocity as well, start building [your first Neon branching workflows](https://neon.com/blog/practical-guide-to-database-branching) on the [Free plan](https://console.neon.tech/signup) and see how far you can go. + + +Keep an eye on Modem! Join Modem’s waitlist at [https://modem.dev/](https://modem.dev/) to get notified when they launch. + diff --git a/content/blog/posts/better-postgres-with-prisma-experience.md b/content/blog/posts/better-postgres-with-prisma-experience.md new file mode 100644 index 0000000000..9dffffe602 --- /dev/null +++ b/content/blog/posts/better-postgres-with-prisma-experience.md @@ -0,0 +1,157 @@ +--- +title: Better Postgres with Prisma Experience +description: One URL to run SQL migrations and scale your applications +excerpt: >- + We’re Neon. We’re building Postgres that helps you confidently ship reliable + and scalable apps. We made Postgres on Neon work seamlessly with Prisma. This + article explains how we did it. We love Prisma, and so do developers. Prisma + ORM makes it easy to perform schema migrations a... +date: '2024-03-07T14:17:20' +updatedOn: '2024-03-07T15:19:32' +category: community +categories: + - community +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/better-postgres-with-prisma-experience/cover.png + alt: null +isFeatured: false +seo: + title: Better Postgres with Prisma Experience - Neon + description: One URL to run SQL migrations and scale your applications + keywords: [] + noindex: false + ogTitle: Better Postgres with Prisma Experience - Neon + ogDescription: >- + We’re Neon. We’re building Postgres that helps you confidently ship reliable + and scalable apps. We made Postgres on Neon work seamlessly with Prisma. + This article explains how we did it. We love Prisma, and so do developers. + Prisma ORM makes it easy to perform schema migrations and map any database + objects with your existing JavaScript […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/better-postgres-with-prisma-experience/social.png +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/better-postgres-with-prisma-experience/image-1024x576-3a07a608.png) + +**We’re Neon. We’re building Postgres that helps you confidently ship reliable and scalable apps. We made Postgres on Neon work seamlessly with Prisma. This article explains how we did it.** + +We love [Prisma](https://www.prisma.io/docs/orm/overview/databases/postgresql), and so do developers. [Prisma ORM](https://www.prisma.io/docs/orm/overview/databases/postgresql) makes it easy to perform schema migrations and map _any_ database objects with your existing JavaScript and TypeScript applications, allowing you to integrate type-safe queries into your codebase. + +Today, we’re pleased to share significant improvements to the developer experience of Neon using Prisma by adding support to schema migrations via pooled connections, making it possible to use Neon’s default connection string to scale your serverless apps and run schema migrations. + +[You can start using Prisma with Neon for free.](https://console.neon.tech) + +For reference, when we first introduced Neon, Prisma users needed a direct database URL, a pooled database URL, and a shadow database URL. Here how your `prisma.schema` file looked like: + +```typescript +// Initially +datasource db { + provider = "postgresql" + url = postgres://sally:@ep-throbbing-boat-918849-pooler.us-east-2.aws.neon.tech/neondb?pgbouncer=true + directUrl = postgres://sally:@ep-throbbing-boat-918849.us-east-2.aws.neon.tech/neondb + shadowDatabaseUrl = postgres://sally:@ep-throbbing-boat-918849.us-east-2.aws.neon.tech/shadowdb +} +``` + +Now, all you need is one database URL. As a bonus, we also removed the need to specify the query parameter `pgbouncer=true` when using pooled connections: + +```typescript +// Now +datasource db { + provider = "postgresql" + url = postgres://sally:@ep-throbbing-boat-918849-pooler.us-east-2.aws.neon.tech/neondb +} +``` + +This article discusses each step of the process and the changes made to Neon, PgBouncer, and Prisma to make this possible, including: + +1. Schema migration support with pooled connections +2. Dropping a shadow database WITH (FORCE) + +## Schema migration support with pooled connections + +In enhancing the experience with Prisma, we [added support for prepared statements](https://github.com/pgbouncer/pgbouncer/releases/tag/pgbouncer_1_21_0) and [`DISCARD ALL/DEALLOCATE ALL` to PgBouncer](https://github.com/pgbouncer/pgbouncer/releases/tag/pgbouncer_1_22_0) to allow for schema migration using pooled connections. Let’s explore why. + +In Postgres, each connection is a backend process that requires memory allocation, which limits the number of concurrent connections. The solution to this problem is connection pooling with PgBouncer, which helps keep the number of active backend processes low. + +PgBouncer becomes increasingly important at scale when using serverless services such as [AWS Lambda](https://aws.amazon.com/pm/lambda/) or [Vercel functions](https://vercel.com/docs/functions), since each function call establishes a new connection. We name database connections that use PgBouncer pooled connections. + +Additionally, [`prisma migrate`](https://www.prisma.io/docs/orm/prisma-migrate/getting-started) uses prepared statements to optimize SQL query performance, and [`DEALLOCATE ALL`](https://github.com/prisma/prisma-engines/blob/4308b705cc0694626ff407996f3145ddef0ad1c6/quaint/src/connector/postgres/native/mod.rs#L507) to release all prepared statements in the current session [before preparing and executing Prisma Client queries](https://www.prisma.io/docs/orm/prisma-client/setup-and-configuration/databases-connections/pgbouncer#add-pgbouncertrue-to-the-connection-url). More on prepared statements in the [PgBouncer 1.22.0 support announcement article](https://neon.tech/blog/pgbouncer-the-one-with-prepared-statements#what-are-prepared-statements). + +Before version 1.22.0, if you attempted to run `prisma migrate` commands using a pooled connection, you might have seen the following error: + +```bash +Error: undefined: Database error +Error querying the database: db error: ERROR: prepared statement "s0" already exists +``` + +To scale using pooled connections and be able to perform schema migrations, you had to specify both pooled and direct database URLs and set `pgbouncer` mode as a query parameter in your schema file. + +Here is how the `datasource db` block in your `prisma.schema` file looked like: + +```typescript +// Before PgBouncer 1.22.0 and Prisma Client 5.10.0 +datasource db { + provider = "postgresql" + url = postgres://sally:@ep-throbbing-boat-918849-pooler.us-east-2.aws.neon.tech/neondb?pgbouncer=true + directUrl = postgres://sally:@ep-throbbing-boat-918849.us-east-2.aws.neon.tech/neondb +} +``` + +Here is how it looks now after adding support for prepared statements and `DISCARD ALL/DEALLOCATE ALL` to PgBouncer: + +```typescript +// With PgBouncer 1.22.0 and Prisma Client 5.10.0 +datasource db { + provider = "postgresql" + url = postgres://sally:@ep-throbbing-boat-918849-pooler.us-east-2.aws.neon.tech/neondb +} +``` + +Note you only need the pooled connection to run Prisma with Postgres. It’s no longer required to specify a direct connection to the database and the `pgbouncer=true` query parameter. The pooled connection is used to scale your queries and run schema migrations. + +This allows Neon to confidently set the default URL to the pooled connection string on the Console and the [Vercel Integration](https://neon.tech/docs/changelog/2024-02-23-console#neon-vercel-integration-improvements). + +## DROP shadow database WITH (FORCE) + +When you run `prisma migrate dev`, Prisma Migrate uses a shadow database to detect schema drifts and generate new migrations. During that process, Prisma creates, introspects, and then drops a shadow database. [More on shadow databases on Prisma’s documentation.](https://www.prisma.io/docs/orm/prisma-migrate/understanding-prisma-migrate/shadow-database) + +However, certain cloud providers do not allow to drop and create databases via SQL, which forces developers to manually create shadow databases and specify them in the `prisma.schema` file: + +```typescript +datasource db { + provider = "postgresql" + url = postgres://sally:@ep-throbbing-boat-918849.us-east-2.aws.neon.tech/neondb + shadowDatabaseUrl = postgres://sally:@ep-throbbing-boat-918849.us-east-2.aws.neon.tech/shadowdb +} +``` + +We have added [support for managing roles and databases via SQL](https://neon.tech/blog/prisma-dx-improvements#removing-the-need-for-manually-creating-the-shadow-database) on Neon, which allowed for removing the need for manually creating a shadow database. Additionally, [Prisma 5.10.0](https://github.com/prisma/prisma/releases/tag/5.10.0) [introduces support for `DROP WITH (FORCE)`](https://github.com/prisma/prisma-engines/pull/4722) as an alternative drop database path in the schema engine, which allows it to dispose of shadow databases. + +So, in your `schema.prisma` file, you would have: + +```typescript +datasource db { + provider = "postgresql" + url = postgres://sally:@ep-throbbing-boat-918849.us-east-2.aws.neon.tech/neondb +} +``` + +## Conclusion + +The improvements included in PgBouncer 1.22.0 have significantly streamlined the experience for developers using Postgres on Neon and Prisma, making is more efficient to scale serverless applications and run schema migrations. + +We would love to get your feedback. Follow us on [X](https://x.com/neondatabase), join us on [Discord](https://neon.tech/discord) and let us know how we can help you build the next generation of web applications. + +Shout out to all contributors for making this possible, including: + +- [Jelte Fennema](https://github.com/JelteF) +- [Martijn Dashorst](https://github.com/dashorst) +- [Konstantin Knizhnik](https://github.com/knizhnik) +- [Sverre Boschman](https://github.com/sboschman) +- [Frank Schmager](https://github.com/fschmager) +- [Oleg Tselebrovskiy](https://github.com/Medvecrab) +- [Alex Chi Z](https://github.com/skyzh) diff --git a/content/blog/posts/big-dx-improvements-for-neon-users-on-vercel.md b/content/blog/posts/big-dx-improvements-for-neon-users-on-vercel.md new file mode 100644 index 0000000000..f24e7806b3 --- /dev/null +++ b/content/blog/posts/big-dx-improvements-for-neon-users-on-vercel.md @@ -0,0 +1,83 @@ +--- +title: Big DX Improvements for Neon Users on Vercel +description: 'Now with automatic branch cleanup, team sync, and project transfers' +excerpt: >- + Good news for Neon users on Vercel. We’ve shipped a few highly requested + updates to the integration, plus a round of behind-the-scenes reliability and + observability upgrades. Automatic Branch Cleanup Until now, Vercel-managed + projects didn’t automatically clean up their database... +date: '2025-11-18T17:42:20' +updatedOn: '2025-11-18T18:07:21' +category: product +categories: + - product +authors: + - gustavo-salomao +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/big-dx-improvements-for-neon-users-on-vercel/cover.jpg + alt: null +isFeatured: true +seo: + title: Big DX Improvements for Neon Users on Vercel - Neon + description: >- + Users managing Neon via Vercel now have automatic branch cleanup, synced + teams, and smarter project management. + keywords: [] + noindex: false + ogTitle: Big DX Improvements for Neon Users on Vercel - Neon + ogDescription: >- + Users managing Neon via Vercel now have automatic branch cleanup, synced + teams, and smarter project management. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/big-dx-improvements-for-neon-users-on-vercel/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/big-dx-improvements-for-neon-users-on-vercel/neon-vercel-1-1024x576-46519dcb.jpg) + +Good news for [Neon users on Vercel.](https://vercel.com/marketplace/neon) We’ve shipped a few highly requested updates to the integration, plus a round of behind-the-scenes reliability and observability upgrades. + + +Neon offers two types of Vercel integrations:
- [Vercel-Managed Integration (Native Integration via the Vercel Marketplace)](https://neon.com/docs/guides/vercel-managed-integration): You can add Neon to your Vercel project directly via the [Vercel Marketplace](https://vercel.com/marketplace/neon)**. This is the integration we’re focusing on in this post.**
- [Neon-Managed Integration (Connected Accounts):](https://neon.com/docs/guides/neon-managed-vercel-integration) This is for teams that prefer to keep billing on the Neon side, but still want to connect their Neon projects to Vercel (e.g. to create a branch per preview). + + +## Automatic Branch Cleanup + +Until now, Vercel-managed projects didn’t automatically clean up their database branches after preview deployments expired, leaving developers with long lists of inactive branches to remove manually. This update automates that process entirely, keeping your Neon project tidy and your storage usage lean. + + + + +[The Neon-managed integration](https://neon.com/docs/guides/neon-managed-vercel-integration) already supported automatic branch deletion based on Git branch cleanup. Now, the same convenience is also present in the Vercel-managed route. + + +When you deploy from Vercel, each Git branch creates a corresponding Neon branch. That Neon branch stays active as long as there’s at least one deployment for that Git branch in Vercel. + +- When multiple deployments exist for the same Git branch, Neon uses a single database branch for all of them. +- When the last deployment for that branch is deleted, whether manually or automatically via [Vercel’s Deployment Retention Policy,](https://vercel.com/docs/deployment-retention) Neon detects it, and deletes the corresponding branch automatically. + +## Team Member Sync + +When a user’s role changes or they’re removed from a Vercel team, those changes now propagate instantly to the corresponding Neon organization – a big security upgrade. Before this fix, users who left a Vercel team could still appear as members in Neon, generating confusion (and plenty of support tickets). Now, Neon continuously monitors for membership updates, ensuring both platforms stay perfectly aligned. + +- When a team member’s role changes in Vercel, Neon updates their access level in the linked organization to match. +- Vercel’s roles don’t map one-to-one with Neon’s, so they’re grouped into two categories: + - Admin: most Vercel roles (like Owner, Developer, Member) map to the Admin role in Neon + - Member: read-only Vercel roles (like Billing or Viewer) map to Member in Neon. +- When a user is removed from a Vercel team, Neon automatically removes them from the corresponding Neon organization. + +## Project Transfers + +When a Vercel project connected to Neon is moved between Vercel teams, the linked Neon project now transfers automatically as well. This avoids the work of re-linking integrations when a project changes ownership inside Vercel. This is a nice upgrade for larger teams reorganizing projects as they grow or restructure. + +## Asynchronous Webhook Handling + +The last update isn’t visible for users, but it makes a big difference behind the scenes for us. Neon’s webhook processing for the Vercel Marketplace integration is now asynchronous, improving reliability and observability across the entire workflow. + +Before this update, webhooks were handled synchronously, e.g. any delay or transient issue during processing could cause the webhook request to fail and trigger retries on Vercel’s side. That sometimes led to duplicate operations and unnecessary noise in the logs. This has also given us complete traceability across API controllers, webhook handlers, and background workers. + +## Set it Up + +Get started by [adding a free Neon Storage to your Vercel project](https://vercel.com/marketplace/neon) directly from the Vercel dashboard. Then [follow these steps](https://neon.com/docs/guides/vercel-managed-integration) to connect both accounts and start building. diff --git a/content/blog/posts/bitso-branching-workflow.md b/content/blog/posts/bitso-branching-workflow.md new file mode 100644 index 0000000000..ff74d3f062 --- /dev/null +++ b/content/blog/posts/bitso-branching-workflow.md @@ -0,0 +1,106 @@ +--- +title: Inside Bitso’s Branch-Based Workflow +description: Database branching gives every engineer a production-like sandbox +excerpt: >- + “Neon’s branching gave us the last missing piece in our RISE (Robust Isolated + Staging Environment): true database isolation. The services that touched + schema changes or write-heavy paths could never share a database safely. Now + every sandbox gets its own isolated Postgres DB when... +date: '2026-01-07T17:01:55' +updatedOn: '2026-01-07T17:55:56' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bitso-branching-workflow/cover.jpg + alt: null +isFeatured: true +seo: + title: Inside Bitso’s Branch-Based Workflow - Neon + description: >- + Bitso uses Neon branching to give 250+ engineers an isolated, + production-like sandbox, eliminating staging bottlenecks at scale. + keywords: [] + noindex: false + ogTitle: Inside Bitso’s Branch-Based Workflow - Neon + ogDescription: >- + Bitso uses Neon branching to give 250+ engineers an isolated, + production-like sandbox, eliminating staging bottlenecks at scale. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bitso-branching-workflow/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bitso-branching-workflow/screenshot-2026-01-07-at-95244-am-1024x572-1861da62.png) + +
+

“Neon’s branching gave us the last missing piece in our RISE (Robust Isolated Staging Environment): true database isolation. The services that touched schema changes or write-heavy paths could never share a database safely. Now every sandbox gets its own isolated Postgres DB whenever required”



(Joe Horsnell, Principal Platform Engineer at Bitso)

+
+ +[Bitso](https://bitso.com/), leader in digital financial services in Latin America, runs a large engineering organization with hundreds of developers and microservices, and a mission-critical system that processes millions of transactions. As the team grew, testing changes safely became increasingly difficult, especially when multiple services and data models needed to evolve at the same time. + +## Scaling Development Across 250+ Engineers + +Like many teams, Bitso first relied on a single shared development environment and a single shared staging environment. Once hundreds of engineers had to push changes simultaneously, these environments quickly became a bottleneck, with dependencies changing minute by minute and faulty deployments breaking the workflow for everyone else. + +Their architecture made this even harder. Bitso’s platform is built on hundreds of Java microservices communicating asynchronously through Kafka, making it impossible for a developer to reproduce the entire stack locally. + +The shared environments became the only place to test changes end-to-end, but they became quite unreliable. Developers often tested a collection of changes from multiple engineers in staging, then deployed a different build to production, meaning the artifact that reached customers had never truly been tested in a clean environment. + +## Step 1: Solve Service Isolation via Signadot + +Bitso’s first step toward a more developer-friendly environment was adopting [Signadot](https://www.signadot.com/?utm_source=partner&utm_medium=partner_blog&utm_campaign=q1_26_partner_marketing). With Signadot sandboxes, engineers can automatically route requests for specific services to isolated versions of those services based on their pull requests, eliminating collisions at the compute and service layer. + + +The Signadot team published a walkthrough of Bitso’s service-isolation architecture, including how sandboxes are provisioned and how traffic is routed. You can [read it here](https://www.signadot.com/blog/how-bitso-is-scaling-branch-based-development-with-signadot-and-neon?utm_source=partner&utm_medium=partner_blog&utm_campaign=q1_26_partner_marketing). + + +Even after solving service-level isolation, one major dependency remained shared across all sandboxes: the database. + +## Step 2: Solve State Isolation with Neon + +For roughly half of Bitso’s services, sharing a database wasn’t an issue, as these services were read-heavy or interacted with tables that rarely changed. But for the long tail of services that required + +- schema changes +- write-heavy testing +- destructive migrations +- or validation against real production-like data, + +a single shared database made true isolation impossible. To fully unlock sandboxed development, Bitso needed a way to give each engineer (or each PR) their own isolated, production-like Postgres instance without incurring massive operational overhead, so they turned to [Neon’s branching](https://neon.com/blog/practical-guide-to-database-branching). + +## Production-Like Databases for Every Sandbox + +[Branching](https://neon.com/docs/introduction/branching) in [Neon](https://neon.com/) offers a way to create new Postgres environments [instantly by taking a point-in-time copy of an existing database](https://neon.com/blog/instantly-copy-tb-size-datasets-the-magic-of-copy-on-write), without duplicating data or provisioning new infrastructure. Branches behave like independent databases, but they share the same underlying storage using a copy-on-write model, making them fast to create and inexpensive to run. + +For Bitso, branching provided exactly what was missing in their workflow: + +- **Each Signadot sandbox can be paired with its own Neon branch,** giving developers a clean environment without affecting the baseline staging data or other engineers. +- **Engineers validate logic against real-world scenarios and edge cases,** since branches inherit the exact state of the parent environment at creation time. +- **Engineers can run destructive testing and migrations with zero risk to anyone else.** When they’re done, branches simply scale back down to zero or are removed entirely via [workflows that can be automated.](https://neon.com/docs/guides/branch-expiration) +- **Every layer is serverless.** Neon’s serverless architecture means branches start fast, [scale automatically](https://neon.com/docs/introduction/autoscaling), and [incur no cost when idle](https://neon.com/docs/introduction/scale-to-zero), perfectly matching Signadot’s short-lived sandbox model. + +## Bitso’s RISE Architecture + +This combination of service isolation through Signadot and state isolation through Neon is what allowed Bitso to build what they call RISE: a Robust Isolated Staging Environment, or a development workflow where every change is tested in a production-like environment, entirely in isolation, before being promoted. + +At the heart of RISE is the idea that **every change should be tested in its own environment, and the exact artifact that passes those tests should be the artifact deployed to production.** Signadot handles the routing: requests meant for a developer’s modified services are sent to that engineer’s sandbox, while all other requests continue hitting the baseline environment. Neon handles the state: each sandbox can be paired with its own Postgres branch, ensuring the database layer reflects the same level of isolation. + +The result is a workflow where + +- every engineer gets a production-like environment by default +- changes are tested independently and safely +- deployments become predictable and repeatable +- staging stability no longer controls team velocity + +
+ +Image +
Source: Signadot
+ +
+ +## Wrap Up + +For a company operating at Bitso’s scale, this development setup is transformative, with fewer broken environments, faster iteration, safer deploys, and a far more predictable path from development to production. If you’re exploring a similar approach, [check out Signadot’s write-up](https://www.signadot.com/blog/how-bitso-is-scaling-branch-based-development-with-signadot-and-neon?utm_source=partner&utm_medium=partner_blog&utm_campaign=q1_26_partner_marketing), [open a free Neon account](https://console.neon.tech/login), and [try your first branching workflows](https://neon.com/blog/practical-guide-to-database-branching). diff --git a/content/blog/posts/branch-chose-neon-for-its-true-postgres-and-serverless-nature.md b/content/blog/posts/branch-chose-neon-for-its-true-postgres-and-serverless-nature.md new file mode 100644 index 0000000000..a0b2f6e9e4 --- /dev/null +++ b/content/blog/posts/branch-chose-neon-for-its-true-postgres-and-serverless-nature.md @@ -0,0 +1,91 @@ +--- +title: Branch chose Neon for its true Postgres and serverless nature +description: 'Going serverless to ship product, not babysit infrastructure' +excerpt: >- + Using Neon has meant our developers can continue to spend their time on things + that meaningfully drive the business forward, instead of babysitting + infrastructure. Adithya Reddy, Developer 4 at Branch Branch is a leader in + innovative home and auto insurance that’s simple to buy a... +date: '2024-01-18T08:41:35' +updatedOn: '2024-02-29T14:50:43' +category: case-study +categories: + - case-study +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branch-chose-neon-for-its-true-postgres-and-serverless-nature/social.jpg + alt: null +isFeatured: false +seo: + title: Branch chose Neon for its true Postgres and serverless nature - Neon + description: >- + Neon is a perfect match for serverless architectures, simplifying database + management while providing flexibility and scalability. + keywords: [] + noindex: false + ogTitle: Branch chose Neon for its true Postgres and serverless nature - Neon + ogDescription: >- + Neon is a perfect match for serverless architectures, simplifying database + management while providing flexibility and scalability. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branch-chose-neon-for-its-true-postgres-and-serverless-nature/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/branch-chose-neon-for-its-true-postgres-and-serverless-nature/image-17-98d00a11.png) + +
+

Using Neon has meant our developers can continue to spend their time on things that meaningfully drive the business forward, instead of babysitting infrastructure.

+Adithya Reddy, Developer 4 at Branch +
+ +[Branch](https://www.ourbranch.com/) is a leader in innovative home and auto insurance that’s simple to buy and built for savings. As their services expanded, they needed a database solution capable of supporting complex queries and diverse data access patterns — a requirement that Amazon DynamoDB and other providers couldn’t fulfill. + +## Beyond NoSQL and the quest for a true serverless database + +
+

We were fully serverless from the start, but as we grew, our database needs evolved beyond what NoSQL could offer.

+
+ +Branch has embraced a fully serverless tech stack from day one and initially used Amazon DynamoDB for all applications. However, they faced limitations with the NoSQL database, particularly in data storage and query flexibility. This led them to explore other options, including Aurora Serverless v1 and v2, and other distributed, and MySQL providers, but none met their needs of fast scale-up or scale-down to zero, resulting in a substantial bill for idle instances. + +
+

DynamoDB served and continues to serve us well, but its query limitations were a roadblock to our evolving data needs.

+
+ +The company’s challenges with its previous database included: + +- **Inefficiency in Data Handling**: DynamoDB’s limitations in handling diverse query patterns. +- **Infrastructure Management**: Challenges with Aurora Serverless, including slow scaling, VPC limitations, and cost inefficiencies. +- **Technical Limitations**: Issues with payload limitations during large migrations and Aurora V2’s inability to scale to zero. + +Ultimately, they chose Neon Serverless Postgres for its true serverless nature, scaling capabilities, and compatibility with their existing systems. + +
+

Neon checks all of our boxes. It’s real Postgres, scales down to zero, has no instances to manage, and only charges for data storage and actual usage

+
+ +## A perfect match for serverless architecture + +Branch’s tech stack, including AWS Lambda, Vercel, Algolia, and Upstash, seamlessly integrates with Neon. Each developer at Branch is provided with their own isolated deployment environment using Neon’s database branching, ensuring a streamlined developer workflow. After extensive testing, Branch transitioned to Neon for its:
+ +- **True Serverless Nature**: Neon’s capacity to scale to zero and handle complex queries seamlessly. +- **Ease of Use**: Simplified developer’s workflows with database branching, allowing developers to focus on app development rather than infrastructure. +- **Flexibility**: Support for a wide range of PostgreSQL features, such as stored procedures and triggers. + +## An integrated part of the growth strategy + +
+

The best measure of Neon’s success for us has been that we have never had to think about Neon.

+
+ +The implementation of Neon led to significant improvements: + +- **Operational Efficiency**: Zero time spent on database capacity and infrastructure management. +- **Scalability**: Efficiently managed increasing data requirements with Neon’s automatic scaling capabilities. +- **Cost-Effectiveness**: Pay-per-use model reduced costs significantly compared to other solutions. + +## Conclusion + +Neon’s serverless Postgres database proved to be an ideal solution for Branch, aligning perfectly with their serverless architecture and simplifying database management while providing the flexibility and scalability required for their growing needs.
For companies seeking a hassle-free, scalable, and efficient database solution, Neon is a prime example of modern database technology, perfectly tailored for serverless architectures. Discover how Neon can eliminate operational burdens and bring standard Postgres into the modern development workflow by [signing up here](https://console.neon.tech/signup). diff --git a/content/blog/posts/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon.md b/content/blog/posts/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon.md new file mode 100644 index 0000000000..6b666dd104 --- /dev/null +++ b/content/blog/posts/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon.md @@ -0,0 +1,119 @@ +--- +title: 'Branching as the Environment: How Sharing Excess Ships Faster on Neon' +description: >- + Sharing Excess’s three-person development team routes perishable food at a + national scale and ships faster with Neon branching, production-like test + data, and scale-to-zero. +excerpt: >- + “For a small team, branching and scale-to-zero just make our lives markedly + easier. We’re able to use real data without risky scripts touching production, + and we don’t pay overnight for what we don’t use.” — Ryan McHenry, Head of + Technology, Sharing Excess What is Sharing Excess?... +date: '2025-10-23T17:49:17' +updatedOn: '2025-10-24T23:37:10' +category: case-study +categories: + - case-study +authors: + - taraneh-dohmer +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon/cover.jpg + alt: sharing excess case study +isFeatured: false +seo: + title: 'Branching as the Environment: How Sharing Excess Ships Faster on Neon - Neon' + description: >- + Sharing Excess’s lean three-person dev team routes perishable food + nationwide faster and safer using Neon’s branching, production-like test + data, and scale-to-zero capabilities. + keywords: [] + noindex: false + ogTitle: 'Branching as the Environment: How Sharing Excess Ships Faster on Neon - Neon' + ogDescription: >- + Sharing Excess’s lean three-person dev team routes perishable food + nationwide faster and safer using Neon’s branching, production-like test + data, and scale-to-zero capabilities. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon/social.jpg +--- + +
+

“For a small team, branching and scale-to-zero just make our lives markedly easier. We’re able to use real data without risky scripts touching production, and we don’t pay overnight for what we don’t use.”

— Ryan McHenry, Head of Technology, Sharing Excess

+
+ +## What is Sharing Excess? + +In the United States, nearly 40% of food goes to waste while 47 million Americans go hungry. [Sharing Excess](https://www.sharingexcess.com/) is a nonprofit food-rescue organization that redirects surplus food from retailers, wholesale markets, and farms to community hunger relief organizations. Their mission is to build a more sustainable and equitable food system that helps reduce waste and feed communities. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon/scheme-3-1024x458-073f5e60.jpg) + +The work is a real-time logistics challenge. Perishable cargo, often in pallets and truckloads, must be split and routed daily across a network of 1,800+ vendors. For example, 30 bins of watermelons (weighing thousands of pounds) from a single pickup must be distributed to multiple partners because no single site can absorb it all. Accuracy and speed are everything.
+ +## Engineering at Sharing Excess: Team, Stack and Workflow + +Sharing Excess runs a deliberately lean engineering organization: a three-person core team ( engineers and analysts) led by Head of Technology Ryan McHenry, with additional support from co-op interns and community volunteers. The team maintains guardrails like point-in-time restore (PITR) and daily pg_dump backups while focusing most cycles on shipping products. + +- **Language:** Typescript (front + backend) +- **Repos & apps:** Monorepo structure using Bun Workspaces with separate client and server apps, as well as packages for db, types/schemas, ui +- **Frontend:** React 18, built with Vite + VitePWA (installable/offline); Chakra UI; Clerk for auth; Tanstack Query, Form + Store, Mapbox, deployed on **Railway** +- **Backend: Bun/TypeScript** with a Hono API +- **Database:** Postgres on Neon with **Drizzle ORM** (schema & connection management) +- **Auth:** Clerk +- **Typing/Validation:** Zod +- **Tooling/Visualization: Retool** (external data visualizations) +- **Analytics + Error Management:** Sentry, Mixpanel, APITally +- **AI + LLM Services:** Anthropic +- **Infrastructure/Hosting:** **Dockerized** setup on **Railway** + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon/image-5-1024x607-17808dd6.png) + +## Platform Architecture + +Sharing Excess’s platform consists of data systems, services, and web/mobile clients that coordinate food rescue nationwide, 24/7. Each environment (production, staging, and per-developer) connects to its corresponding Neon branch for predictable end-to-end behavior. Learn more about it [here](https://github.com/sharingexcess).
+ +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-environment-how-sharing-excess-ships-faster-on-neon/diagram-v2-1024x705-5a8fd2b3.jpg) + +## Why Sharing Excess Migrated to Neon + +After outgrowing Firebase’s NoSQL model, Sharing Excess needed more powerful querying, realistic dev/test environments, and simpler operations. Here’s what Sharing Excess had to say about their switch to Neon: + +- **Unopinionated, low-maintenance Postgres.** Neon lets Sharing Excess choose their app layer (Hono, Drizzle, Clerk) while Neon manages the database. Industry standard PostgreSQL makes interoperability with additional tools + services easy and thoughtless.
+- **Environment parity with branch-per-environment.** Each application environment (per team member development branches, staging, production) connects to its matching database branch, so staging and development mirror production without reconfiguration.
+- **Realistic testing and safe collaboration with production-like, anonymized data.** Branches reset from production and run through on-demand anonymization for easy access in local development.
+- **Operational rigor without toil via built-in guardrails.** PITR and history, plus daily pg_dump backups, keep confidence high and data-loss risk mitigated.
+- **Spend aligned to usage with scale-to-zero.** Idle branches scale down outside operating hours, cutting costs and allow the team to optimize their workflows, not their invoices. + +## Database Branching Workflow + +
+

“The feature that will truly keep me attached to Neon is branching. We can reset from the parent branch at any point, which lets us work with up-to-date data as we develop new features . When you’re working in a real-time logistics environment, that makes all the difference in catching bugs before they ship.”

— Ryan McHenry, Head of Technology, Sharing Excess

+
+ +Neon branching reshapes Sharing Excess’s development workflow in three key ways: + +1. **Dev/Test with realistic environments** +2. **Clean, automated release flow branching** +3. **Create production-like data with anonymization** + +### Dev/Test with Realistic Environments + +Neon branching enables a production branch, a staging branch, and individual child branches for each active developer. Everyone works with up-to-date, production-like data, which prevents bugs and keeps test behavior close to production. + +### Clean, Automated Release Flow Branching + +Branching also significantly lowers operational overhead. GitHub **feature → staging → production** merges map **one-to-one** to Neon branches and environments. Each stage connects to the correct database without manual reconfiguration, reducing deployment mistakes and shortening the path from code merge to realistic integration testing. + +### Create Production-like Data with Anonymization + +Sharing Excess combines Neon branching with an in-house anonymization process, enabling safe sharing with external collaborators while protecting sensitive data. + +
+

“Neon branching is game-changing for how we work. Our workflow is built around clean, simple, and well-synchronized production, staging, and development branches. Neon branching makes it just so much simpler to manage our devops without writing complicated scripts or risking production data.”

— Ryan McHenry, Head of Technology, Sharing Excess

+
+ +## Neon Cuts Development Time and Costs for Small Teams + +Neon helps Sharing Excess focus their operational effort where it matters most. With a branch-per-environment Postgres workflow, PITR, and managed scaling, the team spends less time on routine database maintenance without compromising operational standards. Developers test against up-to-date, production-like datasets and safely share anonymized copies with collaborators. And because scale-to-zero reduces idle costs, more of every dollar goes to rescuing perishable food. + +You can learn more about Sharing Excess’s mission and their amazing projects on [their website](https://www.sharingexcess.com/). Should it resonate with you, consider supporting Sharing Excess by [donating or volunteering](https://www.sharingexcess.com/get-involved). diff --git a/content/blog/posts/branching-as-the-new-standard-for-relational-databases.md b/content/blog/posts/branching-as-the-new-standard-for-relational-databases.md new file mode 100644 index 0000000000..b3b8f68c1f --- /dev/null +++ b/content/blog/posts/branching-as-the-new-standard-for-relational-databases.md @@ -0,0 +1,107 @@ +--- +title: Branching as the New Standard for Relational Databases +description: >- + Modern software needs agile infrastructure, and branching is the missing + primitive for relational data. +excerpt: >- + Over the last decade, nearly every part of the software development stack has + evolved to support faster iteration, better automation, and less oversight. + But one layer has stubbornly resisted this evolution – the relational + database. The stack evolved, the database stayed behind... +date: '2025-07-10T16:51:48' +updatedOn: '2025-07-10T16:51:50' +category: workflows +categories: + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-new-standard-for-relational-databases/cover.jpg + alt: null +isFeatured: false +seo: + title: Branching as the New Standard for Relational Databases - Neon + description: >- + Modern software needs agile infrastructure, and branching is the missing + primitive for relational data. + keywords: [] + noindex: false + ogTitle: Branching as the New Standard for Relational Databases - Neon + ogDescription: >- + Modern software needs agile infrastructure, and branching is the missing + primitive for relational data. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-new-standard-for-relational-databases/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-new-standard-for-relational-databases/neon-flow-1024x576-dd29dece.jpg) + +Over the last decade, nearly every part of the software development stack has evolved to support faster iteration, better automation, and less oversight. But one layer has stubbornly resisted this evolution – the relational database. + +## The stack evolved, the database stayed behind + +In many ways, this makes sense. Databases are supposed to be robust, reliable, boring – they hold your source of truth, you don’t want them to be too exciting. But that mindset came at a cost. Databases became the scary piece of the stack – slow to provision, hard to reset, risky to touch – and working with them started to, in many ways, suck. + +The managed databases most teams interact with run pretty much like they did a decade ago – they’re built on serverful architectures, running on VMs. Some vendors have added a few nice experience touches, and the surface may feel modern – but there’s not much innovation under the hood. This legacy shows up in workflows that haven’t changed in years: + +- Shared staging databases that get manually wiped and re-seeded +- Long-lived environments prone to drift +- Fragile seed files and teardown scripts +- Tests running against data that looks nothing like production… + +## The agent era exposes the cracks + +Developers have been putting up with this for years, but what finally broke the pattern wasn’t the developer pain – it was agents. + +With the [initial launch of Replit Agent](https://blog.replit.com/introducing-replit-agent), the AI era for infrastructure began. Today’s agents can go from prompt to deployed app in seconds – they write code, run migrations, provision databases, and deploy autonomously. This simply doesn’t work with the old, slow, heavy infra model for databases. Agents are pickier than human engineers, who learned to work around the pain—agents won’t. They can’t wait 10 minutes for a database to spin up or resize, they can’t run pg_dump/restore workflows while a user waits to run a test, and the company behind them can’t afford to pay $25 per instance to AWS when it might be thrown away five minutes later. + +It’s time to set a new standard for how databases fit into development workflows. The work those database startups like Neon began years ago (reimagining databases for developers) was made urgent by the needs of agents, which in many ways are fundamentally the same as the needs of developers. + +Which brings us to branching. + +## Database branching: the missing primitive + +Ask developers what they want from a database environment. The wishlist is clear – they want something that’s easy to spin up, automatically scalable, integrated with the rest of the stack, cost-efficient, and pretty much invisible most of the time; they also want something that’s robust and safe to experiment with, something that doesn’t break everything when they test a migration or debug a failing query – a database that’s somehow forgiving. + +This isn’t that different from how developers felt about touching codebases not that long ago. Version control and branching transformed how we work with code, giving engineers the ability to experiment safely, isolate changes, review work in progress, and collaborate without risking the main codebase. Now imagine applying that same model to your database – why not introduce branching to Postgres (the de-facto standard for relational databases used by millions of teams today) and unlock a similar level of agility? + +This is what we’ve been working on at Neon. [From the very beginning](https://neon.com/blog/architecture-decisions-in-neon), we had the concept of a database branch in mind – not as a feature but as a primitive. Something foundational to give developers the ability to create safe environments instantly, to experiment and roll back without consequence and to move fast. + +But the biggest (and most underrated) power of branching databases is confidence. Branching breaks down the fear that’s built up around relational databases and makes them feel safe to work with again. It gives developers a way to try things without hesitation. If something breaks, you reset. If you want to compare two versions side by side, you branch and test. If an agent or a teammate messes something up, you go back in time. You’re much more free to build. + +## Branching Postgres requires a new architecture + +But here’s the catch – you can’t fake this. Branching and legacy database infrastructure don’t mix – you can’t just clone an instance, give it a new name, and call it a “branch.” Building a real branching primitive for Postgres requires heavy architectural work that most database platforms simply haven’t done. Others claim to offer branching but do it by duplicating entire instances or snapshots behind the scenes, but that’s still too slow and heavy. These approaches to “branching”, + +- Require full storage duplication, even if nothing has changed +- Take a long time to provision, especially for large datasets +- Accumulate costs linearly with every copy +- Offer no native reset, no built-in time travel, no real ephemerality + +To make branching real, you need to start with a different foundation. That’s what we did at Neon, where [we rearchitected Postgres from the storage layer up](https://neon.com/blog/get-page-at-lsn) to support instant, copy-on-write branching as a first-class primitive. Branching on Neon is dependent on multiple architectural elements that took us years to build (and that we’re still working on) + +- Decoupled storage and compute with copy-on-write: Each branch must have its own compute endpoint, while sharing the same underlying storage unless it diverges. When a branch is created, it should reference the exact same data pages as its parent – new data is only written on change. This is what makes branch creation instant and storage-efficient, without forcing the user to duplicate data. +- Parent/child branch relationship: The engine must track changes per branch and maintain visibility across versions. This allows you to query or restore the state of any branch at any point in time. +- Time travel: Developers should be able to create a new branch from any past moment instantly, regardless of the size of the database. This must feel lightweight and immediate – not like traditional snapshot or restore systems. +- API lifecycle management: Branches should be created, reset, promoted, or deleted programmatically. They must scale to zero when idle, resume quickly, and fit naturally into automated workflows. + +The combination of all these innovations is what makes branching viable not just as a UX improvement but as a core building block for modern workflows. Once you have it, entirely new patterns emerge. + +## A glimpse into the future + +The future of software won’t be built by humans alone. It’ll be built by agents – and branching is what will power this new development experience where every version is saved, every failure is reversible and iteration is fast and fearless.We’re already seeing this in production. + +Replit already lets users preview and roll back to any version of their app – code and database included. Behind the scenes, [it’s Neon branching doing the work,](https://neon.com/blog/replit-app-history-powered-by-neon-branches) quietly making the whole thing feel effortless. This kind of experience simply wouldn’t be possible on legacy infrastructure. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-as-the-new-standard-for-relational-databases/screenshot-2025-07-09-at-43853percente2percent80percentafpm-653x1024-70aea39a.png) + +We’re convinced branching will become the default way of interacting with databases – not just because it’s faster or cheaper but because it fundamentally reshapes the relationship between developers, agents, and the database. It creates a new mindset – one where iteration is constant, rollback is instant, and the boundary between code and data begins to disappear. + + +If you’re ready to get started experimenting with branching, [check out our new page](https://neon.com/flow). We break down real-world branching architectures and the core concepts behind them. Try it with your [Neon Free account](https://console.neon.tech/signup). + + + diff --git a/content/blog/posts/branching-environments-anonymized-pii.md b/content/blog/posts/branching-environments-anonymized-pii.md new file mode 100644 index 0000000000..24a6cf9ebd --- /dev/null +++ b/content/blog/posts/branching-environments-anonymized-pii.md @@ -0,0 +1,225 @@ +--- +title: 'Branching With or Without PII: The Future of Environments' +description: 'Managing data across dev, test, and staging is painful. We built a way out' +excerpt: >- + Branching lets developers spin up isolated environments that perfectly mirror + production in a single click or API call. This is a major upgrade for + developer experience: teams can skip the tedious work of keeping databases in + sync and focus on shipping code. But until now, there’... +date: '2025-11-11T17:18:38' +updatedOn: '2025-11-11T17:22:47' +category: product +categories: + - product + - workflows +authors: + - monica-steinke +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-environments-anonymized-pii/cover.png + alt: null +isFeatured: true +seo: + title: 'Branching With or Without PII: The Future of Environments - Neon' + description: >- + A new way to manage environments: branch your database with masked PII for + fast and safe staging, dev, and test workflows. + keywords: [] + noindex: false + ogTitle: 'Branching With or Without PII: The Future of Environments - Neon' + ogDescription: >- + A new way to manage environments: branch your database with masked PII for + fast and safe staging, dev, and test workflows. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-environments-anonymized-pii/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-environments-anonymized-pii/screenshot-2025-11-10-at-102516-am-1024x577-a34cc153.png) + +[Branching](https://neon.com/docs/introduction/branching) lets developers spin up isolated environments that perfectly mirror production in a single click or API call. This is a major upgrade for developer experience: teams can skip the tedious work of keeping databases in sync and focus on shipping code. + +But until now, there’s been one limitation. If your production data includes sensitive information, compliance and security rules often prevented you from creating dev or test branches directly. That changes today. + +**When creating a branch in Neon, you can now select** [Anonymized data](https://neon.com/docs/workflows/data-anonymization#create-a-branch-with-anonymized-data)**: Neon will mask sensitive information according to rules you define, giving you realistic, production-shaped data without exposing PII.** When you’re done, simply delete the branch and move on. You can even automate the whole workflow with the Neon API or GitHub Actions. + + + +All those dreadful environment maintenance tasks (manually managing separate staging databases, working with synthetic data, updating seed scripts) are becoming a thing of the past. **The future of developer experience is branching, and we’re gonna tell you why.** + +_To jump straight to the update on anonymized branches, click [here](https://neon.com/blog/branching-environments-anonymized-pii#pii-wont-stop-you-introducing-anonymized-branches)._ + +## Environments Are a Time Sink + +
+

“Our testing process was very manual before Neon. Product would create a test customer in our development environment, the QA team would test and manually run through all the math, then an engineer would have to go into the database, look at all the values, and handwrite them into fixtures for our end-to-end tests… That’s multiple days for every single change”



(Miguel Hernandez, Backend Tech Lead at Neo.Tax)

+
+ +No matter how small or large, every engineering team needs non-production environments – places to test new features, run CI pipelines, reproduce bugs, or review schema changes before they reach production. These environments are supposed to behave like production, but maintaining that parity is one of the most annoying hardest ongoing operational tasks in software development. + +### The manual non-prod data lifecycle + +Most teams start simple – copy the production database and scrub it. They run pg_dump, maybe write shell scripts to remove or redact sensitive data, and restore the dump into a staging instance. The hard reality is that production data changes often, especially as you’re shipping quickly or the business is growing. New tables, columns, and configs are added all the time, and the scripts quickly fall out of date. + +We all know what happens when your environments fall out of sync with production (bugs undetected), so teams spend time trying to prevent this from happening. One option is automating this with nightly jobs or pipelines that recreate staging from production snapshots; you can also go the opposite way and maintain a seed dataset – a small, curated sample of data that’s “good enough” for testing, and do their best to refresh it often. + +Both options are hardly anyone’s favorite to work on and they always introduce problems. + +### Where the process tends to break + +#### Data drift + +Staging data starts drifting from production almost immediately. New records, customer behaviors, or schema changes in production don’t make it into staging at the same pace. Within days, the shape and distribution of your data no longer match. Queries that perform fine in staging might time out in production because the row counts or index selectivity are different. + +#### Schema mismatch + +Database migrations are another failure point. A migration might be applied to production but not staging, or run in a different order, or use slightly different data. That inconsistency leads to subtle issues – columns missing in staging, default values that don’t match, foreign keys referencing outdated IDs. CI pipelines fail, tests pass for the wrong reasons. + +#### Broken referential integrity + +When teams copy or scrub production data manually, they strip or anonymize columns independently. That breaks the relationships between tables in Postgres: foreign keys no longer align, there’s orphaned records and inconsistent IDs. It’s hard testing with this data and reproducing real user flows because the underlying relational context is gone. This is also one of the hardest issues to detect until something breaks downstream. + +#### Weak releases + +Synthetic or seeded datasets rarely reflect the scale and cardinality of real production workloads – they miss outlier values and the edge cases that stress indexes and constraints. Performance regressions, deadlocks, and memory issues stay hidden until production traffic hits. This causes fragile release confidence: your tests may go green but production starts breaking more than you wish. + +### The cost of keeping it all running + +
+

“Our workload ingests hundreds of data points per second and our RDS costs were increasing, especially since we had multiple regions and environments. With Neon, we found a way to scale our setup more efficiently, using branching instead of duplicating instances and autoscaling to match our actual load”



(Thorsten Rieß, Software Architect at traconiq)

+
+ +Maintaining separate environments isn’t just slow and annoying. It’s also expensive, both in engineering team time and in cloud costs. + +- **Each staging or testing database consumes storage and compute continuously, even when idle.** Teams end up maintaining multiple long-lived managed Postgres clusters that serve no customer traffic but still end up costing hundreds or even thousands of dollars a month, especially as the setup bloats after a while. +- **But the human cost is probably the largest one.** Someone on the team becomes _the unofficial data janitor_, responsible for keeping all these environments usable even if they’re supposed to be a developer. This is a common pain in many smaller / leaner engineering teams, where there’s no designated DBA. Maintaining migration scripts, scrubbers, and seed files takes a lot of time. + +### What you get + +Testing environments don’t catch bugs, development and staging environments rarely behave like production. Bugs that depend on real-world data distributions slip through, developers waste hours debugging issues that only exist because the data is different, companies end up with multiple always-on instances that are barely used, and developers end up spending more and more time on maintenance tasks that nobody saw coming. + +This is a pattern that teams end up accepting as normal, but it doesn’t have to be. + +## Branching as the Alternative + +One of the reasons we built Neon was to eliminate the grind described above. In 2025, developers shouldn’t be hand-curating test data – we deserve better. The way out is adding [branching](https://neon.com/docs/introduction/branching) to databases, creating an experience similar to what we’re used to in Git. But this only works if branching is a storage-level primitive built into the database, not a scripted clone of an instance. + +
+

“When we moved to Neon, we consolidated all our non-prod services into a single Neon project. We just load our synthetic dataset once into the main staging branch and spin off child branches as needed. With Aurora, the process was more complex. We had to duplicate databases per service, which meant loading data separately for each environment” 



(Cody Jenkins, Head of Engineering at Invenco)

+
+ +## The Architecture Primitives of Branching + +### Separation of compute and storage + +In traditional Postgres setups, compute (the Postgres process) and storage (the data directory) live on the same machine. If we were to build any sort of environment-cloning feature this way, it would still be slow and expensive. Each would need its own VM, its own disk, its own lifecycle, etc. + +But in Neon, [the architecture is decoupled](https://neon.com/blog/architecture-decisions-in-neon). Compute is stateless and ephemeral, running Postgres but holding no durable data. All persistent data lives in the storage layer, which is shared and distributed across the platform, and has a very particular design (as we’ll see now). + +### WAL-first storage + +A decoupled compute and storage architecture eases the path towards cloning environments, but one still has to solve the problem of duplicating data very quickly. Neon built a [custom storage layer for this](https://neon.com/storage), in which every write is captured as a stream of WAL records which are stored durably by a quorum of lightweight services called [Safekeepers](https://neon.com/docs/introduction/architecture-overview). + +Safekeepers guarantee that once a transaction is acknowledged, it’s safely persisted, even if no compute is currently attached. They also record precise Log Sequence Numbers (LSNs), which act as version markers in the database’s history. + +This is essential for branching. This makes it possible to say: [“Create a new branch at LSN X”](https://neon.com/blog/get-page-at-lsn) which is an exact point in time. That’s how Neon can spin up a new environment from any moment in production history, without needing a dump, restore, or snapshot job. + +From there, the [Pageserver](https://neon.com/docs/introduction/architecture-overview) reconstructs data pages from WAL and stores them as immutable layers in object storage. Branching uses [copy-on-write](https://neon.com/blog/instantly-copy-tb-size-datasets-the-magic-of-copy-on-write) semantics on those layers: the new branch references the same base data as its parent and only diverges when new writes occur. + +For developers, this means every branch is space-efficient, created in seconds, and mirrors production exactly. + +### Per-branch compute lifecycle + +Because compute and storage are decoupled, each branch can attach its own temporary compute, a lightweight Postgres process that runs only when needed. When a branch is idle, its compute scales down to zero automatically while the storage layer remains intact. Spinning it back up is instant, since Postgres simply reattaches to the branch’s timeline. + +In practice, this keeps workflows lean. Teams can create hundreds of short-lived branches (e.g. one per developer, test run, CI job) without paying for hundreds of always-on instances. The database behaves the way infrastructure should: on when you need it, gone when you don’t. + +## The Workflows + +**Together, these architectural components make Neon branching an actual substitute for manually deploying and maintaining separate non-prod instances.** Neon branches can be used as your environments, and they all can be understood as ephemeral – you create them right when needed, you do whatever you need to do, you delete them / resync them with production after. + +Let’s go through a few workflow examples that thousands of developers are already using every day. + +### Development environments + +Developers need their own isolated environment that mirrors production, and these often need to live in the same cloud environment due to security and compliance reasons. Instead of maintaining long-lived dev instances and populating them manually, a developer on Neon can spin up a new branch from production in seconds. The branch has its own compute, independent from prod, and perfectly mirrors its schema and data. You can run migrations, test queries, seed data, and when you’re done, delete or resync the branch with production. + +
+

“We’re a small team, but we’re scaling quickly and doing a lot. We’re shipping multiple times a day – to do that, we need to test stuff quickly and merge to main very quickly as well. Neon branches are a game changer for this”



(Avi Romanoff, Founder at Magic Circle)

+
+ +### Preview environments + +When deploying previews on platforms like [Vercel](https://neon.com/blog/neon-vercel-native-integration), every feature branch can automatically get its own Neon branch. During build, a new database branch is created from production and linked to that preview deployment, with the same production schema and data, giving every preview app a realistic backend for QA and design review. When the PR closes, both the preview and the database branch are deleted automatically. + +
+

“Branching with scale to zero makes our lives markedly easier. We’re able to use real data without risky scripts touching production, and we don’t pay overnight for what we don’t use”



(Ryan McHenry, Head of Technology, Sharing Excess)

+
+ +### Ephemeral testing and CI environments + +[Ephemeral environments](https://neon.com/blog/ephemeral-environments-aws-serverless) are short-lived copies of your system and are ideal for CI pipelines and automated testing, but databases have traditionally been a friction point. With branching, each test run can automatically create a temporary branch from production, execute all tests against it, and delete it afterward. Because branches are copy-on-write and compute scales to zero when idle, you get instant, isolated test databases that cost virtually nothing when not in use – and GitHub Actions and the Neon API make it easy to automate. + +
+

“Anytime we need to test a new feature or extension, we branch off from production, run our checks, and reconnect when we’re ready. It’s simple, safe, and saves us huge amounts of time”



(Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +### Staging environments + +Staging is where most teams try to get close to production but it’s also where data drift and compliance risks creep in. With Neon, you can create a long-lived staging branch derived directly from production and refresh it as often as you need. This ensures schema and data parity without complex sync jobs or downtime. + +
+

“Branching lets us test big migrations safely with production data, and if something breaks, we just delete the branch and start fresh. That makes us a lot faster and more confident when shipping, even as a small team”



(Dominik Koch, CEO and Co-Founder of Marble)

+
+ +## But what about PII? + +Some teams use real data in staging for full-fidelity testing, but others need anonymized data for compliance. Neon now supports both: + +## PII Won’t Stop You. Introducing Anonymized Branches + +Adopting the branching workflows above could get tricky for teams handling sensitive data in prod – things like names, emails, addresses, payment details. That changes with our recent release of anonymized branches. + +[When you create a new Neon branch, you can now choose “Anonymized data” as the data option:](https://neon.com/docs/workflows/data-anonymization#create-a-branch-with-anonymized-data) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-environments-anonymized-pii/image-5-1024x798-37a9264f.png) + +Neon copies your schema and data from the parent branch (e.g. prod) and immediately applies masking rules using the [PostgreSQL Anonymizer](https://postgresql-anonymizer.readthedocs.io/en/stable/) extension. The anonymization process is all handled by the platform: all you have to do is wait until the masking is done. This will be fast for tables with small PII columns, and it might take longer if you have a lot of PII to mask. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-environments-anonymized-pii/image-7-1024x633-61214834.png) + +Regardless of if you have few or lots of PII, **this gives you a database branch that behaves completely like production but holds no sensitive information.** Fields such as names, emails, phone numbers, and addresses are replaced with safe, dummy equivalents, keeping foreign key relationships and constraints intact. You can pick exactly the masking rule you want to apply to each column, e.g. + +- Dummy Free Email – replaces email addresses with realistic dummy addresses. +- Dummy Last Name generates random but plausible names. +- Dummy Phone Number – substitutes phone numbers with valid, formatted dummies. +- Generate Random Value – randomizes numeric values such as age or salary + +**Of course, you can also define masking rules programmatically** [via the API.](https://neon.com/docs/workflows/data-anonymization#api-reference) Each branch can have its own rules, so you can anonymize them differently for different use cases, e.g. for dev or demos, and you can automate all of this. + +### Under the hood + +This first beta version of anonymized branches uses static masking, meaning that: + +- Masking runs once when the anonymized branch is created (or when you rerun anonymization) +- Parent data stays untouched. All masking happens in the new branch +- Anonymization rules are branch-specific + +### Storage costs implications + +**Since we’re doing static masking by now, there’s storage costs implications to anonymizing data.** The masked columns will generate a storage delta that will add to your overall project storage bill; how large this delta will be depends on how much data you anonymize. If you’re masking a few small columns, the increase is minimal; if you’re masking many or large columns, it will be higher. Keep this in mind when estimating your storage usage. + + +It's important to us to stick to our copy-on-write philosophy even with data anonymized, so we’re already working on the next version of anonymized branches which will use dynamic masking. After this update, we will apply masking at query time, rather than storing transformed copies - which will mean no additional storage created and no additional costs for you. Stay tuned. + + +## A New Way to Manage Environments + +With the launch of anonymized branches, we’ve removed one of the last blockers to fully adopt branching workflows. Combined with Neon’s existing branching and scale-to-zero capabilities, this unlocks a complete workflow that frees developers from the old grind of maintaining environments, allowing them to focus their time and budget on more important things. + +
+

production → anonymized staging → anonymized dev/test/preview branches → delete branch

+
+ +[You can explore anonymized branches in Neon today](https://console.neon.tech/signup). Full setup steps, API examples, and masking function references are available [in our docs.](https://neon.com/docs/workflows/data-anonymization) If you’re new to branching workflows altogether, [explore our guide](https://neon-next-git-anon-doc-update-neondatabase.vercel.app/branching). diff --git a/content/blog/posts/branching-postgres-databases-with-the-neon-api.md b/content/blog/posts/branching-postgres-databases-with-the-neon-api.md new file mode 100644 index 0000000000..a47de4c8e4 --- /dev/null +++ b/content/blog/posts/branching-postgres-databases-with-the-neon-api.md @@ -0,0 +1,103 @@ +--- +title: Branching Postgres databases with the Neon API +description: Use Neon API with GitHub Actions +excerpt: >- + In this post, we’ll discuss using database branching and the Neon API from a + CI/CD pipeline. By the end of this post, you’ll have a better understanding of + how database branching can help you test your code and schema changes before + deploying to production. What is a branch? A br... +date: '2022-12-07T17:43:14' +updatedOn: '2023-07-14T08:55:34' +category: community +categories: + - community +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-postgres-databases-with-the-neon-api/cover.jpg + alt: null +isFeatured: false +seo: + title: Branching Postgres databases with the Neon API - Neon + description: Use Neon API with GitHub Actions + keywords: [] + noindex: false + ogTitle: Branching Postgres databases with the Neon API - Neon + ogDescription: >- + In this post, we’ll discuss using database branching and the Neon API from a + CI/CD pipeline. By the end of this post, you’ll have a better understanding + of how database branching can help you test your code and schema changes + before deploying to production. What is a branch? A branch acts as an + isolated environment […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-postgres-databases-with-the-neon-api/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-postgres-databases-with-the-neon-api/neon-branching-and-github-actions-1-1024x576-3fd1442f.jpg) + +In this post, we’ll discuss using database branching and the Neon API from a CI/CD pipeline. By the end of this post, you’ll have a better understanding of how database branching can help you test your code and schema changes before deploying to production. + +## What is a branch? + +A branch acts as an isolated environment for working with your database. It is a copy-on-write clone of your data that you can modify without affecting the originating data, which allows you to integrate production data into your development environment. + +Each Neon project has a root branch called main, and you can create more branches depending on your needs. + +## Integrate branching into your GitHub actions + +End-to-end (e2e) testing ensures that your application works as expected in a real-world environment. However, e2e testing can be challenging, especially when it involves testing with production data. This is where Neon’s database branching comes into play. + +Instead of manually creating a branch from the Neon console, you can automate and extend this capability using the [Neon API](https://api-docs.neon.tech/reference/getting-started-with-neon-api). This opens up a few scenarios, one of which is integrating database branching into your GitHub pipelines for testing. + +With database branching, you can create and manage multiple database versions without impacting your production environment. This means you can test your application against different data sets without worrying about downtime or data loss. + +One of the areas we are exploring is to use of the Neon API with GitHub Actions. We have implemented two experimental GitHub Actions: [neondatabase/create-branch-action](https://github.com/neondatabase/create-branch-action) and [neondatabase/delete-branch-action](https://github.com/neondatabase/delete-branch-action). + +These actions allow you to automate creating and deleting branches with the Neon API and integrate the automation into your development workflow. + +Here is an example of how you can use these actions to set up e2e testing with database branching: + +1. In your repository, create a new workflow file (e.g., e2e-tests.yml) in the .github/workflows directory. +2. In the workflow file, define a new job and specify the trigger (e.g., on: push). +3. In the job, add the `neondatabase/create-branch-action` to create a new branch from your production database. Specify the required input parameters, such as the database connection string and the name of the new branch. +4. Add the steps to run your E2E tests, such as installing dependencies, building your application, and running the tests. +5. Add the `neondatabase/delete-branch-action` to delete the branch created in step 3. This ensures that the branch is removed after the tests are completed. + +```bash +name: Neon Developer Days Actions Demo +run-name: e2e Testing with Neon Actions???? +on: [pull_request] +jobs: + e2e: + runs-on: ubuntu-latest + steps: + - uses: neondatabase/create-branch-action@beta + with: + project_id: ${{ secrets.NEON_PROJECT_ID }} + parent_branch_id: ${{ secrets.NEON_PARENT_BRANCH_ID }} + branch_name: action_demo_branch + api_key: ${{ secrets.NEON_API_KEY }} username: ${{secrets.DBUSERNAME}} Password: ${{secrets.DBPASSWORD}} + id: create-branch + - run: echo project_id ${{ steps.create-branch.outputs.project_id}} + - run: echo branch_id ${{ steps.create-branch.outputs.branch_id}} + - uses: actions/checkout@v2 + - run: npm install + - uses: neondatabase/delete-branch-action@beta + with: + project_id: ${{ steps.create-branch.outputs.project_id}} + branch_id: ${{ steps.create-branch.outputs.branch_id}} + api_key: ${{ secrets.NEON_API_KEY }} +``` + +With this setup, you can run your e2e tests against a copy of your production database, without impacting your production environment. This allows you to test your application with real-world data, and catch potential issues before they affect your users. + +We are working with the developer community to improve these actions and make them suitable for production use. If you have any feedback or suggestions, please feel free to reach out to us or open an issue on GitHub. + +## Summary + +With database branching, Neon becomes a development tool that you can seamlessly integrate into your existing Postgres-based stack. It can be a powerful tool for e2e testing, and GitHub Actions can help you automate the process and integrate it into your development workflow. + +What is your use case for branching? What would you like to see in Neon? Please email us with questions and feedback at feedback@neon.tech. + +Thank you for being part of the Neon community, and we look forward to seeing what you build with Neon. diff --git a/content/blog/posts/branching-with-preview-environments.md b/content/blog/posts/branching-with-preview-environments.md new file mode 100644 index 0000000000..b545c7b9f8 --- /dev/null +++ b/content/blog/posts/branching-with-preview-environments.md @@ -0,0 +1,401 @@ +--- +title: >- + A database for every preview environment using Neon, GitHub Actions, and + Vercel +description: >- + Learn how to create a Neon branch for every preview environment using Neon, + GitHub Actions, and Vercel. +excerpt: >- + In this guide, you will learn how to leverage Neon’s branching feature to + create a database for every preview environment. We will use Vercel as an + example deployment provider and GitHub actions as the CI/CD tool. Staging + environments and why they are not ideal Teams often set up... +date: "2023-04-14T11:30:56" +updatedOn: "2025-10-14T05:51:11" +category: community +categories: + - community + - workflows +authors: + - mahmoud-abdelwahab +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + A database for every preview environment using Neon, GitHub Actions, and + Vercel - Neon + description: >- + Learn how to create a Neon branch for every preview environment using Neon, + GitHub Actions, and Vercel. + keywords: [] + noindex: false + ogTitle: >- + A database for every preview environment using Neon, GitHub Actions, and + Vercel - Neon + ogDescription: >- + Learn how to create a Neon branch for every preview environment using Neon, + GitHub Actions, and Vercel. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/social.png +--- + +In this guide, you will learn how to leverage Neon’s branching feature to create a database for every [preview environment](https://neon.tech/blog/branching-with-preview-environments#preview-environments). We will use [Vercel](https://vercel.com/) as an example deployment provider and [GitHub actions](https://github.com/features/actions) as the  CI/CD tool. + +## Staging environments and why they are not ideal + +Teams often set up a production-like environment, known as “staging”, where they test and validate new features before releasing them. The goal of this pattern is to ensure quality; however, it introduces challenges when you have several developers collaborating on the same project: + +1. If the staging environment goes down, the entire team cannot preview their changes before deploying to production. +2. If a developer wants to preview their changes in isolation, they will prevent other team members from deploying to the staging environment. +3. You likely end up with a queue of changes that need to be reviewed, forcing you into doing a big release that includes several changes. The problem with doing a big release is that if something goes wrong unexpectedly, and you’re unsure of the cause, you must undo all of your changes. + +Thankfully, a new pattern is becoming increasingly popular: **preview environments**. + +## Preview environments: A better way to build software + +Rather than having a shared and fixed staging environment where all developers collaborate, you automatically provision a production-like environment for every new code change a developer wants to introduce. In other words, every pull request will have its own isolated, production-like environment. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/preview-environments-1024x576-ee4deb9b.png) + +This enables developers to build new features in parallel without affecting each other. It also makes it possible to do frequent small releases, making it easier to revert changes if something goes wrong. + +Many deployment platforms are starting to support this flow out of the box by enabling you to set up an automatic Git integration. This way, you do not have to worry about maintaining and managing the necessary infrastructure for handling this flow. + +## Working with databases in preview environments + +When teams want to set up preview environments for an app that is backed by a database, they typically pick one of the following approaches: + +- Setting up a shared database for all previews +- Creating a database for every pull request and seeding it with random data +- Creating a database for every pull request and seeding it with production data + +All of these options are not ideal and present their own drawbacks. + +Creating a database for every pull request and seeding it with random data means that preview environments will not accurately represent production. So you will not be able to make changes to your production database confidently. + +The alternative would be to create a database for every pull request and import a backup of the production database. The problem with this flow is that depending on the size of your database, it can take a long time to deploy a preview environment. + +That is why teams often skip the step of creating a database for every preview and just use a shared database. The drawback is that any changes made to that shared database will impact all active previews. So, if a developer’s changes include evolving the database schema, they must notify their team members about this change and potentially prevent them from deploying. Furthermore, if the shared database goes down, no one will be able to create preview environments. This flow is somewhat similar to having a staging environment, but now the database is the bottleneck. + +The great news is that you can use Neon’s [branching feature](https://neon.tech/docs/introduction/branching?utm_source=devday&utm_medium=blogpost&utm_campaign=blog) to create a production-like database for every preview. This database will contain production data and can be created in seconds. + +## What is Neon? + +Neon is fully managed serverless Postgres. This means you don’t have to pick a size for your database upfront, and it will automatically allocate resources to meet your database’s workload. + +To get started, go ahead and [create an account](https://neon.tech/?utm_source=devday&utm_medium=blogpost&utm_campaign=blog). Next, you will want to create a new project. Choose a Postgres version, pick the region that’s closest to where you want to deploy your app. After you create the project, you will get a connection string that you can use to connect to your database. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/301940761-1814eb55-557b-4d1c-8136-908c816dabd4-1024x576-65eb2e62.png) + +
+

Neon’s architecture separates storage and compute. This makes a Neon Postgres instance stateless, which makes it possible to automatically scale compute resources up or down based on demand.  To learn more, check out Neon’s architecture.

+
+ +### Neon’s object hierarchy + +When you create a project in Neon, a default Postgres cluster is created with a default database called `neondb`.  Within a project’s Postgres cluster, you can have many databases.  + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/neons-object-hierarchy-1024x576-5d2711dd.png) + +This is important to know because it will help you understand how Neon’s branching feature works. + +### Neon branching + +Neon enables you to create copies of your project’s Postgres cluster, where each copy is completely isolated from the other. We call this copying process “branching” and call each copy a “branch”. + +
+ +
+ +The default Postgres cluster that gets created along with a new project is represented by a branch called `main`. + +Neon branches are: + +- **Isolated**: changes made to a branch don’t affect its parent +- **Fast to create**: creating a branch takes ~1 second, regardless of the size of your database. +- **Cost-effective**: you’re only billed for unique data across all branches, and they scale to zero when not in use (you can configure this behavior for every branch). +- **Ready to use**: branches will have the parent branch’s schema and all its data (you can also include data up to a certain point in time) + +### Creating a branch in the Neon console + +To see branching in action, you will first need to create a new table and add some data to it.  Navigate to the SQL Editor and add the following SQL query. + +```sql +CREATE TABLE elements ( element_name VARCHAR(50), atomic_number INT, symbol VARCHAR(10) ); + +INSERT INTO elements VALUES ('Hydrogen', 1, 'H'), ('Helium', 2, 'He'), ('Lithium', 3, 'Li'), ('Beryllium', 4, 'Be'), ('Boron', 5, 'B'), ('Carbon', 6, 'C'), ('Nitrogen', 7, 'N'), ('Oxygen', 8, 'O'), ('Fluorine', 9, 'F'), ('Neon', 10, 'Ne'); +``` + +This query creates a new table called `elements`. The table has three columns: `element_name`, `atomic_number`, and `symbol`. You are then inserting ten elements into that table. + +You can check that the data was added by going to the “Tables” page. You will find that you have a table called `elements` with ten items. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/image-12-1024x640-60674459.png) + +To create a branch, go to the “Branches” page, and click “New Branch”. You will then be redirected to the branch creation page. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/image-13-1024x640-160206fa.png) + +Here, you will first need to specify the parent branch that will be copied. Right now, you only have the main branch, but you can create a branch from other branches as well. + +You will then need to specify the data you want to include. For that, you have several options: + +- **Current point in time**: this will include all of the available data +- **Specific date and time**: this enables you to include all data up to a certain point in time. This is useful if you want to restore your branch to a previous state. You can create a branch from any previous point in time as long as it falls within your [history-retention window](https://neon.tech/docs/introduction/point-in-time-restore#history-retention). +- **Specific Log Sequence Number**: include data up to a certain LSN in the database log. This option allows for precise data recovery + +Choose the main branch as the parent, pick the “**Current point in time**” option, and click “Create new branch”. You will get a new connection string that is different than the main branch. This connection string will be associated with the newly created branch. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/image-14-1024x640-2ce1bb63.png) + +If you go to the tables page, you can select the newly created branch, and you will find that it contains all of the main branch’s data. + +Now, to test that branches are isolated from one another, go back to the SQL Editor, select the newly created branch from the branch selector drop-down in the top-right of the console, and run the following query: + +```sql +INSERT INTO elements VALUES ('Sodium', 11, 'Na'), ('Magnesium', 12, 'Mg'), ('Aluminum', 13, 'Al'), ('Silicon', 14, 'Si'), ('Phosphorus', 15, 'P'), ('Sulfur', 16, 'S'), ('Chlorine', 17, 'Cl'), ('Argon', 18, 'Ar'), ('Potassium', 19, 'K'), ('Calcium', 20, 'Ca'); +``` + +This query adds ten more items to the `elements` table. If you go back to the tables page, you will be able to compare the difference in data between the newly created branch and the main branch. + +## Creating a branch for every Preview environment + +Neon offers an [API](https://api-docs.neon.tech/reference/getting-started-with-neon-api?utm_source=devday&utm_medium=blogpost&utm_campaign=blog) that you can use to manage resources programmatically. You can use it along with a CI/CD tool to achieve the following flow: + +1. A developer will work on changes locally against a local Postgres instance or a Neon branch. These changes could potentially include schema changes. +2. They will then open a pull request which triggers a CI/CD workflow that does the following: + 1. Creates a Neon branch + 2. Applies database migrations if there are any + 3. Creates a preview deployment and uses the newly created Neon branch for the database +3. If everything looks good in the preview environment, the developer will merge their changes. This will trigger another CI/CD workflow that deploys the app to production. This workflow does the following: + 1. Applies the database migrations to the primary Neon branch + 2. Creates a production deployment + 3. Deletes the old Neon preview branch, which is no longer needed + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/creating-a-branch-for-every-preview-environment-1024x576-b99cb9f4.png) + +## Seeing the flow in action + +### Setting up the demo project locally + +To see the flow in action, we created a demo app that displays the data coming from the `elements` table that we created previously. The app is built using [Next.js](https://nextjs.org/), [Prisma](https://prisma.io), and Neon and is deployed to Vercel. + +For our CI/CD pipeline for this project, we’ll use GitHub Actions, which are automated workflows that you define as code. These workflows are written in YAML, and they live in a `.github` folder inside your project’s repository. + +To clone the demo app locally, run the following command: + +```bash +git clone https://github.com/neondatabase/preview-branches-with-vercel + +cd preview-branches-with-vercel + +npm install +``` + +You will then need to copy the `.env.example` file to a new file called `.env`. You can do that by running the following command: + +```bash +cp .env.example .env +``` + +Next, add the following environment variables which you can get from the Neon console. Add the database credentials: + +```bash +DATABASE_URL = "" # the pooled connection to the database. It has a `-pooler` suffix + +DIRECT_URL = "" # the direct connection string to the database +``` + +Finally, run the setup script, which creates the tables and runs a seed script: + +```bash +npm run setup +``` + +You can now start a development server by running the following command: + +```bash +npm run dev +``` + +This command starts a development server which will run at https://localhost:3000. You should see the following UI: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/seeing-the-flow-in-action-1024x576-66a421c6.png) + +### A look at the GitHub actions + +If you open the project in your text editor of choice, you will find that there are three files in the .github folder located at the root of your project: + +- `cleanup-preview.yml`: this workflow deletes Neon preview branches. It runs whenever a pull request is merged/closed +- `deploy-preview.yml`: this workflow creates and deploys to a preview environment. It runs whenever a pull request is opened. +- `deploy-production.yml`: this workflow deploys the app to production. It runs whenever a commit is pushed to the main git branch. + +Before diving into the code for these workflows, you must add a few secrets to your GitHub project’s repository. To do that, navigate to your repository in GitHub, and go to  Settings > Security > Secrets and variables > Actions. You will then need to add the following secrets: + +- `VERCEL_TOKEN` – this is the API key for Vercel. It will be used to create preview environments (also known as [Preview Deployments](https://vercel.com/docs/concepts/deployments/preview-deployments)) and deploy the app to production. You will find it in Vercel, in Settings > Tokens > Create Token. +- `NEON_API_KEY` – this is the API key for the Neon user, which will be used to create the branches. You can find it in your Neon account under Settings > Developer settings > Generate new API key. +- `NEON_PROJECT_ID` – this is the ID of the Neon project. You can find it in the Neon console on the Settings page. +- `DATABASE_URL` – this is the pooled connection string of the production database. It will be used by your app to run queries. You can find in it in the Connection Details widget on the Neon Dashboard. +- `DIRECT_URL` – this is the direct connection string to the database. Prisma Migrate requires this to run migrations reliably. You can find it in the Connection Details widget on the Neon Dashboard. +- `NEON_DATABASE_USERNAME` – the name of the database role that will be used when creating a branch. You can obtain the user name from your Neon connection string. +- `GH_TOKEN` – this is required for commenting on pull requests + +These are all of the secrets you will need. Now, here is an overview of the GitHub actions that this project uses. When you open the `deploy-preview.yml` file, you will see the following: + +```yaml +name: Deploy Preview + +on: [pull_request] + +env: + NEON_DATABASE_USERNAME: ${{ secrets.NEON_DATABASE_USERNAME }} # change this to your database username + GH_TOKEN: ${{ secrets.GH_TOKEN }} # Required for commenting on pull requests for private repos + NEON_API_KEY: ${{ secrets.NEON_API_KEY }} # You can generate a an API key in your account settings + NEON_PROJECT_ID: ${{ secrets.NEON_PROJECT_ID }} # You can find this in your project settings + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + +jobs: + deploy-preview: + permissions: write-all + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Get branch name + id: branch-name + uses: tj-actions/branch-names@v8 + + - name: Create Neon Branch + id: create-branch + uses: neondatabase/create-branch-action@v4 + with: + project_id: ${{ env.NEON_PROJECT_ID }} + # parent: dev # optional (defaults to your primary branch) + branch_name: preview/pr-${{ github.event.number }}-${{ steps.branch-name.outputs.current_branch }} + username: ${{ env.NEON_DATABASE_USERNAME }} + api_key: ${{ env.NEON_API_KEY }} + + - name: Run Migrations + run: | + touch .env + + echo DATABASE_URL=${{ steps.create-branch.outputs.db_url_with_pooler }}?sslmode=require >> .env + echo DIRECT_URL=${{ steps.create-branch.outputs.db_url }}?sslmode=require >> .env + + npx prisma generate + npx prisma migrate deploy + + - name: Install Vercel CLI + run: npm install --global vercel@latest + + - name: Pull Vercel Environment Information + run: vercel pull --yes --environment=preview --token=${{ env.VERCEL_TOKEN }} + + - name: Build Project Artifacts + run: vercel build --token=${{ env.VERCEL_TOKEN }} + + - name: Deploy Preview to Vercel + id: deploy + run: echo preview_url=$(vercel deploy --prebuilt --token=${{ env.VERCEL_TOKEN }}) >> $GITHUB_OUTPUT + + - name: Comment on Pull Request + uses: thollander/actions-comment-pull-request@v2 + with: + # GITHUB_TOKEN: ${{ env.GH_TOKEN }} + message: | + Vercel Preview URL:rocket:: ${{ steps.deploy.outputs.preview_url }} + Neon branch:elephant:: https://console.neon.tech/app/projects/${{ env.NEON_PROJECT_ID }}/branches/${{ steps.create-branch.outputs.branch_id }} +``` + +This workflow runs every time we create a pull request by listening to the pull_request event. It checks out into the repository and gets the pull request’s git branch name to use it in the Neon branch name. + +The workflow then creates a Neon branch using the official [create branch action](https://github.com/neondatabase/create-branch-action) that is provided by Neon. This action takes a Neon project ID, the name of the branch, the database username, and a Neon API Key. By default, branches are created from the primary branch, however you can choose another branch as the parent. + +When naming preview branches, we recommend following the naming convention `preview/pr--` to identify these branches easily. Example: `preview/pr-123-feat/new-login-screen`. + +The next step is applying database migrations. You get the newly created Neon branch’s connection string from the create branch step and then run any pending database migrations against the Neon preview branch. This example workflow uses [Prisma Migrate](https://www.prisma.io/docs/concepts/components/prisma-migrate), which is an imperative database schema migration tool. You can of course swap it out with your tool of choice. + +The next step is using the Vercel CLI to build and deploy a preview that uses the Neon preview branch. + +Finally, we comment on the pull request with the URL of the preview deployment, which we get from the deployment step, along with a link to the Neon console for the project branch. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/branching-with-preview-environments/a-look-at-the-github-actions-1024x576-26f04fa7.png) + +
+

It is a good idea to disable the automatic preview deployments that Vercel creates because they are now triggered using a GitHub action. To do that, you need to enable the “Ignored Build Step” field. Go to your Vercel project settings, select Git, and then add the following command: if [ "$VERCEL_ENV" == "production" ]; then exit 1; else exit 0; fi. This command causes Vercel to skip triggering a build if it’s a preview. You can learn more in this guide.

+
+ +When you open the `deploy-production.yml` file, you will see the following: + +```yaml +name: Deploy Production + +on: + push: + branches: + - "main" + +env: + VERCEL_TOKEN: ${{ secrets.VERCEL_TOKEN }} + DATABASE_URL: ${{ secrets.DATABASE_URL }} + DIRECT_URL: ${{ secrets.DIRECT_URL }} + +jobs: + deploy-production: + permissions: write-all + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Run Prisma Migrate + run: | + npx prisma generate + npx prisma migrate deploy + + - name: Install Vercel CLI + run: npm install --global vercel@latest + + - name: Pull Vercel Environment Information + run: vercel pull --yes --environment=production --token=${{ secrets.VERCEL_TOKEN }} + + - name: Build Project Artifacts + run: vercel build --prod --token=${{ secrets.VERCEL_TOKEN }} + + - name: Deploy Project Artifacts to Vercel + run: vercel deploy --prebuilt --prod --token=${{ secrets.VERCEL_TOKEN }} +``` + +This workflow runs every time we push a commit to the main branch on GitHub by listening to the push event.  Next, the workflow runs any pending database migrations against the production database. It then deploys the app to production. + +Finally, if you open the `cleanup-preview.yml` file, you’ll see the following: + +```yaml +name: Clean up Preview Deployment +on: + pull_request: + types: [closed] + +jobs: + delete-preview: + runs-on: ubuntu-latest + steps: + - name: Delete Neon Branch + uses: neondatabase/delete-branch-action@v3.1.3 + with: + project_id: ${{ secrets.NEON_PROJECT_ID }} + branch: preview/pr-${{ github.event.number }}-${{ github.event.pull_request.head.ref }} + api_key: ${{ secrets.NEON_API_KEY }} +``` + +This workflow runs when a pull request is closed, and it deletes the Neon preview branch that’s associated with the pull request using the official [delete branch action](https://github.com/neondatabase/delete-branch-action). This action takes the branch name or ID. + +## Conclusion + +In this tutorial, you learned how to provision a database for every preview environment by leveraging Neon’s branching feature along with GitHub Actions and Vercel. + +You can swap out GitHub actions with another CI/CD tool or swap out Vercel with another deployment provider, as long as they support the preview environment workflow.  If there are other deployment providers or CI/CD tools you would like us to cover, feel free to reach out to us in our [community forum](https://community.neon.tech/). + +If you want to try out this flow on Neon, you can [sign up today for free](https://console.neon.tech). No credit card required. diff --git a/content/blog/posts/bring-postgres-relationships-to-light.md b/content/blog/posts/bring-postgres-relationships-to-light.md new file mode 100644 index 0000000000..03435f8eac --- /dev/null +++ b/content/blog/posts/bring-postgres-relationships-to-light.md @@ -0,0 +1,115 @@ +--- +title: Bring Postgres relationships to light +description: >- + Entity-relationship diagrams help you understand how your tables relate to + each other +excerpt: >- + Have you ever stared at a complex database and thought, “What the F#!% did I + get myself into?” If so, you’re not alone. Tables multiply, relationships + tangle, and before you know it, you are lost in a monster of your own making, + or worse, somebody else’s. This is where something... +date: '2024-08-08T17:15:00' +updatedOn: '2024-08-08T17:15:03' +category: community +categories: + - community +authors: + - brandon-strittmatter +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/cover.jpg + alt: null +isFeatured: false +seo: + title: Bring Postgres relationships to light - Neon + description: >- + You can use Outerbase to automatically generate ERDs for your Neon database, + making it easier visualize your Postgres relationships. + keywords: [] + noindex: false + ogTitle: Bring Postgres relationships to light - Neon + ogDescription: >- + You can use Outerbase to automatically generate ERDs for your Neon database, + making it easier visualize your Postgres relationships. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/neon-postgres-relationships-1024x576-9b3b50a6.jpg) + +Have you ever stared at a complex database and thought, “What the F#!% did I get myself into?” If so, you’re not alone. Tables multiply, relationships tangle, and before you know it, you are lost in a monster of your own making, or worse, somebody else’s. + +This is where something like an Entity-Relationship Diagram (ERD) comes into play. It’s not new—database designers have been using them for decades. But they’re more relevant than ever in our interconnected, data-driven world. + +## What is an ERD? + +At its core, an ERD is a visual representation of your database. It shows: + +- Entities (usually tables) +- Attributes (columns in those tables) +- Relationships between entities (foreign keys) + +Imagine a simple blogging platform. You might have tables for Users, Posts, and Comments. An ERD would show these as boxes, with lines connecting them to illustrate relationships. At a glance, you can see that a User can have many Posts, and a Post can have many Comments. Or you might have a much more complex schema like in the example here. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/ad4nxdsq1eyvmkpnvyxomsds0wxc9l9odnet6maa9ogptqatd1bdsrvuwjdlxl0epkvzgemmjhdcdhgio8zxetvr5fk2yfmjzsrnxqpn04rye0no-gb1vagfcx1nxeqt9bw1gs9-fw5hiov8ziuhfw9g6xmksca-bd091171.png) + +## How ERDs can help you build your Postgres relations + +Understanding primary and foreign keys in Postgres is critical for data integrity and referential integrity. ERDs visually represent these key constraints, making it easier to manage and maintain your Postgres database. + +Database design is as much art as science. While an ERD can show you if your schema is normalized, it can’t determine if your design makes sense for your specific use case. That still requires thought and domain knowledge. As applications grow more complex, with new features and more stakeholders involved, an ERD can serve as a crucial map in this increasingly intricate landscape. + +Here are three concrete advantages of having ERDs at hand: + +#### Turn abstract database structures into visual representations + +By highlighting primary keys, which uniquely identify records, and foreign keys, which link tables, ERDs make it easier to comprehend the overall design and interactions within your Postgres database. + +#### Make schema design accessible + +Databases can be intimidating, especially for non-technical team members or beginners. ERDs provide a visual tool that simplifies database design, making it accessible for everyone, and ensuring that database structures are easily communicated and comprehended. + +#### Understand the current state + +ERDs make it much easier to visualize the existing models and relationships, essential when adding new features or scaling your application. Understanding how tables relate through primary and foreign keys makes it easier to design a robust, scalable schema. + +## Get ERDs automatically for your Neon databases via Outerbase + +Manually creating and maintaining ERDs can be tedious and time-consuming, but that’s where [Outerbase](https://www.outerbase.com/) comes in. Outerbase recently added a feature to automatically generate ERDs for your Neon database, making it easier than ever to build ERDs for Postgres: + +- No manual diagramming is required—your diagrams are built automatically. +- Your ERD updates automatically as your schema changes, always staying current. + +If you’re already using Neon to manage your database, [Outerbase is just one click away to visualize your table relationships.](https://neon.tech/blog/add-an-interface-to-your-neon-database-via-outerbase) + +## How to view your Neon database ERD + +Go to Neon Console > Integrations. [Add Outerbase](https://neon.tech/docs/guides/outerbase). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/ad4nxdavnikcpiqfvrokboeq0ha038paffonahyplhiqbvedsi0yhfiktahlajy7m1sput5kwjejan2vzeb0xwx3dvycv8ginga29-zfr0sfuzd1ltzp4j4ralj2gkikcoujwsirxc-bkvgmgu6bvi1vlduii-101247ee.png) + +Sign up for an Outerbase workspace or log in if you already have an account and give it access to your Neon database. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/ad4nxepweo2tblrowposnp5zheiut5h17s2oedf7adirkrqdmsmrjxjdizv4m75isoiskroetrbjkzwo6cijwhnm-rzr7t9fqpdmnvy1zt1viresbrmnrfiqrmq0yqu7jdl4kmbgijqbbzbuypobq-t67j690-422a4419.png) + +This will take you back to Neon, choose your database. Click “Open in Outerbase”. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/ad4nxdpxhdark8wdedmqggzwegnvoxeopvtqvh1fnj88j9jtdn8ucghssa-au2jq3uhh4ww46wvmc43nk3dmfgrggsr8ygh5llm5k0osq4x7jiowj9ly83h0jnbjkej9kqhtp6mfcmyjiezg-adxxomuv3gae-784e0e3d.png) + +Now find the database icon (bottom left) and then click on the “Relational Diagram”. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-postgres-relationships-to-light/ad4nxfik1dh0na6dyxkh2nvw7ihvdg2tm84b5af5vkcfqnqb4r9eownkoo0euslmj1oukifhxz5f-m37xjlopgwfezurnktdcb9q5swj1ucfpl7zr4hbwxqfppnfvf4pa2cenjpdcvyasr26u3mmlelnbn9zof-472a6473.png) + +That’s it, it’s that easy, you now have your ERD. Click on a table to see all of it’s relationships, filter schemas to drill down, and watch it update as you make changes – it’s a powerful way to visualize your database structure. + +Bonus: With your database now connected, you can do even more in Outerbase: build dashboards, collaborate with your team, and use AI to generate SQL queries. Each feature crafted to make your data management experience so much more powerful. + +## Looking forward + +Tools like Neon and Outerbase’s automatic diagram generation are making databases more accessible. We are moving away from the days when DBAs were the gatekeepers; now, every developer wants and needs database access. + +With Outerbase’s EZQL, auto-generating SQL is easier than ever. In the near future, you might even collaborate with AI to optimize your database schema, visualizing differences and improvements. + +This democratization of data is powerful. It allows developers to better understand and control their data structures, facilitates communication between technical and non-technical team members, and makes systems more transparent and robust. + +The next time you’re stuck on a database issue or explaining your data model to a colleague, generate a diagram with Outerbase. A picture is worth a thousand words, and with instant generation and automatic maintenance, there’s no reason not to visualize your database. diff --git a/content/blog/posts/bring-your-own-extensions-to-serverless-postgresql.md b/content/blog/posts/bring-your-own-extensions-to-serverless-postgresql.md new file mode 100644 index 0000000000..963be73c71 --- /dev/null +++ b/content/blog/posts/bring-your-own-extensions-to-serverless-postgresql.md @@ -0,0 +1,95 @@ +--- +title: Bring Your Own Extensions to Serverless PostgreSQL +description: Announcing Dynamic Extension Loading +excerpt: >- + Extensions in PostgreSQL are comparable to libraries in programming languages + or plugins in web browsers. They are pivotal in the PostgreSQL ecosystem, + providing additional functionalities ranging from encryption and AI to + handling time series and geospatial data. More complex ex... +date: '2024-01-17T14:07:36' +updatedOn: '2024-03-27T11:33:26' +category: postgres +categories: + - postgres + - community +authors: + - anastasia-lubennikova + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bring-your-own-extensions-to-serverless-postgresql/cover.png + alt: null +isFeatured: false +seo: + title: Bring Your Own Extensions to Serverless PostgreSQL - Neon + description: Announcing Dynamic Extension Loading + keywords: [] + noindex: false + ogTitle: Bring Your Own Extensions to Serverless PostgreSQL - Neon + ogDescription: >- + Extensions in PostgreSQL are comparable to libraries in programming + languages or plugins in web browsers. They are pivotal in the PostgreSQL + ecosystem, providing additional functionalities ranging from encryption and + AI to handling time series and geospatial data. More complex extensions can + transform PostgreSQL into a graph or analytical database, and some companies + even create custom […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bring-your-own-extensions-to-serverless-postgresql/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-your-own-extensions-to-serverless-postgresql/cover.png) + +Extensions in PostgreSQL are comparable to libraries in programming languages or plugins in web browsers. They are pivotal in the PostgreSQL ecosystem, providing additional functionalities ranging from encryption and AI to handling time series and geospatial data. More complex extensions can transform PostgreSQL into a graph or analytical database, and some companies even create custom private extensions for specific business logic. + +Neon’s compute in stateless PostgreSQL runs as a VM or a Kubernetes pod. The compute image comes with a list of [supported extensions](https://neon.tech/docs/extensions/pg-extensions). However, supporting a wide range of PostgreSQL extensions can pose performance and security risks in a multi-tenant serverless environment like Neon. This is why we are excited to announce we added [support for private and custom extensions](https://neon.tech/docs/extensions/pg-extensions#custom-built-extensions) using Dynamic Extension Loading. + +This feature is currently in beta on request only. You need to [have an account](https://console.neon.tech/) and [contact support](https://support@neon.tech) if you want to bring your own extensions to Neon. In this article, we’ll introduce Dynamic Extension Loading, its implementation, its benefits, and our future plans. + +## Extensions in PostgreSQL + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bring-your-own-extensions-to-serverless-postgresql/image-14-1024x511-63c80259.png) + +PostgreSQL is a robust and versatile database system that is further enhanced by its support for extensions. Some of the most popular extensions are [PostGIS](https://postgis.net/) for geolocation, [pg_stat_statement](https://www.postgresql.org/docs/current/pgstatstatements.html), or [pgvector](https://github.com/pgvector/pgvector) for vector similarity search. + +Extensions in PostgreSQL come in various forms: + +1. **SQL Object Packages**: These are the most common, comprising domain-specific data types, functions, triggers, etc. +2. **Procedural Languages**: Extensions like [PLPython](https://www.postgresql.org/docs/current/plpython.html) or [PLV8](https://github.com/plv8/plv8) enable the use of different programming languages within PostgreSQL. +3. **Internal API Enhancements**: Written in C, these powerful extensions can introduce new storage methods, volume replication, background jobs, and configuration parameters. +4. **Extensions in Other Languages**: Beyond C, extensions can be developed in languages like C++ or Rust, broadening the scope of functionality. + +To use an extension, it must be built against the correct major version of PostgreSQL. The installation involves placing files in the shared directory and library files in the libdir, paths that vary across platforms. After placing the files, the `CREATE EXTENSION` command is executed in the database, prompting PostgreSQL to locate and run the installation scripts for the extension. + +## Extension support limitations in serverless environments + +In Neon’s serverless PostgreSQL environment, each compute runs as an ephemeral Kubernetes pod or VM. A compute instance can be scaled up, down, and descheduled whenever the workload changes. Therefore, supporting a wide range of PostgreSQL extensions presents significant challenges such as: + +- **Compatibility**: Many extensions are not designed for serverless architectures, particularly those needing persistent storage or deep system integration, such as [pg_cron](https://github.com/citusdata/pg_cron) and [file_fdw](https://www.postgresql.org/docs/current/file-fdw.html). +- **Performance Issues**: Embedding all extensions in the compute image significantly increases its size, leading to slower start times and reduced performance. +- **Maintenance Overhead**: Traditional methods require frequent updates to the entire compute image for each extension update, causing potential service disruptions. +- **Security Risks**: A larger set of extensions in the base image increases the potential attack surface, especially with extensions that remain unused by many users. +- **Limited Customization**: The open-source nature of compute images restricts the inclusion of custom or closed-source extensions, limiting tailored solutions for specific customer needs. + +Therefore, the conventional method of bundling extension files into compute images is impractical due to the sheer number of extensions and the varied needs of users. This led us to rethink how we provide extensions with Dynamic Extension Loading. + +## Dynamic Extension Loading: A New Approach + +At Neon, we’ve addressed these challenges with our dynamic extension loading mechanism. Here’s how it works: + +1. **Building and Storing Extensions**: We build extensions in a separate repository and store the resulting files in an S3 bucket. +2. **Configuring Extensions**: Extensions are configured per user in the Neon control plane, enhancing customization. +3. **On-Demand Loading**: Compute instances download control files at startup, and library files are fetched as needed when extension functions are called. + +With Dynamic Extension Loading, private and default extensions can be added to compute instances without restarting, reducing maintenance overhead. Additionally, it brings performance benefits to Neon. Our plans with Dynamic Extension Loading include moving all default-supported extensions to the extension storage, resulting in a smaller compute image size and faster start times. + +## How to bring your own extension to Neon + +To request support for a Postgres extension, paid plan users can [open a support ticket](https://console.neon.tech/app/projects?modal=support). Free plan users can submit a request via the feedback channel on our [Discord Server](https://discord.com/invite/92vNTzKDGp). + +Our engineers will then evaluate the compatibility of your extensions with Neon, build it, and upload the artifacts to the extension storage once it pass all the security tests. + +## Conclusion + +This feature is currently in beta, with plans for general availability in the near future. This development marks a significant step forward in making PostgreSQL more adaptable and efficient in a serverless environment. + +What about you? Do you use PostgreSQL extensions in your projects? Join us on Discord and let us know which extensions you use and how we can enhance your PostgreSQL experience in the cloud. diff --git a/content/blog/posts/bringing-mcp-to-the-cloud.md b/content/blog/posts/bringing-mcp-to-the-cloud.md new file mode 100644 index 0000000000..82462c125f --- /dev/null +++ b/content/blog/posts/bringing-mcp-to-the-cloud.md @@ -0,0 +1,175 @@ +--- +title: Bringing MCP to the Cloud +description: How Neon Implemented a Remote MCP Server +excerpt: >- + We recently released Neon’s remote MCP server. In this post, we’ll dive deep + into how we implemented Neon’s remote MCP server. We’ll explore using + Server-Sent Events (SSE) for communication and integrating OAuth 2.1 for + authentication. So let’s dive in. Note: If you’d rather watc... +date: '2025-04-11T18:01:05' +updatedOn: '2025-04-16T19:12:50' +category: ai +categories: + - ai +authors: + - shridhar-deshmukh + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bringing-mcp-to-the-cloud/cover.png + alt: null +isFeatured: true +seo: + title: Bringing MCP to the Cloud - Neon + description: How Neon Implemented a Remote MCP Server + keywords: [] + noindex: false + ogTitle: Bringing MCP to the Cloud - Neon + ogDescription: >- + We recently released Neon’s remote MCP server. In this post, we’ll dive deep + into how we implemented Neon’s remote MCP server. We’ll explore using + Server-Sent Events (SSE) for communication and integrating OAuth 2.1 for + authentication. So let’s dive in. Note: If you’d rather watch a video about + remote MCP server implementation, there is a great […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bringing-mcp-to-the-cloud/cover.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bringing-mcp-to-the-cloud/image-1024x576-4c8f1bc1.png) + +We recently released Neon’s remote MCP server. In this post, we’ll dive deep into how we implemented Neon’s remote MCP server. We’ll explore using Server-Sent Events (SSE) for communication and integrating OAuth 2.1 for authentication. So let’s dive in. + +_Note: If you’d rather watch a video about remote MCP server implementation, there is a great video by Sentry’s founder, David Cramer, who explains his in details._ + + + +## What is MCP in the Context of Neon? + +The Model Context Protocol (MCP) is an open standard that allows large language models (LLMs) and AI agents to interact with external systems in a structured way. In simpler terms, MCP defines how an AI agent (like an IDE plugin or Claude Desktop app) can talk to an MCP **server** that exposes certain actions or tools. + +Neon’s MCP Server bridges natural language commands from an AI agent to actual Neon API calls on the Neon Postgres platform. For example, Neon’s MCP server provides “tools” for tasks like creating projects, running SQL queries, managing branches and others, allowing an AI assistant to execute commands like _“List all my Neon projects”_ or _“Create a new Postgres database called my_database”_ on behalf of a user. + +Originally, Neon’s MCP Server ran locally using (via `npx @neondatabase/mcp-server-neon`) along with the client. The server required a Neon API key for authentication. The MCP client (e.g. Claude Desktop) would spawn the server as a subprocess and communicate via standard I/O. This local MCP setup proved that an AI agent can manage Neon resources using natural language, but there were a couple of pain points that stood out: + +1. **Installation experience: Requiring manual installation and a Neon API key created friction for users.** +2. **Upgrades**: Users had to manually update the local MCP server to get access to new tools. Not ideal. +3. **Security**: Handing around long-lived API keys isn’t without a risk, and we wanted a more secure, user-friendly authentication method. + +There was another driver, too: the MCP spec itself was starting to evolve in this direction, with native support for remote transports and OAuth authorization flows. We wanted to build toward that future. + +## Designing the Remote MCP Architecture + +To enable remote access, the team had to implement the following: + +- **SSE + HTTP for bi-directional communication:** The local server used a direct stdio pipe to communicate with the AI client. We used a package called [mcp-remote](https://github.com/geelen/mcp-remote) that bridges local stdio-based MCP clients with remote MCP servers over HTTP/SSE. mcp-remote mimics a local server from the client’s perspective (via `stdio`), but under the hood it sends requests to the remote endpoint and pipes responses back. +- **An OAuth 2.1 Authorization Flow:** Rather than requiring a Neon API key in plaintext, the remote server implements an OAuth flow where users log in to Neon and authorize the MCP server. This way, the server obtains a Neon access token and issues its own token to the MCP client. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bringing-mcp-to-the-cloud/image-1-1024x640-0185ba53.png) + +Additionally, we: + +- Refactor the server to support both stdio (for local mode) and sse (for remote mode) operation. +- Introduce an Express.js web server to handle HTTP connections for SSE and OAuth. +- Implement the OAuth flow with Neon as the upstream provider, including support for refresh tokens. +- Use a persistent store to save OAuth client registrations, authorization codes, and tokens. +- Perform a security review focusing on OAuth best practices (PKCE, redirect URI validation, secure token handling). + +[Pull Request **#36: “Support for** **/sse** **and OAuth flow”**](https://github.com/neondatabase-labs/mcp-server-neon/pull/36) encapsulated most of this work, spanning changes across the codebase. Let’s break down how these pieces were implemented. + +## Implementing SSE for remote communication + +The MCP protocol involves a back-and-forth message exchange between client and server. The protocol allows you to use SSE for server→client messages, and HTTP POST for client→server messages. We added a new Express route at /sse to handle SSE connections, and a /messages endpoint for incoming messages. + +When a client connects, it issues a GET request to /sse with an Authorization header. The Neon MCP server treats this as the start of a session. In the implementation, the server code creates a new **SSE transport** instance for the connection and associates it with a unique session ID: + +```javascript +app.get('/sse', bodyParser.raw(), requiresAuth(), async (req, res) => { + const accessToken = extractBearerToken(req.headers.authorization as string); + const transport = new SSEServerTransport('/messages', res); + transports.set(transport.sessionId, transport); + logger.info('new sse connection', { sessionId: transport.sessionId }); + res.on('close', () => { + logger.info('SSE connection closed', { sessionId: transport.sessionId }); + transports.delete(transport.sessionId); + }); + // Create an MCP server instance bound to this user's Neon token and attach it: + const server = createMcpServer(accessToken); + await server.connect(transport); +}); +``` + +In this snippet, the server first extracts the bearer token (more on what token this is in the OAuth section). It then creates an SSEServerTransport (provided by the MCP SDK) bound to the Express response object, which will stream events to the client. The createMcpServer(accessToken) call builds a new instance of the Neon MCP server logic for this session, using the provided Neon API access token to authorize Neon API calls. Finally, it connects that MCP server to the SSE transport, which begins the protocol handshake over SSE. + +On the other side, the client (for example, Cursor or another IDE) opens an EventSource to https://mcp.neon.tech/sse and receives events. But how does the client send commands (like “create_project” or “run_sql”)? This is where the companion /messages endpoint comes in. The SSEServerTransport uses a **long-lived SSE channel for outbound data**, and expects clients to POST their requests to a separate URL. Neon’s implementation maintains an in-memory map of sessionId -> transport so that when a POST comes in, it knows which session to route it to: + +```javascript +app.post('/messages', bodyParser.raw(), async (req, res) => { + const sessionId = req.query.sessionId as string; + const transport = transports.get(sessionId); + if (!transport) { + logger.warn('No transport found for sessionId', { sessionId }); + return res.status(400).send('No transport found for sessionId'); + } + // Hand off the request to the transport to process the MCP message + await transport.handlePostMessage(req, res); +}); +``` + +The client includes the session ID (provided by the server via SSE welcome message or as part of the SSE URL) with each POST. The server then finds the right SSEServerTransport and lets it handle the message, which ultimately passes it to the MCP server logic for processing. Responses or results from that command are then sent back as SSE events. + +Under the hood, Neon’s MCP server code registers all the supported “tools” (actions like list_projects, create_branch, etc.) with the MCP protocol server object, and it ensures each tool’s handler is invoked with the proper Neon API client context. The refactoring introduced a pattern where each handler function receives a neonClient parameter, so it can make API calls on behalf of that specific session. For example, the handler for “create_project” now looks like: + +```javascript +async function handleCreateProject(neonClient: ApiClient, name: string) { + const response = await neonClient.createProject({ name }); + if (response.status!== 201) { + throw new Error(`Failed to create project: ${response.statusText}`); + } + return response.data; +} +``` + +All Neon API interactions are scoped to the neonClient created with the user’s access token, ensuring user isolation in the multi-session environment. + +## Integrating OAuth 2.1 for Secure Authentication + +The goal here is to allow users to connect and authorize the MCP server to act on their behalf without manually sharing API keys. We achieved this with an OAuth 2.1 authorization flow using the MCP Server as the **OAuth provider** for downstream and Neon as an OIDC client for upstream, and the MCP server itself acts as an OAuth **client** to Neon as well as an OAuth **server** to the MCP client. Let’s unpack that: + +- **Upstream provider (Neon)**: The user will log in to Neon (or use their Neon account session) to grant the MCP server access, just like granting permissions to a third-party app. +- **MCP server as OAuth client**: The MCP server is registered with Neon’s OAuth service (with a client ID and secret). It requests scopes to the Neon API on behalf of the user. +- **MCP server as OAuth server**: To the MCP **client** (the AI agent application), the Neon MCP server itself presents OAuth endpoints (/authorize, /callback, /token). The MCP client doesn’t need to know about Neon’s own OAuth; it only interacts with the MCP server’s endpoints to obtain an access token. + +We implemented the following endpoints to complete the authorization flow: + +- Dynamic **Client Registration** (**POST /register\*\***)\*\*: The server generates a client ID and secret for the MCP client and stores it. We use a simple storage via Keyv to save the client info. +- **Authorize** (**GET /authorize\*\***)\*\*: Neon’s server doesn’t directly validate the user here; instead it initiates the upstream Neon OAuth flow. The code parses the incoming request, packages the parameters (including a generated state and the PKCE challenge) and then redirects the user’s browser to Neon’s OAuth authorization page. +- **Callback** (**GET /callback\*\***)\*\* the MCP client receives a code from Neon’s MCP server. +- **Token Exchange** (**POST /token\*\***)\*\* – In the last step, the MCP client exchanges the MCP authorization code for an access token. . +- **Refresh Tokens** – The /token endpoint handles grant_type=refresh_token and returns a new access/refresh pair to the client. +- + +## Deployment + +For deployment, we evaluated AWS, Cloudflare, and Fly. We ended up using a Docker image and deployed the MCP server on Fly for simplicity. But we are exploring other platforms for future deployments. The server typically listens on a public URL (mcp.neon.tech) and clients can connect to the `/sse` endpoint. + +## Looking Ahead + +The remote MCP server is live. You can install it on Cursor or Windsurf as follow: + +```json +"Neon": { + "command": "npx", + "args": ["-y", "mcp-remote", "https://mcp.neon.tech/sse"] +} +``` + +Although remote servers simplify the installation process, it’s still at an early stage. There are a few known issues such as: + +1. **Dependency on Node.js**: As you can see on the command above, the server uses mcp-remote. The package makes it easy to migrate a local MCP server into a remote one, but it requires the client to have Node.js installed, which is not an ideal experience. +2. **Refresh tokens**: We noticed that MCP clients at times re-open the OAuth flow instead of using refresh tokens. + +As we expect a broader adoption of MCP servers in the future, by developers and non-developers, we are looking into simplifying the installation and configuration process and testing other tools to build MCP servers. + +If you’re building with our MCP or curious about plugging into the Neon platform with AI tools, we’d love to hear from you. And if you’re interested in the code, the whole journey is public at [neondatabase-labs/mcp-server-neon](https://github.com/neondatabase-labs/mcp-server-neon). + +Thanks for reading, and happy hacking! diff --git a/content/blog/posts/bringing-psqls-d-to-your-web-browser.md b/content/blog/posts/bringing-psqls-d-to-your-web-browser.md new file mode 100644 index 0000000000..9b15d78ac5 --- /dev/null +++ b/content/blog/posts/bringing-psqls-d-to-your-web-browser.md @@ -0,0 +1,103 @@ +--- +title: Bringing psql’s \d to your web browser +description: Neon’s SQL Editor now supports Postgres’s introspection commands +excerpt: >- + Different database systems provide different ways to list or describe the + things they hold. For instance, to find a particular table and column in + MySQL, you run SHOW TABLES followed by SHOW COLUMNS FROM my_table. In SQLite, + you do .tables and then .schema my_table. And in Postgr... +date: '2024-04-17T13:42:50' +updatedOn: '2024-04-17T14:04:42' +category: postgres +categories: + - postgres +authors: + - george-mackerron +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bringing-psqls-d-to-your-web-browser/cover.png + alt: null +isFeatured: false +seo: + title: Bringing psql's \d to your web browser - Neon + description: Neon’s SQL Editor now supports Postgres’s introspection commands + keywords: [] + noindex: false + ogTitle: Bringing psql's \d to your web browser - Neon + ogDescription: >- + Different database systems provide different ways to list or describe the + things they hold. For instance, to find a particular table and column in + MySQL, you run SHOW TABLES followed by SHOW COLUMNS FROM my_table. In + SQLite, you do .tables and then .schema my_table. And in Postgres, the + commands ared (for describe) followed by d […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bringing-psqls-d-to-your-web-browser/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/bringing-psqls-d-to-your-web-browser/image-34-1024x576-4e2ca775.png) + +Different database systems provide different ways to list or describe the things they hold. For instance, to find a particular table and column in MySQL, you run `SHOW TABLES` followed by `SHOW COLUMNS FROM my_table`. In SQLite, you do `.tables` and then `.schema my_table`. And in Postgres, the commands are `\\d` (for describe) followed by `\\d mytable`. + +# \\d wasn’t working + +When I got access to my first Neon Postgres database, almost the first thing I did was go to the web-based SQL Editor and type `\\d`. I was a little sad when the response I got back was: `ERROR: syntax error at or near "\\" (SQLSTATE 42601)`. + +It turns out that `\\d`, `\\l`, and their relatives are a psql feature. That is, these introspection commands are found in [the C code that implements the psql client](https://github.com/postgres/postgres/blob/master/src/bin/psql/describe.c), not the C code that implements the Postgres server. Each time you issue one of these commands in psql, you’re running a little local routine that constructs one or more SQL queries, sends them to execute on the server one by one, and stitches the results together into a nice little table. + +
+

You may already know that you can see those underlying SQL queries by running psql with the -E or --echo-hidden option. \d issues only one query, but try \d pg_a* and you’ll see dozens. This is a useful way to explore some of Postgres’ internals. It was helpful, for example, in putting together the queries Zapatos uses to fetch type information from your database.

+
+ +Of course, it’s not just `\\d` and `\\l`: there are lots of other useful backslash commands. You can get a brief cheat-sheet with `\\? `, or look up [more details in the Postgres docs](https://www.postgresql.org/docs/current/app-psql.html#APP-PSQL-META-COMMANDS). Some personal favourites are `\\dconfig`, which lists configuration parameter values; `\\du` and `\\dx`, which list users and installed extensions; `\\sf` and `\\sv`, which display the source of functions and views; and `\\h`, which provides a syntax reference for SQL commands. + +# \\d **works now** + +Fast-forward a year or two. Having put together Neon’s [serverless driver](https://neon.tech/docs/serverless/serverless-driver) — which runs in environments that don’t offer raw TCP connections, such as web browsers — I was tasked to upgrade our web-based SQL Editor to make use of it. This enables interactive sessions and transactions in the SQL Editor, and reduces memory usage on our back-end, amongst other things. + +
+

Easter egg alert! As a as result of this work, if you open your browser’s dev tools in the SQL Editor, you can also run queries there using the currently-connected serverless driver client. The client is a property of the window, named rawClient. It’s so named because all parsing has been turned off: it returns the raw Postgres text format for each data type. Try pasting in rawClient.query('SELECT now()').then(console.log), for instance.

+
+ +While moving the SQL Editor to the serverless driver, I took the opportunity to make a few other tweaks along the way. Probably the one I was most excited to get to work on was supporting the psql backslash commands. + +As I mentioned above, these commands are implemented in C. An obvious place to start might thus have been to use [emscripten](https://emscripten.org/) to compile that C to WASM, and I did begin by looking at that approach. But I also noticed that much of the code was pretty repetitive: lots of basic string manipulation to assemble SQL queries, and some simple logic to interpret what came back. So I wondered if we needed to actually compile the C, or whether we couldn’t instead directly translate the C to JavaScript. + +That second approach — translating C to JavaScript — was the approach I ended up taking: + +- I turned C syntax into JS syntax using a bunch of RegExp search-and-replace. For example, I replaced `int x` with `let x`; concatenated consecutive strings (`"a" "b"` becomes `"ab”`); turned the `->` arrow operator into `. `; and so on. (I felt a little bit dirty doing this with RegExps, and I did briefly look at using a C parser and transforming the AST instead … but RegExps were **so** much quicker). +- I reimplemented some C and Postgres functions. That includes dead simple ones like `function atoi(str) \{ return parseInt(str, 10); \}` or `function pg_tolower(ch) \{ return ch.toLowerCase(); \}`, and also some slightly more complex ones, like a minimal `sprintf` function, or the psql code to pretty-print a table. +- Last, I wrote down [examples of all the different psql introspection commands](https://github.com/neondatabase/psql-describe/blob/main/test/tests.txt), and a test script that compares their results in psql to the results generated by my code. I then went through these, one by one, until every last one matched. Almost all the required fixes were in places the original C code was doing something with pointers — pointer arithmetic, pointer dereferencing, and so on — without any direct JavaScript equivalent. It didn’t take too long this way to get a JavaScript implementation up and running. You can use it right now in the SQL Editor in the Neon console. You can also [see the code on GitHub](https://github.com/neondatabase/psql-describe), and do whatever you like with it (within the Postgres license terms) from there. + +
+

Easter egg alert! When running backslash-commands in the SQL Editor on a machine with a keyboard, hold the Shift key as you click ‘Run’ to see the underlying SQL commands interleaved with the output, just like you’d get when running psql with the -E or --echo-hidden option.

+
+ +I can’t promise that this is the only or necessarily even the best approach to the problem, and it would of course be nice in the long run to have an automated process to bring updates across from the psql C code. But it works, and I think it’s a good start. + +# Significant semicolons + +Once I had the introspection commands working from JavaScript, the other challenge in bringing them to the SQL Editor was to pick them out of a line-up of SQL queries. + +Neon’s SQL Editor can run several queries at a time, and shows one result tab for each. Previously, we could just send one big opaque SQL string to a server API, and let the server handle splitting it into individual queries. But now, instead, we need to do that splitting client-side, because the introspection commands have to be handled separately. + +It’s hopefully obvious why a simple `sql.split(';')` doesn’t work here. It will fail on the following string, for example, which is a single query followed by a comment: + +```sql +SELECT ';' /* select a ';' literal */ +``` + +There’s [an emscipten build of the Postgres query parser](https://github.com/pganalyze/pg-query-emscripten). But it currently only supports up to Postgres version 10. And it would kind of be using a sledgehammer to crack a nut, since we don’t need to fully parse each query: we only need to figure out which semicolons we care about and which ones we don’t. + +To figure out which semicolons are the significant ones, there are three syntax elements we need to understand: [identifiers, comments, and string literals](https://www.postgresql.org/docs/current/sql-syntax-lexical.html). + +- Identifiers are the simplest: they’re always double-quoted, and any (unlikely) double-quotes within them are doubled up. +- Comments are only a little trickier. Single-line comments extend from a double-dash to the end of a line. C-style `/* … */` block comments also exist, and can be nested. +- Strings are the most complex. They can be plain single-quoted, in which case the character-escape behaviour inside them depends on the Postgres server’s `standard_conforming_strings` configuration parameter. Or they can be ‘escape strings’, which have an `e` or `E` before the opening quote. Both these sorts of strings can combine across whitespace (but only if it includes a newline). Or, of course, they can be dollar-quoted strings. + +In any case, the upshot is that I wrote [another little open-source package](https://github.com/neondatabase/semicolons) to do only this much parsing. It makes heavy use of [sticky RegExps](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/sticky) — a JS parser-writer’s best friend, [as I’ve mentioned elsewhere](https://neon.tech/blog/parsing-json-from-postgres-in-js) — to efficiently locate both comments and statement-terminating semicolons in SQL string. + +We also allow backslash commands to be newline-terminated, which is just a small extra step on top of the work done by the package. + +This enables us to quickly split your SQL Editor selection into an array of SQL queries and backslash commands. As you may have already noticed, that means we can also now tell you ahead of time how many separate result tabs are going to be presented when you hit the Run button. + +If you have any feedback on any of these new features, please [let us know on Discord](https://discord.com/invite/92vNTzKDGp). diff --git a/content/blog/posts/btree_gist.md b/content/blog/posts/btree_gist.md new file mode 100644 index 0000000000..b53354632d --- /dev/null +++ b/content/blog/posts/btree_gist.md @@ -0,0 +1,347 @@ +--- +title: How and when to use btree_gist +description: Be a wizard of space and time with Postgres +excerpt: >- + The right indexes make big SQL queries fast. If you’ve been using Postgres for + more than 5 minutes, you’re almost certainly familiar with the everyday B-Tree + index. This can deal with data that has a one-dimensional ordering: numbers, + timestamps, text, and so on. And if you’ve ha... +date: '2024-07-08T13:29:31' +updatedOn: '2024-10-02T13:45:55' +category: postgres +categories: + - postgres +authors: + - george-mackerron +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/btree_gist/cover.jpg' + alt: null +isFeatured: false +seo: + title: btree_gist - How and when to use it - Neon + description: Be a wizard of space and time with Postgres + keywords: [] + noindex: false + ogTitle: btree_gist - How and when to use it - Neon + ogDescription: >- + The right indexes make big SQL queries fast. If you’ve been using Postgres + for more than 5 minutes, you’re almost certainly familiar with the everyday + B-Tree index. This can deal with data that has a one-dimensional ordering: + numbers, timestamps, text, and so on. And if you’ve had anything to do with + spatial data in PostGIS, or indeed […] + image: 'https://cdn.neonapi.io/public/images/pages/blog/btree_gist/social.jpg' +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/btree_gist/neon-wizard-btreegist-1024x576-eeb63d76.jpg) + +The right indexes make big SQL queries fast. If you’ve been using Postgres for more than 5 minutes, you’re almost certainly familiar with the everyday [B-Tree index](https://www.postgresql.org/docs/current/indexes-types.html#INDEXES-TYPES-BTREE). This can deal with data that has a one-dimensional ordering: numbers, timestamps, text, and so on. + +And if you’ve had anything to do with [spatial data in PostGIS](https://postgis.net/), or indeed with Postgres’ own [geometry types](https://www.postgresql.org/docs/current/datatype-geometric.html), you’ve probably also encountered [GiST-based indexes](https://www.postgresql.org/docs/current/gist-intro.html). These can handle 2D and higher-dimensional data, enabling speedy queries on proximity, containment, overlap, and so on. + +Both index types can work with multiple data items. For example, if you need to find people by their name and date of birth, you can create a multi-column B-Tree index that enables fast searching on the two together. + +But what if you need to search by multiple columns, and the data in those columns is a mix of one- and multi-dimensional? A case in point: what if you need to search simultaneously over space and time? Well, Postgres has you covered here too. + +The TL;DR is: use the [btree_gist](https://www.postgresql.org/docs/current/btree-gist.html) extension. Some context and a worked example follow. + +## Crimes + +This issue cropped up while I was working with the [detailed crime data published by UK police forces](https://data.police.uk/data/). The data set has one row per crime: each row has the crime’s type, a latitude and longitude (generalised slightly for anonymity), and the month and year it was reported. + +I needed a big join query to count crimes occurring near millions of specific point-locations at particular times. + +## Get the data + +The data I used was [the April 2017 archive](https://data.police.uk/data/archive/2017-04.zip) (which is the latest that goes all the way back to 2010). It’s organised into folders named by year and month (`2017-04`, `2017-03`, etc.). In each folder there are CSV files for each police force, and for a few different sorts of data. I was interested in street-level crime data, so the files I wanted look like `[year-month]/[year-month]-[police-force]-street.csv` (for instance, `2017-04/2017-04-thames-valley-street.csv`). + +When I’m exploring CSV data and loading it into Postgres, I generally find [`xsv`](https://github.com/BurntSushi/xsv) invaluable. Let’s use it to take a look at what we have in these CSVs. + +### How many crimes are recorded? + +```bash +> xsv cat rows **/*-street.csv | xsv count +38,595,130 +``` + +### What do the columns represent? + +```bash +> xsv headers 2010-12/2010-12-avon-and-somerset-street.csv +1 Crime ID +2 Month +3 Reported by +4 Falls within +5 Longitude +6 Latitude +7 Location +8 LSOA code +9 LSOA name +10 Crime type +11 Last outcome category +12 Context +``` + +### What are the types of crime? + +Note that I’m using `pv` with the row count calculated above to monitor progress, and we need the `--limit` argument because `xsv` frequency tables default to showing only the top 10. + +```bash +> xsv cat rows **/*-street.csv \ + | pv -l -s 38595130 \ + | xsv frequency --select 'Crime type' --limit 0 \ + | xsv table + +field value count +Crime type Anti-social behaviour 13869607 +Crime type Violence and sexual offences 4047849 +Crime type Other theft 3258648 +Crime type Criminal damage and arson 3129836 +Crime type Burglary 2866490 +Crime type Vehicle crime 2477833 +Crime type Other crime 2147319 +Crime type Shoplifting 1890071 +Crime type Violent crime 1673219 +Crime type Drugs 987402 +Crime type Public order 792808 +Crime type Robbery 391709 +Crime type Bicycle theft 369895 +Crime type Theft from the person 346421 +Crime type Public disorder and weapons 242145 +Crime type Possession of weapons 103878 +``` + +Now let’s load just the subset of data we need into Postgres. + +First, create the schema: + +```sql +> create database crimes_db; +CREATE DATABASE + +> \c crimes_db +You are now connected to database "crimes_db" as user "george". + +> create type crime_type as enum ( + 'Anti-social behaviour', + 'Violence and sexual offences', + 'Other theft', + 'Criminal damage and arson', + 'Burglary', + 'Vehicle crime', + 'Other crime', + 'Shoplifting', + 'Violent crime', + 'Drugs', + 'Public order', + 'Robbery', + 'Bicycle theft', + 'Theft from the person', + 'Public disorder and weapons', + 'Possession of weapons' +); +CREATE TYPE + +> create table crimes ( + month date, + longitude real, + latitude real, + crime crime_type +); +CREATE TABLE +``` + +Then, load in the data with `COPY … FROM`. Again I use `pv` to monitor progress. I use `xsv` to select specific columns. And I use `sed` to put the year-month dates into a format Postgres understands, by appending `-01` (i.e. the first day of the month) to each. + +```bash +> xsv cat rows **/*-street.csv \ + | pv -l -s 38595130 \ + | xsv select Month,Longitude,Latitude,'Crime type' \ + | sed -E 's/^([0-9]+-[0-9]+)/\1-01/g' \ + | psql crimes_db -c 'copy crimes from stdin csv header' +``` + +As quick check that the data have loaded OK, let’s see how crimes are distributed over years with a quick [crosstab](https://www.postgresql.org/docs/current/tablefunc.html): + +```sql +> create extension tablefunc; +CREATE EXTENSION + +> select * from crosstab( + $$ select crime, to_char(month, 'YYYY') as year, count(*) from crimes group by crime, year order by crime asc, year asc $$, + $$ select gs from generate_series(2010, 2017) gs $$ +) as ("Crime" text, "2010" int, "2011" int, "2012" int, "2013" int, "2014" int, "2015" int, "2016" int, "2017" int); + + Crime | 2010 | 2011 | 2012 | 2013 | 2014 | 2015 | 2016 | 2017 +------------------------------+--------+---------+---------+---------+---------+---------+---------+-------- + Anti-social behaviour | 201016 | 2792756 | 2369431 | 2191485 | 2035155 | 1875251 | 1850397 | 554116 + Violence and sexual offences | | | | 477713 | 839521 | 1049988 | 1232210 | 448417 + Other theft | | 251482 | 737010 | 582768 | 515839 | 505800 | 496783 | 168966 + Criminal damage and arson | | 207293 | 563360 | 526600 | 516930 | 549252 | 570616 | 195785 + Burglary | 37893 | 505841 | 476006 | 455384 | 427448 | 410673 | 409102 | 144143 + Vehicle crime | 29416 | 411012 | 392300 | 378911 | 356789 | 368599 | 394087 | 146719 + Other crime | 142705 | 1530533 | 182205 | 71454 | 53501 | 63296 | 74160 | 29465 + Shoplifting | | 104179 | 303366 | 321207 | 331093 | 339237 | 361982 | 129007 + Violent crime | 57580 | 729387 | 676732 | 209520 | | | | + Drugs | | 71007 | 208111 | 198252 | 177470 | 151401 | 137739 | 43422 + Public order | | | | 91823 | 152113 | 194164 | 254129 | 100579 + Robbery | 5731 | 75068 | 68042 | 60648 | 52416 | 51651 | 56042 | 22111 + Bicycle theft | | | | 73342 | 95021 | 88193 | 86670 | 26669 + Theft from the person | | | | 68442 | 81964 | 83056 | 83483 | 29476 + Public disorder and weapons | | 51555 | 147405 | 43185 | | | | + Possession of weapons | | | | 14133 | 21415 | 25168 | 31218 | 11944 +(16 rows) +``` + +This looks sensible enough. It also tells us that crime classifications changed a few times between 2010 – 2017, which is why some of the categories are a bit oddly overlapping. + +## Make it spatial + +The next step is to create a point geometry out of the latitude and longitude coordinates we’ve been given. + +As is annoyingly common, no [coordinate reference system](https://en.wikipedia.org/wiki/Spatial_reference_system) is specified in the data set. The [gov.uk](https://gov.uk/) standard is British National Grid (BNG). But that’s clearly not what we have here, since that would give us eastings and northings in metres, not latitude and longitude in degrees. + +For now, I’m going to guess we’ve got WGS84, which is the GPS coordinate system (EPSG code 4326). And then I’m going to transform to BNG (EPSG code 27700), because having the coordinates in metres [makes distance calculations quick and easy](https://postgis.net/workshops/postgis-intro/geography.html). + +Two small caveats: (1) I’m using PostGIS’ built-in parametric transformation between the two coordinate systems, which may be off by a few metres. For really precise work, I’d use [OSTN15](https://gis.stackexchange.com/questions/396967/using-ostn15-in-postgis). And (2) BNG isn’t really appropriate for the Northern Ireland data, but it should still work well enough here. + +```sql +> create extension postgis; +CREATE EXTENSION + +> select addgeometrycolumn('crimes', 'location', 27700, 'point', 2); + addgeometrycolumn +------------------------------------------------------ + public.crimes.location SRID:27700 TYPE:point DIMS:2 +(1 row) + +> update crimes set location = st_transform(st_setsrid(st_makepoint(longitude, latitude), 4326), 27700); +UPDATE 38595130 +``` + +The final step in loading the data is to update table statistics, helping the query planner make good decisions in what follows. + +```sql +> analyze crimes; +ANALYZE +``` + +## Ask questions + +Let’s start by counting crimes within 1km of Stratford station (near the Olympic Park) around the 2012 Olympics in London. Stratford is at grid point E 538566, N 184375, so: + +```sql +> \timing +Timing is on. + +> select count(*) from crimes + where st_dwithin(location, st_setsrid(st_makepoint(538566, 184375), 27700), 1000) + and month between '2012-07-01' and '2012-08-31'; + + count +------- + 1644 +(1 row) + +Time: 8639.014 ms (00:08.639) +``` + +Unsurprisingly, since there were so many more people about, it turns out that crime in these months is a bit higher than usual (the highest July/August crime count in any other year in the data set is 1,430). + +It also turns out that this is a pretty slow query, taking about 8 seconds on a fast MacBook Pro. That, of course, is because we haven’t indexed anything. + +If we `explain analyze` the query, there’s a `Parallel Seq Scan on crimes` in there. It’s nice that Postgres can parallelise it, but it’s still a sequential scan: every row of the database has to be retrieved and checked. Yawn. + +```sql +> explain analyze select count(*) from crimes + where st_dwithin(location, st_setsrid(st_makepoint(538566, 184375), 27700), 1000) + and month between '2012-07-01' and '2012-08-31'; + + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Finalize Aggregate (cost=200458563.81..200458563.82 rows=1 width=8) (actual time=7371.148..7371.993 rows=1 loops=1) + -> Gather (cost=200458563.60..200458563.81 rows=2 width=8) (actual time=7371.088..7371.988 rows=3 loops=1) + Workers Planned: 2 + Workers Launched: 2 + -> Partial Aggregate (cost=200457563.60..200457563.61 rows=1 width=8) (actual time=7351.030..7351.030 rows=1 loops=3) + -> Parallel Seq Scan on crimes (cost=0.00..200457563.48 rows=45 width=0) (actual time=6116.709..7351.010 rows=548 loops=3) + Filter: ((month >= '2012-07-01'::date) AND (month <= '2012-08-31'::date) AND st_dwithin(location, '0101000020346C0000000000008C6F204100000000B8810641'::geometry, '1000'::double precision)) + Rows Removed by Filter: 12864495 + Planning Time: 0.193 ms + Execution Time: 7372.042 ms +(10 rows) +``` + +## Index on space + +The first thing I tried, therefore, was a standard spatial index: + +```sql +> create index crime_location_idx on crimes using gist(location); +CREATE INDEX +``` + +When we now re-run the query, the time drops to 32 milliseconds — about 250 times quicker. An `explain analyze` shows that a Bitmap Index Scan is being used, which is [a kind of middle ground between a sequential and index scan](https://pganalyze.com/docs/explain/scan-nodes/bitmap-index-scan). + +```sql +> explain analyze select count(*) from crimes + where st_dwithin(location, st_setsrid(st_makepoint(538566, 184375), 27700), 1000) + and month between '2012-07-01' and '2012-08-31'; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Finalize Aggregate (cost=237562.75..237562.76 rows=1 width=8) (actual time=27.974..30.290 rows=1 loops=1) + -> Gather (cost=237562.53..237562.74 rows=2 width=8) (actual time=12.074..30.225 rows=3 loops=1) + Workers Planned: 2 + Workers Launched: 2 + -> Partial Aggregate (cost=236562.53..236562.54 rows=1 width=8) (actual time=3.980..3.981 rows=1 loops=3) + -> Parallel Bitmap Heap Scan on crimes (cost=759.01..236562.42 rows=45 width=0) (actual time=2.027..3.944 rows=548 loops=3) + Filter: ((month >= '2012-07-01'::date) AND (month <= '2012-08-31'::date) AND st_dwithin(location, '0101000020346C0000000000008C6F204100000000B8810641'::geometry, '1000'::double precision)) + Rows Removed by Filter: 16301 + Heap Blocks: exact=948 + -> Bitmap Index Scan on crime_location_idx (cost=0.00..758.98 rows=27525 width=0) (actual time=3.804..3.804 rows=50546 loops=1) + Index Cond: (location && st_expand('0101000020346C0000000000008C6F204100000000B8810641'::geometry, '1000'::double precision)) + Planning Time: 0.323 ms + Execution Time: 30.375 ms +(13 rows) + +Time: 31.234 ms +``` + +This is very much better. But the whole point of this post is that we can do better still. + +## Index on space and time + +For this particular search we should really be indexing over both space _and_ time. That will then allow Postgres to do a simple Index Scan encompassing all conditions simultaneously. + +For this, we use the [btree_gist](https://www.postgresql.org/docs/current/btree-gist.html) extension, which reimplements a BTree index using GiST infrastructure. + +Ordinarily, btree_gist has no advantage (and some disadvantages) over the native BTree implementation. What it’s useful for, though, is combining a BTree index with GiST index types. Which we can do like so: + +```sql +> create extension btree_gist; +CREATE EXTENSION + +> create index crime_month_location_idx on crimes using gist(location, month); +CREATE INDEX +``` + +When we now re-run the query, it takes about 8 milliseconds: almost 1000x quicker than the unindexed version, and about 4x quicker than the spatial-only index version. An `explain analyze` shows a simple Index Scan as the search method, which is what we were hoping to see. + +```sql +> explain analyze select count(*) from crimes + where st_dwithin(location, st_setsrid(st_makepoint(538566, 184375), 27700), 1000) + and month between '2012-07-01' and '2012-08-31'; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Aggregate (cost=12813.23..12813.24 rows=1 width=8) (actual time=8.047..8.048 rows=1 loops=1) + -> Index Scan using crime_month_location_idx on crimes (cost=0.54..12812.96 rows=109 width=0) (actual time=0.216..7.912 rows=1644 loops=1) + Index Cond: ((location && st_expand('0101000020346C0000000000008C6F204100000000B8810641'::geometry, '1000'::double precision)) AND (month >= '2012-07-01'::date) AND (month <= '2012-08-31'::date)) + Filter: st_dwithin(location, '0101000020346C0000000000008C6F204100000000B8810641'::geometry, '1000'::double precision) + Rows Removed by Filter: 149 + Planning Time: 0.368 ms + Execution Time: 8.094 ms +(7 rows) +``` + +Of course, there is a trade-off to be made, and in this case it’s primarily about disk space. Running psql’s `\\di+` and `\\dt+` commands we can see that the spatial-only index takes 1.9GB of storage, while the space-and-time index requires 2.7GB. This is a significant fraction of the size of the data itself: the `crimes` table is 5.5GB. + +Still, this is a nice, flexible technique that extends to various sorts of one-dimensional data that you might want to combine with a GiST index. I hope you find it useful. diff --git a/content/blog/posts/bugbounty-and-pentests-at-neon.md b/content/blog/posts/bugbounty-and-pentests-at-neon.md new file mode 100644 index 0000000000..e5a658a825 --- /dev/null +++ b/content/blog/posts/bugbounty-and-pentests-at-neon.md @@ -0,0 +1,74 @@ +--- +title: Bugbounty and Pentests at Neon +description: Joining forces with HackerOne to bulletproof our security +excerpt: >- + At Neon, security is at the core of everything we do. Our serverless platform + was built with a vision for innovation, but we also know that a commitment to + security is paramount. That’s why we’re excited to announce the launch of our + Neon’s Bug Bounty Program in partnership with... +date: '2024-11-25T16:43:38' +updatedOn: '2024-11-25T16:43:40' +category: company +categories: + - company +authors: + - busra-demir +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bugbounty-and-pentests-at-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: Bugbounty and Pentests at Neon - Neon + description: >- + We’re launching a Bug Bounty Program in partnership with HackerOne, offering + security researchers the opportunity to test Neon. + keywords: [] + noindex: false + ogTitle: Bugbounty and Pentests at Neon - Neon + ogDescription: >- + We’re launching a Bug Bounty Program in partnership with HackerOne, offering + security researchers the opportunity to test Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/bugbounty-and-pentests-at-neon/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/bugbounty-and-pentests-at-neon/neon-bugbounty-27fc83e5.jpg) + +At Neon, security is at the core of everything we do. Our serverless platform was built with a vision for innovation, but we also know that a commitment to security is paramount. **That’s why we’re excited to announce the launch of our Neon’s Bug Bounty Program in partnership with HackerOne.** + +We’ve designed this program to engage with talented security researchers and ethical hackers worldwide. By joining forces with HackerOne’s community, we can further enhance our security and maintain the trust our users place in us. + +## Why Launch a Bug Bounty Program? + +A secure database environment is critical for our users. As a company dedicated to open-source technology and continuous improvement, we recognize that cybersecurity threats evolve, and our defenses must be equally dynamic. While our internal security practices are robust, this program reflects our commitment to collaboration, welcoming researchers to help us identify vulnerabilities we might have missed. + +## Proactive Security Measures: Our 2024 Pentesting Results + +This year, we reinforced our security practices by conducting three comprehensive penetration tests across different areas of our infrastructure. These tests resulted in the identification of **58 vulnerabilities**, which were promptly resolved within our SLA timelines. Each finding was an opportunity to strengthen our platform, and by addressing them, we have further ensured the reliability of our systems. + +## Key Features of the Bug Bounty Program + +Our private Bug Bounty Program offers security researchers the opportunity to test specific, high-impact areas of our platform. Participants will be rewarded based on the severity and impact of their findings, and our security team is committed to swift communication and resolution for any identified vulnerabilities. + +Here’s a brief overview: + +- **Scope:** The program will cover core aspects of our Neon platform, focusing on components crucial to maintaining user security and privacy, such as authentication, data protection, and API security. In addition to the production environment, we’ll also include **staging in scope**, allowing researchers to test new functionality before it is launched in production. This ensures that potential vulnerabilities are identified and resolved early, reinforcing our commitment to proactive security and safeguarding our users’ trust. +- **Rewards**: Bounties range from **$150 to $3,000**, with rewards calculated based on the severity of each vulnerability.**We’ll increase the rewards** throughout the year based on the engagement. +- **Response Times**: We strive to acknowledge reports within 2 business days, triage within another 10 days, and issue bounty payments within 30 days, depending on the complexity. + +## Join Us in Our Mission for a Secure Database Platform + +Our goal is to create a resilient and secure platform, and we believe this can best be achieved by collaborating with the security community. Through this Bug Bounty Program, we aim to strengthen our platform and continue offering our users a trusted environment for their data. + +Security researchers and ethical hackers: we invite you to join us on this journey. Your insights and expertise will be instrumental in helping us fortify Neon and protect our users. + +**Together, we can make Neon a platform built not only for innovation but also for the highest standards of security.** + +If you have a Hackerone account, reach out to [security@neon.tech](mailto:security@neon.tech) with your account URL to be added to the program. + +Additionally if you found a vulnerability, submit it via [this form](https://hackerone.com/8777433c-7051-4c92-aed4-430278521656/embedded_submissions/new). + +--- + +_Neon is a serverless Postgres platform that helps you ship faster via instant provisioning, autoscaling, and database branching. We have a generous Free Plan – sign up [here](https://console.neon.tech/signup)._ diff --git a/content/blog/posts/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify.md b/content/blog/posts/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify.md new file mode 100644 index 0000000000..f9d5de7f1d --- /dev/null +++ b/content/blog/posts/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify.md @@ -0,0 +1,571 @@ +--- +title: 'Build a Dynamic E-Commerce Store UI with Astro, Neon Postgres, and AWS Amplify' +description: A step-by-step guide to building an e-commerce store UI in Astro and Postgres +excerpt: >- + This guide covers the step-by-step process of building an e-commerce store UI + powered by Neon in Astro and Postgres. Upon completing the guide, you will + understand how to build dynamic pages in Astro by querying your Postgres + database over HTTP requests and automating deployments... +date: '2024-04-09T12:02:05' +updatedOn: '2024-04-11T13:31:07' +category: community +categories: + - community +authors: + - rishi-raj-jain +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/cover.png + alt: null +isFeatured: false +seo: + title: >- + Build a Dynamic E-Commerce Store UI with Astro, Neon Postgres, and AWS + Amplify - Neon + description: >- + A step-by-step guide to building an e-commerce store UI in Astro and + Postgres + keywords: [] + noindex: false + ogTitle: >- + Build a Dynamic E-Commerce Store UI with Astro, Neon Postgres, and AWS + Amplify - Neon + ogDescription: >- + This guide covers the step-by-step process of building an e-commerce store + UI powered by Neon in Astro and Postgres. Upon completing the guide, you + will understand how to build dynamic pages in Astro by querying your + Postgres database over HTTP requests and automating deployments on Git + commits using AWS Amplify. Prerequisites To follow along this […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-10-1024x576-a7fc5dde.png) + +This guide covers the step-by-step process of building an e-commerce store UI powered by Neon in Astro and Postgres. Upon completing the guide, you will understand how to build dynamic pages in Astro by querying your Postgres database over HTTP requests and automating deployments on Git commits using AWS Amplify. + +## Prerequisites + +To follow along this guide, you will need the following: + +- [Node.js 18](https://nodejs.org/en) or later +- A [Neon](https://console.neon.tech/signup) account +- An [AWS](https://aws.amazon.com/free) account + +# Steps + +- [Provisioning a Serverless Postgres powered by Neon](#provisioning-a-serverless-postgres-powered-by-neon) +- [Create a new Astro application](#create-a-new-astro-application) + - [Add Tailwind CSS to the application](#add-tailwind-css-to-the-application) + - [Enabling Server Side Rendering in Astro with AWS Amplify](#enabling-server-side-rendering-in-astro-with-aws-amplify) +- [Setting up a Postgres Database Connection and Schema](#setting-up-a-postgres-database-connection-and-schema) + - [Set up the database connection](#set-up-the-database-connection) + - [Create the database client](#create-the-database-client) + - [Create mock data](#create-mock-data) + - [Create the database schema](#create-the-database-schema) + - [Test the database setup locally](#test-the-database-setup-locally) +- [Define the Astro application routes](#define-the-astro-application-routes) + - [Create a Shared Layout in Astro](#create-a-shared-layout-in-astro) + - [Building Index Route as Product Listing Page](#building-index-route-as-product-listing-page) + - [Building the Product Display Page](#building-the-product-display-page) +- [Deploy to AWS Amplify](#deploy-to-aws-amplify) + +## Provisioning a Serverless Postgres powered by Neon + +Using Serverless Postgres database powered by Neon helps you scale down to zero. With Neon, you only have to pay for what you use. + +To get started, go to the [Neon console](https://console.neon.tech/app/projects) and enter the name of your choice as the project name. You can pick a region near where you will deploy your Astro application. By default, version 16 of Postgres is used. Finally, click on **Create Project** to create the Postgres database named `neondb` (by default). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-1-1024x555-18d72445.png) + +You will then be presented with a dialog that provides a connecting string of your database. Click on **Pooled connection** on the top right of the dialog and the connecting string automatically updates in the box below it. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-2-1024x555-75efea36.png) + +All Neon connection strings have the following format: + +```bash +postgres://:@.neon.tech:/?sslmode=require +``` + +- `user` is the database user. +- `password` is the database user’s password. +- `endpoint_hostname` is the host with neon.tech as the [TLD](https://www.cloudflare.com/en-gb/learning/dns/top-level-domain/). +- `port` is the Neon port number. The default port number is 5432. +- `dbname` is the name of the database. “neondb” is the default database created with each Neon project. +- `?sslmode=require` an optional query parameter that enforces the [SSL](https://www.cloudflare.com/en-gb/learning/ssl/what-is-ssl/) mode while connecting to the Postgres instance for better security. + +Save this connecting string somewhere safe to be used as the `POSTGRES_URL` further in the guide. Proceed further in this guide to create a Astro application. + +## Create a new Astro application + +Let’s get started by creating a new Astro project. Open your terminal and run the following command: + +```bash +npm create astro@latest store-astro-postgres +``` + +`npm create astro` is the recommended way to scaffold an Astro project quickly. + +When prompted, choose: + +- `Empty` when prompted on how to start the new project. +- `Yes` when prompted if plan to write Typescript. +- `Strict` when prompted how strict Typescript should be. +- `Yes` when prompted to install dependencies. +- `Yes` when prompted to initialize a git repository. + +Once that’s done, you can move into the project directory and start the app: + +```bash +cd store-astro-postgres +npm run dev +``` + +The app should be running on [localhost:4321](https://localhost:4321/). + +Next, in your first terminal window, run the command below to install the necessary libraries and packages for building the application: + +```bash +npm install @neondatabase/serverless dotenv +npm install -D tsx +``` + +The above command installs the packages passed to the install command, with the -D flag specifying the libraries intended for development purposes only. + +The libraries installed include: + +- `@neondatabase/serverless`: Neon’s PostgreSQL driver for JavaScript and TypeScript. +- `dotenv`: A library for handling environment variables. + +The development-specific libraries include: + +- `tsx`: To execute and rebuild TypeScript efficiently. + +Further, make the following additions in your `tsconfig.json` file to make relative imports within the project easier: + +```bash +{ + "extends": "astro/tsconfigs/strict", ++ "compilerOptions": { ++ "baseUrl": ".", ++ "paths": { ++ "@/*": ["src/*"] ++ } ++ } +} +``` + +Let’s move on to integrating Tailwind CSS in the Astro application. + +## Add Tailwind CSS to the application + +For styling the app, you will be using Tailwind CSS. Install and set up Tailwind at the root of our project’s directory by running: + +```bash +npx astro add tailwind +``` + +When prompted, choose: + +- `Yes` when prompted to install the Tailwind dependencies. +- `Yes` when prompted to generate a minimal `tailwind.config.mjs` file. +- `Yes` when prompted to make changes to Astro configuration file. + +With choices as above, the command finishes integrating TailwindCSS into your Astro project. It installed the following dependency: + +- `tailwindcss`: TailwindCSS as a package to scan your project files to generate corresponding styles. +- `@astrojs/tailwind`: The adapter that brings Tailwind’s utility CSS classes to every `.astro` file and framework component in your project. + +Let’s move on to enabling server side rendering in the Astro application. + +## Enabling Server Side Rendering in Astro with AWS Amplify + +To fetch and render products dynamically, you’re going to enable server-side rendering in your Astro application. Execute the following command in your terminal: + +```bash +npm install astro-aws-amplify +``` + +The libraries installed include: + +- `astro-aws-amplify`: An adapter that prepares Astro websites to be deployed on AWS Amplify. + +Then, make the following additions in the `astro.config.mjs` file: + +```javascript +import tailwind from "@astrojs/tailwind"; +import { defineConfig } from 'astro/config'; ++ import awsAmplify from 'astro-aws-amplify'; + +// https://astro.build/config +export default defineConfig({ ++ output: "server", ++ adapter: awsAmplify(), + integrations: [tailwind()] +}); +``` + +The additions above begin with importing the default module of `astro-aws-amplify` and use it as the `adapter` with the Astro application. Also, it sets `output` configuration key as `server` so that the pages are generated dynamically, per request. + +Then, create a `amplify.yml` at the root of repository with the following code: + +```yaml +version: 1 +frontend: + phases: + preBuild: + commands: + - npm ci + build: + commands: + - env >> .env + - npm run build + - mv node_modules ./.amplify-hosting/compute/default + - mv .env ./.amplify-hosting/compute/default/.env + artifacts: + baseDirectory: .amplify-hosting + files: + - '**/*' + cache: + paths: + - node_modules/**/* +``` + +The code above represents a flow of commands that AWS Amplify would execute during the build phase. The commands in `preBuild` are ran before AWS Amplify reaches the build step. On reaching the build step, it executes the commands specified under the `build` section. The `build` commands are as follows: + +- `env >> .env`: Outputs all the environment variables into a file `.env` at the root of the project. +- `npm run build`: Invokes `astro build` to build the Astro project. +- `mv node_modules ./.amplify-hosting/compute/default`: Moves the `node_modules` directory from the root of the project to the `.amplify-hosting/compute/default` directory. +- `mv .env ./.amplify-hosting/compute/default/.env`: Moves the `.env` from the root of the project to the `.amplify-hosting/compute/default` directory. + +Let’s move on to setting up Postgres instance to insert and retrieve product(s). + +## Setting up a Postgres Database Connection and Schema + +In this section, you’ll learn how to configure a secure connection to the Postgres database, create a client to interact with it, and populate the tables in the database. + +### Set up the database connection + +Create an `.env` file in the root directory of your project with the following enviroment variable to initiate the setup of a database connection: + +```bash +# Neon Postgres Pooled Connection URL +POSTGRES_URL="postgres://:@.neon.tech:/?sslmode=require" +``` + +The file, `.env` should be kept secret and not included in Git history. Ensure that .env is added to the .gitignore file in your project. + +### Create the database client + +First, create a `postgres` directory in the `src` directory by running the following command: + +```bash +mkdir src/postgres +``` + +Then, to create a client that interacts with your serverless postgres, create a `setup.ts` file inside the `src/postgres` directory with the following code: + +```javascript +// File: src/postgres/setup.ts + +// Load the environment variables +import 'dotenv/config' + +// Load the postgres module +import { neon } from "@neondatabase/serverless"; + +// Create a in-memory SQL query function so that it's cached for multiple calls +export default neon(`${process.env.POSTGRES_URL}`); +``` + +The code imports the `dotenv` configuration, making sure that all the environment variables in the `.env` file are present in the runtime. Then, the code imports the `@neondatabase/serverless` library, retrieves the database URL from the environment variables, and uses it to create a new SQL query function, which is subsequently exported. + +### Create mock data + +In the `postgres` directory, create a file named `data.ts` with the following code: + +```javascript +// File: src/postgres/data.ts + +export interface ProductProps { + name: string; + slug: string; + price: string; + image: string; + currency: string; + description?: string; +} + +export const products: ProductProps [] = [ + { + name: "iPhone 9", + slug: "iphone-9", + price: "993", + image: "https://source.unsplash.com/random/300×300?a=1", + currency: "USD", + description: "An apple mobile which is nothing like apple", + }, + { + name: "MacBook Pro", + slug: "macbook-pro", + price: "1999", + image: "https://source.unsplash.com/random/300×300?a=6", + currency: "USD", + description: + "MacBook Pro 2021 with mini-LED display may launch between September, November", + }, +] +``` + +The code begins with exporting an interface that represents the fields associated with each product stored in the database. Further, it exports a `products` array that contains two sample products. Let’s use `postgres` instance to populate these products in the database. + +### Create the database schema + +In the `postgres` directory, create a file named `schema.ts` with the following code which will allow you to create and populate the products in a database table. + +```javascript +// File: src/postgres/schema.ts + +import { products } from "./data"; +import postgres from "@/postgres/setup"; + +async function createSchema() { + // Create the producs table if it does not exists + await postgres` + CREATE TABLE IF NOT EXISTS products ( + id SERIAL PRIMARY KEY, + image TEXT, + name VARCHAR(255), + price DECIMAL(10, 2), + currency VARCHAR(10), + slug VARCHAR(255), + description TEXT + ); + `; + await Promise.all( + products.map( + ({ image, name, price, currency, slug, description }) => + postgres`INSERT INTO products ( + image, name, price, currency, slug, description + ) VALUES ( + ${image}, ${name}, ${price}, ${currency}, ${slug}, ${description} + )` + ) + ); + console.log("Finished setting up the database."); +} + +createSchema(); +``` + +The code above defines how data will be stored, organized and managed in the database. Using the `postgres` database instance, it executes an SQL query to create a `products` table within the database if it does not already exist. This table comprises of seven columns: + +- An `id` column for storing random identifiers for each product in the table, generated automatically. +- An `image` column for storing absolute URLs of each product in the table. +- A `name` column for storing name of each product in the table. +- A `price` column for storing the cost of each product in the table, expected string to be without currency symbol. +- A `currency` column for storing the currency symbol of each product in the table. +- A `slug` column for storing a identifier of each product in the table, expected string to be in lowercase and contain no special characters. +- A `description` column for storing all the details of each product in the table. + +Further, the code loops over the existing `products` array and creates a SQL query for each to be inserted into the `products` table in your Postgres database. + +After executing the two SQL queries, a message is printed to the console if there’s an error during the execution. + +Finally, to execute the code in the schema file, make the following addition in the `scripts` of your `package.json` file: + +```json +{ + // ... + "scripts": { + // ... ++ "db:setup": "tsx src/postgres/schema.ts" + // ... + } + // ... +} +``` + +### Test the database setup locally + +To execute the code within `schema.ts` to set up the database, run the following command in your terminal window: + +```bash +npm run db:schema +``` + +If the command is executed successfully, you will see no logs in your terminal window except `Finished setting up the database. `, marking the completion of the schema setup in your Postgres Database powered by Neon. + +## Define the Astro application routes + +With Astro, creating a `.astro` file in the `src/pages` directory maps it to a route in your application. The name of the file created maps to the route’s URL pathname (with the exception of `index.astro`, which is the index route). + +The structure below is what our `src/pages` directory will look like at the end of this section: + +```bash +├── index.astro +├── p/ +├──── [slug].astro +``` + +- `index.astro` will serve responses to index route b dynamically fetching products from Neon Postgres. +- `p/[slug].astro` will serve responses to a product route by dynamically fetching product details from Neon Postgres. + +| **URL** | **Matched Routes** | +| ----------------- | -------------------------- | +| `/` | `src/pages/index.astro` | +| `/p/product-slug` | `src/pages/p/[slug].astro` | + +### Create a Shared Layout in Astro + +[Layouts in Astro](https://docs.astro.build/en/basics/layouts/) are built for providing the common HTML elements shared across pages. Each page is bound to have the basic HTML document elements. Further, with `` you’re able to specify where all the children elements (created in individual pages) be rendered. + +First, create a `layouts` directory in the `src` directory by running the following command: + +```bash +mkdir src/layouts +``` + +Then, create a file named `layout.astro` with the following code: + +```javascript +--- + + + + + + + +Astro + + +
+
+
+

Shop

+
+ +
+
+ + +``` + +The code above contains the basic HTML document elements and shared styled `body` elements. Further, it uses `` to render each page unique content below the title `Shop`. + +Let’s move on to creating the index route for listing all the products. + +### Building Index Route as Product Listing Page + +Open the `src/pages/index.astro` file and replace the existing code with the following: + +```javascript +--- +import postgres from "@/postgres/setup"; +import Layout from "@/layouts/Layout.astro"; +import type { ProductProps } from "@/postgres/data"; + +const products = + (await postgres `SELECT image, name, price, currency, slug FROM products;`) as ProductProps []; +--- + + +

Time to suit up

+
+ { + products.map(({ name, price, currency, image, slug }) => ( + + +
+

{name}

+ + {currency} + {price} + +
+ + )) + } +
+ +``` + +The code above begins with importing the `postgres` instance, the shared layout, and the interface representing each product fields. Further, it creates a SQL query using the `postgres` instance to obtain all the products stored in the database. Upon receiving a succesful response, HTML elements are rendered by looping over the products array. Each product is shown as a hyperlink (to it’s unique page) with it’s name, currency and price. + +Let’s move on to creating each product’s page, dynamically. + +### Building the Product Display Page + +Create a file named `src/pages/p/[slug].astro` file with the following code: + +```javascript +--- +import postgres from "@/postgres/setup"; +import Layout from "@/layouts/Layout.astro"; +import type { ProductProps } from "@/postgres/data"; + +const { slug } = Astro.params; + +const [product] = + (await postgres `SELECT * FROM products WHERE slug = ${slug}`) as ProductProps []; +--- + + + + ← Back To The Shop + +
+ +
+

{product.name}

+ {product.currency}{product.price} +
+ {product.description} +
+
+
+ +``` + +The code above begins with importing the `postgres` instance, the shared layout, and the interface representing each product fields. Further, it obtains the `slug` of the page from [`Astro.params`](https://docs.astro.build/en/reference/api-reference/#astroparams) API.`slug` is just a variable name for the dynamic part of the URLs after `/p/`. For example, in URLs ending with `/p/iphone` and `/p/samsung`, `iphone` and `samsung` would be the value of `slug`, respectively. Further, it creates a SQL query to find only the product whose `slug` field matches with the `slug` obtained from the URL. Using product details obtained, the HTML elements pertaining to the product are rendered. + +## Deploy to AWS Amplify + +The code is now ready to deploy to AWS Amplify. Use the following steps to deploy: + +- Start by creating a GitHub repository containing your app’s code. +- Then, navigate to the AWS Amplify Dashboard and click on **Get Started** under **Host your web app** section. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-3-1024x556-d183d4b5.png) + +- Select **GitHub** as the source of your Git repository. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-4-1024x553-71dc3923.png) + +- Link the new project to the GitHub repository you just created. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-5-1024x553-6ab41ab4.png) + +- Give a name to your project, and click on **Advanced Settings**. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-6-1024x555-8957c0f0.png) + +- In **Advanced Settings**, update the **Environment Variables** to match those in your local `.env` file, and `PORT` as 3000. Click **Next** to proceed. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-7-1024x556-dc03bf33.png) + +- Click **Save and Deploy** to deploy your website. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-8-1024x557-05f760b3.png) + +- Grab the deployment URL under the **Domain** title in the succesful build information. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-a-dynamic-e-commerce-store-ui-with-astro-neon-postgres-and-aws-amplify/image-9-1024x553-52e59750.png) + +## Summary & Final thoughts + +In this guide, you learned how to build a dynamic e-commerce store in Astro by using Serverless Postgres Database (powered by Neon) as the data source. Further, you learned how to prepare an Astro server-side rendered application to be deployed to AWS Amplify. + +For more, join us on [Discord](https://neon.tech/discord) to share your experiences, suggestions, and challenges. diff --git a/content/blog/posts/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres.md b/content/blog/posts/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres.md new file mode 100644 index 0000000000..8ecba7276e --- /dev/null +++ b/content/blog/posts/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres.md @@ -0,0 +1,481 @@ +--- +title: >- + Build and Deploy a Global Serverless Nuxt SSR App with Cloudflare Hyperdrive + and Postgres +description: >- + Deploy a Nuxt app on Cloudflare using Neon Postgres, Drizzle, and Cloudflare + Hyperdrive +excerpt: >- + Introduction In this post, we’ll create and deploy a Nuxt fullstack + application on Cloudflare Pages that uses server routes (API endpoints) to + access Neon Serverless Postgres with Cloudflare Hyperdrive. Cloudflare + Hyperdrive, now Generally Available, is a serverless application t... +date: '2024-04-05T17:40:09' +updatedOn: '2024-04-13T18:45:01' +category: community +categories: + - community + - workflows +authors: + - stephen-siegert +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/cover.jpg + alt: Neon Postgres with Cloudflare Hyperdrive +isFeatured: false +seo: + title: >- + Build and Deploy a Global Serverless Nuxt SSR App with Cloudflare Hyperdrive + and Postgres - Neon + description: >- + Deploy a Nuxt app on Cloudflare using Neon Postgres, Drizzle, and Cloudflare + Hyperdrive + keywords: [] + noindex: false + ogTitle: >- + Build and Deploy a Global Serverless Nuxt SSR App with Cloudflare Hyperdrive + and Postgres - Neon + ogDescription: >- + Introduction In this post, we’ll create and deploy a Nuxt fullstack + application on Cloudflare Pages that uses server routes (API endpoints) to + access Neon Serverless Postgres with Cloudflare Hyperdrive. Cloudflare + Hyperdrive, now Generally Available, is a serverless application that + proxies and accelerates queries to your database. This makes it faster to + access your data from across […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/social.jpg +--- + +![Neon Postgres with Cloudflare Hyperdrive](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/neon-hyperdrive-1024x576-edc2dd48.jpg) + +## Introduction + +In this post, we’ll create and deploy a [Nuxt](https://nuxt.com/) fullstack application on [Cloudflare Pages](https://pages.cloudflare.com/) that uses server routes (API endpoints) to access Neon Serverless Postgres with [Cloudflare Hyperdrive](https://developers.cloudflare.com/hyperdrive/). + +Cloudflare Hyperdrive, [now Generally Available](https://blog.cloudflare.com/making-full-stack-easier-d1-ga-hyperdrive-queues), is a serverless application that proxies and accelerates queries to your database. This makes it faster to access your data from across the globe, irrespective of your users’ location. Hyperdrive works by maintaining a globally distributed pool of database connections and routing queries to the closest available connection. + +Neon complements this architecture by providing a serverless Postgres database that scales compute resources automatically, optimizing performance based on demand. + +Hyperdrive is available as a binding in Cloudflare Pages when using Pages Functions _but_ there are some runtime compatibility caveats for setup. For example, full support for [node-postgres](https://node-postgres.com/) is not yet available. This situation can lead to some challenges, especially when running the Nuxt build command locally to create the deployment artifacts. With that in mind, the [Drizzle ORM Postgres HTTP Proxy](https://orm.drizzle.team/docs/get-started-postgresql#http-proxy) can be used to connect to Hyperdrive through a Cloudflare Worker that’s configured with the binding that uses Neon. + +This provides two options for integrating a Nuxt application with Neon Postgres on Cloudflare: + +1. Use the [Neon Serverless driver](https://neon.tech/docs/serverless/serverless-driver) and connect to Neon directly from the Nuxt server route with the [Cloudflare Pages Postgres](https://neon.tech/docs/guides/cloudflare-pages) integration. +2. Or, deploy a Worker configured with the Hyperdrive configuration that’s accessible using the Drizzle Postgres HTTP Proxy driver from the Pages app. + +In both scenarios, Drizzle can be used to access and query Neon databases. We’ll cover the second approach that uses _both Hyperdrive and Neon Postgres_. Using the Worker in this setup offers a flexible workaround for integrating other dependencies or services too, if needed. + +## Overview + +Below are the integration points between a Nuxt SSR app deployed to the Cloudflare Pages app that utilizes Neon Postgres configured with Cloudflare Hyperdrive. + +![Nuxt with Cloudflare Pages, Workers, Hyperdrive, and Neon Serverless Postgres](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/nuxt-hyperdrive-neon-drizzle-1024x243-024acad5.png) + +At a high level: + +1. A client request is made to the Nuxt server route +2. The server route uses Drizzle to submit a query to the Neon Postgres database +3. The database driver, configured with the Drizzle HTTP Proxy, points to the Cloudflare Worker configured with the Cloudflare Hyperdrive binding for Neon Postgres +4. Hyperdrive evaluates the query and executes or returns a cached response +5. That response is returned through the Nuxt Server Route to the requesting client + +To configure and deploy this solution, we’ll follow these steps: + +1. Create a Nuxt App +2. Configure the app for Cloudflare Pages deployment +3. Create a Hyperdrive configuration for Neon Postgres +4. Create the Drizzle ORM driver for the HTTP Proxy +5. Configure Cloudflare Worker Postgres HTTP Proxy service with Hyperdrive +6. Configure the database schema in the Pages app with Drizzle +7. Update the Nuxt server route to query the database and return results + +This approach can be used with other frameworks that support server-side compute functionality like [Remix](https://remix.run/), [Next](https://nextjs.org/), and [Astro](https://astro.build/).

_**Note:** _ ** _The ex_ ample app _should never be completely accessible to the internet in a production environment without security measures in place. This code will need to be modified for production use to restrict access to the Nuxt Server Route and the Postgres Proxy._** + +## Prerequisites + +This post assumes familiarity with Cloudflare, Drizzle, and Nuxt. Additionally, a Neon Postgres account is needed to create the Postgres database and a Cloudflare Workers Paid plan is needed to access the Hyperdrive functionality(the service itself is free). + +For Neon, visit the [Neon Console](https://console.neon.tech/), sign up, and create your first project by following the prompts in the UI. You can use an existing project or create another if you’ve already used Neon. + +## Frontend: Create the Nuxt App + +Start by creating a Nuxt app using `nuxi` to deploy to Cloudflare Pages. + +```bash +pnpm dlx nuxi@latest init nuxt-hyperdrive-ai +``` + +```bash +dependencies: ++ nuxt 3.11.1 ++ vue 3.4.21 ++ vue-router 4.3.0 + +Done in 17s +✔ Installation completed. +✔ Initialize git repository? +Yes +ℹ Initializing git repository... +Initialized empty Git repository in +/nuxt-hyperdrive-ai/.git/ + +✨ Nuxt project has been created with the v3 template. Next steps: + › cd nuxt-hyperdrive-ai + › Start development server with pnpm run dev +``` + +This Pages application will be the frontend along with the API routes using Nuxt server routes. + +Additionally, the `wrangler` dependency is needed if you’re testing the app locally (emulating the Cloudflare environment), and to deploy the built app to Cloudflare. + +```bash +pnpm add wrangler +``` + +Now update and save the `nuxt.config.js` to specify Cloudflare Pages as the deploy target using the `nitro preset`. This zero-config preset within the [Nitro](https://nitro.unjs.io/) server takes care of all the rearranging and build output configuration to make the app “deployable” to Cloudflare. Additionally, server-side rendering (SSR) routes are deployed to the correct resources. Meaning, compute will be associated with server routes, once created, automatically.

_Read more about the_ [Nitro Cloudflare Pages](https://nitro.unjs.io/deploy/providers/cloudflare) _preset._ + +```json +export default defineNuxtConfig({ + ... + nitro: { + preset: "cloudflare-pages", + }, +}); +``` + +Now, test the app locally by building and emulating the Cloudflare environment. First, commit your changes to the working branch (i.e.`main`) and build. + +```bash +git add . +git commit -m'init' + +pnpm build +pnpm dlx wrangler pages dev dist +``` + +_Note: Make sure to build the app before testing (and deploying) if there have been changes._ + +Once you’ve tested the app locally, deploy the app. + +```bash +npx wrangler pages deploy dist/ +``` + +And, verify the live deployed endpoint. + +![Nuxt Getting Started Page](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/welcome-to-nuxt-1024x755-685b0dce.png) + +## Backend: Create the Worker API with Hyperdrive integration + +Now that the base front-end portion of the app is up and running, create ** _a new_ ** Cloudflare Worker to act as a data proxy for a Hyperdrive configuration. + +### Create the Postgres HTTP Proxy Worker + +This application will use the [Hono](https://hono.dev/) web framework to manage the routes and endpoints. Hono is an _ultrafast_ framework optimized for serverless and edge environments. + +```bash + pnpm create hono postgres-hyperdrive-ai +``` + +```bash +create-hono version 0.6.2 +✔ Using target directory … postgres-hyperdrive-ai +? Which template do you want to use? cloudflare-workers +✔ Cloning the template +? Do you want to install project dependencies? yes +? Which package manager do you want to use? pnpm +✔ Installing project dependencies +🎉 Copied project files +Get started with: cd postgres-hyperdrive-ai +``` + +This worker will provide access to the Neon database (via Hyperdrive). Install [`node-postgres`](https://node-postgres.com/) into the project. + +```bash +pnpm add pg +pnpm add @types/pg -D +``` + +### Create the Postgres Hyperdrive Configuration + +In Neon, create a new project named **hyperdrive-ai**. This will be the Postgres cluster used with Hyperdrive. Optionally, [create a new role](https://neon.tech/docs/manage/roles) (i.e.**hyperdrive-ai**) for the connection configuration. + +Select the **Pooled connection** option and copy the connection string. + +![Neon Serverless Postgres project dashboard](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/neon-project-dashboard-1024x511-fe4ceae1.png) + +Within your new Worker’s project directory, create the Hyperdrive configuration using `wrangler`. + +```bash +npx wrangler hyperdrive create $NAME --connection-string="postgres://user:password@HOSTNAME_OR_IP_ADDRESS:PORT/database_name" +``` + +For example: + +```bash +npx wrangler hyperdrive create hyperdrive-postgres --connection-string="postgresql://:@ep-falling-breeze-a444fbohr-pooler.us-east-2.aws.neon.tech/neondb?sslmode=require" +``` + +This creates a new configuration similar to the output below: + +```bash +🚧 Creating 'hyperdrive-postgres' +✅ Created new Hyperdrive config + { + "id": "", + "name": "hyperdrive-postgres", + "origin": { + "host": "ep-falling-breeze-a544ohr-pooler.us-east-2.aws.neon.tech", + "port": 5432, + "database": "neondb", + "scheme": "postgresql", + "user": "hyperdrive-ai" + }, + "caching": { + "disabled": false + } +} +``` + +If you check in the Cloudflare console, the configuration will be present now. + +![Cloudflare Hyperdrive configuration](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/hyperdrive-configuration-1024x109-541460d0.png) + +The default caching behavior for Hyperdrive is set with a **max_age** of 60 seconds, during which cached query responses are served. Responses might be removed from the cache sooner if not frequently accessed. Additionally, the **stale_while_revalidate** setting allows Hyperdrive to serve stale cache results for an extra 15 seconds while updating the cache, typically completing revalidation quickly. The caching behavior can and should be tuned based on your app requirements. Additionally, it’s possible, using this pattern, to create more than 1 Hyperdrive configuration, if needed, to work across data sources.

With the **`id`** in the output from the command above, update the `wrangler.toml` with the Hyperdrive binding: + +```yaml +name = "postgres-hyperdrive-ai" +compatibility_date = "2023-12-01" + +node_compat = true # required for database drivers to function + +[[hyperdrive]] +binding = "HYPERDRIVE" +id = "" +``` + +Also, make sure that `node_compat = true` is present. This is required for the use of the `node-postgres` driver. + +**Save** the `wrangler.toml` file. + +This configuration will bind the Hyperdrive connection with the worker and allow the database to be queried via the Hyperdrive reference. + +Now that the binding is set up, the Worker handler can be configured to connect to Hyperdrive, which then queries the Neon database. This proxy will follow the pattern documented for the [Drizzle Postgres HTTP Proxy](https://orm.drizzle.team/docs/get-started-postgresql#http-proxy). Subsequently, in the next section, the Nuxt app from earlier will be modified to use Drizzle to point to this proxy from within the SSR API routes. This proxy will enable the Pages app to take advantage of Hyperdrive. + +Update the Worker `index.ts` code to add a `/query` route. + +```typescript +// index.ts + +import { Hono } from "hono"; +import { Client, QueryConfig } from "pg"; + +export type Env = { + HYPERDRIVE: Hyperdrive; +}; + +const app = new Hono<{ Bindings: Env }>(); + +app.post("/query", async (c) => { + const { sql, params, method } = await c.req.json(); + + const client = new Client({ + connectionString: c.env.HYPERDRIVE.connectionString, + }); + + await client.connect(); + + // prevent multiple queries + const sqlBody = sql.replace(/;/g, ""); + + try { + const result = await client.query({ + text: sqlBody, + values: params, + rowMode: method === "all"? "array": undefined, + } as QueryConfig); + + return c.json(result.rows); + } catch (e: any) { + return c.json({ error: e }, 500); + } +}); + +export default app; +``` + +Once running (and deployed), this worker API route will now: + +1. Bind the Hyperdrive configuration from the `wrangler.toml` to the Worker and make it accessible to the environment and available in the app `Context` object with `type Env = \{ HYPERDRIVE: Hyperdrive \};` +2. Listen for POST requests made to the `/query` endpoint +3. Parse the request parameters (these will come from Drizzle) +4. Creates a `node-postgres` `Client` connection to Hyperdrive (which is leveraging our Neon Postgres database) +5. Executes the requested query +6. Returns the results + +_**Note: The published endpoint should never be completely accessible to the internet in a production environment without security measures in place. This code will need to be modified for production use.**_ + +This portion of the architecture can now be deployed. To deploy using `wrangler`, run: + +```bash +pnpm run deploy +``` + +The API will now be live 🎉. Next, the full-stack frontend app will be integrated. + +## Update the Nuxt App to make a request to the Neon Hyperdrive Worker + +In the Nuxt Pages app (_nuxt-hyperdrive-ai_) Drizzle needs to be configured to point to the new Hyperdrive Neon proxy that was just created. + +Install `drizzle-orm` into the project: + +```bash +pnpm add drizzle-orm +``` + +### Configure Drizzle for the Hyperdrive Postgres Proxy + +In the root of the project, create `lib/db.ts`. Add the below Drizzle HTTP Proxy configuration and replace the requested endpoint with the Worker endpoint that was created earlier for the Hyperdrive proxy. + +```typescript +// lib/db.ts + +// Connect to the Neon Hyperdrive Postgres Proxy Server +// with the Drizzle ORM + +import { drizzle } from "drizzle-orm/pg-proxy"; + +export const db = drizzle(async (sql, params, method) => { + try { + const res = await fetch( + "", + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + sql, + params, + method, + }), + } + ); + + const rows = await res.json(); + + return { rows: rows || [] }; + } catch (e: any) { + console.error("Error from pg proxy server: ", e.response.data); + return { rows: [] }; + } +}); +``` + +The code above creates a Drizzle driver that uses the remote Worker proxy as the database. In this architecture, that means that queries using Drizzle will go to Hyperdrive, and Hyperdrive will query or cache the Neon Postgres responses. With this integration, the developer experience of using Drizzle will remain the same but Neon will be behind Hyperdrive.

Replace the **``** in the code above with your Workers URL. Consider using an environment variable or secret to inject this value once the deployment URL is known. Or, if using a custom domain, then both Page and Worker can be [mapped with routes on the same top-level domain](https://developers.cloudflare.com/workers/configuration/routing/routes/). + +The only thing that is missing now is the Drizzle schema configuration and updating the Nuxt server route to query and return data from our Neon database. + +### Update the Schema and Nuxt Server Route + +Create a table in your `hyperdrive-ai` Neon Postgres project and add records using the SQL statements below: + +```sql +CREATE TABLE "Element" ( + id TEXT PRIMARY KEY, + element_name TEXT NOT NULL, + symbol VARCHAR(3) NOT NULL, + atomic_number INTEGER NOT NULL + ); +``` + +Populate the table with the below records. + +```sql +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsneiyxk0001dksafvwnhm83', 'Helium', 2, 'He'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsneiz830002dksaxog1ut03', 'Lithium', 3, 'Li'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsneizic0003dksa8p71q9uo', 'Beryllium', 4, 'Be'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsneizsq0004dksa00ee6tvh', 'Boron', 5, 'B'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsnej03b0005dksa9m7p20qi', 'Carbon', 6, 'C'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsnej0dk0006dksa70fneon1', 'Nitrogen', 7, 'N'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsnej0nx0007dksafl3j0r98', 'Oxygen', 8, 'O'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsnej0yk0008dksa4uz6k194', 'Fluorine', 9, 'F'); +INSERT INTO "Element" (id, element_name, atomic_number, symbol) VALUES ('clsnej18x0009dksaljva7jqh', 'Neon', 10, 'Ne'); +``` + +![Rows in Neon Postgres Table Viewer](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/records-in-neon-db-1024x436-809b65f4.png) + +### Add the Drizzle Schema + +In `lib/db.ts`, add in the schema for the above `"Element`” table. This will be used to query against the above table structure. + +```typescript +// lib/db.ts +... + +import { integer, pgTable, text, varchar } from "drizzle-orm/pg-core"; + +export const Elements = pgTable("Element", { + id: text("id").primaryKey(), + name: text("element_name").notNull(), + symbol: varchar("symbol", { length: 3 }).notNull(), + atomicNumber: integer("atomic_number").notNull().primaryKey(), +}); + +... +``` + +In a production setup, consider setting up [database migrations with Drizzle and Neon](https://neon.tech/blog/api-cf-drizzle-neon) to speed up development iteration and harden your workflow. + +### Update the Nuxt Server Route + +Now, with the database configured and schema added to match, update the Nuxt server route to use the Drizzle db driver to query and return results from the endpoint. + +The server routes in Nuxt are effectively API endpoints that use server-side compute. The `server/` directory is specific to this functionality and is used to register API and server handlers to your application. + +In `server/api/hello.ts`, update the code to match the snippet below. + +```typescript +// server/api/hello.ts + +import { db, Elements } from "@/lib/db"; + +export default defineEventHandler(async ({ context }) => { + if (!db) { + return context.text("Could not connect to database"); + } + + const result = await db.select().from(Elements); + + return { + result, + }; +}); +``` + +Once this is saved. Build and deploy the Workers Pages app again. + +```bash +pnpm run build +npx wrangler pages deploy dist/ +``` + +_Note: Unlike Workers, Pages deployments get unique URLs. Make sure to check the latest endpoint after deployment._ + +After deployment, access the Nuxt server route endpoint by visiting the `/api/hello` route at `https://..pages.dev/api/hello`. + +![Nuxt Sever Route API](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/nuxt-api-results-1024x856-13aeb10c.png) + +After making requests, view the Hyperdrive cache metrics in the Cloudflare console. + +![Cloudflare Hyperdrive cache metrics](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/hyperdrive-cache-metrics-1024x116-fe26c7fe.png) + +![Cloudflare Hyperdrive Cache metrics](https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-global-serverless-nuxt-ssr-app-with-cloudflare-hyperdrive-and-postgres/hyperdrive-cache-metrics-2-1024x568-a6de83c5.png) + +## Cleanup + +Delete and remove the Cloudflare Pages app and worker. Additionally, remove the Hyperdrive configuration if no longer needed. + +## Conclusion + +In this post, we’ve walked through the process of creating a global serverless Nuxt SSR application, leveraging Cloudflare Hyperdrive and Neon Postgres. Hyperdrive optimizes the query routing to reduce latency, while Neon provides scalable, cloud-optimized serverless Postgres.

By integrating Cloudflare Hyperdrive, we’ve seen how to efficiently route queries to optimize our app’s responsiveness globally. The use of Nuxt, Drizzle, and Hono has streamlined the development process, offering a smoother experience.

To get started with incorporating Serverless Postgres with your Cloudflare infrastructure, sign up and [try Neon for free](https://console.neon.tech/signup). Follow us on [Twitter/X](https://twitter.com/neondatabase), join us on [Discord](https://neon.tech/discord), and let us know how we can help you build the next generation of applications. diff --git a/content/blog/posts/build-and-deploy-progressive-web-apps-with-glide-and-neon.md b/content/blog/posts/build-and-deploy-progressive-web-apps-with-glide-and-neon.md new file mode 100644 index 0000000000..5ab36f2bd2 --- /dev/null +++ b/content/blog/posts/build-and-deploy-progressive-web-apps-with-glide-and-neon.md @@ -0,0 +1,129 @@ +--- +title: Build and deploy progressive web apps with Glide and Neon +description: >- + The easiest way to build custom interfaces and dashboards for managing + Postgres data +excerpt: >- + Neon makes it easy to get scalable, fully-managed Postgres instances up and + running. It’s a dream for software developers. But what if you need to let + non-technical users manage your Postgres data? SQL clients are overwhelming, + and it takes too long to code something from scratch... +date: '2024-02-07T16:02:42' +updatedOn: '2024-02-29T11:24:07' +category: community +categories: + - community +authors: + - andy-claremont +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-progressive-web-apps-with-glide-and-neon/cover.png + alt: null +isFeatured: false +seo: + title: Build and deploy progressive web apps with Glide and Neon - Neon + description: >- + The easiest way to build custom interfaces and dashboards for managing + Postgres data + keywords: [] + noindex: false + ogTitle: Build and deploy progressive web apps with Glide and Neon - Neon + ogDescription: >- + Neon makes it easy to get scalable, fully-managed Postgres instances up and + running. It’s a dream for software developers. But what if you need to let + non-technical users manage your Postgres data? SQL clients are overwhelming, + and it takes too long to code something from scratch. Thankfully, those + aren’t your only options. You can give […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-and-deploy-progressive-web-apps-with-glide-and-neon/cover.png +--- + +Post image + +Neon makes it easy to get scalable, fully-managed Postgres instances up and running. It’s a dream for software developers. But what if you need to let non-technical users manage your Postgres data? + +SQL clients are overwhelming, and it takes too long to code something from scratch. + +Thankfully, those aren’t your only options. + +You _can_ give your users the freedom to access and manage Postgres data, _without_ having to teach them a complex tool or spending your time writing code. + +Here’s how. + +## Do it with Glide + +[Glide](https://glideapps.com?utm_source=neontech&utm_medium=referral&utm_campaign=ecosystem) lets you quickly build and deploy progressive web apps without code. You can think of it as the perfect front-end counterpart to Neon’s serverless back-end. + + + +Glide apps look and feel like native apps, but since they run in the browser, there’s nothing for your users to download or install. + +Here’s how to use Glide and Neon to build a progressive web app in under an hour. + +## Getting started with Neon + Glide + +### Connect your data + +Connect to your Neon-hosted database using Glide’s [PostgreSQL data source](https://www.glideapps.com/docs/postgresql). + +Once connected, you can add an entire table with live two-way sync, or run a custom query to pull a read-only version. The returned data will appear as a table in Glide’s Data Editor. + +If it’s a full table with two-way sync, you’ll be able to add and modify values in Glide, and see those changes pushed back to Neon. + +Post image + +You can connect to [multiple data sources](https://www.glideapps.com/docs/essentials/data-sources) in a single Glide app, too. + +Let’s say you want to combine data from Postgres with business data stored in Excel or Google Sheets. You can add those additional data sources and build relations between them using [computed columns](https://www.glideapps.com/docs/automation/computed-columns). + +Computed columns are Glide’s way of extending your data without affecting the data source. They let you run local calculations and operations along with AI, API calls, and 3rd party integrations. + +### Customize your layout + +With your data connected to Glide, you’re ready to build an interface with screens and components. + +[Components](https://www.glideapps.com/docs/essentials/components) are the building blocks of layouts in Glide. Use them to display values from your data, or to add input and interaction through forms and buttons. + +Post image + +You don’t need to fuss over styling details, either. Glide’s opinionated design system handles the heavy lifting by giving each component a set of appearance options. + +Glide’s Layout Editor is fully interactive, so you can toggle between mobile and large screen displays to test your interface while you build. + +### Add actions and workflows + +Components can trigger [actions](https://www.glideapps.com/docs/automation/actions), like showing a notification or updating data values. + +Use the Actions Editor to chain multiple actions together in a custom workflow with branching and conditional logic. These action workflows can then be assigned to components in the Layout Editor. + +Action sequences can use integrations. For example, you could use Glide’s [OpenAI integration](https://www.glideapps.com/docs/automation/integrations/openai-and-glide) to generate a prompt response, then store that prompt response as plaintext in a table. Your Glide app can then reference that value through a relation lookup. + +Post image + +### Publish and share + +The final step is to confirm your app settings. + +Review your appearance, privacy, and integration configurations, making final changes as needed. + +When you’re happy with your settings, hit Publish. + +Post image + +Glide hosts everything for you, so there’s no need to think about deployment. You can stick with the default subdomain that Glide provides, or use a custom domain. + +Published apps will automatically update when you make a change in the builder. This ensures your users always have the latest version of your app. You can also flip to the manual publishing mode if you’d rather push the changes live yourself. + +### Putting it all together + +Here’s an example of a Glide app connected to a Postgres table hosted on Neon. + + + +## What’s next? + +Neon is the easiest way for dev teams to get up and running with serverless Postgres. Likewise, Glide is the easiest way to build custom interfaces and dashboards for managing Postgres data. + +Dev teams use Glide for in-house solutions that would otherwise pull resources away from other projects. It’s the best of both worlds. You get a fully managed PWA with no DevOps overhead, and your users get a familiar UI that behaves like the other apps they use every day. + +[Learn more about Glide and start building for free.](https://glideapps.com?utm_source=neontech&utm_medium=referral&utm_campaign=ecosystem) diff --git a/content/blog/posts/build-internal-tools-neon-stackauth-vercel.md b/content/blog/posts/build-internal-tools-neon-stackauth-vercel.md new file mode 100644 index 0000000000..f27e526ac1 --- /dev/null +++ b/content/blog/posts/build-internal-tools-neon-stackauth-vercel.md @@ -0,0 +1,106 @@ +--- +title: 'Build Internal Tools Using Neon, StackAuth, and Vercel' +description: 'A template with a database, API routes, authentication, and access control' +excerpt: >- + Almost every tech company, from small startups to Fortune 500 enterprises + relies on internal tools. Larger organizations often have dedicated services + and structured procedures in place, but for many others, internal tools are… + messy. They often take the form of an insecure setup... +date: '2025-07-08T16:22:43' +updatedOn: '2025-10-06T16:07:05' +category: community +categories: + - community +authors: + - sam-harrison +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-internal-tools-neon-stackauth-vercel/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Build Internal Tools Using Neon, StackAuth, and Vercel - Neon' + description: >- + We built a template using Neon, StackAuth, and Vercel’s free plans to give + you a secure and scalable starting point for internal tools. + keywords: [] + noindex: false + ogTitle: 'Build Internal Tools Using Neon, StackAuth, and Vercel - Neon' + ogDescription: >- + We built a template using Neon, StackAuth, and Vercel’s free plans to give + you a secure and scalable starting point for internal tools. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-internal-tools-neon-stackauth-vercel/cover.jpg +--- + +Almost every tech company, from small startups to Fortune 500 enterprises relies on internal tools. Larger organizations often have dedicated services and structured procedures in place, but for many others, internal tools are… messy. They often take the form of an insecure setup hosted behind a VPN or worse, an overly complex solution cobbled together from a surprising number of often times costly AWS services. + +But if you’re like me, you just want to share tools easily, securely, cheaply, and without feeling like you’re managing a full-fledged production application (that’s already your 9 to 5). + +That’s what drove me to create [this template](https://github.com/sam-harri/internal_tooling_neon_stack). + +Built using Neon, StackAuth, and Vercel’s free plans, it provides a secure and scalable foundation, enabling you to quickly launch your next internal tool in minutes: [https://github.com/sam-harri/internal_tooling_neon_stack](https://github.com/sam-harri/internal_tooling_neon_stack) + +It comes preconfigured with a Postgres database, API routes, authentication, authorization, and a built-in admin panel – everything you need to get an application running fast. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-internal-tools-neon-stackauth-vercel/ad4nxfj6c0nuxt89qqfyneetach634xaq9jgw2q5ojp8ybftxjwapxvnojpvw1it1vltqu4xwmlk3fycosttbptlecppct9rdarwne8ml5kq5iskuynom8y2ufefxpkyh52ahjzaqca-9bee3474.gif) + +## Getting Started with the Template + +Using the template, you can get your internal tool up and running in minutes, and all you need are Neon, Stack Auth, and Vercel accounts. + +``` +git clone https://github.com/sam-harri/internal_tooling_neon_stack.git +``` + +Then open that new project in your editor of choice, and install the dependencies: + +``` +npm install +``` + +Next, set up the environment variables by copying over the .env.example to a new .env file. Fill these in with the connection string from your Neon database project in the console, and the Stack Auth keys from the Auth tab. + +Since the template comes with some admin logic already, you’ll need to sync your database by applying the schema: + +``` +npx drizzle-kit push +``` + +## Set Up Roles and Claim Your Project + +Head to the Auth tab in the Neon console and claim the project with your Stack Auth account. From there, create two project roles: `admin` and `user,` where `user` is contained within `admin`. Your Project Permissions section should look like this afterward: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-internal-tools-neon-stackauth-vercel/ad4nxestncszpceuwuozduko8kgejehm6oaqlmhkwlmbjpknte4dqmiiif4h9psez0kcwan0pl49qqnfss5fvzs7o1bgqoc3mzfoajqrnv7mnlwg2h6unwx98ionx8zinpnsvp8q-d59584e5.png) + +Once that’s done, boot up the project locally, log in, and head over to `/setup` to claim your admin privileges. + +You can then deploy your app with Vercel. Don’t forget to add your Vercel-provided domain to the allowed domains in Neon’s Auth tab, or your custom domain if you’re using one. + +## Authentication and Access Control with StackAuth + +You obviously don’t want just anyone to be able to access your internal tools. Luckily, Neon comes with an auth solution supporting social and password sign-on, role-based access control, and much more. + +In this template, only authorized users can access the tools. The application admin can whitelist domains, allowing anyone with an email from a specific domain (e.g.`yourcompany.tld`) and a verified email to sign up and get access. Likewise, you can grant and revoke access to specific external email addresses, perfect for contractors or partners. Emails can also be explicitly blocked even if their domain is whitelisted. + +Managing access is straightforward via the built-in admin panel at `/tools/admin`. Admins can assign or revoke privileges to other users, giving you complete control over your application’s access policies. These rules are meant to cover most of the common scenarios you might face, but since it’s a template, you can easily update it to add features like regex validation or subdomain whitelisting. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/build-internal-tools-neon-stackauth-vercel/ad4nxdcglkl6jusdo7m7dhoobjwx6q1nn0sb2lcudwrmm0ykgxkju98l9zooaue3txalrmsppij1phvcbirutr8ktgs-adomvkryohvcgwjhoxykcrpep7xt1froy7k9dsja7k-zca-68303c2f.png) + +## Database and Logic with Neon and Next.js + +Internal tools inherently require business logic and data storage. Neon provides a free serverless Postgres database, along with all the most popular Postgres extensions, and offers a serverless database driver that’s perfect for single-shot queries in Next.js serverless functions. + +For business logic, you can use Next’s server actions and API routes without the overhead of managing and deploying a separate API—ideal for simple internal tools. Vercel makes hosting your Next.js app painless, even allowing you to attach a custom domain so that you can host it at something like `yourtool.yourcompany.tld`. + +## Customize the App for Your Team + +Customization is designed to be simple. Just fill out your company-specific details in `config/app.ts`, then add your custom tools to the `app/tools` directory and declare them within the configuration file. To get started, play around with `app/tools/tool1/page.tsx` + +## Get Started + +[Clone the template](https://github.com/sam-harri/internal_tooling_neon_stack), connect your Neon and StackAuth accounts, and deploy to Vercel. Your next internal tool is just a few commands away! + +--- + +_Neon is a serverless Postgres platform built to help developers ship and scale faster via autoscaling, branching, and instant restores. We have a Free Plan – sign up [here](https://console.neon.tech/signup)._ diff --git a/content/blog/posts/build-with-confidence-with-schema-diff-protected-branches.md b/content/blog/posts/build-with-confidence-with-schema-diff-protected-branches.md new file mode 100644 index 0000000000..6ece36f16a --- /dev/null +++ b/content/blog/posts/build-with-confidence-with-schema-diff-protected-branches.md @@ -0,0 +1,143 @@ +--- +title: Build with confidence with Schema Diff & Protected Branches +description: Easily identify schema changes and safeguard sensitive data +excerpt: >- + Neon helps teams ship with confidence without compromising development + velocity. One of its features that contributes to that is database branching. + In this post, we will explore two new features related to database branching: + schema diff and protected branches. Database branches... +date: '2024-04-16T13:33:07' +updatedOn: '2024-04-17T14:21:28' +category: community +categories: + - community +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-with-confidence-with-schema-diff-protected-branches/cover.png + alt: null +isFeatured: false +seo: + title: Build with confidence with Schema Diff & Protected Branches - Neon + description: Easily identify schema changes and safeguard sensitive data + keywords: [] + noindex: false + ogTitle: Build with confidence with Schema Diff & Protected Branches - Neon + ogDescription: >- + Neon helps teams ship with confidence without compromising development + velocity. One of its features that contributes to that is database + branching. In this post, we will explore two new features related to + database branching: schema diff and protected branches. Database branches in + Neon are copy-on-write clones of your database that you can use for + development, […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/build-with-confidence-with-schema-diff-protected-branches/social.png +--- + +Post image + +Neon helps teams ship with confidence without compromising development velocity. One of its features that contributes to that is database branching. In this post, we will explore two new features related to database branching: schema diff and protected branches. + +Database branches in Neon are copy-on-write clones of your database that you can use for development, testing, or experimentation without compromising the original database. + +Ever since we introduced the database branching feature to Neon, developers have asked for ways to set permission rules on individual branches and identify differences between parent and child branches. And here it is. + +Today, we’ll explore two of our newest features, which will provide you with greater confidence. + +# Schema Diff + +Similar to diffs in Git, the Neon schema diff feature compares schemas between the current and past state of the branch. Schema diffs are important to development workflows as they allow you to easily track how your database schema has evolved for better debugging, code review, and cross-team collaboration. For example, you can compare schemas after your peer has merged their PR and applied migrations. + +Join us on [Discord](https://neon.tech/discord) and let us know what you think and how you use schema diff in your workflows. + +We detailed how Neon storage and ephemeral branches work in the [Point In Time Recovery Under the Hood in Serverless Postgres](https://neon.tech/blog/point-in-time-recovery-in-postgres#ephemeral-branches) article. In short, Neon’s storage engine saves Write-Ahead-Log records and can reconstruct a Postgres page at any given timestamp or Log-Sequence Number, allowing for time travel queries. + +Under the hood, schema diff creates short-lived, ephemeral branches (TTL=10 seconds) set at a specific time (and LSN), then queries the Pageservers to retrieve the past schema, and then compares it with the current one to effectively display the changes. + +Let’s see an example of schema diff on the Console. For that, we will create a Neon project and run the following query to create a user table: + +Post image + +```sql +CREATE TABLE users ( + user_id SERIAL PRIMARY KEY, + first_name VARCHAR(50), + last_name VARCHAR(50), + email VARCHAR(100) UNIQUE NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); +``` + +Once done, we can compare the current state of the database with the user table to its previous state, where I had no tables. To do so, we first need to navigate to the Restore page, select a branch, and click on Schema Diff. + +Post image + +Below is what the result will look like: + +Post image + +As expected, the previous state of my database is empty because I had just created the project. Let’s now modify my schema by adding a new `phone_number` column: + +```sql +ALTER TABLE users +ADD COLUMN phone_number VARCHAR(20); +``` + +Let’s compare again with the state of my database at 10:52 PM, the time after I ran the schema changes. The result should look like this: + +Post image + +We can observe that lines 60 and 61 were modified as a result of the schema change. + +# Protected branches + +Protected branches prevent unauthorized applications, users, and roles from accessing personally identifiable information (PII) or other sensitive data within your branch. This feature is available for users who are on [the Scale plan](https://neon.tech/pricing) + +The first feature that’s available in this release is “IP Allow”, which restricts database access exclusively to trusted IP addresses. We plan on introducing more rules in the future. + +If there are other ways we can protect your database branches, let us know on [Discord](https://neon.tech/discord) or [X](https://x.com/neondatabase). + +You have a limit of 5 protected branches in your project. To set your branch as protected, simply follow these steps: + +1. Go to the branches page. +2. Find the database branch you wish to protect. +3. Click on “More” and then select “Set as protected.” + +Post image + +Check out the documentation for [more details on protected branches](https://neon.tech/docs/manage/branches#protected-branch). + +# Conclusion + +The addition of Schema Diff and Protected Branches to Neon allows developers to easily identify schema changes and safeguard sensitive data, and equipped to build with confidence.
[You can try Schema Diff and Protected Branches on Neon](https://console.neon.tech) today. Join us on [Discord](https://neon.tech/discord), and let us know how we can help you build better and ship faster with Neon. diff --git a/content/blog/posts/building-301-pro-multi-cloud-with-neon-postgres.md b/content/blog/posts/building-301-pro-multi-cloud-with-neon-postgres.md new file mode 100644 index 0000000000..7cd4ec6a52 --- /dev/null +++ b/content/blog/posts/building-301-pro-multi-cloud-with-neon-postgres.md @@ -0,0 +1,102 @@ +--- +title: Building 301.Pro Multi-Cloud With Neon Postgres +description: Why I chose to move away from AWS RDS Postgres +excerpt: >- + “AWS RDS is overly complex to set up, especially when using fine-grained user + permissions. I tried Neon after talking to a few friends who had recommended + it and I was able to transition our entire analytics operation from AWS RDS to + Neon in about two hours. That tells you how di... +date: '2024-10-31T16:10:03' +updatedOn: '2024-11-14T07:28:19' +category: community +categories: + - community +authors: + - scott-cate +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-301-pro-multi-cloud-with-neon-postgres/cover.jpg + alt: null +isFeatured: false +seo: + title: Building 301.Pro Multi-Cloud With Neon Postgres - Neon + description: >- + We take you through the journey of building the backend analytics for + 301.Pro, a new link shortener, sharing all discoveries along the way. + keywords: [] + noindex: false + ogTitle: Building 301.Pro Multi-Cloud With Neon Postgres - Neon + ogDescription: >- + We take you through the journey of building the backend analytics for + 301.Pro, a new link shortener, sharing all discoveries along the way. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-301-pro-multi-cloud-with-neon-postgres/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-301-pro-multi-cloud-with-neon-postgres/neon-multi-cloud-1-c99354f2.jpg) + +
+

“AWS RDS is overly complex to set up, especially when using fine-grained user permissions. I tried Neon after talking to a few friends who had recommended it and I was able to transition our entire analytics operation from AWS RDS to Neon in about two hours. That tells you how different the UX was and the simplicity of Neon”

+
+ +Hi 👋 I’m Scott Cate and I’m the founder of [**301**](https://301.pro/).Pro, a new link shortener platform that brings advanced features like geolocation-based redirects and time-sensitive rules to the classic URL shortener. + +In this blog post series, I’ll take you through my journey of building the backend analytics for 301.Pro using Neon, sharing all the nerdy details and real-time discoveries along the way so we can learn together if you’re also a developer tackling your backend. + +This first post will cover the first steps right after moving from RDS to Neon. In the next one, we’ll dive into Neon’s fancier features—like [branching](https://neon.tech/flow)—and explore how I end up configuring my Neon setup of projects, branches, etc. + +## The App: 301.Pro – Adding a rules engine to Pro Links + + + +301.Pro is not your grandparents link shortener. At first glance, we have all the same-old-features other’s have—but then Pro Links take things up a notch with unique features that have never been seen before in a link routing rules engine. + +The lie of utm_source. If you have every setup utm_source in your expanded url, you knew it felt dirty because it’s never accurate. Example: Say you setup [301.pro/announcement](https://301.pro/announcement) as a Pro Link, and your expanded destination was [yourdomain.com/blog/announcement?utm_source=social](https://yourdomain.com/blog/announcement?utm_source=social). What is that hardcoded social? What about Which Social? What about Who on your team shared the link? The 301.Pro rules engine is full of helpful variables, that you set up ONE TIME and the rule works for every link. + +Imagine using this goodness! + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-301-pro-multi-cloud-with-neon-postgres/212-ec6aa5c1.jpg) + +This rule as a team or campaign rule will work for every link associated and automatically change the utm_source based on the referral if there is one. When there is no Referrer – the default of social kicks in to fill in the blank! How awesome is that? By the way – we know referer is spelled wrong, but we can’t change [internet legend](https://en.wikipedia.org/wiki/HTTP_referer)! + +Another example is geolocation-based redirects, which are one of the standout features of a Pro Link. Imagine you’re a company running a national ad promotion with one branded Pro Link or the same QR-Code (Every 301.Pro Link has it’s own customizable QRCode) you have printed on billboards and magazines. Instead of sending everyone to the same generic landing page, 301 Pro allows you to redirect users to the location nearest to them automatically. + +Another nice feature is time-sensitive rules. Think of setting up a Pro-Link to promote a webinar or online event. Before the event starts, the Pro Link directs users to a registration page, and after, it takes them straight to the recording. All while only setting up the Pro Link once, so the Pro LInk continues to be valuable after the webinar has completed. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-301-pro-multi-cloud-with-neon-postgres/36-f229ac2f.jpg) + +Thanks to these dynamic rules, 301.Pro isn’t just shortening links: it’s making them smarter and more efficient for your campaigns. A big piece of this is to also give you control and insights. For example, you can see detailed analytics for each click right from the 301 platform, not just a basic number. It’s what we call **Click Truth AnalyticsTM**. + +## The Backend: Multi-Cloud And Serverless + +When it came to building the backend for 301.Pro, I knew I wanted something fast, serverless, and scalable across multiple platforms. The whole app is built on a multi-cloud architecture, with half of the operations running in **AWS Lambda** and the other half deployed as **Edge Cloudflare Workers**. + +This setup lets us deliver lightning-fast results (under 100ms in most cases) no matter where the user is when they click your Pro LInk. Cloudflare’s CDN allows us to handle geolocation-based redirects at the edge to give users a more personalized experience, e.g., sending them to the nearest store location or tailoring content based on their time zone. + +## Setting up Neon For Analytics Collection + +Now, let’s talk about [Neon](https://neon.tech/), which handles the analytics for 301.Pro. I tried AWS RDS first, but AWS RDS is too expensive for the volume of Analytics and overly complex to set up, especially when using fine-grained user permissions. + +So, I tried Neon after talking to a few friends who had recommended it. **I was able to transition our entire analytics operation from AWS RDS to Neon in about two hours. That tells you how different the UX was and the simplicity of Neon.** The dashboard is amazingly simple to use. A detail I loved right away was the [browser based table viewer / editor.](https://neon.tech/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page) + +Neon also fits really well into this multi-cloud, serverless setup. Both AWS Lambda and Cloudflare can direct connect to the Neon, each with their appropriate credentials. You could accomplish this 100 different ways, but Neon was super simple and easy to understand. For 301.Pro, Neon sits perfectly in the middle of our Multi Cloud setup, acting as the centralized data hub for analytics collection and reporting. + +Neon also comes with connection pooling, which is essential for this setup. + +However, **there’s a tip I’d like to share regarding connection pooling**: + +When handling SET OPERATIONS (think database set, create tables, or application schema migrations), it’s better to not use connection pooling… Pooling is optimized for managing high volumes of short-lived, concurrent connections, not for long-running operations required for database setup. I found much better results in switching direct connections for the migrations and reserved pooled connections for handling analytics high-speed writes. This is a good little forcing function as for DBO connection permissions. + +In my opinion, applications should not be running with DBOwner permissions. I want to run with teh least permissions as possible, so designating a user with SET permissions that runs on the NON-POOLED connection string was awesome. That’s where my app migrations run, not my end user code execution. + +## The Next Steps: Branching Workflows And Branch vs Project Design + +Now that we have the basics covered, my next big focus is exploring some of the more advanced features Neon has to offer. First up, I’m looking into Neon’s [branching](https://neon.tech/docs/introduction/branching). The idea of being able to create [ephemeral environments for testing without duplicating entire datasets](https://neon.tech/blog/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora) sounds really cool, where I can branch off a main branch that I keep in sync with a testing dataset, run tests or experiments, and then delete those branches automatically. + +Also, I’m still figuring out whether I should create a separate Neon project per customer or manage them within branches. The key here is making sure each customer’s data is isolated for faster queries and better performance, especially as 301 Pro grows. We’ll share where I end up in a future blog post! + +--- + +_If you are new to Neon Postgres, we have a Free Plan – you can get started [here](https://console.neon.tech/signup). Don’t forget to check out [https://301.Pro](https://301.Pro) and to [follow Scott on X](https://x.com/scottcate) to see more of what he’s building._ diff --git a/content/blog/posts/building-a-cli-client-for-model-context-protocol-servers.md b/content/blog/posts/building-a-cli-client-for-model-context-protocol-servers.md new file mode 100644 index 0000000000..24dfcaf753 --- /dev/null +++ b/content/blog/posts/building-a-cli-client-for-model-context-protocol-servers.md @@ -0,0 +1,275 @@ +--- +title: Building a CLI Client For Model Context Protocol Servers +description: Going beyond Claude Desktop +excerpt: >- + The Model Context Protocol (MCP) keeps gaining traction in the AI space, and + since the launch of the Neon MCP Server (~2 weeks ago), the community has + built dozens of these servers across a wide spectrum of domains. However, the + Claude Desktop app has established itself as the de... +date: '2024-12-20T17:21:38' +updatedOn: '2025-03-06T15:13:40' +category: engineering +categories: + - engineering + - ai +authors: + - pedro-figueiredo +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-cli-client-for-model-context-protocol-servers/cover.jpg + alt: Building a CLI Client For Model Context Protocol Servers +isFeatured: true +seo: + title: Building a CLI Client For Model Context Protocol Servers - Neon + description: >- + Learn how to build an MCP CLI client to leverage the versatility of + Anthropic's protocol without the limitations of a desktop client. + keywords: [] + noindex: false + ogTitle: Building a CLI Client For Model Context Protocol Servers - Neon + ogDescription: >- + Learn how to build an MCP CLI client to leverage the versatility of + Anthropic's protocol without the limitations of a desktop client. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-cli-client-for-model-context-protocol-servers/social.jpg +--- + +![Building a CLI Client For Model Context Protocol Servers](https://cdn.neonapi.io/public/images/pages/blog/building-a-cli-client-for-model-context-protocol-servers/building-a-cli-client-for-model-context-protocol-servers-1024x576-1e7b300c.jpg) + +The Model Context Protocol (MCP) keeps gaining traction in the AI space, and since the launch of the [Neon MCP Server](https://neon.tech/blog/let-claude-manage-your-neon-databases-our-mcp-server-is-here) (~2 weeks ago), the community has built dozens of these servers across a [wide spectrum of domains](https://github.com/modelcontextprotocol/servers/#-third-party-servers). However, the Claude Desktop app has established itself as the default MCP Client, with most servers having exclusive instructions on how to integrate it with this client. + +But MCP is not coupled to Claude Desktop and it can be used with any other LLM client that supports it. With that in mind, we’ve decided to build an MCP CLI client that demonstrates this. This MCP client can be used to test MCP servers much more quickly as well. + +## How to build an MCP client + +All MCP Clients are built with the same core principles and follow the same protocol. For tool usage (our use case), these are the main concepts that need to be implemented: + +- **MCP Server Connection**: The first step is to connect to the MCP Server, so that it can discover and use the tools available on the server. + +```typescript +const mcpClient = new Client( + { name: 'cli-client', version: '1.0.0' }, + { capabilities: {} }, +); + +// This function will connect the MCP Client to the MCP Server +await mcpClient.connect(new StdioClientTransport(serverConfig)); +``` + +- **Tool Listing**: We need to fetch the available tools from the MCP Server. This allows the LLM to know which tools it can use during our interaction + +```typescript +// This function will return a list of tools available on the MCP Server +const toolsAvailable = await this.mcpClient.request( + { method: 'tools/list' }, + ListToolsResultSchema, +); +``` + +- **Tool Usage**: Once the LLM has decided which tool to use, we need to call its handler on the MCP Server. + +```typescript +// This function will call the tool handler on the MCP Server +const toolResult = await this.mcpClient.request( + { + method: 'tools/call', + params: { + name: toolName, + arguments: toolArgs, + }, + }, + CallToolResultSchema, +); +``` + +- **LLM Integration**: this is a multi-step process that connects the LLM to the available tools: + +1. Send the initial prompt to the LLM +2. Wait for the LLM to respond with a tool use +3. Call the tool handler on the MCP Server +4. Inject the tool result into the LLM’s context +5. Send the next prompt to the LLM + +And since we are using the [Tools API](https://docs.anthropic.com/en/docs/build-with-claude/tool-use) from the Anthropic API, it’s way simpler if we just rely on their [official SDK](https://github.com/anthropics/anthropic-sdk-typescript). + +```typescript +// 1- send the initial prompt +const response = await this.anthropicClient.messages.create({ + messages: [ + { + role: 'user', + content: 'Can you list my Neon projects?', + }, + ], + model: 'claude-3-5-sonnet-20241022', + max_tokens: 8192, + tools: this.tools, +}); + +for (const content of response.content) { + // 2- Wait for the LLM to respond with a tool use + if (content.type === 'tool_use') { + const toolName = content.name; + const toolArgs = content.input; + // 3- Call the tool handler on the MCP Server + const toolResult = await this.mcpClient.request( + { + method: 'tools/call', + params: { + name: toolName, + arguments: toolArgs, + }, + }, + CallToolResultSchema, + ); + + // 4- inject the tool result into the LLM's context + const contextWithToolResult = [ + ...previousMessages, + { role: 'user', content: toolResult.content }, + ]; + + // 5- Send the next prompt to the LLM + const nextResponse = await this.anthropicClient.messages.create({ + messages: contextWithToolResult, + model: 'claude-3-5-sonnet-20241022', + max_tokens: 8192, + }); + } +} +``` + +## Building the CLI Client + +Once we have all the core pieces in place, all we need to do is to build a cool CLI client that can be used to interact with the MCP Server. + +1. **LLM handling** – Handle the LLM messages and tools usage + +It’s important that we persist the messages between each interaction, so that we can inject the tool result into the LLM’s context. + +```typescript +private async processQuery(query: string) { + try { + // 1 - Send the user's query to the LLM + this.messages.push({ role: 'user', content: query }); + const response = await this.anthropicClient.messages.create({ + messages: this.messages, + model: 'claude-3-5-sonnet-20241022', + tools: this.tools, + }); + + // 2 - Handle the LLM response + for (const content of response.content) { + if (content.type === 'text') { + process.stdout.write(content.text); + } + + // 3 - Handle the tool use + if (content.type === 'tool_use') { + const toolResult = await this.mcpClient.request({ + method: 'tools/call', + params: { + name: content.name, + arguments: content.input, + } + }); + + // 4 - Add the tool result to the conversation + this.messages.push({ + role: 'user', + content: JSON.stringify(toolResult) + }); + + // 5 - Get Claude's response to the tool result + const nextResponse = await this.anthropicClient.messages.create({ + messages: this.messages, + model: 'claude-3-5-sonnet-20241022' + }); + + // 6 - Display Claude's response + if (nextResponse.content [0].type === 'text') { + process.stdout.write(nextResponse.content [0].text); + } + } + } + } catch (error) { + console.error('Error during query processing:', error); + } +} +``` + +2\. **Chat Loop** – Create a chat loop that will be used to send messages to the LLM and handle the response. + +```typescript +private async chat_loop() { + while (true) { + try { + const query = (await this.rl.question(styles.prompt)).trim(); + // process the query + await this.processQuery(query); + } catch (error) { + console.error(styles.error('\\\\nError:'), error); + } + } +} +``` + +3\. **Entry Point –** Setup a main entry point for the client that will initialize the MCP Client, fetch the tools and start the chat loop + +```typescript + // This is the main entry point for the client + async start() { + try { + console.log(styles.info('🤖 Interactive Claude CLI')); + console.log( + styles.info(`Type your queries or "${EXIT_COMMAND}" to exit`), + ); + + // 1 - Connect the MCP Client to the MCP Server + await this.mcpClient.connect(this.transport); + + // 2 - Fetch the tools available on the MCP Server + await this.initMCPTools(); + + // 3 - Start the chat loop + await this.chat_loop(); + } catch (error) { + console.error(styles.error('Failed to initialize tools:'), error); + process.exit(1); + } finally { + this.rl.close(); + process.exit(0); + } + } +``` + +4\. **Run –** Start the client + +Now that we have built an all-purpose MCP Client, we can run it by passing the MCP Server URL and whatever other arguments it needs. + +```typescript +const cli = new InteractiveCLI({ + command: '../dist/index.js', + args: ['start', process.env.NEON_API_KEY!], +}); +cli.start(); +``` + +### Improvements + +There are 2 main caveats with this simple implementation: + +- **Streaming**: This client doesn’t support streaming, so the responses may seem a bit slower from a user perspective. +- **Multiple Tool Calls**: This client doesn’t follow up on multiple tool calls, it will always stop after the first tool call. + +Luckily, both of these issues have been solved in the [MCP Client CLI](https://github.com/neondatabase/mcp-server-neon/tree/main/mcp-client) that we built at Neon. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-cli-client-for-model-context-protocol-servers/ad4nxcswahsks5tljrl3rrv42qdku20znqw9wnoorsrpp1axeojfwziqafnnk2ui3iyubnbpqalpcjlauh80z9i6u0iirhutjeaeibvcjlwtoeqsabovz3ajo7cu6vdvmon60adrashg-e8c09142.gif) + +## Try it + +Use this tool with any MCP Server to see how it works or use it as a base to build your own MCP Client. You can check out our GitHub [repository](https://github.com/neondatabase/mcp-server-neon/tree/main/mcp-client), and give us any feedback on [our Discord server](https://neon.tech/discord)! + +--- + +_Neon is a serverless Postgres platform that helps teams ships faster via instant provisioning, autoscaling, and database branching. We have a Free Plan – you can [get started](https://console.neon.tech/signup) without a credit card._ diff --git a/content/blog/posts/building-a-deep-research-agent-with-neon-and-durable-endpoints.md b/content/blog/posts/building-a-deep-research-agent-with-neon-and-durable-endpoints.md new file mode 100644 index 0000000000..bcdb2dc997 --- /dev/null +++ b/content/blog/posts/building-a-deep-research-agent-with-neon-and-durable-endpoints.md @@ -0,0 +1,321 @@ +--- +title: Building a Deep Research Agent with Neon and Durable Endpoints +description: 'Using Inngest, Claude Code, and Neon to vibe code our own agent' +excerpt: >- + Every AI lab is shipping research agents. OpenAI’s Deep Research, Perplexity, + and Gemini’s research mode. These products are not simple RAG pipelines. + Recent papers like DeepResearcher and Step-DeepResearch formalize what makes + them work: a recursive loop of planning, searching,... +date: '2026-02-24T17:14:34' +updatedOn: '2026-02-26T17:50:50' +category: community +categories: + - community + - ai +authors: + - charly-poly +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/cover.jpg + alt: null +isFeatured: false +seo: + title: Building a Deep Research Agent with Neon and Durable Endpoints - Neon + description: >- + Learn how to vibe code your own Deep Research agent using Neon, Inngest, and + Claude Code. + keywords: [] + noindex: false + ogTitle: Building a Deep Research Agent with Neon and Durable Endpoints - Neon + ogDescription: >- + Learn how to vibe code your own Deep Research agent using Neon, Inngest, and + Claude Code. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/neon-deep-research-2-2-1-1024x576-158a35a4.jpg) + + +This blog post is a collaboration with [Inngest](https://www.inngest.com/), a platform for building serverless workflows and durable API endpoints that automatically handle retries, state, and long-running logic. + + +Every AI lab is shipping research agents. OpenAI’s Deep Research, Perplexity, and Gemini’s research mode. These products are not simple RAG pipelines. Recent papers like [DeepResearcher](https://arxiv.org/abs/2504.03160) and [Step-DeepResearch](https://arxiv.org/abs/2512.20491) formalize what makes them work: a recursive loop of planning, searching, learning, and reflecting, where the agent decides when to go deeper and when to stop. + +The interesting part isn’t that AI can search the web. Three things stand out from the research: + +- **The recursive structure**: research is not a single query. It is a tree of queries that branches based on what the agent learns at each level. +- **The four atomic capabilities** (from Step-DeepResearch): planning and task decomposition; deep search and information gathering; reflection and cross-validation; and report generation. +- **Memory across sessions**: most research agents start from scratch every time. What if yours got smarter? + +This article walks through building a recursive research agent that implements these ideas with a practical stack: [Neon](https://neon.tech/) (serverless Postgres + pgvector) for persistence and semantic memory, [Inngest](https://www.inngest.com/?ref=neon-deep-research-blogpost)’s durable endpoints for fault-tolerant API endpoints, and Claude Code to vibe-code the implementation. + +**What we are building:** + +- A recursive research agent with real-time progress tracking +- Durable execution that survives failures, retries, and restarts +- Semantic memory with pgvector, so new research builds on past findings +- A Next.js UI that shows live progress, source discovery, and research history + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/image-2-1024x443-2de45b81.png) + +The full source code is [available on GitHub](https://github.com/inngest/deep-research-neon-durable-endpoints). + +**The stack – and why each piece matters:** + +| Component | Role | +| --------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [Neon](https://neon.com/) | Sessions, events, sources, and learnings in one database.
Embeddings live next to relational data, no separate vector DB.
The serverless driver means no connection pool headaches. | +| [Durable Endpoints](https://www.inngest.com/docs/learn/durable-endpoints?ref=neon-deep-research-blogpost) | Each API call, search, and LLM invocation is a retriable step. If Exa times out on search #14 of 30, it retries that step rather than failing the API request. | +| [Claude](https://claude.ai/login) | Generates clarification questions, search queries with reasoning, learning extraction with source rationale, and the final report with citations. | +| [Exa](https://exa.ai/) | Neural web search that returns full page content, not just snippets. | +| OpenAI | `text-embedding-3-small` for 1 `536-dim` embeddings stored in pgvector. | +| Next.js | The front-end and API stack powering the demo. | + +## The database layer: schema, events, and semantic memory + +Everything lives in one Postgres database. Sessions, progress events, relational data, and vector embeddings. No separate vector database, no Redis for events, no S3 for reports. One connection string, one deployment. + +### The four tables + +The system state lives in four Postgres tables: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/1-2-1024x666-9ac890af.jpg) + +- **research_sessions** is the session ledger: topic, status, and the final report. +- **research_events** stores progress events with sequence numbers for cursor-based polling. +- **sources** holds web sources with pgvector embeddings for semantic recall. +- **learnings** contains extracted insights with embeddings, linked back to their session. + +### Polling-based real-time progress + +The client polls `/api/research/events?researchId=X&cursor=N` every 500ms. Events have monotonic sequence numbers. The cursor ensures no duplicates and no missed events: + +No WebSockets needed. This works in any serverless environment because each poll is a stateless HTTP request. + +### Semantic memory: learning across sessions + +Most AI tools treat each session as a blank slate. Our agent remembers. Before generating search queries for a new session, it queries pgvector for semantically similar learnings and sources from past sessions. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/7-1024x769-d1e3edbd.jpg) + +This is a practical implementation of the “reflection” capability from the Step-DeepResearch paper. The agent is not just reflecting on the current session. It is reflecting across its entire research history. + +The recall step runs a pgvector query using cosine distance. + +Prior knowledge gets injected into the search query generation prompt as context: + +``` +Here are relevant findings from previous research sessions: + +Prior Insights: +- [From "AI in healthcare"] FDA approved 171 AI medical devices in 2023 +- [From "AI in healthcare"] Most medical AI focuses on radiology and pathology + +Build on this existing knowledge rather than duplicating it. +``` + +This turns “generate queries about X” into “generate queries about X, knowing that previous research found Y and Z.” The agent naturally avoids re-searching known territory and focuses on what is new. + +### Why Neon fits + +Three things make Neon a good fit for this architecture: + +- [Serverless driver](https://neon.com/docs/serverless/serverless-driver) (`@neondatabase/serverless`): each query is an HTTP request. No persistent connections, no pool management. This is critical when running inside Inngest’s [durable endpoint](https://www.inngest.com/docs/learn/durable-endpoints?ref=neon-deep-research-blogpost), where long-lived connections are not practical. +- [pgvector built-in](https://neon.com/docs/extensions/pgvector): embeddings live in the same database as sessions and events. One connection string, one deployment. +- [Neon MCP server](https://neon.com/docs/ai/neon-mcp-server): Claude Code can create the database, run the schema, and query data directly from the terminal during development. + +### Claude Code prompt: database schema, helpers, and event store + +
+

Set up the Neon database layer.



Create schema.sql with four tables: research_sessions (id, topic, status, clarifications JSONB, report, sources_count, learnings_count, created_at, completed_at), research_events (session_id FK, seq auto-incrementing via subquery, event_type, data JSONB, event_key with UNIQUE constraint on session_id + event_key), sources (session_id, title, url, content, favicon, embedding vector(1536), UNIQUE on session_id + url), learnings (session_id, insight, source_url, source_rationale, connection, embedding vector(1536), UNIQUE on session_id + insight). Add IVFFlat indexes on the embedding columns.



Create src/lib/db.ts wrapping @neondatabase/serverless with a lazily-initialized sql tagged template function. Create src/lib/embeddings.ts with generateEmbedding(text) and generateEmbeddings(texts) using the Vercel AI SDK’s embed/embedMany with OpenAI text-embedding-3-small.



Create src/inngest/event-store.ts with createSession() (INSERT into research_sessions), emitProgress() (idempotent INSERT with ON CONFLICT DO NOTHING and auto-incrementing seq), and getEventsSinceCursor() (SELECT with cursor returning events array, next cursor, and session status).



Create src/types.ts with shared frontend types: ClarificationQuestion, Source, ResearchState, LogEntry, DurabilityMetrics, PastSession.

+
+ +## The research engine: LLM functions, algorithm, and durable execution + +### From papers to architecture + +[DeepResearcher](https://arxiv.org/abs/2504.03160) describes emergent cognitive behaviors that arise from training research agents: planning (formulating and adjusting a search strategy), cross-validation (verifying findings across sources), self-reflection (refining queries when results don’t match), and epistemic honesty (knowing when to stop). + +[Step-DeepResearch](https://arxiv.org/abs/2512.20491) takes this further. The paper formalizes these behaviors as four atomic capabilities and frames Deep Research as long-horizon decision-making over them. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/2-2-1024x279-1c480073.jpg) + +Our implementation maps these capabilities onto concrete functions. Plan is `generateClarificationQuestions()` and `generateSearchQueries()`. Search is Exa web search. Reflect is `extractLearnings()` with source rationale, connections, and follow-up queries. Synthesize is `generateReport()`. + +### Structured LLM output + +Before starting the research, the agent generates clarification questions with suggested options. This implements Step-DeepResearch’s “planning” capability, decomposing a broad topic into a focused research direction. Claude generates structured output validated by Zod: + +```json +{ + questions: [ + { + id: "q1", + question: "What aspect of quantum computing interests you most?", + options: ["Hardware", "Algorithms", "Error correction", "Applications"] + }, + ... + ] +} +``` + +Each search query includes a reasoning field (why this angle matters) and an angle field (what perspective it covers). This mirrors DeepResearcher’s emergent planning behavior: the agent explains its search strategy before executing it. The reasoning is not just for debugging. It gets displayed in the UI and emitted as progress events, so the full reasoning chain is persisted in Neon. + +When extracting learnings from search results, the agent also produces: + +- Source rationale: why this source was relevant to the query +- Learning connections: how this finding relates to previously accumulated learnings +- Follow-up queries: what questions this finding raises (this drives the recursive loop) +- Synthesis notes: meta-observations about how findings from different sources connect + +The follow-up queries are what make the algorithm recursive. After extracting learnings at depth 3, the agent generates new questions to explore at depth 2, informed by what it just learned. + +### The recursive algorithm + +The user enters a topic, Claude generates clarification questions, then search queries with reasoning. From there, the agent enters a depth loop: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-deep-research-agent-with-neon-and-durable-endpoints/3-2-954x1024-802a203a.jpg) + +At each depth level, the agent searches all queries in parallel, extracts learnings from the results (also in parallel), then collects follow-up queries and recurses. + +### Durable execution with Inngest + +A deep research session makes 30+ external API calls across several minutes: Exa searches, Claude analyses, OpenAI embeddings, Neon inserts. Any single call can fail. Without durability, a timeout on search #14 means starting over from search #1, a rate limit on Exa’s API means losing all accumulated progress, and a server restart means the entire session is gone. + +Inngest’s `step.run()` wraps each external call. Each step is: + +- Retriable: if it fails, only that step retries (up to 3 times) +- Memoized: if the whole function replays, completed steps return their cached result instantly +- Identifiable: step IDs like `search-d3-a1b2c3d4` ensure deterministic replay + +The main research endpoint is defined as an Inngest [durable endpoint](https://www.inngest.com/docs/learn/durable-endpoints?ref=neon-deep-research-blogpost) using `inngest.endpoint()`. It receives a standard Request and orchestrates the full workflow through sequential durable steps: + +```javascript +export const GET = inngest.endpoint(async (req: NextRequest) => { + // Step 0: Create session in Neon + await step.run("create-session", async () => { + await createSession(researchId, topic, clarifications); + }); + + // Step 1: Recall prior research (semantic memory) + const priorKnowledge = await step.run("recall-prior-research", async () => { + const topicEmbedding = await generateEmbedding(topic); + const relatedLearnings = await sql` + SELECT l.insight, s.topic as from_topic + FROM learnings l + JOIN research_sessions s ON l.session_id = s.id + WHERE s.id!= ${researchId} + ORDER BY l.embedding <=> ${JSON.stringify(topicEmbedding)}::vector + LIMIT 10 + `; + return { relatedLearnings, relatedSources }; + }); + + // Step 2: Generate search queries (informed by prior knowledge) + const queries = await step.run("generate-queries", async () => { + return await generateSearchQueries(topic, clarifications, breadth, priorKnowledge); + }); + + // Step 3: Recursive deep research (many nested steps) + await deepResearch(researchId, topic, queries, depth, depth, breadth, accumulated, existingUrls); + + // Step 4: Generate report + const report = await step.run("generate-report", async () => { + return await generateReport(topic, accumulated); + }); +}); +``` + +**Idempotency at every layer** + +Durability requires idempotency. The system handles this at three levels: + +1. Step level: deterministic step IDs (`search-d$\{depth\}-$\{hash(query)\}`) prevent duplicate execution on replay +2. Event level: progress events use content-based keys, with `ON CONFLICT DO NOTHING` in Postgres +3. Data level: sources and learnings tables have `UNIQUE (session_id, url)` and `UNIQUE (session_id, insight)` constraints + +This matters because Inngest replays the entire function on step failure. Without idempotent writes, a retry could duplicate sources or emit duplicate progress events. The `ON CONFLICT DO NOTHING` pattern in Postgres makes every `INSERT` safe to re-execute. + +### Claude Code prompt: search and LLM functions + +
+

Create the search and LLM integration layer. In src/inngest/search.ts, wrap the Exa SDK (exa-js) to search the web using searchAndContents() with numResults: 5, useAutoprompt: true, and text: { maxCharacters: 2000 }. Return results as { title, url, content, favicon } objects.



In src/inngest/types.ts, define types for Source (with content field), AccumulatedResearch, QueryWithReasoning (query + reasoning + angle), LearningWithReasoning, and ExtractedLearnings. In src/inngest/utils.ts, create hashQuery(query) using crypto SHA-256 (first 8 hex chars) and calculateProgress(depth, maxDepth).



In src/inngest/llm.ts, implement four functions using Claude Sonnet via the Vercel AI SDK with Zod schema validation: (1) generateClarificationQuestions(topic) returning questions with options, (2) generateSearchQueries(topic, clarifications, breadth, priorKnowledge) returning queries with reasoning and angles, injecting prior knowledge context when available, (3) extractLearnings(topic, query, sources, existingLearnings) returning learnings with source rationale and connections plus follow-up queries with reasoning, (4) generateReport(topic, accumulated) generating a markdown report with inline [N] citation references.

+
+ +### Claude Code prompt: recursive algorithm + +
+

Build the recursive deep research algorithm in src/inngest/deep-research.ts.



The function deepResearch takes a research ID, topic, queries array, depth, maxDepth, breadth, an AccumulatedResearch accumulator, and a Set of existing URLs.



At each depth level: (1) search all queries in parallel using Exa via step.run(), deduplicating by URL against existingUrls, (2) extract learnings from each result using Claude via step.run(), collecting follow-up queries, (3) persist new sources and learnings with OpenAI embeddings to Neon via step.run(), using ON CONFLICT DO NOTHING for idempotency.



Emit progress events between steps using emitProgress() from the event store.



After processing all queries at a depth level, collect follow-up queries, halve breadth (nextBreadth = Math.ceil(breadth / 2)), cap follow-ups to nextBreadth * queries.length, and recurse with depth – 1.



Step IDs must include depth and a hash of the query for deterministic replay (e.g. search-d${depth}-${hashQuery(query)}).

+
+ +### Claude Code prompt: API routes and durable endpoints + +
+

Create the API routes.



(1) src/app/api/research/clarify/route.ts: an Inngest durable endpoint (https://www.inngest.com/docs/learn/durable-endpoints) that takes a topic query param and returns clarification questions by calling generateClarificationQuestions() inside a step.run(). (2) src/app/api/research/route.ts: the main Inngest durable endpoint that receives researchId, topic, clarifications, depth, and breadth as query params.



It orchestrates: step.run(“create-session”) to create the session, step.run(“recall-prior-research”) to embed the topic and query both learnings and sources tables using pgvector cosine distance (<=> operator), step.run(“generate-queries”) to generate search queries with prior knowledge context, then deepResearch() for the recursive loop, then step.run(“generate-report”) to synthesize findings, and finally update the session as complete.



Emit progress events between each phase. (3) src/app/api/research/events/route.ts: a plain Next.js route handler (not durable) that takes researchId and cursor query params and returns events via getEventsSinceCursor(). (4) src/app/api/research/history/route.ts: a plain route handler that queries research_sessions and their associated sources, returning past sessions ordered by date.

+
+ +## The frontend: real-time research visualization + +The interface cycles through six states: idle (topic input with past sessions list), loading-clarifications, clarifying (answering questions), researching (live progress), complete (report view), and error. + +A useResearch hook manages all state transitions and event processing. During research, the hook polls `/api/research/events` every 500ms, processing each event to update progress, sources, reasoning, and step statuses. + +### Live progress during research + +During the researching state, the UI displays: + +- A progress bar that fills as depth levels complete +- Current reasoning activity with expandable history +- Source cards appearing in real-time as they are discovered +- A “prior knowledge” banner when the agent recalls past findings + +The reasoning display is important. It shows the agent’s decision-making in real time, including query angles, source rationale, and synthesis notes. This makes the research process transparent rather than treating it as a black box. + +### Past sessions and memory visibility + +The idle state shows past research sessions. Each card displays the topic, date, source count, and status. Clicking a completed session loads its full report with all sources. + +This makes the semantic memory tangible. The user can see knowledge accumulating across sessions. When the agent recalls prior learnings during a new session, the connection to past work is visible. + +### Claude Code prompt: research UI + +
+

Build the research UI as a Next.js app.



Create a useResearch hook in src/hooks/useResearch.ts managing six states: idle, loading-clarifications, clarifying, researching, complete, error. The hook should poll /api/research/events every 500ms during research, processing events by type (prior-knowledge, clarify-complete, queries-generated, search-start, source-found, learning-extracted, synthesis, follow-up-reasoning, depth-complete, report-generating, complete, error, step-retry, step-recovered) to update progress, sources, reasoning history, and step statuses.



Deduplicate sources by URL to handle Inngest replays. Create these components: (1) TopicInput with a textarea for the topic and a list of past research sessions, (2) ClarificationForm with numbered questions, clickable option chips, and text input fallbacks, (3) ResearchProgress with a progress bar, live reasoning display with expandable history, source cards appearing in real-time, and a prior knowledge indicator, (4) ResearchComplete with a completion summary, collapsible sources list, and a markdown report rendered with react-markdown and clickable citation badges, (5) ExecutionLog showing timestamped events with color-coded types, (6) shared ui.tsx primitives (LoadingSpinner, ProgressBar, CitationText).



Wire everything in src/app/page.tsx which renders the appropriate component based on the current research state.

+
+ +## Wrap up + +This project implements the core ideas from DeepResearcher and Step-DeepResearch in a practical full-stack application: + +- **Recursive search** with breadth reduction and follow-up generation +- **Four atomic capabilities**: planning (clarification + query generation), search (Exa), reflection (learning extraction with rationale), and synthesis (report with citations) +- **Durable execution** via Inngest, where every step is retriable and memoized +- **Semantic memory** via Neon pgvector, where knowledge compounds across sessions +- **Real-time progress** via cursor-based polling on Neon + +**The full source code is** [available on GitHub](https://github.com/inngest/deep-research-neon-durable-endpoints). + +--- + +## Appendix: Taking it further with Claude Code + +Here are some prompts to extend this demo: + +**Curated memory.** The current implementation saves all sources and learnings to memory automatically. A more useful approach would let users choose what to keep: + +
+

Add a post-research review step where the user can select which sources and learnings to persist to memory. Add checkboxes to the ResearchComplete view next to each source and learning. Only embed and store the selected items.

+
+ +**Source quality scoring.** Not all sources are equally valuable. Weighting them improves recall: + +
+

Add a quality_score column to the sources table. After extracting learnings, have Claude rate each source on relevance (1-5). Use the score as a weight when recalling prior knowledge: ORDER BY (1 – (embedding <=> $vector)) * quality_score DESC.

+
+ +**Branching research.** Sometimes you want to fork a completed session and explore a different direction: + +
+

Allow users to fork a completed research session into a new one that starts with all the parent’s accumulated learnings. Implement this as a new API route that copies the session’s learnings as the initial priorKnowledge for the child session.

+
diff --git a/content/blog/posts/building-a-keycloak-email-change-plugin.md b/content/blog/posts/building-a-keycloak-email-change-plugin.md new file mode 100644 index 0000000000..964b96f304 --- /dev/null +++ b/content/blog/posts/building-a-keycloak-email-change-plugin.md @@ -0,0 +1,218 @@ +--- +title: Building a Keycloak Email Change Plugin +description: >- + How to extend Keycloak to securely handle email change events and sync + external systems +excerpt: >- + In this post, I’ll share my experience creating a custom Keycloak plugin that + adds functionality for email change events. This plugin solves a common + integration challenge: how to trigger actions in your application when users + change their email addresses through Keycloak. The Pr... +date: '2025-04-09T23:18:26' +updatedOn: '2025-04-09T23:18:32' +category: engineering +categories: + - engineering +authors: + - adi-griever +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-keycloak-email-change-plugin/cover.jpg + alt: null +isFeatured: true +seo: + title: Building a Keycloak Email Change Plugin - Neon + description: >- + Learn how we built a Keycloak extension that notifies external systems when + email changes occur. + keywords: [] + noindex: false + ogTitle: Building a Keycloak Email Change Plugin - Neon + ogDescription: >- + Learn how we built a Keycloak extension that notifies external systems when + email changes occur. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-keycloak-email-change-plugin/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-keycloak-email-change-plugin/neon-email-change-1-1-1024x576-96ac6966.jpg) + +In this post, I’ll share my experience creating a custom Keycloak plugin that adds functionality for email change events. This plugin solves a common integration challenge: how to trigger actions in your application when users change their email addresses through Keycloak. + +## The Problem: Keeping Systems in Sync After Email Changes + +While Keycloak provides excellent user management capabilities out of the box, including email change functionality, it doesn’t natively support notifying external systems when these changes occur. This can be problematic when you need to sync user data across multiple services or perform specific actions after an email change is confirmed. + +## Building a Keycloak Plugin to Hook Into Email Change Events + +To address this challenge, I developed a custom Keycloak extension (plugin) that enhances the built-in email update process. The plugin is implemented as a Keycloak Service Provider Interface ([SPI](https://www.keycloak.org/docs/latest/server_development/index.html#_providers)). + +Keycloak supports many possible interface extensions allowing addition of custom functionality. More interfaces we take advantage of for example are [AbstractClaimMapper](https://www.keycloak.org/docs-api/latest/javadocs/org/keycloak/broker/oidc/mappers/AbstractClaimMapper.html) to receive `identity_provider_uid` (the unique identifier provided by identity providers) and `org.keycloak.authentication.Authenticator` to create a custom step for authentication flow. + +The extension consists of two main components: + +1. **A REST endpoint (`update-user-email`)** – This allows the backend to initiate an email change request. It verifies the request, generates an email verification token, and sends a confirmation email to the new address. +2. **A token handler (`NeonUpdateEmailActionTokenHandler`)** – This processes the token when the user clicks the confirmation link, verifies the email update, and ensures data consistency by updating external systems accordingly. + +## How It Works + +### 1. User Requests an Email Change + +When a user requests to change their email, our backend calls the `update-user-email` endpoint. This is implemented as part of a [`RealmResourceProvider`](https://www.keycloak.org/docs-api/latest/javadocs/org/keycloak/services/resource/RealmResourceProvider.html): + +```java +@PUT +@Path("/update-user-email/{clientId}") +@Produces(MediaType.APPLICATION_JSON) +@Consumes(MediaType.APPLICATION_JSON) +public Response updateUserEmail(@PathParam("clientId") String clientId, String newEmail) { + auth.require(AccountRoles.MANAGE_ACCOUNT); + event.event(EventType.UPDATE_EMAIL).detail(Details.CONTEXT, UserProfileContext.ACCOUNT.name()); + + UserModel userFromToken = getUserFromToken(session); + UserModel user = session.users().getUserById(realm, userFromToken.getId()); + + if (user == null) { + return Response.status(Response.Status.NOT_FOUND).entity("User not found").build(); + } + + NeonUpdateEmailActionToken actionToken = new NeonUpdateEmailActionToken( + user.getId(), + Time.currentTime() + TIMEOUT, + user.getEmail(), newEmail, clientId, true + ); + + UriInfo uriInfo = session.getContext().getUri(); + String link = Urls.actionTokenBuilder(uriInfo.getBaseUri(), actionToken.serialize(session, realm, uriInfo), clientId, "", "") + .build(realm.getName()).toString(); + + try { + session.getProvider(EmailTemplateProvider.class) + .setRealm(realm) + .setUser(user) + .sendEmailUpdateConfirmation(link, TimeUnit.SECONDS.toMinutes(TIMEOUT), newEmail); + } catch (EmailException e) { + LOG.error("Failed to send email for email update", e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } + + return Response.ok().entity("Email sent successfully").build(); +} + +private UserModel getUserFromToken(KeycloakSession keycloakSession) { + AccessToken accessToken = Tokens.getAccessToken(keycloakSession); + if (accessToken.getSessionId() == null) { + return TokenManager.lookupUserFromStatelessToken(keycloakSession, realm, accessToken); + } + + UserSessionProvider sessions = keycloakSession.sessions(); + UserSessionModel userSession = sessions.getUserSession(realm, accessToken.getSessionId()); + + if (userSession == null) { + userSession = sessions.getOfflineUserSession(realm, accessToken.getSessionId()); + } + + return userSession.getUser(); + } +``` + +This method: + +- Ensures the user is authorized. +- Generates an email verification token. +- Sends a confirmation email to the new address with a verification link. + +### 2. User Confirms the Change + +- When the user clicks the verification link, the `NeonUpdateEmailActionTokenHandler` processes the request. This class is based on [UpdateEmailActionTokenHandler](https://github.com/keycloak/keycloak/blob/66f0d2ff1db6f5ec442b0ddab4580bdd652d8877/services/src/main/java/org/keycloak/authentication/actiontoken/updateemail/UpdateEmailActionTokenHandler.java). +- The function `handleToken` gets invoked on email verification link confirmation. +- As part of the email change functionality, we decided to unlink all existing social provider links the user has, as they are based on the email, and by changing the email, the associations with those providers could cause confusion by pointing to the wrong account, potentially causing authentication issues. + +```java +@Override + public Response handleToken( + NeonUpdateEmailActionToken token, + ActionTokenContext tokenContext + ) { + KeycloakSession session = tokenContext.getSession(); + + AuthenticationSessionModel authenticationSession = tokenContext.getAuthenticationSession(); + UserModel user = authenticationSession.getAuthenticatedUser(); + + LoginFormsProvider forms = session + .getProvider(LoginFormsProvider.class) + .setAuthenticationSession(authenticationSession) + .setUser(user); + + String newEmail = token.getNewEmail(); + + UserProfile emailUpdateValidationResult; + try { + emailUpdateValidationResult = UpdateEmail.validateEmailUpdate(session, user, newEmail); + } catch (ValidationException pve) { + return forms.setErrors(Validation.getFormErrorsFromValidation(pve.getErrors())) + .createErrorPage(Response.Status.BAD_REQUEST); + } + + UpdateEmail.updateEmailNow(tokenContext.getEvent(), user, emailUpdateValidationResult); + + if (Boolean.TRUE.equals(token.getLogoutSessions())) { + AuthenticatorUtil.logoutOtherSessions(token, tokenContext); + } + + tokenContext.getEvent().success(); + + // verify user email as we know it is valid as this entry point would never have gotten here. + user.setEmailVerified(true); + + // remove any required actions to update or verify their email as we know it is now verified and updated + user.removeRequiredAction(UserModel.RequiredAction.UPDATE_EMAIL); + tokenContext.getAuthenticationSession().removeRequiredAction(UserModel.RequiredAction.UPDATE_EMAIL); + user.removeRequiredAction(UserModel.RequiredAction.VERIFY_EMAIL); + tokenContext.getAuthenticationSession().removeRequiredAction(UserModel.RequiredAction.VERIFY_EMAIL); + + // unlink all social providers links from Keycloak + RealmModel realm = session.getContext().getRealm(); + UserProvider users = session.users(); + users.getFederatedIdentitiesStream(realm, user) + .forEach(link -> users.removeFederatedIdentity(realm, user, link.getIdentityProvider())); + + try { + String oldEmail = token.getOldEmail(); + notifyExternalService(user, newEmail, oldEmail); + } catch (SQLException e) { + throw new RuntimeException("ERROR updating database after email change for keycloak user " + user.getId(), e); + } + + return forms.setAttribute("messageHeader", forms.getMessage("emailUpdatedTitle")) + .setSuccess("emailUpdated", newEmail) + .createInfoPage(); + } +``` + +This handler: + +- Validates the token and the new email. +- Updates the email in Keycloak. +- Verifies the email to prevent additional verification prompts. + +### 3. External Systems are Updated + +To maintain data consistency, we update our backend systems after the email change. + +`notifyExternalService` takes care of updating our external service. + +## How to Set Up the Plugin in Your Keycloak Instance + +To use the plugin, you’ll need to: + +1. Add a text file that points to the added provider: `META-INF/services/org.keycloak.services.resource.RealmResourceProviderFactory` +2. Build and deploy the plugin jar to your Keycloak instance: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-keycloak-email-change-plugin/image-6-1024x156-4632ecda.png) + +3\. Use update-user-email API extension to on user request to change email. + +## Final Thoughts + +By leveraging Keycloak’s extensibility, this solution provides a secure and efficient way to manage email updates while keeping external systems in sync. It ensures that users go through a verification process before their email is changed and that all relevant systems reflect the update without manual intervention. diff --git a/content/blog/posts/building-a-news-app-with-replit-agent-a-step-by-step-guide.md b/content/blog/posts/building-a-news-app-with-replit-agent-a-step-by-step-guide.md new file mode 100644 index 0000000000..3ba3b0b52a --- /dev/null +++ b/content/blog/posts/building-a-news-app-with-replit-agent-a-step-by-step-guide.md @@ -0,0 +1,218 @@ +--- +title: 'Building a News App with Replit Agent: A Step-by-Step Guide' +description: Build and Deploy a Neon-powered full-stack app using Replit Agent +excerpt: >- + Replit released Replit Agent, an AI Software Engineer that helps build + full-stack apps. Replit Agent includes a planner, a code editor, integrations + to third-party APIs such as Discord and Stripe, and the ability to create and + deploy Postgres databases powered by Neon. In this tu... +date: '2024-09-17T14:51:22' +updatedOn: '2024-11-12T22:59:46' +category: ai +categories: + - ai +authors: + - mervin-praison +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/cover.png + alt: null +isFeatured: false +seo: + title: 'Building a News App with Replit Agent: A Step-by-Step Guide - Neon' + description: Build and Deploy a Neon-powered full-stack app using Replit Agent + keywords: [] + noindex: false + ogTitle: 'Building a News App with Replit Agent: A Step-by-Step Guide - Neon' + ogDescription: >- + Replit released Replit Agent, an AI Software Engineer that helps build + full-stack apps. Replit Agent includes a planner, a code editor, + integrations to third-party APIs such as Discord and Stripe, and the ability + to create and deploy Postgres databases powered by Neon. In this tutorial, + we’ll walk through how to use Replit Agent to build […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/image-1-1024x576-e1bf5cae.png) + +**Replit released** [Replit Agent](https://docs.replit.com/replitai/agent)**, an AI Software Engineer that helps build full-stack apps. Replit Agent includes a planner, a code editor, integrations to third-party APIs such as Discord and Stripe, and the ability to create and deploy Postgres databases** [powered by Neon](https://neon.tech/blog/neon-replit-integration). + +In this tutorial, we’ll walk through how to use **Replit Agent** to build a news app that scrapes data from Hacker News, stores it in a Postgres database, and displays the front end. But first, let’s dive into what **Replit Agent** is and why it’s a powerful tool for developers. + +Prefer watching over reading? You can watch the full tutorial on YouTube on [how to create a scraper using Replit Agent](https://www.youtube.com/watch?v=w-okKzhGPkc&t=162s), where we walk through the process of building this news app. + +For those who enjoy a written guide, continue reading to learn how to use Replit Agent to build a fully functional news app that scrapes Hacker News, stores data in a Postgres database, and renders it using Next.js. + +## What is Replit Agent? + +**Replit Agent** is an AI-powered development tool within the Replit ecosystem that allows you to easily create, debug, and prototype applications. By simply providing natural language prompts, you can instruct Replit Agent to generate fully functional apps, install dependencies, create back-end and front-end structures, and even deploy the final product. + +You can think of Replit Agent as an intelligent coding assistant that helps you turn your ideas into reality quickly—whether it’s a stock data visualization app, a map of local landmarks, or, in our case, a news scraper app. It’s integrated with various databases like Postgres, and third-party applications such as Google Docs, Slack, Discord, and more. + +Note: + +It’s important to note that Replit Agent is currently in early access. This means the tool is still experimental, and while it does its best to fulfill requests, users may occasionally encounter errors or unexpected behavior. Replit is actively working on improvements and appreciates user feedback during this phase. + +## What Can You Do with Replit Agent? + +Replit Agent can help you: + +- Rapidly prototype apps with minimal coding +- Integrate with databases like Postgres +- Create both back-end and front-end code +- Debug and deploy applications +- Integrate external APIs and services such as Slack or Google Docs + +Now, let’s go step by step to build our news app. + +## Step-by-Step Guide: Building a News App with Replit Agent + +### Step 1: Sign Up for Replit and Enable Replit Agent + +1\. Go to [Replit](https://replit.com/) and sign up for an account if you haven’t already. + +2\. Replit Agent is available in **Replit Core** and **Replit Teams**. Make sure you have access to one of these plans. + +### Step 2: Start a New Replit Agent Project + +1\. Once signed in, navigate to the Replit Agent interface. + +2\. Use a **natural language prompt** to instruct the agent to start building your news app. Here’s the prompt we used: + +```bash +I want to build a news app that +1) continuously scapes news.yconbinator.com, +2) populate the data in a postgres database, +3) render the news in a frontend +``` + +Pro Tip: Click on “Improve Prompt” to generate a detailed prompt. + + + +3\. Click on **“Start Building”**. Replit Agent will then propose a step-by-step plan for the project. + +### Step 3: Approve the Plan + +1\. Review the proposed steps, which may include: + +– Implementing user authentication + +– Creating an API endpoint for scraping + +– Adding search functionality + +– Implementing caching + +2\. Approve the plan, and the agent will begin building the files and dependencies for your project. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/screenshot-2024-11-12-at-235203-1024x551-a46c9b5b.png) + +### Step 4: Watch the Project Build + +As the agent starts working, you’ll see: + +– Files being automatically generated (e.g., scraper, database models, API endpoints) + +– Dependencies like Postgres being installed + +– A live console where the app is being compiled and executed + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/screenshot-2024-11-12-at-235341-1024x551-649e9683.png) + +### Step 5: Customize the Scraper + +The scraper will continuously collect data from the **Hacker News** website. It’s built using: + +– **SQLAlchemy** to manage database interactions + +– **Flask** for handling API requests + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/screenshot-2024-11-12-at-235411-1024x551-3a419df7.png) + +You can inspect the scraper code, which will look something like this: + +```python +import requests +from bs4 import BeautifulSoup +from database import add_news_item + +def scrape_hacker_news(): + url = 'https://news.ycombinator.com/' + response = requests.get(url) + soup = BeautifulSoup(response.text, 'html.parser') + + items = soup.find_all('tr', class_='athing') + + for item in items: + title_tag = item.find('a', class_='titlelink') + if title_tag: + title = title_tag.text + url = title_tag ['href'] + item_id = item ['id'] + + subtext = item.find_next_sibling('tr').find('td', class_='subtext') + score = subtext.find('span', class_='score') + score = int(score.text.split() [0]) if score else 0 + + author = subtext.find('a', class_='hnuser') + author = author.text if author else 'Unknown' + + add_news_item(item_id, title, url, author, score) + +if __name__ == '__main__': + scrape_hacker_news() +``` + +### Step 6: Set Up the Postgres Database + +The Replit Agent automatically integrates Postgres to store your scraped news data. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/screenshot-2024-11-12-at-235804-1024x551-378b1714.png) + +You can manage and view the schema within the Replit interface. Navigate to the Postgres section to see the schema. + +### Step 7: Create the Front End + +The Replit Agent creates `base.html` and `index.hmtl` pages in the template folders to display the frontend. The Replit Agent handles: + +– Routing for different pages (e.g., Home, News Detail) + +– Rendering the scraped news data + +– Styling using CSS or a framework like Tailwind CSS + +### Step 8: Deploy the App + +After ensuring the app works as expected, you can deploy it directly from Replit. Simply click the **Deploy** button, and Replit will handle hosting and deployment for you. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-news-app-with-replit-agent-a-step-by-step-guide/screenshot-2024-11-12-at-235502-1024x551-b5f91cfb.png) + +### Step 9: Review the Results + +You’ll now have a fully functioning news app with: + +– A back-end that scrapes data from Hacker News in real-time + +– A Postgres database that stores the scraped news items + +– A front-end built to display the news + +## Why Use Replit Agent for Building Apps? + +– **Speed**: What would normally take days can now be done in minutes. + +– **Simplicity**: The natural language interface allows even non-coders to build apps. + +– **Versatility**: Integrations with third-party services like Postgres, Stripe, Slack, and more make it easy to expand your app’s functionality. + +– **End-to-End Development**: Replit Agent handles everything from building and testing to deployment. + +### Final Thoughts + +Using Replit Agent can significantly accelerate your development process, allowing you to quickly bring ideas to life with minimal hassle. Whether you’re building a quick prototype or a full-featured app, Replit Agent’s AI-driven capabilities make it easy and efficient. + +**Ready to give it a try?** Head to [Replit](https://replit.com/) and start building your next project today! diff --git a/content/blog/posts/building-a-rag-application-with-llama-3-1-and-pgvector.md b/content/blog/posts/building-a-rag-application-with-llama-3-1-and-pgvector.md new file mode 100644 index 0000000000..fbd33325f7 --- /dev/null +++ b/content/blog/posts/building-a-rag-application-with-llama-3-1-and-pgvector.md @@ -0,0 +1,434 @@ +--- +title: Building a RAG application with Llama 3.1 and pgvector +description: Trying out the latest model by Meta +excerpt: "The AI wars have begun. Not the one where the machines enslave us and use our body heat to power their compute–that’s at least 18 months off. \U0001F642 No, this AI war is between the tech giants and between closed, proprietary models and open source. In one corner, we have OpenAI, suppo..." +date: '2024-07-30T16:58:41' +updatedOn: '2024-07-30T17:03:35' +category: ai +categories: + - ai +authors: + - andrew-tate +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-rag-application-with-llama-3-1-and-pgvector/cover.jpg + alt: null +isFeatured: false +seo: + title: Building a RAG application with Llama 3.1 and pgvector - Neon + description: >- + We use the latest and greatest Meta has to offer to build out a RAG + application that uses Neon Postgres with pgvector as the vector store. + keywords: [] + noindex: false + ogTitle: Building a RAG application with Llama 3.1 and pgvector - Neon + ogDescription: >- + We use the latest and greatest Meta has to offer to build out a RAG + application that uses Neon Postgres with pgvector as the vector store. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-rag-application-with-llama-3-1-and-pgvector/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-rag-application-with-llama-3-1-and-pgvector/neon-rag-1024x576-4ed4f4ae.jpg) + +The AI wars have begun. Not the one where the machines enslave us and use our body heat to power their compute–that’s at least 18 months off. 🙂 No, this AI war is between the tech giants and between closed, proprietary models and open source. + +In one corner, we have OpenAI, supported by Microsoft, building out the strongest closed models with their GPT line. These models have set the benchmark for performance in natural language processing tasks, demonstrating remarkable capabilities in understanding and generating human-like text across a wide range of applications. + +In the other, we have Zuck and his Llamas. And the Llamas are growing strong. The latest Llama model, [Llama 3.1](https://ai.meta.com/blog/meta-llama-3-1/), represents a significant leap forward in open-source AI technology, offering comparable performance to the top proprietary models while providing the freedom to study, modify, and deploy the model without the restrictions and costs of closed-source alternatives. + +So that’s what we’re going to do here. We will use the latest and greatest Meta has to offer to build out a RAG application that allows us to augment the model’s response with our own specific knowledge. + +But first, we have to answer this question: + +## What is RAG? + +RAG, or [Retrieval-Augmented Generation](https://arxiv.org/abs/2005.11401), is an AI technique that combines the power of large language models (LLMs) with external knowledge retrieval. **The idea is to fix one of LLMs’ key limitations: their inability to access or update their knowledge after training.** + +In a RAG system, when a query is received, it first goes through a retrieval step. This step searches a knowledge base to find relevant information. Once relevant information is retrieved, it’s fed into the LLM along with the original query. This allows the model to generate responses based not only on its pre-trained knowledge but also on the most up-to-date and relevant information from the external knowledge base. + +There are a few ways to do this, but the most common current technique is using [embeddings](https://stackoverflow.blog/2023/11/09/an-intuitive-introduction-to-text-embeddings/). Embeddings are a mathematical representation of the text in the knowledge base. Using a specific embedding model, you convert text into a dense vector representation (called an embedding). + +When the text is represented as this vector, performing similarity searches on these numbers is much easier. So the basic process goes: + +1. You have a lot of text or documents. After some preprocessing (splitting it and cleaning it up), you create an embedding for each element of the text. +2. You store these in a vector database–a specialized (or not 😉) database that efficiently stores these long embeddings. +3. When a user makes a query, you then run that query through the embedding model to create its own embedding. +4. You then search through your database looking for similar embeddings. +5. You return the text associated with the N similar embeddings from the database, which are then passed to the LLM along with the original query. +6. The LLM uses the query and the returned text to formulate a more specific, relevant, and up-to-date answer for the user. + +You can see the applications. You can add your documentation to a vector database and build a model that allows users to pull the exact way to use your API. You can add all your past customer service interactions to a vector database and build a model that understands the biggest problems for your customers. You can add all your company’s internal knowledge base articles to a vector database and build a model that provides accurate and up-to-date information to employees across different departments. + +RAG is extremely useful. And the underpinning technology of RAG is the vector database. **Many specific vector database tools are available for this application, but as the saying goes, “[Just Use Postgres](https://www.amazingcto.com/postgres-for-everything/).”** + +Creating a vector database using Postgres is astonishingly easy, and we’ll do that here. + +## Building our AI app + +OK, so what are we going to build? We’re going to create a fairly simple application that peps us up with inspirational quotes. We’ll create these quotes ourselves and store them for retrieval. + +### The tech stack + +Here’s what we’re going to use: + +- **Llama 3.1 for our model.** Llama 3.1 is Meta’s latest open-source large language model, providing state-of-the-art performance for natural language AI tasks without the restrictions of proprietary models. +- **Neon for our vector database.** You probably know this, but [Neon](https://neon.tech) is a serverless Postgres database that offers vector operations via pgvector. Its serverless compute and storage scaling makes it ideal for storing and querying our embeddings efficiently. +- **[OctoAI](https://octo.ai/) to stitch everything together.** OctoAI is a platform that simplifies the deployment and management of open-source AI models, allowing us to easily integrate Llama 3.1 and our Neon database into an application. + +### Creating a vector database in Neon + +Neon has a Free plan: to start, [create an account here](https://console.neon.tech/signup) and follow [these instructions](https://neon.tech/docs/get-started-with-neon/signing-up#step-1-sign-up) to connect to your database. + +Once you’re connected, to turn Neon into a vector database just takes three words: + +```sql +CREATE EXTENSION vector; +``` + +That’s it. **Neon ships with [pgvector](https://github.com/pgvector/pgvector), a Postgres extension that enables efficient storage and similarity search of embeddings.** We can then create a quotes table that includes embeddings alongside the rest of our data: + +```sql +CREATE TABLE quotes ( + id BIGSERIAL PRIMARY KEY, + quote text, + author text, + embedding VECTOR(1024) +); +``` + +So this table has four columns: + +1. `id`: A BIGSERIAL PRIMARY KEY serving as the primary identifier for each quote. +2. `quote`: A text column that stores the actual text of the inspirational quote. +3. `author`: A text column that stores the name of the person who said or wrote the quote. +4. `embedding`: A VECTOR(1024) column that stores the 1024-dimensional vector representation of the quote generated by an embedding model. + +This vector allows for efficient similarity searches in the vector space later. Why 1024? Well, because that is the length of the output vector from the [embedding model](https://huggingface.co/thenlper/gte-large) we’re going to use. If you were using the OpenAI embedding model, this number would be up to [3072](https://platform.openai.com/docs/guides/embeddings/what-are-embeddings). This is a good place to be cautious. The longer the embedding, the more storage it will use and the higher the cost. + +Now we have our vector database (yes, that was all you had to do). Let’s populate it. + +### Creating embeddings + +In this case, we’ve created a few fake quotes and stored them in a CSV. We’ve done this purely to show that the model is pulling from our RAG data rather than grabbing the quotes from somewhere else. We can then do steps 1-3 from above. + +Luckily, as we’ve faked the data, we don’t need to do any cleanup, and we can get straight into creating and storing our embeddings. Here’s the Python code: + +```python +import requests +import os +import csv +from psycopg2 import pool +from dotenv import load_dotenv +import time + +# Load .env file +load_dotenv() + +def load_csv(filename): + quotes, authors = [], [] + with open(filename, 'r') as file: + reader = csv.reader(file) + for row in reader: + if row: + quotes.append(row [0]) + authors.append(row [1]) + return quotes, authors + +def get_embeddings(quotes): + embeddings = [] + url = "https://text.octoai.run/v1/embeddings" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {os.getenv('OCTO_API_TOKEN')}" + } + + for quote in quotes: + data = { + "input": quote, + "model": "thenlper/gte-large" + } + response = requests.post(url, headers=headers, json=data) + if response.status_code == 200: + response_json = response.json() + embeddings.append(response_json ['data'] [0] ['embedding']) + else: + print(f"Request failed with status code {response.status_code}") + # Optional: Add delay to avoid hitting the rate limit + time.sleep(1) + + return embeddings + +def insert_into_db(quotes, authors, embeddings): + connection_string = os.getenv('DATABASE_URL') + connection_pool = pool. SimpleConnectionPool(1, 10, connection_string) + + if connection_pool: + print("Connection pool created successfully") + + try: + conn = connection_pool.getconn() + cur = conn.cursor() + + for quote, author, embedding in zip(quotes, authors, embeddings): + cur.execute( + "INSERT INTO quotes (quotes, author, embedding) VALUES (%s, %s, %s)", + (quote, author, embedding) + ) + + conn.commit() + cur.close() + finally: + connection_pool.putconn(conn) + connection_pool.closeall() + +def main(): + quotes, authors = load_csv('fake_quote.csv') + embeddings = get_embeddings(quotes) + + if len(quotes)!= len(embeddings): + print("Mismatch between number of quotes and embeddings") + return + + insert_into_db(quotes, authors, embeddings) + +if __name__ == "__main__": + main() +``` + +What do each of these functions do? + +- **`load_csv(filename)`**: We’re parsing our CSV of quotes and returning the list of quotes and authors. +- **`get_embeddings(quotes)`**: This is the heart of the code. We generate an embedding for each quote using the “helper/gte-large” model via the OctoAI API and return this list of embeddings. +- **`insert_into_db(quotes, authors, embeddings)`**: Now we can add everything to Neon. We utilize a connection pool for efficient database connections and execute an SQL INSERT statement for each quote-author-embedding trio, committing the transaction at the end. It ensures proper closure of database connections and the connection pool. + +With that done, we can see our data in the [Neon tables page](https://neon.tech/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page): + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-rag-application-with-llama-3-1-and-pgvector/screenshot-2024-07-30-at-94721percente2percent80percentafam-1024x302-4aa7a1df.png) + +This is also a good way to see what embeddings are–just long lists of numbers. These embeddings underpin everything within the AI revolution. It’s all about matching these numbers. + +Now we can start using this data. + +### Building a RAG model with Llama 3.1 + +Now we’re at the business end. Let’s start with the code, and then we’ll walk through it: + +```python +import requests +import sys +import os +import json +from psycopg2 import pool +from dotenv import load_dotenv +from octoai.text_gen import ChatMessage +from octoai.client import OctoAI + +# Load .env file +load_dotenv() + +def get_input_embedding(text): + url = "https://text.octoai.run/v1/embeddings" + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {os.getenv('OCTO_API_TOKEN')}" + } + data = { + "input": text, + "model": "thenlper/gte-large" + } + + response = requests.post(url, headers=headers, json=data) + + if response.status_code == 200: + response_json = response.json() + embedding_vector = response_json ['data'] [0] ['embedding'] + return embedding_vector + else: + print(f"Request failed with status code {response.status_code}") + return None + +def get_quotes(input_embedding): + # Get the connection string from the environment variable + connection_string = os.getenv('DATABASE_URL') + + # Create a connection pool + connection_pool = pool. SimpleConnectionPool( + 1, # Minimum number of connections in the pool + 10, # Maximum number of connections in the pool + connection_string + ) + + # Check if the pool was created successfully + if connection_pool: + print("Connection pool created successfully") + + try: + # Get a connection from the pool + conn = connection_pool.getconn() + + # Create a cursor object + cur = conn.cursor() + + cur.execute(f"SELECT * FROM quotes ORDER BY embedding <-> '{input_embedding}' LIMIT 2;") + + # Fetch the results + quotes = cur.fetchall() + + # Extract and print the quote text and author name + retrieved_quotes = [] + for item in quotes: + quote_text = item [1] + author_name = item [2] + retrieved_quotes.append(f'"{quote_text}" - {author_name}') + + # Commit the transaction + conn.commit() + + # Close the cursor + cur.close() + + return retrieved_quotes + + finally: + # Return the connection to the pool + connection_pool.putconn(conn) + + # Close the connection pool + connection_pool.closeall() + +def generate_response(user_input, retrieved_quotes): + # Construct the system prompt with retrieved quotes + system_prompt = ( + "You are helping people get motivated. Here are some quotes related to your input:\n" + + "\n".join(retrieved_quotes) + + "\nPlease provide a response related to the input and consider the above quotes." + ) + + client = OctoAI( + api_key=os.getenv('OCTO_API_TOKEN'), + ) + completion = client.text_gen.create_chat_completion( + model="meta-llama-3.1-405b-instruct", + messages=[ + ChatMessage( + role="system", + content=system_prompt, + ), + ChatMessage(role="user", content=user_input), + ], + max_tokens=150, + ) + + message_content = completion.choices [0].message.content + + # Print the message content + print(message_content) + +def main(): + if len(sys.argv) < 2: + print("Usage: python input.py 'Your text string goes here'") + return + + user_input = sys.argv [1] + + # Get the embedding vector for the input text + embedding_vector = get_input_embedding(user_input) + + if embedding_vector: + # Get quotes similar to the input text + retrieved_quotes = get_quotes(embedding_vector) + if retrieved_quotes: + # Generate and print the response + generate_response(user_input, retrieved_quotes) + else: + print("No quotes retrieved from the database.") + else: + print("Failed to get embedding vector") + +if __name__ == "__main__": + main() +``` + +This starts with the user input. We’re just using the command line here, but you can imagine this input coming from an app or other frontend. This is a motivational app, so let’s ask it about some dreams: + +```python +python input.py 'I want to know about dreams' +``` + +
That text is passed to `get_input_embedding`. In this function, we’re doing exactly what we did above and creating an embedding for this string. Ultimately, we want a 1024-length vector we can check against the stored 1024-length vectors in our Neon vector database. + +This `embedding_vector` is then passed to `get_quotes`. This sets up a connection to Neon again, but instead of inserting elements, this time it runs this SQL query: + +```sql +SELECT * FROM quotes ORDER BY embedding <=> '{input_embedding}' LIMIT 2; +``` + +Where `input_embedding` is the embedding of our user input. This query performs a similarity search in the vector space of our stored quotes. The ‘<=>’ operator calculates the cosine distance between the input embedding and each embedding in the database. By ordering the results based on this distance and limiting them to 2, we retrieve the two most similar quotes to our input. In this case, this function will output: + +```sql +Quote: Dream big, work hard, stay focused. +Author: Harper Bennett +Quote: The future belongs to those who believe in the beauty of their dreams. +Author: Sophie Montgomery +``` + +You can see the ‘similarity,’ with both of these quotes talking about dreams. Now, RAG’s second magic trick is adding these quotes to the context of our larger model. Let’s just show this code again and then step through it: + +```python +def generate_response(user_input, retrieved_quotes): + # Construct the system prompt with retrieved quotes + system_prompt = ( + "You are helping people get motivated. Here are some quotes related to your input:\n" + + "\n".join(retrieved_quotes) + + "\nPlease provide a response related to the input and consider the above quotes." + ) + + client = OctoAI( + api_key=os.getenv('OCTO_API_TOKEN'), + ) + completion = client.text_gen.create_chat_completion( + model="meta-llama-3.1-405b-instruct", + messages=[ + ChatMessage( + role="system", + content=system_prompt, + ), + ChatMessage(role="user", content=user_input), + ], + max_tokens=150, + ) + + message_content = completion.choices [0].message.content + + # Print the message content + print(message_content) +``` + +The function takes two parameters: the user’s input and the quotes we retrieved from our vector database. It then uses those to construct the system prompt, using the retrieved quotes as context. + +We then initialize the OctoAI client. This is where we’re calling the Llama 3.1 model through OctoAI. We’re using the chat completion endpoint, providing our constructed system prompt and the user’s input as messages. We’re limiting the response to 150 tokens. + +Finally, we extract and print the generated response from the model’s output. This is what we get for the above input: + +```sql +Dreams! The spark that ignites the fire of motivation within us. As Sophie Montgomery so eloquently put it, "The future belongs to those who believe in the beauty of their dreams." This quote reminds us that our dreams are the foundation upon which we build our futures. They give us direction, purpose, and a sense of what's possible. + +But, as Harper Bennett added, it's not just about having dreams; it's about taking action towards making them a reality. "Dream big, work hard, stay focused." This quote highlights the importance of putting in the effort required to bring our dreams to life. It's not just about wishing for something to happen, but about creating a plan, working towards it, and staying committed +``` + +This embodies the core of our RAG system: it takes the context we’ve retrieved from our vector database (the quotes) and uses it to inform the language model’s response to the user’s input. We get those quotes back in our response, adding more relevancy to the model’s output. + +We’ve created a RAG model with Llama 3.1 and Neon. What is the cost of all the AI calls here? + +**One cent.** And that’s with all the tests while building. + +## Your AI apps have a home: Postgres + +Hopefully, you’ve learned three things with this post: + +1. **RAG is extremely powerful.** This is just a toy example, but imagine having thousands of documents in a vector database and being able to add all that knowledge to a regular LLM. It can make the applications you build much more relevant to your users. +2. **Open-source models are up for the fight.** We’re obviously barely scratching the surface of what Llama 3.1 can do, but the benchmarks put it up against OpenAI, Claude, and Cohere, and for a fraction of the cost. +3. **Postgres is a vector database.** Like with Llama 3.1, we’ve barely started exploring the possibilities of Postgres and vectors. You can learn more about optimizing [Neon for embeddings](https://neon.tech/docs/extensions/pgvector), and this is a great read on how [Postgres compares to specialized Vector databases](https://lantern.dev/blog/postgres-vs-pinecone). + +If you are using Neon, you already have a vector database. If you aren’t, [sign up for free](https://console.neon.tech/signup) and start building your AI apps. diff --git a/content/blog/posts/building-a-vscode-chat-extension-to-order-lunch.md b/content/blog/posts/building-a-vscode-chat-extension-to-order-lunch.md new file mode 100644 index 0000000000..1178f7e916 --- /dev/null +++ b/content/blog/posts/building-a-vscode-chat-extension-to-order-lunch.md @@ -0,0 +1,236 @@ +--- +title: Building a VSCode Chat Extension to Order Lunch +description: Want some cheesburgers? +excerpt: "Are you ever so deeply in the developer grindset that you forget to eat? Me neither. But like most VC-backed companies, I wanted to solve a problem that does not exist \U0001F642 (not yet at least!) So, we decided to build an extension able to order us cheesburguers for lunch. The Stack..." +date: '2025-03-07T02:21:47' +updatedOn: '2025-03-07T16:56:04' +category: community +categories: + - community +authors: + - andrew-hamilton +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/cover.jpg + alt: null +isFeatured: false +seo: + title: Building a VSCode Chat Extension to Order Lunch - Neon + description: >- + The team at Layer went to work with an essential mission: to build an + extension able to order them cheesburguers for lunch. + keywords: [] + noindex: false + ogTitle: Building a VSCode Chat Extension to Order Lunch - Neon + ogDescription: >- + The team at Layer went to work with an essential mission: to build an + extension able to order them cheesburguers for lunch. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/neonvscode675-1024x576-5711b9f3.jpg) + + +This is a guest blog post by our friends from [Layer](https://www.buildwithlayer.com), which helped us built [Neon Copilot](https://marketplace.visualstudio.com/items?itemName=buildwithlayer.neon-integration-expert-15j6N). Chech them out. + + +Are you ever so deeply in the developer grindset that you forget to eat? Me neither. But like most VC-backed companies, I wanted to solve a problem that does not exist 🙂 _(not yet at least!)_ + +So, we decided to build an extension able to order us cheesburguers for lunch. + +## The Stack + +First, a look at the architecture we will be using. Here is the general flow: + +1. **VSCode Chat API**: Developer asks Copilot to “Order lunch” +2. **LLM Determines tool calls**: GET_LUNCH_OPTIONS +3. **Copilot responds**: Copilot will list options from the restaurant of what the developer can order +4. **Developer responds**: “Cheeseburger” +5. **LLM Determines tool calls**: ORDER_LUNCH_ITEM +6. **Copilot responds**: “Your cheeseburger had been ordered, sir” + +## Reverse Engineering Grubhub API like a Sigma Developer + +At first, we wanted to use Doordash, but they use server-side rendering to display their menus which would make our jobs real hard. So, we settled on using Grubhub instead. + +Grubhub doesn’t have an API to order food from that is public; they do have this [API](https://developer.grubhub.com/), but this is for merchants, which we are not. So we needed to reverse engineer the API. To do this, we used Chrome Dev Tools & [Postman Interceptor](https://chromewebstore.google.com/detail/postman-interceptor/aicmkgpgakddgnaphhhpliifpcfhicfo?hl=en). + +## The First “Accidental” Cheeseburger + +In order to intercept all the requests, we needed to place an order. With our Postman interceptor listening and the company card details ready, we walked through the checkout process, and clicked “Submit”. + +Soon, hundreds of requests poured out of our computer as our credit card information was sent to the Grubhub servers. We then rapidly tried to cancel the order, but it was too late, and it arrived 30 minutes later. + +Here it is in its full glory: + +```` + |\ /| /|_/| + |\||-|\||-/|/| + \\|\|//||/// + _..----.._ |\/\||//|||| + .' o '. |||\\|/\\ || + / o o \ | './\_/.' | + |o o o| | | + /'-.._o __.-'\ | | + \ ````` / | | + |``--........--'`| '.______.' + \ / + `'----------'` + +I forgot to take a picture, enjoy this ascii art +```` + +The burgers were as good as we imagined them would be. More importantly, we had all the request information we needed to start the reverse engineering. + +We found out it only takes 4 POST and 1 PUT request on Grubhub to make an order. To save you all the time, here they are: + +1. **POST `/carts`:** This route creates a new cart on the user’s account +2. **POST `/carts/\{cart_id\}/lines`:** This allows us to add an item to the cart we just created +3. **PUT `/carts/\{cart_id\}/delivery_info`:** This updates the delivery address for the cart +4. **POST `/carts/\{cart_id\}/payments`:** This attaches a payment method to the cart +5. **POST `/carts/\{cart_id\}/checkout`:** This places the order + +Now, there are a few more routes we are going to add to make the VSCode checkout experience smoother—but these 5 routes are all you need to place an order using the Grubhub. + +## Building The VS Code Extension + +Now, the next step. VSCode extensions are a bunch of Typescript accessing bunch of [APIs](https://code.visualstudio.com/api/get-started/your-first-extension). You can actually start one with a single command here: + +```bash +npx --package yo --package generator-code -- yo code` +``` + +Let us step back for a moment, and take a look at the full project structure: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/diagram1-1024x552-d5da843c.jpg) + +You can see there are two parts to our extension: + +1. The stuff (on the left-hand side) that VSCode requires for us to render a participant. And for that, I will refer you to [these docs](https://code.visualstudio.com/api/extension-guides/chat-tutorial), as they are pretty good. +2. The stuff (on the right-hand side) required to call the tools / use the LLM, which is what the rest of this blog will focus on. + +## How do we Call an API With an LLM? + +Function calling basically works like this: + +
+

User:

Hey LLM I have this function called add that takes parameters {num1: int, num2: int} only respond with JSON so I can parse it from the response. Please add 5 and 9



Assistant:

{num1: 5, num2: 9}

+
+ +While the LLMs that produce these JSON schemas no longer need to be prompted in this fashion, fundamentally, this is how function calling works. Here is an example of one of the tool schemas for `/get_restaurant_items`: + +```json +"inputSchema": { + "type": "object", + "properties": { + "restaurant_id": { + "type": "string", + "description": "The ID of the restaurant" + } + }, + "required": ["restaurant_id"] +} + +--> Expected response from LLM +{ + "restaurant_id": "38427391" +} +``` + +This response is easy to parse with JSON.loads() and then can be validated with something like [Zod](https://zod.dev/) or [Pydantic](https://docs.pydantic.dev/latest/) to ensure it is correct. These tool schemas are declared in the `package.jsonfile` in the extension, which you can find [here](https://github.com/andrewlayer/grubhub/blob/main/package.json). + +## Function Calling + +Now that we have our JSON, we need to use it to invoke a function. In the case of calling an API endpoint, that mean we need to take our parameters and shove them into javascript `fetch`. Here is how we got that done for `/get_restaurant_items`: + +```typescript +export class GetRestaurantItemsTool implements vscode.LanguageModelTool { + async invoke( + options: vscode. LanguageModelToolInvocationOptions, + _token: vscode. CancellationToken + ) { + try { + const res = await grubhubClient.getRestaurantItems(options.input.restaurant_id); + + const itemsList = response.items.map(item => + `- ${item.item_name} (ID: ${item.item_id})\n + ${item.item_description || 'No description available'}` + ).join('\n\n'); + + return new vscode. LanguageModelToolResult([ + new vscode. LanguageModelTextPart( + itemsList || 'No items found' + ) + ]); + } catch (error) { + return new vscode. LanguageModelToolResult([ + new vscode. LanguageModelTextPart( + `Failed to get restaurant items: ${error instanceof Error ? error.message : 'Unknown error'}` + ) + ]); + } + } +} +``` + +In the code above, we implement the `vscode.LanguageModelTool` class which requires the `invoke` function. This line of code is ultimately what does the “calling” of the tool: + +```typescript +const res = await grubhubClient.getRestaurantItems(options.input.restaurant_id); +``` + +You can see that we ascertained the restaurant ID. You might be asking, “How did you parse the JSON?” By implementing the language model tool class, this is done automatically for us as long as we provide a JSON schema. + +## Workflows (a Quick Aside) + +**In order to make any agentic experience nice, you really need workflows.** _Why is this?, you might be asking._ Well, let me show you a hypothetical conversation to illustrate: + +_**Hungry Developer**: Hey, can you list my restaurants?_ + +_**AI (internally panicking)**: You need to make a session first before I can list your restaurants, let me do that. (Frantically makes API calls in the background)_ + +_**Still-Hungry Developer**: ok can you do it now please_ + +_**AI (sweating)**: Getting your favorite restaurants. Here they are_ + +- _Restaurant 123421_ +- _Restaurant 60552_ +- _Restaurant 6646_ + +
_**Hangry Developer**: What?! I want the names of the restaurants, not their IDs 😡_ + +_**AI (having an existential crisis)**: Ah, I see. I need to get the restaurant names using this route for each ID. Here they are:_ + +- _Beighly’s Burgers and Bananas_ +- _Jared’s Jive_ +- _Dave’s Delicious Driveway_ + +_(Phew, crisis averted… until the next API call)_ + +The above conversation is the (moderately dramatized) flow of API calls required for Copilot to list restaurants for Grubhub. This obviously isn’t very user-friendly. You see, most APIs out-of-the-box are not ready to be used by AI agents because they provide bad UX and require additional information that we, as users, and LLMs, don’t care about. + +Thus, we must clean and simplify the API. So how can we accomplish these workflows? + +A fully managed VSCode extension is a pain to maintain for most companies. Here a way to evaluate if you should build your own: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/diagram2-1024x621-d9c30737.jpg) + +In this project, I hardcode them all. But if you are interested in _effortlessly_ cleaning your API for agents to use effectively, you should check out [Layer](https://buildwithlayer.com/). + +## Are we Done Yet? + +For the most part, yes. But don’t you want to order some food? 🙂 + +1. **Install the extension** [here](https://marketplace.visualstudio.com/items?itemName=buildwithlayer.grubhub). This will open a tab in VSCode where you can then actually add the extension. +2. **Get your bearer token & POINT**: So, this took us so long anyways—so we’re still not handling auth well. You can get your bearer token and POINT by intercepting the `https://api-gtm.grubhub.com/restaurants/availability_summaries` request made as such: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/image-1024x474-d7e46092.png) + +3\. Input those values into the VSCode Grubhub extension settings:
+ +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-a-vscode-chat-extension-to-order-lunch/image-1-1024x656-2df16a45.png) + +4\. Restart VSCode, and voilà 🎉! + +_Bon Appétit._ diff --git a/content/blog/posts/building-ai-agents-just-got-faster-with-wordware-and-neon.md b/content/blog/posts/building-ai-agents-just-got-faster-with-wordware-and-neon.md new file mode 100644 index 0000000000..935ad8b576 --- /dev/null +++ b/content/blog/posts/building-ai-agents-just-got-faster-with-wordware-and-neon.md @@ -0,0 +1,117 @@ +--- +title: Building AI agents just got faster with Wordware (and Neon) +description: >- + Wordware enables anyone to develop, iterate, and deploy useful AI Agents. + Built on Neon +excerpt: >- + “Wordware is all about building and iterating quickly, so there’s alignment + with Neon. With Neon’s preview branches, we can catch issues early (like a + migration breaking on a copy of the main database) and fix them before they + hit production. By spotting and fixing problems ear... +date: '2024-08-19T15:23:42' +updatedOn: '2024-08-29T18:10:13' +category: community +categories: + - community + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-ai-agents-just-got-faster-with-wordware-and-neon/cover.png + alt: null +isFeatured: false +seo: + title: Building AI agents just got faster with Wordware (and Neon) - Neon + description: >- + Wordware has built an IDE for prompt engineering, allowing you to build + useful AI Agents much faster. Backed by Neon. + keywords: [] + noindex: false + ogTitle: Building AI agents just got faster with Wordware (and Neon) - Neon + ogDescription: >- + Wordware has built an IDE for prompt engineering, allowing you to build + useful AI Agents much faster. Backed by Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-ai-agents-just-got-faster-with-wordware-and-neon/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/building-ai-agents-just-got-faster-with-wordware-and-neon/screenshot-2024-08-19-at-83009percente2percent80percentafam-1024x523-be6395b1.png) + +
+

“Wordware is all about building and iterating quickly, so there’s alignment with Neon. With Neon’s preview branches, we can catch issues early (like a migration breaking on a copy of the main database) and fix them before they hit production. By spotting and fixing problems early, we can move fast while ensuring stability in production.”

+Robert Chandler, CTO at Wordware.ai +
+ +[Wordware.ai](https://www.wordware.ai) is set to transform the way you build with AI. Think of it as an IDE for prompt engineering; with its user-friendly, Notion-like interface, it allows you to build AI applications much faster by streamlining the feedback loop through natural language programming. You can create AI agents by writing prompts in plain English, iterating with your team on the results while Wordware handles the heavy lifting. To help you get started quickly, Wordware also offers a variety of useful templates: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/building-ai-agents-just-got-faster-with-wordware-and-neon/wordware-templates-27313961.gif) + +## The origin story + +The Wordware cofounders, [Robert](https://www.linkedin.com/in/robertjhchandler/) and [Filip](https://www.linkedin.com/in/filipkozera/), began working with AI almost a decade ago. Shortly after they met at Cambridge, Filip focused on augmenting human memory, _battling_ with GPT-2—while Robert joined [Five AI,](https://www.five.ai/) helping it grow before its acquisition by Bosch. + +Inspired by their personal experiences (and challenges) in building AI products, they started working on a project together. A turning point came with the release of the new GPT models, which showcased unprecedented reasoning capabilities in AI. For several months, Robert and Filip focused on building AI agents and experimenting with various technologies and frameworks. + +## Building better agents by iterating with expert feedback + +
+

“If prompting is the new programming, tools must involve domain experts and enable rapid iteration. You want the fastest possible cycle between changing the prompt, running it, and reviewing the output. This prompt-first approach allows domain experts and engineers to iterate together, making the AI development environment both efficient and effective”

+Robert Chandler, CTO at Wordware.ai +
+ +During these exploratory months, the cofounders expected to spend much of their time building amazing AI products. Instead, they discovered that working with AI often meant getting lost in implementation details. + +The key to creating effective AI agents lay in crafting high-quality prompts, yet most AI tech based on LLMs buried the prompts deep within highly complex code, slowing down the refinement process. The Wordware cofounders realized that by doing the opposite—streamlining the process of refining prompts with human feedback—the development of useful AI agents could be much faster. + +What engineers needed was an environment that allowed them to involve domain experts directly in the development process (e.g., salespeople for sales agents, lawyers for legal assistants) to incorporate their feedback and continuously refine the prompts. + +And just like that, Wordware was born. The result is a web-based IDE for building AI apps that feels intuitive and familiar, much like Notion. It facilitates collaboration between engineers and domain experts, improving the quality of the final product—the AI agent—while speeding up development. + +## How Wordware uses Neon to ship faster and safer + +Since Wordware is all about building and iterating quickly, their architecture couldn’t fall behind. When they discovered Neon, it was a match. + +[Neon](https://neon.tech/) is a serverless Postgres database that prioritizes development speed. With serverless features like scale-to-zero and autoscaling, it saves developers the hassle of sizing servers and paying for unused capacity. + +On top of it, Neon’s [native database branching capabilities via copy-on-write](https://neon.tech/docs/introduction/branching) allow Wordware to instantly create previews on real data with minimal setup. The compounded value of this feature is “immense”, they said—it enables them to identify and fix problems early, preventing major system outages and reducing the need for emergency fixes. + +
+

“Creating previews on real data gives us another level of confidence that, when we merge changes into prod, things will actually work. In Wordware, everything – from Stripe webhooks to just new code that’s being run— gets put on some preview URL using a Neon branch”

+(Robert Chandler, CTO at Wordware.ai +
+ +Wordware’s development workflow is designed for speed and efficiency, focusing on minimizing friction to maximize velocity: + +- When a developer picks up a ticket in Linear, they create a new Git branch for their work. As soon as they push this branch, it gets deployed as a fresh instance via [Vercel](https://neon.tech/docs/guides/vercel). +- This deployment process includes hooking up a new Neon branch for the database, making the environment an exact copy of production. [Everything is set up automatically](https://neon.tech/flow). +- Every change, whether it’s code, environment variables, or database migrations, is deployed to a preview URL. +- The team can then review and test the merge request without having to check out the code or configure their local environments. This setup allows anyone to run and interact with the new feature without any manual set up, ensuring any issues are identified early. + +As developers continue to make changes and push updates, these get built and deployed continuously. This rapid feedback loop helps Wordware catch bugs as soon as they emerge, highlighting any differences between local and production environments. + +## The flexibility of Postgres, but serverless + +
+

“Neon is truly serverless. In some “serverless” databases, you still pay for reserved capacity, and that’s not very helpful if you want to use branch-based development. What you need is the ability to quickly spin up branches and test them in real production settings”

+Robert Chandler, CTO at Wordware.ai +
+ +Another area where Neon and Wordware align is in their serverless approach. Wordware is built on serverless architecture; serverless unlocks scalability without the need to manage infrastructure, and Neon brings the serverless experience to Postgres. Neon has an [API-first feel](https://neon.tech/docs/reference/api-reference), robust [connection pooling](https://neon.tech/docs/connect/connection-pooling), and works seamlessly with an architecture that relies heavily on AWS Lambdas, like Wordware’s. + +Neon’s uniqueness lies in its ability to apply serverless practices while maintaining the flexibility of traditional Postgres. While other serverless databases like DynamoDB excel at scale by enforcing specific access patterns through a key-value store architecture, this can be limiting during the early stages of a startup. In DynamoDB, adding new records requires careful planning on how they will be read, written, updated, and queried. Running joins is challenging, and denormalizing data can be cumbersome. + +In contrast, Neon is just Postgres—without the server management. It offers the flexibility and compatibility essential in early development, allowing Wordware to leverage Postgres fully without giving up a serverless experience to build and scale fast. + +
+

“I love DynamoDB at scale, if you know what your access patterns are going to be—but when you’re early in a startup, you don’t always know. That’s when you need Postgres”

+Robert Chandler, CTO at Wordware.ai +
+ +## Try it out: build your first AI app + +Both Wordware and Neon have free plans. [Click here](https://app.wordware.ai/register) to get started with Wordware, and [create a Neon account](https://console.neon.tech/signup) to get the feel for serverless Postgres. + +PS: Want to play with an app built on Wordware? Check this out: [https://twitter.wordware.ai](https://twitter.wordware.ai/) + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/building-ai-agents-just-got-faster-with-wordware-and-neon/screenshot-2024-08-16-at-105522percente2percent80percentafam-1024x581-f68e8983.png) diff --git a/content/blog/posts/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres.md b/content/blog/posts/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres.md new file mode 100644 index 0000000000..fecb7192b0 --- /dev/null +++ b/content/blog/posts/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres.md @@ -0,0 +1,517 @@ +--- +title: 'Building an AI-powered ChatBot using Vercel, OpenAI, and Postgres' +description: A step-by-step guide with optimization techniques. +excerpt: >- + ChatGPT has demonstrated how powerful and attractive large language models + are. OpenAI API allows you to use word embeddings to tailor a ChatGPT-like app + for your business. This article covers the following: To illustrate how to + build an app that uses pgvector and the OpenAI API,... +date: '2023-03-09T18:18:37' +updatedOn: '2025-10-14T06:23:03' +category: ai +categories: + - ai +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Building an AI-powered ChatBot using Vercel, OpenAI, and Postgres - Neon' + description: A step-by-step guide with optimization techniques. + keywords: [] + noindex: false + ogTitle: 'Building an AI-powered ChatBot using Vercel, OpenAI, and Postgres - Neon' + ogDescription: >- + ChatGPT has demonstrated how powerful and attractive large language models + are. OpenAI API allows you to use word embeddings to tailor a ChatGPT-like + app for your business. This article covers the following: To illustrate how + to build an app that uses pgvector and the OpenAI API, we created Ask Neon, + a ChatGPT-like app that answers […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres/social.png +--- + +ChatGPT has demonstrated how powerful and attractive large language models are. OpenAI API allows you to use word embeddings to tailor a ChatGPT-like app for your business. This article covers the following: + +1. What word embeddings are +2. Why embeddings are useful +3. How the `pgvector` Postgres extension is used to store word embeddings and perform similarity analysis +4. How to create an API for a chatbot app using Neon, OpenAI, and Vercel Edge Functions +5. How to perfect the model by fine-tuning + +To illustrate how to build an app that uses `pgvector` and the OpenAI API, we created _Ask Neon_, a ChatGPT-like app that answers Postgres and Neon questions. + + + +You can ask the app questions such as “How to create a branch with the API” and it returns the following: + +_“You can create a branch with the Neon API by using the POST /projects/\{project_id\}/branches method. You must specify the project_id for the Neon project, and you can specify the parent_id of the branch you are branching from. You can also include an endpoints attribute to create a compute endpoint, which is required to connect to the branch.”_ + +Using a prompt can save time researching the answer in the documentation. However, although impressive, the model could be better. That’s why, in some cases, the models require fine-tuning to answer the questions more accurately (which we will cover at the end of the article). + +You can test the [app](https://ask-neon.vercel.app/) and follow the instructions on the [GitHub repository](https://github.com/neondatabase/postgres-qa) to deploy your own. + +We built the app using Neon, NextJS, Vercel Edge Functions, and OpenAI API. Below is the overall architecture: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres/architecture-11093b1b.png) + +The app does the following: + +1. **Sends a POST request with the `question`** (step 1) +2. **Creates the question embedding** using OpenAI API create embeddings method (steps 2 and 3) +3. **Gets context** by calculating the cosine distances between the question embedding and the embeddings stored in Neon (steps 4 and 5) +4. **Creates text completion** and answers the `question` using the `context` with the OpenAI completion method (steps 6, 7, and 8.) + +This article describes the above steps and fundamentals of word embeddings using the OpenAI API. + +## 1. What are embeddings? + +At a high level, word embedding is a way to represent words as vectors (arrays of numbers). Each word is assigned a vector of floating-point numbers representing its meaning in some abstract sense. The idea behind word embeddings is that words with similar meanings will have smaller distance between corresponding vectors. For example, “dog” and “cat” are both animals; thus, their word embeddings should be closer to one-another in the vector space than to the word “apple”. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres/what-are-embeddings-d8a53634.png) + +### Why is this useful? + +One advantage of word embeddings is that they can make our models more efficient and easier to train by using relatively low-dimensional feature vectors to represent words. Instead of one-hot vector that is the same length as the number of words in our vocabulary (171,146 words in the English language, for example), you can use a dense vector of a fixed size (usually a few hundred dimensions). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres/why-is-this-useful-60272c3e.png) + +Another advantage of word embeddings is that we can improve the performance of our models on smaller data sets by transferring the knowledge from Large Language Models such as OpenAI’s GPT-3. + +OpenAI offers a powerful API for natural language processing, which you can use to create word embeddings. Word embeddings have applications in a variety of natural language processing tasks, such as named entity recognition, sentiment analysis, and text classification. For example, you can use word embeddings to compare different pieces of text to determine similarity. And that’s what we will cover in this article. + +## 2. How to use embeddings in Postgres + +We use the Postgres `pgvector` extension to store embeddings to calculate distances. The pgvector extension uses the nearest neighbor algorithm to speed up search on multi-dimentional vectors. + +You can install pgvector using the following command: + +``` +CREATE EXTENSION IF NOT EXISTS vector; +``` + +The app uses a table called `documents` that contains the embeddings. + +``` +CREATE TABLE documents (text text, n_tokens integer, embeddings vector(1536)); +``` + +OpenAI uses 1536 tokens in their new `text-embedding-ada-002` model to represent each piece of text. This helps to create more accurate embeddings for natural language processing tasks. + +### About OpenAI tokens + +Language models process text in chunks referred to as tokens. In English, a token can range from a single character to a complete word such as “a” or “apple.” In certain languages, tokens can even be shorter than one character or longer than one word. + +For instance, the sentence “Neon is Serverless Postgres” is divided into seven tokens: [“Ne”, “on”, ” is”, ” Server”, “less”, ” Post”, “gres”]. + +The number of tokens included in an API call has several implications. First, it affects the cost of the API call as you are charged per token. Second, it affects the duration of the API call, as processing time increases with more tokens. Lastly, the total number of tokens must be within the model’s maximum limit for the API call to function properly. + +### Similarity analysis with pgvector + +We use the following query to perform a similarity analysis, which calculates the distances between embeddings for an input vector embedding equal to `'[0.006762247998267412, …, -0.028751766309142113]’`: + +```sql +SELECT text, n_tokens, embeddings, + + (embeddings <=> ' [0.006762247998267412, ..., -0.028751766309142113]') AS distances +``` + +The embedding ‘[0.006762247998267412, …, -0.028751766309142113]’ was generated by creating an embedding using the OpenAI API for the following question “What is Neon?”. + +Here are the two first rows of the result set for the query shown above: + +```bash +text | n_tokens | embeddings | distances + +'obs. Careers — | |[0.010606693,… | +Neon …o recent | 200 |-0.007342569] | 0.32074564 +searches' | | | + +--- + +'bout us. … | |[0.022020863, …, | +service for every | 75 |-0.0066274726] | 0.30041761 +developer.' | | | +--------------------------------------------------------------------------------------- +``` + +The distance calculation is done using the “<=>” operator, which is a cosine distance. The extension also supports inner product (<#>) and euclidian distance (<->). + +But before that, we need some data in our database. You can learn more about how to prepare the data in the [ask-neon project repository Readme](https://github.com/neondatabase/ask-postgres#prepare-the-data). + +### Use indexing for performance + +Indexing speeds up queries with an approximate index. We’ll add an index for the cosine distance function. We can specify the number of inverted lists. A good place to start is `4 * sqrt(rows)`, with rows = 4400. + +``` +CREATE INDEX ON items USING ivfflat (embedding vector_cosine_ops) WITH (lists = 265); +``` + +Now that we know more about embeddings and `pgvector`, let’s apply this to our API. + +## 3. Building the application’s backend + +The backend uses Neon’s serverless driver, Vercel Edge Functions, and OpenAI to do the following: + +1. Accept the question as an input +2. Generate embeddings for the question using OpenAI API +3. Calculate the distances between embeddings and return the context +4. Generate an answer using OpenAI’s completion method + +We are using Edge Functions over regular Serverless Functions to speed up output streaming. This [video](https://youtu.be/9Q9_CQxFUKY?t=191) from Vercel explains why we chose Edge over Serverless. + +Neon’s serverles driver is based on the node-postgres `pg` package with the difference that it opens WebSocket connections instead of TCP. And this is important since Edge Functions do not support TCP. + +The first step of the app is to do a similarity analysis between the question embedding and all the embeddings stored in the database. We keep the rows for which embeddings are the most similar to the question. The concatenation of row text is called context. + +The context for the example in the previous section is: + +```bash +// context +'obs. Careers — Neon … 'bout us. About Us … service for every developer.' +``` + +Which is the concatenation of the text column of the most similar embeddings: + +```bash +text | n_tokens | embeddings + +'obs. Careers — | | [0.010606693,…,-0.007342569] +searches' | | +'bout us. … service | | +for every | 75 | +developer.' | | + +--- + +Neon …o recent | 200 | [0.022020863, …, -0.0066274726] +``` + +Then, we use the `question` and the `context` in the prompt to generate the `answer` with the OpenAI API. + +Let’s look at how to do that in code, starting with creating the question embedding. + +### Create the question embedding + +Creating the question embedding aims to compare apples-to-apples with the embeddings stored in our database. We need to vectorize the question into an embedding using the OpenAI API to do so. + +```javascript +const response = await fetch('https://api.openai.com/v1/embeddings', { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, + }, + method: 'POST', + body: JSON.stringify({ + model: 'text-embedding-ada-002', + input, + }), + }); + +const responseJson = await response.json(); +const q_embeddings = responseJson ['data'] [0] ['embedding']; +const q_embeddings_str = q_embeddings.toString().replace(/\.\.\./g, ''); + +return q_embeddings; +``` + +### Get the question’s context + +The goal of this step is to search the Postgres documentation text in our database the embedding that is the most similar to the question embedding. We calculate the distance between two vectors: the smaller distance, the more similarities. + +The context is the body of text containing the question’s answer. We get the context by calculating the cosine distances between the stored embeddings and question embedding and ordering in ascending order. + +We want to use the same query as before, but there is a limited amount of OpenAI tokens available to us. + +```bash +-- Returns all rows +SELECT text, n_tokens, embeddings, + (embeddings <=> ' [0.006762247998267412, ..., -0.028751766309142113]') AS distances +``` + +Let’s modify the query to include the max token constraint and only return entries with a cumulative token count of less than or equal to 1500. + +```sql +SELECT text +FROM ( + SELECT text, n_tokens, embeddings, + (embeddings <=> ' [0.006762247998267412, ..., -0.028751766309142113]') AS distances, + SUM(n_tokens) OVER (ORDER BY (embeddings <=> ' [0.006762247998267412, ..., -0.028751766309142113]')) AS cum_n_tokens FROM documents +) subquery +WHERE cum_n_tokens <= 1500 +ORDER BY distances ASC; +``` + +The cumulative number of tokens is, in this case, arbitrary and depends on your prompt (input). OpenAI’s `text-davinci-003` model accepts 4096 tokens that can be split between the prompt and the answer. The prompt we use in the app uses ~1500 tokens for the context and ~400 tokens for the input and question. This way, we know there is a maximum of approximately 2000 tokens for the answer. + +```javascript +const client = new Client({ connectionString: process.env.DATABASE_URL }); + +await client.connect(); +const query = ` + SELECT text + FROM ( + SELECT text, n_tokens, embeddings, + (embeddings <=> ' [${q_embeddings_str}]') AS distances, + SUM(n_tokens) OVER (ORDER BY (embeddings <=> ' [${q_embeddings_str}]')) AS cum_n_tokens + FROM documents + ) subquery + WHERE cum_n_tokens <= $1 + ORDER BY distances ASC; +`; + +const max_tokens = 1500; +const queryParams = [max_tokens]; + +console.log('Querying database...'); +const { rows } = await client.query(query, queryParams); + +await client.end(); + +const context = rows.reduce((acc, cur) => { + return acc + cur.text; + }, ''); + +return context +``` + +### Create text completion + +Now we need OpenAI to interpret the prompt (composed of the question and context) to answer the question. For that, we use the [OpenAI Completion](https://platform.openai.com/docs/api-reference/completions) method. + +```javascript +const questionPrompt = `You are an enthusiastic Postgres developer who loves Neon database and has a passion for helping answering developers might have. Answer the question asked by developers based on the context below. If the question can't be answered based on the context, say "Sorry :( I don't know."\n\nContext: ${context}\n\n---\n\nQuestion: ${input}\nAnswer:`; + +const res = await fetch('https://api.openai.com/v1/completions', { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ''}`, + }, + method: 'POST', + body: JSON.stringify({ + model: 'text-davinci-003', + prompt, + temperature: 0.5, + top_p: 1, + frequency_penalty: 0, + presence_penalty: 0, + max_tokens: 1700, + stream: false, + n: 1, + }), +}); + +return await res.json(); +``` + +You may have noticed in the `createCompletion` paramaters, we set `stream` to `false`, which means that we expect the result to be returned all at once. However, OpenAI allows you to return a _read_ of words, similar to how ChatGPT shows results. This part was inspired by the [Twitter Bio](https://github.com/Nutlope/twitterbio) app. + +```javascript +import { + createParser, + ParsedEvent, + ReconnectInterval, +} from 'eventsource-parser'; + +And replace the previous code with the following function. + +const createCompletion = async (prompt: string) { +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); +let counter = 0; +const res = await fetch('https://api.openai.com/v1/completions', { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ''}`, + }, + method: 'POST', + body: JSON.stringify({ + model: 'text-davinci-003', + prompt, + temperature: 0.5, + top_p: 1, + frequency_penalty: 0, + presence_penalty: 0, + max_tokens: 1700, + stream: true, + n: 1, + }), +}); + +const stream = new ReadableStream({ + async start(controller) { + // callback + function onParse(event: ParsedEvent | ReconnectInterval) { + if (event.type === 'event') { + const data = event.data; + if (data === ' [DONE]') { + controller.close(); + return; + } + try { + const json = JSON.parse(data); + const text = json.choices [0].text; + + if (counter < 2 && (text.match(/\n/) || []).length) { + // this is a prefix character (i.e., "\n\n"), do nothing + return; + } + + const queue = encoder.encode(text); + controller.enqueue(queue); + counter++; + } catch (e) { + // maybe parse error + controller.error(e); + } + } + } + + // stream response (SSE) from OpenAI may be fragmented into multiple chunks + // this ensures we properly read chunks and invoke an event for each SSE event stream + const parser = createParser(onParse); + + // https://web.dev/streams/#asynchronous-iteration + for await (const chunk of res.body as any) { + parser.feed(decoder.decode(chunk)); + } + }, + }); + + return stream; +} +``` + +With streaming, users do not have to wait for the entire answer to generate before seeing results. This method also executes a few seconds faster than the full completion approach. + +## 4. Fine-tuning + +There is no perfect model when it comes to natural language programs. As impressive as ChatGPT is, there are still some improvements in the quality of answers the AI generates. Fortunately, there is a way to improve your model with fine-tuning. Visit the [OpenAI documentation](https://platform.openai.com/docs/guides/fine-tuning) to learn more about fine-tuning. + +Fine-tuning starts with gathering the ideal generated text for each prompt. + +```json +[object Object] +``` + +You can create a table openai_ft_data to save user feedback. + +```sql +CREATE TABLE openai_ft_data ( + id SERIAL PRIMARY KEY, + query TEXT NOT NULL, + answer TEXT NOT NULL, + suggested_answer TEXT, + user_feedback BOOLEAN +); +``` + +You can imagine scenarios where users are asked to provide feedback on whether they liked the generated response. If no, users can share their expected answer, which would help improve your model. + +Here is what the function that collects user feedback looks like as a Vercel Serverless Function: + +```javascript +import { NextApiRequest, NextApiResponse } from 'next'; +import { Pool } from 'pg'; + +const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + ssl: { + rejectUnauthorized: false, + }, +}); + +export default async (req: NextApiRequest, res: NextApiResponse) => { + try { + const { query, answer, suggestedAnswer, feedback } = req.body; + + if (!query ||!answer) { + return res + .status(400) + .json({ message: 'Question and answer are required' }); + } + + const client = await pool.connect(); + const pgQuery = + 'INSERT INTO openai_ft_data (query, answer, suggested_answer, user_feedback) VALUES ($1, $2, $3, $4)'; + const values = [query, answer, suggestedAnswer, feedback]; + + await client.query(pgQuery, values); + + client.release(); + + return res.status(200).json({ message: 'Data added successfully' }); + } catch (error) { + console.error(error); + return res.status(500).json({ message: 'Error adding data' }); + } +}; +``` + +OpenAI recommends collecting at least 500 feedback examples. Results get better every time the number of examples doubles. Once you have collected enough data, here are the steps to create a fine-tuned model: + +1. Install the OpenAI CLI: + +``` +pip install --upgrade openai +``` + +2. Set your OPENAI_API_KEY: + +``` +export OPENAI_API_KEY="" +``` + +3. Export your data: + +You can use the following query to display the data in the `openai_ft_data` table in JSON format: + +```sql +SELECT json_build_object('prompt', query, 'completion', answer) FROM openai_ft_data; +``` + +Let’s use the above query to export data to JSONL format: + +```bash +psql ?sslmode=require -c "SELECT json_build_object('prompt', query, 'completion', answer) FROM openai_ft_data;" -t -A | sed 's/}$/},/' | sed '$ s/,$//' > output.jsonl +``` + +Expected output: + +```json +// output.jsonl + +{ + "prompt": "what's neon", + "completion": " Neon is a fully managed serverless PostgreSQL with a generous free tier. Neon separates storage and compute and offers modern developer features such as serverless, branching, bottomless storage, and more. Neon is open source and written in Rust." +} +``` + +4. Prepare your data using the OpenAI CLI: + +``` +openai tools fine_tunes.prepare_data -f output.jsonl +``` + +5. Create your fine-tuned model: + +``` +openai api fine_tunes.create -t -m +``` + +## Conclusion + +In this article, we covered the fundamentals of word embeddings and their importance in natural language processing. We have explored how to use OpenAI to create embeddings, store them in Postgres using the pgvector extension, and how to use Vercel Edge Functions along with the @neondatabase/serverless driver to get the context to generate an OpenAI completion. + +We also discussed ways to improve performance and user experience, and to reduce cost by returning a stream and using PolyScale to reduce latency. + +Finally, we discussed fine-tuning the model using user feedback and gathering data using the OpenAI CLI. + +I hope you found this post helpful and informative, and we encourage you to share ideas, feedback, and any questions you have about building end-to-end chatbots with word embeddings and Neon. We look forward to hearing from you! + +I hope you found this post helpful and informative, and we encourage you to share ideas, feedback, and any questions you have about building end-to-end chatbots with word embeddings, and Neon. Your input will help us create better content and improve the experience for all readers. So don’t hesitate to share your thoughts in the comments section or contact us directly. We look forward to hearing from you! diff --git a/content/blog/posts/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres.md b/content/blog/posts/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres.md new file mode 100644 index 0000000000..23b67b2c75 --- /dev/null +++ b/content/blog/posts/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres.md @@ -0,0 +1,299 @@ +--- +title: >- + Building Production API Services with Encore, TypeScript, and Neon Serverless + Postgres +description: Deploy Backends Faster with Encore and Neon +excerpt: >- + Starting today, Encore now supports using Neon Postgres for all Encore + environments, including those in the cloud on Amazon Web Services (AWS) and + Google Cloud Platform (GCP). “This means you can connect your Neon account to + use highly scalable serverless Postgres for all your En... +date: '2024-05-08T13:41:20' +updatedOn: '2024-05-08T13:54:58' +category: community +categories: + - community +authors: + - stephen-siegert +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Building Production API Services with Encore, TypeScript, and Neon + Serverless Postgres - Neon + description: Deploy Backends Faster with Encore and Neon + keywords: [] + noindex: false + ogTitle: >- + Building Production API Services with Encore, TypeScript, and Neon + Serverless Postgres - Neon + ogDescription: >- + Starting today, Encore now supports using Neon Postgres for all Encore + environments, including those in the cloud on Amazon Web Services (AWS) and + Google Cloud Platform (GCP). “This means you can connect your Neon account + to use highly scalable serverless Postgres for all your Encore managed + environments, in the same seamless way youʼre already using […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/cover.jpg +--- + +![Deploy backends faster with Neon and Encore](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/neon-deploy-faster-1-1024x576-2f2a5213.jpg) + +Starting today, [Encore](https://encore.dev/) now supports using Neon Postgres for all Encore environments, including those in the cloud on Amazon Web Services (AWS) and Google Cloud Platform (GCP). + +_“This means you can connect your Neon account to use highly scalable serverless Postgres for all your Encore managed environments, in the same seamless way youʼre already using Encore to orchestrate your Cloud SQL and RDS databases.” –_ [Encore x Neon announcement post](https://encore.dev/blog/neon-serverless-postgres) + +Encore offers a streamlined development platform for building cloud backend applications, enabling developers to focus on building cloud-first solutions rather than struggling with vendor-specific infrastructure complexities. Pairing Encore with Neon Serverless Postgres enhances this developer experience, providing a flexible and robust foundation for building scalable API services. + +![Neon Postgres with Encore](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/neonxencore-1024x939-3cc49c6b.png) + +Now, with Encore, you can leverage Neon Postgres through your entire development lifecycle. From local development to production. This includes services and APIs, Databases (as with Neon), Cron Jobs, Pub/Sub & Queues, and Secrets. + +In this post, we’ll walk through setting up a [URL Shortener Web Service](https://encore.dev/docs/tutorials/rest-api#1-create-a-service-and-endpoint-1) using an Encore template and CLI. Then, after deploying the local environment to a staging environment, we’ll push to a production environment. + +This web service will use the [Encore TypeScript SDK](https://encore.dev/blog/encore-for-typescript), now Generally Available, to build out the backend cloud infrastructure. + +## Prerequisites + +To follow along, you’ll need: + +- A [Neon account](https://console.neon.tech/signup) +- The [Encore CLI](https://encore.dev/docs/install#install-the-encore-cli) installed and configured +- Docker desktop running to test locally to test the app + +If you’d like to deploy the final application to AWS or GCP, then you’ll need to configure those accounts separately and link within the Encore console. The application in this post will be deployed to the Encore Cloud. + +## Encore’s Development Flow + +Encore’s TypeScript SDK speeds up backend development by allowing frontend and full-stack developers to build production-ready applications and distributed systems using a language that is already familiar. Leveraging TypeScript, developers can transition from frontend to backend development, ensuring type safety throughout the process. The TypeScript SDK simplifies backend development by allowing developers to declare common infrastructure primitives directly in application code, streamlining workflows and facilitating rapid iteration and debugging. + +Additionally, If you use Neon for your production or testing database, you can now configure Encoreʼs built-in ephemeral Preview Environments to automatically branch off this database, seeding your Preview Environment with real-world data to simplify testing. + +Here’s how Encore facilitates a seamless development workflow: + +**Local Development** + +- Encore’s local development workflow reduces the hassle of setting up infrastructure manually. With just a few commands, developers can instantiate a local environment that mirrors the cloud setup. +- The local environment accurately mirrors the cloud environment, helping to maintain consistency and minimizing discrepancies during development. +- Encore’s built-in tools, such as type-aware infrastructure and automatic API documentation generation, enhance developer productivity. + +**Testing** + +- Encore simplifies testing by providing built-in support for service/API mocking and dedicated test infrastructure. +- Developers can leverage preview environments for each Pull Request, facilitating comprehensive end-to-end testing before deployment. + +**DevOps** + +- Encore provides automatic infrastructure provisioning in the cloud, eliminating the need for manual configuration. +- By leveraging the Backend SDK, developers can declare infrastructure semantics within their application code, enabling seamless infrastructure management across environments. +- Encore offers a suite of DevOps tools, including least-privilege IAM setup, infrastructure tracking, and cost analytics, empowering teams to focus on product development rather than infrastructure management. + +If you aren’t using AWS or GCP directly, you can deploy your apps directly to Encore’s cloud infrastructure. + +## Using Neon Serverless Postgres with Encore + +Neon Serverless Postgres complements Encore by offering a serverless Postgres service optimized for cloud environments. Here’s how Neon Serverless Postgres enhances the development experience: + +- Neon abstracts away the complexities of managing Postgres databases, allowing developers to focus on application logic. +- With Neon’s serverless architecture, [databases automatically scale](https://neon.tech/blog/1-year-of-autoscaling-postgres-at-neon#architecture-of-autoscaling-a-refresher) to accommodate fluctuating workloads, providing optimal performance without manual intervention. +- Neon enables [database branching](https://neon.tech/blog/move-fast-and-branch-things), making it fast and cost effective to create a separate database instance for each environment, such as development, staging, and production. + +## Create and Deploy the URL shortener app + +Let’s dive into the quick start guide to build your first app in just a few minutes using the Encore CLI, TypeScript SDK, and Neon. + +First, Install the Encore CLI. The Encore CLI provisions your local development environment and simplifies application setup: + +```bash +brew install encoredev/tap/encore +``` + +After the CLI is installed, use `encore app create` flow to create an app locally. + +```bash +encore app create + + Select language for your application [Use arrows to move] + + Go + Build performant and scalable backends with Go + + │ TypeScript + │ Build backend and full-stack applications with TypeScript and Node.JS +``` + +Select TypeScript and the URL Shortener template. + +```bash + ✔ Language: TypeScript + + Template [Use arrows to move] + Hello World + A simple REST API + + Uptime Monitor + Microservices, SQL Databases, Pub/Sub, Cron Jobs + + │ URL Shortener + │ REST API, SQL Database + + Empty app + Start from scratch (experienced users only) +``` + +Follow the prompts to pick a name for your app and select the URL Shortener template. If you’re only doing local development or managing your own deployments, no Encore account is needed. For this guide, we’ll be using Encore’s cloud platform to automate cloud deployments, so if it’s your first time using Encore you’ll need to sign up for a free account. + +```bash + ✔ Language: TypeScript + ✔ Template: URL Shortener + + App Name [Use only lowercase letters, digits, and dashes] + > neon-short-url +``` + +And just like that, your app is set up locally. You’ll also have an app project created in the Encore web dashboard. + +```bash +Successfully created app neon-short-url! +App ID: neon-short-url-a7f2 +Web URL: https://app.encore.dev/neon-short-url-a7f2 + +Useful commands: + + encore run + Run your app locally + + encore test + Run tests + + git push encore + Deploys your app + +Get started now: cd neon-short-url && encore run +``` + +Open the generated TypeScript file (`url.ts`) in your code editor to explore the template. You’ll notice that defining API endpoints is straightforward and follows standard TypeScript conventions. + +View the App in the Encore Dashboard locally with: + +```bash +encore run +``` + +In this example, Encore is using Docker to recreate the environment and pull in the Postgres Docker image for testing. You’ll also notice that the database migrations are run as part of this command execution. This migration is located in the project directory at `url/migrations/url/1_create_tables.up/sql`. + +```bash +neon-short-url encore run (main) + ✔ Building Encore application graph... Done! + ✔ Analyzing service topology... Done! + ✔ Creating Postgres database cluster... Done! + ✔ Pulling Postgres docker image... Done! + ✔ Running database migrations... Done! + ✔ Starting Encore application... Done! + + Encore development server running! + + Your API is running at: https://127.0.0.1:4000 + Development Dashboard URL: https://localhost:9400/neon-short-url-a7f2 +``` + +The app is now running locally and can be inspected using the Development Dashboard URL. + +![Encore local development dashboard](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-local-dev-dash-1024x287-fbf9f7fd.png) + +Open the Local Development Dashboard at [https://localhost:9400](https://localhost:9400/) to view the API explorer and service catalog. + +Now, we’ll push the new app and deploy. Push your changes to deploy your application to Encore’s free development Cloud: + +```bash +git add -A . +git commit -m 'Init deploy 🚀' +git push encore +``` + +When prompted, connect to Encore’s Git, `git.encore.dev`. This will trigger a deployment through the Encore cloud. You can also [integrate your app with GitHub](https://encore.dev/docs/how-to/github), connect to a specific app repo, and activate preview environments for each pull request (PR). Additionally, preview environments allow you to take advantage of [Neon’s branching](https://encore.dev/docs/deploy/infra#development-infrastructure) automatically through Encore for a DB per PR. + +## View the app in the Encore dashboard + +In Encore, each app has environments. When you first push the app from the local environment, a live (deployed) staging environment is created. This app will, by default, have a Neon database provisioned and managed for you by Encore. You can see this by viewing the infrastructure for the `staging` environment for the app. + +For example, when the URL shortener app is deployed initially, you’ll see the provisioned Postgres database. + +The infrastructure and Neon Database is managed by Encore. This is why it is not showing in your account (if you’ve already configured your Neon API key). + +![Encore Dashboard Infrastructure](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-stage-db-1024x533-04ddad11.png) + +## Deploy a Prod Environment with Neon + +Next, you’ll create a new production environment and integrate your Neon account for database use. + +### Configure Neon in your Encore Account + +Visit the [Neon Console](https://console.neon.tech/), sign up, and create your first project by following the prompts in the UI. You can use an existing project or create another if you’ve already used Neon. + +![Neon Postgres Getting Started](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/neon-getting-started-1024x789-2ea15d9a.png) + +### Add the Neon API Key + +Next, [create a Neon API ke](https://neon.tech/docs/manage/api-keys#create-an-api-key) y for your Neon account. Add this API key to the Neon section of the URL Shortener App Settings at `https://app.encore.dev//settings/integrations/neon` and **Save**. + +![Encore link Neon account](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-neon-link-1024x692-7339bfaa.png) + +This allows Encore to provision and configure Neon projects in your Neon account using the [Neon API](https://neon.tech/docs/reference/api-reference). So, instead of the production database being hosted in the Encore account; the production Neon database will be in your personal Neon account. + +![Linked Neon account in Encore](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/linked-neon-account-1024x397-34d6d294.png) + +Next, you’ll create the prod environment. + +### Create a Production Encore environment + +Create a new environment, `prod`, in the application and **Save**. + +![Encore create environment](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-create-env-1024x742-c7cf304f.png) + +For the application database, select Neon. Adjust the settings depending on your preferred region and **Encore > Neon** integration name (from the Neon API key settings). + +![Encore create environment Postgres database](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-create-env-db-978x1024-4dbf3acb.png) + +**Create** the environment. + +### View the deployed app resources and Postgres database + +Once the deployment is complete (which only takes a few seconds), you can test the app and view the database resources. + +![Encore deployed Neon database](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-deployed-neon-db-1024x468-dad65805.png) + +Not only is the app deployed and operational, but the Neon Postgres database has been created in your account, and the migrations have been applied. Using Neon Postgres Branching, Encore created a `prod` Postgres branch from the parent branch for the environment. + +![Encore Neon branches](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-neon-branches-1024x211-4ae3393d.png) + +Inspecting the database tables, we can see that the migrations were applied. + +![Neon Postgres tables](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-schema-neon-1024x440-ec87ec26.png) + +## Testing the URL shortener with cURL + +Now, you can test the API service by using the Encore Cloud Dashboard with cURL. + +```bash +curl 'https://.encr.app/url' -d '{"url":""}' +``` + +After a successful request, you’ll see the data written into the integrated database in your Neon account. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-production-api-services-with-encore-typescript-and-neon-serverless-postgres/encore-neon-populated-table-1024x493-1e8afc4f.png) + +## Clean up + +To clean up your application infrastructure. Navigate through each environment and remove the environment from the **environments > environment-name > settings** section. + +## Conclusion + +Using [Neon with Encore](https://encore.dev/blog/neon-serverless-postgres) offers a robust foundation for developing scalable API services across multiple environments and cloud platforms. Encore streamlines application development and deployment, while Neon manages your serverless Postgres database, enabling you to focus on innovation and speed. + +Starting today, Encore supports using Neon Postgres for all Encore environments, including those in the cloud on AWS and GCP. This capability is now available for developers to take advantage of, enabling seamless database management across different environments and cloud providers. + +To accelerate your development process and leverage the power of Neon Serverless Postgres, [sign up and try Neon for free](https://console.neon.tech/signup). Stay updated by following us on [Twitter/X](https://twitter.com/neondatabase), and join our [Discord](https://neon.tech/discord) community to share your experiences and explore how we can support you in building the next generation of applications. diff --git a/content/blog/posts/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows.md b/content/blog/posts/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows.md new file mode 100644 index 0000000000..f57e04be5d --- /dev/null +++ b/content/blog/posts/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows.md @@ -0,0 +1,538 @@ +--- +title: Building Slack notifications to monitor pg_dump and restore workflows +description: >- + Automate nightly dump/restores from RDS to Neon and get real-time job status + alerts via Slack +excerpt: "\U0001F4DA This article is part of a series on setting up Neon for dev, test, and staging environments while keeping your prod database in RDS. Find out what we’re talking about. In a previous blog post, we showed you how to create a Neon Twin using a GitHub Action that automatically run..." +date: '2024-08-01T16:36:26' +updatedOn: '2025-05-06T09:53:56' +category: workflows +categories: + - workflows +authors: + - rishi-raj-jain +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/cover.jpg + alt: null +isFeatured: false +seo: + title: Building Slack notifications to monitor pg_dump and restore workflows - Neon + description: >- + Learn how to set up Slack notifications for monitoring the status of your + Postgres pg_dump and restore jobs. + keywords: [] + noindex: false + ogTitle: Building Slack notifications to monitor pg_dump and restore workflows - Neon + ogDescription: >- + Learn how to set up Slack notifications for monitoring the status of your + Postgres pg_dump and restore jobs. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/neon-slack-1024x576-43860fff.jpg) + +
+

📚 This article is part of a series on setting up Neon for dev, test, and staging environments while keeping your prod database in RDS. Find out what we’re talking about. 

+
+ +In a [previous blog post](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon), we showed you how to create a Neon Twin using a GitHub Action that automatically runs a pg_dump of your RDS production database and restores it to Neon on a recurring nightly schedule. + +In this blog post, we’ll guide you through setting up a Slack Webhook to send notifications to a public Slack channel. This will inform your team about the latest pg_dump and restore activities. + +By using Slack instead of relying on GitHub email notifications, you can avoid the need to add every developer who needs this information to the GitHub Repository where the Action runs. + +We’ll also cover how to surface more detailed information about the dump and restore including the database name, size how long the job took to complete, and how to handle failures. + +
+Image +
How the success/failure messages will look like in your Slack
+
+ +## Prerequisites + +Before diving in, ensure you have the following: + +- **Completion of previous steps.** Make sure you have completed [all the steps outlined in the previous article](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon), particularly the setup of the Neon Twin and the GitHub Action for pg_dump and restore. +- **Slack API admin access.** You’ll need administrator privileges to your company’s Slack workspace to create and manage Slack apps and webhooks. +- **GitHub repo access.** Ensure you have access to the GitHub repository where the Actions will run, including permissions to manage Actions and Secrets. + +## Quick start + +**👉 All the code shown in this article can be found on this GitHub repo:** [create-neon-twin-slack.yml.](https://github.com/neondatabase/rds-to-neon-twin) + +There are two additional Actions that you might like to use for reference: + +- [create-neon-twin-default.yml](https://github.com/neondatabase/rds-to-neon-twin/blob/main/.github/workflows/create-neon-twin-default.yml): A simple workflow that handles just a pg_dump and restore +- [create-neon-twin-ssl.yml](https://github.com/neondatabase/rds-to-neon-twin/blob/main/.github/workflows/create-neon-twin-ssl.yml): An example of how to handle SSL Certificates as outlined in this community blog post: [How to Use PostgreSQL SSL certificates in GitHub Actions](https://www.paulie.dev/posts/2024/07/how-to-use-postgresql-ssl-certificates-in-github-actions/). + +## How to build Slack notifications to monitor pg_dump and restores + +In this section, we will guide you through how to set up Slack notifications for monitoring your pg_dump and restore workflows. The process looks like this: + +1. Creating a Slack channel in your company’s Slack workspace for receiving notification +2. Setting up and configuring a Slack App using the Slack developer console +3. Modify your existing GitHub Actions workflow ([from the previous blog post](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon)) +4. Develop JavaScript functions to post formatted notifications to your Slack channel +5. Commit changes and deploy the action + +Let’s get to it. + +### Create the Slack channel + +To start, set up a dedicated channel in your company’s Slack workspace for receiving notifications. For this example, we’ll use the channel name **#rds-to-neon-twin.** + +### Create a Slack app + +[We’ll use the Slack developer console to create and configure an app.](https://api.slack.com/apps) This involves naming the app, granting it access to your Slack workspace, and generating a Webhook URL for sending notifications. + +If you have administrator privileges to your company’s Slack account, head over to [https://api.slack.com/apps](https://api.slack.com/apps) and click **Create New App**. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-create-slack-app-1-1024x640-b10bb380.jpg) + +#### Create Slack app from manifest + +From the available options select, **From an app manifest**. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-create-slack-app-2-1024x640-49d7ad23.jpg) + +#### Select workspace + +From the dropdown list: **Pick a workspace to develop your app**. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-create-slack-app-3-1024x640-9abd0648.jpg) + +#### Slack app name + +Edit the JSON data and give your application a **name**. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-create-slack-app-4-1024x640-766b7544.jpg) + +#### App summary + +Click **Create** to finish creating your Slack application. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-create-slack-app-5-1024x640-d4a416c3.jpg) + +#### Basic Information + +With your app created, navigate to: **Basic Information** from the sidebar and update the **Display Information**. These details will be visible when you add the app to your company’s Slack workspace. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-create-slack-app-6-1024x640-ee0f1c6a.jpg) + +#### Active Slack webhooks + +Navigate to: **Incoming Webhook** and click the toggle switch to **On**. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-slack-webhook-1-1024x640-049687e9.jpg) + +#### Add webhook to workspace + +On the same page, scroll down and click the **Add New Webhook to Workspace** button. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-slack-webhook-2-1024x640-febe2f09.jpg) + +#### Select channel + +From the dropdown list select the channel you’d like to post notification to. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-slack-webhook-3-1024x640-935ac7a9.jpg) + +#### Webhook URL + +You will now be able to **Copy** the Webhook URL. Add this as an env var in your project and name it `SLACK_WEBHOOK_URL` as you’ll need this in the next step. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/article-3-images-slack-webhook-4-1024x640-2f55b4b4.jpg) + +### Update GitHub Action + +Now, it’s time to update the workflow we set up [in the previous blog post.](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon) To post a notification to Slack with information about the pg_dump/restore job there are a number of additions that need to be made to the .yml file, and three new JavaScript files that are responsible for posting a formatted message to Slack for both success and failure scenarios. + +To begin with, add the following to your .yml file and we’ll explain what each part is for. + +```yaml +name: Create Neon Twin + +on: + schedule: + - cron: '0 0 * * *' # Runs at midnight ET (us-east-1) + workflow_dispatch: # Enable manual triggering + +env: + PROD_DATABASE_URL: ${{ secrets.PROD_DATABASE_URL }} # Production or primary database + DEV_DATABASE_URL: ${{ secrets.DEV_DATABASE_URL }} # Development database + PG_VERSION: '17' + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} // [!code ++] + NODE_VERSION: '20.x' // [!code ++] + +jobs: + capture-start-time: // [!code ++] + runs-on: ubuntu-latest // [!code ++] + + steps: // [!code ++] + - name: Capture start time // [!code ++] + id: capture-start-time // [!code ++] + run: | // [!code ++] + echo "start_time=$(date --utc +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_OUTPUT // [!code ++] + + outputs: // [!code ++] + start_time: ${{ steps.capture-start-time.outputs.start_time }} // [!code ++] + + dump-and-restore: + runs-on: ubuntu-latest + needs: // [!code ++] + - 'capture-start-time' // [!code ++] + + steps: + - name: Install PostgreSQL + run: | + sudo apt update + yes '' | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh + sudo apt install -y postgresql-${{ env.PG_VERSION }} + + - name: Set PostgreSQL binary path + run: echo "POSTGRES=/usr/lib/postgresql/${{ env.PG_VERSION }}/bin" >> $GITHUB_ENV + + - name: Dump from RDS and Restore to Neon + run: | + $POSTGRES/pg_dump "${{ env.PROD_DATABASE_URL }}" -Fc -f "${{ github.workspace }}/prod-dump-file.dump" + $POSTGRES/pg_restore -d "${{ env.DEV_DATABASE_URL }}" --clean --no-owner --no-acl --if-exists "${{ github.workspace }}/prod-dump-file.dump" + + db-query: // [!code ++] + runs-on: ubuntu-latest // [!code ++] + needs: // [!code ++] + - dump-and-restore // [!code ++] + + steps: // [!code ++] + - name: Install PostgreSQL // [!code ++] + run: | // [!code ++] + sudo apt update // [!code ++] + yes '' | sudo /usr/share/postgresql-common/pgdg/apt.postgresql.org.sh // [!code ++] + sudo apt install -y postgresql-${{ env.PG_VERSION }} // [!code ++] + + - name: Set PostgreSQL binary path // [!code ++] + run: echo "POSTGRES=/usr/lib/postgresql/${{ env.PG_VERSION }}/bin" >> $GITHUB_ENV // [!code ++] + + - name: Database Query // [!code ++] + id: db-query // [!code ++] + run: | // [!code ++] + echo "database_size=$($POSTGRES/psql "${{ env.PROD_DATABASE_URL }}" -t -c "SELECT pg_database_size(current_database());")" >> $GITHUB_OUTPUT // [!code ++] + echo "database_name=$($POSTGRES/psql "${{ env.PROD_DATABASE_URL }}" -t -c "SELECT current_database();")" >> $GITHUB_OUTPUT // [!code ++] + + outputs: // [!code ++] + database_size: ${{ steps.db-query.outputs.database_size }} // [!code ++] + database_name: ${{ steps.db-query.outputs.database_name }} // [!code ++] + + capture-end-time: // [!code ++] + runs-on: ubuntu-latest // [!code ++] + needs: // [!code ++] + - db-query // [!code ++] + + steps: // [!code ++] + - name: Capture end time // [!code ++] + id: capture-end-time // [!code ++] + run: | // [!code ++] + echo "end_time=$(date --utc +"%Y-%m-%dT%H:%M:%SZ")" >> $GITHUB_OUTPUT // [!code ++] + + outputs: // [!code ++] + end_time: ${{ steps.capture-end-time.outputs.end_time }} // [!code ++] + + post-to-slack-success: // [!code ++] + runs-on: ubuntu-latest // [!code ++] + needs: // [!code ++] + - capture-start-time // [!code ++] + - db-query // [!code ++] + - capture-end-time // [!code ++] + if: ${{ success() }} // [!code ++] + + env: // [!code ++] + DATABASE_SIZE: ${{ needs.db-query.outputs.database_size }} // [!code ++] + DATABASE_NAME: ${{ needs.db-query.outputs.database_name }} // [!code ++] + JOB_START_TIME: ${{ needs.capture-start-time.outputs.start_time }} // [!code ++] + JOB_END_TIME: ${{ needs.capture-end-time.outputs.end_time }} // [!code ++] + + steps: // [!code ++] + - name: Checkout repository // [!code ++] + uses: actions/checkout@v4 // [!code ++] + + - name: Install dependencies // [!code ++] + run: npm ci // [!code ++] + + - name: Run Success script // [!code ++] + run: | // [!code ++] + node src/slack-success.js // [!code ++] + + post-to-slack-failure: // [!code ++] + runs-on: ubuntu-latest // [!code ++] + needs: // [!code ++] + - dump-and-restore // [!code ++] + - db-query // [!code ++] + if: ${{ failure() }} // [!code ++] + + steps: // [!code ++] + - name: Checkout repository // [!code ++] + uses: actions/checkout@v4 // [!code ++] + + - name: Install dependencies // [!code ++] + run: npm ci // [!code ++] + + - name: Run Failure script // [!code ++] + run: | // [!code ++] + node src/slack-failure.js // [!code ++] +``` + +#### Environment variables + +There are two new variables this action needs. The Slack Webhook URL from the previous step (this one also needs to be added to your GitHub Secrets) and a Node version which is set “globally” so that later steps can run the success and failure JavaScript scripts. + +#### Capture start time + +The `capture-start-time` job does as the name describes and captures the time the job started. Using GitHub Actions `outputs`, the value of the current time is stored as `start_time` so it can be referenced by the `post-to-slack-success`, which we’ll explain in a later step. + +#### Query database + +The `db-query` job is where you can query the production database and determine the `database_size`, and `database_name`. You could perform any queries in this job that suit your needs. Both values are stored as outputs so they can also be referenced by the `post-to-slack-success` job. This job `needs` the `dump-and-restore` job. + +#### Capture end time + +Similar to the start time job this job captures the value of the current time and stores it as `end_time` so it can also be referenced by the `post-to-slack-success` job. + +#### Post success + +This job is triggered by GitHub Actions built in `if: $\{\{ success() \}\}` condition. Only if the previous jobs as defined by `needs` are successful will this job run. The four environment variables are set using values from the previous jobs. They are: + +- DATABASE_SIZE +- DATABASE_NAME +- JOB_START_TIME +- JOB_END_TIME + +Setting environment variables in this way makes them available to JavaScript / Node environments. + +The last part of this job is to run the `src/slack-success.js` file which we’ll cover in a moment. + +#### Post failure + +Similar to the success job, this job is triggered by `if: $\{\{ failure() \}\}` and is only run if any of the previous jobs defined by `needs` fail. Unlike the success job, there are no environment variables. + +The last part runs the `src/slack-failure.js` file which we’ll cover in the following steps. + +### Create Slack notification scripts + +To complete the setup, we need to create the scripts that will handle sending notifications to Slack. There are three essential files required for this: + +- **src/slack-success.js:** This script is triggered when all jobs complete successfully and sends a success notification to the Slack channel. +- **src/slack-failure.js:** This script is triggered when any job fails and sends a failure notification to the Slack channel. +- **src/format-date.js:** A utility function to format the date and time values captured by the previous jobs. + +Additionally, you need to install the [dotenv](https://www.npmjs.com/package/dotenv) dependency to allow these scripts to access your environment variables and GitHub Secrets. + +To install the dependency, run: + +```bash +npm install dotenv +``` + +#### Format date + +Create a directory named `src`, then create a file named `format-date.js` and add the following code. + +```javascript +// src/format-date.js + +const formatDate = (dateString) => { + const date = new Date(dateString).toLocaleString('en-US', { + day: 'numeric', + month: 'long', + weekday: 'long', + year: 'numeric', + }); + + const time = new Date(dateString).toLocaleTimeString('en-US', { + hour12: true, + hour: 'numeric', + minute: 'numeric', + }); + + return { + date, + time, + }; +}; + +export default formatDate; +``` + +#### Slack success + +Create a file named `slack-success.js` and add the following code. This file is used when all jobs complete successfully. + +```javascript +// src/slack-success.js + +import dotenv from 'dotenv'; +dotenv.config(); + +import formatDate from './format-date.js'; + +const init = async () => { + const bytes = parseInt(process.env.DATABASE_SIZE, 10) || 0; + const gigabytes = (bytes / (1024 * 1024 * 1024)).toFixed(2); + const name = process.env.DATABASE_NAME || 'undefined'; + const start = new Date(process.env.JOB_START_TIME) || new Date(); + const end = new Date(process.env.JOB_END_TIME) || new Date(); + const duration = end - start; + const hours = Math.floor(duration / (1000 * 60 * 60)); + const minutes = Math.floor((duration % (1000 * 60 * 60)) / (1000 * 60)); + const seconds = Math.floor((duration % (1000 * 60)) / 1000); + + try { + fetch(process.env.SLACK_WEBHOOK_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + blocks: [ + { + type: 'header', + text: { + type: 'plain_text', + text: '☝️ A new Neon Twin is available!', + emoji: true, + }, + }, + { + type: 'divider', + }, + { + type: 'section', + text: { + type: 'plain_text', + text: `Created: ${formatDate(start).date}`, + }, + }, + { + type: 'context', + elements: [ + { + type: 'mrkdwn', + text: `• Size: ${gigabytes} GB\n• Name: ${name}\n• Start: ${formatDate(start).time}\n• End: ${ + formatDate(end).time + }\n• Duration: ${hours} hours, ${minutes} minutes, ${seconds} seconds\n`, + }, + ], + }, + ], + }), + }); + } catch (error) { + console.error(error); + } +}; + +init(); +``` + +The above code will create a formatted message using Slack special message syntax and includes information about your job. The message that appears in Slack will look similar to the below. You can read more about posting messages in the [Slack documentation](https://api.slack.com/messaging/webhooks#advanced_message_formatting). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/ad4nxepagt-o9oztk0l2dh7ka5mtrl8wzdsk-plft6w5zg06syrvlp18y9synsfu26fbibqqnsj9vruq3z7d7sqiiyun3kyb4powmr6oboe6d2hu7actfe0apqxrpfhabwpc5liu7pk3jmf7fyfkcxqtw8s2xpn-8413a656.png) + +You can use [Slack’s Block Kit Builder](https://app.slack.com/block-kit-builder/T070FFUDNH3#%7B%22blocks%22:%5B%7B%22type%22:%22header%22,%22text%22:%7B%22type%22:%22plain_text%22,%22text%22:%22%E2%98%9D%EF%B8%8F%20A%20new%20Neon%20Twin%20is%20available!%22,%22emoji%22:true%7D%7D,%7B%22type%22:%22divider%22%7D,%7B%22type%22:%22section%22,%22text%22:%7B%22type%22:%22plain_text%22,%22text%22:%22Created:%20Wednesday,%20July%2010,%202024%22,%22emoji%22:true%7D%7D,%7B%22type%22:%22section%22,%22text%22:%7B%22type%22:%22mrkdwn%22,%22text%22:%22%E2%80%A2%20List%20item%201%5Cn%E2%80%A2%20List%20Item%202%5Cn%22%7D%7D%5D%7D) to create a message format that suits your needs and or create different environment variables that can be used to surface important information within the message. + +#### Slack failure + +Create a file named `slack-failure.js` and add the following code. This file is used when any jobs fail. + +```javascript +// src/slack-failure.js + +import dotenv from 'dotenv'; +dotenv.config(); + +import formatDate from './format-date.js'; + +const init = async () => { + try { + fetch(process.env.SLACK_WEBHOOK_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + blocks: [ + { + type: 'header', + text: { + type: 'plain_text', + text: '🚨 A Neon Twin failed', + emoji: true, + }, + }, + { + type: 'divider', + }, + { + type: 'section', + text: { + type: 'plain_text', + text: `Failed: ${formatDate(new Date()).date}`, + }, + }, + ], + }), + }); + } catch (error) { + console.error(error); + } +}; + +init(); +``` + +The above code will create a formatted message using Slack special message syntax but doesn’t include information about your job. The message that appears in Slack will look similar to the below. You can read more about posting messages in the [Slack documentation](https://api.slack.com/messaging/webhooks#advanced_message_formatting). + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/ad4nxclo6hsxxbrwurfqi0xp8wivdligvya7dljr8rv1dq99c1z5rtfmrtqmzdbn2bc-egsmfn1qfet9r1zrzhdrbfzszyafrhxfptqgrrjfdhvkhqmqz4pcdcr4uaaaai08741k96xwyixl7xgz3jyyof8y-3178f9dd.png) + +You can use [Slack’s Block Kit Builder](https://app.slack.com/block-kit-builder/T070FFUDNH3#%7B%22blocks%22:%5B%7B%22type%22:%22header%22,%22text%22:%7B%22type%22:%22plain_text%22,%22text%22:%22%F0%9F%9A%A8%20A%20Neon%20Twin%20failed%22,%22emoji%22:true%7D%7D,%7B%22type%22:%22divider%22%7D,%7B%22type%22:%22section%22,%22text%22:%7B%22type%22:%22plain_text%22,%22text%22:%22Failed:%20Thursday,%20July%2011,%202024%22,%22emoji%22:true%7D%7D%5D%7D) to create a message format that suits your needs. + +### Deploy Action + +The final step is to commit your changes and deploy the action. To complete the deployment, + +1. Navigate to your GitHub repo +2. Go to **Settings > Secrets and variables > Actions** +3. Click on **New repository secret** +4. Add the following environment variable: `SLACK_WEBHOOK_URL` + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/ad4nxckvr2ute97kqpyvshjbjqcpsdy3blg2ttztqqenjzmsqc4m9ptifs9pi3jjjw9mutpximemdr71d3cxvlcl2ao4pgfjtv-jxm8tvqgj64rjcrppj8u5mz6luqlrgtifnl7fn6cilqus6ojxgctboeaetj1-4fadfa0e.png) + +During development, you can manually trigger the workflow from the GitHub UI: + +1. Navigate to **Actions** in your repository +2. Select the **create-neon-twin** workflow +3. Click on **Run** workflow + +And you can run the JavaScript files directly from your terminal for testing, although some environment variables, `DATABASE_SIZE`, `JOB_START_TIME`, etc, won’t be available. + +```javascript +node src/slack-failure.js +``` + +## Finished + +And that’s it. You now have a full pg_dump and restore of your production database ready and waiting for you as a Neon Twin, plus, a convenient way for everyone involved to be notified when a new Twin is available! + +## Next steps + +**Continue building the workflow:** Navigate to [Part IV of the series – How to deploy a chance tested in Neon to prod in RDS](https://neon.tech/blog/neon-twin-deploy-workflow) + +## In case you missed it + +**Part I:** [Building a Neon Twin: Move Dev/Test/Staging to Neon, Keep Production on RDS](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon) + +Part II: [Optimize your AWS RDS Dev Environments with Neon Postgres](https://neon.tech/blog/development-environments-for-aws-rds-using-neon-postgres) + +Check out the [Twin Thing app](https://neon.tech/dev-for-rds#github-action-builder?twin=true&twinWorkflowName=Create+Neon+Twin&twinSchedule=0+0+*+*+*&twinJob=default&twinSSLName=prod-us-east-1.pem&reverseTwin=false&reverseTwinWorkflowName=Run+Migrations&reverseTwinJob=sql&reverseTwinSubJob=null&pgVersion=16)—it helps you build your Github Action workflows to build your Neon Twins: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows/screenshot-2024-10-02-at-65644percente2percent80percentafpm-1024x981-55e55105.png) diff --git a/content/blog/posts/building-tanstack-com-with-neon-and-tanstack-duh.md b/content/blog/posts/building-tanstack-com-with-neon-and-tanstack-duh.md new file mode 100644 index 0000000000..115f492747 --- /dev/null +++ b/content/blog/posts/building-tanstack-com-with-neon-and-tanstack-duh.md @@ -0,0 +1,113 @@ +--- +title: Building TanStack.com with Neon and TanStack (duh) +description: 'Combining TanStack Start, TanStack Query, Drizzle, and Neon' +excerpt: >- + “Neon was so easy to get started, and it worked flawlessly with TanStack Start + and TanStack Query. When you start a new project with TanStack Start and + select Neon in the CLI, you get a working database instantly and later you can + go claim it, which is one of the coolest things I... +date: '2026-01-20T17:17:15' +updatedOn: '2026-01-20T17:20:32' +category: product +categories: + - product + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-tanstack-com-with-neon-and-tanstack-duh/cover.jpg + alt: null +isFeatured: true +seo: + title: Building TanStack.com with Neon and TanStack (duh) - Neon + description: >- + What better proof than production? See how the TanStack team runs + tanstack.com using Neon, TanStack Start, and TanStack Query. + keywords: [] + noindex: false + ogTitle: Building TanStack.com with Neon and TanStack (duh) - Neon + ogDescription: >- + What better proof than production? See how the TanStack team runs + tanstack.com using Neon, TanStack Start, and TanStack Query. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-tanstack-com-with-neon-and-tanstack-duh/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-tanstack-com-with-neon-and-tanstack-duh/neon-object-storage-1024x576-7091f146.jpg) + +
+

“Neon was so easy to get started, and it worked flawlessly with TanStack Start and TanStack Query. When you start a new project with TanStack Start and select Neon in the CLI, you get a working database instantly and later you can go claim it, which is one of the coolest things I’ve seen”



(Tanner Linsley, founder of TanStack)

+
+ +[TanStack](https://tanstack.com/) is one of the most exciting projects in web development right now, and it’s never been a secret that we’re fans (we’re actually one of its [partners](https://tanstack.com/partners?status=active)). **Recently, everything came full circle: the TanStack team also started using Neon to power [tanstack.com](http://tanstack.com) itself**, which turned out to be a great way to test the TanStack stack + Neon developer experience. Teaser: it didn’t disappoint. + +## In case you don’t know TanStack yet… + +TanStack is an open-source ecosystem of tools for building modern web applications that started as a few focused libraries and has now grown into a cohesive stack used by teams building all kinds of things, from side projects to large production apps. + + + +We invite you to [explore the tools](https://tanstack.com/) and [take a look what people are building with them](https://tanstack.com/showcase?page=1), but here’s a flash list of what’s available today – with many more things in the works: + +- [TanStack Start](https://tanstack.com/start/latest), a full-stack framework for React and Solid +- [TanStack Router](https://tanstack.com/router/latest), type-safe routing for React and Solid +- [TanStack Query](https://tanstack.com/query/latest), for data fetching, caching, and synchronization across client and server +- [TanStack Table](https://tanstack.com/table/latest), a headless UI for building data grids and tables +- [TanStack DB](https://tanstack.com/db/latest), client-side database that works beautifully with the rest of the ecosystem +- [TanStack AI](https://tanstack.com/ai/latest), a framework-agnostic AI SDK for building AI features +- [TanStack Form](https://tanstack.com/form/latest), a headless UI for building performance and type-safe forms +- [TanStack Virtual,](https://tanstack.com/virtual/latest) a headless UI for rendering large data sets efficiently +- [TanStack Pacer](https://tanstack.com/pacer/latest), utilities for scheduling, rate-limiting, and controlling async work +- [TanStack Store](https://tanstack.com/store/latest), a framework-agnostic data store, +- and [TanStack Devtools](https://tanstack.com/devtools/latest), a devtool panels for inspecting and debugging TanStack apps
+ +
+ +
Check out the showcase
+
+ +Across these libraries, TanStack follows a consistent set of principles: + +- Strong type safety by default +- Composable, headless primitives over rigid abstractions +- Framework awareness without lock-in +- Tooling that scales with real applications, not demos + +TanStack is possible thanks to a core team of sponsored open-source maintainers supported by a much larger group of contributors from around the world. [Shoutout to them](https://tanstack.com/maintainers?viewMode=compact&groupBy=none&sortBy=none). + + +If you’re also building an open source project that uses Postgres, we’re happy to help you scale. [Apply to our open source program](https://neon.com/programs/open-source) to get credits, referral payouts, and marketing support. + + +## Running the stack (plus Neon) to power tanstack.com + +Of course, [tanstack.com](http://tanstack.com) became the obvious place to put TanStack itself to the test. The site isn’t just a simple marketing page – it hosts TanStack’s documentation and community resources, serving millions of visitors per year. + +What’s behind it: + +- At the core is TanStack Start, which TanStack uses as the full-stack framework driving the site. Start handles routing, server functions, and rendering, while keeping everything type-safe end to end. +- On the data side, TanStack Query powers client and server data fetching, caching, and synchronization. +- For persistence, the team uses [Neon](https://neon.com/) with [Drizzle](https://orm.drizzle.team/), backing many pieces of the site – e.h. authentication tables, TanStack Stats (which caches and serves npm download data), and the community showcase where users can submit and browse projects built with TanStack. + +The architecture itself is intentionally straightforward: a serverless, lambda-style runtime, React on the frontend, and Postgres on the backend. + +Being just Postgres, Neon fit well into the TanStack stack since the start, but a few things stood out that made the experience especially smooth: + +- **Works seamlessly with TanStack Start and TanStack Query,** without requiring any special adapter or workarounds. +- **Instant setup, no friction.** When starting a new TanStack project, selecting Neon immediately provisions a working Postgres database. You can build with it right away, and if you decide to keep it, you simply claim the database later. Is that easy. +- **No need to manually provision or configure anything.** In Neon, compute autoscaling does the job of resizing the database and storage scales as needed. It’s pretty much _set it and forget it._ +- **Branching for DX.** The TanStack team takes advantage of Neon [branches](https://neon.com/docs/introduction/branching) for development, including [schema-only branches](https://neon.com/docs/guides/branching-schema-only) with no data. +- **Compatibility with AI-assisted coding.** Much of the TanStack site was built using AI coding tools like Claude Code, which proved very good working with Neon. + +## Give it a try + +If you want to try the same setup, go ahead and give it a go. [Run TanStack Start](https://tanstack.com/start/latest) and select Neon in the CLI, experiment for a while, and if it clicks, claim your database, and keep going. + + +The claimable Neon database experience that TanStack offers is built on [Instagres](https://neon.new/). This setup makes it easy to embed a claimable database flow to open-source projects and frameworks, so your end users can get a working database instantly, no signup required. [Check out the Instagres docs to learn more.](https://neon.com/docs/reference/instagres) + diff --git a/content/blog/posts/building-versioning-for-ai-generated-apps.md b/content/blog/posts/building-versioning-for-ai-generated-apps.md new file mode 100644 index 0000000000..c0e798193d --- /dev/null +++ b/content/blog/posts/building-versioning-for-ai-generated-apps.md @@ -0,0 +1,105 @@ +--- +title: Building Versioning for AI-Generated Apps +description: How Dyad built a complete versioning system with code + data +excerpt: >- + For anyone building apps with AI, iteration is constant: you try different + prompts, tweak the logic, and regenerate code until it finally works. At Dyad, + a free, local, open-source AI app builder, this cycle is part of every user’s + experience. From the start, Dyad made it easy to... +date: '2025-11-12T16:43:33' +updatedOn: '2026-01-02T17:40:17' +category: app-platform +categories: + - app-platform + - community + - ai +authors: + - will-chen +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-versioning-for-ai-generated-apps/cover.jpg + alt: null +isFeatured: false +seo: + title: Building Versioning for AI-Generated Apps - Neon + description: >- + Dyad shares how they built full-stack versioning for their agent, syncing + code and database changes safely using Neon branching. + keywords: [] + noindex: false + ogTitle: Building Versioning for AI-Generated Apps - Neon + ogDescription: >- + Dyad shares how they built full-stack versioning for their agent, syncing + code and database changes safely using Neon branching. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/building-versioning-for-ai-generated-apps/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/building-versioning-for-ai-generated-apps/neon-building-versioning-1-2c498ecb.jpg) + + +This blog post is a collaboration with [Dyad](https://www.dyad.sh/) as part of our Agent Builders series, where agent platform teams share how they built, scaled, and refined their systems. + + +For anyone building apps with AI, iteration is constant: you try different prompts, tweak the logic, and regenerate code until it finally works. At [Dyad](https://github.com/dyad-sh/dyad), a free, local, open-source AI app builder, this cycle is part of every user’s experience. + +From the start, Dyad made it easy to roll back code changes when something broke, but database changes were a different story. Schema edits and data migrations weren’t covered by the same versioning system, which meant they were riskier to experiment with. + +That gap eventually led us to design a database versioning model, bringing code and data state together for the first time. Once we could pair code commits with database state, users no longer had to worry about breaking data or schema changes. Versioning finally felt complete. + +This is how we built it and how we’re planning to improve our initial design. + +## The Initial Design + +### Part I: Integrating with Neon + + +Dyad is open-source ([github.com/dyad-sh/dyad](https://github.com/dyad-sh/dyad/)) so you can see exactly how everything works, including the Neon integration. + + +The first thing we did was integrate Dyad with [Neon](https://neon.com/), since they provided the building blocks needed to get the versioning experience we wanted. When a user creates a new Dyad app, + +1. Dyad creates a Neon database for the user using [Neon’s OAuth integration](https://neon.com/docs/guides/oauth-integration) +2. It automatically creates another database [branch](https://neon.com/docs/introduction/branching) for preview +3. Dyad links the database state to each code version, pairing the two as part of its versioning system. + +[Neon’s branching feature](https://neon.com/branching) is essential in this setup. When users preview older versions of their app, Dyad switches the app to use the preview branch in Neon, so previewing or reverting to a previous version doesn’t touch the active development branch. + +Meanwhile, as the user keeps developing on the main branch, Dyad records the database’s current timestamp alongside every commit. In practice, this ties the Git commit and the database timestamp together. When the user undoes a version, Dyad uses that timestamp to roll back both the code and the database to the same point in time. + +Undoing a version in Dyad means undoing the code and the data. The two stay perfectly in sync. + +### Part II: Building on Payload CMS + +Our initial implementation of this integration was developed as part of a Portal template, built on top of [Payload CMS](https://payloadcms.com/). This gives us a full-stack setup where users can generate an entire application in a couple of clicks, database, CMS, and all, with Dyad handling all the wiring behind the scenes. + +## What We’re Planning to Do Next + +This first integration achieved our main goals (linking database state and code versioning), but we’re already thinking about making some changes. + +### Using snapshots for versioning + +When the team first built this integration, [Neon’s Snapshot API](https://neon.com/blog/checkpoints-for-agents-with-neon-snapshots) wasn’t available yet. Snapshots are a natural fit for our use case because they capture the entire database state at a point in time and make it possible to: + +- Roll back or forward between versions without losing intermediate history +- Preserve continuity when restoring data +- Avoid destructive restores, since snapshots don’t overwrite previous states + +With snapshots, we can make database time travel symmetrical, going back and forward between versions safely. Right now, Dyad’s timestamp-based rollback means that when you restore to an earlier version, the later part of the database history is lost. Snapshots will remove that limitation, allowing full, bi-directional versioning. + +### Simplifying the experience by adding auth + +As we mentioned earlier, Dyad’s Neon integration is currently tied closely to the Payload CMS framework. Payload gives us a robust structure but the setup process can be intimidating for first-time users, e.g. when it comes to configuring basics like an email provider. + +To simplify that experience, we’re exploring [Neon Auth](https://neon.com/docs/neon-auth/quick-start/nextjs). Using it will let us offer built-in authentication flows (like password resets) without requiring an external email service, which is a step toward making the full-stack Dyad + Neon setup accessible even to those without backend experience. + +## Wrapping Up + +We’re excited about what Neon is providing for AI app builders like us. From the generous free tier to time-travel capabilities, it’s a stack that fits how our users build and iterate. Our experience so far has given us two takeaways: + +- **Version everything, not just code.** When AI is generating both your frontend and backend, your database deserves the same level of version control as your source files. +- **Design for reversibility.** Vibe coders will break things often and that’s part of the creative process. Safety nets like Neon branching and snapshots will help you transform that chaos into confident iteration for your users. + + +[Explore Dyad on GitHub](https://github.com/dyad-sh/dyad) or [download it locally](https://www.dyad.sh/) to play directly with it. + diff --git a/content/blog/posts/calling-all-yc-startups-we-have-a-special-deal-for-you.md b/content/blog/posts/calling-all-yc-startups-we-have-a-special-deal-for-you.md new file mode 100644 index 0000000000..34657d7d55 --- /dev/null +++ b/content/blog/posts/calling-all-yc-startups-we-have-a-special-deal-for-you.md @@ -0,0 +1,62 @@ +--- +title: 'Calling all YC startups: we have a special deal for you' +description: '$100k /year worth of credits. Plus, we’ll match any competitor’s offer' +excerpt: >- + YC startups: we know the grind (customer obsession, rapid iterations, the need + to scale with tight budgets) so we’re rolling out a special deal for you. If + you’re in Y Combinator, you can claim $100k worth of Neon credits for your + first year. And if you stumble upon a better deal... +date: '2024-05-07T15:45:40' +updatedOn: '2024-05-07T15:54:32' +category: company +categories: + - company +authors: + - atli-cervantes +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/calling-all-yc-startups-we-have-a-special-deal-for-you/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Calling all YC startups: we have a special deal for you - Neon' + description: '$100k /year worth of credits. Plus, we’ll match any competitor’s offer' + keywords: [] + noindex: false + ogTitle: 'Calling all YC startups: we have a special deal for you - Neon' + ogDescription: >- + YC startups: we know the grind (customer obsession, rapid iterations, the + need to scale with tight budgets) so we’re rolling out a special deal for + you. If you’re in Y Combinator, you can claim $100k worth of Neon credits + for your first year. And if you stumble upon a better deal somewhere else, + let us […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/calling-all-yc-startups-we-have-a-special-deal-for-you/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/calling-all-yc-startups-we-have-a-special-deal-for-you/neon-yc-startups-1024x576-8807df5a.jpg) + +YC startups: we know the grind (customer obsession, rapid iterations, the need to scale _with tight budgets_) so we’re rolling out a special deal for you. + +**If you’re in Y Combinator, you can claim $100k worth of Neon credits for your first year. And if you stumble upon a better deal somewhere else, let us know. We’ll match it.** + +To apply, just leave us your email [here](https://neon.tech/yc-startups). + +## Why building on Neon + +Two words: development velocity. Neon’s serverless nature allows you to spin up databases in seconds, and the ability to branch your data and schema allows you to ship faster than ever before: + +- Onboarding new developers takes no time. [They can work on their own isolated dev data branches](https://neon.tech/docs/introduction/branching#development), experimenting with new features or updates without the risk of data inconsistencies that slow everyone down. When their work is done, they can [reset their dev branch from parent in one click](https://neon.tech/docs/manage/branches#reset-a-branch-from-parent) so it mirrors the latest state of production, and start again. +- [You can automatically test schema migrations before they ship](https://www.youtube.com/watch?v=EOVa68Uviks) by integrating database branching directly into your CI/CD pipelines via the Neon API or GitHub Actions, creating one preview database branch per PR. +- Bugs can be fixed faster. Neon database branches [can be created from any past point](https://neon.tech/docs/introduction/branching#what-is-a-branch), enabling your team to reproduce and diagnose issues and fix regressions easily. + +Multiple database branches can be used simultaneously to test different fixes or updates without any risk for conflicts. This is sustainable also: [branching uses a copy-on-write system in Neon](https://neon.tech/blog/architecture-decisions-in-neon), making database branches very resource-efficient and virtually free. + +
+

With branching, we can test with actual data and spot and fix errors easily. Since every engineer can work on an individual branch that mirrors production, we can collaboratively develop, test, and debug in a synchronized manner.

+Topo.io (W24) is building a mind-blowing AI sales agent using Neon +
+ +## Jump aboard + +If you’re a YC startup, [send us your contact info](https://neon.tech/yc-startups) and claim your deal. diff --git a/content/blog/posts/cascade-and-neon-mcp.md b/content/blog/posts/cascade-and-neon-mcp.md new file mode 100644 index 0000000000..a9f8067ab8 --- /dev/null +++ b/content/blog/posts/cascade-and-neon-mcp.md @@ -0,0 +1,132 @@ +--- +title: Using Windsurf Cascade and Neon MCP for Agent-Driven Database Interaction +description: Let Cascade deploy and manage your Neon databases +excerpt: >- + Imagine an IDE that not only writes your code but also provisions, migrates, + and queries your Postgres database. Stop imagining—this is possible today with + Windsurf. In this blog post, we will explore how Windsurf can be integrated + with the Neon MCP server to allow agents to inte... +date: '2025-03-24T16:50:27' +updatedOn: '2025-04-15T19:19:25' +category: community +categories: + - community +authors: + - akshat-agrawal +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Using Windsurf Cascade and Neon MCP for Agent-Driven Database Interaction + - Neon + description: >- + You can integrate Windsurf with the Neon MCP server to allow agents to + interact directly with your database. + keywords: [] + noindex: false + ogTitle: >- + Using Windsurf Cascade and Neon MCP for Agent-Driven Database Interaction + - Neon + ogDescription: >- + You can integrate Windsurf with the Neon MCP server to allow agents to + interact directly with your database. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/neon-mcp-1-1024x576-3c0aa326.jpg) + +Imagine an IDE that not only writes your code but also provisions, migrates, and queries your Postgres database. Stop imagining—this is possible today with [Windsurf](https://codeium.com/windsurf). + +In this blog post, we will explore how Windsurf can be integrated with the [Neon MCP server](https://neon.tech/blog/let-claude-manage-your-neon-databases-our-mcp-server-is-here) to allow agents to interact directly with your database. We’ll walk through an example where we build a simple Todo application using Next.js, Typescript, Drizzle ORM, and Neon, and then see how [Cascade](https://codeium.com/cascade) can iterate on the project, **adding features and managing migrations without writing a line of code**. + +## What is MCP? + +If you’ve been anywhere near the internet lately, you probably know this by now. MCP, or the Model Context Protocol, is [an open standard developed by Anthropic](https://www.anthropic.com/news/model-context-protocol) designed to help AI models integrate with and receive context from external data sources. Similar to how a USB port is a standardised way to connect with other devices, MCP is a standardised way for LLMs to connect with other services. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/ad4nxdkxgr7wnjuuqymthmu5e5x5qqcaescnjeuxmvovebp1dpfdjqknwel1omjynswlajod9jwuucjnlc5lofpd6eyuisbfotsyxnv2fzrif9orgzyxeo-elbz6cp2f7so1v1emrz-9b2cb87a.png) + +In our case today, Windsurf is set up as an _MCP client_, meaning it can send commands and interpret feedback from Neon, which acts as an _MCP server,_ allowing Windsurf to securely interact with your Postgres database. + +## Setting up Neon MCP in Windsurf + +The first step to manage databases from Windsurf is configuring Neon’s MCP server. + +The process is simple: you’ll first need to head over to the [Neon Console](https://console.neon.tech/app/projects) and grab an API key by going to `Settings > API Keys > Create new API key`, then naming the key anything you’d like and copying it to clipboard. + +Next, boot up Windsurf and open up [Cascade](https://codeium.com/cascade) by hitting `Ctrl + L`, then hit configure `MCP Server` to open up your `mcp_config.json` and add the following: + +```json + "mcpServers": { + "neon": { + "command": "npx", + "args": [ +"-y", +"@neondatabase/mcp-server-neon", +"start", +"" +] + } + } +``` + +If you’re more of a visual learner, follow this quick video to see how to set everything up in just 30 seconds! + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/ad4nxfmguspwqnw7xlqjkpupwxj1fwlnfkzb2osumzlek-bsjnb84ihtdxtjjf6qeqauv55k-kmpveutg34lzvfx5zbagm4h9ycikdiyqnp5hakbnfq8w4tv7c4gk7mnmpjabu4ucaq-0a5547c0.gif) + +Once you’ve done this, your Neon MCP server is up and running in Windsurf. + +## Using the Neon MCP Server to deploy databases with Cascade + +Now, it’s time to put MCP into action. Let’s build a simple Todo application to demonstrate how smoothly Cascade interacts with Neon. + +Open the Cascade AI Agent in Windsurf, then prompt it to: + +
+

Create a todo application using Next.js, Typescript, Drizzle ORM, Tailwind, and Neon Postgres.

+
+ +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/ad4nxeyhtw3qcjpkxjxfoxymosbw4b8fso4ro0d0zzgtwnlwtl1gmz1fedqdfaw96zk7avdhwidh-i4lxcmlibpz66c5yxxgefkxuoofuy4uzbbeee-jsf-euqjqz4bjpuc0w2mu-ad3db9cd.gif) + +As soon as the request is sent, Windsurf’s MCP Client kicks into gear. + +Just as a developer would iteratively create an application, Cascade breaks the prompt down into a series of more clearly defined tasks, tackling them each one by one. In the case of our prompt, Cascade does the following: + +- **Project initialization:** Sets up the Next.js project, and all the npm packages that will be used +- **Database provisioning:** Creates a new Neon Postgres database instance using the Neon MCP server +- **Schema creation:** Designs and applies the schema directly to your database +- **API and UI generation**: Creates the API routes, then connects those to the frontend components, and styles everything using Tailwind CSS + +In just moments, your application has been taken from a single prompt into a working Next.js application connected to a Postgres database: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/ad4nxd25uzyxdxw7to7hlx6nxq3aqdb6gchqtm7lv8w87tdigbduwpv4yea2oj2oj4i3ejjn5fs0dsny8hi9tkdxexcwnzxs6ttqrhrx550ynqgb5hollsivwwgvdrjh8241sgkua-c6d3049b.gif) + +## Adding features + +After your Todo application is up and running, it’s time to take it a step further and add a feature. + +We’ll instruct Cascade to integrate a priority label to the todos, and watch how it’s able to develop this feature, create a migration file, and apply it to the database using MCP. + +No more than 1 minute later, Cascade developed a plan and executed on it start to finish: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/ad4nxdrgjb4dmtyrsf4kll-sotkdanuitmiczxoxnbybmqwjyz9xlq8oqai7apn0dms6w5bd3tkuzmfbhdd0aud93rxsewkcs1yplpnakt6dpvherdxteme901buxjh7nywbqdvg-394bd7ca.gif) + +What’s more, if we look at the generated migration files, we can see Cascade created a custom enum type for priorities instead of just using plain text. This small detail shows the agent’s ability to generate clean and thoughtful schemas. + +```sql +CREATE TYPE "public"."priority" AS ENUM('low', 'medium', 'high'); +ALTER TABLE "todos" ADD COLUMN "priority" "priority" DEFAULT 'medium' NOT NULL; +``` + +And looking over in the Neon console, the database changes were successfully applied with the new priority column. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/cascade-and-neon-mcp/ad4nxctx8qp6gng2t0rs5dbyayhblfdmbyl7yzqotksahmvijpedgv2kpvwrz6lt5x80idldxxll2llopdacdq689a54p7kekoyxxbqdlyomjt2hfqyhnult8wwhmj-likyzowvrk3q-45e7bd75.png) + +## Wrap up + +Using Windsurf together with Neon MCP makes database management accessible for any developer. Cascade can automatically provision Postgres databases, run schema migrations, execute queries, and more. This tight integration means your database remains completely synchronized with your application, guaranteeing consistency, while also reducing the amount of manual work you need to worry about. + +If you haven’t tried Windsurf yet, [get started with the Free plan](https://codeium.com/pricing)! diff --git a/content/blog/posts/cdc-with-materialize.md b/content/blog/posts/cdc-with-materialize.md new file mode 100644 index 0000000000..4fe7cf9908 --- /dev/null +++ b/content/blog/posts/cdc-with-materialize.md @@ -0,0 +1,177 @@ +--- +title: Change Data Capture with Neon and Materialize +description: 'Replicate Neon data in real time to build better, faster products' +excerpt: >- + Now that Neon supports logical replication, your head might be filled with + questions about how to turn this new feature into interactive data apps, + better customer experiences, and other use cases ripe for fresh data. Do you + need to rethink your architecture? Does your team need... +date: '2023-12-21T12:23:12' +updatedOn: '2024-03-01T14:00:38' +category: community +categories: + - community +authors: + - marta-paes +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cdc-with-materialize/cover.png + alt: null +isFeatured: false +seo: + title: Change Data Capture with Neon and Materialize - Neon + description: 'Replicate Neon data in real time to build better, faster products' + keywords: [] + noindex: false + ogTitle: Change Data Capture with Neon and Materialize - Neon + ogDescription: >- + Now that Neon supports logical replication, your head might be filled with + questions about how to turn this new feature into interactive data apps, + better customer experiences, and other use cases ripe for fresh data. Do you + need to rethink your architecture? Does your team need to catch up with all + the streaming jargon? In […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cdc-with-materialize/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/cdc-with-materialize/image-10-1024x576-8d261962.png) + +Now that [Neon supports logical replication](https://neon.tech/blog/change-data-capture-with-serverless-postgres), your head might be filled with questions about how to turn this new feature into interactive data apps, better customer experiences, and other use cases ripe for **fresh data**. Do you need to rethink your architecture? Does your team need to catch up with all the streaming jargon? In this blog post, we’ll walk through how you can use Materialize to replicate data from Neon in real-time without needing to rethink much. + +## Why Materialize? + +[Logical replication](https://neon.tech/docs/guides/logical-replication-materialize) allows capturing `INSERT` s, `UPDATE` s, and `DELETE` s as these operations happen in your Neon database, making this data available to an external system for processing — a pattern known as Change Data Capture (CDC). If your use case requires more than simple mirroring, Materialize can help you transform the replicated data into **actionable results** that are **always fresh and consistent**. + +Materialize shines at [operational use cases](https://materialize.com/blog/operational-data-warehouse/), where an analytical data warehouse would be too slow or too costly, and a stream processor would introduce too much complexity. It can handle arbitrarily complex transformations on changing data using — you’ve guessed it — just SQL. For CDC from a PostgreSQL database like Neon, it has the added benefit of requiring **no additional infrastructure** (_e.g._ Debezium), while guaranteeing **transactional consistency**. + +Enough talk! + +Let’s see what it takes to build a CDC pipeline to replicate data from Neon to Materialize in real time. If you want to follow along, sign up for a [Materialize free trial](https://materialize.com/register/)! + +## Get started with Neon + Materialize + +### Configure logical replication in Neon + +**1.** First, enable logical replication for your Neon project: + +1. Select your project in the [Neon Console](https://console.neon.tech/app/projects). +2. On the Neon Dashboard, select Settings. +3. Select Beta. +4. Click Enable to enable logical replication. + +
+

🖐️ It’s important to note that enabling logical replication in Neon cannot be reverted, and triggers a restart of all active compute endpoints in your Neon project. Any active connections will be dropped and have to reconnect.

+
+ +**2.** Double check that logical replication is enabled by running the following query: + +```sql +SHOW wal_level; + + wal_level +----------- + logical +``` + +Once logical replication is enabled, the next step is to create a publication with the tables that you want to replicate to Materialize. It’s also recommended to [create a dedicated user](https://neon.tech/docs/manage/roles) for replication. + +**4.** Grant the replication user the required permissions on the tables you want to replicate: + +```sql +GRANT CONNECT ON DATABASE neon_demo TO neon_repl; +GRANT USAGE ON SCHEMA public TO neon_repl; +GRANT SELECT ON playing_with_lr TO neon_repl; +``` + +For each table that you want to replicate to Materialize, set the [replica identity](https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-REPLICA-IDENTITY) to FULL: + +```sql +ALTER TABLE playing_with_lr REPLICA IDENTITY FULL; +``` + +`REPLICA IDENTITY FULL` ensures that the replication stream includes the previous data of changed rows, in the case of `UPDATE` and `DELETE` operations, and enables Materialize to ingest Neon data with minimal in-memory state. + +**5.** Create a [publication](https://www.postgresql.org/docs/current/logical-replication-publication.html) with the tables you want to replicate: + +```sql +CREATE PUBLICATION mz_demo FOR TABLE playing_with_lr; +``` + +The mz_demo publication will contain the set of changes generated for the specified table, and will be used to ingest the replication stream into Materialize next! + +### Create a source in Materialize + +Logical replication in Neon follows PostgreSQL’s native replication protocol, so you can jump straight into action using Materialize’s [direct PostgreSQL source](https://materialize.com/docs/sql/create-source/postgres/). + +**1.** Log in to Materialize. You’ll be dropped into the **SQL shell**, which you can use to run the commands in the next steps. + +**2.** Use the connection details of your Neon endpoint to create a [connection](https://materialize.com/docs/sql/create-source/postgres/#creating-a-connection): + +```sql +-- Your Neon password goes here +CREATE SECRET neonpass AS 'safe12345'; +CREATE CONNECTION neon TO POSTGRES ( + -- Your Neon endpoint goes here + HOST 'ep-dry-bread-********.eu-central-1.aws.neon.tech', + PORT 5432, + -- Your Neon replication user goes here + USER 'neon_repl', + PASSWORD SECRET neonpass, + SSL MODE 'require', + DATABASE 'neondb_cdc' + ); +``` + +**2.** Create a [PostgreSQL source](https://materialize.com/docs/sql/create-source/postgres/) using the connection you just created, as well as the publication you created upstream: + +```sql +CREATE SOURCE neon_demo + IN CLUSTER default + FROM POSTGRES CONNECTION neon (PUBLICATION 'mz_demo') + FOR ALL TABLES; +``` + +Materialize will create a subsource for each table in the publication, which will first get hydrated with a snapshot of the existing data, and then carry on receiving new data as it arrives: + +```sql +SHOW SUBSOURCES ON neon_demo; + + name | type +--------------------+----------- + neon_demo_progress | progress + playing_with_lr | subsource + +SELECT * FROM playing_with_lr; + + col +------------- + hello + logical + replication +``` + +### Transform data in Materialize + +To get a quick taste of what Materialize can do, let’s keep track of the row count in `playing_with_lr`, and see how adding and deleting data in Neon immediately affects the results in Materialize. + +**1.** In Materialize, create a view with a count aggregate function: + +```sql +CREATE VIEW lr_cnt AS +SELECT COUNT(*) +FROM playing_with_lr; + +SELECT * FROM lr_cnt; + +count +------- + 3 +``` + +**2.** Switch back to Neon, and insert some new data into the `playing_with_lr` table. Then, query the `lr_cnt` view in Materialize one more time: + +This isn’t a very complex task, but…it’s still somewhat magic to see the count instantly update, no? Hopefully, your use case will be strictly more complex than this, and Materialize will make you Ooh! and Aah! at how efficiently it can handle CDC pipelines (which can be…[surprisingly costly](https://materialize.com/blog/capturing-cdc-data/)). + +### What’s next? + +If you’re as excited as we are about logical replication support in Neon, we encourage you to learn more about Materialize and how it can help you **build better products, faster**. Check out the [Materialize customer stories](https://materialize.com/customer-stories/) to see operational use cases in the wild, and [reach out](https://materialize.com/demo/) if you’d like to get a hands-on walkthrough with our team. diff --git a/content/blog/posts/change-data-capture-with-serverless-postgres.md b/content/blog/posts/change-data-capture-with-serverless-postgres.md new file mode 100644 index 0000000000..f112df7815 --- /dev/null +++ b/content/blog/posts/change-data-capture-with-serverless-postgres.md @@ -0,0 +1,168 @@ +--- +title: Change Data Capture with Serverless Postgres +description: Announcing Logical Replication in Neon +excerpt: >- + Modern applications often require loosely coupled components and services that + help teams and systems to scale. These data pipelines generate continuous data + streams that need to be replicated, processed, or analyzed. However, moving + data between different data stores can serious... +date: '2023-12-21T12:23:07' +updatedOn: '2024-03-27T11:38:57' +category: community +categories: + - community +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/change-data-capture-with-serverless-postgres/cover.png + alt: null +isFeatured: false +seo: + title: Change Data Capture with Serverless Postgres - Neon + description: Announcing Logical Replication in Neon + keywords: [] + noindex: false + ogTitle: Change Data Capture with Serverless Postgres - Neon + ogDescription: >- + Modern applications often require loosely coupled components and services + that help teams and systems to scale. These data pipelines generate + continuous data streams that need to be replicated, processed, or analyzed. + However, moving data between different data stores can seriously compromise + the quality and reliability of your decisions because inconsistent data or + corruption occurs during […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/change-data-capture-with-serverless-postgres/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/change-data-capture-with-serverless-postgres/image-8-1024x576-cd71c190.png) + +Modern applications often require loosely coupled components and services that help teams and systems to scale. These data pipelines generate continuous data streams that need to be replicated, processed, or analyzed. + +However, moving data between different data stores can seriously compromise the quality and reliability of your decisions because inconsistent data or corruption occurs during transformation. This is why [Change Data Capture (CDC)](https://en.wikipedia.org/wiki/Change_data_capture) has emerged as one of the most popular methods to synchronize data across multiple data stores. One way to use CDC in Postgres is with [logical replication](https://www.postgresql.org/docs/current/logical-replication.html#:~:text=Logical%20replication%20is%20a%20method,byte%2Dby%2Dbyte%20replication.). + +Today, we’re excited to announce the release of logical replication in beta on Neon. This feature lets you stream your data hosted on Neon to external data stores, allowing for change data capture and real-time analytics. + +## Why CDC matters? + +CDC refers to the process of capturing changes made to data in a database – such as inserts, updates, and deletes – and then delivering these changes to downstream processes or systems. + +CDC operates by monitoring and capturing data changes in a source database as they occur. This is a departure from traditional batch processing, where data updates are transferred at scheduled intervals. CDC ensures that every change is captured and can be acted upon almost instantaneously. + +Why CDC Matters + +- **Data synchronization**: In a distributed system architecture, keeping data synchronized across various platforms and services is critical. CDC facilitates this by providing a mechanism for real-time data replication. +- **Minimizing Latency**: By capturing changes as they happen, CDC minimizes the latency in data transfer. This is essential for applications where even a slight delay in data availability can lead to significant issues, such as financial trading systems. +- **Enabling Event-Driven Architectures**: CDC is a cornerstone for building event-driven systems. In such architectures, actions are triggered in response to data changes, making real-time data capture essential. +- **Data warehousing and real-time analytics**: For organizations relying on data warehouses and analytics tools for decision-making, CDC ensures that the data in these systems is current, enhancing the accuracy of insights. + +Now that we understand it better, let’s explore the technical mechanics of how CDC is implemented in Postgres through logical replication. + +## Logical replication: under the hood + +In Postgres, logical replication is one of the methods of implementing CDC and streaming data from your database to an external source. It uses a publisher-subscriber model. + +Your Neon database works as a publisher, copying first a snapshot of the data and then streaming changes to one or more target data stores (subscribers). This model allows for selective replication, where only specified tables or even specific columns within a table can be replicated. + +Learn more about connecting [Neon to different data stores](https://neon.tech/docs/guides/logical-replication-guide) in the documentation. + +The [Write-Ahead-Log (WAL)](https://www.postgresql.org/docs/current/wal-intro.html) is a fundamental component in Postgres, designed to ensure data integrity and facilitate recovery. It records every change made to the database, including transactions and their states. + +For logical replication, the WAL serves as the primary data source. The WAL captures the comprehensive sequence of data changes, which are then decoded for replication purposes. Logical replication transforms the WAL to a format accepted by the subscriber through logical decoding, and the `walsender` then streams the transformed data using the replication protocol. + +The `walsender` initiates the logical decoding of the WAL using an output plugin. Postgres ships with several logical decoding plugins that can output the data in various formats. In addition, new plugins can be developed. + +For instance, in a Postgres-to-Postgres logical replication, the standard `pgoutput` plugin transforms the data changes to the logical replication protocol. The transformed data is subsequently streamed using the replication protocol, which maps it to local tables and applies the changes in the exact sequence of the original transactions. However, integrations with non-Postgres systems require an output format different from the standard one specifically designed for Postgres-to-Postgres logical replication. + +Today’s data pipelines involve more than one data store type. For example, you can integrate all your Postgres databases into a data warehouse or streaming platform, such as [Materialize](https://materialize.com/) or [Kafka](https://kafka.apache.org/), to process and analyze data at higher scales. This is why, with the release of logical replication on Neon, we added support for [wal2json](https://github.com/eulerto/wal2json), which outputs changes in the JSON format to be easily consumed by other systems and data stores. + +You can read more on [Change Data Capture using Neon and Materialize](https://neon.tech/blog/cdc-with-materialize) by our friend Marta Paes, to learn how to integrate your database with external systems. + +## Logical vs. physical replication + +Logical replication differs from physical replication in that it replicates changes at the data level (row-level changes) rather than replicating the entire database block. This allows for more selective replication and reduces the amount of data transferred. Unlike snapshot replication, which provides a full copy of the data at a specific point in time, logical replication ensures continuous streaming of changes, making it more suitable for applications that require near real-time data availability. + +This comparison highlights the distinct characteristics, advantages, and applications of logical and physical replication. + +| **Logical Replication** | **Physical Replication** | +| ------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Row-Level Changes**: focuses on replicating specific row-level changes (INSERT, UPDATE, DELETE) in selected tables. | **Block-Level Replication**: replicates the entire database at the block level. It creates an exact copy of the source database, including all tables and system catalogs. | +| **Flexibility**: Offers the flexibility to replicate specific tables and even specific columns within tables. | **Limitations**: Doesn’t allow for selective table replication and requires the same PostgreSQL version on both the primary and standby servers. | +| **WAL-based**: Uses the WAL for capturing changes, but with logical decoding to convert these changes into a readable format for the subscriber. | **Streaming Replication**: Changes are streamed as they are written to the WAL, minimizing lag. | +| **Use Cases**: Ideal for situations requiring selective replication, minimal impact on the source database, or cross-version compatibility. | **Use Cases**: Best suited for creating read-only replicas for load balancing, high availability, and disaster recovery solutions. | + +## Get started with logical replication + +To enable logical replication, navigate to your project’s settings in the console and click on the “Beta” tab, locate Logical Replication then on the “Enable” button. + +Note that enabling logical replication will restart your compute instance, which will drop existing connections. A subscriber may also keep the connection to your Neon database active, preventing your Neon instance from scaling to zero. + +This action is also irreversible, and you will not be able to disable logical replication for your project. + + + +Ensure that logical replication is enabled by running the following query in the [SQL Editor](https://neon.tech/docs/get-started-with-neon/query-with-neon-sql-editor) within the Neon console or using `psql` on your terminal. + +```sql +SHOW wal_level; + + wal_level +----------- + logical +``` + +### Create a publication + +Let’s assume you have the following users table: + +```sql +CREATE TABLE users ( + + id SERIAL PRIMARY KEY, + + username VARCHAR(50) NOT NULL, + + email VARCHAR(100) NOT NULL + +); +``` + +Execute the following query to create a publication for the users table: + +```sql +CREATE PUBLICATION users_publication FOR TABLE users; +``` + +Learn more about [how to connect Neon to different data stores](https://neon.tech/docs/guides/logical-replication-guide) in the documentation. + +## Limitations + +While logical replication in Neon Postgres offers numerous benefits for real-time data synchronization and flexibility, it has some limitations: + +**Publisher, not a subscriber** + +This release of logical replication on Neon is in beta, and for security reasons, it does not include subscriber capabilities at the moment. We are currently working on these security constraints, which should be supported in future releases. + +**Logical replication and Auto-suspend** + +In a logical replication setup, a subscriber may keep the connection to your Neon publisher database active to poll for changes or perform sync operations, preventing your Neon compute instance from scaling to zero. Some subscribers allow you to configure connection or sync frequency, which may be necessary to continue taking advantage of Neon’s Auto-suspend feature. Please refer to your subscriber’s documentation or contact their support team for details. + +**Data Definition Language (DDL) Operations** + +Logical replication in Postgres primarily handles Data Manipulation Language (DML) operations like INSERT, UPDATE, and DELETE. However, it does not automatically replicate Data Definition Language (DDL) operations such as CREATE TABLE, ALTER TABLE, or DROP TABLE. This means that schema changes in the publisher database are not directly replicated to the subscriber database. + +Manual intervention is required to replicate DDL changes. This can be done by applying the DDL changes separately in both the publisher and subscriber databases or by using third-party tools that can handle DDL replication. + +**Replication Lag** + +In high-volume transaction environments, there is potential for replication lag. This is the time delay between a transaction being committed on the publisher and the same transaction being applied on the subscriber. + +It’s important to monitor replication lag and understand its impact, especially for applications that require near-real-time data consistency. Proper resource allocation and optimizing the network can help mitigate this issue. + +## Conclusion + +Logical replication is undoubtedly one of the most important features for modern applications. As we continue to develop its capabilities, we encourage you to test, experiment, and push the boundaries of what logical replication can do. Join us on [Discord](https://neon.tech/discord), and share your experiences, suggestions, and challenges with us. + +We can’t wait to see what you build with Neon. diff --git a/content/blog/posts/chat-with-neon-postgres-using-natural-language.md b/content/blog/posts/chat-with-neon-postgres-using-natural-language.md new file mode 100644 index 0000000000..480b30da4c --- /dev/null +++ b/content/blog/posts/chat-with-neon-postgres-using-natural-language.md @@ -0,0 +1,89 @@ +--- +title: Chat with Neon Postgres using natural language +description: AskYourDatabase is the ChatGPT for SQL databases +excerpt: >- + Interacting with SQL databases can be challenging for non-technical team + members, often requiring custom-built GUI tools. If you’re not an expert, + writing SQL queries can become a barrier to accessing data—this is where AI + can help. What is AskYourDatabase? AskYourDatabase is an... +date: '2024-07-19T13:28:59' +updatedOn: '2024-07-19T13:31:14' +category: community +categories: + - community +authors: + - sheldon-niu +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/cover.jpg + alt: null +isFeatured: false +seo: + title: Chat with Neon Postgres using natural language - Neon + description: AskYourDatabase is the ChatGPT for SQL databases + keywords: [] + noindex: false + ogTitle: Chat with Neon Postgres using natural language - Neon + ogDescription: >- + Interacting with SQL databases can be challenging for non-technical team + members, often requiring custom-built GUI tools. If you’re not an expert, + writing SQL queries can become a barrier to accessing data—this is where AI + can help. What is AskYourDatabase? AskYourDatabase is an AI-powered tool + that enables natural language interaction with SQL databases. It allows + users […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/neon-askyourdatabase-1-1024x576-d9a6acb5.jpg) + +Interacting with SQL databases can be challenging for non-technical team members, often requiring custom-built GUI tools. If you’re not an expert, writing SQL queries can become a barrier to accessing data—this is where AI can help. + +## What is AskYourDatabase? + +[AskYourDatabase](https://www.askyourdatabase.com/) is an AI-powered tool that enables natural language interaction with SQL databases. It allows users to perform many database tasks, including data analysis, business intelligence, CRUD operations, data visualization, and schema migrations, without writing SQL queries. + +With persistent memory and assistant behavior powered by GPT-4, AI will automatically understand your schema, make queries, correct them if there are syntax errors, and explain the results to you in a human-understandable way. + +For example, when asked to “List the four most popular users’ databases and visualize it,” the AI: + +1. Creates the correct SQL query. +2. Executes the query. +3. Explains the results. +4. Presents a visualization, like a pie chart. + +These steps are performed autonomously without additional user instructions. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/screenshot-2024-07-19-at-32528percente2percent80percentafpm-1024x844-738961b6.png) + +## Who can benefit from this? + +AskYourDatabase is particularly useful for teams where somebody not proficient in SQL might need quick access to database information. For example: + +- CEOs and managers can make data-driven decisions without needing to learn SQL. +- Customer Support Teams can quickly access and update customer records. +- Business Analysts can generate reports and insights on demand. + +Even technical teams can offload routine data tasks to the AI, freeing up time for more complex projects. + +## Combining AskYourDatabase with Neon branching + +Neon is uniquely advantageous for AskYourDatabase due to their branching feature. [Neon allows you to create isolated copies, or “branches,” of your database environment for development:](https://neon.tech/docs/introduction/branching) each branch is a full, independent clone of your database, including its data and schema, created without impacting the production environment. + +Instead of granting non-technical teams access to the production database, with Neon you can easily connect AskYourDatabase to a development branch. This setup allows your non-technical teammates to query and manipulate data safely without impacting the live environment. + +## How to integrate AskYourDatabase with Neon + +Once you have downloaded the AskYourDatabase app [here](https://www.askyourdatabase.com/download), all you need to do is to click on `Connect to your database`, and paste the URL of your Neon dev branch: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/screenshot-2024-07-19-at-32622percente2percent80percentafpm-1024x639-944617d3.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/screenshot-2024-07-19-at-32631percente2percent80percentafpm-1024x639-6fc3119d.png) + +That’s it! Once the connection is established, a new chat will open, and you can start asking your database questions. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/chat-with-neon-postgres-using-natural-language/screenshot-2024-07-19-at-32722percente2percent80percentafpm-1024x576-30f977cd.png) + +## Request your discount + +We are so excited for you to try this integration. If you’re a Neon user, you can claim a special discount. Shoot me an email at [sheldon@askyourdatabase.com](mailto:sheldon@askyourdatabase.com) if you’re interested. diff --git a/content/blog/posts/checkpoints-for-agents-with-neon-snapshots.md b/content/blog/posts/checkpoints-for-agents-with-neon-snapshots.md new file mode 100644 index 0000000000..ee56b3e4f2 --- /dev/null +++ b/content/blog/posts/checkpoints-for-agents-with-neon-snapshots.md @@ -0,0 +1,162 @@ +--- +title: Build Checkpoints For Your Agent Using Neon Snapshots +description: Your users will be able to jump between app versions as seen in Replit and v0 +excerpt: >- + You can now create Neon snapshots via API. This new capability isn’t just + useful for backups or disaster recovery, but also serves as a powerful + building block for one of the most requested features in agentic platforms: + versioning (or checkpoints). Neon’s snapshots, built on our... +date: '2025-09-16T22:52:22' +updatedOn: '2026-01-02T17:41:04' +category: app-platform +categories: + - app-platform + - product + - ai +authors: + - andre-landgraf + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/checkpoints-for-agents-with-neon-snapshots/cover.jpg + alt: null +isFeatured: true +seo: + title: Build Checkpoints For Your Agent Using Neon Snapshots - Neon + description: >- + Build checkpoints into your agent with Neon’s Snapshots API. Restore code, + schema, and data instantly - just like versioning in Replit or v0. + keywords: [] + noindex: false + ogTitle: Build Checkpoints For Your Agent Using Neon Snapshots - Neon + ogDescription: >- + Build checkpoints into your agent with Neon’s Snapshots API. Restore code, + schema, and data instantly - just like versioning in Replit or v0. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/checkpoints-for-agents-with-neon-snapshots/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/checkpoints-for-agents-with-neon-snapshots/neon-snapshots-api-1024x576-c6dedd99.jpg) + +You can now [create Neon snapshots via API](https://api-docs.neon.tech/reference/createsnapshot). This new capability isn’t just useful for backups or disaster recovery, but also serves as a [powerful building block for one of the most requested features in agentic platforms: versioning (or checkpoints).](https://neon.com/docs/ai/ai-database-versioning) + +Neon’s snapshots, built on our [copy-on-write branching](https://neon.com/blog/instantly-copy-tb-size-datasets-the-magic-of-copy-on-write), make it simple and cost-effective to implement this feature – unlocking a magical user experience for your users. With snapshots, you can give your agents the ability to create checkpoints after each change, so your users can jump between app versions and restore not only the code, but also the exact database schema and data that version was built on. + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + + +## The Use Case: Jump Between Versions of Your App + +[If you’re building an agentic platform](https://neon.com/use-cases/ai-agents), your users are probably already asking for the ability to go back in time. This is a tremendously useful feature to have when you’re vibe coding: it compensates for potential misunderstandings between you and the agent, and it allows for a lot of iteration – giving you an easy way to compare different versions and roll back to the best one. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/checkpoints-for-agents-with-neon-snapshots/image-2-1024x768-90bd2539.png) + +With code alone, it’s relatively easy to build a rollback feature: just deploy a previous commit or version of the generated code. But for agent-built apps, the database schema and data often change together with the code, and if you revert the code without also reverting the database, you risk giving your end users a wonky experience with broken queries, failed migrations, and mismatched data that causes more bugs down the line. + +The alternative is to build a checkpoint abstraction that also understands database state: + +- Every time the agent modifies the app, it also saves a database snapshot. +- Each snapshot represents the exact schema and data at that moment. +- Restoring a snapshot puts the database back into that state, so the code “just works” + +## Building Agent Checkpoints with Neon: Step-by-Step + +In case you’re not familiar, [Neon](https://neon.com/) is a serverless Postgres database with an architecture that makes it ideal for building a feature like this. At the core is our architecture with decoupled compute and storage, with a [custom storage engine](https://neon.com/blog/what-you-get-when-you-think-of-postgres-storage-as-a-transaction-journal) built from the ground up. This storage system uses copy-on-write to track changes over time, which means we can reference any previous state of the database without duplicating all the data. + +This is perfect for building checkpoints, and **we’ve built a live demo to show exactly how this works:** + +[https://snapshots-as-checkpoints-demo.vercel.app/](https://snapshots-as-checkpoints-demo.vercel.app/) + + + +In this demo, + +- An agentic platform is building an app via prompts +- Every prompt affects both the app code and the underlying Postgres database +- After each change, the platform takes a Neon snapshot of the production branch, saving the complete state of the app after each user prompt +- These snapshots are stored alongside metadata in a separate meta database that keeps track of the timeline of checkpoints +- If the end user wants to go back to a previous version, whether to preview it or fully restore it, the platform looks up the associated snapshot ID in the meta database and calls Neon’s [restore snapshot API](https://api-docs.neon.tech/reference/restoresnapshot) +- Neon instantly reverts the production branch to the exact schema and data from that checkpoint, so the rolled-back code runs without broken queries, failed migrations, or mismatched data. + + +If you’re looking to build something similar, [we’ve prepared a guide that walks you through it step-by-step.](https://neon.com/docs/ai/ai-database-versioning) + + +### Under the hood + +The guide above explains you everything you need to know to use the snapshots API to build your own versioning feature, but let’s walk you through the basic steps, using our demo code as an example. All the code for the demo lives here: + +[https://github.com/neondatabase-labs/snapshots-as-checkpoints-demo](https://github.com/neondatabase-labs/snapshots-as-checkpoints-demo) + +#### Setting up your databases + +The first thing to notice is that the demo references two separate Postgres databases: + +- App database: Where your agent-generated app lives. Its schema evolves as the user interacts with it (e.g. adding columns like name, email, role, tags). +- Meta database: This is used to track checkpoints via a checkpoints table, recording the version IDs and associated snapshot IDs. + +#### Identifying the production branch + +Now that you have your databases identified, it’s time to get familiar with the concept of branches, since they’re the primitive that works behind the snapshots API you’re going to use for your checkpoints. + +Differently from other Postgres platforms, Neon is organized in [branches](https://neon.com/docs/introduction/branching). Think of branches living in the same project as [separate environments](https://neon.com/branching) that share the same storage / can be synced together (pretty cool). + +For our demo app, we told the system which branch is our production branch, so it can set it up as the root for changes. The file [lib/neon/branches.ts](https://github.com/neondatabase-labs/snapshots-as-checkpoints-demo/blob/main/lib/neon/branches.ts) is responsible for identifying the right branch ID via Neon’s API, so you always snapshot or restore against the correct branch reference. + +#### Creating the snapshots + +Now you can understand better what’s happening after each prompt-driven mutation: + +1. The agent modifies the app database schema/data +2. Metadata is recorded in the meta database, e.g. a new row is created in your checkpoints table + +The demo then calls the Neon Snapshots API via [lib/neon/create-snapshot.ts](https://github.com/neondatabase-labs/snapshots-as-checkpoints-demo/blob/main/lib/neon/create-snapshot.ts). This is what snapshots the current state of the production branch and returns a snapshot ID, which is stored with the checkpoint metadata + +#### Navigating between versions + +Once snapshots are set up, what happens when a user jumps to an older version? + +1. The app looks up the desired checkpoint in the meta database to get its snapshot_id +2. It calls the lib/neon/branches.ts helper (or Neon’s list-branches API directly) to fetch the current target_branch_id for the branch named “production” +3. It then calls the restore API (lib/neon/apply-snapshot.ts) with finalize_restore: true and target_branch_id: <current production branch id> +4. The response includes operation IDs representing the restore progress +5. lib/neon/operations.ts polls the operations API until all operations reach a terminal state (finished, skipped, or cancelled) +6. After the restore completes, the UI fetches and renders + - App data from the app database + - The app UI for that version + - The updated schema (information_schema.columns) + - The list of available checkpoints (from the meta DB) + +| Key component | Role | +| ----------------------------------------------------- | --------------------------------------------------------------------------------- | +| App database | Stores the evolving app schema and data | +| Meta database | Tracks version history and associated `snapshot_id` s | +| `lib/neon/branches.ts` | Resolves the current ID of the “production” branch (don’t hard-code) | +| `lib/neon/create-snapshot.ts` | Creates a snapshot after each prompt/change | +| `lib/neon/apply-snapshot.ts + lib/neon/operations.ts` | Calls restore with `finalize_restore:true` and polls operations to terminal state | +| UI (Next.js) | Displays the app, schema, and checkpoint timeline; triggers create/restore flows | +| `lib/contacts.ts` | Schema mutations, CRUD, and schema/data queries for the app DB | +| `lib/checkpoints.ts` | Creates/reads checkpoint rows in the meta DB and links them to `snapshot_id` | + + +Treat the branch name (“production”) as your stable label and resolve its ID right before each restore. The underlying branch ID may change across restores, so avoid caching it long-term. If your code caches connection metadata tied to a branch ID, refresh it after the restore completes. + + +## Build Your Full-Stack Agent Platform on Neon + +Neon isn’t just a place to store your app’s data – it’s a version-aware Postgres engine you can control entirely from code. + +As we introduced earlier, Neon has a [unique architecture](https://neon.com/blog/architecture-decisions-in-neon) that separates compute (the Postgres process) from storage (our custom engine). This separation enables some capabilities that are rare / flat-out impossible to build with conventional Postgres setups, and that directly enable this snappy versioning experience that remembers database state: + +- **Copy-on-write storage.** In Neon, [every database change is stored as a delta](https://neon.com/blog/what-you-get-when-you-think-of-postgres-storage-as-a-transaction-journal), so we can reference any historical state without making a full duplicate. +- **Instant speed.** Because snapshots are just references to existing data, they can be created and restored without the heavy I/O you’d expect from a backup/restore cycle. The restore process is just as fast and efficient – Neon just swaps the branch pointer to the snapshot’s state. +- **API-first provisioning and control.** Everything in Neon is exposed via API – from creating databases and branches, to controlling compute consumption, to (now!) creating and restoring snapshots. You can wire up this entire checkpointing system into your agent’s workflow without touching a UI. + +The [snapshots API](https://api-docs.neon.tech/reference/createsnapshot) makes it simple to build agent features that let users jump between versions of their app. Code rollbacks alone can’t guarantee that – but with Neon, you get schema and data rollbacks as well. + +Try the [live demo](https://snapshots-as-checkpoints-demo.vercel.app/) to see how it works. Then, [sign up for Neon](https://console.neon.tech/signup) and start building your agent. We’ve built a step by step guide [here](https://neon.com/docs/ai/ai-database-versioning). + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + diff --git a/content/blog/posts/cli.md b/content/blog/posts/cli.md new file mode 100644 index 0000000000..1bcdb5acd2 --- /dev/null +++ b/content/blog/posts/cli.md @@ -0,0 +1,145 @@ +--- +title: Postgres at your fingertips with the Neon CLI +description: >- + Today, we are releasing Neon CLI, a command-line interface that enables + developers to manage Neon resources directly from the terminal! With Neon + CLI, you can handle authentication, and manage projects, branches, + databases, roles, and much more without leaving your command line. Get + started by installing the CLI using the following command: Use neonctl auth + command […] +excerpt: >- + Today, we are releasing Neon CLI, a command-line interface that enables + developers to manage Neon resources directly from the terminal! With Neon CLI, + you can handle authentication, and manage projects, branches, databases, + roles, and much more without leaving your command line.... +date: '2023-07-12T12:57:50' +updatedOn: '2023-12-20T21:00:54' +category: community +categories: + - community + - workflows +authors: + - raouf-chebri + - daniel-price +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/cli/image-34.png' + alt: null +isFeatured: false +seo: + title: Postgres at your fingertips with the Neon CLI - Neon + description: >- + Today, we are releasing Neon CLI, a command-line interface that enables + developers to manage Neon resources directly from the terminal! With Neon + CLI, you can handle authentication, and manage projects, branches, + databases, roles, and much more without leaving your command line. Get + started by installing the CLI using the following command: Use neonctl auth + command […] + keywords: [] + noindex: false + ogTitle: Postgres at your fingertips with the Neon CLI - Neon + ogDescription: >- + Today, we are releasing Neon CLI, a command-line interface that enables + developers to manage Neon resources directly from the terminal! With Neon + CLI, you can handle authentication, and manage projects, branches, + databases, roles, and much more without leaving your command line. Get + started by installing the CLI using the following command: Use neonctl auth + command […] + image: 'https://cdn.neonapi.io/public/images/pages/blog/cli/social.png' +--- + +![Old computer with a terminal being displayed, placed on a colorful background](https://cdn.neonapi.io/public/images/pages/blog/cli/image-33-1024x576-ac5d9c0c.png) + +Today, we are releasing Neon CLI, a command-line interface that enables developers to manage Neon resources directly from the terminal! With Neon CLI, you can handle authentication, and manage projects, branches, databases, roles, and much more without leaving your command line. + + + +Get started by installing the CLI using the following command: + +```bash +npm i -g neonctl +``` + +Use `neonctl auth` command to connect to your Neon account, or create a new one. + +## What Can You Do with the Neon CLI? + +The CLI offers an integrated approach to managing other aspects of your projects, including CRUD (Create, Read, Update, Delete) operations on your projects, branches, databases, and roles. Here are a few examples of what you can do using the CLI: + +### Projects + +With Neon CLI you can create a Postgres database and get a connection string in seconds: + +```bash +neonctl projects create +``` + +Output: + +```bash +┌───────────────────┬──────┬───────────────┬──────────────────────┐ +│ Id │ Name │ Region Id │ Created At │ +├───────────────────┼──────┼───────────────┼──────────────────────┤ +│ cold-lab-971294 │ cli │ aws-us-east-2 │ 2023-07-05T12:05:02Z │ +└───────────────────┴──────┴───────────────┴──────────────────────┘ +┌───────────────────────────────────────────────────────────────────────────────────────┐ +│ Connection Uri │ +├───────────────────────────────────────────────────────────────────────────────────────┤ +│ postgres://:@ep-lingering-moon-792025.us-east-2.aws.neon.tech/neondb │ +└───────────────────────────────────────────────────────────────────────────────────────┘ +``` + +The `projects create` returns your new project-id (cold-lab-971294) and the connection string so you can immediately use it in your project. + +### Branches and connection string + +You can also list your branches and create a new one to use for local development: + +```bash +neonctl branches list +``` + +Output: + +```bash +┌────────────────────────┬──────┬──────────────────────┬──────────────────────┐ +│ Id │ Name │ Created At │ Updated At │ +├────────────────────────┼──────┼──────────────────────┼──────────────────────┤ +│ br-purple-tooth-335860 │ main │ 2023-07-05T12:06:49Z │ 2023-07-05T12:11:04Z │ +└────────────────────────┴──────┴──────────────────────┴──────────────────────┘ +``` + +The CLI allows you to create Neon branches, similarily to when you create a local Git branch for local development. Neon branches are isolated copy of your database that you can modify without compromising your main database, ideal for development and testing. You can create a branch using the following command: + +```bash +neonctl branches create –-project-id= +``` + +The branches and projects `create` commands return a connection string. However, you can use the following command to display your connection string at any moment: + +```bash +neonctl cs --project-id= +``` + +Once you are done with your feature and have merged your code and schema changes, you can dispose of the branch: + +```bash +neonctl branches delete --project-id +``` + +### Integrated the CLI in your CI/CD pipelines + +You can use the CLI in your developer workflows and pipelines. If not authenticated, the CLI expects a `NEON_API_KEY` as an argument, that you can create in the [Neon console](https://console.neon.tech). You can also pass the API key as a parameter like in the command below: + +```bash +npx neonctl branches create --project-id= --api-key= +``` + +We invite you to visit the [documentation](https://neon.tech/docs/reference/neon-cli) for more information about the supported commands. + +## We’d love to hear your feedback + +We are currently actively developing CLI, so more features to the CLI are to come. [Share feedback](https://community.neon.tech/) and let us know what you’d like to see in Neon CLI. + +We thank you for your valuable contributions, encourage you to explore Neon CLI, and would love to hear from you to make the experience managing your Postgres databases better. diff --git a/content/blog/posts/coding-with-cursor-and-windsurf-side-by-side.md b/content/blog/posts/coding-with-cursor-and-windsurf-side-by-side.md new file mode 100644 index 0000000000..b5d3bfbc8e --- /dev/null +++ b/content/blog/posts/coding-with-cursor-and-windsurf-side-by-side.md @@ -0,0 +1,87 @@ +--- +title: Coding With Cursor and Windsurf Side by Side +description: Getting a taste of vibe coding +excerpt: >- + Since the release of GitHub Copilot, AI integration in IDEs has evolved + dramatically. Now, two IDEs are being talked about more than any other: + Windsurf and Cursor. Both tools are forks of VSCode, offer a similar set of + features, and use Anthropic’s Claude model under the hood. I... +date: '2025-03-20T00:41:40' +updatedOn: '2025-03-26T19:56:11' +category: ai +categories: + - ai +authors: + - sam-harrison +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/coding-with-cursor-and-windsurf-side-by-side/cover.png + alt: null +isFeatured: true +seo: + title: Coding With Cursor and Windsurf Side by Side - Neon + description: >- + Cursor and Windsurf are AI-powered IDEs based on VSCode. We try them both + for tab completions, context management, and agentic workflows. + keywords: [] + noindex: false + ogTitle: Coding With Cursor and Windsurf Side by Side - Neon + ogDescription: >- + Cursor and Windsurf are AI-powered IDEs based on VSCode. We try them both + for tab completions, context management, and agentic workflows. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/coding-with-cursor-and-windsurf-side-by-side/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/coding-with-cursor-and-windsurf-side-by-side/neon-pgversus-1024x576-662c2ec8.png) + +Since the release of GitHub Copilot, AI integration in IDEs has evolved dramatically. Now, two IDEs are being talked about more than any other: Windsurf and Cursor. Both tools are forks of VSCode, offer a similar set of features, and use Anthropic’s Claude model under the hood. + +I vibe coded with both IDEs and took some notes, using my experience developing the Neon Python driver as a guiding example. + +## What feels the same? + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/coding-with-cursor-and-windsurf-side-by-side/ad4nxcyibkhej0hw179mbmibie7hui3afd18m-zm-cic979agfsyqvsdwylsi2oy9ufbd-pkco3rmosq3yepbazbriyxxwkabrm44d9rdxahcjkfijgx8szz5ud8uob3wzicnv6vea-9b5492a6.gif) + +On the surface, Windsurf and Cursor feel strikingly similar. So much so that at first glance, you might think you’re simply looking at two VSCode windows. This means you get the familiar editor you already know and love, with the extensive extension marketplace and setting customization still available. + +Both editors provide the same general interface with the chat window on the right, and the same feature set you’ve come to expect from AI-powered IDEs, like better tab completions, multi file refactoring, agent-based workflows, and context management. + +## Tab Completions: Cursor’s Tab vs. Windsurf’s Tabs + +I find tab completions one of the most important features of any AI-powered IDE, and it’s where I noticed the biggest productivity gains when developing the Neon Python driver compared to my old setup. + +While both Windsurf and Cursor offer far better autocomplete experiences than GitHub Copilot, Cursor’s tab completions are particularly impressive. Specifically, I’d often make a small change at the top of a file that would need further adjustments below, and Cursor consistently predicted exactly what I needed next—I could just `tab-tab-tab` my way through the file. + +For a long time, Cursor held the edge in tab completions when compared to Windsurf’s SuperComplete. But Windsurf is shipping fast. As I was writing this article, they released Windsurf Wave 5, which introduced the Windsurf Tab, and the gap has narrowed. Previously, Cursor felt more aware of the codebase for tabs and provided more useful completions, but Windsurf autocompletions now benefit from the same context engine that powers its AI Agent Cascade, along with a larger and more powerful model. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/coding-with-cursor-and-windsurf-side-by-side/ad4nxfe0qatfcgdpjt6sawiu9vj11cswfm-u6rbzxwf5yot0urewlwoljuorn0zh8pravqu5uqr415s92rgs0cbhyypp23ydnshzitx6qzn3hsdlb27krqaehlv3ke2z6jcciocoof-pa-d5a8d8c7.gif) + +## Context management: Attaching the relevant information + +Context management, or the way IDEs handle relevant information to inform their AI suggestions, is another important differentiator. Cursor offers robust support for custom rules that can be conditionally applied based on file type, directory, and other factors. For instance, I easily defined a set of Python rules with strict type hinting, NumPy-style docstrings, and other linting preferences. Throughout my development, Cursor adhered perfectly to these rules, saving me a lot of time in docstring writing and code-style preference refactoring. Windsurf also has built-in rule management, with support for both global rules and rules local to the project, but not for rule file pattern matching or semantic description attachment. + +Yet Windsurf excels in one crucial area: context indexing. Its automatic indexing and understanding of the project structure felt superior. For example, when I made changes to the core logic of the application, Windsurf would reach out to the tests directory and update the relevant unit and integration tests. Cursor is also capable of this, but it requires prompting to get it there, which gets tedious and can easily be forgotten. On the other hand, it offers a wider variety of options for attaching content including most recent changes, git history, lint errors, and most importantly custom docs. Here, at the beginning of my project I was able to create a custom documentation context for Psycopg which I could easily reference. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/coding-with-cursor-and-windsurf-side-by-side/ad4nxdjx5u3jdi1tambdz2fswfm454p5psa7vtk0clidp8loq8kcmwngwledwimuiai9rwvc0hkvqx50ueruzqng4ci5kb07wpu4kjx3flacktvpp0dcus70fe428gp3f1hzelazbh-c89e6363.gif) + +Windsurf offers only a pre-set documentation library without the means to extend it, though you can attach the link of the documentation as web context to achieve a very similar result. Despite this limitation in customizable context attachments, Windsurf’s automatic context management is really a standout feature and as AI models continue to evolve, this strength will only become more pronounced. + +## Agent mode: Cursor’s Composer vs Windsurf’s Cascade + +Both Windsurf’s Cascade and Cursor’s Composer offer agentic workflows, which allow LLMs to intelligently interact with your codebase, automate repetitive tasks, and implement entire features. + +In practice, I found that both still struggled with more complex coding scenarios. For instance, when implementing Python-to-Postgres type conversions in the Neon driver, both agents had difficulty leveraging Psycopg effectively, because the conversion needed to happen without establishing an actual database connection. Even after I devised a clear implementation plan and explicitly provided it in the prompt instructions, neither IDE could successfully execute without significant manual prompting. + +All hope is not lost, as both agents shined in many other tasks like tests and frontend development. In the case of testing, they generated robust, comprehensive unit and integration tests, accelerating development by at least a few hours without any detailed prompting. For this alone, the agents are well worth having integrated in the IDE. What’s more, to see how each of these fared in more line-heavy development, I tried both out in a React project of mine. After asking for the same component, they each generate working solutions, though Windsurf clearly had a broader understanding of my codebase, utilising some of my custom hooks and API endpoints many directories away. + +## Conclusion + +Both Cursor and Windsurf significantly boosted my productivity when porting the SQL-over-HTTP driver to Python. They both have strengths in different areas —to me, + +- **Cursor** excelled in tab completions, and offered tighter, more intuitive control, providing better confidence in the AI coding workflow. +- **Windsurf** offered better and automatic context indexing, leading to a more “magical” feeling agentic workflow. + +This vibe coding experiment not only provided insights into these two IDEs but also unveiled a new way to use Neon. You can now leverage Neon more effectively in Python serverless functions (e.g., AWS Lambdas or Azure Functions) by using SQL over HTTP for efficient single-shot queries and transactions. + +Check out the repository [here](https://github.com/sam-harri/neon-pyserverless)! diff --git a/content/blog/posts/cold-starts-just-got-hot.md b/content/blog/posts/cold-starts-just-got-hot.md new file mode 100644 index 0000000000..e939e20885 --- /dev/null +++ b/content/blog/posts/cold-starts-just-got-hot.md @@ -0,0 +1,103 @@ +--- +title: Cold starts just got hot +description: How we cut cold start time in half +excerpt: >- + tl;dr> Over the past few months, a bunch of efforts by the engineering team + have greatly reduced our “cold start time” for compute resources for idle + computes that become active. This post explores the problem and how the Neon + team worked on this problem. Background: What’s a cold s... +date: '2023-07-25T19:07:11' +updatedOn: '2024-03-01T16:00:08' +category: engineering +categories: + - engineering +authors: + - sam-kleinman +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cold-starts-just-got-hot/cover.jpg + alt: null +isFeatured: false +seo: + title: Cold starts just got hot - Neon + description: How we cut cold start time in half + keywords: [] + noindex: false + ogTitle: Cold starts just got hot - Neon + ogDescription: >- + tl;dr> Over the past few months, a bunch of efforts by the engineering team + have greatly reduced our “cold start time” for compute resources for idle + computes that become active. This post explores the problem and how the Neon + team worked on this problem. Background: What’s a cold start and why does it + matter? Let’s get one thing […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cold-starts-just-got-hot/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/cold-starts-just-got-hot/neon-cold-starts-1024x576-dfe1b468.jpg) + +
+

tl;dr> Over the past few months, a bunch of efforts by the engineering

team have greatly reduced our “cold start time” for compute resources

for idle computes that become active. This post explores the problem

and how the Neon team worked on this problem.

+
+ +## Background: What’s a cold start and why does it matter? + +Let’s get one thing straight about this serverless thing: there are still, somewhere, in some way, servers. You, as a user of Neon don’t have to think about that, which is great, but we have to. The great benefit of Neon is that the server–we call them computes–that you connect to is ephemeral and stateless: when you’re not using it, you/we can turn it off. When the compute turns off, the cost–for everyone–of the “database” drops significantly. This is huge, and the idea is that if we can keep the compute off, except when it’s actually being used, there are a lot of workloads that become incredibly
resource efficient. + +The problem is that there are still servers, someone has to “turn on the compute” (us) and while the compute is stateless in the sense that it’s easy to quickly recreate it and there’s nothing that happens in the compute that can’t be recreated, it does have some transient state, and it takes (a small amount of time.) for this to get setup. + +In the history of Neon, the approach to this “startup” problem has been “let’s just make it fast:” until now, when you tried to connect to your endpoint (database) our proxy would tell our control plane “quick go start this compute,” and then the control plane would go and do this. That process takes somewhere between 3 and 6 seconds in the ideal case, but there was a lot of variance based on region and network activity, and as we add more features (extensions, etc,) it tends to get slower. “Just be fast,” at a certain point, doesn’t work anymore. + +## Making Cold Starts Warmer + +The rest of this post is about these 3-6 seconds, and what we’ve done to be able to get a compute, up, running into about 500 milliseconds (an order of magnitude! neat,) in many cases: we think we can get it even faster and have some projects on our roadmap to keep shaving off time, though maybe not _quite_ another order of magnitude! + +![Image](https://cdn.neonapi.io/public/images/pages/blog/cold-starts-just-got-hot/3-1024x562-4a60162d.jpg) + +Let’s be clear, allocating and configuring resources takes actual time, and there’s no black magic going on here. We’ve applied my two favorite optimization strategies: “if you have a really hard problem that you can’t solve, decide that you have a different problem,” and “the fastest way to do something is to not (need to) do it at all.” Though the applications of these strategies are pretty cool: + +### Only Reconfigure When you Need To + +Neon has a UI that lets users make changes to their database’s configuration, for managing users, databases, roles, and extensions. These configuration options translate to an idempotent operation that starts the instance, if needed, applies changes that you make in the console to your database. This operation takes between 1 and 1.5 seconds, and it’s always safe to run, and if (for whatever reason,) your branch was out of date or sync, applying the configuration would help keep things running as you expected. + +The big change here is that we don’t do this anymore: unless this is the first time you connect to your compute endpoint, there’s no need to keep track of that. Internally we track changes to configuration and use that to tell the compute if it needs to apply the configuration during start up. Most of the time, you don’t need to, and, we just saved ourselves a lot of time. + +We still apply the config, just to be safe, if your compute is idle and our availability health check wakes the endpoint up. Also, most of the time, if you make a configuration-impacting change on the console, we can just tell the compute to apply the configuration changes without needing to restart your compute, which we had to do more often previously. + +There are still times when we do have to apply configuration changes, but they are quite uncommon, and we’ve just saved everyone a lot of time by not doing something! Great! + +### Compute Pools + +If the “apply configuration” changes are “don’t do something that takes a long time,” our compute pools are “choosing to solve a different tractable problem in the face of an intractable problem.” + +Since its always going to take _some_ amount of time to start a compute, we decided to (mostly) accept that the ongoing cost of keeping compute starts consistently fast across our entire fleet wasn’t something that we could do reliably: so we didn’t. Instead we’re (basically) always starting computes before anyone asks for them, and then when we get a request for a compute, we just take an “empty” compute, give it some configuration and… that’s it? You have a compute in a few hundred milliseconds. + +I jest, but only a little. There were a lot of little changes that went into making this work: before our application and system we didn’t really have a concept of a “compute” separate from the endpoint, so we had to create that. Also, there’s a bunch of finesse and science in choosing at what rate we should start computes and the ideal number, location, and size of “empty computes” to maintain. We also don’t want computes to be idle in the pool for too long, both because we don’t want idle workloads that could take resources from a “real” workload, and also, we want every compute to be as up to date as possible, so we make sure to recycle computes after a period of time. + +We want to make sure that when you make a request for an idle compute there’s always something there waiting for your application! Sometimes, if we don’t have a compute in the pool that suits your needs, the system gracefully falls back to creating computes in the old on-demand way: it’s slower, but it works well. We’re collecting data whenever this happens so we can work to prevent it in the future. + +Having said all this, the largest part of the compute pool project was probably _not_ the compute pools themselves, but the changes to the way computes get their configuration. In the old model, we could give the compute the config when it started, but now the compute needs to be able to get the config for itself and handle getting a new configuration once it starts. This took a little bit of elbow grease, but it also enabled the “on-demand” config application features above. While we’re just rolling out compute pools now, the new configuration system has been running for a while. + +To be honest, I was expecting that the compute pools would take us from about 5 seconds to about 1.25 seconds, but in practice, it took us down to about .6 or .75 seconds, and the configuration loading changes brings us down another .2 seconds. But this is a little bit of a simplification: + +### The Odds and Ends + +While compute pools and configuration change tracking got us (and therefore you!) a lot of time back, there were other changes that have helped: networking changes to improve the way that we configure networking for new computes, caching some internal IP addresses to avoid waiting for internal DNS routing, tactically applying concurrency, and improving how we wait for computes to become active, and, of course, a little bit of good old fashion optimization of some particularly hot code paths. + +## The Road from Here + +But wait! There’s more! Half a second is good, but it’s not the end: most notably, these speedy starts are the fastest in our us-east-2 region, and slowest in ap-southeast-1. We are in the middle of a herculean task–sort of like replacing the engine on an airplane without landing it–to divide our control plane up so that it can run closer to the computes it manages, at which point we expect the `us-east-2` speeds to be the same everywhere else. + +Related to this effort, autoscaling–currently in beta–will pair really nicely with the new model: rather than needing to maintain a bunch of pools for every compute size, we can maintain one pool of scalable computes, and then when a request for a new compute comes in, we pass the configuration _and_ the size of the compute, and the compute _becomes_ the right size. We can even make sure the compute starts lightly above a minimum, to ensure a speedy Postgres start up, before reducing the allocation (if necessary.) This is actually really close and should start to land in the next couple of weeks. + +Not only is this (super!) cool, but it makes the problem of “how big should the pool be” much simpler. There’s also a bunch of smarts that we could implement to proactively change the size of the pools to more accurately and responsively address demand, but maybe autoscaling will afford us another opportunity to apply the good old “solve a hard problem (forecasting) by applying a different solution (dynamic resource allocation).” + +There are, as always, lots of other changes, too: making Postgres faster at starting up, streamlining some of our internal protocols to remove unnecessary round trips, optimizing the management process inside of the compute, and maybe providing a way for compute instances to require less intervention from the control plane. + +I’m skeptical that we’ll get to 50 milliseconds (but we’ll get close!), although I guessed wrong before. + +## Conclusions + +This has been a great project, and I hope everyone is able to take advantage of it and use it to build great and efficient things with Neon! + +Well, to be honest, I hope everyone quickly forgets that cold starts were ever a thing, and this post remains the only reminder that the amount of time it took for a sever to startup was ever a consideration: it is serverless, after all. diff --git a/content/blog/posts/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai.md b/content/blog/posts/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai.md new file mode 100644 index 0000000000..bcd548f152 --- /dev/null +++ b/content/blog/posts/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai.md @@ -0,0 +1,94 @@ +--- +title: Comigo.ai is using Neon to transform ADHD support with AI +description: >- + Thanks to Neon, Comigo can focus less on their database more on building their + app +excerpt: >- + “With Neon, we can start small and scale up. We don’t have to think about some + level of operational stuff. That’s awesome.” Paul Dlug, CTO of Comigo.ai + Comigo.ai is a digital companion and coach for individuals with ADHD. The app + leverages LLMs to offer support traditionally give... +date: '2024-08-02T17:02:50' +updatedOn: '2024-08-02T17:02:53' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai/cover.jpg + alt: null +isFeatured: false +seo: + title: Comigo.ai is using Neon to transform ADHD support with AI - Neon + description: >- + Neon allows startups like Comigo.ai to build and scale quickly while + dedicating minimal time and resources to managing Postgres. + keywords: [] + noindex: false + ogTitle: Comigo.ai is using Neon to transform ADHD support with AI - Neon + ogDescription: >- + Neon allows startups like Comigo.ai to build and scale quickly while + dedicating minimal time and resources to managing Postgres. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai/neon-comigoai-1024x576-ce4d9bb7.jpg) + +
+

“With Neon, we can start small and scale up. We don’t have to think about some level of operational stuff. That’s awesome.” 

+Paul Dlug, CTO of Comigo.ai +
+ +[Comigo.ai](https://www.comigo.ai/) is a digital companion and coach for individuals with ADHD. The app leverages LLMs to offer support traditionally given by a body double—someone who helps individuals with ADHD stay focused and productive. It addresses the unique needs of people with ADHD, which are personalized productivity and therapeutic support, reducing the cognitive load to help people get into the right mind space to be productive, something absent in other apps aimed at those with ADHD. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/comigo-ai-is-using-neon-to-transform-adhd-support-with-ai/ad4nxcnfrhjabspnplh9utxs-cxe5dclf-uled9uzb3lta-pvtu1h6wu1yppy2ogd4hwbu9acc09z37ojrunqponaqozr4gqrydfhi5p0t7cuz2tcwkqm1zvtag9m54roikkgs7npezk5casqibizrpmbgta-eeaa4e81.png) + +## The next generation of ADHD support + +
+

“We’ve built Comigo to be an effective companion for people with ADHD, such as myself. Neon helps us focus a bit less on tech and more on directly achieving product market fit.”

+Jason Curry, CEO at Comigo.ai +
+ +The Comigo app combines multiple proven techniques within a single platform, ensuring 24/7 accessibility for everyone: + +- **AI-powered body doubling**. Comigo acts as a virtual [body double](https://add.org/the-body-double/), a practice where an individual with ADHD works alongside another person to stay focused and productive. You now have a virtual companion giving you real-time support and reminders to stay on task. +- **Therapy on-the-go.** The app incorporates empirically supported therapeutic exercises, including Cognitive Behavioral Therapy (CBT), Dialectical Behavioral Therapy (DBT), and Acceptance and Commitment Therapy (ACT). The AI guides users through these exercises, helping them manage their thoughts, feelings, and behaviors effectively—offering a form of digital therapy that mimics the support traditionally provided by a human therapist. +- **Personalized task management**. Comigo also acts as an executive function coach, helping users with day-to-day organization, planning, and self-regulation. The app includes a tool that assists users in organizing and prioritizing their tasks, breaking down large tasks into manageable chunks and providing time estimates based on the user’s activity patterns. + +## The overhead of building on Cloud SQL for PostgreSQL + +
+

“In GCP, we had to constantly think about provisioning new instances and migrating data, which added operational overhead.”

+Paul Dlug, CTO at Comigo.ai +
+ +When they first built Comigo, the team initially managed their database using Cloud SQL for PostgreSQL. But managing this setup involved many manual processes that were time-consuming and prone to errors. The prospect of scaling their infrastructure to meet increasing demands without overloading the team was becoming a major concern. + +## Offloading the DB management to Neon + +When Comigo tried [Neon](https://neon.tech/), they realized they’d found what they were looking for. Neon is a serverless Postgres platform built to enable developers to ship faster without babysitting their database. + +Due to its unique architecture, Neon is much simpler to maintain than other Postgres solutions. Instead of provisioning CPU / memory / storage upfront, [Neon autoscales resources automatically in response to load.](https://neon.tech/docs/introduction/autoscaling) It’s also developer-friendly in nature, with an lo API-first feel and [integrations](https://neon.tech/docs/guides/integrations) with all the popular frameworks, tools, and ORMs. Even [small teams can manage thousands of Postgres databases](https://neon.tech/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases) on Neon. + +## The magic of database branching + +
+

“Neon branching is a big win for us. We can create full data copies at zero cost. For example, we can script a fresh branch every night for our staging server or for each deploy to run integration tests, all without additional costs.”

+Paul Dlug, CTO at Comigo.ai +
+ +But one of the most unique advantages of Neon is its [branching](https://neon.tech/docs/introduction/branching) capability. Neon allows you to “branch” your databases in order to create instant copies of data and schema via copy-and-write which feel similar to how Git branches code. + +This enables teams like Comigo to adopt [database branching workflows](https://neon.tech/flow) for their deployment and testing pipelines. Comigo uses branching for: + +- **Efficiently managing development environments.** Branching allows the creation of full database copies at zero cost. For example, a script can be written to create a fresh branch every night for the staging server, ensuring the development site always has an up-to-date copy of the data. +- **Improving CI/CD processes.** Each deploy can have a fresh copy of the data, enabling integration tests to run on current data without additional costs. This means CI/CD pipelines can spin up a fresh copy of the production data, run data checks, execute scripts, and perform necessary actions without incurring extra costs, paying only for the data used. +- **Reproducing bugs easily.** Bugs can be isolated and reproduced safely without impacting the production environment. Branches off the production database can be created, allowing developers to write data and troubleshoot issues efficiently. This eliminates the need to refresh the staging database, saving significant time and effort. + +## Get started with Neon (it’s free!) + +If Comigo’s story has sparked your curiosity, [create a Neon account](https://console.neon.tech/signup). You can use the Free Plan to get a sneak peek into the platform without even adding your credit card. And to learn more about Comigo.ai, check out their website—[you can also try it for free.](https://app.comigo.ai/) diff --git a/content/blog/posts/comparing-local-first-frameworks-and-approaches.md b/content/blog/posts/comparing-local-first-frameworks-and-approaches.md new file mode 100644 index 0000000000..b3af19f392 --- /dev/null +++ b/content/blog/posts/comparing-local-first-frameworks-and-approaches.md @@ -0,0 +1,191 @@ +--- +title: Comparing local-first frameworks and approaches +description: Any sufficiently advanced technology is indistinguishable from magic. +excerpt: >- + Cloud applications can feel like magic. You write text in a Google Doc on your + laptop, and it magically appears in the doc on your phone. Or you’re working + on some code in an online IDE like Replit, and your entire team can build + together. But, like magic, we all know a lot is go... +date: '2024-09-19T16:36:45' +updatedOn: '2024-09-29T01:29:21' +category: community +categories: + - community + - workflows +authors: + - rishi-raj-jain +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/comparing-local-first-frameworks-and-approaches/cover.jpg + alt: null +isFeatured: false +seo: + title: Comparing local-first frameworks and approaches - Neon + description: >- + Local-first frameworks are making a comeback. We explore some of them, + together with an alternative to build locally - Neon branches via CLI. + keywords: [] + noindex: false + ogTitle: Comparing local-first frameworks and approaches - Neon + ogDescription: >- + Local-first frameworks are making a comeback. We explore some of them, + together with an alternative to build locally - Neon branches via CLI. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/comparing-local-first-frameworks-and-approaches/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/comparing-local-first-frameworks-and-approaches/neon-local-1-1024x576-b254e183.jpg) + +Cloud applications can feel like magic. You write text in a Google Doc on your laptop, and it magically appears in the doc on your phone. Or you’re working on some code in an online IDE like Replit, and your entire team can build together. + +But, like magic, we all know a lot is going on under the surface that we’re not supposed to see. And, like magic, when it goes wrong, it goes horribly wrong. Such as when you’re writing a Google Doc, the network fails, and the doc hangs. You can’t edit it because the document can’t be updated from the cloud. Suddenly, you wish for Windows 95 and the luxury of local. + +Luckily, local isn’t a disappearing technology. In fact, local is making a renaissance with local-first frameworks that put your data in your hands first. If the cloud is there, great. If not, ¯\\_(ツ)_/¯. This is such a nascent field that several local-first frameworks are being built to approach this problem from different sides. Here, we want to highlight a few to give you a feel for what to look for in local-first applications. + +## Merging Data with CRDTs + +In 2019, University of Cambridge computer scientist [Martin Kleppmann](https://martin.kleppmann.com/) (of [Designing Data-Intensive Applications](https://www.oreilly.com/library/view/designing-data-intensive-applications/9781491903063/) fame) and authors from digital research lab [Ink & Switch](https://www.inkandswitch.com/) published a manifesto titled, “[Local-first software: you own your data, in spite of the cloud](https://www.inkandswitch.com/local-first/).” + +The manifesto laid out their thinking on how the cloud-first data storage model had gone awry. First, you get the issue mentioned above–cloud apps become unusable offline, limiting usability. But the problems go beyond that. When data is stored primarily in the cloud, users become “borrowers” of their own data. The service provider has ultimate control over access and preservation. It can be difficult to migrate data between cloud services, and if the cloud service shuts down, users lose access to _their_ data and work. + +The manifesto proposed seven ideals for local-first software: + +1. **No spinners**: Fast performance by working with local data +2. **Multi-device**: Seamless sync across all user devices +3. **Offline**: Full read/write functionality without internet access +4. **Collaboration**: Real-time collaborative editing for multiple users +5. **Longevity**: Access and edit data for decades, outliving servers +6. **Privacy**: Encrypted data that’s not accessible to service providers +7. **User control**: Ownership of data, including the ability to copy, modify, and delete + +They proposed a number of technical concepts to achieve these aims, with the critical one being the Conflict-Free Replicated Data Type, or CRDT. CRDTs are data structures that allow multiple replicas of a dataset to be updated independently and concurrently without coordination between the replicas while ensuring that all replicas eventually converge to the same state. This makes them particularly well-suited for local-first software architectures. The key properties of CRDTs include: + +- Strong eventual consistency. All replicas will converge to the same state given the same set of updates, regardless of order received +- Commutativity. The order of applying updates does not affect the final state +- Associativity. Updates can be grouped and applied in any order + +Say you have a collaborative text document that two people are working on. Each can work offline, and CRDTs will take care of merging. They assign unique identifiers to each character or operation, allowing the system to track and merge changes intelligently. For example, if one user inserts text at the beginning of the document while another adds content at the end, the CRDT algorithm can seamlessly integrate both changes without conflict. + +Ink & Switch released [automerge](https://automerge.org/) to _auto_ matically achieve this _merge_. If you have two documents you are [collaboratively editing](https://automerge.org/docs/documents/text/), you can use automerge to make concurrent changes. + +```javascript +import { next as Automerge } from "@automerge/automerge" + +let doc = Automerge.from({text: "hello world"}) + +// Fork the doc and make a change +let forked = Automerge.clone(doc) +forked = Automerge.change(forked, d => { + // Insert ' wonderful' at index 5, don't delete anything + Automerge.splice(d, ["text"], 5, 0, " wonderful") +}) + +// Make a concurrent change on the original document +doc = Automerge.change(doc, d => { + // Insert at the start, delete 5 characters (the "hello") + Automerge.splice(d, ["text"], 0, 5, "Greetings") +}) + +// Merge the changes +doc = Automerge.merge(doc, forked) + +console.log(doc.text) // "Greetings wonderful world" +``` + +This approach enables true local-first collaboration, where users can work offline and sync their changes when they reconnect, without the need for a central server to arbitrate conflicts. + +## Syncing Data with PouchDB + +“The Database that Syncs!” shouts the [PouchDB](https://pouchdb.com/) homepage. PouchDB is another new local-first/sync database. PouchDB is a JavaScript database that runs in the browser, allowing developers to create applications that work offline and sync with server-side databases when online. It’s designed to be compatible with (and is inspired by) Apache’s NoSQL [CouchDB](https://docs.couchdb.org/en/stable/index.html). + +The core principles of PouchDB align well with the local-first manifesto: + +- **Offline-first**: Works entirely offline, syncing when a connection is available +- **Multi-device sync**: Seamlessly syncs data across devices +- **Open source**: Allows for community contributions and audits +- **Cross-platform**: Runs in browsers, Node.js, and mobile devices + +PouchDB uses a document-based data model, where each document is a JSON object. Here’s a simple example of creating and querying a PouchDB database: + +```javascript +// Create a database +var db = new PouchDB('my_database'); + +// Add a document +db.put({ + _id: 'mittens', + name: 'Mittens', + species: 'cat', + age: 3 +}).then(function () { + // Find all documents where species is 'cat' + return db.find({ + selector: {species: 'cat'} + }); +}).then(function (result) { + console.log(result.docs); +}).catch(function (err) { + console.log(err); +}); +``` + +One of PouchDB’s key features is its sync protocol. When online, PouchDB can sync with a CouchDB server, allowing for real-time collaboration: + +```javascript +var localDB = new PouchDB('my_database'); +var remoteDB = new PouchDB('https://example.com/my_database'); + +localDB.sync(remoteDB, { + live: true, + retry: true +}).on('change', function (change) { + console.log('Data changed:', change); +}).on('error', function (err) { + console.log('Sync error:', err); +}); +``` + +PouchDB doesn’t use CRDTs for conflict resolution. Instead, it uses a [deterministic algorithm](https://pouchdb.com/guides/conflicts.html#:~:text=By%20default%2C%20CouchDB%20will%20choose,time%20to%20resolve%20the%20conflict.) based on revision history. When conflicts occur, PouchDB chooses a winning revision and stores the others as conflicts, which can be accessed and resolved manually. If you want to nerd out, here’s a good read on [how CouchDB provides eventual consistency](https://docs.couchdb.org/en/stable/intro/consistency.html). + +While PouchDB provides a robust solution for offline-first apps, it’s not without challenges. Large datasets can impact performance. And like other local-first solutions, developers need to carefully consider data modeling and conflict resolution strategies. + +PouchDB shows us that local-first isn’t just about new, bleeding-edge tech–it’s about reimagining existing databases for a world where offline is the norm, not the exception. + +## Sync from Postgres with ElectricSQL + +[ElectricSQL](https://electric-sql.com/) is especially exciting to us at Neon because it uses Postgres as the underlying data store. It is one level broader than a local-first solution: The core component of ElectricSQL is a service called a Sync Engine that runs on your servers between local devices and your Postgres database. So you can use ElectricSQL as a key component of a local-first architecture but it doesn’t solve all your needs. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/comparing-local-first-frameworks-and-approaches/use-cases-1024x473-1eb14d99.png) + +The ElectricSQL service listens for changes in your database using [Postgres logical replication](https://neon.tech/docs/guides/logical-replication-concepts) and efficiently syncs little subsets of data into connected local apps. + +The sync engine is only the server-side component though, you still need a local component to consume the data. [ElectricSQL has a Typescript client](https://www.npmjs.com/package/@electric-sql/client) that gives you basic functionality for consuming data from the sync engine: + +```javascript +import { ShapeStream, Shape } from '@electric-sql/client' + +const stream = new ShapeStream({ + url: `${BASE_URL}/v1/shape/foo`, +}) +const shape = new Shape(stream) + +// Returns promise that resolves with the latest shape data once it's fully loaded +await shape.value + +// passes subscribers shape data when the shape updates +shape.subscribe(shapeData => { + // shapeData is a Map of the latest value of each row in a shape. +} +``` + + +If you're using ElectricSQL with Neon, make sure to enable logical replication on your database via the Neon UI to make it work. + + +More examples of local-first with ElectricSQL can be found in the examples directory of their repo. [https://github.com/electric-sql/electric/tree/main/examples](https://github.com/electric-sql/electric/tree/main/examples) + +## Local By Default + +If you’ve ever wondered how Linear, Figma, or Excalidraw work, this is it. Local-first approaches are changing how we think about data ownership and application architecture. They’re not just solving the offline problem; they’re making us rethink the entire user-data relationship. + +This is hardly an exhaustive list. While most local-first frameworks are roughly following one of the above approaches, each has a fun implementation that might make it the best option for you. To take your research further, here’s [an evaluation paradigm for assessing local-first solutions](https://jaredforsyth.com/posts/in-search-of-a-local-first-database/), and here’s [a list of local-first tools](https://electric-sql.com/docs/reference/alternatives) that you can explore. By putting data back in users’ hands, these frameworks are paving the way for more resilient, privacy-respecting, and user-centric applications. diff --git a/content/blog/posts/control-planes-for-database-per-user-in-neon.md b/content/blog/posts/control-planes-for-database-per-user-in-neon.md new file mode 100644 index 0000000000..738ab3d1be --- /dev/null +++ b/content/blog/posts/control-planes-for-database-per-user-in-neon.md @@ -0,0 +1,152 @@ +--- +title: Control Planes for Database-Per-User in Neon +description: Designing a control plane to scale your multi-tenant app +excerpt: >- + Due to its serverless architecture, Neon is a great option for building + multi-tenant, database-per-user applications in Postgres. In a previous post, + we explored the various approaches to multi-tenancy in Postgres, with a + particular focus on the database-per-user architecture and... +date: '2024-09-11T08:32:29' +updatedOn: '2024-11-15T17:33:20' +category: workflows +categories: + - workflows +authors: + - dian-m-fay +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/control-planes-for-database-per-user-in-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: Control Planes for Database-Per-User in Neon - Neon + description: >- + This post dives into a key component of scaling and maintaining + database-per-user systems: the control plane. + keywords: [] + noindex: false + ogTitle: Control Planes for Database-Per-User in Neon - Neon + ogDescription: >- + This post dives into a key component of scaling and maintaining + database-per-user systems: the control plane. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/control-planes-for-database-per-user-in-neon/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/control-planes-for-database-per-user-in-neon/neon-control-planes-1-1024x576-cc6a7102.jpg) + +Due to its serverless architecture, [Neon](https://neon.tech/) is a great option for building multi-tenant, database-per-user applications in Postgres. [In a previous post](https://neon.tech/blog/multi-tenancy-and-database-per-user-design-in-postgres), we explored the various approaches to multi-tenancy in Postgres, with a particular focus on the database-per-user architecture and its advantages for isolating customer data. + +In this second post, we dive into a crucial component of managing database-per-user systems: the control plane. Understanding and implementing an effective control plane is key to scaling your application while maintaining operational efficiency and security. + +## The Neon object hierarchy + +Before we get into the specifics of control planes, let’s review the core concepts that make up the Neon ecosystem so that you can follow through the logic of the sections below. + +[Neon](https://neon.tech) is a Postgres service with a [custom-built storage engine](https://neon.tech/blog/get-page-at-lsn) that separates compute from persistence. When you build on Neon, you organize your backend in a [hierarchy](https://neon.tech/docs/manage/overview) of three primary elements: **projects**, **branches**, and **databases**. + +- A **project** in Neon acts as a container for all your database resources. Each project can host multiple branches and databases, making projects the top-level unit within your Neon organization. +- **Branches** in Neon are similar to branches in version control systems like Git. They allow you to create copies of your database environment at specific points in time and make changes in an isolated sandbox. This feature is particularly useful for testing, development, or experimenting with new features without affecting the main branch. +- Within each branch, you can have one or more Postgres **databases**. These databases store the actual data and are isolated from each other, even within the same branch. + +When talking about database-per-user architectures in Neon, we’re usually speaking of a [project-per-user](https://neon.tech/blog/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers) design. Projects offer the highest level of isolation, and features like branching and [PITR](https://neon.tech/docs/guides/branch-restore) are difficult if not impossible to use safely without each user’s data in its own project. + +If you’re creating one Neon project per user, the number of Neon projects will quickly grow. Managing them individually becomes increasingly complex and time-consuming — so a centralized approach will be necessary to stay on top of operations. This is where the **control plane** comes into play. + +## What does a control plane look like? + +The term “control plane” originated in network administration, describing routing rules over a “data plane” of packet traffic. More recently, it’s been generalized to mean systems that facilitate other systems’ management. + +The [Neon console](https://console.neon.tech/) and tools like [neonctl](https://neon.tech/docs/reference/neon-cli) are themselves part of a control plane. Diving into those yourself is an important first step in learning how Neon works and getting your systems off the ground, but building a more centralized solution will be a critical part of scaling your database-per-user architecture. + +In this article, we’re going to discuss the control plane as a basically singular software system—but it’s a much fuzzier concept than might immediately be evident. The control plane extends not just to tools you build to manage your application, but to other systems entirely, from CI/CD to observability platforms to schema migration frameworks. Nor does this meta-system stop at software: your first steps into the control plane will tend to look more like standard operating procedures and runbooks. + +The control plane also grows with your system and your organization. You’ll likely start by supplementing the general-purpose capabilities of the Neon console and `neonctl` with individual specialized tools — shell scripts, spreadsheets, little glue programs, admin modes or modules. These automations help ensure that routine tasks don’t become major timesinks. But in the long run, it’s hard to standardize these many discrete moving parts, which is solved by building a consolidated, consistent management platform. + +**Disclaimer: You probably shouldn’t invest a lot of time into organizing control plane software before you have customers.** Exceptions exist, but it’s important to engineer for the problems you’re actually facing. In most cases, runbooks and elbow grease will get your first deployments out as effectively. Evolving past that tends to be part of the [chasm-crossing effort](https://en.wikipedia.org/wiki/Crossing_the_Chasm), in Geoffrey Moore’s classic formulation: automation and standardization are how you scale your distribution channel to be able to serve more and more customers. + +## The catalog database + +At its core, the **catalog database** is a special database that serves as a centralized hub for tracking and managing all your Neon projects and databases. This should be its own self-contained Neon project, ensuring it is isolated from user data and user-facing systems. + +The [Neon console](https://console.neon.tech/) provides a standard set of metadata for each project—details like project age, active branches, and compute sizes. But this built-in functionality may not be sufficient for more specific operational needs, such as tracking the current deployed schema version or monitoring customer billing status. Centralizing this critical but more specialized information in the catalog database is the first step toward effectively managing your projects at scale. + +The first and most important component of the catalog database is a **manifest of users** (or accounts, or however you name them in your system). This manifest is essentially a list of all your customers or organizational units for whom you provision Neon projects. From this starting point, you can build out the catalog database to include your other metadata: + +
+Image +
A catalog db schema example, including simple payment tracking and allowing multiple Neon projects for each user. Each project tracks its currently deployed schema version, among other quantities.
+
+ +### Advice for designing your catalog database schema + +- Linking tables like `project` and `payment` to the `account` table using foreign keys helps maintain data integrity and ensures that each project or payment is correctly associated with an account. +- Use data types like `citext` for case-insensitive text fields, `uuid` for unique identifiers to avoid surfacing information about your sequences, and `timestamptz` [for anything to do with tracking concrete, real-world time](https://wiki.postgresql.org/wiki/Don't_Do_This#Don.27t_use_timestamp_.28without_time_zone.29). +- Capture critical operational data, such as `schema_version`, in the `project` table. **This information will be essential for managing updates and ensuring consistency across your Neon projects.** +- Don’t forget about indexing! While the catalog is and will likely remain much smaller than all but the emptiest per-user database it manages, it will grow, especially if you’re tracking repeated events like payments. You’ll be spending a fair amount of time in the control plane – it’s important that it continues to perform well as you scale. +- While it’s tempting to include every possible piece of data, start with the essentials and plan for extensions as your needs evolve. +- Standard Neon metadata like compute size or branch information are already available in the console. You most likely don’t need to replicate this information in the catalog database as well, unless having to look it up separately is becoming a significant time sink or adding a lot of complexity to management workflows. + +## Entering the control plane + +The control plane has many uses, but among the earliest and most important is **getting new customers onboarded to your system**. In a database-per-user setting, this means standing up a new Neon project at minimum, but it can mean many other things as well: + +- Billing and invoicing +- Updating the catalog database with the new Neon project information +- Storing some initial information in _their_ database: the organization name, the registering email address so the first user can provision others, and so on +- In isolated environments, provisioning a new application instance +- Notifying customers that the activation process is complete + +When you’re just starting your automation journey, the control plane probably won’t do all of this, or even necessarily most of it. If your target clients are fewer and bigger, onboarding could even be rare enough that you might find it a better use of effort to concentrate on other control plane functions entirely. + +But if you find you’re spending a lot of time and effort on onboarding — or anything else — and begin taking steps to automate it, that’s a control plane. It might be some horror story of an undocumented, uncontrolled shell script on a founding engineer’s laptop that reads a CSV emailed manually when someone signs up, but there are good control planes and bad control planes. + +_Good control planes_ are systematically thought out, provide clear and unambiguous information and capabilities, and are accessible to anyone who needs them (and only to those people). _Bad control planes_ grow organically as emergencies come up but are never reorganized with a holistic sense for what both the organization and its engineers need. They are unclear, unreliable, present information partially or inconsistently. Often, only one person understands them or is allowed to operate them, although this isn’t an infallible sign at smaller organizations. + +## Control planes, control towers + +Monitoring and observability is a big space in the 2020s. When developing on Neon, basic metrics will split between the Neon console and whatever observability platform you deploy or integrate into your application. Both these tools are part of your control plane in that fuzzier meta-system sense. So why, and for what, should you consider adding monitoring capabilities to your control plane in the specific sense? What third type of operational information can’t the other two options surface easily? + +**There’s a whole class of useful data which neither the console nor traditional monitoring or observability software makes visible: the sorts of things someone asks, “can we run a query to see **\_\_\_\_\_**?”.** There’s any number of ways to fill in that blank. If you sell features, support tiers, or anything else à la carte; if counts, frequencies, and other point-in-time aggregates need to be reported; if you have all-Postgres background processes that can’t easily push metrics to an integrated provider: **the dedicated control plane is the obvious place to make this kind of information visible.** + +
+Image +
A mature control plane offers visibility into system status and usage, along with mechanisms for deploying new Neon projects and managing or modifying existing projects.
+
+ +**Control plane data are most easily queried from the catalog database.** It’s absolutely possible to look up information in each per-user database — that’s how a shared application environment approach works in the first place — but it’s a lot simpler to make one query for all projects than it is to make one query _per_ project. A central data repository also makes it possible to connect a dashboarding or business intelligence system to your control plane. + +Of course some data, such as in-database process performance metrics, are resolutely local. And in fully isolated application environments, you’ll need to have information like feature enablement stored in-project so the application instance can use it. However, ease of surfacing information from the catalog database compared with local dbs is only one factor exerting pressure toward centralized data architecture. We’ll get into some others when we discuss the shared and isolated instance strategies. + +## Life on the control plane + +Day-to-day control plane operations, such as maintenance, troubleshooting, and recovery, tend to happen to databases individually and at different times or cadences. This means the console and `neonctl` are likely able to fulfill many of your needs for longer even as you scale, compared to something like the immediate and specialized requirements of onboarding. + +However, if you make extensive or routine use of some of Neon’s more powerful features, turning common tasks into push-button operations can make them substantially faster and safer. [Managing logical replication](https://neon.tech/docs/guides/logical-replication-guide), for example, involves several steps in both the source and destination, and getting an argument wrong or missing a setting can be difficult and frustrating to debug. + +And there’s one big part of maintenance that it’s crucial to centralize sooner rather than later. Software changes; so do database schemata. At some point, you’re going to have to roll out an upgrade. **Schema upgrades, or migrations**, are never trivial even for a single database. When each user has their own Neon project, it becomes not just a problem of correctness but of synchronization, especially in a shared application environment. We’ll discuss the implications and tactics for managing migrations in future posts, but the control plane has an important general role to play here. + +**One of the most important things the control plane can tell you, full stop, is the deployed schema version for each and every per-user database.** It’s critical to use a migration framework such as [sqitch](https://sqitch.org/), [graphile-migrate](https://github.com/graphile/migrate), [Flyway](https://www.red-gate.com/products/flyway/community/), or whatever built-in migration support is supplied by your web framework or data access layer of choice; every one of these worth its salt has an in-database version registry your control plane can query. Many also support a “baseline” operation that converts an existing schema to an initial migration script. + +
+Image +
The 17.0 migration is tested in a branched database from each project, and something in diagonal-feldspar is causing a failure. The fact that all projects are versioned in lockstep suggests that this is likely a shared application environment. The problem will need to be found and fixed before it’s safe to roll out 17.0; if the resolution requires a change to the migration script, and an older version of that script has been applied to the older projects, the schema divergence could be very difficult to track down and reconcile.
+
+ +When you know which schema version is deployed to a database, the next logical step is to do something with that information. Triggering and monitoring the progress of upgrade rollouts is as if not more important than onboarding: you usually onboard each customer a maximum of one time, but upgrades happen forever after. + +## Key takeaways: Designing an effective control plane + +
As we’ve explored, building and managing a database-per-user architecture requires a strategic approach to control plane design. The details depend on your organization’s data and deployment needs, on your choice to deploy a single shared or many isolated environments, and on the engineering effort you’re able to invest into getting ahead of scale. No two implementations are exactly alike. However, there are still a few general governing principles: + +- **Start simple, scale thoughtfully**. The Neon console and tools like `neonctl` will help you get off the ground, but as your system grows and the number of your Neon projects increases, you’ll need more. Prioritize automation for tasks that consume the most time and effort, like onboarding or schema migrations. +- **The control plane is an internal product**. Treat it like one: understand who needs it, who uses it, who _should_ use it and isn’t, and the why and how of each. Cultivate its growth. Take a conscious approach to design and avoid growing the control plane reactively in a “fire-fighting” mode. +- **Centralize your metadata**. By centralizing vital metadata—like user manifests, billing, and per-project operational data—you can streamline your own operations and reduce the complexity of managing multiple projects. +- **Pick your battles wisely**. Spending a lot of time on your control plane before you have projects to manage is almost always a bad bet. Be sure you have a problem before you start solving it, and gradually build a more centralized and consistent management platform as your needs evolve. +- **Use the catalog database for insights**. When managing multiple projects, it’s more efficient to query a central catalog database rather than accessing each per-user database individually. This centralization allows you to monitor, maintain, and upgrade your entire system more effectively, supporting your ability to scale. + +## So: What does _your_ control plane look like? + +The one thing all control planes have in common is that they integrate and standardize operations on the data plane. They render it legible, in the sense [James C. Scott](https://yalebooks.yale.edu/9780300078152/seeing-like-a-state) uses the term to imply “a viewer whose place is central and whose vision is synoptic”, that is, comprehensive and summarizing. + +Individual views of and individual actions on individual databases under management are difficult to trace, to reason about, and to perform consistently and safely, especially as your customer base grows. More than an application, the control plane is a strategy for centralizing governance of your data resources and minimizing the risks of technical operations, by channeling actions through a consistent set of abstractions and procedures. diff --git a/content/blog/posts/create-read-replicas-in-the-free-plan.md b/content/blog/posts/create-read-replicas-in-the-free-plan.md new file mode 100644 index 0000000000..2e3a0b0021 --- /dev/null +++ b/content/blog/posts/create-read-replicas-in-the-free-plan.md @@ -0,0 +1,91 @@ +--- +title: Create Read Replicas in The Free Plan +description: Our free tier keeps expanding. +excerpt: >- + We keep bringing you new things, following up on the 10/10 spirit. Today’s + news: read replicas are now available in the Free Plan. This is particularly + great if you’ve been curious to test out how awesome they are. Why are we + doing this? Read Replicas are just another way to use... +date: '2024-10-23T16:02:29' +updatedOn: '2024-10-24T15:56:12' +category: company +categories: + - company +authors: + - brad-van-vugt +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/cover.jpg + alt: null +isFeatured: false +seo: + title: Create Read Replicas in The Free Plan - Neon + description: >- + You can now create read replicas in the Neon Free plan. Use them as + read-only access points for your branches. + keywords: [] + noindex: false + ogTitle: Create Read Replicas in The Free Plan - Neon + ogDescription: >- + You can now create read replicas in the Neon Free plan. Use them as + read-only access points for your branches. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/neon-read-replicas-1-2-1024x576-f7b3259c.jpg) + +We keep bringing you new things, following up on the [10/10 spirit](https://neon.tech/blog/10x-projects-on-free-plan). Today’s news: **read replicas are now available in the Free Plan.** This is particularly great if you’ve been curious to test out how awesome they are. + +Why are we doing this? Read Replicas are just another way to use your [compute hours](https://neon.tech/docs/introduction/usage-metrics#compute). Our Free Plan includes 190 CU-hours, enough to run the smallest compute 24/7 if that’s what you prefer. But if you don’t need your database running all the time, you can invest your compute hours in more projects ([you now get ten](https://neon.tech/blog/10x-projects-on-free-plan)), higher capacity (via [autoscaling](https://neon.tech/docs/introduction/usage-metrics#compute)) or, now, in read replicas. + +## Neon read replicas crash course + +Read replicas may not sound like _the most exciting feature_: they’ve been around for a while as a table-stakes feature of managed databases, acting as tools for offloading reporting and analytics tasks from the primary database. Traditional read replicas work by asynchronously copying data from the primary database to a secondary instance using log-based replication mechanisms: changes are recorded in a Write-Ahead Log (WAL) and replayed on the replica to keep it in sync with the primary. [We discussed the state of the art of replicas in a previous blog post.](https://neon.tech/blog/the-problem-with-postgres-replicas) + +But Neon takes a fundamentally different approach to read replicas, taking advantage of its unique serverless architecture. [Neon separates storage and compute while implementing a custom-built storage layer that allows multiple compute endpoints to be attached to the same storage unit.](https://neon.tech/blog/architecture-decisions-in-neon) As a result, Neon’s read replicas don’t need to copy or duplicate data—both the primary compute and the read replicas access the same data source directly from Neon’s storage layer, which persists data across all replicas. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/ad4nxcglqrezpea8dy5jpbgnqcllh9wkvbdqur28vqd-djdie6pkpgvfcob7zhtdfemc1bcsvzgdwmjmnskhmuwzf6mnkn4dudhsrswyjcgevizkgwcjf89g3guxkfmyqd8kq3msa8xyoisu7cchjbn2g6i-95c36d1f.png) + +This architectural difference brings three key advantages: + +- **Lighter and cost-efficient**. In traditional setups, read replicas need additional storage to maintain a copy of the primary database, so they quickly get heavy and expensive. In Neon, all replicas read from a shared storage backend. You can create hundreds of them, and you only pay for storage once. +- **Worry-free management**. Neon’s read replicas scale to zero when idle. This not only saves you even more costs but also eases _maintenance pressure_. You don’t have to worry about your read replicas once they’re created. They’re virtually free if nobody’s using them. +- **Fast to deploy**. Since no data needs to _actually_ be _replicated_, read replicas in Neon are created and deleted almost instantly. If someone needs to query the database, they can immediately get their isolated connection string. + +## Two use cases + +### Safe, low-cost read access for teams + +The most popular use case for read replicas in Neon is granting read access to your data without compromising your primary database. For example, there might be team members who need to run queries for reporting or analysis; thanks to the lightweight nature of Neon’s read replicas, you can create dedicated replicas for each person, each with their own access URL. All while ensuring that: + +- **Your main compute stays safe**. You don’t have to worry about someone running an inefficient or heavy SQL query that could jeopardize the performance of your database or, worse, break something critical. +- **Costs stay low by default**. You can create many replicas without worrying about the cost. If nobody is actively querying a replica, it automatically suspends itself. + +### Horizontal scaling + +Another popular use case for Neon read replicas is helping teams scale compute horizontally. If you’re running a write-heavy workload in production, you could use read replicas to offload read queries, relieving pressure on the primary compute and ensuring that your application continues running with optimal performance. + +But you can even do this if your primary workload isn’t that heavy. Instead of scaling your primary database by [increasing the compute autoscaling limit](https://neon.tech/docs/guides/autoscaling-guide#configure-autoscaling-defaults-for-your-project), you could create multiple read replicas and distribute your queries across them, keeping all your computes small. + +## Create your first read replica + +In your [Neon Free account](https://console.neon.tech/signup), navigate to the branch you’d like to add a read replica to and select `Add Read Replica`: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/ad4nxfgmm-yf0qy-ktmc9l0pb5ikgfah5wbt6avmpgbwtxzegxiykwj0mmintozikdopzxqkz4btd53eddcv9wsagmhvvljybwicbbe4briel3gqwubnpauw9fv4dq77f6kjnvfduhhvljogsf-arejktb7nw-f9d6b4e0.png) + +Next, configure your read replica as you would for any other compute. Remember, a read replica in Neon is simply an additional compute endpoint attached to your branch with read-only access to your database. + +For example, in the screenshot below, the read replica is configured with a fixed 0.25 CPU and 2 GB RAM capacity, but you can also enable autoscaling with a higher compute max. Make sure to keep autosuspend enabled so your read replicas can scale to zero when not in use. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/ad4nxfyaixmv0nyd83x4xok3ctufnyllgf3jqoiw2r5oa4ir4vyocfo4e1kr-xryhbt0khroxfwdk9rpsjggavqp6h0qyiithjuugfxxohgf-ryavkzthn8tkv3mwj-lcoam3uzwmsbtshechw8mfc3bmvzg-0ded8e30.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/ad4nxfd5d3mo9gighgghh3ilgpxkqwowci5irivskjlabdmd8mbr2ood3c9n1uyqptti66o5o4ennisqgspdnv9u8q6npd7dfqasizab901fhn8p-axscw9c1ty0tlpiozm5bexlzfhrhrexvskf9xpr0po-2a2d059a.png) + +Once you click `Create`, your read replica will appear in the list of computes attached to your branch. If you want to connect to your branch via the read replica (or allow someone else to), you’ll find a dedicated connection string under `Connect`: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/ad4nxdrf93edbz7qqxig0wacfsgetssnraojth3enn6pwf7cprsludxmrpemhij24esyluiisc2rgkfivblgwukt7npxkhcodzt-kyjeagynsdpcq8y-1ufgs6vo-cie-vrqfqtxhbnrqbpwcfbfey4qt7i-1028a941.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/create-read-replicas-in-the-free-plan/ad4nxfwszbahn-q5vsuhf7gqmui0nk0obtaemv0lmsfz1-1s0ao1uzhav01cj1nmrpz2g-idrhkc3ta4ymyuh6fy1buwtvxneeyt7v1kuotcpucjg5f8dmbwiy4cr9jw7eueg9cn7c8cbr86b90xccotyujo-20db5fcb.png) + +Go ahead and replicate! If you have any questions, feel free to ask us in [Discord](https://discord.gg/92vNTzKDGp). diff --git a/content/blog/posts/create-up-to-1000-neon-projects-without-extra-cost.md b/content/blog/posts/create-up-to-1000-neon-projects-without-extra-cost.md new file mode 100644 index 0000000000..4322bd5bb7 --- /dev/null +++ b/content/blog/posts/create-up-to-1000-neon-projects-without-extra-cost.md @@ -0,0 +1,89 @@ +--- +title: 'Create up to 1,000 Neon projects without extra cost' +description: Our partner plans now include more projects +excerpt: >- + At Neon, we are committed to listening to our customers and partners, and + constantly striving to improve our services based on your feedback. That’s why + we’re excited to announce a significant improvement to our partnership + program: partners can now create up to 1,000 projects without cost as part of + their partnership agreement. This increased project ... +date: '2024-05-10T16:03:40' +updatedOn: '2024-05-10T16:03:41' +category: company +categories: + - company +authors: + - will-adams +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/create-up-to-1000-neon-projects-without-extra-cost/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Create up to 1,000 Neon projects without extra cost - Neon' + description: >- + Become a Neon partner and expand your offering with a managed database, + without getting into the business of database hosting yourself. + keywords: [] + noindex: false + ogTitle: 'Create up to 1,000 Neon projects without extra cost - Neon' + ogDescription: >- + Become a Neon partner and expand your offering with a managed database, + without getting into the business of database hosting yourself. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/create-up-to-1000-neon-projects-without-extra-cost/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/create-up-to-1000-neon-projects-without-extra-cost/neon-1000-projects-1-1024x576-e2f9d4d1.jpg) + +At Neon, we are committed to listening to our customers and partners, and constantly striving to improve our services based on your feedback. That’s why we’re excited to announce a significant improvement to our [partnership program](https://neon.tech/partners): **partners can now create up to 1,000 projects without cost** as part of their partnership agreement. + +This increased project limit, combined with Neon’s [scale to zero](https://neon.tech/docs/introduction/auto-suspend) capabilities, enables partners to focus on growing their customer base instead of worrying about overhead costs. They can create a dedicated project for every new customer, giving each a managed database to play with right away. This accelerates time to value and enhances customer satisfaction; at the same time, these databases scale to zero, keeping costs as low as possible. + +## What is the Neon partnership program? + +The Neon partnership program includes [special pricing plans](https://neon.tech/partners) for those companies looking to expand their own platforms with a managed database, but don’t want to get into the business of database hosting themselves. By becoming partners, they can integrate Neon into their applications via OAuth or API-based integrations. + +The OAuth integration allows companies to integrate Neon into their platforms (e.g. [you can create a Neon project via Hasura](https://neon.tech/blog/neon-hasura-integration)), while the API integration enables partners to offer managed databases as a fully incorporated element of their products (e.g. [RetoolDB is powered by Neon](https://neon.tech/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases)). + +## What is a Neon project? + +In the context of the Neon platform, a [project](https://neon.tech/docs/manage/overview) represents the highest level of isolation for managing serverless Postgres databases, ideal for maintaining distinct environments for different customers. By allowing partners to create up to 1,000 projects without additional costs, Neon significantly enhances the ability for partners to scale their offerings. Each project can be dedicated to a specific customer, ensuring maximal isolation and security, and facilitating growth as partners can simply add more projects as their client base expands. + +## What are the benefits of becoming a partner? + +- Scale on demand without managing servers. Just focus on getting more users, Neon does the rest. +- Take advantage of Neon’s scale-to-zero to only pay for active database usage, significantly reducing overhead costs. +- Automate all database management tasks via integrations and APIs. Focus on your own application, not on managing databases. +- Provide a top-notch user experience with instant database provisioning. Don’t keep your users waiting. +- Implement secure connections and manage user permissions efficiently through Neon’s security guarantees. +- Gain access to Neon’s team of Postgres experts, ensuring reliable support for any technical challenges. +- Leverage joint marketing efforts with Neon to enhance your visibility and reach a broader audience. + +## Which companies are Neon partners? + +Companies like Retool, Vercel, Koyeb, Replit, Hasura, and many others are partners. Explore our [case studies](https://neon.tech/case-studies) to read more about how they’re using Neon. + +
+

“Neon’s serverless philosophy is aligned with our vision — no infrastructure to manage, no servers to provision, no database cluster to maintain — making them the obvious partner to power our serverless Postgres offering”

+CTO at Koyeb +
+ +
+

“We’ve been able to automate virtually all database management tasks via the Neon API. The scale-to-zero functionality of Neon allows us to offer dedicated databases to our customers without worrying about the cost of idle resources”

+Sr Engineer at Retool +
+ +
+

“The combination of flexible resource limits and nearly instant database provisioning made Neon a no-brainer to power our hosted Postgres offering”

+Infrastructure Engineer at Replit +
+ +
+

“By partnering with Neon, Vercel’s frontend platform is now the end-to-end serverless solution for building on the Web, from Next.js all the way to SQL”

+CEO at Vercel +
+ +## How can I apply? + +If you’re interested in hearing more about the partnership plans, fill out the form [here](https://neon.tech/partners) and we’ll get back to you shortly. diff --git a/content/blog/posts/ctrl-c-in-psql-gives-me-the-heebie-jeebies.md b/content/blog/posts/ctrl-c-in-psql-gives-me-the-heebie-jeebies.md new file mode 100644 index 0000000000..8cdf1a54a5 --- /dev/null +++ b/content/blog/posts/ctrl-c-in-psql-gives-me-the-heebie-jeebies.md @@ -0,0 +1,210 @@ +--- +title: Ctrl-C in psql gives me the heebie-jeebies +description: Postgres has been YOLOing query cancellation for 30 years +excerpt: >- + There are a few different reasons to hit the brakes on a Postgres query. Maybe + it’s taking too long to finish. Maybe you realised you forgot to create an + index that will make it orders of magnitude quicker. Maybe there’s some reason + the results are no longer needed. Or maybe you,... +date: '2026-03-05T16:32:34' +updatedOn: '2026-03-05T16:59:50' +category: postgres +categories: + - postgres +authors: + - george-mackerron +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ctrl-c-in-psql-gives-me-the-heebie-jeebies/cover.jpg + alt: null +isFeatured: false +seo: + title: Ctrl-C in psql gives me the heebie-jeebies - Neon + description: Postgres has been YOLOing query cancellation for 30 years + keywords: [] + noindex: false + ogTitle: Ctrl-C in psql gives me the heebie-jeebies - Neon + ogDescription: >- + There are a few different reasons to hit the brakes on a Postgres query. + Maybe it’s taking too long to finish. Maybe you realised you forgot to + create an index that will make it orders of magnitude quicker. Maybe there’s + some reason the results are no longer needed. Or maybe you, or your LLM + buddy, […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ctrl-c-in-psql-gives-me-the-heebie-jeebies/cover.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/ctrl-c-in-psql-gives-me-the-heebie-jeebies/neon-ctrl-c-1024x538-a995d0bc.jpg) + +There are a few different reasons to hit the brakes on a Postgres query. Maybe it’s taking too long to finish. Maybe you realised you forgot to create an index that will make it orders of magnitude quicker. Maybe there’s some reason the results are no longer needed. + +Or maybe you, or your LLM buddy, made a mistake in the SQL, and you only noticed it while you were waiting for it to return. Maybe it’s even a mistake [scary chords play] that could have valuable production data as collateral damage. But let’s hope it isn’t (or, if it is, that you’re using a system that does time-travel, like Neon). + +Whatever the reason, if you’re a psql command-line user, Ctrl-C is in your muscle memory. So now you’re looking at the words `Cancel request sent`, followed shortly after by the not-really-an-error message `ERROR: cancelling statement due to user request`. But what’s going on behind the scenes? + +## How CancelRequest works + +To cancel a Postgres query, the Postgres client makes a new and additional connection to the server, in the form of a [CancelRequest](https://www.postgresql.org/docs/18/protocol-flow.html#PROTOCOL-FLOW-CANCELING-REQUESTS). The server distinguishes this from an ordinary client connection via a magic protocol version number at the beginning of the startup message: the latest Postgres protocol is v3.2 (or `0x00030002`), but a CancelRequest claims to be v1234.5678 (or `0x04d2162e`). + +A CancelRequest targets a connection rather than a specific query. The target connection is identified to the server by two numbers that the server originally provided to the client at the end of their connection handshake (via the [BackendKeyData](https://www.postgresql.org/docs/18/protocol-flow.html#PROTOCOL-FLOW-START-UP) message). + +The numbers are a 4-byte process ID and a secret random key value, traditionally also 4 bytes long. Aside from an initial length-of-message value, that’s everything the client sends. No credentials are required except that 4-byte secret key. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/ctrl-c-in-psql-gives-me-the-heebie-jeebies/image-4-1024x724-65e3d548.png) + +It’s perhaps slightly surprising that Postgres cancels by connection rather than by query. It leads to a race condition: we risk cancelling a different query to the one that was running at the moment we asked to cancel it (this isn’t great, but the heebie-jeebies are pretty mild so far: maybe a 2 or 3 out of 10). + +But here’s a bigger surprise: **psql always sends this CancelRequest unencrypted.** Even if the connection carrying the query to be cancelled has the strictest possible TLS settings (`sslmode=verify-full`, `channel_binding=require`, and so on), psql always goes right ahead and cancels in plaintext. + +The Postgres server on the other end of the exchange has _accepted_ CancelRequest messages over TLS ever since it got TLS support. But until Postgres 17 — that is, less than 18 months ago — there was simply no support for encrypting a CancelRequest in libpq, the client-side Postgres C library on which psql is built. + +Since Postgres 17, libpq [does have functions](https://www.postgresql.org/message-id/E1rk5BP-003RWD-U4@gemulon.postgresql.org) to send CancelRequest messages over TLS. And many drivers that are based on libpq, [such as ruby-pg](https://github.com/ged/ruby-pg/blob/master/ext/pg_cancel_connection.c), now use these new, encrypting functions. + +But psql itself _still doesn’t use them_. As of today, hit Ctrl-C in psql and your CancelRequest goes over the wire naked as the day it was born, in unencrypted plaintext. + +## Hello, Denial-of-Service? + +As [someone who cares](https://neon.com/blog/postgres-needs-better-connection-security-defaults) about Postgres security, this makes me a bit uneasy (heebie-jeebies level: a solid 6 at least). There’s a potential Denial of Service attack lurking here. + +It’s not that the Postgres developers aren’t well aware of the problem. There are [architectural reasons](https://www.postgresql.org/message-id/E1rmHjd-004U0Q-3A@gemulon.postgresql.org) why psql doesn’t yet use libpq’s encrypted cancellation functions (it “would need a much larger refactor to be able to call them due to the new functions not being signal-safe”) – but a patch to do the necessary refactoring [is in the works](https://commitfest.postgresql.org/patch/6314/) for some future release. + +And Neon’s own Heikki described the risk that [4-byte secret keys get brute-forced](https://www.postgresql.org/message-id/508d0505-8b7a-4864-a681-e7e5edfe32aa@iki.fi) two years back, resulting in the [first Postgres protocol update](https://neon.com/postgresql/postgresql-18/security-improvements) in over twenty years. Protocol v3.2 differs from v3.0 only in that the secret key for cancellation can be up to 256 bytes long. But libpq and psql [still don’t use this new version](https://www.postgresql.org/docs/current/protocol-overview.html#PROTOCOL-VERSIONS) unless you explicitly specify `min_protocol_version=3.2` on the end of your connection string. + +So, what if you do use the latest psql, and you also specify protocol 3.2 so as to get a secret key that’s too big to brute-force? Even then, as soon as you cancel a query, anyone who can see your network traffic (e.g. anyone on the same open WiFi network) [can mount a Denial of Service attack](https://www.postgresql.org/message-id/489C969D.8020800@enterprisedb.com). Specifically, they can cancel any and all future queries on the same connection, just by repeatedly replaying the intercepted cancellation request. + +## No TLS = SNI is MIA + +The fact that CancelRequest connections commonly travel unencrypted has some slightly subtler impacts that Postgres-adjacent developers need to deal with. In fact, these are what led me to dive deeper into this little backwater of the Postgres protocol. + +I created and maintain [Elephantshark](https://github.com/neondatabase/elephantshark), an open-source Postgres network traffic monitor. It’s like Wireshark, but specialised for Postgres and implemented as a proxy. + +Elephantshark’s first release could only proxy one Postgres connection at a time. That had been fine for my own use-cases, but first contact with the real world demonstrated it was a pretty dumb limitation. + +As one example, you now know that when you press Ctrl-C in psql, the [CancelRequest](https://www.postgresql.org/docs/current/protocol-flow.html#PROTOCOL-FLOW-CANCELING-REQUESTS) message that’s sent to the Postgres back-end travels over a new and separate connection. As another example: did you also know that [Bun.SQL](https://bun.com/docs/runtime/sql) defaults to connecting a pool of 10 clients, eagerly and in parallel, even if you only send one query? + +It’s safe to say that neither of those things plays nicely with one-at-a-time connection support. So Elephantshark got support for concurrent connections in v0.2. That fixed its Bun.SQL issue. + +But Ctrl-C in psql still wasn’t working. And, after a little digging, the non-encryption of CancelRequest messages turned out to be the culprit. + +That’s because, when you connect to Postgres via Elephantshark over TLS, Elephantshark typically works out your target server using the SNI (Server Name Indication) extension to TLS, plus a customisable suffix. + +For instance, to monitor a connection to `ep-adj-noun-abc1234.region.aws.neon.tech`, you run elephantshark and then connect to `ep-adj-noun-abc1234.region.aws.neon.tech.local.neon.build` instead. Any subdomain of .local.neon.build resolves to 127.0.0.1, where Elephantshark is running. And Elephantshark knows that, if the hostname it gets via SNI ends in .local.neon.build, it should strip that off before it makes the onward connection. + +Marvellous. Until, that is, you hit the brakes on a query. Because your CancelRequest travels unencrypted, there is no SNI, and that means no record of the intended destination host. Luckily, Elephantshark recognises when it’s being asked to forward both from and to localhost, and would politely bail in preference to sending your message in an infinite loop. Unluckily, until Elephantshark v0.3, this would bring you no closer to cancelling your query. + +The fix for this turns out to be old news in the Postgres proxy world. It’s [in Neon’s proxy](https://github.com/neondatabase/neon/blob/main/proxy/src/cancellation.rs), for one. On receiving an ordinary client connection, you update a data structure that maps (process ID, secret key) values to destination hostnames. On subsequently receiving a CancelRequest connection without SNI, you look up the destination hostname using the (process ID, secret key) it specifies. Elephantshark v0.3 keeps a Ruby Hash for this purpose, so now you can both cancel and monitor your cancelling. + +## Wrapping up + +Let’s hope encrypted CancelRequest messages land in psql soon. In the meantime, if you care a lot about security, do some or all of the following: use Postgres 18 and `min_protocol_version=3.2`; use a VPN; don’t use Ctrl-C in psql; and check any other Postgres clients or drivers you use to see if they’re encrypting their CancelRequests. + +On that last point, [Elephantshark](https://github.com/neondatabase/elephantshark) can help. And watch this space — I have an idea to make this kind of check even easier in the near future. + +_Appendix: what happens when you press Ctrl-C in psql, using Postgres protocol v3.2, courtesy of Elephantshark_ + +
+ + + #1 listening on 127.0.0.1 port 5432 … + #2 listening on 127.0.0.1 port 5432 … + #1 connected at t0 = 2026-03-04 17:42:07 +0000 + #1 client -> script: "\x00\x00\x00\x08\x04\xd2\x16\x2f" = SSLRequest + #1 script -> client: "S" = SSL supported + #1 TLSv1.3/TLS_AES_256_GCM_SHA384 connection established with client + #1 server name via SNI: ep-winter-morning-abv2sxkp.eu-west-2.aws.neon.tech.local.neon.build + #1 client -> script: "\x00\x00\x00\x56" = 86 bytes of startup message + #1 "\x00\x03" = protocol major version 3 "\x00\x02" = protocol minor version 2 + #1 "user\x00" = key "neondb_owner\x00" = value + #1 "database\x00" = key "neondb\x00" = value + #1 "application_name\x00" = key "psql\x00" = value + #1 "client_encoding\x00" = key "UTF8\x00" = value + #1 "\x00" = end + #1 connecting to Postgres server: ep-winter-morning-abv2sxkp.eu-west-2.aws.neon.tech + #1 script -> server: "\x00\x00\x00\x08\x04\xd2\x16\x2f" = SSLRequest + #1 server -> script: "S" = SSL supported + #1 TLSv1.3/TLS_AES_256_GCM_SHA384 connection established with server + #1 forwarding client startup message to server + #1 script -> server: "\x00\x00\x00\x56" = 86 bytes of startup message + #1 "\x00\x03" = protocol major version 3 "\x00\x02" = protocol minor version 2 + #1 "user\x00" = key "neondb_owner\x00" = value + #1 "database\x00" = key "neondb\x00" = value + #1 "application_name\x00" = key "psql\x00" = value + #1 "client_encoding\x00" = key "UTF8\x00" = value + #1 "\x00" = end + #1 forwarding all later traffic + #1 server -> client: "R" = Authentication "\x00\x00\x00\x2a" = 42 bytes "\x00\x00\x00\x0a" = AuthenticationSASL + #1 "SCRAM-SHA-256-PLUS\x00" = SASL mechanism + #1 "SCRAM-SHA-256\x00" = SASL mechanism + #1 "\x00" = end + #1 ^^ 43 bytes forwarded at +0.17s + #1 client -> server: "p" = SASLInitialResponse "\x00\x00\x00\x36" = 54 bytes + #1 "SCRAM-SHA-256\x00" = selected mechanism "\x00\x00\x00\x20" = 32 bytes follow + #1 "n,,n=,r=6Dv+CvxsNyKjNgqj9ICgfr0A" = SCRAM client-first-message + #1 ^^ 55 bytes forwarded at +0.17s + #1 server -> client: "R" = Authentication "\x00\x00\x00\x5c" = 92 bytes "\x00\x00\x00\x0b" = AuthenticationSASLContinue + #1 "r=6Dv+CvxsNyKjNgqj9ICgfr0AmTrd19svrM64paqVlEwutk7d,s=wJpvuPu9tcsKybvQsHxu+w==,i=4096" = SCRAM server-first-message + #1 ^^ 93 bytes forwarded at +0.18s + #1 client -> server: "p" = SASLResponse "\x00\x00\x00\x6c" = 108 bytes + #1 "c=biws,r=6Dv+CvxsNyKjNgqj9ICgfr0AmTrd19svrM64paqVlEwutk7d,p=iFJwCSajkRUUpTF5J9Sb7mqumKK3JyEZtW1DrGTWil0=" = SCRAM client-final-message + #1 ^^ 109 bytes forwarded at +0.19s + #1 server -> client: "R" = Authentication "\x00\x00\x00\x36" = 54 bytes "\x00\x00\x00\x0c" = AuthenticationSASLFinal + #1 "v=U52Ytx3KNXJPcOgMs0UdGwJBjHUdw91OTUawbAB0EK0=" = SCRAM server-final-message + #1 server -> client: "R" = Authentication "\x00\x00\x00\x08" = 8 bytes "\x00\x00\x00\x00" = AuthenticationOk + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x17" = 23 bytes "in_hot_standby\x00" = key "off\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x19" = 25 bytes "integer_datetimes\x00" = key "on\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x11" = 17 bytes "TimeZone\x00" = key "GMT\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x1b" = 27 bytes "IntervalStyle\x00" = key "postgres\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x20" = 32 bytes "search_path\x00" = key "\"$user\", public\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x15" = 21 bytes "is_superuser\x00" = key "off\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x1a" = 26 bytes "application_name\x00" = key "psql\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x26" = 38 bytes "default_transaction_read_only\x00" = key "off\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x1a" = 26 bytes "scram_iterations\x00" = key "4096\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x17" = 23 bytes "DateStyle\x00" = key "ISO, MDY\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x23" = 35 bytes "standard_conforming_strings\x00" = key "on\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x27" = 39 bytes "session_authorization\x00" = key "neondb_owner\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x19" = 25 bytes "client_encoding\x00" = key "UTF8\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x22" = 34 bytes "server_version\x00" = key "17.8 (6108b59)\x00" = value + #1 server -> client: "S" = ParameterStatus "\x00\x00\x00\x19" = 25 bytes "server_encoding\x00" = key "UTF8\x00" = value + #1 server -> client: "K" = BackendKeyData "\x00\x00\x00\x14" = 20 bytes + #1 "\x00\x00\x03\x6d" = process ID: 877 + #1 "\x6b\x32\xfb\x77\x81\x02\x50\xee\x49\xe2\x92\x73" = secret key + #1 server -> client: "Z" = ReadyForQuery "\x00\x00\x00\x05" = 5 bytes "I" = idle + #1 ^^ 522 bytes forwarded at +0.22s + #1 client -> server: "Q" = Query "\x00\x00\x00\x19" = 25 bytes "SELECT pg_sleep(10);\x00" = query + #1 ^^ 26 bytes forwarded at +1.41s + #3 listening on 127.0.0.1 port 5432 … + #2 connected at t0 = 2026-03-04 17:42:10 +0000 + #2 no SSLRequest: continuing in plaintext + #2 client -> script: "\x00\x00\x00\x18" = 24 bytes of CancelRequest message + #2 "\x04\xd2" = cancel code: 1234 "\x16\x2e" = cancel code: 5678 + #2 "\x00\x00\x03\x6d" = target backend process ID: 877 + #2 "\x6b\x32\xfb\x77\x81\x02\x50\xee\x49\xe2\x92\x73" = target backend secret key + #2 using remembered host for CancelRequest connection + #2 connecting to Postgres server: ep-winter-morning-abv2sxkp.eu-west-2.aws.neon.tech + #2 script -> server: "\x00\x00\x00\x08\x04\xd2\x16\x2f" = SSLRequest + #2 server -> script: "S" = SSL supported + #2 TLSv1.3/TLS_AES_256_GCM_SHA384 connection established with server + #2 forwarding client startup message to server + #2 script -> server: "\x00\x00\x00\x18" = 24 bytes of CancelRequest message + #2 "\x04\xd2" = cancel code: 1234 "\x16\x2e" = cancel code: 5678 + #2 "\x00\x00\x03\x6d" = target backend process ID: 877 + #2 "\x6b\x32\xfb\x77\x81\x02\x50\xee\x49\xe2\x92\x73" = target backend secret key + #2 forwarding all later traffic + #2 server hung up + #2 connection end + #1 server -> client: "T" = RowDescription "\x00\x00\x00\x21" = 33 bytes "\x00\x01" = 1 columns follow + #1 "pg_sleep\x00" = column name "\x00\x00\x00\x00" = table OID: 0 "\x00\x00" = table attrib no: 0 + #1 "\x00\x00\x08\xe6" = type OID: 2278 "\x00\x04" = type length: 4 "\xff\xff\xff\xff" = type modifier: -1 "\x00\x00" = format: text + #1 server -> client: "E" = ErrorResponse "\x00\x00\x00\x68" = 104 bytes + #1 "S" = severity "ERROR\x00" = value + #1 "V" = unlocalized severity "ERROR\x00" = value + #1 "C" = SQLSTATE code "57014\x00" = value + #1 "M" = message "canceling statement due to user request\x00" = value + #1 "F" = file "postgres.c\x00" = value + #1 "L" = line "3428\x00" = value + #1 "R" = routine "ProcessInterrupts\x00" = value + #1 "\x00" = end + #1 server -> client: "Z" = ReadyForQuery "\x00\x00\x00\x05" = 5 bytes "I" = idle + #1 ^^ 145 bytes forwarded at +3.13s + +
diff --git a/content/blog/posts/cutting-storage-costs.md b/content/blog/posts/cutting-storage-costs.md new file mode 100644 index 0000000000..4419c0dec9 --- /dev/null +++ b/content/blog/posts/cutting-storage-costs.md @@ -0,0 +1,76 @@ +--- +title: We’re Cutting Storage Costs (Automatically) +description: We’ve implemented archive storage so you pay less for inactive branches +excerpt: >- + Starting December 1st, many Neon users will see lower storage costs. Inactive + branches—those idle for at least a day and older than 2 weeks—will now be + charged at a reduced per-GB rate. If this applies to you, there’s nothing you + need to do—it all happens automatically. Simply en... +date: '2024-11-21T00:26:43' +updatedOn: '2024-11-21T00:26:45' +category: company +categories: + - company +authors: + - anna-stepanyan +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cutting-storage-costs/cover.jpg + alt: null +isFeatured: false +seo: + title: We’re Cutting Storage Costs (Automatically) - Neon + description: >- + All branches idle for at least one day and older than two weeks will now be + charged a reduced per-GB rate, saving users money. + keywords: [] + noindex: false + ogTitle: We’re Cutting Storage Costs (Automatically) - Neon + ogDescription: >- + All branches idle for at least one day and older than two weeks will now be + charged a reduced per-GB rate, saving users money. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/cutting-storage-costs/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/cutting-storage-costs/neon-cut-cost-1-1-1024x576-6733c820.jpg) + +**Starting December 1st, many Neon users will see lower storage costs. Inactive branches—those idle for at least a day and older than 2 weeks—will now be charged at a reduced per-GB rate. If this applies to you, there’s nothing you need to do—it all happens automatically. Simply enjoy the savings.** + +Here’s a deeper dive into what’s behind this price reduction, if you’re interested: + +## What you get with our standard per-GB price + +Neon uses a [custom storage engine](https://neon.tech/blog/get-page-at-lsn), so its “regular” per-GB price covers more than just basic storage. It also includes: + +- **Cache for high performance**. Neon is built for speed, with a cache component we’re calling the [Pageserver](https://neon.tech/blog/get-page-at-lsn). The Pageserver stores frequently accessed data on high-performance SSDs to ensure low latencies. For HA, Neon also runs secondary Pageservers in different availability zones that maintain up-to-date copies of the project’s data. +- **Safekeepers for WAL**. Neon’s storage also includes Safekeepers, which capture [every change made to your database through WAL.](https://neon.tech/blog/what-you-get-when-you-think-of-postgres-storage-as-a-transaction-journal) This is how Neon can offer features like branching and [time travel queries](https://neon.tech/docs/guides/time-travel-assist). +- **Object storage for the long term**. While the Pageserver handles fast access, Neon uses cloud object storage (Amazon S3 or Azure Blob Storage) as the durable layer that holds the bulk of your data. + +
+Image +
The Neon storage architecture. Diagram shows how failures of Safekeeper or Pageserver services are recovered across Availability Zones in case of failure.
+
+ +This design allows us to offer a better developer experience (with performance, durability, branching, and HA). But reflecting on our architecture, we asked ourselves—is it necessary to keep all these components fully operative all the time? + +The answer, of course, is that certain components (Pageservers, Safekeepers) are not active if this data is not being accessed, which leads us to the introduction of archive storage. + +## Archive storage: What changes + +Now, when a branch has been inactive for at least one day and is older than two weeks, Neon will automatically “archive” it. **This process doesn’t involve transferring data; rather, the Pageserver simply evicts the branch’s data from its SSD cache.** + +The branch remains stored in cloud object storage, but it no longer uses the high-performance SSD cache. When you query an archived branch again, it’s “reactivated” automatically. Once reactivated, the Pageserver pulls the branch’s data back into SSD storage, restoring it to full performance. + +
+Image +
Since Archive storage is hosting inactive branches, we can lower storage costs by deactivating pageserver and safekeepers. These elements are automatically reactivated once branches get accessed.
+
+ +Here’s the tradeoff (there’s always one): while the branch’s data remains securely in cloud storage, accessing it will come with a small performance delay _the first time you query it_ as it’s reloaded to the Pageserver again. + +But this tradeoff is well worth it. By implementing achive storage, we can lower down the costs of our storage infrastructure, passing down this price reduction to our customers. We’ll bill archive storage at a reduced rate—$0.1 per GB-month. Plus, your monthly subscription for the Launch, Scale, and Business plans will include a substantial amount of archive storage. We’ll share more details soon. + +## Wrap up + +This storage model enables us to pass down storage savings to our customers with minimal consequences. Once again, no action is required from you—you can just enjoy the savings when they come. If you have questions, [ask us in Discord](https://discord.gg/92vNTzKDGp), we’ll be happy to chat. diff --git a/content/blog/posts/dat-streams-millions-of-staff-messages-through-neon.md b/content/blog/posts/dat-streams-millions-of-staff-messages-through-neon.md new file mode 100644 index 0000000000..172b292f1a --- /dev/null +++ b/content/blog/posts/dat-streams-millions-of-staff-messages-through-neon.md @@ -0,0 +1,135 @@ +--- +title: DAT Streams Millions of Staff Messages Through Neon +description: 'They power mission-critical field operations for 16,000+ staff' +excerpt: >- + “Our system can’t afford any downtime. We manage field staff operations for + thousands of workers through WhatsApp, and if a message fails to write, it’s + lost. We’re leveraging all of Neon’s multi-region flexibility, autoscaling, + and read replicas to optimize reliability at scale”... +date: '2025-09-29T15:55:24' +updatedOn: '2025-09-29T15:56:05' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dat-streams-millions-of-staff-messages-through-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: DAT Streams Millions of Staff Messages Through Neon - Neon + description: >- + Learn how DAT powers mission-critical field operations for +16k workers + taking advantage of Neon's branching, autoscaling, and replicas. + keywords: [] + noindex: false + ogTitle: DAT Streams Millions of Staff Messages Through Neon - Neon + ogDescription: >- + Learn how DAT powers mission-critical field operations for +16k workers + taking advantage of Neon's branching, autoscaling, and replicas. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dat-streams-millions-of-staff-messages-through-neon/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dat-streams-millions-of-staff-messages-through-neon/neon-dat-systems-1024x576-4a4ad5e3.jpg) + +
+

“Our system can’t afford any downtime. We manage field staff operations for thousands of workers through WhatsApp, and if a message fails to write, it’s lost. We’re leveraging all of Neon’s multi-region flexibility, autoscaling, and read replicas to optimize reliability at scale” (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +DAT is a Pakistan-based startup who 2 years ago set out to solve a uniquely local problem: how to effectively manage and monitor field staff operations at scale. Today, the company powers mission-critical operations for the Lahore Waste Management Company (LWMC), Pakistan’s largest public sector employer with more than 16,000 staff across multiple districts. + +Their flagship product integrates directly with WhatsApp, Telegram, and WeChat to manage field staff operations for thousands of workers. Staff in the field send updates through messaging apps (text, photos, videos, location pins, and voice notes), while managers monitor everything in real time through dashboards built on [Retool](https://retool.com/). + +
+Image +
One of the dashboards built by DAT, used to track staff activity in real-time. Frontend powered by Retool
+
+ +## A Write-Heavy, Real-Time Data Pipeline + +
+

“Every message in our pipeline triggers four or five jobs, all written into the database. We’re processing around 10 million transactions a month”  (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +DAT’s workload is a combination of continuous ingestion, multiple writes per event, background jobs, and complete audit trails. Here’s how their pipeline looks like: + +- Messages flow in from WhatsApp Business (their primary channel) as well as Telegram and WeChat. These include text, location pins, photos, videos, and voice notes. +- Each incoming message results in at least three separate writes: + - The message itself + - Background jobs queued for processing (four to five per message) + - Audit log entries recording every change to the application +- Every message is written into Postgres on Neon, and then triggers 4–5 background jobs for tasks such as image resizing, anomaly detection, or geolocation validation. All jobs are persisted in the DB using [Oban](https://github.com/oban-bg/oban), Elixir’s job queue. +- Every change in the application is also logged with a complete record, to provide accountability and traceability for public sector clients. These logs are written into Postgres. +- To keep the database lean, audit logs are regularly streamed out of Postgres and moved to blob storage. + +## Why DAT Runs Postgres on Neon + +From the very beginning, DAT built on [Neon](https://neon.com/), starting on Neon’s Free Plan to start their PoC and growing quickly from there. This choice was deliberate: having worked with large cloud providers in the past, they knew the pain of committing to fixed instances, dealing with region lock-in, and leaking money on idle resources. + +They’ve built a system that takes advantage of Neon’s serverless features to service their workload with top efficiency: + +### Read replicas to optimize performance + +
+

“Since our workload is write-heavy, we use Neon’s primary instance for nonstop ingestion, and then spin up read replicas for dashboards and analytics” (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +DAT’s database layer is constantly hammered with writes from thousands of messages, background jobs, and audit logs, so they split reads and writes taking advantage of the flexibility of Neon’s [read replicas](https://neon.com/docs/introduction/read-replicas). This allows them to ensure ingestion never slows down while supervisors and managers still get real-time reporting. + +Read replicas in Neon [are architected differently from traditional Postgres replicas](https://neon.com/blog/the-problem-with-postgres-replicas): instead of being tied to a fixed VM, each replica is a serverless compute node that connects to the same durable storage layer. Replicas can be created instantly, scale independently from the primary, and scale down to zero when idle. + +### Autoscaling to cover for peak events without hussle + +
+

“During the Eid cleanup, we have thousands of workers sending nonstop updates for three straight days. Neon just scales with the load – sometimes the write side, sometimes the read side – and we don’t have to touch a thing”  (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +Every application faces moments of extreme load when traffic suddenly surges – sometimes planned, sometimes unexpected. Instead of manually resizing databases or committing to oversized instances in advance, Neon’s [autoscaling](https://neon.com/docs/introduction/autoscaling) is an efficient, hands-off solution for handling these bursts automatically. Compute scales up when traffic spikes and quickly goes back down when things slow down, giving companies like DAT a worry-free solution for managing mission-critical peaks. + +### Scale-to-zero to keep consumption efficient + +
+

“When workers go on break, traffic completely stops for a while. Our Neon databases scale down to zero so we’re not paying for idle compute. The same goes for our dev and test environments” (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +Unlike traditional cloud databases, Neon automatically scale down during quiet periods – even all the way down to zero to save you any compute costs. This behavior also extends to development and test environments, which in Neon you create via [branches](https://neon.com/docs/introduction/branching). + +### Branching for realistic development and testing environments + +
+

“Anytime we need to test a new feature or extension, we branch off from production, run our checks, and reconnect when we’re ready. It’s simple, safe, and saves us huge amounts of time” (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +In other managed Postgres, creating a realistic development environment involves duplicating an instance, copying data, paying for the extra capacity 24/7, and constantly work to keep things updated with production. This is a slow and costly process. + +Neon takes a different approach with [branching](https://neon.com/docs/introduction/branching). Branches on Neon are instant, copy-on-write clones of a production database: they include the full schema and data, only consume storage as changes are made, and are able to scale to zero. + +Implementing [branching workflows](https://neon.com/branching) lets teams like DAT ship faster while keeping costs under control. They use branches every time they need to test a new extension, feature, or schema change. As soon as testing is complete, the branch is deleted. This is all done via API and all the operations are near-instant. + +## Using Retool for Client Dashboards + +
+

“Retool lets us move really fast: we can duplicate an app for a new client, tweak it, and deploy instantly. All the authentication, maps, tables, and forms are already there” (Ahsan Nabi Dar, CTO and co-founder of DAT)

+
+ +While Neon powers the database layer, Retool is the presentation layer that DAT uses to serve its clients. Every organization they work with gets a personalized dashboard built in Retool. These dashboards provide supervisors and managers with real-time visibility into staff operations, e.g. + +- Maps to track worker locations +- Tables and filters for reviewing large volumes of field data. +- Forms for editing staff records, team structures, or assignments +- Authentication and SSO out of the box for secure access + +By building on Retool, DAT Systems avoids the work of maintaining and hosting a custom frontend. Instead of reinventing all the components above, Retool handles it – giving them a speed that lets them serve multiple clients at once without adding extra headcount. + +Retool has been a trusted Neon partner for years. Not only both products work great together; [Retool itself also uses Neon as the backend for some of their offerings.](https://neon.com/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases) + +## Wrap-Up + +From a three-person startup to powering mission-critical operations for 16k+ field workers, DAT Systems has grown a resilient system on Neon. [You can get started the same way they did: on Neon’s Free Plan.](https://console.neon.tech/signup) Spin up a Postgres database in seconds, experiment with branching and scale-to-zero, and see how much time (and money) you save. + +--- + +_Thank you so much to Ahsan and DAT for building on Neon and for sharing their story. If you’d also like to be featured,_ [tell us on Discord.](https://discord.gg/92vNTzKDGp) diff --git a/content/blog/posts/database-branching-for-postgres-with-neon.md b/content/blog/posts/database-branching-for-postgres-with-neon.md new file mode 100644 index 0000000000..2e31b58193 --- /dev/null +++ b/content/blog/posts/database-branching-for-postgres-with-neon.md @@ -0,0 +1,106 @@ +--- +title: ketteQ uses Neon branching for scenario analysis +description: Testing hundreds of scenarios on production databases cleanly with zero risk +excerpt: >- + ketteQ is a supply chain planning and automation platform built on Salesforce + and AWS. Its scenario management framework allows its users to model capacity, + forecast demand, plan operations, manage orders and handle disruptions by + enabling them to model various scenarios in their... +date: '2022-12-06T15:14:30' +updatedOn: '2025-10-10T08:55:40' +category: case-study +categories: + - case-study +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-branching-for-postgres-with-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: ketteQ uses Neon branching for scenario analysis - Neon + description: Testing hundreds of scenarios on production databases cleanly with zero risk + keywords: [] + noindex: false + ogTitle: ketteQ uses Neon branching for scenario analysis - Neon + ogDescription: >- + ketteQ is a supply chain planning and automation platform built on + Salesforce and AWS. Its scenario management framework allows its users to + model capacity, forecast demand, plan operations, manage orders and handle + disruptions by enabling them to model various scenarios in their business. + For example, a Supply Chain Manager could run a scenario to understand […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-branching-for-postgres-with-neon/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-branching-for-postgres-with-neon/neon-database-branching-with-neon-1-1024x576-5b08fe13.jpg) + +[ketteQ](https://www.ketteq.com) is a supply chain planning and automation platform built on Salesforce and AWS. Its scenario management framework allows its users to model capacity, forecast demand, plan operations, manage orders and handle disruptions by enabling them to model various scenarios in their business. For example, a Supply Chain Manager could run a scenario to understand how a 30% uplift in demand would affect their stock and their ability to meet their customers’ needs. She could tweak different parameters for each scenario and run simulations. + +Each scenario is a snapshot of the current working data. The problem for ketteQ was that some of their large clients maintain hundreds of scenarios at any time. On traditional Postgres offerings like RDS, creating snapshots or provisioning a new database for each one of them is hard to manage and time-consuming. + +With Neon and database branching, ketteQ creates a branch for each user’s scenario to allow its customers to model many different scenarios simultaneously and compare the results of their analysis to find the optimal business outcome. They can create hundreds of branches to scale to meet the customers’ demands without provisioning additional resources. + +We spoke with Vasu Narayan, Senior Vice President of Product, to discuss how ketteQ uses database branching for scenario analysis. + +_“Certain clients of ours, with a large user base, need to maintain hundreds of scenarios at any time. Neon capabilities allow us to scale to meet our customers’ needs to be able to maintain and run these scenarios. Branching also provides a way to maintain a hierarchical relationship among the scenarios to track their provenance and versioning.”_ + + + +## What is database branching, and why should you care? + +A branch is a copy of your database that you can use in your development environment and for testing. + +A typical development shop has production, staging, and development databases that share the same schema but not always the same data. + +When you build features that require “real” data for verification and testing, you might add scripts that stream data to your non-production environments, use snapshots, or even generate data to test with. But some common problems with these approaches include + +1. Adding unwanted complexity to your architecture. +2. Having to write and maintain more code +3. Having to maintain production-size databases in your staging and development environments. +4. They are difficult to automate + +With database branching, your other environments, such as staging and development, are just a branch of your production database, which you can create instantly. Branches include all of the data in your parent branch at the point of branch creation. You can easily spin up new branches with a single click or API call. + +Another typical issue developers face is when they work in parallel on features that involve changes to the database. The chance of introducing conflicts and bugs in your code base is much greater in these situations. And let’s face it, this is often a significant source of frustration among the development team members. + +## How does branching help? + +So, how does branching address the common pain points we discussed above? Before we get to that, let’s take a closer look at how branching works. + +The space complexity of creating a branch is _O(parent-branch-metadata),_ which makes branching computationally inexpensive. Neon uses the copy-on-write technique to create branches. This means that when you create a branch, you only increase your storage when you update and insert data in the branch. + +![Branching](https://cdn.neonapi.io/public/images/pages/blog/database-branching-for-postgres-with-neon/branching-1024x690-8e7b7e72.png) + +Neon separates storage and compute, which enables branching. Our co-founder Heikki Linnakangas [discusses Neon’s architecture](https://neon.tech/blog/architecture-decisions-in-neon/) in a different blog post. In addition to branching, one of the benefits of Neon’s architecture is the ability to scale compute nodes to zero. + +Copy-on-write combined with scale-to-zero works well for development teams that only want to consume the necessary resources—no more VMs with development databases that run 24/7 but are used only 10% of the time. + +Neon enables every developer to have a branch of the production database for development and testing features with “real” data. + +As a developer, you know way too well how important it is to test your code in your CI/CD before deploying it to production. But what about your database? With the [Neon API](https://api-docs.neon.tech/reference/getting-started-with-neon-api), you can implement branching into your CI/CD pipeline to ensure that your database changes won’t break your application or cause other problems in production. + +Tune in to [our talk about the Neon API](https://www.youtube.com/watch?v=NI2x1mhB6uI) to learn more about using database branching to address common pain points in your workflows. + +## So, what else can you do with branching? + +We have discussed how branching can be used in the development and that everyone in your team can have a branch of your production database to run tests. Now, let’s look at other ways that you can branch, but before that, let’s touch on the structure of the Neon account and the Neon API for a moment + +In your Neon account, you can create multiple projects. Every project comes with a main branch and can have several other branches. Every branch comes with a default `neondb` database, and you can create as many other databases as you need. So, any project that you create starts with this hierarchy: + +`project xyz -> main branch -> neondb database` + +You can learn more about [Neon account hierarchy](https://neon.tech/docs/manage/overview/) in the documentation. + +## Recap + +To summarize, database branching allows you to create new database environments and test features with production data to avoid issues after deployment. It saves you the hassle of maintaining additional code and tools for streaming data to your staging and development environments. + +Since branching uses copy-on-write and compute-nodes scale down to zero, each team member can get a branch to work simultaneously without breaking your development database. + +Additionally, branching opens the door to a world of possible use cases. Some users use database branching and the Neon API to create hundreds of branches to run simulations. + +What could you do with h branching? Let us know what your use case is! + +Don’t have an account? [Sign-up today](https://console.neon.tech/sign_in)! diff --git a/content/blog/posts/database-branching-workflows-a-guide-for-developers.md b/content/blog/posts/database-branching-workflows-a-guide-for-developers.md new file mode 100644 index 0000000000..6a98f91ad2 --- /dev/null +++ b/content/blog/posts/database-branching-workflows-a-guide-for-developers.md @@ -0,0 +1,75 @@ +--- +title: 'Database branching workflows: A guide for developers' +description: >- + We've put together step-by-step guides for two database branching workflows: + one preview per PR and one local dev branch per engineer. +excerpt: >- + Modern development demands speed and flexibility, but this is nowhere to be + found in database development. Even when using managed databases, database + workflows are still slow and prone to errors: setting up new instances takes + time, undoing changes is risky, keeping data consist... +date: '2024-05-09T17:50:40' +updatedOn: '2024-05-09T17:50:43' +category: postgres +categories: + - postgres + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-branching-workflows-a-guide-for-developers/cover.png + alt: null +isFeatured: false +seo: + title: 'Database branching workflows: A guide for developers - Neon' + description: >- + We've put together step-by-step guides for two database branching workflows: + one preview per PR and one local dev branch per engineer. + keywords: [] + noindex: false + ogTitle: 'Database branching workflows: A guide for developers - Neon' + ogDescription: >- + We've put together step-by-step guides for two database branching workflows: + one preview per PR and one local dev branch per engineer. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-branching-workflows-a-guide-for-developers/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-branching-workflows-a-guide-for-developers/cover.png) + +Modern development demands speed and flexibility, but this is nowhere to be found in database development. Even when using managed databases, database workflows are still slow and prone to errors: setting up new instances takes time, undoing changes is risky, keeping data consistent across environments becomes a pain over time—the list goes on. For many teams, the database is still the place where it’s way too easy to make a fatal mistake and slow everything down. + +This is a problem that we’re very familiar with in Neon. In fact, it is one of our main [product inspirations](https://neon.tech/blog/hello-world). We believe that data and databases should be treated as everything else in our codebase —with the same capacity for safe development and collaboration, iterative improvement, and potential for automation, all without compromising security or jeopardizing engineering time. + +## Database development done right: ship faster with these database branching guides + +To get there, we’re working on a bunch of exciting things in Neon that will be coming soon—but a big leap forward can already be made today. To demonstrate this, we’ve put together a **[database branching workflows guide](https://neon.tech/flow)** in which we walk you through two particular workflows step by step: + +- **Preview environments (one per PR)** **— [repo](https://github.com/neondatabase/preview-branches-with-fly?tab=readme-ov-file)** + - This workflow covers how to automatically create a [preview environment for every pull request](https://github.com/neondatabase/preview-branches-with-fly?tab=readme-ov-file) with its associated database branch. We’ll use Fly.io, Neon, and Drizzle. + - Once the PR is closed, the preview environment and its associated database branch are automatically deleted. +- **Local dev environments (one per developer)** + - This workflow covers how to create personalized dev development environments for every engineer in a team using database branching. Each engineer will have instant access to an isolated “copy” of production-like data. + - When the work is done, they can sync their dev branch so it reflects the latest state of schema + data in production.
+ +We review the workflows in this video: + + + +## Why adopt database branching workflows? + +The two-word summary: _you’ll ship faster._ + +If you want the longer answer, here are some customer favorites: + +- Rapid onboarding. Every engineer on the team can instantly spin up dev environments with an isolated copy of data plus schema. +- Reduced risk. Since branches can be created from past points in time, developers can explore changes with a safety net. Mistakes or unwanted modifications can be easily reverted. +- Higher quality code. It’s easier to fix bugs when using realistic, production-like data in development environments. +- Improved team collaboration. Engineers work in parallel without stepping on each other’s toes. Branches are very affordable, so this isolation costs the team very little. +- Effortless data consistency. All dev branches can be reset from main in one second, mirroring the same data conditions and schema. +- No prod-like data in personal machines. + +## Get started + +To get started with the workflows, [create a Neon project](https://console.neon.tech/signup) and follow the steps in the guide. If you implement this, we want to hear about it. Find us on [Twitter](https://x.com/neondatabase) and [Discord](https://neon.tech/discord). diff --git a/content/blog/posts/database-per-user-architecture-with-isolated-application-environments.md b/content/blog/posts/database-per-user-architecture-with-isolated-application-environments.md new file mode 100644 index 0000000000..c9202e1b2e --- /dev/null +++ b/content/blog/posts/database-per-user-architecture-with-isolated-application-environments.md @@ -0,0 +1,104 @@ +--- +title: Database-per-User Architecture With Isolated Application Environments +description: Design advice +excerpt: >- + Previously in this series, we discussed implementing database-per-user + architecture in Neon with a shared application environment communicating with + individual user databases. That approach keeps operational complexity + contained by minimizing the number of software systems deploy... +date: '2024-10-21T16:01:08' +updatedOn: '2024-10-21T16:01:11' +category: workflows +categories: + - workflows +authors: + - dian-m-fay +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-per-user-architecture-with-isolated-application-environments/cover.jpg + alt: null +isFeatured: false +seo: + title: Database-per-User Architecture With Isolated Application Environments - Neon + description: >- + Design advice for building apps with not only a database per customer but a + copy of the entire system for each customer. + keywords: [] + noindex: false + ogTitle: Database-per-User Architecture With Isolated Application Environments - Neon + ogDescription: >- + Design advice for building apps with not only a database per customer but a + copy of the entire system for each customer. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-per-user-architecture-with-isolated-application-environments/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-per-user-architecture-with-isolated-application-environments/isolated-application-environment-1-1024x576-02e03898.jpg) + +Previously in this series, we discussed implementing [database-per-user architecture](https://neon.tech/blog/multi-tenancy-and-database-per-user-design-in-postgres) in Neon with a [shared application environment](https://neon.tech/blog/shared-application-environment) communicating with individual user databases. + +That approach keeps operational complexity contained by minimizing the number of software systems deployed and limiting the flexibility you can offer your customers. But the world is a complex place, and sometimes your customers need more than the “one size fits all” service approach allows. + +**Isolated application environments** are exactly what they sound like. It’s not just databases which are separated from each other in this mode: you’re deploying and managing a copy of the entire system for each customer. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-per-user-architecture-with-isolated-application-environments/ad4nxexipaf31liufnapw-iy7xefka-upt7udnoo7cqgr4gg80direzyhv9px9utkpybjqj8v6yvhyzvtoezms4vklyononqp6xt5t8xvg8l90sbvpo56k8kcol2ln7rwqfsptwpd7crvbnst-m4emqh0dtd-97b3e634.png) + +## Why Even Consider Isolated Environments? + +This is not multitenancy by any stretch of the definition, although a well-equipped control plane can let you manage customer deployments almost like it is. First, though, why go back to this seemingly archaic way of doing things? + +Let’s say upfront that this is not a path to embark upon lightly. Shared resources allow economization on scale that you just don’t get with a deployment per customer; there has to be a very good reason, or two, for adopting this approach. + +First, there are geographical and political requirements. Depending on the type of data your customers store, governments may regulate where you’re allowed to store it through _data sovereignty_ or _data residency_ laws. Customers who operate in specific regions can also simply want their systems to be close by; computers and networks may have gotten faster over the decades, but [Hopper’s nanoseconds](https://dataphys.org/list/grace-hopper-nanoseconds/) are still as long as they ever were. + +Second, there’s the ability for customers to dictate the pace of upgrades. Some customers — in particular, older, bigger customer organizations in more institutional sectors — may expect to control this in order to minimize and manage disruptions to their own end users’ work. This isn’t possible in a shared environment, since the pace of application development dictates the only allowed version of the database schema. + +There is another reason, which is that you intend to tailor the application, and/or the database, to individual customers. **This is not a good reason**: bespoke customizations, especially in the schema, effectively make each deployment a separate software product. Instead of maintaining a single product line and stamping out instance after identical instance, you take on responsibility for adapting bugfixes and upgrades to each and every variation. We’ll talk about how to handle the inevitable one-off requests in the next section. + +## Designing for Isolated Environments + +Isolated environments shouldn’t be confused with on-premises deployments. You’re still the service provider and manage the infrastructure, including the Neon project that underpins each instance. This includes networking too: it’s up to you to determine how a given customer user connects to their instance. You could set each customer up with a subdomain, which wildcard certificates make particularly easy, but can also get more exotic with “bring your own URL” options. + +Unlike the shared environment strategy, there isn’t much to do internal to the system being deployed! There’s no need to identify a user with their customer organization in your catalog database, no need to route queries and mutations to the appropriate database. This strategy pushes complexity into provisioning and deployment. You build the software itself like it’s a single-tenant system — mostly. + +Because each deployment serves a single tenant, there’s more of a temptation (and it’s more possible) to customize. This, again, is your worst enemy. Instead, you should take care to think of opinions and requests for bespoke work as indicating customer [needs and wants](https://apenwarr.ca/log/?m=202110) that point a possible direction to the future of the whole product. Channel them into your standard requirement evaluation, design, and development processes rather than doing unreproducible one-off work. + +Capacity limits and enabled features are always constant within each deployment. This means there’s less call for feature flag management compared to a shared system, but it’s important to keep the source of truth for enablement in the catalog database. This allows you to validate and compare these settings in one place. However, it doesn’t mean that customer applications should all check in with the catalog database whenever they need to verify a feature setting! Instead, settings should be forwarded to the application database when configured in order to minimize roundtrip time for user requests. + +## Managing Isolated Environments + +Like any other [database-per-user](https://neon.tech/blog/multi-tenancy-and-database-per-user-design-in-postgres) system, isolated environments need to be provisioned, monitored, and maintained. Also like other such systems, this is all best done from a [central control plane and catalog database](https://neon.tech/blog/control-planes-for-database-per-user-in-neon). Even if a problem for one user stands no chance of being a problem for others as happens in shared application environments, those isolated problems are still hard to investigate and remediate if you don’t know how to find them. + +The main distinguishing factor for control planes in an isolated-environment setting is the complexity of provisioning. A new customer needs not just a Neon project, but a whole new application deployment with its attendant delivery pipeline, network configuration (some of which may require customer input, for example requesting a TLS certificate for a custom URL), and observability infrastructure. Automation is very much a journey in this kind of system, and the pace will be determined by the frequency with which you onboard new customers. If your customers are fewer and larger, it’s possible to get by for longer on engineering documentation and active account management. + +Next to onboarding and feature enablement, it’s also crucial to surface deployed versions in the control plane. As your application and your tools for managing it continue to mature, the control plane will be where you go to roll out upgrades. + +## Software Development Lifecycles and Isolated Environments + +Independent versioning per customer is one of the main reasons to take an isolated-environments approach, but using it effectively requires strict discipline in your development lifecycle. Versioning is never wholly linear, even if there’s a “main line” of major version releases; each major release has a lifecycle of its own, with minor versions fixing bugs and making more modest improvements. Development effort too is usually divided between work toward the next major release and updates to already-released versions. + +The figure below shows three views of a series of software releases. The first release, 1.0.0, receives ongoing support in the form of patches and feature releases while work continues on version 2. After 2.0.0 is released, a bug is discovered and patches are backported to the 1.1.x and 1.2.x series’ database schemata: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-per-user-architecture-with-isolated-application-environments/ad4nxc66cqi8uixl4vmf1xtvns1hz2bzgfoz-fhlnjr2zkthypvv1irkapmxnr8my7cr6lk0kkqr15p8p2ke6m9qniglajmnrbvusgxxts7jr1xrhu6ouohh-tugldhheh0yh2m1oa2knst9h1qa4q5k-la-2dca84e5.png) + +The first view, on the left, shows the release lineage in a SemVer-style major/minor/patch hierarchy. The second, at top right, shows the actual order in which each version arrived. + +Finally, the third view shows the flow of schema changes between versions, which is much more complicated! There’s a “main line” from 1.0.0 through 1.1.0 and 1.2.0 to 2.0.0, but even patch releases may or may not inform the schemata of subsequent versions. Here, for example, the fix in 1.0.1 is also included in 1.1.0 to forestall the same bug happening in that series, but the fix in 1.1.1 only affects the 1.1.x series – changes in 1.2.0 have rendered it moot in later versions. + +This isn’t just complicated to manage, it also poses extra difficulty when customers do want to upgrade to the next release. Because changes and fixes from “later” in the version hierarchy can get backported to “earlier” release series, it’s important to linearize upgrades to keep the number of possible upgrade paths to a minimum. The path from 1.0.1 to 1.2.0 goes _through_ the 1.1.x series, in other words. Even though, for example, 1.2.1 contains the same schema change as 1.1.2, the upgrade process as a whole is much more reliable if an older schema gets it from 1.1.2. + +Idempotent upgrades are a very effective tool in this situation. If the same columns must all be added in 2.0.1 and the backport releases 1.2.1 and 1.1.2, the higher-versioned scripts can check for their presence, types, and constraints before modifying the schema. This prevents conflicts and migration failures when a deployment that received the 1.1.2 patch goes through later upgrades. + +It takes a lot of effort to manage versions in a scenario where individual systems can be at any version at any time. But overall, it’s much easier for you to maintain a few well-defined upgrade routines that migrate from the latest patch on one major-minor lineage to the next .0 or .0.0 release and resume the main-line sequence of schema changes from there than it is to deliver point-to-point upgrades from any version to any greater version. + +## Conclusion + +Isolated environments are an investment. They offer your customers flexibility in geographic location, version management, and more — but that flexibility in turn adds substantially to your operational overhead. Still, for the right circumstances, there’s no substitute. + +--- + +This article is part of a series. Check out the previous three articles on the topic of building database-per-user architectures: + +- [Part I: Multi-tenancy and Database-per-User Design in Postgres](https://neon.tech/blog/multi-tenancy-and-database-per-user-design-in-postgres) +- [Part II: Control Planes for Database-Per-User](https://neon.tech/blog/control-planes-for-database-per-user-in-neon) +- [Part III: Shared Application Environments](https://neon.tech/blog/shared-application-environment) diff --git a/content/blog/posts/database-recovery-strategies-to-help-you-sleep-at-night.md b/content/blog/posts/database-recovery-strategies-to-help-you-sleep-at-night.md new file mode 100644 index 0000000000..7bea069e0f --- /dev/null +++ b/content/blog/posts/database-recovery-strategies-to-help-you-sleep-at-night.md @@ -0,0 +1,111 @@ +--- +title: Database recovery strategies to help you sleep at night +description: 'We hope you never need this, but Neon has instant PITR' +excerpt: >- + As you’re building your architecture, much thought goes into making database + systems resilient to failures. But shhh… stuff happens; misconfigurations, + human errors, or accidentally signing up little Bobby Tables to your + service—sometimes, databases fall over. If you don’t have a... +date: '2024-06-24T19:32:13' +updatedOn: '2024-06-24T19:34:38' +category: workflows +categories: + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-recovery-strategies-to-help-you-sleep-at-night/cover.jpg + alt: null +isFeatured: false +seo: + title: Database recovery strategies to help you sleep at night - Neon + description: >- + We hope you never need this, but it's important to have a data recovery + strategy in place in case your database fails. + keywords: [] + noindex: false + ogTitle: Database recovery strategies to help you sleep at night - Neon + ogDescription: >- + We hope you never need this, but it's important to have a data recovery + strategy in place in case your database fails. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-recovery-strategies-to-help-you-sleep-at-night/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-recovery-strategies-to-help-you-sleep-at-night/neon-database-recovery-1-1024x576-31fd7c17.jpg) + +As you’re building your architecture, much thought goes into making database systems resilient to failures. But shhh… stuff happens; misconfigurations, human errors, or accidentally signing up [little Bobby Tables](https://xkcd.com/327/) to your service—sometimes, databases fall over. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-recovery-strategies-to-help-you-sleep-at-night/exploitsofamom2x-1024x316-49e26700.png) + +If you don’t have a data recovery strategy in place, this will be a huge problem. + +## Backups are not enough: you have to _recover_ from backups + +We’ll talk about backups in this post (_of course_) but first, there’s something you need to hear: [backups are not backups unless you have tested restoring from them.](https://www.reddit.com/r/sysadmin/comments/5rlcz1/backups_are_not_backups_unless_you_have_tested/) + +Backups are essential for data protection and disaster recovery. They act as a safety net, ensuring that your valuable data is not lost in case of failures. But simply having backups is not sufficient. The true value of a backup lies in its ability to be restored successfully when needed. + +**The illusion of safety** + +It’s easy to fall into the trap of complacency once you have a backup system in place. But without regular testing of these backups, this sense of security can be dangerously misleading. A backup that has never been tested in a recovery scenario might as well not exist because there’s no guarantee that it can be restored effectively. + +**The critical factor: time to restore from backups** + +When you’re getting to production, not only is it essential to verify the integrity and functionality of your backups, but it’s also crucial to consider how much time it will take to restore from a backup if the worst happens. The time to restore (TTR) is a vital metric in any disaster recovery plan, as it directly impacts your system’s downtime and, consequently, your business operations. + +This is important, so it’s worth elaborating. In the context of Postgres, restoring a large database can be a time-consuming process. Several factors influence the restoration time, including: + +- Database size: larger databases naturally take longer to restore. +- the volume of data changes since the last backup (you’ll have to replay WAL). +- The method used for backup (e.g. pg_dump vs pg_basebackup). +- CPU, memory, and I/O resources available during the process. +- Network speed. + +**Real-world example** + +Imagine your primary Postgres server hosting a mission-critical application crashes. The database size is 1TB, and you have been using pg_basebackup for nightly backups stored on a remote server. Something goes terribly wrong and you have to restore. The process will look more or less like this: + +1. First, you download the latest backup. +2. Once you have the backup, you copy the files to the Postgres data directory. +3. After copying the files, you replay the WAL files to cover the data changes since the last backup. +4. Lastly, you verify the integrity and consistency of the restored data. + +This process takes you at least a few hours, assuming no issues arise during the restoration process. Yikes. + +## What can you do to protect yourself: database recovery strategies + +### Strategy #1 – Use a managed database + +The previous section pretty much summarizes why so many developers choose managed databases. Managed database services handle most of the heavy lifting for you, usually offering automated backups, point-in-time recovery, and other tools to ensure your data is safe and quickly recoverable. Unless you’re experienced in running databases in production (and you’re open to investing significant resources into managing the DB), a managed Postgres will be a wise choice. + +### Strategy #2 – Take your own nightly backups via Github actions + +Needless to say, [Neon](https://neon.tech/home) is one of those managed Postgres options you can choose for your workload. As such, we take care of your data; [Neon keeps a copy of your data in highly durable object storage](https://neon.tech/blog/architecture-decisions-in-neon), so you definitely don’t have to take your own backups. + +That said, sometimes you want to be extra sure. We get it. If you want to also run your own nightly backups, **not as your recovery path in case something goes wrong** but as a safety net, [this article walks you through how to schedule nightly backups in Neon via a GitHub Action and store them to S3.](https://thenewstack.io/how-to-schedule-postgresql-backups-with-github-actions/) + +### Strategy #3 – If you’re not using Neon, set up a standby/HA replica + +As we said earlier, having backups doesn’t necessarily mean you can recover from them in a timely manner. Managed database or not, you have to consider what happens when your database fails, not only in terms of data loss but also availability. How long will your system be down if production fails? + +If you’re using “traditional” databases like RDS, setting up a standby replica is a common way to protect yourself against the long periods of downtime you might experience while waiting for recovery from a backup to finish. This setup involves creating a replica that continuously receives updates from the primary database. In case of a production failure, your system can automatically switch to the replica, which becomes the primary while the original instance is being repaired. + +Standby replicas can save you in case of disaster, and they’re a recommended practice in production. However, be aware that while they minimize downtime, they also effectively **double your database costs** since you are running an additional instance at all times. + +### Strategy #4 – Chose Neon for instant PITR via branching + +If you’re using Neon, you have some special tools 🛠️ for database recovery, derived from the unique architecture of Neon. First, [Neon’s storage is inherently durable](https://neon.tech/blog/get-page-at-lsn), since it implements an architecture that uses a combination of safekeepers and pageservers running in high-performance SSDs with cloud object storage to ensure data durability. + +Second, Neon offers a different approach to recovery: [you can run instant PITR via database branching.](https://neon.tech/blog/announcing-point-in-time-restore) Since Neon’s storage is [brancheable](https://neon.tech/docs/guides/branch-restore), instead of relying on a slow recovery-from-backup process or on expensive standby replicas, you simply create a [database branch](https://neon.tech/docs/guides/branch-restore) from any time in the past within the project’s [history retention window](https://neon.tech/docs/manage/projects#configure-history-retention). + +Running this PITR in Neon doesn’t incur additional costs, and **it’s an extremely fast process regardless of how large your dataset is**. [You can even run queries before restoring to a precise point in the past, to make sure you’re restoring to the right timestamp.](https://neon.tech/docs/guides/time-travel-assist) If this has picked your curiosity, watch this demo to see it in action: + + + +## From nightmares to sweet dreams + +When you’re getting your project to production, knowing you have a quick data recovery plan in case something goes wrong will help you sleep at night. In Neon, that’s instant PITR – we hope you never have to use it though 🙂 + +If you’re concerned about a particular scenario, [don’t hesitate to reach out to us at the Neon team](https://discord.gg/92vNTzKDGp). diff --git a/content/blog/posts/database-testing-with-fixtures-and-seeding.md b/content/blog/posts/database-testing-with-fixtures-and-seeding.md new file mode 100644 index 0000000000..c02c2a2db0 --- /dev/null +++ b/content/blog/posts/database-testing-with-fixtures-and-seeding.md @@ -0,0 +1,376 @@ +--- +title: Database testing with fixtures and seeding +description: >- + There's a catch with testing a new database: how do you test something that's + empty? +excerpt: >- + To test a new database, you need data, and not just any data. You need + consistent, reliable data that behaves the same way every time you run your + tests. Otherwise, how can you trust that your database and code is working + correctly? This is where the twin concepts of seeding and... +date: '2024-07-02T15:57:39' +updatedOn: '2024-07-02T15:57:40' +category: workflows +categories: + - workflows +authors: + - rishi-raj-jain +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-testing-with-fixtures-and-seeding/cover.jpg + alt: null +isFeatured: false +seo: + title: Database testing with fixtures and seeding - Neon + description: >- + To test a new database, you need testing data that behaves consistently. + This is where the concepts of seeding and fixtures come into play. + keywords: [] + noindex: false + ogTitle: Database testing with fixtures and seeding - Neon + ogDescription: >- + To test a new database, you need testing data that behaves consistently. + This is where the concepts of seeding and fixtures come into play. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/database-testing-with-fixtures-and-seeding/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-testing-with-fixtures-and-seeding/neon-database-testing-1-1024x576-40cc951e.jpg) + +To test a new database, you need data, and not just any data. You need consistent, reliable data that behaves the same way every time you run your tests. Otherwise, how can you trust that your database and code is working correctly? This is where the twin concepts of seeding and fixtures come into play, each serving a role in populating your database for different purposes. One provides the foundation your application needs to function; the other sets up specific scenarios for your tests to validate. + +Let’s take a closer look at each and explain why you should set up both for robust database testing. + +## Seed your database with the data you’ll always need + +Static seeding is where you insert predefined, fixed (i.e., “static”) data into your database. This can be helpful in two ways.
First, database initialization. If you’ve ever used `createsuperuser` in Django, you are statically seeding your database. You need an initial admin role for the deployment to work correctly. But you can also bucket in the broader set of data that you’ll need for your application here. Let’s say you have a dropdown for users to enter their country and state. You also need to seed this information into the database. How are you going to do that? In Python, it’s pretty straightforward: + +```python +from your_app.models import Country, State + +def seed_countries_and_states(): + # Seed countries + usa = Country.objects.create(name="United States") + canada = Country.objects.create(name="Canada") + + # Seed states for USA + us_states = [ + "Alabama", "Alaska", "Arizona", "Arkansas", "California", + # ... other states ... + "Wisconsin", "Wyoming" + ] + for state_name in us_states: + State.objects.create(name=state_name, country=usa) + + # Seed provinces for Canada + canadian_provinces = [ + "Alberta", "British Columbia", "Manitoba", "New Brunswick", + "Newfoundland and Labrador", "Nova Scotia", "Ontario", + "Prince Edward Island", "Quebec", "Saskatchewan" + ] + for province_name in canadian_provinces: + State.objects.create(name=province_name, country=canada) + +# Call this function in your Django management command or migration +seed_countries_and_states() +``` + +In Ruby on Rails, it is even easier thanks to `rails db:seed`: + +```java +# db/seeds.rb +require 'csv' + +# Ensure we have the necessary models +# app/models/country.rb +class Country < ApplicationRecord + has_many:states +end + +# app/models/state.rb +class State < ApplicationRecord + belongs_to:country +end + +puts "Seeding countries and states..." + +# Read countries from CSV +countries = {} +CSV.foreach(Rails.root.join('db', 'seed_data', 'countries.csv'), headers: true) do |row| + country = Country.create!(name: row ['name']) + countries [row ['code']] = country +end + +# Read states/provinces from CSV +CSV.foreach(Rails.root.join('db', 'seed_data', 'states.csv'), headers: true) do |row| + country = countries [row ['country_code']] + country.states.create!(name: row ['name']) +end + +puts "Seeding completed!" + +# To run this seed file, use: +# rails db:seed +``` + +To run this seed file, you then use: + +```bash +rails db:seed +``` + +The fact that Rails has a command expressly for this shows the importance of seeding. + +The second reason you’ll need seeding is testing. By seeding your test database with a known set of data, you can write predictable and repeatable tests. + +Let’s say you’re building an e-commerce platform. You might want to seed your test database with various products, users, and orders to test different scenarios. Here’s how you might approach this in Rails: + +```java +# db/seeds/test_seeds.rb + +puts "Seeding test data..." + +# Create test users +alice = User.create!(username: 'alice', email: 'alice@example.com', password: 'password123') +bob = User.create!(username: 'bob', email: 'bob@example.com', password: 'password456') + +# Create some products +laptop = Product.create!(name: "Laptop", price: 999.99, stock: 10) +phone = Product.create!(name: "Smartphone", price: 499.99, stock: 20) +headphones = Product.create!(name: "Wireless Headphones", price: 149.99, stock: 30) + +# Create some orders +Order.create!(user: alice, product: laptop, quantity: 1) +Order.create!(user: bob, product: phone, quantity: 2) +Order.create!(user: alice, product: headphones, quantity: 1) + +puts "Test data seeded successfully!" +``` + +You can then load this test seed data in your test setup: + +```java +# In your test helper or spec helper +Rails.application.load_seed +load "#{Rails.root}/db/seeds/test_seeds.rb" +``` + +By using static seeding for your tests, you ensure that they work with the same set of data every time they run. This makes your tests more reliable and easier to reason about. If a test fails, you know it’s because of a change in your code, not because of a difference in the test data. + +When creating your static seeding data, some key aspects to consider are: + +1. Deterministic data: The seed data is pre-determined and doesn’t change unless explicitly modified in the seed script or configuration. +2. Version control: Seed data is stored in version-controlled files (e.g., JSON, YAML, CSV, or SQL scripts) alongside the application code. +3. Idempotency: Well-designed seed scripts are idempotent, meaning they can be run multiple times without changing the result beyond the initial application. +4. Environment independence: Static seeds provide consistent data across different environments (development, staging, testing), ensuring reproducibility. + +If we’re going to the trouble of naming this “static” seeding, does that suggest there is also a dynamic version? Yes. + +Dynamic seeding involves generating random or varied data. This can be useful for stress testing or uncovering edge cases you might not have thought of with your hand-crafted static data. An example might be generating users with varied characteristics or creating products with random prices and stock levels. This approach can help simulate more realistic scenarios and potentially uncover issues that might not be apparent with a small, fixed dataset. + +For this, you can use libraries such as [Faker](https://faker.readthedocs.io/en/master/) to generate data on the fly: + +```python +from django.contrib.auth.models import User +from your_app.models import Product, Order +from faker import Faker +from random import choice, randint +from decimal import Decimal + +# Initialize Faker +fake = Faker() + +def seed_dynamic_data(num_users=50, num_products=100, num_orders=200): + # Create users + users = [] + for _ in range(num_users): + user = User.objects.create_user( + username=fake.user_name(), + email=fake.email(), + password=fake.password(length=12) + ) + user.first_name = fake.first_name() + user.last_name = fake.last_name() + user.save() + users.append(user) + + # Create products + products = [] + for _ in range(num_products): + product = Product.objects.create( + name=fake.catch_phrase(), + description=fake.paragraph(), + price=Decimal(fake.pydecimal(left_digits=3, right_digits=2, positive=True)), + stock=randint(0, 100) + ) + products.append(product) + + # Create orders + for _ in range(num_orders): + user = choice(users) + product = choice(products) + quantity = randint(1, 5) + + Order.objects.create( + user=user, + product=product, + quantity=quantity, + order_date=fake.date_time_this_year() + ) + + print(f"Created {num_users} users, {num_products} products, and {num_orders} orders.") + +# Usage +if __name__ == "__main__": + seed_dynamic_data() +``` + +Dynamic seeding offers several benefits: + +1. Scalability testing: You can easily generate large amounts of data to test how your application performs under load. +2. Edge case discovery: Random data may uncover scenarios you didn’t anticipate when writing your static seeds. +3. Realistic data distribution: Adjusting the random generators to mimic real-world distributions can create more realistic test scenarios. +4. Stress testing: You can generate extreme cases (like products with zero stock or very high prices) to ensure your application handles them correctly. + +Use static seeds for core data your application needs to function and dynamic seeds for additional data to test various scenarios. + +## Use fixtures to test the specifics of your data model + +Here’s a fixture: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/database-testing-with-fixtures-and-seeding/ad4nxftstzgkqc7fycr6wcfwflscstw1cilnstghxzseraop3sdmjekfjvlpve8oe1rpdxufeekqktxskzd6hxbcl0rlssvlpbtiyz3vsktwu9zbkx7ghs-o1xadczzjj9uwffgezhcmlqqqza8osiayke4u-c5b9b7a5.jpg) + +In engineering (the physical kind), fixtures securely hold a workpiece in a precise position during testing. They ” fix” them, ensuring consistency and repeatability. + +This is effectively what fixtures are in database testing. They are a fixed state of a set of data for consistent test execution. + +- Whereas static seeding can be used for testing but is more broadly about adding startup data to your application, fixtures are a specific testing concept that allows you to create a known, consistent environment for running tests. +- Whereas static seeds are loaded from JSONs or CSVs and are implemented via scripts (like above) or database migrations, fixtures are usually integrated into testing frameworks and defined in specific formats supported by the testing tool. +- Whereas the point of seeding is to persist data in your database, fixtures are usually loaded at the beginning of a test and cleaned up after. + +Some examples: + +1. In Django (Python), fixtures are typically defined in JSON or YAML files and can be loaded using the loaddata management command or within test cases. +2. In Rails (Ruby), fixtures are defined in YAML files and are automatically loaded for tests unless configured otherwise. +3. In Jest (JavaScript), fixtures can be defined as JavaScript objects or loaded from JSON files using Jest’s require function. +4. In JUnit (Java), fixtures are often defined using annotations like @BeforeEach or @BeforeAll to set up test data. +5. In NUnit (C#), fixtures can be set up using the [SetUp] attribute on methods that prepare the test environment. + +Using fixtures in your tests allows you to ensure consistency across test runs, isolate tests from each other by providing a fresh set of data for each test, and improve test readability by separating test data from test logic. + +Let’s say we’re working in Django. Then, a fixture definition in YAML might look like this: + +```javascript +# products.yaml +- model: myapp. Product + pk: 1 + fields: + name: Laptop + price: 999.99 + description: A high-performance laptop + +- model: myapp. Product + pk: 2 + fields: + name: Smartphone + price: 499.99 + description: The latest smartphone model + +# users.yaml +- model: auth. User + pk: 1 + fields: + username: johndoe + email: john@example.com + password: pbkdf2_sha256$... # This should be a hashed password + +- model: auth. User + pk: 2 + fields: + username: janedoe + email: jane@example.com + password: pbkdf2_sha256$... # This should be a hashed password + +# orders.yaml +- model: myapp. Order + pk: 1 + fields: + user: 1 + product: 1 + quantity: 2 + order_date: "2023-06-26T10:00:00Z" + +- model: myapp. Order + pk: 2 + fields: + user: 2 + product: 2 + quantity: 1 + order_date: "2023-06-26T11:00:00Z" +``` + +We can then use that data in our tests.py: + +```python +# tests.py + +from django.test import TestCase +from django.contrib.auth.models import User +from myapp.models import Product, Order + +class OrderTestCase(TestCase): + fixtures = ['products.yaml', 'users.yaml', 'orders.yaml'] + + def test_order_creation(self): + # Check if fixtures are loaded correctly + self.assertEqual(Product.objects.count(), 2) + self.assertEqual(User.objects.count(), 2) + self.assertEqual(Order.objects.count(), 2) + + # Test specific order details + order = Order.objects.get(pk=1) + self.assertEqual(order.user.username, 'johndoe') + self.assertEqual(order.product.name, 'Laptop') + self.assertEqual(order.quantity, 2) + + def test_total_order_value(self): + order = Order.objects.get(pk=1) + expected_total = order.product.price * order.quantity + self.assertEqual(order.total_value(), expected_total) + + def test_user_order_count(self): + user = User.objects.get(username='johndoe') + self.assertEqual(user.order_set.count(), 1) +``` + +If all is good, this returns: + +```bash +Creating test database for alias 'default'... +System check identified no issues (0 silenced). +... +---------------------------------------------------------------------- +Ran 3 tests in 0.789s + +OK +Destroying test database for alias 'default'... +``` + +The subtle differences between seeding and fixtures are starting to come to light. Seeding typically provides initial data for an application to function correctly in various environments. In contrast, fixtures are specifically designed for testing scenarios, providing a controlled and consistent dataset that can be easily reset between test runs. + +In practice, the line between seeding and fixtures can sometimes blur, especially in development environments. Some teams might use the terms interchangeably. However, understanding the nuanced differences can help choose the right approach for different scenarios in your development and testing processes. + +## Branching: the next level of database testing + +While seeding and fixtures are great tools for populating your database with consistent data, [database branching](https://neon.tech/docs/introduction/branching) in Neon offers a powerful UX enhancement to these practices. + +1. **Instantaneous data cloning.** Branching allows for a quick copy of your database. You create a branch (available immediately) and run your tests. If anything goes wrong, you can simply reset the branch to its original state, restore it to a point in time, or start afresh. +2. **Isolated test environments.** Branches act like isolated environments: you can create a separate branch of your database for each feature you’re developing or bug you’re fixing. +3. **Parallel development.** Multiple branches allow different team members to work on various features or fixes simultaneously. +4. **Effortless rollbacks.** If you get to an unexpected state, you can quickly revert to a previous state. This ability to roll back changes provides a safety net, making your development process more robust. + +## Seeds, Fixtures, and Everything in Between + +Database testing is challenging. This is one of the reasons we created [branching](https://neon.tech/docs/introduction/branching) in Neon: to give database developers more maneuverability when testing new databases, schema changes, and queries. + +But seeding and fixtures, and the tooling around them, are ideal starting points for robust database design. Seeding sets the stage with your app’s essential data, while fixtures give you a focused test environment. diff --git a/content/blog/posts/databuddy-is-open-sourcing-privacy-first-analytics-built-on-neon.md b/content/blog/posts/databuddy-is-open-sourcing-privacy-first-analytics-built-on-neon.md new file mode 100644 index 0000000000..f7b2fad0ac --- /dev/null +++ b/content/blog/posts/databuddy-is-open-sourcing-privacy-first-analytics-built-on-neon.md @@ -0,0 +1,88 @@ +--- +title: 'Databuddy Is Open-Sourcing Privacy-First Analytics, Built on Neon' +description: 'A clean, open stack for analytics' +excerpt: >- + “I was surprised by how fast Neon was. It was faster than my self-hosted setup + and Prisma Postgres. This plus the convenience of the Free Plan makes it a + no-brainer for building your projects.” (Issa Nassar, founder of Databuddy) + Databuddy is a new privacy-first web and product a... +date: '2025-08-28T15:50:25' +updatedOn: '2025-08-28T15:50:26' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/databuddy-is-open-sourcing-privacy-first-analytics-built-on-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Databuddy Is Open-Sourcing Privacy-First Analytics, Built on Neon - Neon' + description: >- + Databuddy is a new privacy-first web and product analytics platform that + balances power and simplicity. It's open-source and built on Neon. + keywords: [] + noindex: false + ogTitle: 'Databuddy Is Open-Sourcing Privacy-First Analytics, Built on Neon - Neon' + ogDescription: >- + Databuddy is a new privacy-first web and product analytics platform that + balances power and simplicity. It's open-source and built on Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/databuddy-is-open-sourcing-privacy-first-analytics-built-on-neon/social.jpg +--- + +
+ +
+ +
+

“I was surprised by how fast Neon was. It was faster than my self-hosted setup and Prisma Postgres. This plus the convenience of the Free Plan makes it a no-brainer for building your projects.” (Issa Nassar, founder of Databuddy)

+
+ +[Databuddy](https://www.databuddy.cc/) is a new privacy-first web and product analytics platform that strikes the perfect balance between power and simplicity. Inspired by tools like Plausible and PostHog, it’s built to be more developer-friendly and fully open source, solving a common dilemma for small teams choosing analytics tools – everything feels either too simple or too complex. Databuddy aims to get it just right. [You can use it for free via their Free Plan.](https://app.databuddy.cc/login) + +
+ +
Try it out at https://www.databuddy.cc/
+
+ +## Building on Neon: Top Postgres Performance Without the Hassle + +
+

“I’ve been through the pain of self-hosting: setting everything up, tuning PgBouncer, dealing with connection limits. Neon gives me great performance without the maintenance” (Issa Nassar, founder of Databuddy)

+
+ +Issa first got familiar with [Neon](https://neon.com/) during a hackathon project, loving how easy it was to spin up multiple projects under the [Free Plan](https://neon.com/pricing). When it came time to build Databuddy, he also explored many Postgres options, including Supabase and Prisma Postgres. But here’s what made the difference in Neon’s favor: + +- **Composable stack.** Tools like Supabase are experts in bundling multiple tools into a vertically integrated stack, while Neon’s main focus is to do one thing well (serverless Postgres) and to stay as composable as possible. That philosophy aligns with how Issa approaches his stack. +- **Simplicity.** Issa had experience running Postgres himself: provisioning machines, wiring in PgBouncer for connection pooling, managing daily backups. With Neon, all of that just worked out of the box. +- **Performance.** Neon wasn’t just easy to use – it was fast. Issa was surprised to find that Neon even outperformed his own self-hosted Postgres setup. It’s possible this might have come down to a misconfiguration, but that’s the benefit of Neon: you don’t have to debug it, it just works. + + +We sponsor promising open source projects that start on our Free Plan, increasing their resource limits and helping them grow. If you’re building in public, scaling, and could use a little headroom, apply [here](https://forms.gle/nP9FQ3jKe7t1NPgu9). + + +## Databuddy’s Architecture And Tech Stack + +_You can check out all of Databuddy’s code_ [in this repo.](https://github.com/databuddy-analytics/Databuddy) + +Let’s take a closer look at Databuddy’s composable stack: + +- **Frontend:** Built with Next.js, styled with Tailwind CSS, and powered by Bun for blazing-fast builds and dev workflow. +- **API layer:** A self-hosted backend using Elysia and tRPC. +- **ORM:** Drizzle handles all database interactions with Neon, using pooled connections for performance and efficiency. +- **Database layer:** + - Neon stores all structured metadata and takes care of the low-latency relational operations. + - ClickHouse stores raw events and analytics data at scale, taking care of the heavy analytical queries. +- **Auth & routing logic:** Every tracked website is assigned a unique website_id. Incoming events are first verified against Neon to confirm ownership and then passed on to ClickHouse. +- **Dev tooling:** Bun drives the entire workflow – compilation, testing, database migrations, SDK builds, even running in the docker containers + +
+

“This is a really simple setup that works. Neon handles the transactional side, ClickHouse handles the heavy lifting, and I get to choose exactly how everything else fits together”  (Issa Nassar, founder of Databuddy)

+
+ +## Start Building + +[Databuddy](https://www.databuddy.cc/) is open source and growing fast. Give the project a ⭐️ on [GitHub](https://github.com/databuddy-analytics/Databuddy), follow [@izadoesdev](https://x.com/izadoesdev) for updates, and [spin up a Neon database](https://console.neon.tech/signup) to start building your own stack. diff --git a/content/blog/posts/databutton-neon-integration.md b/content/blog/posts/databutton-neon-integration.md new file mode 100644 index 0000000000..92eacdf890 --- /dev/null +++ b/content/blog/posts/databutton-neon-integration.md @@ -0,0 +1,121 @@ +--- +title: 'Databutton Just Made Their Agent Smarter, with Postgres and Auth Built In' +description: >- + Every Databutton app now ships with production-ready infrastructure, no setup + required. Powered by Neon +excerpt: >- + “Integrating Neon was a no-brainer. It gives every Databutton app a + production-grade Postgres database in seconds, with zero overhead. Our AI + agent can now create, manage, and debug the entire stack, not just code” + (Martin Skow Røed, CTO and co-founder of Databutton) Databutton i... +date: '2025-06-19T15:21:55' +updatedOn: '2025-10-24T18:43:50' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/databutton-neon-integration/cover.jpg + alt: null +isFeatured: true +seo: + title: >- + Databutton Just Made Their Agent Smarter, with Postgres and Auth Built In - + Neon + description: >- + Databutton is a vibe coding platform that helps you deploy full-stack apps + with zero code. Now with Postgres and Auth powered by Neon. + keywords: [] + noindex: false + ogTitle: >- + Databutton Just Made Their Agent Smarter, with Postgres and Auth Built In - + Neon + ogDescription: >- + Databutton is a vibe coding platform that helps you deploy full-stack apps + with zero code. Now with Postgres and Auth powered by Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/databutton-neon-integration/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/databutton-neon-integration/neon-databutton-1024x576-52821599.jpg) + + +On October 23 2025, Databutton became Riff. New branding, same full-stack capabilities: [riff.new](https://riff.new/) + + +**“Integrating Neon was a no-brainer. It gives every Databutton app a production-grade Postgres database in seconds, with zero overhead. Our AI agent can now create, manage, and debug the entire stack, not just code”** ([Martin Skow Røed](https://www.linkedin.com/in/martinsroed/overlay/about-this-profile/), CTO and co-founder of Databutton) + +[Databutton](https://databutton.com/) is a vibe coding platform that helps anyone create and deploy full-stack web applications with zero code. At its heart is the [Databutton AI Agent](https://docs.databutton.com/getting-started/meet-the-databutton-ai-agent), an intelligent assistant that writes React frontends, FastAPI backends, tests your code, debugs errors, and manages secrets all guided by your prompts. + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + + +## Postgres and Auth Built Right Into Every Databutton App + +Databutton just made their Agent a whole lot more powerful. As of today, every new app created on the platform ships with a fully integrated Postgres database and built-in user authentication, with no configuration or external services required. + + + +This launch (powered by Neon) eliminates one of the biggest sources of friction for Databutton users. Previously, setting up data persistence and authentication meant wiring up third-party tools like Supabase. That added complexity, extra configuration steps, and a significant number of support tickets, creating overhead for the Databutton team and a frustrating experience for users. + +Now, that complexity is gone. Instead, every new Databutton app includes: + +- **Out-of-the-box data persistence:** A native Postgres database is provisioned instantly with each new app, no setup required – your data is stored and ready to use from the start. +- **User authentication:** Built-in login, signup, and user management flows are now part of every app. There’s no need to wire up external auth services or handle tokens manually. +- **Database management handled by AI:** The Databutton AI Agent can now take care of schema design, data model migrations, and even fix common issues, giving non-technical builders a faster path to working apps with less troubleshooting. + +This change doesn’t just simplify setup, it makes the agent much more capable – and Databutton users are already putting it to work. Some recent examples built with the new native stack include: + +- A customer feedback dashboard that aggregates reviews across platforms +- A real-time sentiment monitor for social media, complete with alerts +- An inventory management system that tracks stock levels and auto-generates purchase orders + +Check out Databutton’s [templates](https://databutton.com/templates) for more inspiration on what to build. + +## Behind the Scenes: Infra for Agents, Powered by Neon + +While Databutton users won’t see it (_and that’s the magic_), the infrastructure behind this launch is powered by [Neon](https://neon.com/). + +[Neon](https://neon.com/) is serverless Postgres that provisions in seconds, scales automatically, and is managed entirely through API. It’s purpose-built for platforms like Databutton, where every user or app needs its own isolated database but without the high cost or operational burden that typically comes with managing fleet-scale infrastructure in the cloud. + +Every time someone creates a new Databutton app, Neon silently spins up a dedicated Postgres project behind the scenes, complete with its own storage, compute, and connection strings. The setup is fully automated and invisible to the user, but the infrastructure is real, production-grade, and ready from the first query. + +### Postgres databases that spin up in a second, and scale to zero + +Neon separates storage and compute, which means databases can be provisioned in under a second, [suspended automatically when idle](https://neon.com/docs/introduction/scale-to-zero), and resumed in milliseconds. For Databutton, this enables a [one-database-per-app model](https://neon.com/use-cases/database-per-tenant) without incurring the typical overhead of managing thousands of long-lived instances. + +Most platforms can’t afford to give every user a dedicated cloud instance. With Neon, that’s not only possible but highly efficient. Apps that sit idle don’t burn compute, and active apps scale on demand. It’s a perfect match for agent-native workflows, where infrastructure needs to appear instantly, do its job, and get out of the way. + +### Infrastructure that the agent can control + +But what also makes this launch so powerful isn’t just that every app has Postgres – it’s that Databutton’s AI agent can now control it directly. Thanks to [Neon’s developer-first API](https://neon.com/blog/provision-postgres-neon-api) and 100% Postgres compatibility, the agent can: + +- Create and evolve schemas as your app changes +- Run data migrations automatically +- Fix database-related issues in context, without human help +- Handle everything from first query to final deployment + +### Built for agent platforms at scale + +The Neon API also gives Databutton full control over provisioning, usage, and cost management. + +- Databases are isolated by default (one Neon project per app) +- Quotas can be defined per project for storage, compute, and write limits +- Inactive databases are automatically suspended to save on cost +- Usage can be tracked in real time, feeding internal dashboards or usage-based billing +- If a user wants to take their data with them, Databutton can transfer ownership of the Neon project in one API call + +This is what lets Databutton ship serious infrastructure without hiring a DevOps team. It’s also why [agent platforms keep choosing Neon as their backend engine](https://neon.com/blog/replit-app-history-powered-by-neon-branches). + +## Start Building + +If you’ve been waiting for a faster way to build full-stack apps, now’s the time to try Databutton. Get started at [databutton.com](https://databutton.com), describe what you want to build, and let the agent do the rest. + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + diff --git a/content/blog/posts/debugging-with-cursor.md b/content/blog/posts/debugging-with-cursor.md new file mode 100644 index 0000000000..975678b35b --- /dev/null +++ b/content/blog/posts/debugging-with-cursor.md @@ -0,0 +1,68 @@ +--- +title: Debugging with Cursor as Your Rubber Ducky +description: Conversational Debugging in Your IDE +excerpt: >- + Have you ever explained a bug or issue you’re having to someone, only to + discover the solution mid-conversation, as if simply describing the problem + helped you catch your own mistake? This is actually quite common and has been + used by software engineers for decades, with the offi... +date: '2025-06-20T16:44:12' +updatedOn: '2025-07-01T18:39:57' +category: ai +categories: + - ai +authors: + - sam-harrison +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/debugging-with-cursor/cover.jpg + alt: null +isFeatured: false +seo: + title: Debugging with Cursor as Your Rubber Ducky - Neon + description: >- + With codebase indexing as their strength, AI-integrated IDEs like Cursor + have great skills as a debugging partner. + keywords: [] + noindex: false + ogTitle: Debugging with Cursor as Your Rubber Ducky - Neon + ogDescription: >- + With codebase indexing as their strength, AI-integrated IDEs like Cursor + have great skills as a debugging partner. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/debugging-with-cursor/social.jpg +--- + +
+Post image +
+ +Have you ever explained a bug or issue you’re having to someone, only to discover the solution mid-conversation, as if simply describing the problem helped you catch your own mistake? This is actually quite common and has been used by software engineers for decades, with the official name of **rubber duck debugging**. + +By explaining your issue, even to an inanimate object, you’re forced to evaluate your assumptions and logic from a different perspective. Discussing your problem with another person can add even more value, since they might ask basic questions that help uncover overlooked assumptions and can keep the conversation going. Ideally, you’d always talk to a colleague familiar with the domain, but interrupting a coworker every time you hit a snag won’t make you very popular in the office. + +## Extending the Rubber Duck Experience in Cursor + +This is where I think AI-integrated IDEs, like [Cursor](https://www.cursor.com/en), have a very strong use case as a debugging partner. Cursor can act like a better rubber ducky, giving you more than the silent receptiveness of an inanimate object. With the strength of the codebase indexing, it can highlight logical gaps or misinterpretations of your own code, or ask meaningful follow-up questions that help you think more deeply about the issue. + +This approach is significantly better than the usual AI debugging methods, like copy-pasting the error message and instantly applying the response, for many reasons. In the short term, you might solve the issue 9/10 times using a “send-and-pray” strategy, but you can quickly lose sight of your codebase if you apply fixes you don’t understand, making subsequent change and future debugging harder as the amount of blindly accepted code increases. + +[More importantly, however, you will slowly lose the ability to learn, effectively solve problems, and critically think.](https://www.microsoft.com/en-us/research/wp-content/uploads/2025/01/lee_2025_ai_critical_thinking_survey.pdf) By having the LLM act as a coworker that probes your understanding and guides your line of questioning, you can reap the benefits of an integrated IDE without chipping away at the skills you rely on for more complex problems. + +When compared to more structured AI-driven debugging, where you would provide the full error stack, relevant code snippets, and detailed context, I’d argue the conversational approach still comes out ahead. With complex bugs, these structured reports are typically still not enough, and lead to incorrect recommendations. By contrast, having a conversation gradually builds quality context around the issue, ultimately improving both your understanding and the suggestions provided by Cursor. + +## Productivity Gains + +Given the frequency at which debugging occurs, rewriting almost the same prompt each time isn’t very productive. Instead of repeatedly prompting the agent manually, I created a Cursor ruleset in a .cursor/rules/rubberducky.mdc file. Then, when I wanted to initiate a session, I could reference it with @rubberducky.mdc, and the conversation would start according to my predefined standards. This also has the added benefit of making sessions more consistent, allowing me to tweak the prompt over time. + +Another huge help is setting up voice-to-text (like WisprFlow or SuperWhisper), so you can speak to the agent rather than needing to type ideas out. To me, speaking aloud makes the interaction more natural, improves my thought clarity, and gives a much higher words-per-minute throughput compared to typing. + +Here, it’s as simple as opening the agent tab, pulling in your ruleset, and off you go. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/debugging-with-cursor/ad4nxfmvnfhxlbr440p5bh5fxskryvbobuqyl66kv452af8q6xog6ruz98zvciwdcloui3k45ksdabjc2ldixhg1bu0qqnqcgbcmdz2gzlj1gb-req2st8eajwe5a0nvejwikvxv1v-a56e7d53.gif) + +## Rubber Ducking Isn’t Dead, It’s Just Upgraded + +Obviously, not every issue requires a full blown conversation, and there’s a time and a place for simply throwing the error into the agent chatbox, briefly reading the response, and applying it if it seems right. + +You shouldn’t, however, only do this. Rubber ducky debugging helps build quality context, deepens your understanding of the changes you’re making, and prepares you to tackle more difficult issues. diff --git a/content/blog/posts/delayed-start-compute-operations-triggering-event.md b/content/blog/posts/delayed-start-compute-operations-triggering-event.md new file mode 100644 index 0000000000..d7d24fd573 --- /dev/null +++ b/content/blog/posts/delayed-start-compute-operations-triggering-event.md @@ -0,0 +1,96 @@ +--- +title: Delayed Start Compute Operations – Triggering Event +description: >- + This post is part of a series discussing the Neon outages on 2025-05-16 and + 2025-05-19 in the AWS us-east-1 region. In this post, we cover the triggering + cause of the outage, a change in a Postgres execution plan that ultimately + caused idle Computes to be unable to be suspended. +excerpt: >- + For further details, read the top-level Post-Mortem. Summary The Neon Control + Plane service is backed by a Postgres database. A scheduled job in the Control + plane, Activity Monitor, is responsible for identifying Computes that are + ready to be suspended. A Postgres query executed... +date: '2025-05-30T20:24:56' +updatedOn: '2025-05-30T21:10:48' +category: engineering +categories: + - engineering +authors: + - mihai-bojin + - matt-sherman +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/delayed-start-compute-operations-triggering-event/social.png + alt: null +isFeatured: false +seo: + title: Delayed Start Compute Operations - Triggering Event - Neon + description: >- + This post is part of a series discussing the Neon outages on 2025-05-16 and + 2025-05-19 in the AWS us-east-1 region. In this post, we cover the + triggering cause of the outage, a change in a Postgres execution plan that + ultimately caused idle Computes to be unable to be suspended. + keywords: [] + noindex: false + ogTitle: Delayed Start Compute Operations - Triggering Event - Neon + ogDescription: >- + For further details, read the top-level Post-Mortem. Summary The Neon + Control Plane service is backed by a Postgres database. A scheduled job in + the Control plane, Activity Monitor, is responsible for identifying Computes + that are ready to be suspended. A Postgres query executed by this job + against the region’s control plane database changed its execution […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/delayed-start-compute-operations-triggering-event/social.png +--- + +_For further details, read the_ [top-level Post-Mortem](https://neon.com/blog/postmortem-delayed-start-compute-operations)_._ + +# Summary + +The Neon Control Plane service is backed by a Postgres database. A scheduled job in the Control plane, _Activity Monitor,_ is responsible for identifying Computes that are ready to be suspended. A Postgres query executed by this job against the region’s control plane database changed its [execution plan](https://www.postgresql.org/docs/current/using-explain.html). + +Different execution plans can cause dramatic performance changes – in this case, using a suboptimal index – which in turn can make a query run slower and consume more resources than before. In turn, this led to database CPU saturation, and caused the _Activity Monitor_ to be unable to suspend _Compute_ VMs, ultimately resulting in a significant increase in concurrently running _Computes_, above the cluster’s planned capacity of 6,000, but below the tested ceiling of 10,000 VMs. + +This resulted in IP Allocation failures – the specifics of which are described in the [top-level Postmortem](https://neon.com/blog/postmortem-delayed-start-compute-operations). + +# Architecture + +Behind the Neon Console is the Neon Control Plane service. It is a regional Golang service which orchestrates many supporting functions, but primarily it starts and suspends Neon Postgres VMs, which we call “_Computes_”. Neon’s Serverless architecture requires the Control Plane to start _Computes_ when customers execute SQL statements, and suspend them if there are no statements executed for a (configurable) period of time. We call this functionality, [scale-to-zero](https://neon.com/docs/guides/scale-to-zero-guide). + +The mechanism that suspends _Computes_ is called the “_Activity Monitor_”, a scheduled job running inside the Neon Control Plane that evaluates running computes, and identifies those which can be suspended. + +## Chain of events + +The _Activity Monitor_ relies on a Postgres SQL query, which contains several joins, including one against the `computes` table. The `computes` table contains rows for every _Compute_ which has been started or made idle. + +The triggering event for the 2025-05-16 outage was a change in Postgres execution plan for queries made by the Activity Monitor. This led to a broad scan on the `computes` table, causing CPU saturation on the control plane’s backing Postgres database. + +Due to the high CPU usage, all _fetchEndpointWithCompute_ queries’ execution times increased from a few hundred milliseconds to over 100 seconds. In turn, this resulted in the control plane becoming unable to suspend _Compute VMs_, which led to an increase in the number of concurrently running VMs, ultimately resulting in [IP exhaustion](https://neon.com/blog/postmortem-delayed-start-compute-operations) inside the cluster. + +The `computes` table has tens of millions of rows, as it holds a recent history of all start and suspend events of Neon _Computes_. However, the number of _active_ _Computes_ (rows) at any given moment is much lower. + +When execution plans use the correct indexes, this table can be queried in milliseconds. However, if the wrong index is selected, its performance can fall off a cliff, scanning millions of pages to return a couple thousand rows. + +This change in execution plan didn’t uniformly happen in all regions, as the Control Plane and backing databases are region-isolated for resilience. As a result, each database’s planner independently decides the most efficient execution plan for each executed statement. The _us-east-1_ region was the first to be affected by this plan change. In parallel with mitigating the outage in that region, we have taken steps to prevent the same failure pattern from occurring in other regions in the future. + +During the incident, and in other regions afterwards: + +- We ran `ANALYZE computes`, which refreshed the statistics and brought the execution plan back to normal. +- We refactored _fetchEndpointWithCompute_ to wrap the potentially expensive sub-query in a [materialized CTE](https://www.postgresql.org/docs/current/queries-with.html#QUERIES-WITH-CTE-MATERIALIZATION). The CTE acts as an optimization fence, causing the planner to evaluate it once, use an appropriate index, and return a small set of relevant rows. This prevents the planner from occasionally choosing a suboptimal index and causing poor performance. + +
+Image +
By using a materialized CTE, we are seeing more stable results from the Postgres query planner.
+
+ +**What other improvements have we considered?** + +- We’re implementing a more active Garbage Collection strategy to make future regressions to this query plan less expensive. We expect this change to result in an order of magnitude reduction in rows in this table (from tens of millions, to millions of rows), reducing total index size and reducing the occurrence of dead tuples by enabling `AUTOVACUUM` to run more often. +- In the medium-term, we intend to move all historical data outside of the ‘hot’ control plane OLTP backing database into an OLAP system. +- We have considered moving the _Activity Monitor_ to read from a Postgres read replica. Unfortunately, this is not a viable option due to its need for reading consistent (not stale) results and making transactional writes. + +# Next steps + +One item we’re currently working on is understanding the exact conditions that lead to a change in execution plan. We observed different behaviours by region, depending on the shape of the `computes` table in each backing database. + +We will publish further findings, once we have concluded this investigation. diff --git a/content/blog/posts/deno-2-0-is-here-and-its-scaly.md b/content/blog/posts/deno-2-0-is-here-and-its-scaly.md new file mode 100644 index 0000000000..8b1cfa37bb --- /dev/null +++ b/content/blog/posts/deno-2-0-is-here-and-its-scaly.md @@ -0,0 +1,224 @@ +--- +title: 'Deno 2.0 Is Here, and It’s Scaly' +description: Offering backward compatibility with Node.js and npm +excerpt: >- + Deno is now six years old. Built by Node.js creator Ryan Dahl to address what + he saw as the mistakes of Node, the motivation behind Deno was simple. Node.js + was too complex, too insecure, and had drifted away from JavaScript in how it + worked. Deno would align with ECMAScript and... +date: '2024-10-15T16:00:19' +updatedOn: '2024-10-15T16:00:22' +category: community +categories: + - community +authors: + - andrew-tate +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deno-2-0-is-here-and-its-scaly/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Deno 2.0 Is Here, and It’s Scaly - Neon' + description: >- + Deno 2.0 brings Node.js and npm compatibility directly into Deno: developers + can keep using their favorites libraries and frameworks. + keywords: [] + noindex: false + ogTitle: 'Deno 2.0 Is Here, and It’s Scaly - Neon' + ogDescription: >- + Deno 2.0 brings Node.js and npm compatibility directly into Deno: developers + can keep using their favorites libraries and frameworks. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deno-2-0-is-here-and-its-scaly/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/deno-2-0-is-here-and-its-scaly/neon-deno-1-1024x576-71147cfe.jpg) + +Deno is now six years old. Built by Node.js creator Ryan Dahl to address what he saw as the mistakes of Node, the motivation behind Deno was simple. Node.js was too complex, too insecure, and had drifted away from JavaScript in how it worked. Deno would align with ECMAScript and be secure and simple by default. + +By this definition, Deno has been an enormous success. But, it has struggled to gain widespread adoption in the JavaScript ecosystem. Despite its improvements over Node.js in terms of security and simplicity, developers and companies have remained hesitant to switch from the well-established Node.js environment where their entire code lives, especially without any compatibility between the two ecosystems. + +That is changing with [Deno 2.0](https://deno.com/blog/v2.0-release-candidate). Deno 2.0 brings Node.js and npm compatibility directly into Deno, meaning developers can take advantage of the advances Deno has made within JS while still using the libraries and frameworks they’ve used for the past 15 years. + +## Deno 1: Simple & Modern + +If you’re using Node.js to connect to Neon, you might write something like this: + +```javascript +const { Client } = require("pg"); + +const databaseUrl = process.env.DATABASE_URL; +if (!databaseUrl) { + console.error("DATABASE_URL environment variable is not set"); + process.exit(1); +} + +const client = new Client({ + connectionString: databaseUrl, +}); + +async function main() { + try { + await client.connect(); + + const result = await client.query( + "SELECT name, value FROM playing_with_neon LIMIT 5" + ); + + console.log("result:", result.rows); + } catch (error) { + console.error("An error occurred:", error); + } finally { + await client.end(); + } +} + +main(); +``` + +You then run it with: + +```bash +node index.js +``` + +Node.js relies on npm and package.json for dependency management, which means as soon as you npm install a package, you get all this in your directory: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/deno-2-0-is-here-and-its-scaly/ad4nxc3vkobzcymlgtmlt2gsfs29hi1d9z42scw73bl1ovhzbf8s36kotyam19bc6weybvemmqlv34u2yq084a-o5cbanjrxu1m2qnbd4xv14-2u618llcwz3ujrccchawervmfldpuwb9vtonntvfteiemq-32175ead.png) + +Bear in mind we only installed the pg package–that’s it. A core difference between Node and Deno was these imports. In Deno 1.x, the code would have been similar because it is still JavaScript/TypeScript, but the import totally different: + +```javascript +import { Client } from "https://deno.land/x/postgres@v0.17.0/mod.ts"; + +const databaseUrl = Deno.env.get("DATABASE_URL"); +if (!databaseUrl) { + console.error("DATABASE_URL environment variable is not set"); + Deno.exit(1); +} + +const client = new Client(databaseUrl); + +async function main() { + try { + await client.connect(); + + const result = await client.queryObject<{ id: number; name: string }>( + "SELECT name, value FROM playing_with_neon LIMIT 5" + ); + + console.log("result:", result.rows); + } catch (error) { + console.error("An error occurred:", error); + } finally { + await client.end(); + } +} + +main(); +``` + +Deno 1.x uses direct URL imports, eliminating the need for a package manager. All that is in your directory is your code. This approach to dependency management significantly simplifies the process. No installs, no package.json, no node_modules folder. Dependencies are fetched and cached on the first run, ensuring that your project remains lightweight and portable. + +This streamlined approach offers several advantages: + +1. **Reduced project complexity**: Without the need for a package.json file or a node_modules folder, project structure remains clean and straightforward. +2. **Improved portability**: Since dependencies are specified directly in the code, sharing projects becomes easier. Anyone can run your code without first installing dependencies. +3. **Version locking**: URL imports can include specific versions (e.g. the code above is locked to version 0.17), ensuring consistency across different environments. +4. **Better security**: Deno downloads and caches dependencies on the first run, then checks integrity on subsequent runs, reducing the risk of supply chain attacks. +5. **Faster startup**: With no installation step, projects can be run immediately, speeding up development and deployment processes. +6. **Explicit dependencies**: Each file lists its own dependencies, making it clear what’s being used where. + +How you run the file looks different as well: + +```bash +deno run --allow-net --allow-env index.ts +``` + +This is a factor of Deno’s security model. Node.js grants full system access by default. This means that every file you run has complete access to the file system and complete access to the network. Mostly OK, until a file you run contains malicious code or a vulnerability is exploited. + +By default, Deno scripts have no system access. You must explicitly grant permissions when running the script to perform operations like network requests or file system access. This is what the –allow-net and –allow-env flags do in the example command. We need to access Neon over the network and access the DATABASE_URL environment variable, so we set those two flags. But if we wanted to write the output to a CSV, we couldn’t do that with the above command–we’d also need –allow-write. + +This granular permission system offers several benefits: + +1. **Enhanced security**: It limits the potential damage from malicious code or compromised dependencies. +2. **Transparency**: Developers and users can easily see what permissions a script requires. +3. **Least privilege principle**: Scripts only get the permissions they need to function. +4. **Easier auditing**: The required permissions can serve as a quick indicator of a script’s behavior. + +These are only a couple of the Deno differences. Deno also provides a different set of APIs (e.g., Deno.env.get(), Deno.exit()), more closely aligned with browser APIs, and supports TypeScript out of the box (e.g. queryObject<\{ id: number; name: string \}>). Deno also provides built-in tooling for testing, formatting, and bundling, reducing the need for external tools and configuration. + +All of these are excellent choices to move the JS/TS/ECMAS ecosystem forward. But as well-designed, performant, and simple as Deno is, developers continue to use Node.js–that’s where their code is, and the libraries frameworks they use are. Node.js has had twice as long to build up a robust ecosystem, and for all its faults developers want to use it. So when the mountain won’t come to you, you must go to the mountain. + +## Deno 2: Scale & Stability + +This is what it says in the [Deno 1.x to 2.x migration guide](https://docs.deno.com/runtime/reference/migration_guide/): + +
+

While we’ve accomplished a ton in Deno 1.x, the next major version is focused on using Deno at scale. This means seamless interoperability with Node.js and npm JavaScript infrastructure and supporting a wider range of projects and development teams, all without sacrificing the simplicity, security, and “batteries included” nature that developers love.

+
+ +
This is the crux of Deno 2.0–keep what works while allowing developers to use Deno better to scale. At its core, that means two things. + +### Node.js and npm compatibility + +Deno stabilized npm support with the npm: specifier in [Deno 1.28](https://deno.com/blog/v1.28). Since then, they have gradually improved Node.js and npm support throughout the runtime. Deno 2.0 provides backward [compatibility with Node.js and npm](https://docs.deno.com/runtime/fundamentals/node/): + +1. **Node.js Built-in Modules**: Deno now supports most Node.js built-in modules, which can be imported using the `node: prefix`. For example, `import * as os from "node:os";`. +2. **CommonJS Support**: Deno 2.0 improves CommonJS support, allowing developers to execute CommonJS files directly with the `.cjs` extension. Developers can import CommonJS modules in ES modules using `import` statements or use `require()` to import ES modules (as long as they don’t use top-level await). +3. **npm Packages**: Developers can use npm packages directly in Deno using the `npm: specifier`, making leveraging the vast npm ecosystem easier. +4. **package.json and node_modules**: Deno 2.0 understands `package.json` files and can work with `node_modules` directories, facilitating easier migration of existing Node.js projects. +5. **Global Variables**: Deno 2.0 introduces the process global variable widely used in Node.js applications and frameworks. + +If we revisit our examples from above, we can port our Node.js example to Deno. In fact, all we have to do for that is: + +1. Run `deno install` in the directory to read the package.json +2. Change the file extension to `.cjs` as the code includes a CommonJS module +3. Run `deno run  --allow-net --allow-env --allow-read node_index.cjs` with the additional `--allow-read` flag so Deno can read the `node_modules` + +Alternatively, we can change the Deno code to use the npm version of the Postgres library. We change our import to: + +```javascript +import pg from "npm:pg"; +const { Client } = pg; +``` + +Then, we also have to change the query call itself as the pg library doesn’t support `queryObject`: + +```javascript +const result = await client.query( + "SELECT name, value FROM playing_with_neon LIMIT 5" + ); +``` + +With these trivial examples, you can already see the compatibility between Deno and Node. But this is really all about a better developer experience for serious developers who need to scale their projects. With these Node.js compatibility improvements, Deno 2.0 starts to support the Node.js frameworks almost all developers now use, like Next.js, SvelteKit, Remix, and Nuxt. You should be able to run deno task dev instead of npm run dev for the same result. + +This all allows the gradual migration of existing Node.js projects to Deno while allowing developers to use familiar Node.js tools and libraries within Deno projects. Importantly, the flip is also true–Node developers can leverage Deno’s security features, tooling, DX, and modern JavaScript support while maintaining access to the npm ecosystem. + +### Deno LTS + +Another critical part of Deno 2.0 that matters in terms of making Deno a viable solution for large builds is [long-term support](https://docs.deno.com/runtime/fundamentals/stability_and_releases/). Starting with v2.1.0, Deno will provide an LTS channel with six months of support for bug fixes and critical performance improvements. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/deno-2-0-is-here-and-its-scaly/ad4nxecnvpx-qizdhrjzo2pgs37xr-iqqkzkspeflbccuge20nzsocmwfken2ea0wsokhh3hdue3pwmtgcfha9xv-nkwkxl2rwbl58u5c9rdj4ly-2lyl1-kvg99vww0kc4601klnefvi-5y3-ugr3r0k1s-487891e5.png) + +LTS channels are important because they provide stability and predictability for large-scale projects and enterprises. Organizations can rely on these versions to remain consistent over an extended period, reducing the risk of unexpected behavior changes that could break their applications. Organizations can plan their upgrade cycles more effectively with a defined support period. This is especially important for large projects where upgrades must be carefully scheduled and tested. + +Due to internal policies or compliance requirements, many enterprises require LTS versions for their production environments. Offering LTS makes Deno a more viable option for these organizations. + +For the Deno ecosystem, this means: + +1. Increased adoption in production environments +2. More confidence for businesses to invest in Deno-based projects +3. A clearer path for organizations transitioning from Node.js to Deno +4. Potential for more third-party tools and services to support Deno officially + +Overall, the introduction of LTS in Deno 2.0 is a significant step towards making Deno a mature, scalable JavaScript runtime. It addresses one of the key concerns that larger organizations and projects might have had about adopting Deno, potentially accelerating its adoption in production environments alongside or as an alternative to Node.js. + +## Scaly Denos + +Did dinosaurs have scales? I’m not sure whether feathers or scales are the current thinking, but Deno definitely has scal..ability now. + +This all might look like capitulation to the Node universe, and the arrival of the package.json and node_modules in an ostensibly Deno project might bring a [dinosaur tear]() to Deno enthusiasts. But this is just about reality and, more importantly, building what developers want or need. + +The Deno team has listened to the community and heard that while they love the simplclity, security, and performance of Deno, they still live and breathe Node, so need the crossover. Deno 2 gives them just that, allowing any developer to keep building in Node and start taking full advantage of the Deno world and slowly transitioning over. diff --git a/content/blog/posts/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale.md b/content/blog/posts/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale.md new file mode 100644 index 0000000000..b8f8f2247f --- /dev/null +++ b/content/blog/posts/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale.md @@ -0,0 +1,468 @@ +--- +title: >- + Deploy a Serverless FastAPI App with Neon Postgres and AWS App Runner at any + scale +description: >- + Create a serverless API using FastAPI, deployed on AWS App Runner and powered + by Neon Postgres +excerpt: >- + In this post, we’ll guide you through setting up a scalable serverless API + using FastAPI, deployed on AWS App Runner and powered by Neon Postgres as the + serverless database. FastAPI is a modern, fast (high-performance), web + framework for building APIs with Python 3.8+ based on st... +date: '2024-02-09T17:16:46' +updatedOn: '2024-03-04T16:13:55' +category: community +categories: + - community +authors: + - stephen-siegert +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/cover.jpg + alt: Neon FastAPI +isFeatured: false +seo: + title: >- + Deploy a Serverless FastAPI App with Neon Postgres and AWS App Runner at any + scale - Neon + description: >- + In this post, we’ll guide you through setting up a scalable serverless API + using FastAPI, deployed on AWS App Runner with Neon Postgres as the + serverless database. + keywords: [] + noindex: false + ogTitle: >- + Deploy a Serverless FastAPI App with Neon Postgres and AWS App Runner at any + scale - Neon + ogDescription: >- + In this post, we’ll guide you through setting up a scalable serverless API + using FastAPI, deployed on AWS App Runner with Neon Postgres as the + serverless database. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/neon-fastapi-ad1e5d21.jpg) + +In this post, we’ll guide you through setting up a scalable serverless API using [FastAPI](https://fastapi.tiangolo.com/), deployed on [AWS App Runner](https://docs.aws.amazon.com/apprunner/latest/dg/what-is-apprunner.html) and powered by Neon Postgres as the serverless database. + +FastAPI is a modern, fast (high-performance), web framework for building APIs with Python 3.8+ based on standard Python type hints. The key features of FastAPI are its speed and ease of use, making it an excellent choice for building robust APIs. FastAPI has quickly become a go-to framework for setting up Python APIs and services. + +AWS App Runner is a fully managed service that makes it easy for developers to quickly deploy containerized web applications and APIs, at scale. These services will automatically scale the instances up or down for your App Runner application in accordance to incoming traffic volume. + +Neon complements this setup by providing a serverless Postgres database that [scales compute resources automatically](https://neon.tech/docs/introduction/autoscaling), optimizing performance based on demand. + +We’ll walk through deploying a FastAPI application with Neon Serverless Postgres, focusing on secure database connection management via AWS Systems Manager (SSM) Parameter Store. This approach allows for flexible application environment management across development, testing, and production stages. + +Let’s get started! + +### Prerequisites + +To follow along and deploy the application in this guide, you will need: + +1. An AWS account, with access to AWS App Runner for deploying and managing your application +2. A GitHub or BitBucket account, for linking to AWS App Runner and enabling CI/CD functionality +3. [A Neon account](https://console.neon.tech/signup) – The FastAPI app will connect to a Neon serverless Postgres database 🚀 + +We’ll start by setting up a local development environment and getting our application running. Then, we’ll connect the application to a Neon Postgres database to ensure it can scale efficiently. Finally, we’ll set up automatic deployment through AWS App Runner, which will deploy our application upon each commit to a Git repository.

This infrastructure provides a flexible architecture that will scale automatically as needed by the API. + +### Set up FastAPI with Poetry + +In this app, we’ll use [poetry](https://python-poetry.org/) to manage the dependencies in the local Python virtual environment. For reference, the Python version used to write this post is 3.11. This version will also match the [App Runner Python 3.11 runtime](https://docs.aws.amazon.com/apprunner/latest/dg/service-source-code-python-releases.html) during deployment. + +To start, use poetry to create a new project: + +```bash +poetry new fastapi-neon +``` + +This will create a project structure that we’ll use to build out the FastAPI app. + +```yaml +fastapi-neon +├── pyproject.toml +├── README.md +├── fastapi-neon +│ └── __init__.py +└── tests + └── __init__.py +``` + +Next, initialize a new _git_ repository within the project directory. + +```bash +git init +``` + +Once the app is working locally, you’ll push the repo to GitHub and then deploy to AWS App Runner using the built-in CI/CD workflow. + +#### Installing FastAPI and uvicorn + +To isolate project dependencies, poetry will create a Python virtual environment associated with the project. To serve the application, we’ll use [uvicorn](https://fastapi.tiangolo.com/deployment/manually/#run-a-server-manually-uvicorn). + +Now, using poetry (or your preferred dependency manager), install `fastapi` and `uvicorn`. + +```bash +poetry add fastapi "uvicorn[standard]" +``` + +Now, add a `main.py` file in the `fastapi_neon` app directory and serve the app: + +```python +# fastapi_neon/main.py +from typing import Union +from fastapi import FastAPI + +app = FastAPI() + +@app.get("/") +def read_root(): + return {"Hello": "World"} + +@app.get("/items/{item_id}") +def read_item(item_id: int, q: Union[str, None] = None): + return {"item_id": item_id, "q": q} +``` + +Serve the app locally with the following command: + +```bash +poetry run uvicorn fastapi_neon.main:app --host 0.0.0.0 --port 8000 +``` + +Awesome! The app will be functional on your local machine. Check the Swagger UI endpoint by visiting `https://0.0.0.0:8000/docs`. + +![fastapi-swagger](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/fastapi-swagger-1024x360-f89904fa.png) + +## Updating the application to connect to Neon + +Now that we have an app working locally, we’ll connect the application code to a Neon Postgres database. + +For configuration management, we’ll use the [Starlette configuration pattern](https://www.starlette.io/config/). This will make it possible to reference predefined environment variables. FastAPI is based on Starlette, so we’ll be able to use this functionality without adding an additional dependency. + +To connect to a Postgres database from the app, we’ll add an [SQLModel](https://sqlmodel.tiangolo.com/) (based on SQLAlchemy). SQLModel is a library for interacting with SQL databases from Python code, with Python objects. + +Since SQLModel is driver agnostic, we’ll need to install a Postgres driver to enable it to connect to Neon. For this, we’ll use [psycopg 3](https://www.psycopg.org/psycopg3/docs/index.html). + +Add the `sqlmodel` and `psycopg` dependencies: + +```bash +poetry add sqlmodel "psycopg[binary]" +``` + +_Note: The default driver behavior for SQLAlchemy is to look for/use psycopg2. If you plan on using psycopg2 then you’ll need to install `psycopg2-binary` and adjust the connection string formatting in the `create_engine` code below._ + +Now, replace the existing `main.py` application code with the app code below: + +```python +# fastapi_neon/main.py +from contextlib import asynccontextmanager +from typing import Union, Optional +from fastapi_neon import settings +from sqlmodel import Field, Session, SQLModel, create_engine, select + +from fastapi import FastAPI + +class Todo(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + content: str = Field(index=True) + +# only needed for psycopg 3 - replace postgresql +# with postgresql+psycopg in settings.DATABASE_URL +connection_string = str(settings.DATABASE_URL).replace( + "postgresql", "postgresql+psycopg" +) + +# recycle connections after 5 minutes +# to correspond with the compute scale down +engine = create_engine( + connection_string, connect_args={"sslmode": "require"}, pool_recycle=300 +) + +def create_db_and_tables(): + SQLModel.metadata.create_all(engine) + +# The first part of the function, before the yield, will +# be executed before the application starts +@asynccontextmanager +async def lifespan(app: FastAPI): + print("Creating tables..") + create_db_and_tables() + yield + +app = FastAPI(lifespan=lifespan) + +@app.get("/") +def read_root(): + return {"Hello": "World"} + +@app.post("/todos/") +def create_todo(todo: Todo): + with Session(engine) as session: + session.add(todo) + session.commit() + session.refresh(todo) + return todo + +@app.get("/todos/") +def read_todos(): + with Session(engine) as session: + todos = session.exec(select(Todo)).all() + return todos +``` + +This example API allows clients to create (POST) and retrieve (GET) todos. The database connection string is imported from settings and used to instantiate the connection in `create_engine`. All of the values are cast as `Secret` to limit their exposure in logs in the event that they are exposed. + +[Lifespan events](https://fastapi.tiangolo.com/advanced/events/) are the recommended way to execute code once the server starts in FastAPI. In this example, the `Todo` model (i.e. table) is created if it doesn’t yet exist in the database. + +The `pool_recycle=300` option is an “optimistic” approach to [prevent the pool from using a connection that has passed a certain age](https://docs.sqlalchemy.org/en/20/core/pooling.html#setting-pool-recycle). In this case, we are setting the value to 5 minutes to correspond with the default [compute auto-suspend in Neon](https://neon.tech/docs/guides/auto-suspend-guide). + +Another option to account for possible stale connections is to use the `pool_pre_ping` option. This option is used to test the availability of a database connection before the connection is used. Note, this can add additional latency to new connections since they are first “checked”. + +Next, we’ll get the connection string for the database. + +#### Integrate Neon Postgres with FastAPI + +Create a `settings.py` file in the `fastapi_neon` directory alongside `main.py`. This will reference the `DATABASE_URL` environment variable. During local development, this value will be populated from an `.env` file. + +```python +# fastapi_neon/settings.py +from starlette.config import Config +from starlette.datastructures import Secret + +try: + config = Config(".env") +except FileNotFoundError: + config = Config() + +DATABASE_URL = config("DATABASE_URL", cast=Secret) +``` + +Create an `.env` file in the root of your project and add the `DATABASE_URL` variable: + +```yaml +# .env +# Don't commit this to source control +# Eg. Include ".env" in your `.gitignore` file. +DATABASE_URL= +``` + +**Note: Do not commit the `.env` file to source control. Add `.env` to the `.gitignore` file to prevent this file from being committed and pushed. This environment variable in this file will only be used during local development.** + +Visit the [Neon Console](https://console.neon.tech/), sign up, and create your first project by following the prompts in the UI. You can use an existing project or create another if you’ve already used Neon. + +![Get started with Neon for free](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/get-started-with-neon-1024x789-d8b0c6a9.png) + +Visit the project Dashboard, check the **Pooled connection** option, and select the **Connection string** option from the **Connection Details** panel. + +The connection string will be in the following format: + +```bash +postgres://alex:AbC123dEf@ep-cool-darkness-123456-pooler.us-east-2.aws.neon.tech/dbname?sslmode=require +``` + +Set this value as your `DATABASE_URL` in the `.env` file. Now, test the application by running the following to start the app server locally: + +```bash +poetry run uvicorn fastapi_neon.main:app --host 0.0.0.0 --port 8000 +``` + +The app is live…locally! + +The application is now using Neon Postgres as the connected database. The `create_db_and_tables` lifespan event created the todo model (table) in the database. You can double-check this by going into the Neon console and viewing the tables in your Neon project – you will see `todo` listed.
You can insert and fetch _todos_ from the API by using the live endpoints. For example, try sending some requests using the built-in FastAPI Swagger UI docs endpoint at `https://0.0.0.0:8000/docs`. + + + +Now, it’s time to deploy the app to AWS App Runner. + +## Setting up the AWS App Runner configuration + +A few items need to be in place before the app can be deployed to App Runner. Mainly, the database connection string will need to be stored in AWS Systems Manager (SSM) Parameter Store. Then, an `apprunner.yaml` configuration file will be added to the project that instructs App Runner how to build and run the app. This is also where the database connection string variable is referenced and associated with the app. + +At a high-level, the steps are as follows. Subsequent sections will guide you through each of these: + +1. Create the ** *apprunner.yaml* **configuration file in the root of the project directory +2. Create a `DATABASE_URL` parameter in SSM Parameter Store +3. Update the `apprunner.yaml` secret value with the SSM parameter ARN to inject the database connection string into the runtime environment +4. Create an _instance role_ that App Runner can use to access the SSM Parameter Store Secure String secret +5. Push to GitHub or BitBucket and set up the deployment in the AWS App Runner console + +#### Create the App Runner _apprunner.yaml_ configuration + +In your project directory, create an `apprunner.yaml` configuration file with the following structure. + +```yaml +version: 1.0 +runtime: python311 +build: + commands: + build: + - echo "Build command..." +run: + runtime-version: 3.11 + pre-run: + - echo "Installing dependencies..." + - pip3 install poetry + - poetry config virtualenvs.create false + - poetry install + + command: poetry run uvicorn fastapi_neon.main:app --host 0.0.0.0 --port 8000 + network: + port: 8000 + secrets: + - name: DATABASE_URL + value-from: "" +``` + +This configuration instructs App Runner to install, build, and run the app. The specified managed runtime is the [revised Python 3.11 App Runner build](https://docs.aws.amazon.com/apprunner/latest/dg/service-source-code.html#service-source-code.build-detail). When the app is pulled from the git repo, the dependencies defined in the `poetry.lock` file are installed, then the app is served using the same `uvicorn` command from the local environment. You’ll notice that there is a placeholder for the `value-from` entry in the `secrets` section. We’ll update this next. + +_Note: The App Runner CI process is very strict when parsing the configuration. Make sure to check the indentation and that the file name is not abbreviated and ends with **.yaml**._ + +#### Create a DATABASE_URL parameter in SSM Parameter Store + +Go to [AWS Systems Manager](https://console.aws.amazon.com/systems-manager/) and navigate to **Application Management** > **Parameter Store** and click **Create Parameter.** + +Enter the following for Name and Type: + +- **Name:** `/fastapi-neon/DATABASE_URL` +- **Type:** SecureString + +For the value, use the database connection URL from **your Neon dashboard** that you’ve tested with previously. + +Click **Create parameter**. Once created, click on the newly created parameter and copy the ARN identifier.
+ +![SSM Parameter Store - Copy parameter ARN](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/ssm-copy-arn-1024x770-e26a0b1a.png) + +Now, replace the placeholder (``) value in `apprunner.yaml` under `secrets` > `value-from` with this ARN. + +```yaml +... +secrets: + - name: DATABASE_URL + value-from: "" +... +``` + +And **save**. + +#### Push to Source Control + +The final project directory structure will be: + +```yaml +. +├── fastapi_neon +│ ├── __init__.py +│ ├── main.py +│ └── settings.py +├── tests/ +├── apprunner.yaml +├── poetry.lock +├── pyproject.toml +├── README.md +├── .gitignore +... +``` + +Confirm that `.env*` is included in the `.gitignore` file in the root of the project. + +```yaml +# Environments +.env* +... +``` + +
Commit and push the code to a repo in GitHub or BitBucket. Remember, **do not commit the `.env` file**. + +#### Create an App Runner instance role + +In order for the App Runner process to access the SSM Parameter Store secret, we’ll need to create an [instance role](https://docs.aws.amazon.com/apprunner/latest/dg/security_iam_service-with-iam.html) with the appropriate permissions to access the SSM resource and attach it to the App Runner service when we create the service. + +Go to the [AWS Identity and Access Management (IAM)](https://console.aws.amazon.com/iam/home) console and click **Create role**. + +Select **Custom trust policy** and add the JSON policy below: + +```json +[object Object] +``` + +1. Click **Next** and skip adding permissions +2. Click **Next** and enter the below information + 1. **Role name**: _`app-runner-fastapi-role`_ +3. Click **Create Role** + +Now, **open** the new _`app-runner-fastapi-role`_. Select **Create** **inline policy** in the **Permissions** tab. + +![AWS IAM - Create inline policy](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/create-inline-policy-1024x205-918ea04b.png) + +In the JSON editor, add the following permission policy with the parameter `/fastapi-neon/DATABASE_URL` ARN from above: + +```json +[object Object] +``` + +The above permissions allow the App Runner [instance role to access the SSM parameter](https://docs.aws.amazon.com/apprunner/latest/dg/env-variable.html) (s) that contains the database connection string. + +Click **Next**. In the **Policy details** section, add: + +- **Policy name**: _apprunner-ssm-policy_ + +Finally, click **Create policy**. + +## Deploying the application to AWS App Runner + +Finally, we are in App Runner. Let’s deploy this service! + +Go to [AWS App Runner](https://console.aws.amazon.com/apprunner/home) and click **Create an App Runner service**. + +![AWS App Runner - Create Service](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/create-app-runner-1024x286-03bd2fd3.png) + +This service will use a **Source code repository** since it will be pulling the code from the repo that was just created. You will need to allow App Runner to access the repo in your account if you have not already authorized the AWS Connector. + +![AWS App Runner - Configure Service](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/source-and-deployment-1024x719-509248db.png) + +Select your git provider and repository details. In **Deployment settings**, **Automatic** is selected to deploy the app on each push to the linked branch.
+ +![AWS App Runner - Automatically deploy app](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/apprunner-github-connecton-1024x883-86f9deef.png) + +Click **Next**. On the next screen, select **Use a configuration file**. This will instruct App Runner to use the ** _apprunner.yaml_ ** configuration file from the repo. Click **Next**. + +In the **Configure service** section: + +- Add a **Service name** for your app in Service settings +- In **Security**, select the instance role that was created earlier. + +![AWS App Runner - Select instance role](https://cdn.neonapi.io/public/images/pages/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale/app-runner-instance-role-1024x397-6971aef8.png) + +The trust relationship that was specified during role creation is what determines if it is available in the dropdown options 🙂. + +Click **Next.** Next, review the setup and click **Create & deploy**. + +_This may take a few minutes to deploy…_ + +Once successfully deployed, test the FastAPI service at `https://..awsapprunner.com/docs`. For example, if you add data to the table using the SQL Editor in the Neon console: + +```sql +INSERT INTO todo (content) VALUES ('Task 1.'); +INSERT INTO todo (content) VALUES ('Task 2.'); +INSERT INTO todo (content) VALUES ('Task 3.'); +``` + +Then you can fetch that data by accessing `https://..awsapprunner.com/todos/`. + +### Clean up + +To wrap up, delete the application if it’s no longer needed. To do this, go into the App Runner service and select **Actions** > **Delete** to remove the app. + +## Conclusion + +Integrating AWS App Runner and Neon Serverless Postgres is an effective way to deploy a robust, scalable serverless FastAPI service. Neon’s unique autoscaling capabilities ensure that your database resources adapt to your application’s demands without manual intervention. This dynamic scalability is crucial for maintaining optimal performance and cost efficiency during off-peak times and traffic bursts. This is perfect for growing teams that want to increase their development velocity while keeping their architecture flexible. + +
To get started with incorporating Serverless Postgres into your FastAPI and Python apps, [sign up and try Neon for free](https://console.neon.tech/signup). Follow us on [Twitter](https://twitter.com/neondatabase) and join us in [Discord](https://neon.tech/discord) to share your experiences, suggestions, and challenges. + +## Resources + +- The code for this guide is in the [fastapi-apprunner-neon](https://github.com/neondatabase/fastapi-apprunner-neon) repo on GitHub diff --git a/content/blog/posts/deploy-agent-api-xpander.md b/content/blog/posts/deploy-agent-api-xpander.md new file mode 100644 index 0000000000..06f210e421 --- /dev/null +++ b/content/blog/posts/deploy-agent-api-xpander.md @@ -0,0 +1,168 @@ +--- +title: Under the hood of a deploy-agent API that works across any runtime environment +description: >- + How we designed a single controller to deploy AI agents across the xpander.ai + cloud and customer clusters, with the same developer experience +excerpt: >- + When we started building xpander.ai’s agent platform, one of our core promises + to enterprise customers was freedom of deployment. Some wanted to run agents + entirely inside xpander cloud, using our managed infrastructure. Others needed + to host agents on their own Kubernetes cluste... +date: '2025-10-27T16:13:17' +updatedOn: '2025-10-27T16:14:30' +category: community +categories: + - community + - ai +authors: + - ran-sheinberg +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deploy-agent-api-xpander/cover.jpg + alt: null +isFeatured: true +seo: + title: >- + Under the hood of a deploy-agent API that works across any runtime + environment - Neon + description: >- + Learn how xpander solved a tricky problem for agent platforms: how to make + deploying AI agents feel identical across clouds and clusters. + keywords: [] + noindex: false + ogTitle: >- + Under the hood of a deploy-agent API that works across any runtime + environment - Neon + ogDescription: >- + Learn how xpander solved a tricky problem for agent platforms: how to make + deploying AI agents feel identical across clouds and clusters. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deploy-agent-api-xpander/social.png +--- + + +This post was created in collaboration with [xpander.ai](https://neon.com/blog/xpander-ai-agents-slack-neon-backend), as part of our agent builders series - where agent platform teams share how they built, scaled, and refined their systems. + + +When we started building [xpander.ai](https://xpander.ai)’s agent platform, one of our core promises to enterprise customers was freedom of deployment. Some wanted to run agents entirely inside xpander cloud, using our managed infrastructure. Others needed to host agents on their own Kubernetes clusters – be it simply in a cloud VPC, private clouds, on-premise environments, edge locations, etc. + +That raised a big question: + +**How do we make deploying a AI agents feel identical, whether it runs on our cloud or theirs?** + +We did not want separate SDKs, different APIs, or special build steps. We wanted one upload, one deploy call, and one consistent lifecycle across every environment – whether the developer calls the deploy API themselves, or their CI/CD pipeline calls it. + +This post explains how we made that happen through what we now call our deploy-agent API, and how a complex, multi-environment system is abstracted under this single API call. + +## The Problem: Two Environments, One Experience + +A typical AI agent runtime at xpander runs as a containerized microservice. Developers can extend, fine-tune, or ship their own agents, often uploading gzipped Docker image archives (`.tar.gz`) produced by their local builds. + +But the deployment flow looked very different depending on where it lived: + +- Cloud tenants needed to stream those images from S3 → ECR → Kubernetes, with AWS credentials and multi-tenant registry management. +- Self deployed tenants uploaded their images directly to local Docker registries, where our system had no access. + +That duality created a maintenance problem. Different storage systems, network rules, authentication methods, and Kubernetes contexts meant two entirely separate workflows. We knew we could not maintain that in the long term. + +So we asked: _What would it take to make both flows converge into one API surface?_ + +## The Design Principle: Mirror, Don’t Fork + +Our guiding design principle became: + +**Same inputs, same outputs, different under the hood.** + +We built a single deploy-agent API that handled every container lifecycle event – upload, deploy, start, stop, and logs – and routed requests dynamically based on tenant type. + +When a developer uploads an agent or a CI/CD pipeline calls the API, the same endpoint runs: + +```bash +POST /api/deploy-agent +``` + +The controller checks the environment which is configured as the runtime for that specific agent: + +- If it is a cloud environment, it returns a presigned S3 URL so the image can be uploaded directly to S3. +- If it is a self-deployed, it exposes a loopback upload stream, decompresses the tarball locally, and pushes it into the private registry running on the customer’s Kubernetes cluster, which was deployed by our engine (Helm chart). + +The code paths then come together again. Both result in a validated image reference and a standardized call to `apply_agent_container()`, the asynchronous function that launches the Kubernetes job. + +From a user perspective, and for the rest of our backend, the deployment process is completely environment-agnostic. + +## Under the Hood: Two Paths That Look Like One + +Here is what happens behind the curtain. + +### Cloud Path (S3 → ECR → Kubernetes) + +1. The controller streams the uploaded image from S3 directly into ECR using an `aws | skopeo copy` pipeline. No temporary disk writes and no Docker daemon.
+2. Credentials are injected securely from AWS ECR tokens, then redacted from logs before storage.
+3. Once uploaded, we mark the deployment in our registry, start the Kubernetes Job, and update the state: `Starting → Up`. + +### Self-deployed environment Path (Upload → Local Registry → Kubernetes) + +1. The controller accepts the file stream and decompresses the `.tar.gz` archive on disk.
+2. It uses skopeo again – same command, same logic – to push the uncompressed tar into a local registry. The only difference is that it skips AWS and disables TLS verification for air-gapped setups.
+3. From here, it triggers the same downstream logic, `apply_agent_container()` and `start_agent_container()`, to run the agent pod. + +
+Image +
Same API, different plumbing.
+
+ +## Bridging to Kubernetes Safely + +Both flows converge when creating the Kubernetes Job (which is the actual agent container runtime) + +Here is what makes that piece both tricky and elegant: + +- The controller loads the kubeconfig context from base64-encoded environment variables. + - For xpander cloud, it connects to our shared managed cluster. + - For self-deploy, it uses the customer’s own context and credentials. +- It injects organization-scoped secrets and environment variables, so every agent pod runs with its own API keys and configuration. +- It waits for pod startup, monitors crash loops, and updates the registry with the result. + +This context-loading method lets us dynamically switch clusters for each request. No static config files and no persistent Kubernetes clients. It is a full multi-cluster orchestrator written in Python. + +## Making It Observable + +Once the pod is live, developers need logs, ideally in real time. + +The challenge was that the Kubernetes Python client’s log stream is blocking, while our API server uses FastAPI’s asynchronous event loop. + +To fix that, we built a threaded log streamer. It tails logs in a background thread, pushes lines into an `asyncio.Queue`, and exposes them over Server-Sent Events (SSE) to the UI. + +
+Image +
Agent execution logs shown in the xpander AI Workbench
+
+ +For longer debugging sessions, we can backfill logs from S3 for historical context. Whether your agent runs in xpander cloud or in your own cluster, `GET /agents/{id}/logs` just works – same endpoint, same interface. + +## Why This Matters + +For us, this was not just an architectural exercise. + +Unifying cloud and on-prem orchestration gives customers tangible benefits: + +- **Predictability** – one deploy-agent API with consistent behavior everywhere. +- **Security** – credentials never leave tenant boundaries and there are no cross-cloud assumptions. +- **Portability** – agents can move from our cloud to theirs with no rebuild or code change. +- **Maintainability** – our engineering team maintains one pipeline instead of two. + +It is one of those engineering decisions that is invisible when it works, but transformative for scale. + +## Closing Thoughts + +In the end, our deploy-agent API became the foundation for xpander’s agent infrastructure when it comes deployment DevEx and CI/CD integration. + +Whether a customer deploys 10 agents in xpander cloud or 1,000 agents across self-hosted clusters, the workflow stays the same – simple, secure, and transparent. + +When we say _“run agents anywhere”_, this is what makes it possible. + +[xpander.ai](https://xpander.ai) **is the ultimate agent toolbox: memory, tools, MCP, storage, slack integrations. Get a production-ready backend for your agents** [via their Free Plan](https://app.xpander.ai/signup). + + +If you're building your own agent platform and need a backend, [take a look at Neon's Agent Plan.](https://neon.com/use-cases/ai-agents) Get special pricing, resource limits, and assistance to get your platform up and running. + diff --git a/content/blog/posts/deploy-mistral-large-to-azure-and-chat-with-langchain.md b/content/blog/posts/deploy-mistral-large-to-azure-and-chat-with-langchain.md new file mode 100644 index 0000000000..024bc5dc99 --- /dev/null +++ b/content/blog/posts/deploy-mistral-large-to-azure-and-chat-with-langchain.md @@ -0,0 +1,250 @@ +--- +title: >- + Deploy Mistral Large to Azure and create a conversation with Python and + LangChain +description: Step-by-step guide to deploying Mistral Large to Azure +excerpt: >- + We’re Neon, and we’re redefining the database experience with our cloud-native + serverless Postgres solution. If you’ve been looking for a database for your + RAG apps that adapts to your application loads, you’re in the right place. + Learn more about Neon and give it a try, and let... +date: '2024-02-27T01:59:48' +updatedOn: '2024-02-27T12:34:43' +category: ai +categories: + - ai +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deploy-mistral-large-to-azure-and-chat-with-langchain/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Deploy Mistral Large to Azure and create a conversation with Python and + LangChain - Neon + description: Step-by-step guide to deploying Mistral Large to Azure + keywords: [] + noindex: false + ogTitle: >- + Deploy Mistral Large to Azure and create a conversation with Python and + LangChain - Neon + ogDescription: >- + We’re Neon, and we’re redefining the database experience with our + cloud-native serverless Postgres solution. If you’ve been looking for a + database for your RAG apps that adapts to your application loads, you’re in + the right place. Learn more about Neon and give it a try, and let us know + what you think. Neon is cloud-native […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/deploy-mistral-large-to-azure-and-chat-with-langchain/social.jpg +--- + +Post image + +**We’re Neon, and we’re redefining the database experience with our cloud-native serverless Postgres solution. If you’ve been looking for a database for your RAG apps that adapts to your application loads, you’re in the right place.** [Learn more about Neon and give it a try](https://neon.tech), **and let us know what you think. Neon is cloud-native Postgres and scales your AI apps to millions of users with pgvector. In this post, Raouf is going to tell you what you need to know about Mistral Large, the most advanced LLM by MistralAI.** + +[Mistral AI](https://mistral.ai/) has recently unveiled its most advanced open-source large language model (LLM) yet, [Mistral Large](https://mistral.ai/news/mistral-large/), alongside its ChatGPT competitor, [Le Chat (beta)](https://chat.mistral.ai/chat). Le Chat includes other models such as Next, and Small, to let you explore Mistral AI’s capabilities. + +Post image + +For those waiting to get their hands on Le Chat but stuck in the queue, this guide will show you how to deploy Mistral Large on Azure and start using it immediately with LangChain. + +Before we dive into the deployment process, let’s briefly explore Mistral Large. + +## Mistral Large + +Mistral Large is Mistral AI’s most advanced model with unparalleled reasoning capabilities across multiple languages, including French, Spanish, German and Italian. It has a generous 32k token context window making interesting for Retrieval Augmented Generation applications. + +
+Post image +
Comparison measuring massive multitask language understanding
+
+ +And most importantly, Mistral Large is pretty good at coding and math. The model ranks the highest in the MassiveText Benchmarks for Programming Problems (MBPP), which covers a wide range of difficulty levels and programming concepts and is designed to evaluate models on several fronts, including accuracy and efficiency. + +Mistral Large also ranks the highest in the GSM8K, which measures the capabilities of AI models in educational contexts and reasoning in mathematics. + +Post image + +But don’t believe the benchmarks. Next, we’ll deploy the Mistral Large model to Azure and try it for ourselves. + +## Deploy your own Mistral Large model to Azure + +As part of the launch, [Mistral AI announced its partnership with Microsoft](https://techcommunity.microsoft.com/t5/ai-machine-learning-blog/mistral-large-mistral-ai-s-flagship-llm-debuts-on-azure-ai/ba-p/4066996), making the Mistral Large model available on Azure. Below are the steps to deploy the model: + +1. **Access Azure AI Studio**: Sign into your Azure account and navigate to [AI Studio](https://aka.ms/aistudio/landing/mistral-large). +2. **Deploy Mistral Large**: Look for the “Deploy” option and select Mistral Large for deployment. + +Post image + +3. **Create a Project**: If you haven’t already, set up a new project, opting for the Pay-As-You-Go plan and choosing France Central as your region. + +Post image + +4. **Review and Create**: Double-check your resource information before finalizing your AI project. + +Post image + +5. **Finalize Deployment**: After creating your AI project, proceed to deploy Mistral Large. Choose a name for your deployment; this will be your inference endpoint’s identifier. +6. **Select a Deployment Name**: This is the name that will be displayed on your inference endpoint. + +Post image + +Congratulations 🎉 You’ve successfully deployed Mistral Large on Azure! + +## How to use Mistral Large with LangChain + +After deployment, you’ll receive an API endpoint and a security key for making inferences. We’ll use those further below. + +Post image + +To use Mistral Large with LangChain, follow these steps: + +1. **Create project** + +```bash +mkdir mistral-large-example +cd mistral-large-example +``` + +2. **Create and activate Python environment:** Run the following command to create an environment. + +```bash +python -m venv myenv +source myenv/bin/activate +``` + +3. **Install packages and project dependencies:** + +```bash +pip install langchain langchain_mistralai +``` + +4. **Create a LangChain conversation:** first, create a file: + +```bash +touch main.py +``` + +Here’s an example of how to create a LangChain conversation chain with Mistral Large: + +```python +from langchain.chains import LLMChain +from langchain.memory import ConversationBufferMemory +from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, MessagesPlaceholder +from langchain.schema import SystemMessage +from langchain_mistralai.chat_models import ChatMistralAI + +# Configuration for prompting +prompt = ChatPromptTemplate.from_messages([ + SystemMessage(content="You are a chatbot engaging in a conversation with a human, often incorporating French cultural references."), + MessagesPlaceholder(variable_name="chat_history"), + HumanMessagePromptTemplate.from_template("{human_input}"), +]) + +# Memory configuration +memory = ConversationBufferMemory(memory_key="chat_history", return_messages=True) + +# Configuring the Mistral model endpoint and API key +chat_model = ChatMistralAI( + endpoint="https://.francecentral.inference.ai.azure.com", + mistral_api_key=" Entering new LLMChain chain... + +Prompt after formatting: + +System: You are a chatbot engaging in a conversation with a human, often incorporating French cultural references. + +Human: Hi there, my friend + +> Finished chain. + + Hello! It's a pleasure to chat with you. As you've noticed, I enjoy incorporating French cultural references into our conversations. Did you know that the Eiffel Tower, one of France's most iconic landmarks, was initially criticized by some of France's leading artists and intellectuals for its design when it was first built? How can I assist you today? +``` + +## Conclusion + +There has never been a better time to develop AI-powered applications. With rapid deployments to robust and scalable infrastructures such as Azure’s, developers can create applications that are more intelligent, interactive, and impactful. + +If you are building a RAG application, or simply need a Postgres database that scales, Neon with its autoscaling capabilities offers elastic vector search and fast index build with pgvector, making your AI apps fast and scalable to millions of users. + +[Start building with Neon for free today](https://console.neon.tech), join us on [Discord](https://neon.tech/discord) and let us know what you’re working on and how we can help you build better apps. + +## Resources + +- [Mixtral 8x7B: What you need to know about Mistral AI’s latest model](https://neon.tech/blog/mixtral-8x7b-what-you-need-to-know-about-mistral-ais-latest-model) +- [Mistral 7B and BAAI on Workers AI vs. OpenAI Models for RAG](https://neon.tech/blog/mistral-7b-and-baai-on-workers-ai-vs-openai-models-for-rag) +- [pgvector: 30x Faster Index Build for your Vector Embeddings](https://neon.tech/blog/pgvector-30x-faster-index-build-for-your-vector-embeddings) +- [Building an AI-powered ChatBot using Vercel, OpenAI, and Postgres](https://neon.tech/blog/building-an-ai-powered-chatbot-using-vercel-openai-and-postgres) diff --git a/content/blog/posts/design-decisions-behind-app-build.md b/content/blog/posts/design-decisions-behind-app-build.md new file mode 100644 index 0000000000..fc57f7f5c8 --- /dev/null +++ b/content/blog/posts/design-decisions-behind-app-build.md @@ -0,0 +1,231 @@ +--- +title: 'Design Decisions Behind app.build, a Prompt-to-App Generator' +description: >- + Building a code generation system that prioritizes working apps over feature + complexity. +excerpt: >- + TL;DR: We built app.build, a reliable open-source AI code generator by: + limiting initial scope to CRUD web apps, using FSM-guided tree-search actors, + implementing extensive validation, and encapsulating context management using + an error analysis feedback loop. These architectural... +date: '2025-06-26T14:36:49' +updatedOn: '2025-10-01T16:46:04' +category: ai +categories: + - ai +authors: + - arseni-kravchenko + - igor-rekun + - evgenii-kniazev +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/cover.jpg + alt: null +isFeatured: true +seo: + title: 'Design Decisions Behind app.build, a Prompt-to-App Generator - Neon' + description: >- + Software architecture decisions behind a code generation system that + prioritizes working apps over feature complexity. + keywords: [] + noindex: false + ogTitle: 'Design Decisions Behind app.build, a Prompt-to-App Generator - Neon' + ogDescription: >- + Software architecture decisions behind a code generation system that + prioritizes working apps over feature complexity. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/neon-decisions-1024x576-f336854e.jpg) + + +Since publishing this post, we’ve shifted focus. The managed version of app.build has been discontinued, but the source code is available - if you’re also building an agent, you can still explore the app.build [agent](https://github.com/appdotbuild/agent) and [platform](https://github.com/appdotbuild/platform) code for reference and implementation examples. We’re also applying this learnings and code to our next project. + + +**TL;DR:** We built [app.build](https://www.app.build/), a reliable [open-source](https://github.com/appdotbuild) AI code generator by: limiting initial scope to CRUD web apps, using FSM-guided tree-search actors, implementing extensive validation, and encapsulating context management using an error analysis feedback loop. + +These architectural choices weren’t arbitrary—they emerged from a fundamental decision about what kind of system we wanted to build. Let’s start with the core philosophy that shaped everything else. + +## Philosophy: Reliability vs Capabilities + +One of the core principles in system design is identifying key tradeoffs. In the code generation context, the most important one is **capabilities** (how advanced the generated features can be) versus **reliability** (how likely they are to work). + +Consider these two extremes: + +- **High reliability, limited capabilities**: When you define types in an ORM and those automatically appear in the database through generated migrations, reliability approaches 100%, but the feature set is very limited. +- **High capabilities, low reliability**: A tech-savvy user prompting an LLM-based assistant can request any advanced feature, but it may fail in obvious ways—or worse, in implicit ways. + +Many AI app builders have emerged recently, supporting a wide range of applications (partially limited by tech stack). Coding assistants and AI-powered IDEs like Claude Code and Cursor don’t even limit your choice of programming languages, despite performing better with popular ones. Unfortunately, neither approach guarantees a final working result (especially in the single shot mode), though they do empower users in the process. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/image-1024x632-4025b2fa.png) + +When working on app.build, we set **reliability** as our core value, even if it meant sacrificing advanced features for the initial release. We wanted to echo the principle popular in some tech communities: “if it compiles, it works.” + +### Generalization through Specialization + +Since app.build was kicked off at [Neon](https://neon.com) (a database company), we started by focusing on the typical OLTP use case—CRUD web apps. Yes, we wouldn’t support building anything user may want, but the final artifacts should be well-functioning—ideally without human intervention, or at worst, requiring only a gentle touch from the user. + +By fixing both the stack and the niche, we could restrict AI generation and introduce a reliable validation pipeline. Our core proposition is the following: + +1. We keep the agent narrow, very opinionated and thus reliable. +2. With reliable agents available, we will widen the scope via horizontal scaling – supporting new stacks and application types + +Support for new tech stacks is already work in progress. For every new vertical, we’re going to stick to our core value: the resulting apps should be usable and pass smoke tests (e.g., end-to-end tests with tools like [Playwright](https://playwright.dev/)). + +## Core Architecture + +With this reliability-first philosophy established, let’s examine how we translated these principles into our core architecture. + +Under the hood, app.build uses three key components + +- a Finite State Machine (FSM) for flow control +- parallel tree search for finding solutions +- specialized actors for different tasks + +While this sounds complex, each piece serves a simple purpose. + +### Evolution from Linear to Graph-Based + +Our very first version was a linear workflow with one step following another. We realized decomposition was key, so we started with sequential generation and gradual context enrichment: data model first, ORM code later, followed by tests and handlers. This worked fine for initial demo apps but quickly became too restrictive. + +The main limitation? No intervention was possible. If an intermediate result didn’t match expectations, users could only start from scratch—not the ergonomics nor the velocity we were aiming for! + +This prompted the next idea: transform the linear pipeline into a graph, granting the required flexibility. Since we needed to combine flow control with guardrails, we introduced a **Finite State Machine (FSM)**. With FSM, we could restrict the level of feedback—the FSM driver (a top-level agent) could only [send guiding events](https://github.com/appdotbuild/agent/blob/e73f2b5b9a7dde12ece87c1c59aa8b7d65bc7f57/agent/api/fsm_tools.py#L68) instead of imperative-style control. Using FSM for agentic tasks becomes a popular topic these days, as [StateFlow](https://arxiv.org/abs/2403.11322v1) and [MAPLE](https://arxiv.org/abs/2505.23596) suggest. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/image-5-702x1024-a10f0ebc.png) + +### Actors and Concurrent Execution + +The FSM itself doesn’t generate anything; it delegates work to actors—subagents focused on solving particular problems. Our initial release includes four actor types: + +- **DraftActor** – for initial data model generation +- **HandlerActor** – for backend logic +- **FrontendActor** – for UI code generation +- **EditActor** – for subsequent changes + +Some actors can run concurrently. For example, we can spawn 20+ actors working on various handlers while one builds the frontend for a single app. Ideally, we’d like to decompose the FrontendActor further so multiple instances could work on specific components—currently, frontend generation often dominates the total app generation latency (because we do it all at once). + +Concurrency execution is instrumental in this design. Universal tools improving the quality of modern LLM apps are typically associated with larger context or more computational budget (either with reasoning or parallel searches, both are compute-heavy). Our design allows us to tune this knob and get reliable results without wasting hundreds of millions of tokens. Additionally, the validations pipeline described below tends to affect the final latency too, so the agent would never be usable without concurrency support. + +### The Search Process + +Each actor’s main job is to [search for the solution](https://github.com/appdotbuild/agent/blob/e73f2b5b9a7dde12ece87c1c59aa8b7d65bc7f57/agent/trpc_agent/actors.py#L33), and modern research [suggests that tree-based search can improve final quality by a large margin](https://arxiv.org/abs/2409.09584). The search process maintains a tree of generation attempts, where each node represents a potential solution state. Key features include: + +- Early termination when any candidate passes all validation stages +- Dramatically reduced generation time for successful paths +- Tailored search parameters (depth vs breadth preference, beam width) +- Task-specific validation checks + +Each actor works in an [isolated environment](https://github.com/appdotbuild/agent/blob/main/agent/core/workspace.py), following the encapsulation principle. Each actor [owns its own LLM message chains with required context](https://github.com/appdotbuild/agent/blob/e73f2b5b9a7dde12ece87c1c59aa8b7d65bc7f57/agent/core/actors.py#L85). Reliability is achieved through the validation pipeline—no actor can return a solution unless it passes all checks. + +### The Special Case: EditActor + +[EditActor](https://github.com/appdotbuild/agent/blob/e73f2b5b9a7dde12ece87c1c59aa8b7d65bc7f57/agent/trpc_agent/diff_edit_actor.py#L128) deserves special mention. It appeared later than others because our first versions (with the aforementioned linear workflow) could only perform a single shot of an app generation. + +Our evolution: + +1. First hacky workaround: refine the data model first, then rerun generation +2. Experimented with advanced FSM transitions +3. Final realization: straightforward file editing works fine if the project structure is properly set during initial generation + +Unlike most actors, EditActor has more flexibility, because its scope can be really huge for some changes (e.g., when the initial model needs changes that are later propagated across components) and minor for others (e.g., “change the header color” translates to a single line change). Similarly to the FrontendActor, it uses tools for more agentic behavior to rely on existing files. Those properties affect the actor’s convergence negatively, but given it starts the work with established project structure and validation pipeline, it is not a blocking problem. + +### Stateless Nature of the Agent + +![Image](https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/image-3-1024x412-1db3626e.png) + +Designing the agent service in a scalable manner was one of the software engineering challenges that resulted in our decision to go stateless. We achieved that by supporting only in-memory transient sessions while serializing / deserializing state from the payload. All the parts mentioned above are very inline with such an approach: an FSM state changes are atomic transactions, and the actors’ inner state is serializable. Stateless apps are very easy to scale, so our production infrastructure was straightforward: a single Docker image deployed to AWS ECS with autoscaling support. The only non-trivial aspect of it was the need to provide a Docker socket inside the application, so our isolated sandboxes could be spawned outside of the main app container. + +## Top-Level Agent + +Even when the FSM could produce viable apps, we weren’t sure about the user interface. It worked fine with an initial prompt, but writing great initial prompts is a rare skill (we fail at it too often!). + +Our journey to the current solution: + +1. **First attempt**: Separate prompt refinement step at the beginning (limited value) +2. **MCP experiment**: Wrapped FSM in an MCP server, controlled via MCP clients like Cursor or Claude Desktop (didn’t feel right) +3. **Final solution**: Separate top-level agent that gathers details from users and passes them to the core generation machine + +This approach aligned with other agents in the market (familiar UX for users), and matches our context encapsulation design. The top-level agent focuses on user needs and communication without handling technical details. It has limited tools to propagate these needs, creating a combination of: + +- Controllable, restricted API +- Flexible communication allowing users to refine requirements as needed + +![Image](https://cdn.neonapi.io/public/images/pages/blog/design-decisions-behind-app-build/image-4-1024x413-0705714a.png) + +Interesting fact: a top-level agent does not need to be too advanced. We currently use gemini-flash in our managed service for its balanced AI capabilities and latency; with more advanced models, we even observed behavior that is “too picky” behavior – better instruction following made them act as truly harsh critics significantly affecting overall latency. + +The top-level agent works fine as a base defense line keeping us from unwanted usage. We did not have a dedicated red teaming process dedicated to prompt hacking, but overall naive approaches to misguide the agent and generate apps for illegal or unethical needs were not successful; the same applied for attempts to use an agent for irrelevant tasks (“hey, how to boil eggs?”). + +## Checks and Validations + +Since we fixed the initial tech stack, we could implement solid checks to enforce app quality. TypeScript was our solid baseline—the language is well-known to all LLMs and provides an advanced type system and toolset to rely on. + +### Validation Evolution + +Our validation pipeline evolved gradually, with each check addressing specific failure modes: + +1. **TypeScript compilation** – The obvious starting point +2. **Unit tests** – Ensured backend API handlers do what they’re supposed to do; enforced separation of concerns. [LLMs are widely used in the industry to improve test coverage](https://arxiv.org/abs/2402.09171), and adding this step was crucial for app.build as well. +3. **Linters with custom rules** – Addressed LLM-specific failure modes (especially in the Sonnet 3.x era), such as: + - Using mocks to bypass tests + - Tricks with renamed imports + +Some of those errors evaporated after the base models were updated. However, some are still valuable – e.g., [a rule checking for empty values](https://github.com/appdotbuild/agent/blob/main/agent/trpc_agent/template/client/eslint.config.js#L7) on the frontend fixed many UI issues. + +4. **Playwright end-to-end smoke tests** – Enforced consistency between user prompts and generated apps; caught client-side errors that passed previous checks + +We kept Playwright as the last check for performance reasons—while linters run in under 100ms, Playwright tests typically require seconds. + +The current validation pipeline is solid, but should not be considered as a final revision – additional types of checks (e.g. property testing) can be introduced later to reduce the long tail of runtime errors happening to exotic generated apps. + +## Context Engineering + +Validation catches errors after generation, but prevention is better than cure. The key to reducing failures lies in what information we provide to our actors—and how we structure it. + +AI engineers these days avoid the term “prompt engineering” because it doesn’t really reflect the whole stack. For any modern LLM app, writing a good prompt isn’t enough—prompts aren’t magic spells, and AI engineering isn’t the Harry Potter universe. + +Early prompt engineers searched for the most effective phrases to express their needs in ways understandable by LLMs. **Context engineering**, however, is broader in scope: it defines what information is needed for LLM operations and how to structure it. Today, the specific wording in a prompt barely matters, but writing detailed instructions and maintaining relevant context is crucial. + +At the same time, overloading the context is a mistake—longer contexts mean less attention to important details and higher latency. Despite modern LLMs being able to handle large contexts in theory, empirical tests like “[needle in a haystack](https://github.com/gkamradt/LLMTest_NeedleInAHaystack)” demonstrate potential drawbacks. + +### Our Context Preparation Principles + +1. **Encapsulation is key** – Each step should only be familiar with the parts of the project relevant to that step. Unreasonably long context has three ways to make things worse: higher latency, higher costs for the API, and – most importantly – attention layer dilution leading to degradation of the final result. +2. **Dynamic user prompts, cached system prompts** – User prompts should be short and dynamic, while system prompts can be longer due to [prompt caching](https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching). We use Jinja2 templates to fill user prompts with project-specific details. +3. **System prompts for common failures** – System prompts can cover typical failures and provide guidance on tech stack nuances where LLMs struggle. +4. **Semi-automated prompt writing** – Most examples and rules in our system prompts were LLM-generated based on logs from failed app generation sessions. This feedback loop helped us solve many common issues in a short period. (Eventually, we’d like to automate this in a continuous manner from production logs.) + +### Example: Type Matching Guidance + +Postgres, Drizzle, and Zod have somewhat different default types for the same semantics. Popular failure modes include: + +- Decimal values (often used for money-related data in business applications) +- Confusion between nullable and optional fields + +Both issues were extracted from debug logs by the LLM (kudos to the Gemini team and their 1M context window!) and later added to the system prompt [here](https://github.com/appdotbuild/agent/blob/acdc1c6550e257dbd83b99e57955331a7e99b397/agent/trpc_agent/playbooks.py#L477). + +Overall, the idea of feeding the context with a few high-quality examples of user prompt – code pairs proved to be very effective in increasing the success rate of the code generation. Those who want to reduce the amount of manual work like writing those high-quality examples could leverage Deep Research-like tools. A single prompt like “_analyze the docs on TECH STACK OF YOUR CHOICE, its typical usage and problems; based on that generate a detailed prompt on how to use it for AI software agent_” can provide you with a really solid baseline for system prompt. + +In the future, we plan to advance the context management towards more automated ways, inspired by [TextGrad](https://github.com/zou-group/textgrad) and other text “differentiation” frameworks. + +## Tools We Rely on + +While our architectural decisions were crucial, none of this would have been possible without choosing the right foundation. Several key tools enabled our approach and deserve recognition for making the entire system viable; others were important during the exploration: + +- **Modern TypeScript ecosystem** – Made the whole solution possible. The combination of Drizzle, tRPC, and Zod was crucial for consistency. +- [uv](https://docs.astral.sh/uv/) – We started with raw requirements.txt and switched to uv later. It’s blazing fast and totally worth it for any Python project. We rate it a 10/10, and fully recommend it. [Astral](https://astral.sh/) is doing a great job building modern tooling for Python devs. +- [Dagger](https://dagger.io/) – Our second sandboxing solution (started with classic Docker). [Provides a modern, agent-native approach](https://docs.dagger.io/features/llm/): chaining methods with effective caching instead of manual container maintenance. Special features: + - Restricted file paths prevent actors from hacking rewards by drifting from project structure + - Elegant DevEx despite being a risky bet for production – we had a kind of adventure fixing concurrency issues too close to the release date. However, currently we consider it stable enough. +- [Langfuse](https://langfuse.com/) – Used for LLM tracing (and cost analysis), generally happy with it. However, we have been changing our internal agent structure too often and didn’t always have time to attach proper tracing. Maybe we’ll give it a second chance now that our code structure is more stable, as the current tracing situation is very basic. +- [TypeSpec](https://typespec.io/) – Early experiments with using TypeSpec to define and later generate APIs were indeed promising, especially for complex apps with non-trivial logic. Furthermore, TypeSpec is very extendable, so one can add custom code emitters on top of the base specification – for instance, we connected it with [Cucumber](https://cucumber.io/) to support BDD. While it was not part of the main release, we found this approach very viable for the future development. + +## Summary and Future Directions + +During this project, we achieved our initial goal: creating reliable apps deployable to Neon’s infrastructure. The limited scope of full-stack CRUD web apps helped us ensure quality. At the same time, we fully understand this approach’s limitations, and are currently working on a more universal solution, enabling users to build more advanced apps using their favorite technologies. + +Among the error logs, we’ve observed clear evidence of users’ needs for more flexibility. The lessons learned during this first version will help tremendously as we expand! + +Give app.build a try [on the managed service](https://www.app.build/) or [hack around locally](https://github.com/appdotbuild/agent) with our fully open-source code! diff --git a/content/blog/posts/designers-who-code.md b/content/blog/posts/designers-who-code.md new file mode 100644 index 0000000000..fb9f13c0b2 --- /dev/null +++ b/content/blog/posts/designers-who-code.md @@ -0,0 +1,90 @@ +--- +title: 'Designers Who Code: Can AI End Your Papercut Backlog?' +description: >- + We’ve implemented Devin and other AI tools in the Neon Design team—and it + worked. +excerpt: >- + Designers often spot a long list of UX papercuts—those small, frustrating + issues that quietly erode a products quality—but development teams are often + focused on bigger priorities and buried under long backlogs. This is where + vibe coding can empower designers to break this cycle.... +date: '2025-04-28T22:56:13' +updatedOn: '2025-06-25T01:55:09' +category: ai +categories: + - ai +authors: + - carl-thomas + - lachezar-petkov +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/designers-who-code/cover.jpg' + alt: null +isFeatured: false +seo: + title: 'Designers Who Code: Can AI End Your Papercut Backlog? - Neon' + description: >- + Designers often spot a long list of small, frustrating issues that quietly + erode a products quality. This is where AI coding can help. + keywords: [] + noindex: false + ogTitle: 'Designers Who Code: Can AI End Your Papercut Backlog? - Neon' + ogDescription: >- + Designers often spot a long list of small, frustrating issues that quietly + erode a products quality. This is where AI coding can help. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/designers-who-code/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/designers-who-code/neon-designers-who-code-1-1024x576-6d1fc31b.jpg) + +Designers often spot a long list of UX papercuts—those small, frustrating issues that quietly erode a products quality—but development teams are often focused on bigger priorities and buried under long backlogs. This is where vibe coding can empower designers to break this cycle. + +Now, with tools like [Devin](https://devin.ai/), we’re changing that story. By leveraging AI codegen, designers at Neon can “vibe code” away the papercut backlog—shipping real, production-ready improvements without waiting in the queue. This shift empowers us to raise the quality bar, act on what we see, and deliver a smoother and more polished experience for our users experience. + +## Shipping improvements with Devin + +Devin is an AI-powered code generation tool designed for software development. It understands our system architecture and repo structure, so designers can prompt it with scoped changes and get back a ready-to-review pull request—turning design intent into real product improvements, fast + +Lets take a look at Devin in action by one of my Product Designers, Lacho. + +**A recent example of using Devin:** + +The front-end engineer I was partnered with was pulled into another project, leaving a minor item unresolved—a simple confirmation modal tied to an action. With access to Devin, I wrote a clear, detailed prompt outlining the requirements, asked it to sanity-check its understanding, and let it take a first pass. The ideal outcome was Devin providing the code needed to implement the modal with an action and prep the PR for review. + +
+

On the Backup & Restore page there are snapshot cards. Each snapshot card has an ellipsis/’more’ icon button. Each of these buttons opens a menu with a single ‘Delete’ action item. If the user selects Delete, their snapshot will be deleted forever.



Add a modal dialog after the user clicks delete with a confirmation message and two buttons: ‘Cancel’ and ‘Delete’. The copy in the modal must be ‘This is a permanent operation. Deleted snapshots cannot be recovered.’ and must be styled as standard body text. Use other modals for reference.

+
+ +Devin submitted the PR, and I looped in the original engineer for a quick review. He flagged a minor adjustment, which Devin handled in a follow-up commit. A few hours later, the change was merged, unblocking the flow–only needing an Engineer to approve the PR. This enabled design to unblock engineering and ensure we hit our release deadlines. + +**What Devin helped launch:** + +![Image](https://cdn.neonapi.io/public/images/pages/blog/designers-who-code/ad4nxfgbv1yuntivp0xcannnhgs7jukmwytgrdchya6wpd9cjwmompddgm4dhsbxrw2tgbpt33-dnbn7khrvb-o0eniq8umrbb3ng1go7thmnd2cc2-1whddsdnxingkvasnjeoq-ea963484.png) + +Imagine if you did this the old way you. You submit a task with a figma screen, it goes into a backlog and you could be waiting weeks for this to be prioritized then shipped. This is why tools like Devin can really aid in unlocking the best capabilities of your design team. + +### Limitations + +Devin is a powerful tool, but it’s not a silver bullet. On larger tasks—like rewriting an entire page layout—it still behaves like a junior developer: it needs a lot of context, guidance, and review. These bigger changes often require hands-on intervention from designers or engineers to get across the finish line. + +But when it comes to smaller, well-scoped improvements, Devin shines. It can quickly scaffold solutions and handle foundational work, giving us a head start and freeing up engineering bandwidth. It’s not magic, but it’s a meaningful shift—putting more capability in designers’ hands and helping us deliver a better experience for our customers, faster. + +## What’s Next: Empowering Designers, Elevating Product + +### Designers Who Ship + +Giving designers the ability to submit code is a game-changer. It empowers us to raise the bar on quality, ship improvements faster, and take pressure off our engineering partners. That’s why my team is doubling down on code-based submissions—working side by side with engineers to ensure Neon delivers the best Postgres experience possible. + +### Supercharging Our Design System + +We’re especially excited about how tools like Devin and V0 can help us evolve our design system. Imagine being able to update components, roll out accessibility improvements, and launch new capabilities—quickly and at scale. This kind of agility keeps our product polished and consistent, while letting us respond to user needs in real time. + +### Bridging Design and Research with Codegen + +We’re also exploring how codegen tools like V0 and Lovable can transform our research process. Instead of relying on static Figma prototypes or waiting on live builds from engineering, we can now spin up interactive, production-like prototypes in record time. This means we can test real experiences with users earlier, learn faster, and make smarter decisions about what to build next. It’s a shift that turns research from a bottleneck into a superpower. + +By unlocking these new capabilities, we’re not just designing better products—we’re building a culture where designers are hands-on makers, collaborators, and drivers of real change. + +--- + +_Neon is a serverless Postgres database used by v0 and Replit Agent. It also works like a charm with AI IDEs like Cursor [via its MCP Server](https://neon.tech/guides/cursor-mcp-neon). [Sign up for Neon](https://console.neon.tech/signup) (we have a Free Plan) and start building._ diff --git a/content/blog/posts/dev-from-heroku-to-neon.md b/content/blog/posts/dev-from-heroku-to-neon.md new file mode 100644 index 0000000000..ce7ad5bfdf --- /dev/null +++ b/content/blog/posts/dev-from-heroku-to-neon.md @@ -0,0 +1,129 @@ +--- +title: 'From Heroku to Neon: The dev.to Story' +description: Serverless Postgres for a platform used by millions +excerpt: >- + “We didn’t just want a better Postgres database, we wanted a partner who + shared our focus on developers. With Neon, we finally have a setup that scales + with our platform and our values.” (Peter Frank, DEV Co-Founder) DEV needs no + introduction. For millions of developers getting s... +date: '2025-06-19T16:04:37' +updatedOn: '2025-06-19T23:20:05' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dev-from-heroku-to-neon/cover.jpg + alt: null +isFeatured: true +seo: + title: 'From Heroku to Neon: The dev.to Story - Neon' + description: >- + Millions of developers visit dev.to daily - now running on Neon. Here’s why + they left Heroku and how we’re supporting devs together. + keywords: [] + noindex: false + ogTitle: 'From Heroku to Neon: The dev.to Story - Neon' + ogDescription: >- + Millions of developers visit dev.to daily - now running on Neon. Here’s why + they left Heroku and how we’re supporting devs together. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dev-from-heroku-to-neon/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dev-from-heroku-to-neon/neon-from-heroku-1-1024x576-0a79df39.jpg) + +
+

“We didn’t just want a better Postgres database, we wanted a partner who shared our focus on developers. With Neon, we finally have a setup that scales with our platform and our values.” (Peter Frank, DEV Co-Founder)

+
+ +[DEV](https://dev.to/) needs no introduction. For millions of developers getting started with programming, [dev.to](https://dev.to/) is their town square. With over 3 million registered users, ~500k daily pageviews, and 500+ new community articles and discussions published daily, the DEV community serves a global audience with tutorials, product announcements, in-depth technical discussions, career advice, and more. + +Behind the scenes, this means handling a constantly growing database workload with user accounts, reactions, comments, page views, editorial tools, partner integrations… + +All of it runs on Postgres. It started on Heroku, now it’s powered by Neon. + +## Why Heroku Wasn’t Enough + +Heroku Postgres was a reliable backend for the early days of DEV, but as the platform scaled to millions of users, they began looking for a service that better fit their needs. + +The biggest issue was rigidity. Heroku’s scaling model made it hard to adjust resources cleanly or predictably – the DEV team was paying for unused capacity across the board, and there was no easy way to right-size compute independently from storage or to scale read replicas on demand. + +
+

“Our Heroku bill was thousands of dollars a month, and we knew this bill was inflated. We either had too many or too few resources allocated to our database, and adjustments typically were a big project with downtime” (Ben Halpern, DEV Co-Founder)

+
+ +Heroku’s rigidity translated into: + +- Heavy instances for modest workloads – aka, high bills +- Limited replica options, with no ability to pause or scale dynamically +- Operational overhead – setting up follower databases for analytics or experimentation required effort +- Lost engineering time + +As DEV’s needs evolved, it became clear that Heroku’s serverful model was holding them back. They needed more control and a better developer experience. + +## Finding Neon + +The team started looking for a new Postgres provider that could support their scale and that aligned with how they wanted to build. They looked at _all the options_, from enterprise-focused platforms to developer-centric tools. Neon stood out immediately. + +
+

“What first attracted us to Neon was the efficient scaling, it solved our core issues right away. But what kept us interested were all the thoughtful developer-experience wins. You just don’t see that from general-purpose cloud Postgres providers.” (Ben Halpern, DEV Co-Founder)

+
+ +### Serverless Postgres with autoscaling (no overprovisioning) + +The immediate win was how Neon eliminated the need to overprovision. With Neon, there are no instance sizes – instead, [Neon is serverless:](https://neon.tech/docs/get-started-with-neon/why-neon) CPU, memory, connections, and storage scale up and down with actual usage and responding to [highly performant algorithms](https://neon.tech/docs/guides/autoscaling-algorithm). + +Capacity spikes? Neon handles it. Quiet periods? Neon scales down automatically. No manual resizing, no guessing at tiers, no paying for “just in case.” It’s that simple. + +### Built for developers, not DBAs + +Neon is Postgres for developers. DEV valued this focus on DX right away: Neon’s intuitive UI and [API-first approach](https://neon.tech/blog/provision-postgres-neon-api) that made it easy to automate database operations, [branches](https://neon.tech/blog/instantly-copy-tb-size-datasets-the-magic-of-copy-on-write) could be created and torn down programmatically, they were integrations with platforms like [Vercel](https://neon.tech/blog/neon-vercel-integration) – making full-stack previews as simple as opening a pull request. + +And just like DEV, Neon was accessible by design. There’s a generous free plan, no onboarding friction, and no vendor lock-in. Just fast, flexible Postgres, built to match the way modern developers work. + +
+

“There was this magic moment with Neon. We could just offer a path for our community members to spin up a serverless Postgres database in seconds. No infra setup, no complications, no sales process – just start building.” (Peter Frank, DEV Co-Founder)

+
+ +### Lightweight read replicas that just work + +The DEV team had experimented with read replicas on Heroku, but every replica was a full instance – expensive to maintain and hard to justify when the bill came. + +Neon’s [replicas](https://neon.tech/docs/guides/read-replica-guide) are fundamentally different. Spinning up a replica doesn’t mean duplicating your whole environment; instead, Neon can launch additional compute nodes that read from the same underlying storage, making it simple to offload read-heavy workloads without the cost and complexity of provisioning full clones. + +The DEV team can now launch lightweight, read-only replicas for analytics or internal automation without any overhead in maintenance or costs. They’re connecting tools like [Metabase](https://www.metabase.com/) and their in-house workflow automation scripts to these replicas, with plans to do more over time. + +### Real data, real resting + +
+

“We plan to make much more flexible use of multiple environments for testing against real data. That’s something Neon makes significantly easier.” (Ben Halpern, DEV Co-Founder)

+
+ +DEV’s architecture is a Ruby on Rails app deeply integrated with Postgres via ActiveRecord. Their main database stores most of the platform’s application logic and state – prod was 1.2TB at the moment of migration and it’s growing steadily. + +With such a large dataset, Neon’s [branching model](https://neon.com/flow) offers a major advantage over conventional setups like Heroku. Instead of provisioning separate database instances for staging or QA (and constantly working to keep them in sync), [branches](https://neon.com/docs/introduction/branching#what-is-a-branch) in Neon act like lightweight clones of your production data. + +Branches share the same underlying storage but operate in isolated compute environments, so you can spin up dev, testing or preview environments in seconds, with no duplication or overhead. And because branches can be created and torn down programmatically, they fit naturally into [modern CI/CD workflows](https://neon.com/docs/get-started-with-neon/workflow-primer). + +## More Than a Migration: A Developer-First Partnership + +
+

“We didn’t just want a better database: we wanted a partner who shared our focus on developers.” (Peter Frank, DEV Co-Founder)

+
+ +This developer focus was in full alignment with DEV’s ethos, and it laid the foundation for a perfect synergy. What began as a technical migration quickly became a deeper collaboration, with the two teams started collaborating regularly on shared content and campaigns to educate and empower developers, together with special offers. + +Speaking of…
+ + +If you’re exploring Neon for the first time, [sign up with this link](https://console.neon.tech/app/?promo=OZHcuQtJHCLlOMhpi3YeZasOF) and receive **$100 in Neon credits**. And if you’re already part of the DEV community, check out the [DEV++](https://dev.to/++) deals for exclusive perks, including extra Neon credits and member-only resources. + + +## Looking Ahead + +Like many others, DEV grew frustrated with the waste and rigidity of serverful architectures. When they found Neon, they didn’t just switch database providers – they entered a partnership built on a shared belief: that developers deserve Postgres infrastructure that’s efficient, modern, and built for how software gets made today. + +Whether you’re scaling a production app or hacking on a side project, Neon makes it easy to get started with Postgres. [Sign up to Free plan](https://console.neon.tech/signup) and start building right away, or [reach out to our team](https://neon.tech/contact-sales) if you (like DEV) are exploring leaner Postgres solutions for your business. diff --git a/content/blog/posts/development-environments-for-aws-rds-using-neon-postgres.md b/content/blog/posts/development-environments-for-aws-rds-using-neon-postgres.md new file mode 100644 index 0000000000..cfc2e5237d --- /dev/null +++ b/content/blog/posts/development-environments-for-aws-rds-using-neon-postgres.md @@ -0,0 +1,185 @@ +--- +title: Optimize your AWS RDS Dev Environments with Neon Postgres +description: 'Neon is Postgres built to increase dev velocity, cost efficiency' +excerpt: >- + AWS RDS for PostgreSQL is the most widely-used hosted Postgres solution there + is, so naturally we hear a lot from developers that use it. While it can be a + solid offering, it’s quite lacking when it comes to developer experience, and + doesn’t support modern developer workflows. We... +date: '2024-07-16T12:24:14' +updatedOn: '2024-12-24T18:23:22' +category: workflows +categories: + - workflows +authors: + - brad-van-vugt +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/development-environments-for-aws-rds-using-neon-postgres/cover.jpg + alt: null +isFeatured: false +seo: + title: Optimize your AWS RDS Dev Environments with Neon Postgres - Neon + description: >- + Switching your development environments from RDS to Neon can speed up your + development process (and save you money). + keywords: [] + noindex: false + ogTitle: Optimize your AWS RDS Dev Environments with Neon Postgres - Neon + ogDescription: >- + Switching your development environments from RDS to Neon can speed up your + development process (and save you money). + image: >- + https://cdn.neonapi.io/public/images/pages/blog/development-environments-for-aws-rds-using-neon-postgres/social.png +--- + +Post image + +[AWS RDS for PostgreSQL](https://aws.amazon.com/rds/postgresql/) is the most widely-used hosted Postgres solution there is, so naturally we hear a lot from developers that use it. While it can be a solid offering, it’s quite lacking when it comes to developer experience, and doesn’t support modern developer workflows. We often hear of developers switching to Neon exactly for this reason, especially for their local environments. + +In this article, we’ll dive into the specific challenges of developing on RDS and show you a solution we often recommend: **using Neon for development, even when you must keep RDS as your production database.** + +
+

📚 This article is Part I of a series. Click here to read Part II: How-to guide for setting up your dev environments in Neon while keeping prod in RDS

+
+ +## Why AWS RDS is slowing you down + +Using RDS has its advantages, and can be a reasonable default choice. But the cost of using RDS for development environments is incredibly high, and developers often need to find alternative (and hacky!) workarounds. + +### Provisioning RDS databases is a slow and manual process + +
+

“The RDS developer experience is not quite there. The AWS console and APIs are quite convoluted and require extensive setup and configuration to achieve even basic tasks”

+Guido Marucci, co-founder at Cedalio +
+ +First, provisioning new RDS instances can take a while, requiring manual configuration and constant oversight to ensure they are appropriately sized and ready for use. Trying to navigate this slows down projects and costs valuable engineering hours. Developers want to build software! They don’t want to spend time provisioning databases. + +### Developer collaboration is hard + +
+

“When we used a shared RDS instance for development, it was hard to properly test new features and changes without impacting the work of other teammates”

+Camelia Smeria, Lead Engineer at Proposales +
+ +RDS is simply not built to support team workflows where multiple developers need to interact with the same dataset at the same time. It’s easy to quickly run into problems involving shared infrastructure, “noisy neighbors”, and even data consistency. Developers need to know their database is available, up-to-date, and not being messed with by testing environments or other developers at the same time. + +### It takes a lot of work to keep data in sync + +
+

“When we were using RDS, we had trouble keeping the same environment on my computer, my developer’s environment, and production”

+Léonard Henriquez, co-founder and CTO, Topo.io +
+ +Some teams choose to use separate RDS instances for development, staging, and testing, which can require complex data consistency pipelines. Managing data across all these environments is time consuming and prone to errors, often leading to data discrepancies and frustrating bugs. Most solutions involve regular data dumps and imports, which can further increase complexity and be disruptive when developers are trying to work. + +### Complexity grows with the number of RDS instances + +
+

“RDS becomes a bottleneck if you don’t have full-time DevOps dedicated to it” 

+Joey Teunissen, CTO at OpusFlow +
+ +The problem with managing RDS instances isn’t just about dealing with outdated consoles and APIs; it’s also about the overall developer experience of provisioning AWS infrastructure. As your fleet of RDS instances grows, the manual setup and configuration work grows too. Tools like Terraform can help, but also add a layer of abstraction that needs to be maintained and debugged. + +## Building on Neon: how it’s different from AWS RDS + +For agile dev teams, the ability to quickly build, test, and deploy new features is critical for success. Teams typically need to create and maintain datasets or local environments that mirror production, and using AWS RDS for this purpose can slow down development cycles and introduce unnecessary complexity during testing phases. + +Enter [Neon](https://neon.tech/), a modern serverless Postgres solution built to streamline development workflows. Due to its unique architecture that natively [decouples storage and compute](https://neon.tech/blog/architecture-decisions-in-neon), **Neon is more agile, more efficient, and more developer-friendly than any other hosted Postgres – which makes it perfect for local dev and testing environments.** + +Let’s break down why. + +### Provisioning new environments takes only a second + +
+

“I created 15 Neon databases in the time it took to spin up one RDS instance”

+Heard on Neon Discord +
+ +With Neon, provisioning new environments is nearly instantaneous. Thanks to its serverless architecture, you can spin up a new database (or branch an existing one) in less than a second. This rapid provisioning accelerates development cycles and allows you to add your database to your existing workflow without being slowed down. + +### Modern development workflows are baked in + +
+

“We no longer have to set up an actual testing database instance and make sure the data is always synced with production. We now spin up a Neon branch when we need to and then tear it down via the create/delete Github Actions”

+Angelina Quach, Software Engineer at Shepherd +
+ +Speaking of workflows: [Neon integrates into your pipelines by allowing you to automatically create branches of your database for testing and development](https://neon.tech/flow). You can spin up an isolated environment that mirrors production, make your changes, and then tear it down when you’re done, all within seconds and with minimal effort. Workflow tools like GitHub Actions can be used to automate the creation and deletion of these branches, ensuring your workflow is both efficient and seamless. + +
+Post image +
Neon allows you to incorporate the database into existing CI/CD pipelines. Collaborating with your team is as easy as collaborating in code.
+
+ +### Develop and test with production-like data + +
+

“Neon allows us to develop much faster than we’ve ever been used to. Instead of putting a lot of effort into getting a synthetic dataset resembling production data that we can reliably run tests within Docker or local Postgres, we just test in a Neon branch, with a perfect copy of production data without leaving our VPC”

+Alex Klarfeld, CEO and co-founder of Supergood.ai +
+ +Neon workflows are based on [database branching](https://neon.tech/docs/manage/branches) — allowing you to develop and test with data that very closely mirrors your production environment. Instead of relying on synthetic datasets or dealing with the complexities of maintaining local data scripts, you can create a branch with a perfect copy of your production data. This approach ensures your local testing is more reliable and reduces the risk of encountering issues when deploying to production. + +
+Post image +
You can branch databases via copy-on-write. All branches are isolated, but they share the same storage, reducing your costs. Branches can be created and deleted immediately.
+
+ +### Database management is easy + +
+

“We’ve been able to manage 300K+ Postgres databases via the Neon API with only one engineer”

+Himanshu Bhandoh, Software Engineer at Retool +
+ +Neon has a straightforward, API-driven developer experience. The Neon API allows for easy automation and management of thousands of databases, reducing the overhead typically associated with database administration. This simplicity means you can focus on developing great software rather than managing infrastructure, even as your team and database needs grow. + +## How much is this going to cost me? + +Perhaps this is the best part: thanks to the economics of database branching and [scale to zero](https://neon.tech/blog/why-you-want-a-database-that-scales-to-zero) functionality, building with Neon is much cheaper than RDS. Even if you keep your production data in an RDS instance, **you can save nearly 40% of your monthly costs by moving your non-production databases to Neon** — all while getting a superior developer experience. + +
+Post image +
In Neon, databases automatically scale to zero after 5 minutes of inactivity. You only pay for compute when you’re using your databases. This is great for pre-prod environments.
+
+ +### Example RDS cost breakdown + +Consider this typical AWS RDS deployment: + +- 1 production database (db.m3.2xlarge: 8 CPU, 30 GB memory) – 100 GB +- 3 test databases (db.m1.medium: 1 CPU, 3.75 GB memory) -Used 3 hours / day on average – 50 GB +- 5 dev databases (db.m1.medium: 1 CPU, 3.75 GB memory) – Used 8 hours / day on average – 10 GB + +Total monthly costs: $1,301.83. + +### Example Neon cost breakdown + +Now, let’s imagine moving the non-production environments to Neon: + +- 1 production database in RDS (db.m3.2xlarge: 8 CPU, 30 GB memory) – 100 GB +- Scale plan in Neon + - Main database branch with a copy of production (2 CU) – 100 GB – Used 2h / day on average for data transfer. +- 3 test database branches (1 CU) – Used 3 hours / day on average – Full copy of main – 100 GB +- 3 test database branches (1 CU) – Used 8 hours / day on average – Full copy of main – 100 GB + +Total monthly costs: $783.68 — **this is 39.8% cost savings**. + +## Next step: Build a Neon Twin + +This blog post is just **Part I** of a series on how to build non-prod environments in Neon. The series continues with comprehensive guides that walk you through the process of setting things up—check them out: + +**Part II:** [How to automatically sync data from RDS to Neon using GitHub Actions](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon) + +**Part III:** [How to set up Slack notifications to automatically monitor the status of your dump/restores](https://neon.tech/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows) + +**Part IV:** [How to deploy changes tested on Neon back to prod in RDS](https://neon.tech/blog/neon-twin-deploy-workflow)
+ +_Even if we use “RDS” in this content, the workflows described in those guides also work for other managed Postgres._ + + +If you're looking for a detailed comparison of Neon vs RDS, check out [neon.tech/rds](https://neon.tech/rds). + diff --git a/content/blog/posts/devin-and-neon-mcp-marketplace.md b/content/blog/posts/devin-and-neon-mcp-marketplace.md new file mode 100644 index 0000000000..5909b74ac4 --- /dev/null +++ b/content/blog/posts/devin-and-neon-mcp-marketplace.md @@ -0,0 +1,82 @@ +--- +title: Devin Integrates with Neon Through New MCP Marketplace +description: Give Devin access to your Neon database using the Neon MCP +excerpt: >- + Cognition Lab’s AI software engineer, Devin, recently had its capabilities + expanded by opening up an MCP (Model Context Protocol) server marketplace. + With this, and Neon’s MCP server, you can now have Devin interact with your + Neon database directly using a comprehensive set of to... +date: '2025-07-24T20:44:35' +updatedOn: '2025-07-24T20:44:37' +category: ai +categories: + - ai +authors: + - sam-harrison +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/devin-and-neon-mcp-marketplace/cover.png + alt: null +isFeatured: true +seo: + title: Devin Integrates with Neon Through New MCP Marketplace - Neon + description: >- + You can now have Devin interact with your Neon database directly, using a + comprehensive set of tools via MCP server. + keywords: [] + noindex: false + ogTitle: Devin Integrates with Neon Through New MCP Marketplace - Neon + ogDescription: >- + You can now have Devin interact with your Neon database directly, using a + comprehensive set of tools via MCP server. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/devin-and-neon-mcp-marketplace/social.png +--- + +Cognition Lab’s AI software engineer, Devin, recently had its capabilities expanded by opening up an MCP (Model Context Protocol) server marketplace. With this, and [Neon’s MCP server](https://neon.com/docs/ai/neon-mcp-server), you can now have Devin interact with your Neon database directly using a comprehensive set of tools. Here, let’s explore a practical scenario using Devin and the Neon MCP to see how one can fully automate a bug fix or feature. + +## A Practical Example: Using Devin to Fix a Slow API + +### Problem identified: performance bottleneck + +Consider an e-commerce website with a “top sellers” table on its homepage, which shows the most sold products of the day. The backend endpoint might look like this: + +```python +@app.get("/top-sellers") +async def top_sellers(limit: int = 5): + sql = """ + SELECT + product_id, + SUM(quantity) AS total_qty + FROM sales + WHERE sold_at >= NOW() - INTERVAL '1 day' + GROUP BY product_id + ORDER BY total_qty DESC + LIMIT $1; + """ + async with app.state.pool.acquire() as conn: + rows = await conn.fetch(sql, limit) + return rows +``` + +However, when running this query with a large volume of sales, there’s quickly a drop in performance, and seconds of latency. To fix this, let’s hook up Devin with Neon’s MCP and GitHub integrations, and ask it to troubleshoot the problem, come up with a fix, and write a report. + +### Diagnosis and resolution: Devin in action + +Right away, Devin looked into the query plan using the `explain_sql_statement` tool which analyzes queries and provides suggestions. Then, using `prepare_query_tuning`, it was able to implement the suggested approach on a new branch to safely test the optimizations. + +Unfortunately, after applying the indexes and analyzing the new query plan, there were no gains. However, because Devin works in an iterative loop, the failed attempt did not stop the process, and instead it tried another approach using a materialized view. + +**This strategy resulted in a 97% performance increase.** Devin’s analysis also included the trade-off of potential data staleness with a materialized view, and recommended using the `pg_cron` extension to refresh the view hourly. + +Following the successful test, Devin created a pull request on GitHub including the necessary change in the API to query the new materialized view, and a detailed explanation of the changes, and the reasoning behind them. The only remaining manual step was to review the report, and merge the changes. + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/devin-and-neon-mcp-marketplace/ad4nxexs3sxi7k1b3ah4lrr7y1tm9q9oqimnjgce4pmiw5nhalhmeh2hsux3emjxoey8npkz1zz4snvcxa-gtby7qlgo8tjxbl-kxle2hayk3gfo2jg4andpj-zubqoore5nkiacxzg-bc9558fd.png) + +Because the Neon MCP server was designed as a work-flow and tool first API for LLMs, rather than just a mirror of the normal programmatic API, Devin can take on these large and complex tasks more safely and confidently. Any query performance optimization test run on the database will be on a new [branch](https://neon.com/docs/introduction/branching), protecting your data if things go awry. + +## Devin’s Agentic Toolkit is Expanding + +With the integration of MCP servers, Devin can now interact directly not only with a Neon database but also with other common development tools like Sentry, DataDog, Linear, and Slack. As the ecosystem of MCP servers grows, the range of challenges these AI agents can take on increases. + +Give [Devin](https://app.devin.ai/settings/mcp-marketplace) a try with your [Neon database](https://console.neon.tech/signup)! diff --git a/content/blog/posts/docker-compose-neon-branches-testing.md b/content/blog/posts/docker-compose-neon-branches-testing.md new file mode 100644 index 0000000000..823084e175 --- /dev/null +++ b/content/blog/posts/docker-compose-neon-branches-testing.md @@ -0,0 +1,186 @@ +--- +title: 'Beyond Docker Compose: An Alternative for Deploying Postgres for Testing' +description: Get consistent test data with Neon branches +excerpt: >- + Testing applications with Postgres presents a common challenge: how do you + provide each developer and CI pipeline with isolated, consistent databases to + test against? Docker Compose has emerged as the standard solution, offering + containerized Postgres instances that can be spun u... +date: '2025-02-24T19:18:32' +updatedOn: '2025-02-24T19:24:34' +category: postgres +categories: + - postgres + - workflows +authors: + - jeff-christoffersen +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/docker-compose-neon-branches-testing/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Beyond Docker Compose: An Alternative for Deploying Postgres for Testing - + Neon + description: >- + If you use Docker to deploy Postgres for your ephemeral testing + environments, try Neon branches—as fast a Docker but with persistent data. + keywords: [] + noindex: false + ogTitle: >- + Beyond Docker Compose: An Alternative for Deploying Postgres for Testing - + Neon + ogDescription: >- + If you use Docker to deploy Postgres for your ephemeral testing + environments, try Neon branches—as fast a Docker but with persistent data. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/docker-compose-neon-branches-testing/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/docker-compose-neon-branches-testing/neon-beyond-docker-1024x576-f6a9953d.jpg) + +Testing applications with Postgres presents a common challenge: how do you provide each developer and CI pipeline with isolated, consistent databases to test against? [Docker Compose](https://docs.docker.com/compose/) has emerged as the standard solution, offering containerized Postgres instances that can be spun up alongside your application with a single command. + +However, as testing requirements grow more complex, teams often discover the limitations of this approach. Populating test data into a local database container can be cumbersome, managing test data between runs can be tricky, and CI pipelines can slow down as they repeatedly spin up new database instances. What started as a simple solution can become a source of testing headaches. + +## Why Use Docker Compose for Testing with Postgres? + +Since running Postgres for testing requires a multi-container setup, one for Postgres and one for your application, it is clear why many developers turn to Docker Compose for testing. It not only provides a way to test your application in an isolated, reproducible, and easy way but also removes the complexity of installing and configuring dependencies like a Postgres database locally. You just need to use the [Postgres official docker image](https://hub.docker.com/_/postgres) in your Compose app, as it contains all the necessary code, tools, dependencies, and libraries that your application needs. Here are some of the benefits of using official Postgres image from Docker: + +- **Quick setup**: You can start the Postgres container that has preconfigured settings and avoid any complex manual setup. +- **Consistency**: Docker ensures that all the team members and CI/CD environments use the same database version and configurations. +- **Automated cleanup**: Once the testing is finished, docker containers can be stopped and removed with very simple commands. This helps in preventing resource leakage. + +A simple `docker compose.yml` file to spin up the Postgres database for testing looks something like this: + +```bash +services: + postgres: + image: postgres:17 + environment: + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + POSTGRES_DB: testdb + ports: + - "5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + +volumes: + pgdata: +``` + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/docker-compose-neon-branches-testing/ad4nxfnkexuare51nd0gslynpalejtuupfwzmjied1x0hkmmo8q9gkdtb2mp8rggztufeuivf1dq6qoowlexn8hd5frmtotbwj7fwyocjbawc67kpbuncodom-6leo-lamdxkqh-d987da26.png) + +Let’s break down this Docker Compose file to understand each component: + +1. `services`: Each service in the services section defines a different container that makes up your application and includes configuration data to ensure they work correctly and can communicate with each other. +2. `services.postgres`: This is the specific service that will be created when you run Docker Compose. This service, named postgres, will run a Postgres database using the official Postgres Docker image and make it available to your local network on port 5432. The service definition also defines the databases name and what credentials can be used to log into the database. +3. `volumes`: This section declares any Docker volumes you want Docker Compose to create and manage as a part of your application. Because the postgres container uses a docker volume named `pgdata` to store persistent data, it is necessary to also declare it as a root-level object in your compose definition under the volumes section. + +Now all that’s left is to start the database. For this, you simply need to run this command: + +```bash +docker compose up +``` + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/docker-compose-neon-branches-testing/ad4nxeaylkp3avrfqxrf-i0-jpuuxzdshd984mrvbgq32elnsrc4-1koldn9kdnks1jykmrvx4ajdvxzepomwesxsv1pjeps-xjrtpxjse2mjvxagm9kdmlvcaqr1utcz0arx8sg-22098b72.png) + +This launches a Postgres container with a test database ready for application testing.
+ +## The Limitations of Docker Compose for Testing + +While Docker Compose simplifies the setup of Postgres for testing, it has a few shortcomings that can impact efficiency, reliability, and scalability. These challenges become more apparent in more complex testing environments, especially CI/CD pipelines and large-scale projects. + +### Populating your database: Ensuring data is present for testing + +Setting up small, static datasets in Postgres containers is relatively straightforward, either by mounting an initialization SQL script or using testing frameworks to generate fixture data during test execution. However, populating more complex or production-like datasets is far more challenging. While exporting a Postgres database volume into a Docker registry is possible, Docker Compose currently lacks support for using exported volumes when deploying services. This leaves developers primarily with a few suboptimal approaches to populating larger datasets for testing: + +- **Embedding data in a custom database image**. Embedding the database data directly in a custom Docker image can lead to excessively large images and cumbersome build processes that are difficult to maintain. +- **Including migration containers in their compose app**. Executing data seeding or migration jobs in separate helper containers before testing can significantly slow down test execution as the compose app must wait for the job to complete before starting their tests. +- **Leveraging shared staging databases**. Using a shared centralized database in a staging or similar test environment can lead to data integrity problems and increase test downtime by introducing a single point of failure. + +All of these options introduce inefficiencies, making dataset management a notable limitation of Docker Compose. + +### Persistence Issues: Cleaning up containers and volumes after testing + +Another significant challenge with Docker Compose is managing the persistence of test data. By default, Postgres stores data inside the container’s file system or in mounted persistent volume (pgdata in the example above). + +This often leads to unintended data persistence between test runs, where the current run can be affected by the data stored in previous runs. This issue can affect test repeatability and lead to [data contamination](https://www.holisticai.com/blog/overview-of-data-contamination) during testing. Also, if the container is not properly cleaned up, Postgres instances may occupy more storage over time, consuming disk space in the environment. Moreover, in the case of CI/CD pipelines, leftover volumes can slow down builds and create state inconsistencies across different test executions. + +The most straightforward solution to this problem is to run the docker compose down -v command after your test execution. However, this also requires additional steps and automation to ensure a fresh start for each test, and means that the database has be be re-initialized for each test run, including possibly seeding or migrating all new test data into the database, which can slow down testing significantly. Another approach could be to leverage Docker’s tempfs file system which uses the container’s RAM instead of a persistent storage volume, but this can significantly increase the container’s RAM usage and lead to resource contention. + +### Resource consumption: Running local databases can be heavy on RAM/CPU + +Postgres is an always-on database that consumes CPU, memory, and disk I/O even when not actively in use. This is why running Postgres with Docker Compose adds overhead to local development and testing environments. + +If you deploy a Postgres container locally alongside other applications, your system can be sluggish. On CI/CD servers, spinning up Postgres in each test run can slow down test execution. Finally, large datasets require higher memory and storage. This can cause performance issues when multiple team members or tests are running concurrently. + +## Database Branches: A Smart Alternative + +The challenges with using a local Postgres container with Docker Compose raise a fundamental question: Do we need to run a full local database instance for every test environment? Modern cloud architectures suggest an alternative approach: database branching. + +Database branching applies Git-like principles to database management. Just as Git creates lightweight copies of your codebase, database branching creates isolated copies of your database. These branches share underlying storage but maintain complete isolation, allowing developers to: + +- Create instant, isolated test environments +- Run parallel tests without resource conflicts +- Clean up automatically after test completion + +The key innovation behind database branching is the separation of storage and compute. Unlike traditional databases, where these are tightly coupled, branching architectures use copy-on-write mechanics to create instant copies of data. When you create a branch: + +1. A new isolated compute instance is provisioned +2. The branch shares the original data storage +3. Any changes are written only to the branch +4. The original data remains completely unchanged + +We can see this in action with [Neon](https://neon.tech/home). Here’s a simple example of creating a test branch: + +```bash +neon branches create --name test-branch + +# Run your tests against the isolated branch +TEST_DATABASE_URL="postgresql://neondb_owner:89ef7ewfbbef@ep-falling-scene-a8f2ngf9-pooler.eastus2.azure.neon.tech/neondb" +npm test + +# Clean up automatically when done +neon branches delete test-branch +``` + +Database branching addresses many of the pain points we encountered local Postgres containers in Docker Compose: + +- **Prepopulated test data.** By creating a branch of an existing database (with test data already in it), developers can instantly gain access to a prepopulated database for testing without having to perform data migrations or rely on shared databases that others are using as well. +- **Clean test states**. Each branch provides a fresh, isolated environment: + - No need for cleanup scripts + - No data persistence between runs + - No state leakage between tests +- **Resource efficiency**. Instead of running full database instances, branches share underlying storage and provision compute only when needed. This means: + - Lower memory and CPU usage + - Faster test startup times + - Reduced infrastructure costs + +With database branching, Neon gives developers all of the benefits of testing against a shared staging environment database without any of the downsides that typically accompany using shared development resources during testing. + +### When to use a local Postgres container vs. Neon branches? + +Use local Postgres container if: + +- You need full local control over the database instance. +- Your team prefers offline development environments. +- You have lightweight testing needs. + +Use Neon Branches if: + +- You want instant, isolated test databases per test run. +- You need scalability and parallel testing. +- You want zero local setup and cloud-based performance. + +## Database Branching for Ephemeral Dev/Test Environments + +The evolution of database testing reflects a broader shift in how we think about development environments. While Docker Compose revolutionized local testing by making it easy to spin up isolated databases, the growing complexity of modern applications demands more scalable solutions. Database branching represents the next step in this evolution, offering the isolation we need without the operational overhead. + +The choice between Docker Compose and database branching isn’t binary – many teams successfully leverage containers for local development while using branches for CI/CD pipelines and complex testing scenarios. The key is understanding your team’s needs and choosing the right tool for each job. + +--- + +_If you’d like to try Neon branches, they’re included in the [Free Plan](https://neon.tech/pricing) (no credit card required). [Create an account](https://console.neon.tech/signup) and get started right away._ diff --git a/content/blog/posts/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost.md b/content/blog/posts/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost.md new file mode 100644 index 0000000000..6ea971f5e2 --- /dev/null +++ b/content/blog/posts/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost.md @@ -0,0 +1,131 @@ +--- +title: Don’t use vector. Use halvec instead and save 50% of your storage cost +description: pgvector keeps getting better +excerpt: >- + pgvector’s latest release includes quantization features that help reduce your + vector and index footprint while speeding up index builds and prewarming times + for RAG applications. All Neon Postgres databases come with pgvector—spin up a + free database here. In this blog post, we w... +date: '2024-07-31T16:47:12' +updatedOn: '2024-07-31T16:48:20' +category: ai +categories: + - ai +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Don't use vector. Use halvec instead and save 50% of your storage cost - + Neon + description: >- + Pgvector’s latest release includes quantization features that help reduce + your vector and index footprint and increase query performance. + keywords: [] + noindex: false + ogTitle: >- + Don't use vector. Use halvec instead and save 50% of your storage cost - + Neon + ogDescription: >- + Pgvector’s latest release includes quantization features that help reduce + your vector and index footprint and increase query performance. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/neon-storage-cost-1024x576-f09817d8.jpg) + +**pgvector’s latest release includes quantization features that help reduce your vector and index footprint while speeding up index builds and prewarming times for RAG applications. All Neon Postgres databases come with pgvector—spin up a free database [here](https://console.neon.tech/signup).** + +In this blog post, we will test how scalar quantization with `halvec` improves and reduces storage costs without compromising the performance and efficiency of pgvector 0.7.x indexes. + +## What is quantization? + +Quantization is a type of compression that allows to reduction of the number of vector dimensions. There are various types of quantizations, but in this article, we will be testing two types: + +- **Scalar Quantization (SQ)**, which uses half-precision vectors `halfvec` to represent floats in 16 bits instead of 32 bits used by the `vector` type. +- **Binary Quantization (BQ)**, which turns each value of the vector into 0 or 1. + +Learn more about [quantization in pgvector in the article](https://jkatz05.com/post/postgres/pgvector-scalar-binary-quantization/) by Johnathan Katz. + +## Testing setup + +For our tests, we used the DBpedia 1M 1536 dimension vector dataset on an 8CPU, 32 GB memory [Neon Postgres instance](https://neon.tech) running on the AWS us-east-2 region. + +We measured the following: + +- Table and Index sizes (MB) +- Index Build and Prewarming Times (s) +- Query Execution Time (ms) +- Recall (%) + +## How scalar quantization improves pgvector performance + +**Our tests show that `halfvec` not only helps you reduce the memory and storage of your tables and indexes by half but also speeds up index build without compromising recall.** + +Specifically, the experiment shows the following benefits of adopting `halfvec`: + +- Reducing vector storage cost by nearly 50% +- Reducing index build time by 23% +- Reducing prewarming time by 50% +- Equivalent query execution time and recall + +We measured index build, prewarming, query execution, and recall for `vector` and `halfvec` using an HNSW index with `m=32` and `ef_construction=256`. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/185-1024x598-ab6d2c1b.jpg) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/186-1024x598-207c2528.jpg) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/187-1024x598-303063d6.jpg) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/188-1024x598-486b5b83.jpg) + +The pgvector extension uses the `vector` type, an array of floats represented in 4 bytes or 32 bits. In the 0.7.0 release, pgvector adds support for the half-precision vector type, or `halfvec`, which uses 16-bit floating point numbers to represent its components. This reduces storage capacity in half. + +For example, the value 0.123 has the following binary representation: + +0∣01111011∣11110100000000000000000 + +- Sign bit: 0 since the number is positive. +- Exponent: 01111011. In binary, 123 is represented as 01111011. +- Significand: 11110100000000000000000. The fractional part 111101, padded to 23 bits. + +On the other hand, half-precision vectors use a different binary representation of 16 bits: 1 bit for the sign, 5 bits for the exponent, and 10 bits for the significand. This means that the half-precision representation of 0.123 is: + +0∣01011∣1111010000 + +For reference, the length of OpenAI’s `text-embedding-3-small` embedding vector is 1536 and requires 6148 bytes per vector (4 bytes x 1536 dimensions + 4). With `halfvec`, the embedding vector requires only 3076 bytes of storage or 3kB. + +## Testing binary quantization + +Now that we have seen the improvements that half-precision vectors bring to pgvector, let’s explore binary quantization. + +Binary quantization is a simple type of vector compression where each dimension is reduced to either 0 if its value is equal to or smaller than 0 or else 1. + +In this case, an OpenAI `text-embedding-3-small` embedding vector of 1536 dimensions is represented by 1536 bits and reduced in size by a factor of 32, which reduces the index size and build time significantly. + +Below, we compare HNSW index performance for `vector` and `halfvec` with and without BQ: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/189-1024x598-e86ab14d.jpg) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/190-1024x598-ec5fbbbf.jpg) + +Although there are significant gains and comparable query performance, recall on the other hand with 1536 dimensional vector embeddings is not significant enough to use this in production. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dont-use-vector-use-halvec-instead-and-save-50-of-your-storage-cost/191-1024x598-7ba11940.jpg) + +However, in an [article by Johnathan Katz comparing SQ and BQ](https://jkatz05.com/post/postgres/pgvector-scalar-binary-quantization/), we can see the author managed to get up to 99% recall, which might be due to the high dimensionality nature of the embedding model. In a future article, we will explore BQ further using OpenAI `text-embedding-3-large` with 3072 dimensions. + +## Conclusion + +pgvector 0.7.x brings significant improvements to vector search in Postgres by introducing advanced quantization features. With the new half-precision `halfvec` type, you can achieve a 50% reduction in vector and index storage while speeding up index builds and prewarming times. + +Taking into account these results, using SQ and the `halfvec` type over `vector` seems like an obvious choice. However, we highly encourage you to experiment with `halfvec` before migrating since your results might depend on your dataset. + +BQ, on the other hand, didn’t result in a good enough recall with embedding vectors of 1536 dimensions. Further experiments with other embedding models and vector lengths are required before reaching a final conclusion. + +What about you? What embedding models are you using? Are you using pgvector 0.7.0 and halfvec? Join us on [Discord](https://neon.tech/discord), follow us on [X](https://x.com/neondatabase), and let us know what you think. diff --git a/content/blog/posts/dyad-brings-postgres-to-local-ai-app-building-powered-by-neon.md b/content/blog/posts/dyad-brings-postgres-to-local-ai-app-building-powered-by-neon.md new file mode 100644 index 0000000000..f1c93bfeb8 --- /dev/null +++ b/content/blog/posts/dyad-brings-postgres-to-local-ai-app-building-powered-by-neon.md @@ -0,0 +1,77 @@ +--- +title: Dyad Brings Postgres to Local AI App Building (Powered by Neon) +description: The local AI coding tool and app builder just got a powerful backend +excerpt: >- + Dyad is a local-first, open-source AI coding assistant that helps you build + and iterate on web apps from your own machine. You can chat with Dyad to + scaffold code, review changes, or debug issues all inside your editor. Unlike + most cloud-based tools, Dyad runs locally, supports a... +date: '2025-08-06T02:26:35' +updatedOn: '2025-09-10T01:03:55' +category: ai +categories: + - ai + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dyad-brings-postgres-to-local-ai-app-building-powered-by-neon/cover.avif + alt: null +isFeatured: false +seo: + title: Dyad Brings Postgres to Local AI App Building (Powered by Neon) - Neon + description: >- + Dyad is an open-source AI coding tool and app builder that just got a + powerful Postgres backend - powered by Neon. + keywords: [] + noindex: false + ogTitle: Dyad Brings Postgres to Local AI App Building (Powered by Neon) - Neon + ogDescription: >- + Dyad is an open-source AI coding tool and app builder that just got a + powerful Postgres backend - powered by Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dyad-brings-postgres-to-local-ai-app-building-powered-by-neon/cover.avif +--- + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + + +[Dyad](https://www.dyad.sh/) is a local-first, open-source AI coding assistant that helps you build and iterate on web apps from your own machine. You can chat with Dyad to scaffold code, review changes, or debug issues all inside your editor. Unlike most cloud-based tools, Dyad runs locally, supports any LLM (including GPT, Claude, and Gemini), and even lets you plug in open weights via Ollama. + + + +## Just Launched: Dyad’s Portal Template, Backed by Neon + +In its latest release ([v0.16.0-beta.1](https://github.com/dyad-sh/dyad/releases)), Dyad introduced a new [Portal template](https://www.dyad.sh/docs/templates/portal) – a full-stack app scaffold that ships with role-based permissions, a real database schema, and built-in undo support for database changes. + + + +**At the core of it is Postgres, powered by Neon.** With [Neon’s serverless Postgres](https://neon.com/) as the backend, AI-generated apps in Dyad now support persistent state without the user needing to set up any infrastructure. The Portal template is already live in Dyad’s UI – you can spin up a working app, with Neon Postgres under the hood, and start coding against it immediately. + +## Using Neon Branches for Undo and Checkpoints + +One of the most powerful things Dyad gains from Neon is the ability to offer a **database undo** feature, letting users rewind time and restore earlier states of their app, including the database. + +Behind the scenes, Dyad uses Neon’s branching and [Instant Restore APIs](https://neon.com/docs/introduction/branch-restore) to implement this workflow. When a user creates or edits an app, Dyad writes those changes to a development branch (a child of the production branch). If something goes wrong, the user can roll back to an earlier point in time. + +This architecture allows Dyad to support: + +- Previewing older states (like git commits for your app + DB) +- Restoring earlier app versions, including the database schema and content +- Safe experimentation, with branching used as a form of checkpointing + + +Neon is a serverless Postgres platform designed for modern development and agentic workflows. It powers backends for agents with an unparalleled developer experience: instant provisioning, seamless no-signup flows, and branching for checkpoints and versioning. Agentic platforms like Replit Agent, Databutton, Create.xyz, and many others use Neon under the hood. + + +## Try It Yourself + +Dyad’s new Portal template shows what’s possible when local-first tools meet modern cloud infra. If you’re already using Dyad, the Portal template is available in the latest beta. If not, [download Dyad](https://dyad.sh/download) and give it a spin! + + +If you’re building a full-stack AI Agent, apply to our Agents Program for higher resource limits, special pricing, and exclusive features. Fill out the form [here](https://neon.com/use-cases/ai-agents) and we’ll respond shortly. + diff --git a/content/blog/posts/dynamically-estimating-and-scaling-postgres-working-set-size.md b/content/blog/posts/dynamically-estimating-and-scaling-postgres-working-set-size.md new file mode 100644 index 0000000000..8fc2d0e3fb --- /dev/null +++ b/content/blog/posts/dynamically-estimating-and-scaling-postgres-working-set-size.md @@ -0,0 +1,230 @@ +--- +title: Dynamically estimating and scaling Postgres’ working set size +description: >- + How Neon automatically scales Postgres instances to keep your working set in + memory +excerpt: >- + With the announcement that Neon’s autoscaling feature is GA, we wanted to take + the opportunity to dive into the implementation of a recent improvement we + made: Scaling to match your workload’s working set size. This can provide + extraordinary speed-ups for real-world workloads, bu... +date: '2024-09-05T18:21:55' +updatedOn: '2025-01-21T15:38:16' +category: postgres +categories: + - postgres + - engineering +authors: + - em-sharnoff +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/cover.jpg + alt: null +isFeatured: true +seo: + title: Dynamically estimating and scaling Postgres’ working set size - Neon + description: >- + A deep dive at the technical details of how Neon estimates the Postgres + working set size to keep the working set in memory via autoscaling. + keywords: [] + noindex: false + ogTitle: Dynamically estimating and scaling Postgres’ working set size - Neon + ogDescription: >- + A deep dive at the technical details of how Neon estimates the Postgres + working set size to keep the working set in memory via autoscaling. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/neon-dynamically-estimate-2-1024x576-fc58a6e2.jpg) + +[With the announcement that Neon’s autoscaling feature is GA](https://neon.tech/blog/neon-autoscaling-is-generally-available), we wanted to take the opportunity to dive into the implementation of a recent improvement we made: Scaling to match your workload’s working set size. This can provide _extraordinary_ speed-ups for real-world workloads, but it’s a complex problem with no single “right” answer. + +In this post, we’ll look at the technical details of how we estimate the working set size and automatically scale your Postgres instance to keep the working set in memory – without breaking the bank. + +_Neon separates storage and compute to provide autoscaling, branching, point-in-time restore, and more. In this post, we’re only referring to scaling_ **_compute_**_. For more on Neon’s architecture, check out [this pos](https://neon.tech/blog/architecture-decisions-in-neon)_ [t](https://neon.tech/blog/architecture-decisions-in-neon)_._ + +## Working set size: Why is it so important? + +The “working set” of a database workload is the set of pages that will be accessed over the course of the workload – both the data and the indexes used to find it. + +(In case you’re curious, this is a concept that also applies to workloads outside of databases – check out [this great post](https://www.brendangregg.com/wss.html) about the memory used by applications, for example).
+ +Workloads often suffer severe performance costs when the working set doesn’t fit in memory, and Neon’s architecture makes this even more critical, because the cost of a cache miss is a network request, rather than just reading from local disk. + +In our own testing, we found that fitting the working set in memory offers up to an 8x improvement in throughput, particularly for read-heavy workloads. Moreover, because of the latency of cache misses when storage is over the network, keeping the working set in memory also provides much more predictable performance. + +So sizing your instance to fit the working set in memory is critically important – but determining the right size is often difficult! And for workloads that vary over time, manually scaling either results in performance degradations under load or overspending at off-peak times. + +But people already rely on autoscaling for this with CPU and memory load – so, why not scale based on the working set size as well? + +## Estimating working set size, part 1: HyperLogLog + +In order to automatically scale to fit the working set in memory, we first need to find some way to estimate it. + +Nowadays, the typical way one measures the approximate size of a set is with HyperLogLog (HLL) – a probabilistic algorithm to estimate the cardinality of a set to a high degree of accuracy with comparatively little memory. + +
+Image +
Example of updating the HyperLogLog registers after hashing an element.



+
+ +As a brief refresher in case you’re not intimately familiar: + +- HyperLogLog estimates cardinality based on the number of leading zeros in the hashes of the elements +- Every time you add an element: + - Find the register index given by the first `log2(num registers)` bits of the hash + - Update the register by identifying the first non-zero bit in the remaining bits of the hash, and setting it in the register – e.g., with simple maximum, bitwise OR, or directly setting just that single bit. +- To calculate the cardinality, use the harmonic mean of the number of leading zeros in the binary representation of each register – and adjust by a constant based on the number of registers. + +(Caveat: There are many equivalent representations – we’re presenting it in this way because it more closely matches how we modify HLL in the next section.) + +HyperLogLog is great for estimating the cardinality of sets with unknown size, but there’s no way to handle incremental changes in cardinality from _removing_ elements – continually storing the maximum in each register means the information from any individual item has already been removed. + +We did originally experiment with a typical HLL implementation by instrumenting page access in our custom resizable cache ([LFC](https://neon.tech/blog/scaling-serverless-postgres#local-file-cache)) to count the number of distinct pages – but since it measured the number of distinct pages accessed since Postgres started, the calculated sizes were far greater than what users were _currently_ using. + +## Estimating working set size, part 2: What’s the “true” size? + +So the actual working set size of your workload at a particular moment in time is somewhere between “nothing” and “everything since the start of time”. But the key question is how to find it? + +One approach might be to modify HyperLogLog for a sliding window – e.g., continuously measuring the number of unique pages accessed over the last 5 or 15 minutes. There’s a couple of papers outlining how to do this efficiently. But with that approach, there’s a critical question: How big a sliding window should we use? + +For steady workloads, as you increase the duration of the sliding window, the estimated size will _tend_ to plateau around the “true” working set size of the workload. So in those cases, we could just pick some large duration that’s “probably enough” – like 1 hour or so. + +But with varying workloads, such a long duration will actually prevent us from scaling down in a timely fashion. There, we need to strike a careful balance: Scaling down to make sure your compute isn’t over-provisioned while making sure not to drop the cache when it’ll incur substantial performance penalties.
From these, it was clear that _just one_ time window won’t do. + +### In search of flexibility: Time-bounded HyperLogLog + +At this point, we knew we’d want a selection of different sliding windows but without knowing those sizes in advance we had a dilemma: How to enable quick iteration and experimentation at a higher level (i.e., without having to frequently modify our extension in the early stages)? + +One assumption that helped was that we knew we only cared about the number of distinct pages accessed between a prior time and _now_ – rather than arbitrary windows in the past.
So, a key insight: Just replace the bits in your typical HyperLogLog with timestamps! Then, when we look for the observed working set size since time _T_, we count the number of leading zero “bits” by looking for the first timestamp after _T_. + +
+Image +
Example of updating registers based on timestamps. Instead of setting the bit in the register to “1”, we update the timestamp to the current time, for the first non-zero bit in the remainder of the hash.



+
+ +Doing this gives us a way to estimate the number of distinct pages accessed since some arbitrary time in the past, which gave us much more data to use in experimentation – and as we’ll see below, plenty of data to use in the final algorithm as well. + +**Pro-tip:** You can try this out yourself! We implemented this as the Postgres function `neon.approximate_working_set_size_seconds(d)`, which returns the HLL estimate of the working set size (i.e. number of distinct pages accessed) for the last d seconds. Check out [our docs](https://neon.tech/docs/extensions/neon) for more information on how to use the `neon` extension. + +### Keeping it simple: Finding a heuristic + +We now come to our second key insight: At any given moment, we can observe how the working set size changes as we increase the size of the window we use. Remember how stable workloads come to a plateau? In order to handle varying workloads, instead of looking for a plateau, we should actually look for the _end_ of a plateau – i.e. the point in the past where looking further back causes the observed working set size to _increase sharply_. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/neon-dynamically-estimate-3-1-1024x543-a48a11c2.jpg) + +This occurs, for example, when there was a heavy workload that recently ended – if the HLL window includes that previous workload, the size estimate will be much larger. So conversely, if _as you increase the window_, you’ll see a sudden jump once it extends far enough into the past to include the previous workload. + +There’s still a couple key questions to answer, though. So far, to estimate the “true” working set size at any particular time, we should increase the HLL window duration until we see a sharp increase. What if we don’t find one? + +To put a reasonable bound on our search, we can restrict it to just the most recent hour – if we get to the end and there’s no sudden increase, that’s fine! Just use the working set size given by the 1-hour window. This is typical for stable workloads _anyways_, and gives us that long time window we were originally looking for. + +Then, we have a concern at the other end: How do we prevent thrashing on bursty workloads? + +Here, we can just start our search with some offset, and then increase the window size until we find a sharp increase in the working set size or get to the end of 1 hour. This means we’ll wait for that initial offset duration until we allow scaling down when the working set size dramatically decreases. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/neon-dynamically-estimate-4-1-1024x519-20772013.jpg) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/neon-dynamically-estimate-5-1-1024x519-0329341a.jpg) + +There’s no single “right” answer for how long to wait to downscale (doing it too quickly causes bursty workloads to remain slow; doing it too slow incurs extra compute costs) – we picked 5 minutes to match the default suspend timeout. + +**Putting it all together,** we currently [export the HLL metrics](https://github.com/neondatabase/neon/blob/06795c6b9a6b4664dadd4c75ccf9f75087b05614/vm-image-spec.yaml#L428-L440) from each compute using [sql_exporter](https://github.com/burningalchemist/sql_exporter): + +```sql +select + x::text as duration_seconds, + neon.approximate_working_set_size_seconds(x) as size +from + (select generate_series * 60 as x from generate_series(1, 60)) as t (x); +``` + +… and then consume these [in our scaling algorithm](https://github.com/neondatabase/autoscaling/blob/27e0c77d04d91a1ef381d67510ba20dd94375873/pkg/agent/core/wss.go#L51-L71) as: + +```go +func EstimateTrueWorkingSetSize(series []float64, cfg WssEstimatorConfig) float64 { + // For a window size of e.g. 5 points, we're looking back from series [t] to series [t-4], because + // series [t] is already included. (and similarly for looking forward to series [t+4]). + // 'w' is a shorthand for that -1 to make the code in the loop below cleaner. + w:= cfg.WindowSize - 1 + + for t:= cfg.InitialOffset; t < len(series)-w; t += 1 { + // In theory the HLL estimator will guarantee that - at any instant - increasing the + // duration for the working set will not decrease the value. + // However in practice, the individual values are not calculated at the same time, so we + // must still account for the possibility that series [t] < series [t-w], or similarly for + // series [t+w] and series [t]. + // Hence, max(0.0, ...) + d0:= max(0.0, series [t]-series [t-w]) + d1:= max(0.0, series [t+w]-series [t]) + + if d1 > d0*cfg.MaxAllowedIncreaseFactor { + return series [t] + } + } + + return series [len(series)-1] +} +``` + +All in all, this ends up pretty simple. However, as most simple heuristics do, it has some edge cases! We mitigated some of them with tuning, but scroll down to the Future improvements section for some ideas we’re thinking of. + +## Scaling to fit + +To close the loop and _autoscale_ based on the estimated working set size, there’s one last piece to consider.
In short: We should scale based on the _projected_ size so that when a workload first starts, we only have to add the data to the cache once – otherwise, we risk reacting slowly and causing unnecessary cache evictions. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/dynamically-estimating-and-scaling-postgres-working-set-size/neon-dynamically-estimate-6-1024x521-181c94ef.jpg) + +The way we currently do this is by looking at the minute-to-minute increases in working set size as window duration increases and projecting upwards. Because we fetch the HLL values every 20 seconds, we only project a fraction of a minute – both to reduce overhead, and because projecting further can cause oscillation under bursty workloads. (The “why” on that is left as an exercise for the reader 😉) + +## Results + +How well does it work? Well in short… it works pretty well! + +There were a few distinct cases we looked at. + +First: in our internal benchmarking of a workload with a highly variable “target transaction rate”, we were able to _match the latency and throughput of a fixed-size compute while 30% cheaper_. Previously, autoscaling wasn’t able to keep up, because cache misses result in _reduced_ CPU usage because the compute’s waiting on network requests – so we never would have scaled up to fit the working set in cache. + +Elsewhere, in a sample “burst” workload (pgbench: cycle 1 minute on, 1 minute off), we found the scaling behavior changed from spiky allocations (1-3 CUs) well above the real CPU usage to a gradual increase over time, settling at a stable 3.5 CUs. We also saw a roughly 3x increase in throughput. + +And finally, we also tested a “batch” workload (pgbench: 1 hour heavy workload, 1 hour reduced workload). Here, _without_ the working set size-aware scaling, we saw the same spiky changes in allocated resources under the steady workload (between 2-4 CU) – and a cache hit rate of 30-70%. With the improved scaling, we saw it stabilize at 3.5 CUs and a cache hit rate of 100%. + +In summary, we found that the improved autoscaling is more cost-efficient than fixed-size nodes with the same performance. This is a good thing, unless cache misses weren’t a concern for you under the previous algorithm _anyways_ — in those cases, you might see increased compute time. But overall, this change brings more efficient performance to the Neon fleet. + +## Future improvements + +There’s a few different paths we’d like to explore! + +Firstly, when the working set size is bigger than the configured maximum compute size, we may end up scaling up for little benefit. This is because performance can sometimes mimic a step function based on whether the working set fits into cache — scaling up without fitting the working set may just result in increased resource usage without increased performance. Unfortunately there is no singular “right” answer here – if we just ignored the working set size entirely, there’s still workloads with incremental benefits from increased caching that might be harmed. + +We’re also looking at expanding the data we provide into this algorithm — currently it’s only ever collected at a single moment in time, and this is honestly a pretty big restriction. It _does_ help with resilience and architectural simplicity (there’s no warm-up time after the decision-maker restarts, and no external metrics database required), but there’s also distinct cases that aren’t possible for us to disambiguate otherwise. + +For example, if your workload routinely fetches its entire working set in under a minute, this can trick the predictive growth step into over-estimating the future working set size. From what we’ve seen in practice, this isn’t a very common issue, but it _is_ trivially possible to construct a workload that reproduces it. + +Lastly, we’re exploring ways to move away from uniform time windows. There’s a couple problems we see: + +1. A fixed set of time windows can produce transient oscillations as momentary spikes in load flip between the boundaries of the windows; and +2. Fixed windows make it harder to be immediately responsive _and_ support longer time ranges – at some point, it’s just too much data to process. + +Switching to an iterative algorithm (in the spirit of binary search) might particularly help with these — making it possible to consistently track the same timestamps of prior spikes with arbitrary precision on smaller time scales. But more to come on that at a later date! + +## Conclusion + +Take that, Aurora! (just kidding. maybe.) + +In all seriousness though, we’re fans of Aurora’s tech here at Neon, and with the recent publication of [their paper on Aurora Serverless v2](https://www.amazon.science/publications/resource-management-in-aurora-serverless), we’re wondering if AWS is doing something similar with working set estimation. From that paper: + +
+

“[Aurora Serverless v2] introduces a metric in the engine to estimate the size of the working set in the buffer cache”

+
+ +Although they also state the following: + +
+

“Aurora Serverless recommends setting the minimum [capacity] to a value that allows each DB writer or reader to hold the working set of the application in the buffer pool. That way, the contents of the buffer pool aren’t discarded during idle periods.”

+
+ +Perhaps there’s room to standardize on something to get merged into Postgres upstream? + +
_If you’ve enjoyed hearing about how this piece of Neon’s autoscaling works, check out [our other engineering blog posts](https://neon.tech/blog/category/engineering)._ diff --git a/content/blog/posts/easier-postgres-fine-tuning-with-online_advisor.md b/content/blog/posts/easier-postgres-fine-tuning-with-online_advisor.md new file mode 100644 index 0000000000..99568d2b32 --- /dev/null +++ b/content/blog/posts/easier-postgres-fine-tuning-with-online_advisor.md @@ -0,0 +1,155 @@ +--- +title: Easier Postgres fine-tuning with online_advisor +description: >- + A Postgres extension that points out index and stats opportunities based on + your real workload +excerpt: >- + You’ve heard this many times before – in order to keep your Postgres database + working smoothly, you need to have proper index planning. Too few indexes, and + your query performance suffers. Misestimated row counts can also trick the + planner into poor choices, and if you’re not usi... +date: '2025-09-16T18:17:18' +updatedOn: '2025-10-06T16:11:03' +category: postgres +categories: + - postgres +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/easier-postgres-fine-tuning-with-online_advisor/cover.jpg + alt: null +isFeatured: true +seo: + title: Easier Postgres fine-tuning with online_advisor - Neon + description: >- + online_advisor is a new Postgres extension that recommends indexes, stats, + and prepared statements to help you optimize performance. + keywords: [] + noindex: false + ogTitle: Easier Postgres fine-tuning with online_advisor - Neon + ogDescription: >- + online_advisor is a new Postgres extension that recommends indexes, stats, + and prepared statements to help you optimize performance. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/easier-postgres-fine-tuning-with-online_advisor/social.png +--- + +You’ve heard this many times before – in order to keep your Postgres database working smoothly, you need to have proper index planning. Too few indexes, and your query performance suffers. Misestimated row counts can also trick the planner into poor choices, and if you’re not using prepared statements, query planning can add overhead. + +The challenge is knowing **where** an index or set of statistics would actually help. That’s where a new Postgres extension can give you a hand: **`online_advisor`**. [https://github.com/knizhnik/online_advisor](https://github.com/knizhnik/online_advisor) + +## What online_advisor can do + +`online_advisor` analyzes your query workload in real time and points out opportunities for indexes, extended statistics, or prepared statements. It hooks into Postgres’ executor (the same mechanism used by the `auto_explain` extension) to monitor queries as they run. From this execution data, it generates actionable recommendations: + +- Suggests indexes when queries filter a lot of rows without one +- Suggests extended statistics when the planner’s row estimates are way off from reality +- Identifies queries for prepared statements when planning time is eating into execution + +The extension doesn’t create anything automatically. You review the proposals, decide what to apply, and then analyze the table so the planner can take advantage of the change. Postgres automatically collects statistics for each column, including distinct counts and histograms. What it does _not_ do by default is track correlations between columns. This can lead to serious underestimation when columns are dependent. + +## A note on Postgres’ statistics + +Postgres automatically collects statistics for each column, including distinct counts and histograms. What it does not do by default is track correlations between columns. This can lead to serious underestimation when columns are dependent. + +For example: + +```sql +SELECT * FROM cars WHERE company = 'Ford' AND model = 'Mustang'; +``` + +By default, Postgres assumes `company` and `model` are independent. If 10% of rows have `company='Ford'` and 10% have `model='Mustang'`, it estimates that 1% of rows satisfy both conditions. In reality, all `Mustang` rows belong to `Ford`. + +Multivariate statistics can capture these correlations, but you have to create them manually.`online_advisor` helps by flagging queries where correlated stats would improve planner estimates. + +## How to use online_advisor: Examples + +If you’re running Postgres on Neon, you can try it right away. Just run this in your SQL editor to enable it: + +```sql +-- Enable the extension in your database +CREATE EXTENSION online_advisor; + +-- Start collecting workload stats +SELECT get_executor_stats(); +``` + +Once activated, `online_advisor` observes your workload in real time. As queries run, it tracks which predicates are filtering lots of rows, where the planner is way off in its row estimates, and how much time is being spent on planning. From this, it produces recommendations that you can review. + +### Use case 1: Do I need a new index? + +Suppose you have a query that frequently filters on customer_id and order_date. Without an index, Postgres might scan the whole table each time. online_advisor notices this if more than 1,000 rows ([the default `online_advisor.filtered_threshold`](https://neon.com/docs/extensions/online_advisor)) are filtered out by a predicate. + +You can then check recommendations with: + +```sql +SELECT create_index, n_filtered, n_called, elapsed_sec +FROM proposed_indexes +ORDER BY elapsed_sec DESC +LIMIT 5; +``` + +If you see something like this: + +```sql + create_index | n_filtered | n_called | elapsed_sec +-------------------------------------------------------+------------+----------+------------ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_orders_customer_date + ON orders(customer_id, order_date); | 54231 | 320 | 184.5 +``` + +That tells you: + +- What to create: an index on `(customer_id, order_date)` +- Why: queries filtered ~54k rows without it +- How often: 320 calls in your workload +- Impact: 184 seconds of total elapsed time could be improved + +You can then run the `CREATE INDEX` statement and follow up with: + +```sql +VACUUM (ANALYZE) orders; +``` + +### Use case 2: Why is the planner making bad choices? + +Sometimes the planner’s estimates are far from reality, causing it to pick suboptimal plans.`online_advisor` flags any node where actual rows are more than 10× higher than estimated (the default `online_advisor.misestimation_threshold`). + +You can check recommendations with: + +```sql +SELECT create_statistics, misestimation, n_called, elapsed_sec +FROM proposed_statistics +ORDER BY misestimation DESC +LIMIT 5; +``` + +You might see an output like: + +```sql + create_statistics | misestimation | n_called | elapsed_sec +------------------------------------------------------+---------------+----------+------------ + CREATE STATISTICS s_customer_order_date + ON customer_id, order_date FROM orders; | 15.3 | 102 | 67.2 +``` + +This means the planner underestimated rows by a factor of ~15. Creating extended statistics on (`customer_id`, `order_date`) will help the planner make better choices in the future. + +### Use case 3: Should I be using prepared statements? + +Planning overhead matters if you’re running lots of short, repetitive queries.`online_advisor` compares planning time to execution time and flags cases where planning takes longer than the query itself (`online_advisor.prepare_threshold` defaults to `1.0`). + +```sql +SELECT * FROM get_executor_stats(false); +``` + +If `avg_planning_overhead` is greater than 1, that’s a sign you could benefit from preparing those queries instead of planning them from scratch every time. + +## Try it on Neon + +[Check out the docs](https://neon.com/docs/extensions/online_advisor) for all the details on online_advisor, [explore the repository](https://github.com/knizhnik/online_advisor) for more, and [get a Neon free account to try it](https://console.neon.tech/signup) if you’re not a user yet. + + +online_advisor supports Postgres 14–17, but on the Neon platform, it’s available for Postgres 17. (Why are they not supporting older versions: they require adding it to shared_preload_libraries, which is less convenient for cloud users.) + diff --git a/content/blog/posts/easiest-way-migrate-postgres-database-neon.md b/content/blog/posts/easiest-way-migrate-postgres-database-neon.md new file mode 100644 index 0000000000..f2f678624b --- /dev/null +++ b/content/blog/posts/easiest-way-migrate-postgres-database-neon.md @@ -0,0 +1,99 @@ +--- +title: The Easiest Way to Migrate Your Postgres Database to Neon +description: >- + Use the Import Data Assistant to move your data to Neon with a single + connection string +excerpt: >- + Moving a Postgres database from one provider to another it’s tedious, even for + simple migrations. You have to export and import dumps, match versions, check + extension support… To take that friction out of this equation, we’ve built a + tool: the Import Data Assistant. Migrate Your... +date: '2025-05-29T17:45:40' +updatedOn: '2025-10-02T00:16:29' +category: product +categories: + - product + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/easiest-way-migrate-postgres-database-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: The Easiest Way to Migrate Your Postgres Database to Neon - Neon + description: >- + We’ve built a tool (the Import Data Assistant) that helps you move your + Postgres data to Neon in one step. No dumps, no manual configs. + keywords: [] + noindex: false + ogTitle: The Easiest Way to Migrate Your Postgres Database to Neon - Neon + ogDescription: >- + We’ve built a tool (the Import Data Assistant) that helps you move your + Postgres data to Neon in one step. No dumps, no manual configs. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/easiest-way-migrate-postgres-database-neon/social.png +--- + +Moving a Postgres database from one provider to another it’s _tedious_, even for simple migrations. You have to export and import dumps, match versions, check extension support… To take that friction out of this equation, we’ve built a tool: the [Import Data Assistant.](https://neon.com/docs/import/import-data-assistant) + +## Migrate Your Data with Just a Connection String + +The Import Data Assistant is a built-in tool in the [Neon Console](https://console.neon.tech/signup) that lets you migrate an existing Postgres database into Neon with just a connection string. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/easiest-way-migrate-postgres-database-neon/import-db-82612b11.gif) + +The tool handles the entire process automatically: + +1. Verifies compatibility, matching Postgres version, extensions, and region +2. Creates a new branch in your target Neon project for the imported data (you pick which project you’d want the data to live in) +3. Imports the data directly – no need to export, upload, or configure anything manually + +The assistant works with any Postgres-compatible source, including: + +- AWS RDS and Aurora +- Supabase +- Heroku +- Self-hosted Postgres +- Other Neon projects + +## When to Use the Import Data Assistant + +We’ve designed this tool for common migration scenarios that would normally involve a mix of manual steps, downtime windows, or setup work. Here are a few ways developers are using it today: + +### Moving to Neon from another provider + +If you’re running Postgres on another managed service, the Import Data Assistant makes it easy to switch over to Neon. You paste your connection string, run a check, and the data comes in cleanly (schema and all) into a new Neon branch. + +### Upgrading Postgres versions + +You can also use the assistant to move data between versions. For example: you might have a production database running on Postgres 16 and want to test an upgrade on 17 – you can just import into a new project and start testing. + +### Switching regions or clouds + +If your data is in the wrong cloud region (e.g. perhaps we start supporting a region closer to you), you can use the Import Data Assistant to migrate it to a Neon project in a new region. + +### Creating a testing or staging environment + +Lastly, if you need to copy some data to test a new feature, you can use the assistant to bring a dataset into a new Neon project. From there, you can start branching for each of your environments. + +## How to Get Started + +You’ll find the Import Data Assistant in the Neon Console under the **Import Database** tab: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/easiest-way-migrate-postgres-database-neon/ad4nxfe74ocvncxsphvm0kublmbams0xdrlgmsftdxad27slpefcawtuf3nu0l54ijfev0cyg5heyhxjviega27qvl8hamz1rlmnl8rjz2wa07yamcccbmvtfrjtrpl9fym03bmsnt6a-954863f7.png) + +To get started, just follow the prompts. The process is very simple: + +1. You paste the connection string for your source Postgres database (e.g. in Supabase) +2. Neon runs an automatic compatibility check +3. If everything looks good, it starts the import. The assistant creates a new branch in your project of choice, and brings your data in. + +That’s it: no setup, no CLI, no exports. You can find full step-by-step instructions in [our docs](https://neon.tech/docs/import/import-data-assistant). + +## Migrating a Large Database? Reach Out to Us + +The Import Data Assistant is ideal for moving small databases, but if you’re planning a migration with multi-terabyte datasets, complex production setups, and/or low-to-zero downtime requirements, [reach out to us.](https://neon.tech/contact-sales) + +We’ve worked with teams to move TB-size production workloads with near zero downtime, and we’ll be happy to share what’s worked and build a personalized migration plan for your use case. diff --git a/content/blog/posts/easy-embeddings-indexing-pipelines-with-redpanda-and-neon.md b/content/blog/posts/easy-embeddings-indexing-pipelines-with-redpanda-and-neon.md new file mode 100644 index 0000000000..7ffb191fa2 --- /dev/null +++ b/content/blog/posts/easy-embeddings-indexing-pipelines-with-redpanda-and-neon.md @@ -0,0 +1,168 @@ +--- +title: Easy Embeddings Indexing Pipelines with Redpanda and Neon +description: >- + Take a stream of data, compute embeddings of each message, and store them in + Postgres on Neon +excerpt: >- + Building high throughput, scalable embeddings indexing pipelines can be a + daunting task, often requiring stitching together systems with code that needs + to scale, handle failures and ensure delivery guarantees. However, with + powerful tools like Redpanda and Neon, this process can... +date: '2024-09-06T17:32:45' +updatedOn: '2024-09-06T17:32:48' +category: community +categories: + - community +authors: + - tyler-rockwood +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/easy-embeddings-indexing-pipelines-with-redpanda-and-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: Easy Embeddings Indexing Pipelines with Redpanda and Neon - Neon + description: >- + Take a stream of data, compute embeddings of each message, and store them in + Postgres on Neon + keywords: [] + noindex: false + ogTitle: Easy Embeddings Indexing Pipelines with Redpanda and Neon - Neon + ogDescription: >- + Building high throughput, scalable embeddings indexing pipelines can be a + daunting task, often requiring stitching together systems with code that + needs to scale, handle failures and ensure delivery guarantees. However, + with powerful tools like Redpanda and Neon, this process can be + significantly simplified. In this blog post, we walk through how to set up + an […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/easy-embeddings-indexing-pipelines-with-redpanda-and-neon/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/easy-embeddings-indexing-pipelines-with-redpanda-and-neon/neon-redpanda-header-a6b5e6e6.jpg) + +Building high throughput, scalable embeddings indexing pipelines can be a daunting task, often requiring stitching together systems with code that needs to scale, handle failures and ensure delivery guarantees. However, with powerful tools like [Redpanda](https://redpanda.com) and Neon, this process can be significantly simplified. + +In this blog post, we walk through how to set up an embeddings indexing pipeline using these technologies, without the need for any code. This streamlined approach leverages Redpanda for data ingestion and Neon for efficient indexing and querying all in a managed environment. Neon is a scalable, serverless postgres database with support for vector similarity search using [pgvector](https://neon.tech/docs/extensions/pgvector). Combined with Redpanda, a unified streaming data platform that is blazing fast and simple to manage, we can create an embedding indexing pipeline that checks all the boxes without needing to write any code. + +## Setting up an embeddings indexing pipeline + +Prerequisites: + +- [Latest version](https://github.com/redpanda-data/connect/releases) of Redpanda Connect +- [Redpanda Serverless Cluster](https://www.redpanda.com/redpanda-cloud/serverless) +- [Neon Postgres Database](https://neon.tech/docs/get-started-with-neon/signing-up) + +As a warm up, you can check out Neon’s blog post on [building a Retrieval-Augmented Generation (RAG) applications with Neon and pgvector](https://neon.tech/blog/building-a-rag-application-with-llama-3-1-and-pgvector). In this post, we’ll demonstrate a way to create a high throughput, scalable embeddings indexing pipeline for RAG applications using [Redpanda Connect](https://docs.redpanda.com/redpanda-connect/about/). + +### Step 1. Understand our data + +The data we have for the pipeline will be invoices from an ecommerce application. We want to index what the customer bought so that they can search their past purchases using natural language. The data has the following form: + +```json +{ + "InvoiceNo": 536368, + "Country": "United Kingdom", + "CustomerID": 13047, + "Date": "12/1/2010 8:34", + "Items": [ + { + "StockCode": 22960, + "Description": "JAM MAKING SET WITH JARS", + "Quantity": 6, + "UnitPrice": 4.25 + }, + { + "StockCode": 22913, + "Description": "RED COAT RACK PARIS FASHION", + "Quantity": 3, + "UnitPrice": 4.95 + }, + { + "StockCode": 22914, + "Description": "BLUE COAT RACK PARIS FASHION", + "Quantity": 3, + "UnitPrice": 4.95 + } + ] +} +``` + +We’ll be using Redpanda as our intermediate buffer between producers and the computation of embeddings for Neon. Redpanda’s compatibility with the Apache Kafka® protocol means you can produce language easily with a client library in your favorite programming language, and easily absorb spikes in traffic or any pauses in the downstream processing. + +### Step 2. Read data from Redpanda + +Reading this data in Redpanda brokers using Redpanda Connect is a matter of defining the connection in YAML: + +```yaml +input: + kafka: + addresses: ["${RP_CLUSTER}.any.us-east-1.mpx.prd.cloud.redpanda.com:9092"] + topics: ["invoices"] + consumer_group: "connect-embeddings-index-pipeline" + tls: {enabled: true} + sasl: + mechanism: SCRAM-SHA-256 + user: ${RP_USER} + password: ${RP_PASS} +``` + +### Step 3. Compute embeddings using Ollama + +Next we need to actually set up our data processors within Redpanda Connect, we will do some small transformations of the data using [bloblang](https://docs.redpanda.com/redpanda-connect/guides/bloblang/about/), compute the embeddings locally using [Ollama](https://docs.redpanda.com/redpanda-connect/components/processors/ollama_embeddings/), then flatten the array into many messages using [unarchive](https://docs.redpanda.com/redpanda-connect/components/processors/unarchive/). + +```yaml +pipeline: + processors: + - label: compute_embeddings + branch: + request_map: |- + let Description = this. Items.map_each(item -> "%d of %s".format(item.Quantity, item. Description)).join(", ") + root = "search_document: The order contained %s".format($Description) + processors: + - ollama_embeddings: + model: nomic-embed-text + result_map: |- + root.embeddings = this + - label: flatten_items + mapping: + root = this. Items.map_each(item -> item.merge(this.without("Items"))) + - label: seperate_items + unarchive: + format: json_array +``` + +### Step 4. Write output into Neon + +Finally we can insert the data into Neon and make it queryable: + +```yaml +pipeline: + processors: + - label: compute_embeddings + branch: + request_map: |- + let Description = this. Items.map_each(item -> "%d of %s".format(item.Quantity, item. Description)).join(", ") + root = "search_document: The order contained %s".format($Description) + processors: + - ollama_embeddings: + model: nomic-embed-text + result_map: |- + root.embeddings = this + - label: flatten_items + mapping: + root = this. Items.map_each(item -> item.merge(this.without("Items"))) + - label: seperate_items + unarchive: + format: json_array +``` + +Now in a few dozen lines of configuration we have a full production-ready indexing pipeline using a Redpanda Cloud Serverless cluster and a Neon database. + +### Say hello to streamlined data processing workflows + +With Redpanda and Neon, setting up a high throughput, scalable embeddings indexing pipeline has never been easier. In this post, we showed how to create a robust pipeline using these tools without writing any code. Redpanda’s Kafka compatibility ensures smooth data ingestion and handling, while Neon’s serverless Postgres architecture and pg_vector extension provide efficient storage and querying of embeddings. + +This setup simplifies the process of building indexing pipelines, making them scalable and high-performing. Whether you’re indexing e-commerce transactions, customer reviews, or any other data type, this solution can handle large volumes and support your vector search query needs. + +To give it a try in your own projects, [sign up for Redpanda Serverless](https://www.redpanda.com/redpanda-cloud/serverless) and get [a free Neon Postgres account](https://console.neon.tech/signup) and start building! diff --git a/content/blog/posts/edit-records-directly-from-the-neon-console-meet-the-new-tables-page.md b/content/blog/posts/edit-records-directly-from-the-neon-console-meet-the-new-tables-page.md new file mode 100644 index 0000000000..41d9218be1 --- /dev/null +++ b/content/blog/posts/edit-records-directly-from-the-neon-console-meet-the-new-tables-page.md @@ -0,0 +1,84 @@ +--- +title: 'Edit records directly from the Neon console: meet the new Tables page' +description: Powered by Drizzle Studio +excerpt: >- + A few weeks ago, we shipped a cool new feature in our console. In the past, + the only way for Neon users to work with their data was via SQL queries; now, + you can modify your data in an intuitive and visual way directly from the + Tables page, powered by Drizzle Studio. You can now... +date: '2024-06-14T15:53:39' +updatedOn: '2024-06-14T15:53:42' +category: workflows +categories: + - workflows +authors: + - lachezar-petkov +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/cover.jpg + alt: null +isFeatured: false +seo: + title: 'Edit records directly from the Neon console: meet the new Tables page - Neon' + description: >- + The Tables page in the Neon console is now powered by Drizzle studio! You + can now browse your tables and make data edits visually and easily. + keywords: [] + noindex: false + ogTitle: 'Edit records directly from the Neon console: meet the new Tables page - Neon' + ogDescription: >- + The Tables page in the Neon console is now powered by Drizzle studio! You + can now browse your tables and make data edits visually and easily. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/neon-edit-tables-1-1-1024x576-2ea841cd.jpg) + +A few weeks ago, we shipped a cool new feature in our console. In the past, the only way for Neon users to work with their data was via SQL queries; now, **you can modify your data in an intuitive and visual way directly from the Tables page, powered by Drizzle Studio.** + +You can now use the console to add, update, and delete records, filter data, add or remove columns, drop or truncate tables, and [export data in .json and .csv formats.](https://neon.tech/blog/export-to-csv-json-and-xlsx-from-the-neon-console) I show you how in this quick video: + + + +## Editing data from the Neon console: no SQL required + +With our new feature, you can now browse your tables and make edits visually and easily. This update is designed to streamline your workflow and make data management more accessible, even for those who may not be as comfortable writing SQL queries. + +If you want to give it a spin: + +1. Log in to your Neon account +2. Optionally, [create a development branch](https://neon.tech/docs/manage/branches#create-a-branch) from your primary branch (named ‘main’ by default). This branch will be an exact copy-on-write clone of all your data and schemas from main, but it’s isolated – it allows you to play around with your data safely. +3. Once you’re in the development branch, navigate to the Tables view and try to make edits directly on the tables (**similarly as you would do on a spreadsheet**) + +After making your changes, you can simply save them, and they’re immediately reflected in the console – and in your dataset _for that particular branch_. Changes won’t be reflected on the main branch, unless you modify your data there as well. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/screenshot-2024-06-14-at-85035percente2percent80percentafam-1024x410-765e2102.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/screenshot-2024-06-14-at-85035percente2percent80percentafam-1-1024x410-7e5f57e0.png) + +## Adding filters and browsing data + +If you have a large table and are looking for a particular record, you can also use Filters vs writing a SQL query. For example, here I’m looking for somebody whose first name is Grace: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/screenshot-2024-06-14-at-85124percente2percent80percentafam-1024x355-94016582.png) + +After you’ve added a filter, your search will be saved as a view so you can easily come back to it later (see `View 1` up there). + +You can add as many filters as needed, download data, and browse records using the pagination widget. Unnecessary columns can also be hidden for a more focused view. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/screenshot-2024-06-14-at-85200percente2percent80percentafam-1024x355-4aad85bb.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/edit-records-directly-from-the-neon-console-meet-the-new-tables-page/screenshot-2024-06-14-at-85220percente2percent80percentafam-1024x355-8a14ff75.png) + +## Remember: changes only affect your current branch + +Like always in Neon, database branches have data isolation. If you modify records from the console in your development branch, they won’t be automatically reflected in the primary branch or its parent. + +This ensures that your production environment remains stable and unaffected by any accidental data changes. Once you know the changes are safe, if you wish so, you can apply them to the main branch. + +## Give it a go + +This renewed Tables view aims to provide you with flexibility in managing your data, all from within the Neon console. We hope this enhances your experience and look forward to your feedback! If you haven’t tried Neon, you can create an account for Free [here](https://console.neon.tech/signup). diff --git a/content/blog/posts/elephantshark-monitor-postgres-network-traffic.md b/content/blog/posts/elephantshark-monitor-postgres-network-traffic.md new file mode 100644 index 0000000000..5be4a04e18 --- /dev/null +++ b/content/blog/posts/elephantshark-monitor-postgres-network-traffic.md @@ -0,0 +1,168 @@ +--- +title: 'Introducing Elephantshark, a tool to monitor Postgres network traffic' +description: An open-source Ruby script published by Neon +excerpt: >- + Elephantshark helps you monitor, understand and troubleshoot Postgres network + traffic: that’s Postgres servers, clients, drivers and ORMs talking to + Postgres servers, proxies and poolers. Elephantshark sits between the two + parties in a Postgres-protocol exchange, forwarding messa... +date: '2025-09-24T16:33:15' +updatedOn: '2025-09-24T16:33:17' +category: postgres +categories: + - postgres +authors: + - george-mackerron +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/elephantshark-monitor-postgres-network-traffic/cover.jpg + alt: null +isFeatured: true +seo: + title: 'Introducing Elephantshark, a tool to monitor Postgres network traffic - Neon' + description: >- + Elephantshark helps you monitor, understand and troubleshoot Postgres + servers, clients, drivers and ORMs talking to Postgres. + keywords: [] + noindex: false + ogTitle: 'Introducing Elephantshark, a tool to monitor Postgres network traffic - Neon' + ogDescription: >- + Elephantshark helps you monitor, understand and troubleshoot Postgres + servers, clients, drivers and ORMs talking to Postgres. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/elephantshark-monitor-postgres-network-traffic/social.jpg +--- + +Post image + +[Elephantshark](https://github.com/neondatabase-labs/elephantshark) **helps you monitor, understand and troubleshoot Postgres network traffic: that’s Postgres servers, clients, drivers and ORMs talking to Postgres servers, proxies and poolers.** + +Elephantshark sits between the two parties in a Postgres-protocol exchange, forwarding messages in both directions while parsing and logging them. It is an open-source Ruby script published by Neon and works with any and all Postgres-protocol network traffic. That includes, but isn’t limited to, traffic to and from Neon databases. + +[https://github.com/neondatabase-labs/elephantshark](https://github.com/neondatabase-labs/elephantshark) + +## Why not just use Wireshark? + +Ordinarily [Wireshark](https://www.wireshark.org/) is great for this kind of thing, but using Wireshark is difficult if a connection is SSL/TLS-encrypted. [`SSLKEYLOGFILE`](https://wiki.wireshark.org/TLS#tls-decryption) support was [recently merged into libpq](https://www.postgresql.org/message-id/flat/CAOYmi%2B%3D5GyBKpu7bU4D_xkAnYJTj%3DrMzGaUvHO99-DpNG_YKcw%40mail.gmail.com#afc7fbd9fb2d13959cd97acae8ac8532), but it won’t be available in a release version for some time. Plus, not all Postgres connections are made with `libpq`. + +To get round this problem, Elephantshark decrypts and re-encrypts a Postgres connection. It then logs and annotates the messages passing through. Or if you prefer to use Wireshark, Elephantshark can enable that too by writing keys to an `SSLKEYLOGFILE`. + +## Elephantshark in action + +Run elephantshark in one terminal: + +```bash +% elephantshark +listening ... +``` + +In a second terminal, connect to and query a Neon Postgres database via Elephantshark by (1) appending `.local.neon.build` to the host name and (2) changing `channel_binding=require` to `channel_binding=disable`: + +```bash +% psql 'postgresql://neondb_owner:fake_password@ep-crimson-sound-a8nnh11s.eastus2.azure.neon.tech.local.neon.build/neondb?sslmode=require&channel_binding=disable' +psql (17.5 (Homebrew)) +SSL connection (protocol: TLSv1.3, cipher: TLS_AES_256_GCM_SHA384, compression: off, ALPN: postgresql) +Type "help" for help. + +neondb=> SELECT now(); + now +------------------------------- + 2025-07-02 11:51:01.721628+00 +(1 row) + +neondb=> \q +``` + +Back in the first terminal, see what bytes got exchanged: + +
+ + + % elephantshark + listening … + connected at t0 = 2025-09-18 09:19:05 +0100 + client -> script: "\x00\x00\x00\x08\x04\xd2\x16\x2f" = SSLRequest + script -> client: "S" = SSL supported + TLSv1.3/TLS_AES_256_GCM_SHA384 connection established with client + server name via SNI: ep-aged-night-a80vx88s.eastus2.azure.neon.tech.local.neon.build + client -> script: "\x00\x00\x00\x56" = 86 bytes of startup message "\x00\x03\x00\x00" = protocol version + "user\x00" = key "neondb_owner\x00" = value + "database\x00" = key "neondb\x00" = value + "application_name\x00" = key "psql\x00" = value + "client_encoding\x00" = key "UTF8\x00" = value + "\x00" = end + connecting to Postgres server: ep-aged-night-a80vx88s.eastus2.azure.neon.tech + script -> server: "\x00\x00\x00\x08\x04\xd2\x16\x2f" = SSLRequest + server -> script: "S" = SSL supported + TLSv1.3/TLS_AES_256_GCM_SHA384 connection established with server + forwarding client startup message to server + script -> server: "\x00\x00\x00\x56" = 86 bytes of startup message "\x00\x03\x00\x00" = protocol version + "user\x00" = key "neondb_owner\x00" = value + "database\x00" = key "neondb\x00" = value + "application_name\x00" = key "psql\x00" = value + "client_encoding\x00" = key "UTF8\x00" = value + "\x00" = end + forwarding all later traffic + server -> client: "R" = Authentication "\x00\x00\x00\x2a" = 42 bytes "\x00\x00\x00\x0a" = AuthenticationSASL + "SCRAM-SHA-256-PLUS\x00" = SASL mechanism + "SCRAM-SHA-256\x00" = SASL mechanism + "\x00" = end + ^^ 43 bytes forwarded at +0.55s, 0 bytes left in buffer + client -> server: "p" = SASLInitialResponse "\x00\x00\x00\x36" = 54 bytes + "SCRAM-SHA-256\x00" = selected mechanism + "\x00\x00\x00\x20" = 32 bytes follow + "n,,n=,r=oyCbUH3BAFTR5K7ky/6sT6sl" = SCRAM client-first-message + ^^ 55 bytes forwarded at +0.55s, 0 bytes left in buffer + server -> client: "R" = Authentication "\x00\x00\x00\x5c" = 92 bytes "\x00\x00\x00\x0b" = AuthenticationSASLContinue + "r=oyCbUH3BAFTR5K7ky/6sT6slO/L2RQWlqi8k5hbEe9Ch4TW1,s=sua0GGw9khvJmqzfirvr4w==,i=4096" = SCRAM server-first-message + ^^ 93 bytes forwarded at +0.65s, 0 bytes left in buffer + client -> server: "p" = SASLResponse "\x00\x00\x00\x6c" = 108 bytes + "c=biws,r=oyCbUH3BAFTR5K7ky/6sT6slO/L2RQWlqi8k5hbEe9Ch4TW1,p=F4I92rJgKR987t7tf93xdumCRuktShWrNvh6MY/rj8M=" = SCRAM client-final-message + ^^ 109 bytes forwarded at +0.65s, 0 bytes left in buffer + server -> client: "R" = Authentication "\x00\x00\x00\x36" = 54 bytes "\x00\x00\x00\x0c" = AuthenticationSASLFinal + "v=ZKr8JIlFdYKw/3GVRnZ1epdKZIfMjXW2Ep3I5JsvNbQ=" = SCRAM server-final-message + server -> client: "R" = Authentication "\x00\x00\x00\x08" = 8 bytes "\x00\x00\x00\x00" = AuthenticationOk + server -> client: "S" = ParameterStatus "\x00\x00\x00\x17" = 23 bytes "in_hot_standby\x00" = key "off\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x19" = 25 bytes "integer_datetimes\x00" = key "on\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x11" = 17 bytes "TimeZone\x00" = key "GMT\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x1b" = 27 bytes "IntervalStyle\x00" = key "postgres\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x20" = 32 bytes "search_path\x00" = key "\\\"$user\\\", public\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x15" = 21 bytes "is_superuser\x00" = key "off\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x1a" = 26 bytes "application_name\x00" = key "psql\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x26" = 38 bytes "default_transaction_read_only\x00" = key "off\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x1a" = 26 bytes "scram_iterations\x00" = key "4096\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x17" = 23 bytes "DateStyle\x00" = key "ISO, MDY\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x23" = 35 bytes "standard_conforming_strings\x00" = key "on\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x27" = 39 bytes "session_authorization\x00" = key "neondb_owner\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x19" = 25 bytes "client_encoding\x00" = key "UTF8\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x22" = 34 bytes "server_version\x00" = key "17.5 (a42a079)\x00" = value + server -> client: "S" = ParameterStatus "\x00\x00\x00\x19" = 25 bytes "server_encoding\x00" = key "UTF8\x00" = value + server -> client: "K" = BackendKeyData "\x00\x00\x00\x0c" = 12 bytes "\x16\xee\x00\x6a" = process ID "\xa0\x00\x89\x24" = secret key + server -> client: "Z" = ReadyForQuery "\x00\x00\x00\x05" = 5 bytes "I" = idle + ^^ 514 bytes forwarded at +0.76s, 0 bytes left in buffer + client -> server: "Q" = Query "\x00\x00\x00\x12" = 18 bytes "SELECT now();\x00" = query + ^^ 19 bytes forwarded at +2.17s, 0 bytes left in buffer + server -> client: "T" = RowDescription "\x00\x00\x00\x1c" = 28 bytes "\x00\x01" = 1 columns follow + "now\x00" = column name "\x00\x00\x00\x00" = table OID: 0 "\x00\x00" = table attrib no: 0 + "\x00\x00\x04\xa0" = type OID: 1184 "\x00\x08" = type length: 8 "\xff\xff\xff\xff" = type modifier: -1 "\x00\x00" = format: text + server -> client: "D" = DataRow "\x00\x00\x00\x27" = 39 bytes "\x00\x01" = 1 columns follow + "\x00\x00\x00\x1d" = 29 bytes "2025-09-18 08:19:08.270142+00" = column value + server -> client: "C" = CommandComplete "\x00\x00\x00\x0d" = 13 bytes "SELECT 1\x00" = command tag + server -> client: "Z" = ReadyForQuery "\x00\x00\x00\x05" = 5 bytes "I" = idle + ^^ 89 bytes forwarded at +2.3s, 0 bytes left in buffer + client -> server: "X" = Terminate "\x00\x00\x00\x04" = 4 bytes + ^^ 5 bytes forwarded at +3.7s, 0 bytes left in buffer + client hung up + connection end + listening … + +
+ +## Get started with Elephantshark + +To find out more and/or to install Elephantshark, [check out the README](https://github.com/neondatabase-labs/elephantshark) on GitHub. You can also [find out more about Neon](https://neon.com/), or [sign up today](https://console.neon.tech/) for free. diff --git a/content/blog/posts/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon.md b/content/blog/posts/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon.md new file mode 100644 index 0000000000..3c98c19335 --- /dev/null +++ b/content/blog/posts/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon.md @@ -0,0 +1,107 @@ +--- +title: 'Empowering developers with production-like snapshots: how Snaplet uses Neon' +description: They leverage copy-on-write branches to provide instant data copies to users +excerpt: >- + “As soon as we found out about Neon’s branching model with copy-on-write, we + knew it was exactly what we were looking for” Julien Goux, Software Engineer + at Snaplet Snaplet uses Neon to power its Snapshot feature. Neon’s partnership + plans make it possible to manage thousands of d... +date: '2024-05-20T15:19:22' +updatedOn: '2024-05-20T15:19:24' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Empowering developers with production-like snapshots: how Snaplet uses Neon + - Neon + description: >- + Snaplet uses Neon to power Snapshots, using database branching to provide + decelopers with preview databases with production-like data. + keywords: [] + noindex: false + ogTitle: >- + Empowering developers with production-like snapshots: how Snaplet uses Neon + - Neon + ogDescription: >- + Snaplet uses Neon to power Snapshots, using database branching to provide + decelopers with preview databases with production-like data. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon/neon-snapshots-1-1024x576-89ab6f62.jpg) + +
+

“As soon as we found out about Neon’s branching model with copy-on-write, we knew it was exactly what we were looking for”

+Julien Goux, Software Engineer at Snaplet +
+ +[Snaplet](https://www.snaplet.dev/) **uses Neon to power its Snapshot feature. [Neon’s partnership plans](https://neon.tech/partners) make it possible to manage thousands of databases without cost overhead or management burden. Companies like** [Retool](https://neon.tech/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases), [Vercel](https://neon.tech/blog/neon-postgres-on-vercel), [Replit](https://neon.tech/blog/neon-replit-integration)**, and [Koyeb](https://www.koyeb.com/blog/serverless-postgres-public-preview) also use Neon to provide serverless Postgres to their user base.** + +Having access to realistic, production-like data is crucial for effective testing and debugging, but it’s easier said than done. Snaplet, a cutting-edge developer tool, aims to make this process as seamless as a snap 🫰 of the fingers. + +## Simplifying access to production-like data for development and testing + +The inspiration behind Snaplet came from the team’s personal experience with large production database dumps. These dumps were necessary for various crucial tasks, such as testing new features to ensure they worked as expected, developing locally to replicate real-world scenarios, debugging issues by reproducing them in a controlled environment, and staging to create pre-production environments that mirrored the live system. + +Ideally, each of these activities required realistic, production-like data. In practice, this proved not only time-consuming to obtain but also made the team nervous due to the presence of sensitive, personally identifiable information (PII). + +Snaplet aims to improve this experience by providing developers with easy access to production-like data without the associated risks. Through [Seed](https://www.snaplet.dev/seed), developers can populate their local and testing environments with realistic, AI-generated seed datasets. For those teams who prefer to access production-like database copies, Snaplet offers [Snapshot](https://www.snaplet.dev/snapshot), which allows users to access isolated database copies with anonymized sensitive data that can be used for testing, shared among team members, and quickly reset as needed. + +## How database branches power Snapshots under the hood + +
+

“Neon is completely committed to developer experience, which is what we’re also obsessed about at Snaplet”

+Jian Reis, COO at Snaplet +
+ +Snaplet relies on Neon’s serverless Postgres to power Snapshot, with database branching being the key feature behind it. Here’s a breakdown of how it works: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon/image-4-1024x479-0b7f3d07.png) + +- **Each Snapshot in Snaplet corresponds to a project in Neon.** Snaplet restores the snapshot to the main branch within Neon, which serves as a base for creating new database branches. +- **When users require a new preview database, Snaplet creates a branch from the main snapshot branch.** This process is quick and efficient, leveraging [Neon’s copy-on-write technology](https://neon.tech/blog/get-page-at-lsn) to avoid unnecessary data duplication. +- **Snaplet users work against these branched databases in isolated environments.** They have the flexibility to delete and create new preview databases as needed, without affecting the main snapshot or other branches. This isolation ensures a secure and efficient development workflow. +- **Developers can use GitHub actions to automate the process.** They can hook a Snapshot into a preview environment, providing both the application and data parts of the preview in a unified, isolated setup. This solves the challenge of having separate environments for frontend, backend, and data, which often leads to inconsistencies and mistakes. +- **Snaplet’s Snapshot includes subsetting and transformation capabilities to solve the PII problem.** Developers can sample a portion of their database if it’s too large (e.g., taking just 10% of the data). They can also anonymize sensitive data using JavaScript functions, with default settings for detecting and transforming probable PII columns. Snaplet’s deterministic transformation library, [copycat](https://github.com/snaplet/copycat), ensures that transformed data remains consistent across environments. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/empowering-developers-with-production-like-snapshots-how-snaplet-uses-neon/quick-start-06-1024x865-a08790e9.webp) + +
+

“Moving to Neon gave us more clarity and comfort around the architecture and pricing predictability, allowing us to move forward with the development of Snapshot with a high degree of confidence”

+Jian Reis, COO at Snaplet +
+ +Before adopting Neon, Snaplet tried a previous design for Snapshot on Fly.io. To manage preview databases in Fly, Snaplet used a workaround: using Postgres databases and Fly Machines, the Snaplet team created a main database with a “template” and copied the data to generate more databases to mimic a branching structure. When a user wanted to deploy a snapshot, Snaplet performed a pg_restore to a hidden template database, from which branches were created. + +This setup worked, but it had significant limitations. Fly.io’s storage did not support file systems with copy-on-write features like ZFS. While the system was fast due to everything being on the same disk, it caused memory usage to essentially double with each new database, making the setup increasingly expensive. + +Managing this architecture was also complex. Snaplet had to deal with VMs and operating systems, which added a layer of complexity that they preferred to avoid in the long run. They wanted to find a way to focus on single databases rather than infrastructure and OS management. + +Discovering Neon and its copy-on-write branching model was a game-changer for Snaplet, as it provided a logical model that aligned perfectly with their needs. This transition to Neon solved the scalability and cost problems they previously faced, enabling them to focus on delivering a superior product to their users. + +## The Snaplet tech stack + +
+

“We’re using the Neon API with an auto-generated TypeScript client thanks to their OpenAPI compliance. The integration went great and without issues”

+Julien Goux, Software Engineer at Snaplet +
+ +Before we wrap up, let’s touch on the rest of Snaplet’s tech stack. They use Node.js for their server-side operations and React for the front end. Their entire stack is 100% JavaScript, with a strong emphasis on TypeScript. To integrate with the database, Snaplet uses the Neon API. Additionally, Snaplet leverages the OpenAI API to enhance its [Seed](https://www.snaplet.dev/seed) product to generate realistic seed datasets. + +## Become a Neon partner + +
+

“This partnership sets the standard for us in terms of how responsive the Neon team has been”

+Jian Reis, COO at Snaplet +
+ +Neon’s [partnership plans](https://neon.tech/partners) are designed to scale with companies like Snaplet, interested in managing a high number of Neon projects for their users. If you’re interested in becoming a partner, [send us your info](https://neon.tech/partners). You can also [take a quick look at Neon](https://console.neon.tech/signup) (it’s free!) diff --git a/content/blog/posts/endform-wants-to-scale-your-playwright-end-to-end-tests.md b/content/blog/posts/endform-wants-to-scale-your-playwright-end-to-end-tests.md new file mode 100644 index 0000000000..0fc7278132 --- /dev/null +++ b/content/blog/posts/endform-wants-to-scale-your-playwright-end-to-end-tests.md @@ -0,0 +1,110 @@ +--- +title: Endform Wants to Scale your Playwright End-to-end Tests +description: >- + They’re building a platform that uses Lambda, Cloudflare Durable Objects, and + Neon to run massive Playwright test suites +excerpt: >- + “Every tech choice we make is about staying lightweight and scalable. Neon + fits that perfectly: we can spin up real Postgres databases in CI, in seconds, + with zero hassle.” (Oliver Stenbom, co-founder of Endform) Before starting + Endform, Oliver spent years working at Mentimeter,... +date: '2025-05-15T23:46:42' +updatedOn: '2025-05-15T23:46:44' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/endform-wants-to-scale-your-playwright-end-to-end-tests/cover.jpg + alt: null +isFeatured: false +seo: + title: Endform Wants to Scale your Playwright End-to-end Tests - Neon + description: >- + Endform is a platform that helps engineering teams run their end-to-end + tests quickly, reliably, and repeatedly. Built on Neon. + keywords: [] + noindex: false + ogTitle: Endform Wants to Scale your Playwright End-to-end Tests - Neon + ogDescription: >- + Endform is a platform that helps engineering teams run their end-to-end + tests quickly, reliably, and repeatedly. Built on Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/endform-wants-to-scale-your-playwright-end-to-end-tests/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/endform-wants-to-scale-your-playwright-end-to-end-tests/neon-endform-1-1024x576-281ccc0f.jpg) + +
+

“Every tech choice we make is about staying lightweight and scalable. Neon fits that perfectly: we can spin up real Postgres databases in CI, in seconds, with zero hassle.” (Oliver Stenbom, co-founder of Endform)

+
+ +Before starting Endform, [Oliver](https://www.linkedin.com/in/oliverstenbom/) spent years working at [Mentimeter](https://v/), a hypergrowth company, with an engineering team that was shipping up to 80 times a day. That kind of velocity exposed some harsh bottlenecks in the current state of E2E testing: pulling this off meant running hundreds of tests per pull request, across every deploy – aka hundreds of runs per day. The team had to build custom infra just to handle it. + +[This is a common wall that fast-moving teams hit](https://endform.dev/blog/the-fastest-playwright-runner), but [Endform](https://endform.dev/) is here to fix that. Designed for [Playwright](https://playwright.dev/), Endform is a platform that helps engineering teams run their end-to-end tests quickly, reliably, and repeatedly. + +
+ +Image +
Source endform.dev
+ +
+ +But good tooling isn’t enough if the environments under test are brittle. Fast tests need fast, repeatable environments: every part of the stack should be scalable, isolated, and quick to spin up, including the database. + +## Why Endform Needed a New Database Layer + +
+

“We started by building on Supabase because it was really easy to get going, but eventually we realized we didn’t want to be locked in. With Neon, we get Postgres without the constraints, and full control over our infrastructure.” (Oliver Stenbom, co-founder of Endform)

+
+ +When the Endform team first started building, they chose Supabase. It offered Postgres (the DB of choice for the founders) and a quick setup, a good match for a scrappy early-stage team. But they quickly wondered of the tradeoffs of being locked into Supabase’s suite early on. + +Authentication was especially problematic. The Endform team wanted to keep the freedom to choose their own auth provider (they now use [Better Auth](https://www.better-auth.com/)) but Supabase favoured coupling auth to its managed stack, and heavily using row-level-security. That constraint made the team reconsider their entire infrastructure: because once auth was decoupled, the database layer was up for debate. + +Both founders had years of experience with Postgres, but preferred something that kept the overhead of managing Postgres at a minimum. That’s when they chose Neon. The team had used Neon at a previous company and knew it could deliver the right blend of performance, scale, and simplicity. + +Neon didn’t force any particular auth model or infrastructure choices: it was just Postgres but serverless and developer-first. + +## How Endform Uses Neon Today + +
+

“I caught a broken migration thanks to a Neon branch that mirrored production. That bug would’ve made it to prod in any other setup” (Oliver Stenbom, co-founder of Endform)

+
+ +Switching away from Supabase also opened the door to smarter workflows. For example, Endform could now start experimenting with Neon’s branching model to create ephemeral databases tied to every pull request. + +Here’s a high-level view of their setup: + +- Every PR gets its own Neon branch, spun up via [Alchemy](https://github.com/sam-goodwin/alchemy), a new Typescript-native infrastructure-as-code tool that the Endform team is loving. +- These branches serve as the backing database for integration and end-to-end tests, both in CI and locally. +- Branches include data from the parent branch, which helps catch issues that wouldn’t surface in empty test databases (e.g, a missing default on a NOT NULL column, _true story_). +- Developers can test locally using the same infrastructure that runs in CI. No need to wait for GitHub Actions. +- Temporary infra is tied together with [linkup](https://github.com/mentimeter/linkup/) to give each branch a complete environment. + +## The Architecture: Serverless, Fast, Built for Scale + +
+

“Weave together a few lightweight primitives (Neon branches, Hyperdrive URLs, Workers) and you get a setup where there’s just no excuse for breaking prod.”  (Oliver Stenbom, co-founder of Endform)

+
+ +Endform’s infrastructure is optimized for two things: speed and scale. Their platform needs to spin up thousands of browser sessions in parallel, run full end-to-end tests, and coordinate it all across ephemeral environments, without bloated DevOps around. + +Here’s how they do it: + +- **Browsers run on AWS Lambda.** Endform launches hundreds to thousands of Playwright browser instances per test run. Lambda scales well, yet has the wide operating system access needed to run browsers.**Cloudflare powers the rest.** The frontend is deployed as a Cloudflare Worker. Communication between components flows through Durable Objects, which orchestrate test execution and browser coordination. +- **Neon is the backing database.** Customers, organizations, test runs, suite definitions live in Neon. Branches are provisioned dynamically for each PR, and used across integration and end-to-end tests. +- **Alchemy ties it together.** Endform uses [Alchemy](https://www.alchemyinfra.dev/) to define infrastructure programmatically. One pull request triggers: + - A Neon branch + - A Cloudflare Worker with a unique Hyperdrive connection + - A fully wired environment, all provisioned in seconds + +## Looking Ahead + +Endform is building the fastest way to run end-to-end Playwright tests, and Neon is a key part of that equation. If you want to experiment with the same kind of workflow, [Neon’s free plan](https://neon.tech/pricing) it’s a great place to start. [Give it a try](https://console.neon.tech/signup), you don’t need a credit card. + + +If you're running end-to-end tests with Playwright, [request access to join Endform’s waitlist](https://endform.dev/) and experience what reliable, production-grade testing can feel like. + diff --git a/content/blog/posts/environments-masked-production-data.md b/content/blog/posts/environments-masked-production-data.md new file mode 100644 index 0000000000..f087285c1c --- /dev/null +++ b/content/blog/posts/environments-masked-production-data.md @@ -0,0 +1,182 @@ +--- +title: Create Environments with Masked Production Data Using Neon Branches +description: >- + Use branching and PostgreSQL Anonymizer to safely replicate entire + environments in seconds, even with PII in production +excerpt: >- + Every engineering team needs realistic, reliable environments to test, debug, + and ship software with confidence. The ideal setup sounds simple – clone your + production database, run your tests, and move on. But “clone your production + database” is easier said than done. Replicating... +date: '2025-05-15T19:18:23' +updatedOn: '2025-07-09T17:48:12' +category: workflows +categories: + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/environments-masked-production-data/cover.jpg + alt: null +isFeatured: false +seo: + title: Create Environments with Masked Production Data Using Neon Branches - Neon + description: >- + Use Neon branching and static masking via PostgreSQL Anonymizer to safely + replicate real environments, even with PII in production. + keywords: [] + noindex: false + ogTitle: Create Environments with Masked Production Data Using Neon Branches - Neon + ogDescription: >- + Use Neon branching and static masking via PostgreSQL Anonymizer to safely + replicate real environments, even with PII in production. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/environments-masked-production-data/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/environments-masked-production-data/neon-masked-data-1024x576-d68db6f5.jpg) + +Every engineering team needs realistic, reliable environments to test, debug, and ship software with confidence. The ideal setup sounds simple – clone your production database, run your tests, and move on. + +But “clone your production database” is easier said than done. Replicating a live Postgres environment means provisioning new instances and running manual dump/restore processes. Most developers end up working on a database that’s already out of sync with production. + +This problem gets even worse when your production database contains personal data (PII) – customer names, emails, payment details, and so on. Now, on top of the already infrastructure-heavy process of provisioning new databases, you also have to figure out how to safely populate them. + +## Why Cloning Production Environments Is Still Hard + +To summarize: + +- **Provisioning environments is slow and manual.** Spinning up a new instance, creating a fresh database, and populating it with data (real or fake) takes time and effort, together with coordination across teams and DBAs. +- **PII makes cloning a non-starter.** Regulatory frameworks like GDPR and HIPAA, internal security controls, and the practical risk of accidentally triggering real email or analytics flow make it impossible to copy production data into dev or test environments. +- **Seed data doesn’t cut it.** But it’s hard to populate non-prod environment with “fake data” in a way that truly mirrors production. The result is often an unrealistic environment that rarely keeps up with schema or business logic changes, and many bugs slip. +- **Environments drift constantly.** As the environments add up, staging lags behind prod, dev diverges from staging, and no one’s quite sure what version the tests are running against. + +
+

“Our testing process was very manual before. Product would create a test customer in our development environment, then generate PDFs; the QA team would test and manually run through all the math; then an engineer would have to go into the database, look at all the values, and handwrite them into fixtures for our end-to-end tests… That’s multiple days for every single change” (Miguel Hernandez, Backend Tech Lead at Neo.Tax)

+
+ +All of this points to one missing primitive – a safe, fast, and repeatable way to create production-like environments without exposing real user data. This is what we’re working on in Neon. + +## The Solution: Branches + PostgreSQL Anonymizer in Neon + +Solving this problem requires two things: + +1. A way to clone production data without the operational overhead of managing new instances +2. A way to anonymize sensitive information before it reaches development or testing environments + +Neon gives you both: + +### Neon branches – Instant, production-grade copies of your DB + +
+Image +
Branching workflows without PII
+
+ +In Neon, a [branch](https://neon.tech/docs/introduction/branching) is a lightweight, copy-on-write clone of a Postgres database. It contains the same schema and data as its parent, but diverges safely in isolation, with its own compute endpoint and connection URL. + +Branches are created instantly no matter _how large the dataset_ and don’t require provisioning a new instance or duplicating storage, it’s all built into Neon’s architecture. They remain “logically connected’ to the parent, so production drift is a thing of the past – child branches can be synced with their parent in one API call, and they’ll reflect up-to-date and schema again. + +Teams use branches where they were using redundant instances or local setups before: + +- Spin up isolated environments for testing, development, or CI +- Work with a snapshot of production without affecting production +- Reproduce issues, test migrations, or preview features in realistic conditions + +
+

“With Neon branches we get a totally isolated copy to test code changes even when they include database migrations. We can test all changes in real data and ensure that by the time we actually merge the PR to main, things really work”  (Avi Romanoff, Founder at Magic Circle)

+
+ +### PostgreSQL Anonymizer – Protect PII with static masking + +
+Image +
Branching workflows with PII
+
+ +Neon branches eliminate the overhead of environment setup – no need to provision a new instance, create a new database, populate it, or worry about environments drifting out of sync. But if your production database contains PII, branching it directly is not an option. + +That’s where the [PostgreSQL Anonymizer extension (anon)](https://neon.tech/docs/extensions/postgresql-anonymizer) comes in. This open-source extension lets you define masking rules on sensitive columns in your database, replacing real values with fake but realistic-looking alternatives. Neon currently supports static masking, meaning the data is permanently rewritten on the branch. + +You can choose from multiple masking strategies, such as + +- `anon.fake`: replaces real values with random ones +- `anon.partial`: masks part of a field, like a credit card or phone number +- `anon.noise: adds` variability to numerical data + +Here’s a simple example using the `users` table. Suppose we have this data: + +```sql +SELECT * FROM users LIMIT 3; + + id | first_name | last_name | email | iban +----+--------------+-------------+------------------------------+------- + 1 | Real First Name 1 | Real Last Name 1 | user1@example.com | REALIBAN1 + 2 | Real First Name 2 | Real Last Name 2 | user2@example.com | REALIBAN2 + 3 | Real First Name 3 | Real Last Name 3 | user3@example.com | REALIBAN3 +``` + +To anonymize this data, we apply masking rules using the faking strategy (`anon.fake`). This strategy replaces real values with randomly generated, fake but realistic data using functions provided by the extension: + +```sql +SECURITY LABEL FOR anon ON COLUMN users.first_name IS 'MASKED WITH FUNCTION anon.fake_first_name()'; +SECURITY LABEL FOR anon ON COLUMN users.last_name IS 'MASKED WITH FUNCTION anon.fake_last_name()'; +SECURITY LABEL FOR anon ON COLUMN users.email IS 'MASKED WITH FUNCTION anon.fake_email()'; +SECURITY LABEL FOR anon ON COLUMN users.iban IS 'MASKED WITH FUNCTION anon.fake_iban()'; +``` + +After running the anonymization command, + +```sql +SELECT anon.init(); +SELECT anon.anonymize_database(); +``` + +You would see something like this when querying the table again: + +```sql +SELECT * FROM users LIMIT 3; + + id | first_name | last_name | email | iban +----+------------+-----------+------------------------+----------------------------- + 1 | Rhonda | Alvarado | bryanalan@example.net | GB34QDZL89198122631902 + 2 | Darius | Reyes | brandon57@example.com | GB96LBQE53732061681569 + 3 | Stefanie | Byrd | barbara40@example.com | GB67CAZQ75813049489060 +``` + +The real data is gone, replaced by randomly generated values using PostgreSQL Anonymizer’s built-in faker functions. This approach is far more realistic and scalable than using handcrafted seed data. [Seed files are hard to maintain](https://neon.tech/blog/how-to-maintain-seed-data) and rarely reflect your actual production schema. As your app evolves with new tables, columns, relationships, your seed data drifts behind. + +The anon extension takes care of masking sensitive data while preserving the structure, schema, and referential integrity of the database. Foreign keys, data types, and relationships remain intact, making the anonymized database safe but fully usable as a production clone. + +## The Workflow: Create Once, Reuse Everywhere + +Once anonymized, the anon branch behaves just like any other Neon branch, and it can serve as the template for all your non-prod environments. You can branch off of it instantly to create as many environments as you’d like. + +The workflow is simple: + +1. **Create a branch from your production database.** This branch is a copy-on-write snapshot of prod, schema, data, and all, but fully isolated and safe to modify. +2. **Anonymize the branch using masking rules.** Apply column-level masking via the PostgreSQL Anonymizer extension. You can do this manually (with SQL commands) or automatically (with a GitHub Action that runs on pull request creation). +3. **Use the anonymized branch as your base.** Now that your sensitive data is masked, you can branch from this anonymized copy as many times as you want for any non-prod use case, for example: + 1. **Per-PR preview environments**. Spin up a database per pull request with realistic test data, without any risk of exposing real users’ information + 2. **CI pipelines**. Test against real schema and realistic data while staying compliant and secure + 3. **Ephemeral environments**. Create and discard environments freely as part of your dev workflow + 4. **Local development environments**. Give every engineer a realistic sandbox + 5. **Contractor or partner access**. Share a branch of your database for testing or demos, without giving access to actual user data + 6. **Safe debugging**. Reproduce bugs or verify migrations using realistic anonymized data, without worrying about leaks + +Since these branches are all copy-on-write, the data overhead is minimal, and the experience is fast. + + +Our docs include an example of [how to use GitHub Actions to create an anonymized Neon branch every time a pull request is opened.](https://neon.tech/docs/workflows/data-anonymization#automate-data-anonymization) The Action installs the extension, applies masking rules, and runs the anonymization, all in one CI job. + + +## Start Building with Anonymized Branches in Neon + +Cloning production environments has always been a pain, especially when sensitive data is involved. But by combining Neon’s branching architecture and  the PostgreSQL Anonymizer extension it’s much easier to create safe, production-like environments. Just branch from production, anonymize once, reuse everywhere. + +To get started, [sign up to Neon](https://console.neon.tech/signup) and follow [this guide](https://neon.tech/docs/workflows/data-anonymization). If you have any questions on how to integrate anonymized branching workflows into your environment, [reach out to our team.](https://neon.tech/contact-sales) + + +If you sign up [via this link](https://fyi.neon.tech/credits), your first $100 are on us! + diff --git a/content/blog/posts/ephemeral-environments-aws-serverless.md b/content/blog/posts/ephemeral-environments-aws-serverless.md new file mode 100644 index 0000000000..4dbe1b3084 --- /dev/null +++ b/content/blog/posts/ephemeral-environments-aws-serverless.md @@ -0,0 +1,411 @@ +--- +title: 'Simplify Serverless in AWS: How to Use Neon and Ephemeral Environments' +description: A step-by-step guide +excerpt: >- + “Ephemeral environments” refers to the ability to create short-lived copies of + your system so that: It’s a powerful practice and works great with services + that charge on a pay-per-use basis. There is no extra charge for these + environments – it’s only the activities that count. Ho... +date: '2025-05-05T16:53:04' +updatedOn: '2025-05-05T16:53:08' +category: workflows +categories: + - workflows +authors: + - yan-cui +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/cover.png + alt: null +isFeatured: true +seo: + title: >- + Simplify Serverless in AWS: How to Use Neon and Ephemeral Environments - + Neon + description: >- + A step-by-step guide for using Neon with ephemeral environments in AWS. + Guest blog post by Yan Cui (AWS Serverless Hero). + keywords: [] + noindex: false + ogTitle: >- + Simplify Serverless in AWS: How to Use Neon and Ephemeral Environments - + Neon + ogDescription: >- + A step-by-step guide for using Neon with ephemeral environments in AWS. + Guest blog post by Yan Cui (AWS Serverless Hero). + image: >- + https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/cover.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/screenshot-2025-05-02-at-55701percente2percent80percentafpm-1024x577-974eb96c.png) + + +This blog post was originally published in [TheBurningMonk.com](https://theburningmonk.com/2025/04/how-to-use-neon-and-ephemeral-environments-to-simplify-serverless-development/). + + +“Ephemeral environments” refers to the ability to create short-lived copies of your system so that: + +- Developers can work in their isolated environments and make changes without affecting others. +- Test data does not pollute shared environments such as dev and staging. +- Once a feature is complete, the ephemeral environment can be safely torn down. + +It’s a powerful practice and works great with services that charge on a pay-per-use basis. There is no extra charge for these environments – it’s only the activities that count. + +However, services that charge by uptime (aka “_serverful_”), such as Amazon RDS, present a logistical problem – your uptime cost increases with the number of environments. + +To mitigate the extra uptime cost, you can [share the same RDS cluster across multiple environments](https://theburningmonk.com/2023/02/how-to-handle-serverful-resources-when-using-ephemeral-environments/). But this adds friction to the development workflow and complicates deployment: + +- The RDS cluster must be deployed separately, so you can no longer review and update the whole system as a single unit. +- You need scripts to create and delete environment-specific databases or schemas. + +Fortunately, [Neon Serverless Postgres](https://neon.tech/?ref=tbm-blog) solves these problems. + +In this article, we will explore how Neon works, and I will provide a step-by-step guide for using Neon with ephemeral environments. + +## What is Neon? + +Neon is a serverless database platform built around PostgreSQL. + +It separates storage and compute, and you pay only for the processing power you use. On paper, it’s similar to Aurora Serverless v2 in many ways, but there are some notable differences. + +### Scaling & Cold Starts + +Both can scale to zero during inactivity. But Neon scales much faster than Aurora Serverless v2, which takes a slower and more conservative approach. + +Neon also cold starts from zero in 500ms vs. 15s for Aurora Serverless v2. This is important for development and ephemeral environments, where there are often long gaps between bursts of database activities. + +### Connection Pooling & Data API + +Neon integrates PgBouncer directly into its architecture, enabling it to handle up to **10,000 concurrent connections** and queue requests during spikes, rather than rejecting them. + +Aurora Serverless v2 calculates the max connections based on ACU. Serverless applications often create many short-lived connections as Lambda execution environments are created and destroyed. This can cause connection pool exhaustion, which is why many serverless applications would use RDS Proxy to scale the number of concurrent connections or switch to Aurora Serverless v2’s Data API. + +Neon also has a data API that allows you to execute queries over HTTP or WebSockets. You can use the data API via Neon’s [serverless driver](https://neon.tech/docs/serverless/serverless-driver?ref=tbm-blog). + +### Data Branching + +Because Neon separates storage from compute, you can easily create a copy of an existing database by branching from it. Think Git branching but for your data! + +This works great with ephemeral environments. + +Want to run some tests without polluting your database? Create a branch for the tests and delete it afterwards. + +Want to let multiple developers work on the same codebase simultaneously? Create a branch for each so they don’t step on each other’s toes. + +Every ephemeral environment can have its own branch of your development database. + +If you need to seed the database first, then seed the data in the development database and every branch will inherit the seed data. + +Importantly, branching a Neon database is **instant**! Which is important for automation and developer productivity. + +By default, branching a database copies both the data and schema. But Neon also supports schema-only branching as well, in case you don’t want any seed data and want to start from a clean slate. + +## Using Neon with Ephemeral Environments + +Let’s use a simple TODO API to demonstrate how to use Neon and how to use it with ephemeral environments. You can find all the relevant source code [here](https://github.com/theburningmonk/ephemeral-env-with-neon). + +### Create the todos database + +First, sign up for an account with Neon at [neon.tech](https://neon.tech/) and create a new project. Let’s call the database “**todos**”. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-318e722e.png) + +By default, this creates a `production` and `development` branch of the new `todos` database. Notice that the `development` branch has less compute power and is not intended for production use or load testing. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-1-2a1ca2f0.png) + +### Seeding the todos database + +In the demo code, there is a [SQL script](https://github.com/theburningmonk/ephemeral-env-with-neon/blob/main/migrations/001_create_todos_table.sql) for creating a “todos” table. + +```sql +CREATE TABLE IF NOT EXISTS todos ( + id UUID PRIMARY KEY, + title VARCHAR(255) NOT NULL, + description TEXT, + completed BOOLEAN DEFAULT FALSE, + created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +); +``` + +You can run this script directly in the Neon console. + +Go to the “**SQL Editor**”, select the `development` branch, and click `Run` to execute the script. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/screenshot-2025-05-02-at-61318percente2percent80percentafpm-1024x590-eb9b161f.png) + +You can also save frequently used scripts. + +Now that you have created a new `todos` table, you can also look at the data directly in the Neon console. + +Go to “Tables” and select the `development` branch. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-3-cbf4c0e1.png) + +Currently, there is no data. However, you can use the “**Add record**” button to add a row directly in the console. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-4-72378cdc.png) + +There is a lot more you can do in the Neon console. For example, + +- See the schema differences between this branch and its parent (think `git diff`). +- Reset a branch and update it to the latest schema and data from its parent (think `git reset`). +- Create read replicas. + +### Connecting to the todos database + +To connect to the database, select the branch you want, and click `Connect`. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-5-44290e57.png) + +Copy the connection string from the pop-up. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-6-b70b4a53.png) + +### The TODO API architecture + +This is the high-level architecture of the TODO API, where a different Lambda function handles each CRUD operation. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-7-67b311ee.png) + +Everything is configured with the [Serverless Framework](https://serverless.com/) and deployed as a single unit. + +As shown below, each function has its own handler module. Shared code between the functions is kept in the same project folder. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-8-663357c7.png) + +The API path for each function is configured as follows. + +```javascript +createTodo: + handler: functions/createTodo.handler + events: + - http: + path: /todos + method: post + cors: true +``` + +During deployment, the Serverless Framework will look for an environment variable called `DATABASE_URL` and use it to populate an environment variable of the same name for each Lambda function. + +```javascript +environment: + DATABASE_URL: ${env:DATABASE_URL} +``` + +This way, our code can use the `DATABASE_URL` environment variable to initialise the [Neon serverless driver](https://neon.tech/docs/serverless/serverless-driver?ref=tbm-blog) at runtime, like this. + +```javascript +import { neon } from '@neondatabase/serverless'; + +const sql = neon(process.env.DATABASE_URL); +``` + +### Understanding the Neon serverless driver + +Take a look at the `lib/todos.js` module. + +Notice that we’re using the `sql` template tag to execute queries (against Neon’s data API) written as a string literal, like this: + +```javascript +import { neon } from '@neondatabase/serverless'; +import { v4 as uuidv4 } from 'uuid'; + +const sql = neon(process.env.DATABASE_URL); + +export const createTodo = async (title, description) => { + const id = uuidv4(); + const result = await sql`INSERT INTO todos (id, title, description, completed, created_at) + VALUES (${id}, ${title}, ${description}, ${false}, ${new Date().toISOString()}) + RETURNING *`; + + return result[0]; +}; +``` + +Your immediate reaction might be “This is vulnerable to SQL injection attacks!”. But rest assured, the `sql` template tag is inherently safe from SQL injection attacks. + +It’s written as a JavaScript tag function, which receives the string literal and its values as separate parameters. Internally, the `sql` template tag converts these into a SQL template with separate parameters. + +It’s a really nice feature and helps keep your code clean and safe at the same time. If you want to learn more about how this works, then read [this article](https://neon.tech/blog/sql-template-tags?ref=tbm-blog) for more details. + +One thing to note, however, is that this only works if you use the `sql` template tag with a string literal directly! The following will not work because the `sql` tag function is called with a string, not a string literal. + +```javascript +export const execute = async (query) => { + return await sql query; +}; + +export const createTodo = async (title, description) => { + const id = uuidv4(); + const result = await execute(`INSERT INTO todos (id, title, description, completed, created_at) + VALUES (${id}, ${title}, ${description}, ${false}, ${new Date().toISOString()}) + RETURNING *`); + + return result[0]; +}; +``` + +### Handling dynamic queries + +Ok, but what if you need to construct the query dynamically? + +For example, in the `updateTodo` function, we only want to update a field if the caller provides a new value. We would need to construct the SQL query dynamically based on the user’s request. + +The Neon serverless driver also has a `query` function for these cases. + +It takes a query string with embedded `$1`, `$2` (etc.) placeholders, followed by an array of query parameters. So you can build up the SQL query string dynamically, like this: + +```javascript +export const updateTodo = async (id, title, description, completed) => { + const updates = []; + const values = []; + let paramCount = 1; + + if (title!== undefined) { + updates.push(`title = $${paramCount}`); + values.push(title); + paramCount++; + } + + if (description!== undefined) { + updates.push(`description = $${paramCount}`); + values.push(description); + paramCount++; + } + + if (completed!== undefined) { + updates.push(`completed = $${paramCount}`); + values.push(completed); + paramCount++; + } + + values.push(id); + + const result = await sql.query(`UPDATE todos + SET ${updates.join(', ')} + WHERE id = $${paramCount} + RETURNING *`, values); + + if (result.length === 0) { + return null; + } + + return result[0]; +}; +``` + +Ok, so that’s the code, what about the ephemeral environments? + +### Creating & deleting ephemeral environments + +You can create a new branch of your database every time you create an ephemeral environment for your application. + +You can use the Neon console to create and delete branches. But we want to automate this process to eliminate manual steps. + +Fortunately, you can use the [Neon API SDK](https://neon.tech/docs/reference/typescript-sdk?ref=tbm-blog) to accomplish this. The [demo repo](https://github.com/theburningmonk/ephemeral-env-with-neon) contains examples of this – have a look in the [scripts folder](https://github.com/theburningmonk/ephemeral-env-with-neon/tree/main/scripts). + +Your workflow might look like this: + +1. Branch your source code to start work on a new feature called `tbm-042`. +2. Run `node scripts/create-branch development tbm-042` to create a new branch called `tbm-042`. The script outputs the connection URL for the new branch. Capture this in an environment variable called `DATABASE_URL`. This is the environment variable that the Serverless Framework will look for in the next step. +3. Run `npx serverless deploy -s tbm-042` to create a new ephemeral environment (also called `tbm-042`) for your application. +4. Iterate on the code and run tests against the database branch. You can find some example tests in the `tests` folder of the demo code. +5. Create a PR with your changes. +6. Run `npx serverless remove -s tbm-042` to delete the ephemeral environment. +7. Run `node scripts/delete-branch.cjs tbm-042` to delete the `tbm-042` branch in Neon. + +### Using ephemeral environments in CI/CD pipelines + +Another common use case for ephemeral environments is in CI/CD pipelines. This ensures that tests are run against a clean, well-defined initial system state and avoids polluting shared environments with test data. + +Neon also offers [several GitHub Actions](https://neon.tech/docs/guides/branching-github-actions) to help you automate the creation and deletion of branches. + +In the demo app, you can see an [example workflow](https://github.com/theburningmonk/ephemeral-env-with-neon/blob/main/.github/workflows/dev.yaml) that: + +1. Create a branch off the latest in the `development` branch. +2. Runs unit tests against the newly created database branch. +3. Create an ephemeral environment for the TODO API and point it to the new database branch. _The_ [serverless-export-outputs](https://www.npmjs.com/package/serverless-export-outputs) _plugin captures the_ `ServiceEndpoint` _stack output in a_`.env` _file._ +4. Runs end-to-end tests against the ephemeral environment. _These tests use the_`.env` _file to find out where the deployed API is._ +5. Deletes both the ephemeral environment and the database branch. +6. Deploy the application changes to the dev environment. + +Again, creating and deleting database branches is instant (see below). It helps to keep the pipeline feeling fast and snappy despite doing quite a lot of things! + +![Image](https://cdn.neonapi.io/public/images/pages/blog/ephemeral-environments-aws-serverless/image-9-01ebd6d0.png) + +Here are the relevant steps for reference. + +```yaml +- name: create Neon branch + id: create-branch + uses: neondatabase/create-branch-action@v6 + with: + project_id: ${{ secrets.NEON_PROJECT_ID }} + api_key: ${{ secrets.NEON_API_KEY }} + # name of the parent branch + parent_branch: development + # name of the new branch + branch_name: gh-${{ github.sha }} + database: todos + role: todos_owner + +- name: npm ci + run: npm ci + +- name: run unit tests + run: npm run test:unit + env: + DATABASE_URL: ${{ steps.create-branch.outputs.db_url }} + +- name: deploy to ephemeral environment + id: deploy-sls + run: npx serverless deploy --stage dev-gh-actions + env: + DATABASE_URL: ${{ steps.create-branch.outputs.db_url }} + SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }} + +- name: run e2e tests + run: npm run test:e2e + env: + DATABASE_URL: ${{ steps.create-branch.outputs.db_url }} + +- name: delete ephemeral environment + if: ${{ always() && steps.deploy-sls.outcome == 'success' }} + run: npx serverless remove --stage dev-gh-actions + env: + DATABASE_URL: ${{ steps.create-branch.outputs.db_url }} + SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }} + +- name: delete Neon branch + if: ${{ always() && steps.create-branch.outcome == 'success' }} + uses: neondatabase/delete-branch-action@v3 + with: + project_id: ${{ secrets.NEON_PROJECT_ID }} + api_key: ${{ secrets.NEON_API_KEY }} + # name of the new branch + branch: gh-${{ github.sha }} + +- name: deploy to dev + run: npx serverless deploy + env: + DATABASE_URL: ${{ secrets.DEV_DATABASE_URL }} + SERVERLESS_ACCESS_KEY: ${{ secrets.SERVERLESS_ACCESS_KEY }} +``` + +### Summary + +In this article, we discussed: + +- Why you should use ephemeral environments. +- Why Neon is a good fit for ephemeral environments. +- How Neon works and how to use the Neon console to manage branches and query your data. +- How to use the Neon serverless driver to query your data. +- How to automate the process of creating and deleting branches, and how to incorporate them into a productive development workflow. +- How to use Neon with ephemeral environments in CI/CD pipelines. + +With Neon and ephemeral environments, standing up a fresh copy of your database is simple, fast and cost-efficient. Make your changes, run your tests, validate your feature, then tear it down. You’ll end up spending less time on infrastructure and more time on building. + +--- + +_Neon has a Free Plan. [Sign up without a credit card](https://console.neon.tech/signup) and start building._ diff --git a/content/blog/posts/escaping-the-aws-rds-cost-spiral-a-better-way-to-scale-postgres.md b/content/blog/posts/escaping-the-aws-rds-cost-spiral-a-better-way-to-scale-postgres.md new file mode 100644 index 0000000000..a7e7699cc1 --- /dev/null +++ b/content/blog/posts/escaping-the-aws-rds-cost-spiral-a-better-way-to-scale-postgres.md @@ -0,0 +1,124 @@ +--- +title: Escaping the AWS RDS Cost Spiral +description: 'RDS starts cheap, but its rigid architecture drives up costs at scale' +excerpt: >- + If you look at RDS pricing, everything looks quite affordable (storage, + compute, backups) – so how are so many teams spending six figures on it? Usage + is part of the story, of course, but those bills are also very inflated. At + scale, there’s more going on. Scaling a database does... +date: '2025-05-23T18:16:19' +updatedOn: '2025-07-01T18:35:33' +category: postgres +categories: + - postgres + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/escaping-the-aws-rds-cost-spiral-a-better-way-to-scale-postgres/cover.jpg + alt: null +isFeatured: false +seo: + title: Escaping the AWS RDS Cost Spiral - Neon + description: >- + As you scale, RDS starts getting expensive - not because of unit prices but + because of architectural traps most teams don’t see coming. + keywords: [] + noindex: false + ogTitle: Escaping the AWS RDS Cost Spiral - Neon + ogDescription: >- + As you scale, RDS starts getting expensive - not because of unit prices but + because of architectural traps most teams don’t see coming. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/escaping-the-aws-rds-cost-spiral-a-better-way-to-scale-postgres/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/escaping-the-aws-rds-cost-spiral-a-better-way-to-scale-postgres/neon-cost-spiral-1024x576-815e734b.jpg) + +If you look at RDS pricing, everything looks quite affordable (storage, compute, backups) – so how are so many teams [spending six figures on it](https://www.reddit.com/r/devops/comments/1gyjwoj/postgres_rds_is_too_expensive/)? Usage is part of the story, of course, but those bills are also very inflated. At scale, there’s more going on. Scaling a database doesn’t just mean more usage, it also means more regions, more environments, stricter recovery policies. That’s when costs really start climbing in AWS. + +RDS’s rigid architecture locks you into structural patterns that quietly inflate your bill as you grow. + +## The 4 Forces Behind the AWS RDS Cost Spiral + +The true cost of RDS reveals itself not in pricing tables, but in how its architecture behaves at scale. Here are four common patterns we’ve seen inflate bills of otherwise well-designed systems. + +### Every environment becomes a full-blown instance + +Most teams don’t just have one production database. They also run staging, development, region-specific mirrors. In RDS, each environment requires a completely separate instance: separate compute, separate storage, separate snapshots. Teams pay for all those environments in full. Add terabytes of data, and the costs multiply fast. + +
+

“We need multi-terabyte dev environments to mirror prod. In RDS, that means duplicating storage, and the costs are ridiculous.”



Engineer at a logistics company. Migrated TB-size workload from RDS to Neon

+
+ +### Storage volumes only go up + +In RDS, you have to provision storage in advance. You can scale a volume up once every six hours if you need more space. And here’s the catch – RDS doesn’t let you shrink a storage volume once it’s been scaled. Even if you move cold data to S3, the EBS volume stays large, and you keep paying for the peak. The result for most teams is half-empty, multi-terabyte storage volumes on their bill. + +### Compute is provisioned for peak and paid for 24/7 + +Another consequence of RDS’s rigid architecture is that you have to provision fixed compute capacity. **But your traffic likely isn’t constant (it rarely is).** Most real-world systems follow usage patterns – daytime spikes, nighttime lulls, weekend dips. RDS doesn’t account for that. You pick an instance size big enough to handle your peak load, and that’s what you pay for. This also applies to non-prod environments that may only be used for a few hours each day. The meter keeps running. + +### Snapshots are the only realistic backup, but they’re costly (and slow) + +Once your Postgres instance in RDS reaches a certain size, full logical backups become impractical. It’d be too slow to recover from them, the downtime would be many hours or even days. That leaves snapshots as the only viable recovery option for instances at scale, but RDS snapshots come with a cost. You pay to store them, you pay to retain them… And restoring from them is still anything but fast (it can still take hours to restore from a multi-TB snapshot). + +
+

“We kept snapshots just in case in RDS, but knew we’d lose hours restoring from them”

+
+ +## A Better Architecture for Scaling Postgres + +When storage and compute are tightly coupled, when environments are full instance clones, and when scaling is manual, costs naturally spiral as you scale – there’s no other way around it. [Neon](https://neon.tech/home) was built to break this pattern. + +If you’re new here: Neon is a [serverless Postgres platform](https://neon.tech/docs/get-started-with-neon/why-neon) built to fix all the inefficiencies of traditional managed databases. By [separating storage and compute](https://neon.tech/docs/introduction/architecture-overview) and implementing an innovative storage design with [copy-on-write branching](https://neon.tech/blog/instantly-copy-tb-size-datasets-the-magic-of-copy-on-write), Neon enables a more flexible and efficient way to manage Postgres at scale. + +### Create branches instead of full instances + +_**No more duplicating multi-terabyte volumes just to test a migration.**_ + +Neon lets you spin up [copy-on-write branches](https://neon.tech/flow) of your database in a second, even if you’re working with tens of terabytes. Branches act like fully isolated environments that don’t duplicate storage and reflect their parent perfectly, but they have their own compute endpoint to avoid interfering with production in any way. Whether you need a dev environment, a test sandbox, or a point-in-time clone, you can spin up a branch. They’re instant and lightweight, and can also be handled programmatically via CI/CD or API. + +### Autoscale compute and storage + +_**You only pay for the compute and storage that your databases actually use.**_ + +[Neon automatically scales compute up or down based on workload](https://neon.tech/use-cases/serverless-apps). If a dev database is not being used, it scales to zero. When demand spikes in production, it scales up in real time, both connections and throughput. Storage also scales dynamically, up or down, with no manual provisioning required. There’s no need to pick an instance size or preallocate storage. Usage drives cost directly, not guesses about peak traffic. + +### Restore from any point in time in seconds + +_**Restores become economical, fast at scale, and testable.**_ + +Neon implements [point-in-time restore (PITR](https://neon.tech/blog/recover-large-postgres-databases)) as a native part of its branching model. You can restore from any historical point instantly (no matter how large your database is) by creating a branch from that point. You can query that branch directly, test migrations against it, or promote it to production. This model avoids the operational and cost overhead of traditional snapshot-based recovery. + +## Summary: How Neon Breaks the AWS RDS Cost Spiral + +**Via branches,** + +- There’s no need to provision full-size instances for staging or QA +- Shared, copy-on-write storage makes dev environments affordable, even at TB scale + +**Via autoscaling,** + +- Compute costs shrink during off-hours, weekends, or idle dev time +- There’s no getting stuck in large storage volumes that cannot be resized down + +**With instant restores built on branching**, + +- There’s no need to retain and store large, infrequent RDS snapshots +- Inspecting or debugging historical data becomes fast and inexpensive + +| Cost driver | AWS RDS | Neon | +| ------------------------------- | -------------------------------------------------------- | ---------------------------------------------------------------- | +| **Multiple environments** | Requires full-size instances per env (compute + storage) | Lightweight branches reuse storage, created instantly | +| **Storage growth** | Volumes only grow, can’t shrink after scaling | Storage scales up and down automatically | +| **Idle compute** | Compute is always-on, even when traffic drops | Compute autoscales and scales to zero when idle | +| **Dev/prod parity** | Duplicating large datasets for dev is costly | Dev branches match prod instantly, no extra storage cost | +| **Backup and recovery** | Snapshots are expensive and slow to restore | Branches enable instant recovery at any point in time | +| **Inspecting historical data** | Requires restoring and provisioning a new instance | Just branch from a timestamp and query directly | +| **Overprovisioning for spikes** | Must provision for peak load, pay for it 24/7 | Compute scales up and down with demand, no need to overprovision | + +## If You’re Running Into These Walls… + +You’re not alone. Neon offers a modern alternative for scaling Postgres without the architectural baggage that translates not only in a cost spiral but also on a degraded developer experience. [Start a free Neon project](https://neon.tech/) to get a feel for the platform, or [book a call with our team](https://neon.tech/contact-sales) to see if we can help reduce your AWS RDS bill. diff --git a/content/blog/posts/event-driven-architectures-using-neon-and-sequin.md b/content/blog/posts/event-driven-architectures-using-neon-and-sequin.md new file mode 100644 index 0000000000..73ceb9480e --- /dev/null +++ b/content/blog/posts/event-driven-architectures-using-neon-and-sequin.md @@ -0,0 +1,186 @@ +--- +title: Event-driven architectures using Neon and Sequin +description: A low-maintenance stack to stream database changes +excerpt: >- + You’ll often want changes in your database to trigger changes in your + application. Or want to replicate a database change to another service. Think + of fanning out work when a new order comes in, updating a materialized view as + sales finalize, or logging a change to an audit syste... +date: '2024-10-22T15:57:27' +updatedOn: '2024-10-22T15:59:20' +category: community +categories: + - community +authors: + - eric-goldman +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/event-driven-architectures-using-neon-and-sequin/cover.jpg + alt: null +isFeatured: false +seo: + title: Event-driven architectures using Neon and Sequin - Neon + description: >- + A lightweight CDC stack to detect, stream, and process every row change in + Neon with no new dependencies using Sequin. + keywords: [] + noindex: false + ogTitle: Event-driven architectures using Neon and Sequin - Neon + ogDescription: >- + A lightweight CDC stack to detect, stream, and process every row change in + Neon with no new dependencies using Sequin. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/event-driven-architectures-using-neon-and-sequin/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/event-driven-architectures-using-neon-and-sequin/event-driven-architectures-using-neon-and-sequin-1-1024x576-f98b77f4.jpg) + +You’ll often want changes in your database to trigger changes in your application. Or want to replicate a database change to another service. Think of fanning out work when a new order comes in, updating a materialized view as sales finalize, or logging a change to an audit system. + +Capturing changes from your database (CDC) to then trigger events or replicate data is commonly implemented [with a combination of Debezium and Kafka](https://neon.tech/docs/guides/logical-replication-kafka-confluent). Debezium captures the changes while Kafka persists those changes until they are processed. But this is a notoriously complex stack: both tools require significant time and expertise to properly configure while also requiring you to master and scale new infrastructure. It’s a heavy dependency compared to the nimbleness of Neon. + +In this post, we present you with an alternative, using [Sequin](https://sequinstream.com/) to detect, stream, and process every row change in Neon with no new dependencies. + +## How Sequin streams rows from Neon + +Sequin is a tool for capturing changes and streaming data from your Postgres database. It uses a logical replication slot to create a strictly ordered view of rows in your tables, making it possible to process every change to a table’s rows as a stream. You can then consume rows using Sequin’s APIs. + +So, when you complete an `insert into order`, Sequin detects the new row and delivers it to your consumers. Sequin will retry delivery of a row until it is processed – just like Debezium / Kafka. + +In effect, Sequin streams rows from your Neon table without copying the data to a new service. Because it’s all Postgres under the hood – Neon’s amazing developer tooling (like branching, recovery, etc) works with Sequin as well. + +Let’s see how to get this set up: + +## Streaming a Neon table with Sequin + +You can use Sequin as an open-source, self-hosted Docker container in front of your Neon database, or take advantage of their hosted service. + +You’ll then connect your Neon database to Sequin, configure the tables you want to stream, and then begin processing changes in real-time using Sequin’s APIs. We’ll walk through the steps below, but feel free to check out the [integration guide in the Neon docs](https://neon.tech/docs/guides/sequin) if you need more details. + +### Connect your Neon database to Sequin + +Sequin uses a logical replication slot to detect changes from your Postgres database. As a first step, [enable logical replication for your project:](https://neon.tech/docs/guides/logical-replication-postgres) + +1. Select your project in the Neon Console. +2. Navigate to settings and select **Logical Replication**. +3. Click **Enable** to enable logical replication. + +You can verify that logical replication is enabled by running the following query in the Neon SQL Editor: + +````sql +``` +SHOW wal_level; +wal_level +----------- +logical +``` +```` + +Now that your Neon database is set up for replication, connect it to Sequin: + +1. In the Neon Console, copy your database connection string from the **Connection Details** section on the **Project Dashboard.** + +1. + +1. Open the Sequin Console in a new tab and click the **Connect Database** button. Select the **Autofill with URL** button and paste your database string. +1. Use the SQL Editor in your Neon project to create a replication slot for Sequin by executing the following SQL query: + +```sql +SELECT pg_create_logical_replication_slot('sequin_slot', 'pgoutput'); +``` + +4. Create a publication to indicate which tables will publish changes to the replication slot. Run the following SQL command: + +```sql +CREATE PUBLICATION sequin_pub FOR ALL TABLES; +``` + +5\. Finally, back in the Sequin Console, give your database a name (e.g., `neondb`) and click **Create Database.** + +### Create a sequence + +Sequin is now connected to your Neon database via a replication slot. You’ll now create a Sequence – which is how Sequin creates a strictly ordered view of the rows in your table that you can consume, rewind, and replay as a stream. + +1. Navigate to the **Sequence** tab and click **Create Sequence.** +2. Select the table you want to stream and then finish setting up the Sequence by selecting a sort column. +3. Click **Create Sequence.** + +Your Neon table is now configured to stream rows as they change. Notably, Sequin’s designed to pull rows right from your table before delivering them to consumers. It doesn’t copy rows into a new system. + +### Setup a new consumer group + +1. Navigate to the **Consumers** tab and click **Create Consumer Group.** +2. Select the Sequence you configured in the previous step and add any filters. Sequin can filter which messages are delivered to your consumer using SQL where conditions. +3. Now, select where in the Sequence the consumer should start. You can have the consumer start at the beginning of the Sequence, at a specific point in the Sequence, or from now onward.. +4. Finally, select **pull** and click **Create Consumer.** + +Every time a row changes in your Neon database, Sequin will add it to the end of the Sequence. The row will be delivered to your consumer until it is acknowledged, providing an exactly-once processing guarantee. + +### Process rows + +Now, work with the rows streaming from your table. Here’s how you might process orders. + +Create a new order: + +1. + +```sql +INSERT INTO orders (order_id, customer_id, order_date, total_amount, status) +VALUES (1001, 'CUST123', '2024-10-11', 199.99, 'PENDING');` +``` + +Now, **receive** the next message from your consumer: + +1. + +```bash + curl --request POST \ + --url https://api.sequinstream.com/api/http_pull_consumers/orders-pull-consumer/receive \ + --header 'Authorization: Bearer YOUR_API_TOKEN' \ + --header 'Content-Type: application/json' \ + -d '{ "batch_size": 10 }' +``` + +Which will return the order you just created: + +```bash +{ + "data": [ + { + "ackToken": "MTYyeJ7abUjl1pO", + "data": { + "record": { + "id": 1001, + "customerId": "CUST123", + "orderDate": "2024-10-11", + "totalAmount": 199.99, + "status": "PENDING" + } + } + } + ] +} +``` + +Once you process the order, your consumer can acknowledge the message: + +```bash + curl --request POST \ + --url https://api.sequinstream.com/api/http_pull_consumers/orders-pull-consumer/ack \ + --header 'Authorization: Bearer YOUR_API_TOKEN'\ + --header 'Content-Type: application/json' \ + --data '{ + "ack_tokens": ["MTYyeJ7abUjl1pO"] + }' +``` + +Every change to your orders table will be captured, added to the Sequence, and made available to consumers until processed. + +## Getting started + +You can start streaming data from your Neon database right now: + +- Create a [Neon project](https://neon.tech/) +- Create a [Sequin account](https://sequinstream.com/) + +If you have any questions, we’ll be happy to help on the [Neon](https://discord.gg/92vNTzKDGp) or [Sequin](https://discord.gg/BV8wFXvNtY) Discord servers. diff --git a/content/blog/posts/expanding-our-partner-program.md b/content/blog/posts/expanding-our-partner-program.md new file mode 100644 index 0000000000..4a7fec45b6 --- /dev/null +++ b/content/blog/posts/expanding-our-partner-program.md @@ -0,0 +1,96 @@ +--- +title: Expanding our Partner Program +description: Bringing reliable and scalable Postgres to your developers +excerpt: >- + At Neon, we aspire to bring Postgres to as many developers as possible, a + vision that is shared by our rapidly growing ecosystem of partners. Since the + launch of our partner program, we have been fortunate to work with partners + like Vercel, Replit, and others to reach hundreds of... +date: '2024-02-20T16:27:05' +updatedOn: '2024-02-29T11:11:46' +category: company +categories: + - company +authors: + - arjun-rajeswaran +cover: + image: >- + https://neon.com/images/social-previews/index.jpg?updated=2026-01-15 + alt: null +isFeatured: false +seo: + title: Expanding our Partner Program - Neon + description: >- + We're announcing two Neon partner plans, designed to make it easier than + ever to start offering serverless Postgres to your developers. + keywords: [] + noindex: false + ogTitle: Expanding our Partner Program - Neon + ogDescription: >- + We're announcing two Neon partner plans, designed to make it easier than + ever to start offering serverless Postgres to your developers. + image: >- + https://neon.com/images/social-previews/index.jpg?updated=2026-01-15 +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/expanding-our-partner-program/neon-partner-program-e7d53b29.jpg) + +At Neon, we aspire to bring Postgres to as many developers as possible, a vision that is shared by our rapidly growing ecosystem of partners. Since the launch of our [partner program](https://neon.tech/blog/partner-program), we have been fortunate to work with partners like [Vercel](https://neon.tech/blog/neon-postgres-on-vercel), [Replit](https://neon.tech/blog/neon-replit-integration), and others to reach hundreds of thousands of developers who are now building their applications on serverless Postgres. + +
+

“By partnering with Neon, Vercel’s frontend platform is now the end‑to‑end serverless solution for building on the Web, from Next.js all the way to SQL”

+Guillermo Rauch, CEO of Vercel +
+ +Today, we are excited to announce an [expansion of our partner program](https://neon.tech/partners) by offering two partner tiers: + +1. **Standard Partners:** the simplest way for a developer platform to become a Neon partner. With fast onboarding, you can start to embed serverless Postgres into your platform, and offer our features to your customers. +2. **Premier Partners:** for larger platforms who would like custom features and dedicated resources, we’re offering the Premier Partners tier, a more dedicated program focused on the needs of the Enterprise. + +Who are these partner options for? The short answer: any developer platform interested in offering scalable databases to their users without having to worry about infrastructure. + +The long answer: we’re partnering with frontend platforms, low-code/no-code platforms, modern IDEs, data tools and essentially companies looking to make it easy for their developers to create and scale transactional databases in their applications. + +If this sounds like you, keep reading. + +## Why become a Neon partner? + +Over the past year, Neon has grown to serve over 500K databases. Developers love Neon’s product velocity, supported by features like [database branching](https://neon.tech/docs/introduction/branching), [scale to zero](https://neon.tech/docs/introduction/auto-suspend), and [autoscaling](https://neon.tech/docs/introduction/autoscaling). And our partners love that they can offer this experience to their customers by seamlessly integrating Neon into their platforms via the [Neon API](https://neon.tech/docs/reference/api-reference), also taking advantage of security enhancements like [IP allow](https://neon.tech/docs/introduction/ip-allow). + +If you’re a partner, you also get access to 24×7 technical support, and as part of our partnership, we will scale up our marketing efforts to help you reach more developers. Our [existing partners](https://neon.tech/partners) have already been able to leverage the Neon advantage to support their product and commercial success. [Vercel](https://neon.tech/blog/neon-postgres-on-vercel) and [Replit](https://neon.tech/blog/neon-replit-integration) incorporated serverless Postgres into their platforms and have been offering it to their developers since early 2023. [Many other](https://neon.tech/blog/neon-bunnyshell-integration) platforms have also decided to leverage Neon to serve their developers’ needs. + +Why are these companies choosing Neon? + +1. **Automated API for databases**. Neon is like Stripe for databases. Our partners just plug the Neon API into their platforms and start generating revenue.. +2. **Efficient cloud model.** Infrastructure management is not your concern. Neon automatically scales idle databases to zero, leading to significant cost savings, and scales up with load. +3. **Loved by developers.** Neon is Postgres, the most loved, most familiar, [most widely adopted database in the world](https://survey.stackoverflow.co/2023/#databases). + +## Introducing our partner plans + +As part of this expansion, we are excited to offer two partner options: + +| | **Standard Partner** | Premier Partner | +| --------------------------- | -------------------------------------------------------------------- | ------------------------------------------------------------ | +| **Commercials** | | | +| Monthly price | Starts at $500/mo
_Including usage_ | Custom | +| Usage | Billed at a 10% discount
versus list price | Custom | +| **Consumption** | | | +| Projects | Unlimited
_If > 200 projects contact for custom pricing_ | Unlimited | +| Storage | Unlimited | Unlimited | +| Branches | 500/project | Custom | +| Compute units | Up to 7 [compute units](https://neon.tech/docs/introduction/billing) | Custom | +| **Features** | All [Scale](https://neon.tech/2024-plan-updates) plan features | All [Enterprise](https://neon.tech/enterprise) plan features | +| Service SLA of 99.99 | ✔ | ✔ | +| Partner API | ✔ | ✔ | +| Custom reporting | | ✔ | +| Level 1 24X7 support | ✔ | ✔ | +| Level 2 shared 24X7 support | | ✔ | +| Dedicated partner manager | | ✔ | +| Custom domains | | ✔ | +| **Marketing** | | | +| Blog | ✔ | ✔ | +| PR | | ✔ | + +## Partner with us + +Interested? Reach out to our team [here](https://neon.tech/partners#partners-apply).

To you and all of our partners: thank you! Together, we’ll bring serverless Postgres to every developer. diff --git a/content/blog/posts/expire-neon-branches-automatically.md b/content/blog/posts/expire-neon-branches-automatically.md new file mode 100644 index 0000000000..03c66c35d0 --- /dev/null +++ b/content/blog/posts/expire-neon-branches-automatically.md @@ -0,0 +1,98 @@ +--- +title: Your Neon Branches Can Now Expire Automatically +description: 'Set an expiration date for your branches, Neon handles the cleanup' +excerpt: >- + Branching is one of the most powerful features in Neon. It lets teams spin up + isolated environments instantly and automate workflows through the API, and as + a result, developers and agents create many branches at scale – hundreds or + even thousands of them. To make managing short-... +date: '2025-08-13T18:45:44' +updatedOn: '2025-08-14T09:36:01' +category: product +categories: + - product + - workflows +authors: + - gustavo-salomao +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/expire-neon-branches-automatically/cover.jpg + alt: null +isFeatured: true +seo: + title: Your Neon Branches Can Now Expire Automatically - Neon + description: >- + You can now set an expiration time for Neon branches, so they're deleted + automatically - this makes it even easier to manage them. + keywords: [] + noindex: false + ogTitle: Your Neon Branches Can Now Expire Automatically - Neon + ogDescription: >- + You can now set an expiration time for Neon branches, so they're deleted + automatically - this makes it even easier to manage them. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/expire-neon-branches-automatically/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/expire-neon-branches-automatically/neon-ttl-branches-1-1-1024x576-47f624f9.jpg) + +[Branching](https://neon.com/docs/introduction/branching) is one of the most powerful features in Neon. It lets teams spin up isolated environments instantly and automate workflows through the API, and as a result, developers and agents create many branches at scale – hundreds or even thousands of them. + +To make managing short-lived branches even easier, we’re launching a new feature: [you can now set an expiration time for Neon branches](https://neon.com/docs/guides/branch-expiration), so it’s deleted automatically. This is a game-changer for managing branches at scale. + +## Supporting Branching Workflows at Scale + +Neon’s [branching model](https://neon.com/flow) is reshaping how teams build with Postgres. Instead of provisioning and managing separate database instances for every environment, you can create instant, production-like copies of your data using branches – each one isolated, resettable, and built on top of a shared storage layer. + +This unlocks a new level of flexibility and speed for modern development teams. A single project can support many branches, for example: + +- One branch per developer for fully isolated dev environments +- One branch per preview to power live feature previews during code reviews +- One branch per test run in CI pipelines for clean, reproducible tests +- Versioned app environments – where agents or users spin up many parallel branches to explore, fork, or checkpoint different versions of an application over time + +These workflows scale beautifully. Neon can handle hundreds of active branches in a single project because branches are lightweight by design – they inherit data and schema from their parent via copy-on-write, without duplicating storage, and they’re instant to create and scale to zero when idle. Branching has become the foundation for highly dynamic, automation-friendly infrastructure. + +But as branching scales, lifecycle management becomes increasingly important. Developers forget to clean up preview environments, test runs leave stale database branches behind, and agents start creating thousands of branches – and it starts getting annoying (and potentially expensive) to clean them up manually. + +## Let Your Branches Expire Themselves + +That’s why we built [expiration rules for Neon branches](https://neon.com/docs/guides/branch-expiration), a simple feature but one that truly improves the experience of managing the lifecycle of short-lived branches. Now, when creating a branch via the API, CLI, or Console, you can set an expiration time: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/expire-neon-branches-automatically/screenshot-2025-08-13-at-115159percente2percent80percentafam-68560f1e.png) + +Once that time is reached, Neon will automatically delete the branch. There’s no need to write cleanup scripts or worry about forgotten environments eating into your project’s storage; everything expires on schedule. + +### Under the hood + +Neon tracks each branch’s expiration timestamp and runs a periodic job to clean up expired branches. If the branch is still in use, you can reset or update the expiration time. But if it’s done, it disappears automatically, freeing up resources and keeping your project clean. + +## 4 Scenarios Where Expiring Branches Are Game-Changing + +This is a small feature that has a big impact, especially for teams running large-scale ephemeral environments. Here are some of the most common use cases: + +### Serverless ephemeral environments at scale + +[In serverless development, it’s common to spin up short-lived environments on demand](https://neon.com/blog/ephemeral-environments-aws-serverless), whether for dev containers, Lambda test runs, or Kubernetes-based previews. These environments are created automatically, used briefly, and then discarded. + +Neon fits perfectly into this model: branches are instant to create, scale to zero when idle, and reuse storage via copy-on-write. With expiration rules, these branches now clean up automatically, completing the serverless loop with zero manual work. + +### Versioned app timelines in agent-driven workflows + +[Neon’s branching lets agents checkpoint full app states (code + data) by creating branches at each version.](https://neon.com/blog/replit-app-history-powered-by-neon-branches) This enables true time travel – users can preview or restore any historical version. Behind the scenes, each checkpoint triggers a new branch at the specified point-in-time snapshot, meaning that the number of branches quickly grows – since agents generate dozens of these timelines per user, per session. + +By assigning expiration times to these branches, agents can generate versioned timelines at scale, without leaving behind clutter. + +### Preview environments per pull request + +Neon users love creating full-stack previews for every pull request using the [Vercel integration](https://neon.com/docs/guides/vercel-overview). These environments are often short-lived, tied to the lifecycle of a feature branch or a review cycle. + +Expiration rules make preview database branches self-cleaning. When the PR is closed, the countdown continues. Once the expiration is reached, Neon deletes the branch automatically. + +### CI pipelines with one branch per test run + +In a typical CI workflow, each test suite runs in its own environment – but with Neon, every test run can spin up a new branch derived from a known-good baseline. Expiration adds a simple lifecycle step: branches expire automatically after a defined window, ensuring that stale environments don’t accumulate over time. This is great for teams running hundreds or thousands of test jobs per day across distributed services. + +## Try it Now + +Expiration rules for Neon branches are available now via the Neon API, CLI, and Console. You can start using them today to bring more automation, scale, and simplicity to your development and testing workflows. [Check out the docs](https://neon.com/docs/guides/branch-expiration) for all the details, and if you haven’t tried Neon yet, [start a free account.](https://console.neon.tech/signup) diff --git a/content/blog/posts/export-to-csv-json-and-xlsx-from-the-neon-console.md b/content/blog/posts/export-to-csv-json-and-xlsx-from-the-neon-console.md new file mode 100644 index 0000000000..b0fa9d4f98 --- /dev/null +++ b/content/blog/posts/export-to-csv-json-and-xlsx-from-the-neon-console.md @@ -0,0 +1,111 @@ +--- +title: 'Export to CSV, JSON and XLSX from the Neon console' +description: 'New features, and new open-source libraries to power them' +excerpt: >- + After spending a bit of time with Neon’s SQL Editor making \d and friends + work, there were a couple of other things I wanted to bring to it, both + related to the query results. The first was a simple expand-to-window control + for the results pane, to get a better and less cramped l... +date: '2024-05-16T18:05:11' +updatedOn: '2024-05-18T00:51:04' +category: workflows +categories: + - workflows +authors: + - george-mackerron +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/cover.jpeg + alt: 'neon console export to csv, json, xlsx' +isFeatured: false +seo: + title: 'Export to CSV, JSON and XLSX from the Neon console - Neon' + description: 'New features, and new open-source libraries to power them' + keywords: [] + noindex: false + ogTitle: 'Export to CSV, JSON and XLSX from the Neon console - Neon' + ogDescription: >- + After spending a bit of time with Neon’s SQL Editor making d and friends + work, there were a couple of other things I wanted to bring to it, both + related to the query results. The first was a simple expand-to-window + control for the results pane, to get a better and less cramped look at query + result tables and […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/social.jpeg +--- + +![neon console export to csv, json, xlsx, zip](https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/neon-export-csv-json-zip-xlsx-1024x576-206e8b05.jpeg) + +After spending a bit of time with Neon’s SQL Editor [making `\\d` and friends work](https://neon.tech/blog/bringing-psqls-d-to-your-web-browser), there were a couple of other things I wanted to bring to it, both related to the query results. + +The first was a simple expand-to-window control for the results pane, to get a better and less cramped look at query result tables and (especially) the graphical `EXPLAIN` output produced by PEV2. You can now find such a button at the bottom right of the SQL Editor any time there are query result on-screen. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/expand-contract-0afee8bc.gif) + +The second was that I wanted to be able to download tables of results. One time, I ran a query in the SQL Editor, hoping to produce a simple chart of what came back, and was disappointed to find that my best next step was actually to open Terminal and connect via `psql` to re-run the same SQL using `\\COPY (...) TO '/path/to.csv' CSV HEADER`. I wanted to fix that. + +But what download formats should be provided? + +- CSV is an obvious choice: `psql` already offers it, almost any tool (Excel, Numbers, R, Stata, etc.) can import it, and it’s not too hard to generate by following [RFC4180](https://datatracker.ietf.org/doc/html/rfc4180) or the W3C’s [best practice recommendations](https://www.w3.org/TR/tabular-data-model/#h-syntax). +- A colleague suggested JSON, which is another widely-supported interchange format, and an easy way to ingest data from your programming language of choice. +- Finally, I thought it would be nice to offer a native Excel (.xlsx) file on top. + +On that last point, my colleagues were not immediately convinced: _Why bother, when Excel reads CSV?_ + +Well, when I download a CSV file for Excel, there’s a bunch of steps that always come next, and I find these get old pretty quickly. First, I drag the file to Excel (I can’t just open the file from Downloads, since CSV is associated with Numbers by default on Mac). Second, I bold, freeze, and auto-filter the headings. Next, I fix the column widths. Then, I make sure dates and times are sensibly represented. Finally, I save to a native Excel file. Wouldn’t it be delightful, I thought, if all that work could be automated away? + +Long story short, all three formats — CSV, JSON, and XSLX — can now be exported from the SQL Editor. The icon is at bottom right, just above the new expand-to-window control, any time that you’re looking at a query result table. It’s all done client-side, in JavaScript, using the query results the browser already has. + +So for results like this: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/screenshot2024-05-03at091800-1024x520-f0cb377c.png) + +You get downloads like this: + +![Export to csv](https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/screenshot2024-05-03at092053-1024x763-f90d6a50.png) + +![Export to json](https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/screenshot2024-05-03at092042-1024x763-e743c0f4.png) + +![Export to xlsx](https://cdn.neonapi.io/public/images/pages/blog/export-to-csv-json-and-xlsx-from-the-neon-console/screenshot2024-05-03at092154-1024x763-821e2cba.png) + +In the rest of this post I set out what I learned in implementing this feature. I also introduce a couple of new npm packages that could help anyone working on similar tasks. + +### CSV + JSON + +There were no big surprises implementing CSV and JSON download. The key thing to ensure for both formats was that very big or precise numbers (Postgres `int8` s and `decimals`) didn’t at any point get parsed to JavaScript numbers, because that would [cause precision to be lost](https://neon.tech/blog/parsing-json-from-postgres-in-js). + +The other tiny wrinkle was ensuring that there are no duplicate JSON object keys even when multiple result columns have the same name. + +### Excel’s XML + +Providing an `.xlsx` download is slightly less straightforward. Modern Office documents consist of a set of XML files combined into an ordinary `.zip` archive. So: first we need to create some XML files in the right format. And then we need to zip them up. + +There are, of course, libraries on npm that can do this for us. But the ones I found were either so basic that they didn’t do what I needed, or so complex that they would add a lot of weight to the Neon console’s bundled JavaScript. And anyway, doing it yourself is fun, educational, and free of worries about what npm horrors have been imported in dependencies many levels deep. 😀 + +Excel’s XML documents turn out to be moderately comprehensible. There are, much as you might expect, `` elements that contain `` (cell) elements that contain `` (value) elements, and so on. Much of this can be figured out simply by unzipping a simple `.xlsx` file, but there are also [useful references online](https://c-rex.net/samples/ooxml/e1/Part4/OOXML_P4_DOCX_SpreadsheetML_topic_ID0EBC63.html). + +There were only a couple of wrong turns: + +- I thought I knew that dates and times in Excel were represented as days since 1 January 1900, so I was surprised when the dates in my initial `.xlsx` files were off by exactly two days. The first problem was that dates in Excel are actually represented as days since 0 January 1900 (you and I might call that 31 December 1899, but Excel doesn’t: try entering zero in an Excel cell, then changing its display to a date format).

The second problem was Excel’s [1900-is-a-leap-year](https://learn.microsoft.com/en-us/office/troubleshoot/excel/wrongly-assumes-1900-is-leap-year) bug, which causes an off-by-one error for any date after 28 February 1900. The bug originated as a compatibility measure with Lotus 1-2-3. After bumping into it myself, I remembered first hearing about it in [a classic Joel on Software post](https://www.joelonsoftware.com/2006/06/16/my-first-billg-review/). +- I’d assumed that Excel column references were just numbers in base 26, using letters of the alphabet. If that were the case, A would represent zero and B would represent one, so the column after Z would be BA. But in Excel, the column after Z is AA. In decimal terms, it’s as if the number after 9 was 00. So this needed [a little bit more thought](https://github.com/jawj/xlsxtable/blob/5c1778679f897a9fbd13b53b90164ab2cf0adfaf/utils.ts#L4). + +### Client-side Zip + +With these issues fixed, I turned to zipping (and unzipping). First, note that if you’re ever tinkering with an `.xlsx` file — unzipping, modifying, and re-zipping it — you may encounter these small but briefly head-scratching obstacles: + +- At zip time, the _order_ of files in the `.zip` archive matters. In particular, a file called `[Content_Types].xml` has to be first, which means that manually zipping the right files in one go doesn’t always work. Instead, use `zip` at the command line to zip `[Content_Types].xml`, then use it again to add the other files. +- At unzip time, Archive Utility on the Mac flatly refuses to unzip an `.xlsx` file that has been renamed to `.zip`. “Unable to expand ‘workbook.zip’. It is in an unsupported format”, it says. This is either very clever or very stupid (is it using `file` to detect the file type?). But again, it’s no big problem to use the command-line `unzip` tool or The Unarchiver on a Mac instead. + +When it comes to creating Zip archives in the browser, I was pleased to find that, since around a year ago, all the major browsers (plus Node and Deno, [but not yet Bun](https://github.com/oven-sh/bun/issues/1723)) support the Compression Streams API. The [`CompressionStream` interface](https://developer.mozilla.org/en-US/docs/Web/API/CompressionStream/CompressionStream) to this can return either raw DEFLATE data, or ZLIB or GZIP data. The latter two include some metadata and a CRC. + +A little disappointingly, none of these formats is quite the same as you find in a Zip (originally [PKZIP](https://en.wikipedia.org/wiki/PKZIP)) archive. But the two key computationally-intensive components of each file inside a Zip archive — the compressed data plus the CRC — turn out to be identical to those used by GZIP. + +That means we can write a very small, very fast Zip library that works in all modern browsers (as well as Node and Deno). All it has to do is to rearrange the binary output of a GZIP `CompressionStream` alongside a few other bit of metadata, and — hey presto! — you can generate `.zip`, `.xlsx`, `.apk` and others in less than 2KB (zipped, of course) of code. + +### Open source + +I’ve released this Zip library under the name _littlezipper_: find it [on npm](https://www.npmjs.com/package/littlezipper) and [on GitHub](https://github.com/jawj/littlezipper). + +I’ve also released the Excel worksheet-writing library that depends on it, and that powers Neon’s Excel downloads: this one is called _xlsxtable_. Again, you’ll find it [on npm](https://www.npmjs.com/package/xlsxtable) and [on GitHub](https://github.com/jawj/xlsxtable). + +As ever, if you have any feedback on any of these new features, [please let us know on Discord](https://discord.com/invite/92vNTzKDGp). diff --git a/content/blog/posts/fan-out-postgres-changes-using-debezium-and-upstash-redis.md b/content/blog/posts/fan-out-postgres-changes-using-debezium-and-upstash-redis.md new file mode 100644 index 0000000000..c0704980e5 --- /dev/null +++ b/content/blog/posts/fan-out-postgres-changes-using-debezium-and-upstash-redis.md @@ -0,0 +1,331 @@ +--- +title: >- + Fan-out from Postgres with Change Data Capture using Debezium and Upstash + Redis +description: >- + Learn how to implement Change Data Capture with Neon's serverless Postgres, + Debezium, and Upstash Redis +excerpt: >- + Neon now has beta support for Postgres Logical Replication. This enables teams + to use Change Data Capture (CDC) to observe database changes – such as INSERT + and UPDATE operations – and stream these changes to downstream systems. We + previously wrote about the benefits of CDC and h... +date: '2024-02-14T23:32:41' +updatedOn: '2024-02-29T11:12:58' +category: postgres +categories: + - postgres + - community +authors: + - evan-shortiss +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/cover.jpg + alt: >- + Learn how to implement Change Data Capture with Neon's serverless Postgres, + Debezium, and Upstash Redis +isFeatured: false +seo: + title: >- + Fan-out from Postgres with Change Data Capture using Debezium and Upstash + Redis - Neon + description: >- + Learn how to implement Change Data Capture with Neon's serverless Postgres, + Debezium, and Upstash Redis + keywords: [] + noindex: false + ogTitle: >- + Fan-out from Postgres with Change Data Capture using Debezium and Upstash + Redis - Neon + ogDescription: >- + Learn how to implement Change Data Capture with Neon's serverless Postgres, + Debezium, and Upstash Redis + image: >- + https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/neon-fan-out-1-1024x576-6c53fc66.jpg) + +Neon now has beta support for Postgres Logical Replication. This enables teams to use [Change Data Capture (CDC)](https://en.wikipedia.org/wiki/Change_data_capture) to observe database changes – such as INSERT and UPDATE operations – and stream these changes to downstream systems. + +We previously wrote about the [benefits of CDC](https://neon.tech/blog/change-data-capture-with-serverless-postgres#why-cdc-matters) and how it enables [Event-Driven Architecture (EDA)](https://en.wikipedia.org/wiki/Event-driven_architecture). An EDA facilitates the implementation of messaging patterns, such as [fan-out](), with your Neon Postgres database at the heart of the system. + +Implementing a fan-out pattern enables you to create applications composed of loosely coupled components. Downstream consumers can work individually or as groups to asynchronously process database events to update other parts of your system in real-time. + +For example, a user sending a message in your application might necessitate sending a push notification to other users. If the message is written to the database and, in turn, a message broker, the broker can facilitate fan-out in a one-to-many fashion to downstream consumers, one of which is repsonsible for delivery to mobile devices. But how exactly do you stream changes from Postgres to a message broker, and why can’t the application layer simply handle this task? Keep reading to find out. + +![Streaming changes from Postgres to a message broker for fan-out delivery.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/71-1024x326-2c627e9b.jpg) + +This guide will show you how to use [Debezium Server](https://debezium.io/documentation/reference/stable/operations/debezium-server.html) with Neon’s serverless Postgres, and [Redis streams](https://redis.io/docs/data-types/streams/) provided by Upstash, to enable message fan-out with at-least-once delivery semantics. A repository with a sample Node.js consumer for the data produced by Debezium to Redis streams is available on [GitHub at evanshortiss/neon-debezium-redis-cdc](https://github.com/evanshortiss/neon-debezium-redis-cdc). + +## The Dual Write Problem + +Before we dive into the solution, let’s understand why it’s recommended to use a CDC platform like Debezium to stream changes from Postgres to a message broker. + +Imagine a scenario where you must insert or update a record in Postgres and notify downstream systems of the write operation in real-time. The delivery requirements for such a system are typically at-least-once, i.e., you need guaranteed delivery to the event consumers. A naive approach to this problem involves [dual writes](https://thorben-janssen.com/dual-writes/). + +A dual write occurs when you alter data in two systems without ensuring consistency. The following pseudocode provides a simple illustration of the dual-write problem: + +```javascript +async function insertAndPublishMessage (data) { + await postgres.query( + SQL` + INSERT INTO messages (from, to, content) + VALUES (${data.from},${data.to},${data.content}) + ` + ) + + await broker.publish('message.insert', data) +} +``` + +In a perfect world, this code persists data in Postgres, and once successful, it publishes an event to a message broker such as Redis streams, Apache Kafka, or Amazon Kinesis. This broker facilitates a fan-out pattern, as shown in the following architecture diagram. + +![An architecture that could suffer from inconsistent state due to dual-writes.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/69-1024x392-e813baaa.jpg) + +If the database transaction fails, the prior code will throw an error and not publish a message to the message broker. However, if the database transaction succeeds but the message broker is down, you could end up with an inconsistent state because the downstream consumers are unaware of the database change(s). Changing the order of operations doesn’t help since you might inform downstream consumers of a change that fails to be committed to the database. + +Maybe you’re thinking that Postgres’ nifty [LISTEN](https://www.postgresql.org/docs/16/sql-listen.html) and [NOTIFY](https://www.postgresql.org/docs/current/sql-notify.html) capabilities offer a solution to this problem; however, those provide an at-most-once delivery mechanism, meaning sent events will be lost if downstream listeners are briefly disconnected from Postgres. + +## Using Debezium to Avoid Dual Writes + +Debezium consumes changes from Postgres’ write-ahead log (WAL) via a [logical replication slot](https://www.postgresql.org/docs/current/protocol-replication.html#PROTOCOL-REPLICATION-CREATE-REPLICATION-SLOT) and streams these changes to messaging infrastructure in real-time. By consuming the WAL, only committed changes to the database are processed by Debezium and reliably forwarded to downstream message brokers, thus avoiding the dual write problem. This is illustrated in the following architectural diagram: + +![Architecture that uses Debezium to ensure each service only writes to one other location.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/70-1024x392-120e098c.jpg) + +Debezium tracks its progress through the WAL by storing its current WAL offset. This means you can safely restart Debezium, and it will resume streaming change events from where it last left off. If the downstream message broker is unavailable, Debezium will retry sending messages until the broker returns online. + +## Get Started with Neon and Logical Replication + +To start using Debezium with Neon’s serverless Postgres, [sign up for Neon](https://neon.tech/docs/get-started-with-neon/signing-up) and create a project. + +![Neon's create project UI.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/createfirstproject-1024x576-e8c8eccb.jpeg) + +Once you have a project, enable logical replication: + +1. Select your project in the Neon console. +2. On the Neon Dashboard, navigate to _Settings > Beta_. +3. Click the **Enable** button. +4. Confirm that you understand the changes and click **Enable** again. + +That’s it! A message will appear stating that logical replication is enabled for your project. + +![A Neon project with Logical Replication enabled.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/screenshot-2024-02-07-at-120434-1024x565-4bf19177.png) + +Before moving to the next section, create a table and add some data using the **SQL Editor** in the Neon console by running the following SQL statements: + +```sql +CREATE TABLE playing_with_neon(id SERIAL PRIMARY KEY, name TEXT NOT NULL, value REAL); + +INSERT INTO playing_with_neon (name, value) +VALUES + ('Mario', random()), + ('Peach', random()), + ('Bowser', random()), + ('Luigi', random()), + ('Yoshi', random()); +``` + +## Configure Postgres as a Debezium Data Source + +To begin, create a workspace folder in your development environment, and within it, create another folder to store your Debezium configuration. + +```bash +mkdir -p debezium-neon-redis/debezium +cd debezium-neon-redis +``` + +Debezium uses a [properties file](https://en.wikipedia.org/wiki/.properties) to store the necessary configuration. Create a file named `application.properties` inside the `debezium/` folder, and add the following configuration to define Postgres as a data source: + +```bash +# PostgreSQL source connector properties +debezium.source.database.hostname=${PGHOST} +debezium.source.database.port=${PGPORT:5432} +debezium.source.database.user=${PGUSER} +debezium.source.database.password=${PGPASSWORD} +debezium.source.database.dbname=${PGDATABASE} +debezium.source.database.server.name=tutorial +debezium.source.snapshot.mode=initial +debezium.source.plugin.name=pgoutput +debezium.source.connector.class=io.debezium.connector.postgresql.PostgresConnector +debezium.source.schema.whitelist=public +table.include.list=playing_with_neon +``` + +This configuration instructs Debezium to: + +- Connect to Postgres using the connection details defined in the `PG` environment variables. +- Accept payloads in `pgoutput` format. Neon also [supports wal2json](https://neon.tech/docs/guides/logical-replication-neon#decoder-plugins). +- Subscribe to changes in the `playing_with_neon` table. +- Perform a table snapshot to inform downstream consumers of the initial state. This is useful if you’re adding Debezium to an existing application that has tables that already contain data and you’d like to process the existing rows and not just new rows. + +Next, add the following lines to the `application.properties`. Comments are provided to explain what these configuration properties do: + +```bash +# Enable human readable logs +quarkus.log.console.json=${JSON_FMT_LOG:false} + +# Exclude the DB schema from event payloads to keep them lean +debezium.format.schemas.enable=false + +# Keep track of the WAL offsets to safely resume on restarts +debezium.source.offset.storage.file.filename=data/offsets.dat +debezium.source.offset.flush.interval.ms=0 +``` + +Nice work! You’ve enabled logical replication for a Neon serverless Postgres database and created a Debezium Server configuration to use it as a data source. + +## Configure Upstash Redis as a Debezium Data Sink + +When Debezium captures database changes, it needs to stream them to a destination known as a “sink”. You’ll be using a Redis instance provided by [Upstash](https://upstash.com/) as a sink; however, [Debezium supports various sinks](https://debezium.io/documentation/reference/stable/operations/debezium-server.html#_sink_configuration), meaning you can stream data to your preferred messaging infrastructure. + +Provision a Redis instance by visiting [console.upstash.com](https://console.upstash.com/), choosing Redis, and clicking the **Create Database** button. Enter the following parameters when creating your Redis instance: + +- Name: `neon-debezium` +- Type: `Regional` +- Region: Select the region closest to your Neon Postgres database. +- TLS (SSL) Enabled: `Yes` +- Eviction: `Yes` + +The form will resemble the following screenshot: + +![Creating a Redis instance on Upstash.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/screenshot-2024-02-09-at-095734-1024x561-649c0009.png) + +Add the following entries to the `application.properties` to define Redis as your data sink: + +```bash +debezium.sink.type=redis +database.user=${REDIS_USER} +debezium.sink.redis.address=${REDIS_ADDRESS} +debezium.sink.redis.password=${REDIS_PASSWORD} +debezium.sink.redis.wait.retry.enabled=true +debezium.sink.redis.ssl.enabled=true +``` + +This configuration will send change events to a Redis stream data structure in your Upstash Redis instance. In typical Redis fashion, all data is stored under a key. The key name will be composed of the “debezium” prefix followed by the database schema and table name, resulting in `debezium.public.playing_with_neon` in this particular example. + +Check the Redis documentation for a detailed description of all [supported configuration properties for the Redis sink](https://docs.redis.com/latest/rdi/installation/debezium-server-configuration/#redis-data-integration-configuration-reference). + +## Start a Debezium Server Container + +Your `application.properties` references environment variables. It’s time to create a `.env` file to store those variables to pass them into the Debezium Server container. You can also run Debezium on your host machine using Java and by [downloading a Debezium Server distribution](https://repo1.maven.org/maven2/io/debezium/debezium-server-dist/), but [Docker](https://www.docker.com/products/docker-desktop/) and [Podman](https://podman-desktop.io/) allow you to test different versions quickly without affecting your host environment. + +Visit the **Dashboard** for your project in the Neon Console, and select the _Parameters only_ option in the **Connection Details** pane. Copy the values displayed and paste them into a `.env` file in the `debezium-neon-redis` folder you created earlier, removing the surrounding single quotes. + +You’ll also need to define the Redis environment variables expected by the `application.properties`. These are displayed in your Redis instance’s **Details** section on the Upstash Console. + +Your `.env` file should resemble the following example: + +```bash +# file: .env + +# Do not wrap the variable values in quotes. Docker doesn't like that... +PGHOST=ep-adj-noun-a1b2c3.us-west-2.aws.neon.tech +PGDATABASE=neondb +PGUSER=yourusername +PGPASSWORD=yourpassword + +REDIS_ADDRESS=usw1-random-animal-12345.upstash.io:33324 +REDIS_USER=default +REDIS_PASSWORD=upstashredispassword +``` + +Before continuing, confirm your folder structure matches this sample: + +```bash +debezium-neon-redis +├── .env +├── debezium +│ └── application.properties +``` + +Now start a Debezium Server container using this command from within the `debezium-neon-redis` directory: + +```bash +docker run --rm \ +--name debezium-server \ +--env-file .env \ +-v $PWD/debezium:/debezium/conf \ +debezium/server:2.5.1.Final +``` + +Debezium Server will start and print a series of logs. The logs include information about the initial snapshot being performed and should end with a line stating that Debezium is `Searching for WAL resume position`. + +## Viewing Changes in Redis and Consuming them with Node.js + +Return to the Neon **SQL Editor** in your project and run the following `INSERT` statement: + +```sql +INSERT INTO playing_with_neon (name, value) VALUES ('Yoshi', random()); +``` + +Debezium Sever should log a message resembling `First LSN 'LSN\{0/2159960\}' received`. This confirms that Debezium is successfully consuming the WAL! + +To confirm data is being sent to Redis: + +1. Return to the [console.upstash.com](https://console.upstash.com/). +2. Select your Redis instance. +3. Navigate to the _Data Browser_ tab. +4. Set the filter to _All Types_ or _Stream_, and search for `debezium*`. + +You should see a key named `debezium.public.playing_with_neon`. This stream contains the 5 initial rows you inserted in your database with matching timestamps. The new row you inserted a moment ago will also be there. + +![Events in the Redis stream data structure, as seen in the Upstash Data Browser.](https://cdn.neonapi.io/public/images/pages/blog/fan-out-postgres-changes-using-debezium-and-upstash-redis/screenshot-2024-02-09-at-103118-1024x574-2ab4c023.png) + +You can now use a Redis client and the [XREADGROUP command](https://xreadgroup) to have consumer groups process messages in parallel as they arrive to the stream. Messages should be acknowledged using the [XACK command](https://redis.io/commands/xack/), since this enables you to restart consumers and have them resume processing where they last left off. + +A simple Node.js program to process messages as part of a consumer group would be similar to this, but with better error handling in a production codebase, of course. A complete example can be found in [this repository on GitHub](https://github.com/evanshortiss/neon-debezium-redis-cdc). + +```javascript +// file: stream-consumer.ts + +import * as redis from 'redis' + +const REDIS_STREAMS_KEY = 'debezium.public.playing_with_neon' +const REDIS_GROUP_ID = 'pwn-lr' + +// Remember to change this value for each replica of this +// program that you're running. It must be stable value +// across restarts! +const REDIS_CONSUMER_ID = 'consumer-0' + +async function main () { + const client = redis.createClient({ + name: 'lr-demo-client', + url: 'redis-stack:6379' + }) + + // The ID of '0' here makes sure the group consumes messages from + // the beginning of the stream. Use '$' if you want to ignore old + // messages that were created prior to the group's creation time + await client.xGroupCreate(REDIS_STREAMS_KEY, REDIS_GROUP_ID, '0'); + + readFromStream() + + async function readFromStream() { + const res = await client.xReadGroup( + REDIS_GROUP_ID, REDIS_CONSUMER_ID, + { key: REDIS_STREAMS_KEY, id: '>' }, + { BLOCK: 0, COUNT: 10 } + ) + + if (res) { + for (const { id, message } of res [0].messages) { + // Add message processing logic here... + + // ...then ack the message! + await client.xAck(REDIS_STREAMS_KEY, REDIS_GROUP_ID, id) + } + } + + // Immediately queue the next xReadGroup call + setImmediate(readFromStream) + } +} + +main() +``` + +## Conclusion + +Neon’s support for Postres’ Logical Replication enables development teams to leverage Event-Driven Architectures to create robust, real-time applications with Postgres at the core. Change Data Capture platforms and technologies like Debezium provide low-code open-source solutions to reliably stream changes from Postgres to messaging systems such as Redis for further processing and analysis. If you’re looking for a Postgres database, [sign up and try Neon](https://neon.tech/blog/python-django-and-neons-serverless-postgres#:~:text=sign%20up%20and%20try%20Neon) for free. Join us in our [Discord server](https://neon.tech/discord) to share your experiences, suggestions, and challenges. diff --git a/content/blog/posts/first-azure-region-available-in-neon.md b/content/blog/posts/first-azure-region-available-in-neon.md new file mode 100644 index 0000000000..85d3c8cf21 --- /dev/null +++ b/content/blog/posts/first-azure-region-available-in-neon.md @@ -0,0 +1,62 @@ +--- +title: First Azure Region Available in Neon +description: You can now create Neon projects (in Beta) in Azure East US 2 +excerpt: >- + Starting today, you can deploy Neon databases on Azure, starting with the East + US 2 region. This marks the first milestone on our Azure roadmap—many more + exciting updates are on the way, including deeper integrations with the Azure + ecosystem. To create your first Neon project on... +date: '2024-10-07T18:02:14' +updatedOn: '2024-12-03T21:29:36' +category: company +categories: + - company +authors: + - bryan-clark +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/first-azure-region-available-in-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: First Azure Region Available on Neon (Beta) + description: You can now create Neon projects in Azure East US 2 + keywords: [] + noindex: false + ogTitle: First Azure Region Available on Neon (Beta) + ogDescription: You can now create Neon projects in Azure East US 2 + image: >- + https://cdn.neonapi.io/public/images/pages/blog/first-azure-region-available-in-neon/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/first-azure-region-available-in-neon/neon-azure-1-1024x576-4304f6d7.jpg) + + +There are now more Azure regions available in the Neon console. Plus, Neon is also available in the Azure Marketplace as an Azure Native Integration. Read more [here](https://neon.tech/blog/neon-is-now-available-as-an-azure-native-integration). + + +**Starting today, you can deploy Neon databases on Azure, starting with the East US 2 region. This marks the first milestone on our [Azure roadmap](https://neon.tech/blog/neon-is-coming-to-azure)—many more exciting updates are on the way, including deeper integrations with the Azure ecosystem.** + +To create your first Neon project on Azure, [click here](https://console.neon.tech/app/projects/new?provider=azure) or navigate to the console and select “Azure” as your cloud provider. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/first-azure-region-available-in-neon/ad4nxdpcnkau2s5g3glhfzioo4obfcs4sxkwy87a1vuxl9df7qy2zorcsmu2ctalcdqoyiwv7fcpfqr9pp0l-a2jlhba-32wpctfy6gzmgcuumhtffs6ggek0isyr2e50waes6g9et7cciqophiggyu7ylso-6915468b.png) + +We’ll be adding more Azure regions soon—let us know what regions you’d like to see next on our [regions page.](https://neon.tech/docs/introduction/regions#request-a-region) + +## How pricing works during the Beta + +We’re launching this first Azure region with a beta label, which we expect to keep for the next few weeks. Please be cautious about using this region for mission-critical workloads during this period—we recommend using it for testing and prototyping, as our regular [SLA](https://neon.tech/neon-business-sla) does not apply during the beta phase. + +The good news: You can test confidently during the beta without worrying about extra costs. While the beta label is in place, users on our Launch, Scale, and Business plans **won’t be billed for consuming additional resources beyond their plan’s usage limits** (you can exceed your plan’s resources without incurring extra charges). + +We would love to hear your feedback on how having Neon on Azure improves your workflows and what additional features you would like to see next. Your input will play a key role in shaping the future of Neon on Azure as we work toward even deeper connections and more seamless experiences. + +Speaking of which… + +## Big news coming soon – Join the Preview + + +This is now available! You can deploy Neon organizations directly from the Azure Portal. Get started [here](https://azuremarketplace.microsoft.com/en-us/marketplace/apps/neon1722366567200.neon_serverless_postgres_azure_prod?tab=overview). + + +This beta phase marks the beginning of an exciting journey in our deepening partnership with Microsoft and Azure. We can’t reveal details just yet, but we’re working on a tighter integration of **Neon within the Azure Marketplace**. diff --git a/content/blog/posts/fixing-javascript-timezone-issues.md b/content/blog/posts/fixing-javascript-timezone-issues.md new file mode 100644 index 0000000000..d9b09641a5 --- /dev/null +++ b/content/blog/posts/fixing-javascript-timezone-issues.md @@ -0,0 +1,80 @@ +--- +title: Fixing JavaScript Timezone Issues +description: How we resolved a common timezone problem in our parking app +excerpt: >- + Working with timezones in JavaScript often feels like navigating a minefield. + Recently, I faced a timezone issue when building a parking booking app while + handling date storage in our database, something you may also encounter if + you’re working across timezones. Here’s what went... +date: '2025-01-31T17:02:39' +updatedOn: '2025-01-31T17:02:41' +category: community +categories: + - community +authors: + - deepesh-genani +isFeatured: false +seo: + title: Fixing JavaScript Timezone Issues - Neon + description: >- + A blog post written by a Neon user as part of Neon's Community series, + sharing how they fixed a timezone issue when building a paking app. + keywords: [] + noindex: false + ogTitle: Fixing JavaScript Timezone Issues - Neon + ogDescription: >- + A blog post written by a Neon user as part of Neon's Community series, + sharing how they fixed a timezone issue when building a paking app. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/fixing-javascript-timezone-issues/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/fixing-javascript-timezone-issues/neon-timezone-1-1024x576-626b99ad.jpg) + + +This blog post is written by a Neon user as part of our Community series. Many thanks to [Deepesh Genari](https://deepeshgenani.vercel.app/) for sharing this writeup! If you use Neon and want to share your own tips with the community, [reach out to us on Discord.](https://discord.gg/92vNTzKDGp) + + +Working with timezones in JavaScript often feels like navigating a minefield. Recently, I faced a timezone issue when building a parking booking app while handling date storage in our database, something you may also encounter if you’re working across timezones. Here’s what went wrong, how we fixed it, and how Neon simplified the process. + +## The problem: Timezone conversions adding unwanted metadata + +Our app allows users to book parking slots for a selected date and time. However, **users in Italy were booking slots for 12:00 AM, but our database (hosted on Neon in the US) stored it as 6:00 PM of the previous day.** + +Here’s what caused the issue: + +```javascript +const date = new Date(selectedDate); +``` + +When creating a `Date` object in JavaScript, it automatically includes the user’s local timezone. This value is then sent to the API and stored in the database, resulting in unwanted timezone conversions. + +For example: + +- User’s input (Italy): `2024-01-01 12:00 AM` +- Database storage (US): `2023-12-31 6:00 PM` + +The root cause wasn’t just a timezone conversion problem; **this issue was caused by JavaScript automatically adding timezone metadata to the date.** This led to storing a different date and time in the database than what users selected. + +To fix this, we needed to store the exact selected time without any timezone information. + +## How we fixed it + +To avoid automatic timezone handling and metadata issues, we created a plain date-time string that doesn’t include any timezone information. Here’s how: + +```javascript +const dateTimeString = `${year}-${month}-${day} ${time}`; +// Example: "2024-01-01 00:00:00" +``` + +
By sending this plain string to the API and storing it directly in the database, we ensured the database received the exact date and time input by the user, without any unintended timezone conversions or metadata. + +## Why we love using Neon + +1. **Easy connectivity**: Neon integrates seamlessly with our applications. It’s simple to set up and connect to our projects. +2. **Free Plan for personal projects**: The [free plan](https://console.neon.tech/signup) is perfect for starting personal projects, like the parking booking app discussed in this post. +3. **Scalability and reliability**: Despite its ease of use and free plan, we know Neon is a robust Postgres database that will handle scaling and performance when we need it. + +--- + +_You can reach out to the author via X [@deepesh_genani](https://x.com/deepesh_genani). His projects: [https://deepeshgenani.vercel.app/projects](https://deepeshgenani.vercel.app/projects)_ diff --git a/content/blog/posts/frictionless-development-experience-with-neon-branching.md b/content/blog/posts/frictionless-development-experience-with-neon-branching.md new file mode 100644 index 0000000000..59706b583d --- /dev/null +++ b/content/blog/posts/frictionless-development-experience-with-neon-branching.md @@ -0,0 +1,94 @@ +--- +title: How Proposales integrated Neon in their Postgres development workflow +description: "Smoothly migrating dev infra from RDS to Neon, reducing costs in the process." +excerpt: >- + Switching databases is like a heart transplant, and migrating all your + database’s production, staging, and development environments across providers + is a risky process that requires significant effort and expertise, all of + which can be quite intimidating. But you don’t have to ch... +date: "2022-12-08T19:35:25" +updatedOn: "2024-02-02T16:03:00" +category: case-study +categories: + - case-study +authors: + - raouf-chebri +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/frictionless-development-experience-with-neon-branching/social.jpg + alt: null +isFeatured: false +seo: + title: How Proposales integrated Neon in their Postgres development workflow - Neon + description: >- + Smoothly migrating dev infra from RDS to Neon, reducing costs in the + process. + keywords: [] + noindex: false + ogTitle: How Proposales integrated Neon in their Postgres development workflow - Neon + ogDescription: >- + Switching databases is like a heart transplant, and migrating all your + database’s production, staging, and development environments across + providers is a risky process that requires significant effort and expertise, + all of which can be quite intimidating. But you don’t have to choose between + developer experience and stability for your code base. In this article, we + […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/frictionless-development-experience-with-neon-branching/social.jpg +--- + +Switching databases is like a heart transplant, and migrating all your database’s production, staging, and development environments across providers is a risky process that requires significant effort and expertise, all of which can be quite intimidating. + +But you don’t have to choose between developer experience and stability for your code base. In this article, we will look at how one of our database-branching early adopters, [Proposales](https://proposales.com/), integrated Neon into their development environment. + +Proposales offers hoteliers tools to build data-driven proposals effortlessly. They have a team of versatile full-stack engineers and use AWS RDS for PostgreSQL. + +Proposales used RDS for their development environment, but it was an expensive choice. Another issue was that sharing a database in their development environment had a couple of drawbacks: + +1. Schema conflicts arose when multiple developers changed the database, leading to errors and data loss. +2. It was difficult for developers to properly test new features and changes without impacting teammates’ work. + +Overall, using a shared database for development hindered collaboration, testing, and security. + +An alternative that Proposales explored was using Postgres with Docker for local development. Docker allows Proposales to create and manage isolated environments for their applications, making it simple to set up and configure a local development environment. + +However, the problem with using Docker is that it increases resource requirements, making it more difficult to run a development environment on a low-powered or resource-constrained machine. + +## Adding Neon to their development workflow + +Proposales found out about Neon and gradually integrated it into their workflows. They decided to keep RDS on production but use Neon and its database branching capabilities for development, replacing Postgres with Docker and RDS. + +Neon fit well with Proposales’ workflow because they got the best of managed and local Postgres setup worlds. This shared cloud-based database can be branched into isolated environments for local development. + +In addition, with database branching, they create a copy of their production-like database for testing features with production data without affecting their production environment. + +In our recent conversation with Camelia Smeria, Lead Engineer at Proposales, she shared that her team uses Neon database branching for their latest feature, _Insight,_ which helps users better understand their proposal workflows. Insight requires production data for testing. Here is what Camelia had to say about their local setup: + +_“We used to have a snapshot of the production data that we used for development… it turned out to be too much development effort and not worth it … everybody ended up using the localhost database._ + +_With the Neon branching feature, our developers can change their database to test their features without affecting other team members. Branching allows us to confidently do our database migrations to production.”_ + +You can watch the full conversation with Camelia Smeria: + + + +Other areas Proposales are exploring include database branching to their end-to-end testing with the Neon API to create a new branch and test their Vercel previews. + +## Even deeper integration with your development workflow + +We’ve seen above how developers are using Neon in development workflows. We believe that database branching opens up many other possibilities for Neon to be a fully integrated development tool. + +One workflow we are experimenting with introduces Git Hooks to manage Neon branches corresponding to local git branches. + +In the video below, we create a git branch `feat-1` using the command `git checkout -b feat-1`. This triggers the creation of a database branch on Neon, which returns a `DATABASE_URL` variable that is added to the `.env` file. + +
+ +
+ +To summarize, you don’t have to migrate your entire database environment to start with Neon and database branching. You can start by moving your current shared or local Postgres database to Neon and use branching to collaborate with teammates and test your features with production data. + +We also showed an example of branch creation with Git Hooks and the Neon API as another way to integrate Neon into your development workflows.
We believe there are many other workflows out there that could benefit from the database branching. Let us know about the challenges you are facing in your workflows in the [Neon community forum](https://community.neon.tech/) and how Neon’s database branching feature could help. diff --git a/content/blog/posts/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle.md b/content/blog/posts/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle.md new file mode 100644 index 0000000000..48e2d57481 --- /dev/null +++ b/content/blog/posts/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle.md @@ -0,0 +1,128 @@ +--- +title: 'From days to minutes: how Neo.Tax accelerated their development lifecycle' +description: The team is much more agile now that they can use database branches +excerpt: >- + “Database branching is the best quality-of-life improvement to my tech stack + that I can think of in recent years. Second to maybe only Copilot” Miguel + Hernandez, Backend Tech Lead at Neo.Tax Neo.Tax specializes in automating the + calculation of tax credits for startups, enterprise... +date: '2024-06-03T16:40:27' +updatedOn: '2024-06-03T16:40:29' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + From days to minutes: how Neo.Tax accelerated their development lifecycle - + Neon + description: >- + By adopting database branches in Neon, they could speed up their end-to-end + testing process and start fixing bugs quickly. + keywords: [] + noindex: false + ogTitle: >- + From days to minutes: how Neo.Tax accelerated their development lifecycle - + Neon + ogDescription: >- + By adopting database branches in Neon, they could speed up their end-to-end + testing process and start fixing bugs quickly. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle/neon-neotax-1-1-1024x576-cee0d102.jpg) + +
+

“Database branching is the best quality-of-life improvement to my tech stack that I can think of in recent years. Second to maybe only Copilot”

+Miguel Hernandez, Backend Tech Lead at Neo.Tax +
+ +[Neo.Tax](https://www.neo.tax/) specializes in automating the calculation of tax credits for startups, enterprises, and mid-market companies through the use of AI and machine learning. With a team armed with previous experience at Intuit and the IRS, Neo.Tax is inspired to make it simple for businesses to uncover (and claim!) valuable tax advantages, maximizing potential savings with 100% accuracy and security. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle/9p2vksgvwnhnoylqoztuuxxg3ukspwggfpwhv25k7wbrcmuh6qeq0l-1ppxxlj8tg440bg9fj-rfablhebxwu8rpwwicza0xgptaoitbn38aeugl3jw9ukfp4v03o0owmnwalm2iqjzzabbelj0fm-db69dca2.png) + +## E2E testing with accurate data is hard + +
+

“Our testing process was very manual before. Product would create a test customer in our development environment, then generate PDFs; the QA team would test and manually run through all the math; then an engineer would have to go into the database, look at all the values, and handwrite them into fixtures for our end-to-end tests… That’s multiple days for every single change”

+Miguel Hernandez, Backend Tech Lead at Neo.Tax +
+ +One of the awesome things about the Neo.Tax platform is that it’s able to aggregate and analyze across many different sources: ticketing data, logs, GitHub pull requests, accounting expenses, payroll information… This is a manual process for most businesses that Neo.Tax automates. From this data, they calculate the most beneficial tax claims for every business. + +To ensure the accuracy of their calculations, the team put a QA process in place that involves generating accurate previews of tax documents to validate any changes and new features. But pulling this off in a regular database is a laborious process, manual and time-consuming. Before Neon, this process looked something like this: + +1. When a new feature or change needed to be added to the platform, the Product team would create a customer in the development environment, generate PDFs for testing, and then manually verify all calculations. +2. Then, engineers then had to transfer these verified values into fixtures for end-to-end tests. This involved performing SQL dumps and pushes to replicate the data produced by the Product team in a development enviroment. + +## How database branching helps + +
+

“Neon shortened the lifecycle for us between making a change in the product, validating it, and generating the PDFs we expect. Before, it used to be terrible: resetting your database, running migrations, all of that. With Neon, we just create a database branch, link it with the ticket, and use that URL in local development. This has significantly streamlined our end-to-end testing process”

+Miguel Hernandez, Backend Tech Lead at Neo.tax +
+ +Now, Neo.Tax uses Neon to streamline this process. The big difference is this: with Neon, the team can create accurate [deploy previews](https://neon.tech/flow) by leveraging [database branching](https://neon.tech/docs/introduction/branching). Here’s how the new process works: + +1. When a PR is open, Neo.Tax uses the [Neon CLI](https://neon.tech/docs/reference/neon-cli) to create a new branch from the development database. The branch name is associated with the GitHub branch name for consistency. +2. The new branch URL is injected into their Cloud Run instances via environment variables in CircleCI. +3. Product and QA teams can now perform manual and automatic tests using this isolated database branch. They can generate PDFs, compare data, and ensure all calculations are accurate. +4. When the PR is merged, the associated database branch is deleted. + +## Using Neon branches to reproduce errors locally + +
+

“Whenever we have an issue that we can’t solve, or need to pair with someone else, you can just send your URL for your Neon database, and they immediately have a reproducible problem”

+Miguel Hernandez, Backend Tech Lead at Neo.tax +
+ +The Neo.Tax team also takes advantage of another great application of database branches: fixing bugs. Instead of following the traditional process of creating database dumps and moving data to local machines, they now follow this process: + +1. When a bug is identified, a branch is created from the production database. This branch is associated with a specific issue or ticket. +2. The branch URL is shared among developers, allowing them to work with the exact same data. This setup ensures that all developers have a consistent environment, making it easier to replicate and troubleshoot issues. +3. Developers work on the branch to fix the issue. Once the fix is implemented, it is validated in the same consistent environment. +4. If data becomes corrupted or needs to be reset for any reason, developers can simply reset the branch from the parent development branch. This takes one second. +5. Once the issue is fixed and the ticket is closed, the database branch is deleted. + +This method is not only more convenient but also more secure, as it avoids the risks associated with handling sensitive data on local devices and ensures compliance with data security policies. It also allows Neo.Tax to avoid the complexity of resyncing data and mitigate variations that can be caused by LLMs, which can introduce inconsistencies in data processing. + +## A note on Neon design: environments, branches, projects + +Especially for those users already familiar with Neon [and its object hierarchy,](https://neon.tech/docs/manage/overview) you might be curious to know how Neo.Tax structured their Neon deployment. Here’s an overview: + +**Environments** + +Neo.Tax uses three primary environments, each one with its own separate [project](https://neon.tech/docs/manage/overview#projects) in Neon: development, staging, and production. Each of these projects has a main branch that serves as the base for creating other branches. + +- `development` isused for day-to-day development tasks. +- `staging` acts as a middle ground where features and fixes from development are tested before being released to production. +- `production` is the live environment where the actual application runs. + +**Branches** + +Within each project, Neo.Tax creates branches for individual tasks, including: + +- Feature development: developers branch off the main development branch to work on new features or bug fixes. +- Bug fixing: for bugs identified in production, branches are created from the production project and named after the associated ticket or issue. +- End-to-end testing: branches are created to generate deploy previews with pre-built accounts to ensure that calculations and document generation are accurate before merging changes into the main branch. +- Reproducible environments: when issues arise that require collaboration, developers can create branches from the main branch and share the branch URL. This allows other team members to work with the exact same data. + +**Single / multi-tenancy** + +In reality, Neo.Tax handles more than one production project: + +- For most users, Neo.Tax uses a shared database with schema-based isolation. Authentication and authorization ensure data isolation within the same database. +- For their enterprise customers, who oftentimes need a dedicated tenant, Neo.Tax creates a separate project for that customer. + +## Less is more + +Database management and taxes have one thing in common: the less you have to do them, the better. + +If you’re also struggling with your database workflows, [try out Neon](https://console.neon.tech/realms/prod-realm/protocol/openid-connect/registrations?client_id=neon-console&redirect_uri=https%3A%2F%2Fconsole.neon.tech%2Fauth%2Fkeycloak%2Fcallback&response_type=code&scope=openid+profile+email&state=9r-s37V5ewTKwMpF_bbqBQ%3D%3D%2C%2C%2C) and [experiment with database branches](https://neon.tech/flow). And if your startup could get some help with taxes (_and who doesn’t?_), [sign up for Neo.Tax for free](https://app.neo.tax/signup?_gl=1*r4t96k*_gcl_au*MTA1OTkxMDQ2Ny4xNzE2MjI4MTMy) and explore the platform. diff --git a/content/blog/posts/from-idea-to-full-stack-app-in-one-conversation-with-create.md b/content/blog/posts/from-idea-to-full-stack-app-in-one-conversation-with-create.md new file mode 100644 index 0000000000..4b2dcc8a5e --- /dev/null +++ b/content/blog/posts/from-idea-to-full-stack-app-in-one-conversation-with-create.md @@ -0,0 +1,110 @@ +--- +title: From Idea to Full Stack App in One Conversation with Create +description: >- + This new AI Agent knows how to deploy 50+ APIs (including OpenAI and + Anthropic)—and now Postgres databases +excerpt: >- + These are wild times. It’s now possible to describe a fancy app in a single + text prompt, e.g., “Make an AI recipe generator with ChatGPT that lets me + enter a food, generate recipes and save my favorites to a gallery for later” + And watch a complete app materialize in seconds: If y... +date: '2025-02-12T01:21:10' +updatedOn: '2025-08-07T17:36:51' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-idea-to-full-stack-app-in-one-conversation-with-create/cover.jpg + alt: null +isFeatured: false +seo: + title: From Idea to Full Stack App in One Conversation with Create - Neon + description: >- + Create.xyz is an AI Agent that knows how to deploy 50+ APIs, including + OpenAI, Anthropic, and Postgres databases. Build in one conversation. + keywords: [] + noindex: false + ogTitle: From Idea to Full Stack App in One Conversation with Create - Neon + ogDescription: >- + Create.xyz is an AI Agent that knows how to deploy 50+ APIs, including + OpenAI, Anthropic, and Postgres databases. Build in one conversation. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-idea-to-full-stack-app-in-one-conversation-with-create/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-idea-to-full-stack-app-in-one-conversation-with-create/neon-createxyz-1024x576-627ceec5.jpg) + + +Create.xyz is now Anything. Their new home: [https://www.createanything.com/](https://www.createanything.com/ ) + + +These are wild times. It’s now possible to describe a fancy app in a single text prompt, e.g., + +“_Make an AI recipe generator with ChatGPT that lets me enter a food, generate recipes and save my favorites to a gallery for later”_ + +And watch a complete app materialize in seconds: + + + +If you’ve been playing with [Replit](https://replit.com), [Bolt](https://bolt.new) or [Lovable](https://lovable.dev), [Create](https://create.xyz/) is next on your list. + +## From Text to Full Stack, Fast + +Create makes building production-ready apps accessible to everyone. There’s two core elements that make users love it: + +- End-to-end +- Speed + +### Speed + +Rather than make users wait, Create’s agent knows how to make targeted updates to the codebase. This is especially important for people building more sophisticated applications that require many iterations. The AI workflow that produces truly useful apps is simple: see your app, test it, and refine it on the fly—but this must happen without delays. + +### End-to-end + +When you build with agents, the apps are often _almost usable UI prototypes but not quite real_—they’re missing something essential and it’s hard to make them go end to end. Create aims to change this. When you give Create an instruction, it turns it into the right pages, components, backend functions, and database setup. You see them appear visually in real-time. + +Create can also automatically add 50+ built-in integrations, and makes it easy to add your own integrations with its backend functions. The entire workflow of adding and managing integrations is abstracted away—just describe what you need, and the agent handles the setup. Here’s a shortlist of what Create can add to your app: + +- **AI & LLMs.** Use OpenAI, Anthropic, Gemini, Llama, and other foundational models to add AI features like content generation, chatbots, or classification. +- **Image generation.** Call DALL·E, Stable Diffusion, and other models. +- **Payments.** Ask Create to add Stripe to start collecting payments for your product right away. + +Since Create is generating code for your app under the hood, you have control over the UI down to the pixel and exact logic. And when you’re ready to go live, Create deploys it on developer-grade infra. + +## Enter Neon: Powering the Database Behind the Scenes + +
+

“Neon’s speed of provisioning and serverless scale-to-zero is critical for us. We can serve users iterating on quick ideas efficiently while also supporting them as they scale, without making them think about database setup.” (Dhruv Amin, Co-founder at Create.xyz)

+
+ +Among all this goodness, **Create bakes-in a Postgres database from the start—and that’s where Neon comes in.** [Neon](https://neon.tech/home) is a serverless Postgres database built for fast, iterative workflows, making it a perfect fit for AI agents: + +- **Provisioning happens almost instantly in Neon.** AI agents can spin up fully functional Postgres database in less than a second. +- **Neon autoscales infrastructure, starting from zero.** Neon is serverless, with a pay-as-you-go model. It automatically scales compute and storage as the app demands it, which is critical for agents: this pricing model ensures very low costs when databases are idle (for the AI company) and good scalability (for the end-user) as usage grows. + +With Neon, Postgres management becomes just another API call. No manual provisioning, no downtime, no headaches—just instant, scalable Postgres that works behind the scenes. + +But Neon has another big advantage for agentic workflows: it supports [branches](https://neon.tech/flow), aka isolated copies of your data and schema that are provisioned instantaneously. **Neon’s branching model allows Create to solve a big pain when building AI agents: how to handle schema migrations.** + +
+

“When users want to iterate on their app, we can branch off their data with Neon and test schema migrations in isolation. We can even auto-fix issues. Neon’s branching model is a huge win for iteration speed” (Dhruv Amin, Co-founder at Create.xyz)

+
+ +Without a structured approach, schema migrations can introduce downtime, which hurts the user experience especially when updates are meant to happen dynamically in response to user prompts. Neon’s branching allows Create to apply an iterative workflow for schema migrations: + +1. **User proposes schema changes.** When a user prompts Create with “Add payments to the app” or “Let users save a twitter link to their profile,” the system automatically generates the necessary database migration and applies it to an isolated Neon branch, which acts as a separate environment where the changes can be tested first. +2. **Changes are tested.** This separate environment is ready immediately, and it’s a perfect copy of the main database (schema + data). The user and the agent can validate migrations in parallel in this environment, without affecting the original version of the app. +3. **Changes are applied.** Once the migration is verified, the agent can apply the changes back into production. + +## Start Building + +You can [sign up to Create’s Free Plan](https://www.create.xyz/login) and start exploring the agent immediately. Share what you’ve built by mentioning them on X: [https://x.com/create_xyz](https://x.com/create_xyz) + +--- + +_If you’re a company building an AI agent,_ [let’s chat.](https://neon.tech/contact-sales) _We have a Postgres API for you._ diff --git a/content/blog/posts/from-shared-chaos-to-isolated-control-with-neon.md b/content/blog/posts/from-shared-chaos-to-isolated-control-with-neon.md new file mode 100644 index 0000000000..d9831cd360 --- /dev/null +++ b/content/blog/posts/from-shared-chaos-to-isolated-control-with-neon.md @@ -0,0 +1,117 @@ +--- +title: From Shared Chaos to Isolated Control with Neon +description: 'A better way to build: spin up a Neon Twin and keep production untouched.' +excerpt: >- + Your traditional database is rock-solid for handling production + workloads—reliable, resilient, and built to withstand just about anything. But + when it comes to the developer experience? Not so much. If you’ve ever had + multiple developers colliding on the same dev or test instance... +date: '2025-03-11T14:30:00' +updatedOn: '2025-03-20T08:57:10' +category: workflows +categories: + - workflows +authors: + - paul-scanlon +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-shared-chaos-to-isolated-control-with-neon/cover.jpg + alt: 'A better way to build: spin up a Neon Twin and keep production untouched.' +isFeatured: true +seo: + title: From Shared Chaos to Isolated Control with Neon - Neon + description: 'A better way to build: spin up a Neon Twin and keep production untouched.' + keywords: [] + noindex: false + ogTitle: From Shared Chaos to Isolated Control with Neon - Neon + ogDescription: >- + Your traditional database is rock-solid for handling production + workloads—reliable, resilient, and built to withstand just about anything. + But when it comes to the developer experience? Not so much. If you’ve ever + had multiple developers colliding on the same dev or test instance, + overwriting each other’s schema changes, accidentally siphoning resources + from production, or dealing with […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-shared-chaos-to-isolated-control-with-neon/social.jpg +--- + +![A better way to build: spin up a Neon Twin and keep production untouched.](https://cdn.neonapi.io/public/images/pages/blog/from-shared-chaos-to-isolated-control-with-neon/neon-shared-chaos-cover-image-1024x576-dbdb13d0.jpg) + +Your traditional database is rock-solid for handling production workloads—reliable, resilient, and built to withstand just about anything. But when it comes to the developer experience? Not so much. If you’ve ever had multiple developers colliding on the same dev or test instance, overwriting each other’s schema changes, accidentally siphoning resources from production, or dealing with over-provisioned environments that sit idle, you know exactly what I’m talking about. + +You might think, _“Why not migrate to a better provider?”_ But let’s be honest—just suggesting a production migration is enough to trigger endless meetings, risk assessments, and so much red tape it barely feels worth it, and there’s a good chance, you’ll end up right back where you started anyway. + +So don’t migrate. Leave production where it is. Instead, supercharge your development and testing environments with Neon, which is built to handle any size, from small projects to massive datasets. + +In this post, I’ll describe some of our best features, walk you through our [Dev/Test workflow](https://neon.tech/docs/use-cases/dev-test), and introduce the process of creating a [Neon Twin](https://neon.tech/docs/guides/neon-twin-intro). + +## What are Neon’s best features? + +Neon is designed to make developer workflows simpler, faster, and safer. Traditional databases come with well-known pain points—shared environments, risky schema changes, and constant resource management. Neon solves these challenges with features built specifically for modern development teams, including: + +1. **Serverless Architecture with Autoscaling**: Neon automatically scales resources to match workload demand, with no manual intervention required. +2. **Branching for Development and Testing:** Instantly create isolated copies of your data and schema—or schema-only. No matter the size of your database, branching is always instant and seamless. +3. **Integration and Compatibility**: Neon supports all the latest Postgres versions and numerous extensions, ensuring compatibility with a wide range of applications and frameworks. +4. **Provisioning Environments**: Neon simplifies the provisioning and management of environments through its console or API. New branches can be spun up instantly, resources can be managed, and settings can be configured—either through a user-friendly console or automated using API calls. + +## See Neon in Action: Customer Case Studies + +- [Adopting Neon branching in CI/CD pipelines: a practical story by Shepherd](https://neon.tech/blog/adopting-neon-branching-in-ci-cd-pipelines-a-practical-story-by-shepherd) +- [Why Invenco Migrated From Aurora Serverless v2 to Neon](https://neon.tech/blog/why-invenco-migrated-from-aurora-serverless-v2-to-neon) +- [How Dispatch speeds up development with Neon while keeping workloads on Aurora](https://neon.tech/blog/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora) +- [How Mindvalley Minimizes Time-To-Launch With Neon Branches](https://neon.tech/blog/how-mindvalley-minimizes-time-to-launch-with-neon-branches) + +## What is the Dev/Test Workflow? + +Our [Dev/Test](https://neon.tech/docs/use-cases/dev-test) workflow is simple: production stays put, while development and testing move to a [Neon Twin](https://neon.tech/docs/guides/neon-twin-intro). In this workflow, your dev and test environments are clones of your existing production or staging databases, kept in sync through two key processes: + +1. Nightly automated dump and restore to keep your data up to date. +2. Dump and restore triggered by GitHub events to prevent schema drift. + +[Branches](https://neon.tech/docs/introduction/branching) of the Twin can be created instantly, giving each member of your team a safe space to build, test, and iterate with real production data and schema without impacting the stability or performance of your production environment—or each other! + +![Neon Twin workflow displaying a traditional database setup with the development and testing environments shutdown.](https://cdn.neonapi.io/public/images/pages/blog/from-shared-chaos-to-isolated-control-with-neon/twin-workflow-1024x316-5bdce811.jpg) + +## What is a Neon Twin? + +A Neon Twin is a full or partial copy of your production or staging database, giving developers isolated, sandboxed environments that closely mirror production + +With a Neon Twin, teams can streamline their workflows, move faster, and stay productive—while avoiding the complexity and costs of traditional dev and test environments. + +We currently have two Twin workflows available in our [docs](https://neon.tech/docs/guides/neon-twin-intro), which include: + +- **Full Twin**: [A Full Twin](https://neon.tech/docs/guides/neon-twin-full-pg-dump-restore) is an exact copy of your production or staging database, including all data and schema. +- **Partial Twin**: [A Partial Twin](https://neon.tech/docs/guides/neon-twin-partial-pg-dump-restore) is an exact copy of your production or staging databases schema but only includes a subset of dat + +GitHub Actions power both workflows and can be added to any existing GitHub repository. You can configure them to run on a recurring schedule that fits your needs or trigger them with specific [GitHub events](https://neon.tech/docs/guides/neon-twin-partial-pg-dump-restore#handling-pull-request-events)—such as when a pull request is closed and merged—to keep schema changes in sync. Plus, both workflows support [concurrency controls](https://neon.tech/docs/guides/neon-twin-partial-pg-dump-restore#add-concurrency-and-conditions) to prevent overlapping runs. + +
+

We are using the Neon Twin workflow. We just install the GitHub action and it takes care of the rest. Developers may not know how to dump and restore well, but they know how to run a GitHub Action. It’s amazing (Alex Co, Head of Platform Engineering at Mindvalley)

+
+ +We’re also exploring additional workflows, including: + +- **Anonymized Twin**: Automatically anonymizes sensitive PII data for safer testing. +- **Synthetic Twin**: Automatically generates realistic synthetic data based on your schema. + +### GitHub Actions limitations + +There are a few limitations to consider when using GitHub Actions. By default, they run on shared infrastructure, meaning you can’t control the execution region or use a fixed IP address. This can be an issue if your production or staging environments require IP allowlisting for access. + +Second, GitHub Actions have a default timeout of six hours. For larger databases that may take longer to complete a full dump and restore, GitHub recommends using a self-hosted runner. + +## Self-hosted runners + +A self-hosted runner can help solve both of these limitations by giving you control over the environment, region, IP address and execution time. I’ve published a guide that walks through how to build and configure your own self-hosted runner: [How to use self-hosted runners with GitHub Actions](https://neon.tech/guides/gihub-actions-self-hosted-runners). + +## Team notifications + +For larger teams, it’s essential to keep developers informed when a new Twin is available so they can [reset their own branches](https://neon.tech/docs/guides/reset-from-parent) to maintain consistency. To help with this, we’ve put together a guide on setting up Slack notifications to keep everyone in the loop: [Building Slack notifications to monitor pg_dump and restore workflows](https://neon.tech/blog/building-slack-notifications-to-monitor-pg_dump-and-restore-workflows). + +![Mockup of Slack interface displyang a message in the engineering-general channel explaining a new Neon Twin is available](https://cdn.neonapi.io/public/images/pages/blog/from-shared-chaos-to-isolated-control-with-neon/twin-slack-notifications-1024x641-272b37b8.jpg) + +If migrating production isn’t an option, use Neon for development and testing—cut through the chaos, reduce infrastructure costs, and get more done. + +## Let’s Connect + +Curious about how Neon can fit into your workflow? Whether you have technical questions, need help with setup, or want to discuss pricing, [let’s chat](https://neon.tech/contact-sales). Our team is happy to help. diff --git a/content/blog/posts/from-webpack-to-vite.md b/content/blog/posts/from-webpack-to-vite.md new file mode 100644 index 0000000000..b04ea8bcb4 --- /dev/null +++ b/content/blog/posts/from-webpack-to-vite.md @@ -0,0 +1,136 @@ +--- +title: What We Learned Migrating From Webpack to Vite +description: Simplifying our setup with fewer plugins and dependencies +excerpt: >- + The Neon Console is a Single Page Application (SPA), or rather, a collection + of SPAs. Static assets like JavaScript and CSS are served through CloudFront, + while the HTML is dynamically rendered by our Go backend. In this post, we’ll + dive into our journey of migrating from webpack... +date: '2024-11-20T17:59:21' +updatedOn: '2024-11-21T00:30:18' +category: engineering +categories: + - engineering +authors: + - roman-zaynetdinov +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-webpack-to-vite/cover.jpg + alt: null +isFeatured: false +seo: + title: What We Learned Migrating From Webpack to Vite - Neon + description: >- + We recently migrated the Neon console from webpack to Vite. We share the + challenges we faced and the improvements we gained with the change. + keywords: [] + noindex: false + ogTitle: What We Learned Migrating From Webpack to Vite - Neon + ogDescription: >- + We recently migrated the Neon console from webpack to Vite. We share the + challenges we faced and the improvements we gained with the change. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-webpack-to-vite/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-webpack-to-vite/neon-vite-1-1024x576-e1ea80ac.jpg) + + +Neon is a serverless Postgres platform that helps teams ship faster via instant provisioning, autoscaling, and database branching. We have a generous Free Plan - sign up [here](https://console.neon.tech/signup). + + +The [Neon](https://neon.tech/) Console is a Single Page Application (SPA), or rather, a collection of SPAs. Static assets like JavaScript and CSS are served through CloudFront, while the HTML is dynamically rendered by our Go backend. + +In this post, we’ll dive into our journey of migrating from [webpack](https://webpack.js.org) to [Vite](https://vite.dev), the challenges we faced, and what we gained along the way. + +## Our Webpack Setup + +Previously, we used a single Webpack configuration file to build all our apps. Unfortunately, build times were slow, and we lacked hot module replacement support, forcing developers to refresh the page after every change. + + +After removing Webpack, we eliminated 17 (!) direct dependencies, including Webpack itself and its plugins. + + +Webpack built all static resources (JS and CSS) and an `assets-manifest.json` file using the `webpack-assets-manifest` plugin. When rendering HTML, our Go backend used this manifest file to serve the correct static resources for each entry point. In the local environment, Webpack watched for file changes, rebuilt the assets, and regenerated the manifest file. On every page load, the Go backend read the manifest and returned updated HTML. + +In addition to static HTML, our Go backend injects user properties required for the app to render. This eliminates the need for an additional HTTP request when rendering the app in the browser. + +- Build time (with Webpack): 20 seconds +- Incremental build time: ~1.5 seconds + +## Vite on The Horizon + +We’ve all heard that Vite is the go-to tool for speed and simplicity. While we might have been able to achieve similar results with Webpack, we decided to give Vite a try. + +Our **migration goals** were: + +- Support hot module replacement (HMR) +- Improve build times +- Simplify the setup by reducing the number of dependencies +- Unify tooling around a single core (we were already using Vitest for unit testing and Vite-based Storybook) + +We started by migrating our admin application and, once confident with the setup, migrated the rest of the apps in a single change. All users were transitioned to the new Vite-based build in the same deployment—reflecting our confidence that everything would work as expected. + +## Migration Challenges + +Unfortunately, we couldn’t simply use a Vite dev server because HTML generation is handled by the backend. This required us to build a custom proxy using the Vite server. + +Fortunately, Vite offers a convenient [JS API](https://vitejs.dev/guide/ssr.html#setting-up-the-dev-server) that is straightforward to use when more control is needed. Additionally, its documentation includes a helpful guide on [integrating Vite into existing backend applications](https://vitejs.dev/guide/backend-integration.html). + +Here’s how it works locally: + +1. All entry points are listed in `vite.config.js` to build JS bundles. We don’t use HTML entry points _(for most apps)_. +2. Production assets are built and included in the backend Docker. On startup, the backend reads the `manifest.json` file generated by Vite and imports all assets while generating HTML. +3. Our custom Vite proxy reads the HTML response and replaces production assets with Vite dev imports. +4. Vite transforms the HTML and injects development scripts. +5. Developers interact with an HMR-enabled app. +6. End-to-end (E2E) tests bypass the proxy entirely, allowing us to test production bundles without being slowed down by Vite. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-webpack-to-vite/screenshot-2024-11-19-at-120139percente2percent80percentafpm-1024x328-74c9aaa3.png) + +Excluding comments, our custom proxy is only 150 lines of code and uses the `http-proxy` dependency—the same one Vite uses internally. + +- Build time (with Vite): 22 seconds + +Surprisingly, our build times didn’t improve and even became slightly slower. This caught us off guard, but the HMR functionality and reduction in dependencies were still significant wins. + + +While for most of our apps we specify a JS entrypoint, we do have a single maintenance app that is just static assets. We were positively surprised to learn that Vite works well with both entrypoint types and can build both JS and HTML assets at the same time. + + +## Surprises That We Hit Along the Way + +1. We tried to implement a proxy ourselves using `fetch`. This worked for basic scenarios but failed in others. Switching to `http-proxy` made all flows work. +2. Our backend might return multiple SPA apps in a single HTML. The custom Vite proxy needed to support that as well. +3. Finding the entry point for a production asset was a challenge. The backend responds with production assets, so the custom Vite proxy needed to locate the assets in the HTML and replace them with a single dev import referencing the correct entry point. + - We used HTML comments to mark the assets. +4. We have both frontend and backend developers working with the same dev setup. We wanted our change to be as unnoticeable as possible. + - This meant running the custom Vite proxy in a container behind Nginx. +5. Supporting E2E headless tests + - The Vite dev setup loads every file in a separate request. For our main app, the browser needs to load ~1500 files, which makes cold (empty cache) page loads slow. + - For E2E tests, we introduced a second way to run our services locally: we skip the proxy altogether. Nginx forwards requests directly to the backend, and production assets are served and tested. +6. We had to rename one entry point because it conflicted with an API endpoint. The entry point JS file had the same name as the API endpoint, and Vite couldn’t handle it. +7. The `@vitejs/plugin-react-swc` extension panicked on syntax errors, forcing developers to restart the proxy. We switched back to `@vitejs/plugin-react` due to its better stability and error handling. The babel-based plugin was already a huge win for us, especially with HMR. +8. We encountered `Unable to preload CSS for /assets/assets/index-I-pmUzqQ.css` errors in production. It seems we are [not the only ones experiencing this](https://github.com/nuxt/nuxt/issues/26972). +9. We wanted to use Content Security Policy (CSP) headers in the local environment as well. The custom Vite proxy extends CSP to support Vite’s WebSocket server. +10. Global `process.env.*` variables are not defined by default. +11. No incremental builds + +- Developers who skip the custom Vite proxy won’t see any UI changes until they rebuild the app completely. + +## Final Outcomes + +- Build times stayed the same. +- HMR works great +- The Vite configuration is significantly simpler, with fewer plugins and dependencies +- A custom Vite proxy had to be implemented +- Frontend developers are happy 🙂 +- Configuration is now unified across apps, Vitest, and Storybook +- Vite build artifacts are split into more modules, whereas Webpack produced a single JS and CSS import per app +- Initial page loads in the local environment take slightly longer, as the browser must fetch thousands of files—but we now refresh much less frequently + +## Follow-Up Work + +Not having faster build times was a huge disappointment for all of us. [After analyzing the flamegraphs that Vite produces](https://vite.dev/guide/troubleshooting.html#performance-bottlenecks), we found we were hitting Rollup limits. We are looking forward to the future of Vite powered by [Rolldown](https://rolldown.rs/about)! + +However, we noticed that our Ace Editor dependency could be patched to reduce the number of modules Vite needs to process. Using the `yarn patch` utility, we modified the library to depend on ~450 fewer modules. This change alone sped up our build times by 10 seconds (~50%)! diff --git a/content/blog/posts/from-xampp-to-ephemeral-postgres.md b/content/blog/posts/from-xampp-to-ephemeral-postgres.md new file mode 100644 index 0000000000..869bf4e54e --- /dev/null +++ b/content/blog/posts/from-xampp-to-ephemeral-postgres.md @@ -0,0 +1,246 @@ +--- +title: 'From XAMPP to Ephemeral Postgres: Where “Works on My Machine” Has Led Us' +description: How lightweight cloud environments are replacing local dev +excerpt: >- + “It works on my machine” is now a trope—a horror story to tell young + developers (or AI coding models) about what it was like to build without + today’s stack. Like all clichés, it was once true. Hodgepodge local installs, + no containers, and massive environment drift were challenges... +date: '2025-02-27T00:51:30' +updatedOn: '2025-02-27T01:14:29' +category: community +categories: + - community +authors: + - andrew-tate +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-xampp-to-ephemeral-postgres/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + From XAMPP to Ephemeral Postgres: Where “Works on My Machine” Has Led Us - + Neon + description: >- + “It works on my machine” is now a horror story to tell young developers (or + AI models) about what it was like to build in earlier days. + keywords: [] + noindex: false + ogTitle: >- + From XAMPP to Ephemeral Postgres: Where “Works on My Machine” Has Led Us - + Neon + ogDescription: >- + “It works on my machine” is now a horror story to tell young developers (or + AI models) about what it was like to build in earlier days. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/from-xampp-to-ephemeral-postgres/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-xampp-to-ephemeral-postgres/neon-work-on-my-machine-1-1-1024x576-d4cb1aff.jpg) + +“It works on my machine” is now a trope—a horror story to tell young developers (or AI coding models) about what it was like to build without today’s stack. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-xampp-to-ephemeral-postgres/screenshot-2025-02-26-at-44810percente2percent80percentafpm-1-1024x735-f5d46236.png) + +Like all clichés, it was once true. Hodgepodge local installs, no containers, and massive environment drift were challenges for local developers. But like all challenges, these real constraints led to genuine innovations in each era of local development. + +## The 1990s: The Wild West of Local Development + +If you were in web development back in the 90s, you were living the absolute dream. Living in Seattle, Drinking this new ‘Starbucks,’ going to Nirvana gigs, and downloading [Apache](https://httpd.apache.org/download.cgi). What a life. + +Local development in the 1990s required extensive manual setup and system knowledge. Developers needed to download and compile their web servers, interpreters, and databases from source code. A typical Apache setup involved downloading source files via FTP, running configure scripts, resolving dependencies, and manually editing httpd.conf files. + +A typical handcrafted Apache httpd.conf configuration that sets up a local PHP dev environment might look like this: + +```bash +# xampp/apache/conf/httpd.conf +LoadModule php5_module modules/php5apache2_2.dll +PHPIniDir "C:/xampp/php" +LoadModule rewrite_module modules/mod_rewrite.so + +# Virtual hosts for different framework projects + + DocumentRoot "C:/xampp/htdocs/symfony_project/web" + ServerName symfony.local + + AllowOverride All + Require all granted + + +``` + +This configuration would: + +- Load PHP as an Apache module +- Set up a local development site at mysite.local +- Enable .htaccess file usage (AllowOverride All) +- Configure logging for debugging +- Set directory permissions + +Getting this working typically involved several rounds of editing, restarting Apache, and checking error logs. A single syntax error could prevent Apache from starting. + +Environment management was particularly challenging. Each developer’s machine had a unique configuration of manually installed components. Package management was minimal–most installations were done through .exe files on Windows or by compiling from .tar.gz archives on Unix systems. Version conflicts between libraries were common and difficult to diagnose. + +Database setup posed significant technical hurdles. MySQL and Postgres installations required careful system configuration, including setting correct environment variables and managing initial user permissions. Different versions of the same database often couldn’t coexist on one machine, making it difficult to work on multiple projects. + +Because each developer had a unique setup, this was truly the era of “it works on my machine.” + +However, this era fostered deep technical understanding and creativity. Without widespread, standardized toolchains, developers found unique solutions that sometimes led to innovations in open-source communities. Freed from mainframes or expensive servers, developers could experiment locally with minimal hardware cost, driving grassroots software growth. + +Developers gained extensive knowledge of system architecture, build processes, and configuration management through necessity. While everything done in seconds today required time and thought then, this laid the groundwork for future automation. The limitations of the time drove important innovations in build tools and the early foundations of what would later become package management systems. These limitations directly influenced the development of the next set of tools, like XAMPP and, eventually, Docker. + +## The 2000s: Turn Your \*AMP to 11 + +We’re through the dotcom boom and bust, pets.com is gone, but this Bezos fellow seems to be sticking around. You’ve moved your local setup from Seattle to San Francisco for the vibes. + +The 2000s brought the first real attempt at standardizing local development environments through \*AMP stacks. [XAMPP](https://www.apachefriends.org/), [WAMP](https://www.wampserver.com/), [LAMP](), and [MAMP](https://en.wikipedia.org/wiki/MAMP) offered one-click Apache, MySQL, PHP, and Perl installations. A developer could download XAMPP, run the installer, and have a working web server in minutes instead of days. + +Framework adoption drove complexity in local environments. Ruby on Rails required specific Ruby versions, PHP frameworks needed particular PHP configurations, and .NET development required specific Windows components. A typical XAMPP configuration looked like this: + +```bash +# xampp/apache/conf/httpd.conf +LoadModule php5_module modules/php5apache2_2.dll +PHPIniDir "C:/xampp/php" +LoadModule rewrite_module modules/mod_rewrite.so + +# Virtual hosts for different framework projects + + DocumentRoot "C:/xampp/htdocs/symfony_project/web" + ServerName symfony.local + + AllowOverride All + Require all granted + + +``` + +Database management improved but remained problematic. While XAMPP included phpMyAdmin for MySQL administration, teams struggled with database synchronization. Developers often shared database dumps through email or FTP, leading to version mismatches and lost data. + +Where’s Postgres in all this? While MySQL came bundled with these stacks, Postgres required separate installation and configuration. A Postgres workflow for developers would be: + +- Download the installer from postgresql.org, +- run initdb to create a database cluster, +- Manually edited pg_hba.conf and postgresql.conf files for authentication and performance settings. + +This extra complexity meant that Postgres was often chosen by more experienced teams that needed its advanced features and strict SQL compliance. Importantly though, if you were to run a true “local Postgres”, this isn’t too far away from what developers have to deal with today. Yes, you have Homebrew or a GUI to help you, but local Postgres still requires thinking about the local configuration. + +New challenges emerged. Environment drift became common as developers delayed updating their XAMPP installations. Sharing project configurations meant zipping entire directories with vendor folders and database dumps. Version control systems like SVN helped, but dependency management remained manual–there was no composer.json or package.json yet. + +Despite these issues, the era marked significant progress. New developers could start building web applications without deep system knowledge. Community forums filled with XAMPP-specific solutions created a shared knowledge base. Most importantly, developers could focus more on application logic and less on environment configuration–an early sign of productivity gains to come. + +## The 2010s: The Docker Revolution + +The hipster coffee shop where you’re coding has better WiFi than your first three jobs combined, and you’ve just discovered that containers aren’t just what your Blue Bottle comes in. + +In 2013, Solomon Hykes [stood on stage at PyCon](https://www.youtube.com/watch?v=wW9CAH9nSLs&ab_channel=dotcloudtv), showing a demo that would change everything. ‘Docker,’ he called it. No one knew it yet, but XAMPP’s days were numbered. + +[Docker](https://www.docker.com/) transformed local development by introducing containers–lightweight, isolated environments that package everything an application needs to run. No more “but it works on my machine” excuses because your machine wasn’t running the code anymore–a containerized slice of Linux was. Here’s what a typical database-driven application setup looked like: + +```javascript +# Dockerfile for API service that connects to Postgres +FROM node:14 +WORKDIR /app +COPY package*.json ./ +RUN npm install +COPY . . +# Add wait-for-it script to handle Postgres startup timing +COPY wait-for-it.sh /wait-for-it.sh +RUN chmod +x /wait-for-it.sh +EXPOSE 3000 +CMD ["/wait-for-it.sh", "postgres:5432", "--", "npm", "start"] +``` + +But single containers weren’t enough. Enter [Docker Compose](https://docs.docker.com/compose/), which lets developers orchestrate multiple services together. A typical stack might look like this: + +```javascript +version: '3.8' +services: + api: + build: . + ports: + - "3000:3000" + environment: + DATABASE_URL: postgres://user:password@postgres:5432/myapp + REDIS_URL: redis://cache:6379 + depends_on: + - postgres + - cache + + postgres: + image: postgres:13 + ports: + - "5432:5432" + environment: + POSTGRES_USER: user + POSTGRES_PASSWORD: password + POSTGRES_DB: myapp + volumes: + - postgres_data:/var/lib/postgresql/data + - ./init.sql:/docker-entrypoint-initdb.d/init.sql + + cache: + image: redis:alpine + volumes: + - redis_data:/data + +volumes: + postgres_data: + redis_data: +``` + +One command (`docker-compose up`) and suddenly you had a web server, Postgres database, and Redis cache all spinning up in harmony. Well, when it worked. When it didn’t, you were diving into container logs, checking network configurations, and wondering why your Postgres volume mounts were showing stale data from your previous feature branch. + +Data management became its own special challenge. Volumes helped persist Postgres data between container restarts, but developers still struggled with synchronizing databases across teams. You might pull the latest code, but your colleague’s migrations from last week were missing, leading to the classic “works in prod” scenarios. + +Resource management brought new headaches too. Running multiple containerized services could bring even powerful laptops to their knees. Developers learned to juggle Docker Desktop’s resource settings and became intimate with the task manager, watching their RAM disappear into the container void. + +But the benefits were transformative. Teams could version control their entire development environment alongside their code, and new team members could be productive in hours instead of days. The rise of microservices architecture became practical because each service could run in its own container with its own dependencies without conflicting with others. + +Most importantly, Docker helped bridge the gap between development and operations. The same container that ran locally could be deployed to any cloud provider. The “it works on my machine” excuse transformed into “it works in my container”–and that container could run anywhere. + +Meme begets meme: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/from-xampp-to-ephemeral-postgres/screenshot-2025-02-26-at-50158percente2percent80percentafpm-737x1024-1eb8f7b4.png) + +The era wasn’t perfect, but it laid the groundwork for modern development practices. Teams learned to think in terms of isolated services, reproducible environments, and infrastructure as code. These lessons would prove invaluable as cloud-native development took hold in the following decade. + +## The 2020s: The Rise of Hosted Environments + +You’re working remotely now, making artisanal coffee at home, and your Docker Desktop is competing with Zoom for memory. But something’s changing–you don’t need all this running locally anymore. + +The 2020s marked a shift away from local development entirely. Instead of wrestling with Postgres installations or Docker volumes, developers connect to hosted services. A typical configuration now looks remarkably simple: + +```javascript +// Modern database configuration +const config = { + database_url: process.env.DATABASE_URL, // Points to hosted Postgres + cache_url: process.env.REDIS_URL, // Points to hosted Redis +} +``` + +Postgres itself evolved from a local installation to a fully managed service. Cloud services with a Free plan like [Neon](https://neon.tech/home) popularized this workflow, adding features to the table that would have seemed magical a decade ago—especially [branching](https://neon.tech/docs/introduction/branching). Creating a new Postgres environment became as simple as: + +```bash +# Create a new database branch for a feature +neon branches create --name feature-user-auth + +# Get the connection string +neon connection-string feature-user-auth +``` + +For many developers, the entire development environment became cloud-based. As with every era, this shift has brought new challenges. Internet connectivity becomes critical–no more coding on planes unless you’ve set up local fallbacks (plane WiFi still sucks). Teams also have to manage cloud costs and think about data governance in new ways. + +But the benefits are compelling. Development environments finally achieved true parity with production. Features like Neon branching allows each pull request to have its own isolated data environment. Sensitive data no longer needs to live on developer laptops. Teams can collaborate on features by sharing database branch URLs instead of SQL dumps. + +The “it works on my machine” era has ended, replaced by **it works in my branch**–but that branch can be shared, replicated, and torn down at will. + +## 2025 and Beyond: Local Environments in the AI Era + +[Replit](https://replit.com/) has shown an entire generation of new developers that they never needed a local environment in the first place–just a browser and an internet connection. [GitHub Codespaces](https://github.com/features/codespaces) and similar tools prove this model works for enterprise development, too: click a button, and your entire development environment springs to life in a browser tab. AI-powered development environments like [v0](https://github.com/features/codespaces) can generate entire applications from prompts. Your IDE, your runtime, your databases–all running in the cloud, accessible from anywhere. + +The real transformation is in how we collaborate. When an engineer says, “check out this feature,” they’re not sending you a database dump or Docker compose file–they’re sharing a URL that spins up an exact copy of their environment. Real-time debugging sessions happen in shared cloud environments where everyone sees the same state. + +We’re moving beyond the “it works on my machine” era entirely. Your machine isn’t running the code anymore–it’s just a terminal into a vast, shared cloud development environment. The tools we use are becoming more ephemeral, more collaborative, and more powerful. + +Each era of development brought its own innovations. The 1990s taught us system architecture through necessity. The 2000s gave us integrated stacks that made web development accessible. Docker showed us the power of containerization. Now, cloud-native development is teaching us that maybe we don’t need to run anything locally after all. Local is now about **where you code**, not where your code runs. diff --git a/content/blog/posts/full-text-search-cms-pgsearch.md b/content/blog/posts/full-text-search-cms-pgsearch.md new file mode 100644 index 0000000000..942df89826 --- /dev/null +++ b/content/blog/posts/full-text-search-cms-pgsearch.md @@ -0,0 +1,426 @@ +--- +title: Build Your Own Full-Text Search CMS with Neon and pg_search +description: An example project using pg_search with Neon +excerpt: >- + A CMS is a “must-have” for building a website. And your choices are many. If + you want full-stack, you can go with WordPress or Webflow. If you want + headless, you can go with Sanity or Contentful. Heck, with the aid of a GitHub + Action, you can just turn your Notion pages directly... +date: '2025-07-07T16:22:34' +updatedOn: '2026-03-19T19:38:54' +category: postgres +categories: + - postgres +authors: + - tristan-partin +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/cover.jpg + alt: null +isFeatured: false +seo: + title: Build Your Own Full-Text Search CMS with Neon and pg_search - Neon + description: >- + An example project showing how to build a full-text search app with + pg_search and Neon Postgres. + keywords: [] + noindex: false + ogTitle: Build Your Own Full-Text Search CMS with Neon and pg_search - Neon + ogDescription: >- + An example project showing how to build a full-text search app with + pg_search and Neon Postgres. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/neon-full-text-search-1-1024x576-511e69b4.jpg) + + +As of March 19, 2026, `pg_search` is longer available for new Neon projects. For alternatives, see [The pg_search extension](https://neon.com/docs/extensions/pg_search). + + +A CMS is a “must-have” for building a website. And your choices are many. If you want full-stack, you can go with WordPress or Webflow. If you want headless, you can go with Sanity or Contentful. Heck, with the aid of a GitHub Action, you can just [turn your Notion pages directly into a CMS](https://www.notion.com/templates/cms?srsltid=AfmBOoqknO2b0epM9mx_d8uhUGNerwexj-VVV-vUkLn7Gk63I85jRG4x). + +But sometimes these are overkill. If you really just need a place to store words, then that is literally what databases are designed for. All CMSes are DBs under the hood – why not just remove the middleman? + +That is what we’re going to do here, using Neon and the [pg_search](https://neon.com/docs/extensions/pg_search) extension. By adding `pg_search` into the mix, we can drastically improve our CMS experience by adding full-text search capabilities straightforwardly into our application. Let’s do it. + +## Setting up Neon as a CMS + +This is very simple. If you don’t already have a Neon account, sign up for one here. Then, create a new project. Call it whatever you want, but make sure you create it using AWS (`pg_search` is only currently available with AWS on Neon. Sorry, Azurers.). + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxfnv4ljafhe6esk5c9jjg0hgbxozrrszm8fcyqykbsupyes8temqvcp2xaaelpq1tk5vsw58au7fvxfas4zxe-xwawsnync4eykexuevrv8gaph1kaqjbuyqsh4gbf5cw1phpgw-46032239.png) + +Once created, grab your connection URL as we’ll need that within our application. + +We just need one SQL command right now: + +```sql +CREATE EXTENSION IF NOT EXISTS pg_search; +``` + +This will allow us to use `pg_search` with Neon. There is an optional step you could take here: creating your data tables. However, we’re going to use Prisma to do that directly from our application. + +## Creating our application with Next.js, Prisma, and Zod + +You know how to create an application with Next.js. If not, just check out their docs here for the latest version. You can find all the code for this CMS application in this [repo](https://github.com/argotdev/neon-cms). + +Apart from Neon and pg_search, the core part of this application is [Prisma](https://www.prisma.io/orm). Prisma is a type-safe database toolkit that makes working with databases easy. It auto-generates TypeScript types from your schema, handles migrations elegantly, and provides an intuitive query API to make complex database operations simpler. + +To use Prisma, first install it along with its CLI: + +``` +npm install prisma @prisma/client +npm install -D @types/node +``` + +Next, initialize Prisma in your project: + +``` +npx prisma init +``` + +This creates a prisma directory with a `schema.prisma` file. This is the only thing you’ll need to update to connect to your Neon database: + +```typescript +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +model Post { + id String @id @default(dbgenerated("gen_random_uuid()")) @db.Uuid + title String + slug String @unique + body String + tags String [] + createdAt DateTime @default(now()) @map("created_at") @db.Timestamptz + + @@map("posts") +} +``` + +Here’s what each part does: + +- **Generator block**: Tells Prisma to generate a JavaScript/TypeScript client that you’ll use in your code. +- **Datasource block**: Configures the Postgres connection using the DATABASE_URL from your environment variables. +- **Model post**: Defines your database table structure: + - `id`: Primary key using Postgres’ native UUID generation (gen_random_uuid()) + - `title`: Plain text field for the post title + - `slug`: Unique URL-friendly identifier for each post + - `body`: Main content field + - `tags`: Postgres array field for categorization + - `createdAt`: Timestamp with timezone, automatically set on creation + - `@map("created_at")`: Maps the Prisma field name to a snake_case column in the database + +`@@map("posts")` maps the model name to a lowercase table name. This means your database table will be named `posts` instead of `Post`, following Postgres naming conventions where tables are typically lowercase and plural. + +After defining your schema, generate the Prisma client and push the schema to your database: + +``` +npx prisma generate +npx prisma db push +``` + +The first command generates the TypeScript client with all the type definitions based on your schema. The second creates the actual table in your Neon database (here, we’ve [seeded our database](https://github.com/argotdev/neon-cms/blob/main/prisma/seed.ts) with a few posts to start): + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxe31awbxedbepr7lcsqya15hkgd2picl94iumjxrbdehrkkhpkvs0qq-7v79opf07zbm6agkgixdrfa3c7zddbdibq2jkd1ctmkyecpq3stgy-1f0pigk7af-obogjonogsirpq-30961bd2.png) + +You can import and use the Prisma client anywhere in your application. We’ll use it within a [lib/db.ts](https://github.com/argotdev/neon-cms/blob/main/lib/db.ts) file to set up our database connection: + +```typescript + +import { Pool } from '@neondatabase/serverless'; +import { PrismaClient } from '@prisma/client'; +import { PoolClient } from 'pg'; + +declare global { + var prisma: PrismaClient | undefined; +} + +export const prisma = global.prisma || new PrismaClient(); + +if (process.env.NODE_ENV!== 'production') { + global.prisma = prisma; +} + +const pool = new Pool({ + connectionString: process.env.DATABASE_URL, +}); + +export async function withClient( + callback: (client: PoolClient) => Promise +): Promise { + const client = await pool.connect(); + try { + return await callback(client); + } finally { + client.release(); + } +} +``` + +This file sets up the Prisma client. The `withClient` function is the critical function here as it provides direct Postgres access through Neon’s serverless driver. This is crucial for `pg_search` operations that require raw SQL queries beyond Prisma’s capabilities (we’ll get to those in a moment). + +We’re also using [Zod](https://zod.dev/). Zod is used for runtime validation, ensuring that data coming from API requests matches our expected schema before it hits the database. While Prisma gives us compile-time type safety, Zod protects us at runtime from malformed or malicious input. + +```typescript +//schema.ts +import { z } from 'zod'; + +export const postSchema = z.object({ + id: z.string().optional(), + title: z.string().min(1).max(255), + slug: z.string().min(1).max(255).regex(/^[a-z0-9-]+$/), + body: z.string().min(1), + tags: z.array(z.string()).min(1), +}); + +export type Post = z.infer; +``` + +The schema enforces that titles and slugs can’t exceed 255 characters, slugs must be URL-friendly (lowercase letters, numbers, and hyphens only), and posts must have at least one tag. The z.infer utility extracts a TypeScript type from the schema, giving us a single source of truth for both validation and typing. + +That is the core of our application. Now let’s move on to the fun part. + +## Using pg_search to enable full-text search + +As we said above, we’re not going to be using the built-in Prisma client query functionality. Instead, we want to roll our own so we can take complete advantage of `pg_search`. Before we start with the code, let’s create a special index in our Neon database: + +```sql +CREATE INDEX posts_search_idx ON posts USING bm25 (id, title, body, tags) WITH (key_field='id'); +``` + +This index is critical. It creates a [BM25 index](https://docs.paradedb.com/documentation/concepts/index#bm25-index) across all our searchable fields. BM25 is a ranking algorithm specifically designed for full-text search that considers both term frequency and document length. Unlike simple keyword matching, BM25 provides relevance scoring that naturally promotes documents where search terms appear more frequently while penalizing overly long documents that might dilute relevance. + +We’ll contain all our Postgres search functionality within one file, [lib/search.ts](https://github.com/argotdev/neon-cms/blob/main/lib/search.ts). Let’s step through it to understand what is happening: + +```typescript +export interface SearchResult { + id: string; + title: string; + body: string; + snippet: string; + score: number; +} +``` + +This is the shape of our search results. Each result includes the full post data plus a snippet field containing a highlighted excerpt of the matching text and a score representing the BM25 relevance ranking. + +```typescript +export interface SearchOptions { + /**Maximum rows to return. Default = 20*/ + limit?: number; + /**Tag that wraps the first character of each hit. Default = */ + startTag?: string; + /**Closing tag for the hit wrapper. Default = */ + endTag?: string; + /**Levenshtein distance for fuzzy search. Default ≈ 20 % of query length (1‑3)*/ + fuzzyDistance?: number; +} +``` + +This is our configuration object for search behavior. The `startTag` and `endTag` options let you customize how matching terms are highlighted in snippets (defaulting to bold tags). The `fuzzyDistance` parameter enables typo tolerance – a value of 2 means the search will match terms even if they’re off by up to 2 character edits, making searches more forgiving of user mistakes. + +```typescript +export async function searchPosts( + query: string, + options: SearchOptions = {}, +): Promise { + const { + limit = 20, + startTag = '', + endTag = '', + fuzzyDistance = Math.min(3, Math.max(1, Math.floor(query.length * 0.2))), + } = options; + + return withClient(async (client) => { +``` + +Finally, before we get into the SQL, this is our main search function that accepts a query string and optional configuration. The fuzzy distance calculation is particularly nice. It scales with query length, allowing one character difference for short queries (5 characters or less), 2 for medium queries (6-10 characters), and maxing out at 3 for longer queries. This adaptive approach ensures that short searches remain precise while longer searches become more forgiving of typos. + +We’re going to have a three-fold approach to make sure we find the best documents. The first search will be a BM25 keyword search on the title and the body: + +```typescript +/** 1 ▸ BM25 keyword search on title+body --------------------------- */ + const bm25 = await client.query( + ` + WITH bm25 AS ( + SELECT id, + title, + body, + paradedb.snippet(body, start_tag => $2, end_tag => $3) AS snippet, + paradedb.score(id) AS base_score, + (title @@@ $1)::int AS hit_title, + (body @@@ $1)::int AS hit_body + FROM posts + WHERE title @@@ $1 OR body @@@ $1 + ) + SELECT id, + title, + body, + snippet, + base_score* + CASE + WHEN hit_title = 1 AND hit_body = 1 THEN 2.5 + WHEN hit_title = 1 THEN 2.0 + ELSE 1.0 + END AS score + FROM bm25 + ORDER BY score DESC + LIMIT $4; + `, + [query, startTag, endTag, limit], + ); + if (bm25.rows.length) return bm25.rows; +``` + +The query here uses pg_search’s `@@@` operator to perform BM25 searches on both title and body fields. The `paradedb.snippet()` function generates highlighted excerpts showing where matches occur, while `paradedb.score()` provides the raw BM25 relevance score. + +We’ve included some score-boosting logic so if a search term appears in both the title and body, we multiply the score by 2.5. Title-only matches get a 2x boost, while body-only matches keep their base score. This reflects the reality that title matches are typically more relevant than body matches, and documents matching in multiple fields are the most relevant of all. If this search returns any results, we return them immediately without trying other search strategies. + +If not, we move on to the next strategy: + +```typescript +/** 2 ▸ Phrase search (words close together) ------------------------ */ + const words = query.split(/\s+/).filter(Boolean); + if (words.length > 1) { + const slop = Math.min(2, words.length - 1); + + const phrase = await client.query( + ` + WITH phrase AS ( + SELECT id, + title, + body, + paradedb.snippet(body, start_tag => $3, end_tag => $4) AS snippet, + paradedb.score(id) AS base_score + FROM posts + WHERE id @@@ paradedb.phrase('title', $1::text [], slop => $2) + OR id @@@ paradedb.phrase('body', $1::text [], slop => $2) + ) + SELECT id, title, body, snippet, base_score AS score + FROM phrase + ORDER BY score DESC + LIMIT $5; + `, + [words, slop, startTag, endTag, limit], + ); + if (phrase.rows.length) return phrase.rows; + } +``` + +This handles multi-word queries by searching for the words appearing near each other, even if not in exact order. The `slop` parameter (set to the number of words minus 1, capped at 2) determines how many positions apart the words can be while still counting as a phrase match. For example, searching “full text search” with slop=2 would match “full powerful text search” or “text full search”. + +The `paradedb.phrase()` function is more restrictive than the basic BM25 search – it requires all words to be present and relatively close together. This is perfect for when users type what they remember as a phrase, but might not recall the exact wording. If the first strategy found nothing, this approach helps catch documents where the search terms appear together as a meaningful unit rather than scattered throughout the text. + +If that doesn’t return any results, we move on to our third option: + +```typescript +/** 3 ▸ Fuzzy match fallback --------------------------------------- */ + const fuzzy = await client.query( + ` + WITH fuzzy AS ( + SELECT id, + title, + body, + paradedb.snippet(body, start_tag => $3, end_tag => $4) AS snippet, + paradedb.score(id) AS base_score + FROM posts + WHERE id @@@ paradedb.match('title', $1, distance => $2) + OR id @@@ paradedb.match('body', $1, distance => $2) + ) + SELECT id, title, body, snippet, base_score AS score + FROM fuzzy + ORDER BY score DESC + LIMIT $5; + `, + [query, fuzzyDistance, startTag, endTag, limit], + ); + return fuzzy.rows; // may be empty +``` + +This is our last resort for typo-tolerant searching. The `paradedb.match()` function with a distance parameter performs fuzzy matching using Levenshtein distance – it finds documents containing words that are similar to the search query within the specified number of character edits (insertions, deletions, or substitutions). + +This catches common misspellings like “postgress” for “postgres” or “prizma” for “prisma”. The adaptive fuzzy distance we calculated earlier ensures we’re not too permissive with short queries (where a single character change could completely alter meaning) while being more forgiving with longer queries where typos are more likely. Even if this returns an empty array, at least we’ve exhausted all reasonable search strategies before giving up. + +So, let’s try this in action. We can fire up our CMS with: + +``` +npm run dev +``` + +Then start using it. First, we’ll just go through the basics CRUD operations in the CMS. Showing our posts: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxc8-ub9q0boekabgppemx-y508pxmmjxb92t2xnnliplbrt7m6osggs4hxjlschuaq7us1yuy004-scgb8l52erdxyjts2g-com5jj-esaxmnvgbgpelvhfzyhnk2ghwucs-e3ef5dbe.png) + +Creating a new post: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxd4edfdwzbkor6ldlz15kamrdaqcnuhsiwdvtnjplaa6li6vw2aqiy7nkkt92cufa1z6tc9zfqh9vwa2ns3ubiqa9lh9ywhliuiuaeirrkwty3pgquixndh1uatbm3jvs05n-1998eaf9.png) + +Displaying a post: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxeppjldzznqcxixp1xz-fdrgogeepgjlxuzkhvlildtvakqw1vnafyh-0fkgrluexybgchxxgpkayj8waft3sihtmu7f5ph52jx29ugzqeanqdvec86qlkczlwcgtf-bkog-d6f3737a.png) + +Editing a post: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxcyqhrzjbfch6fawfrwr3s2faobbfekbnqwfbu37xwvpflme6nmxkqeftbmjjqt-8v4lqghl3mnejkdosrl9t-hdudyba-7jlxkzeqyqze8horf87thnxfds04t3cwmvdkdow-e2eb411d.png) + +And deleting a post: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxfphqxjw0kfctpdw03gjlzuzhft9fhge6zzqd8bikldxv7rye1n4besnkpj22dlv4kubc-7cb9mwppphdofa-g3gg7jmao5a42228alyfbdwcp3ug5mzdmqx4i-2ci6qdpwg-566648a9.png) + +A fully functioning CMS! But what we really want is search. If we head to the search page, we can search for some of our articles. Let’s try searching for ‘postgres’: + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxcrgg6kiics7hh7tebkzrrc6mylkigqqxiiqikznhxyiilhainbtg4krfzepiwefwiimjc2ipnvqimx1htpxbqmnozqtsvz9rngf5b3demuazodmyjdzdiwgu5txjvxpdhn2fla-e4761c03.png) + +We get two matches, the post with “Postgres” in the title, but also the one with “PostgreSQL” in the title. Our search is smart enough to recognize that “postgres” and “PostgreSQL” are related terms, thanks to BM25’s partial matching capabilities. + +What about if we drop the ‘t’? + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxfgq7xzh2khkv-utgz8eearnq3x5kiwm79knbboscm0gshzmzm-khqlxdba9njadmkjo6djllweo1obu8af6a10wkleqrrp61qi0dfkgzolppwr08iuv7efwo1dohfrspqsopviw-8735c0c0.png) + +Our search still works because the fuzzy matching strategy kicks in when the exact BM25 search fails. With a Levenshtein distance of 1, “posgres” matches “postgres” by recognizing the single missing character. This typo tolerance is crucial for real-world usage, where users might misspell technical terms or type quickly. The scores are slightly lower than exact matches, but the results are still ranked by relevance, ensuring users find what they’re looking for even with imperfect queries. + +What about a phrase, like “neon postgres”? + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/full-text-search-cms-pgsearch/ad4nxenmpaergutjctdbrqmk3-ua1lvgblggsf5pbcaznlhtqnj7kju6awmumpd8krkbqslk-dwdibu6ak1movtuqjrh6mvqb2ae5b7yzczykrd29g-5qxtuivfydtcfiau6obe0mjdtw-4735319e.png) + +Yep, still works. This time, it’s using the phrase search strategy since we have multiple words. The search finds documents where “neon” and “postgres” appear near each other, with the slop parameter allowing them to be up to 2 positions apart. + +## Taking Your CMS Further + +And there you have it – a fully functional CMS with enterprise-grade search capabilities, all running on a single Postgres database. No Elasticsearch clusters to manage, no sync issues between your database and search index, and no complex infrastructure to maintain. Just pure, fast, ACID-compliant search that scales with your data. + +The beauty of this approach is its simplicity. By leveraging `pg_search` on Neon, you’ve eliminated an entire layer of complexity from your stack while gaining features that rival dedicated search engines. Your search results update in real-time with your data, your relevance scoring adapts to your content, and typos don’t break the user experience. + +There’s much more you can explore with `pg_search` to enhance your CMS: + +- **Hybrid search**: Combine BM25 scores with `pgvector` embeddings for semantic search capabilities +- **Faceted search**: Enable category filtering and aggregations for e-commerce-style browsing +- **Advanced tokenizers**: Use ICU or Lindera tokenizers for better multilingual support +- **Fast fields**: Optimize aggregations and sorting with columnar storage +- **Query-Time boosting**: Dynamically adjust field importance based on context +- **Regex and wildcard queries**: Support complex pattern matching in searches +- **More like this**: Find similar documents based on content similarity +- **Custom scoring**: Implement your own relevance algorithms +- **Highlighting options**: Customize snippet generation with different strategies + +What else could you build? With Neon and `pg_search`, the possibilities extend far beyond a simple CMS: + +- **Knowledge base platform**: Build a Stack Overflow clone with instant search across millions of questions and answers +- **E-commerce search**: Create a product catalog with faceted search, price ranges, and typo-tolerant product discovery +- **Documentation site**: Develop a technical docs platform with code-aware search and version-specific results +- **Legal document repository**: Build a searchable archive with phrase matching for exact legal terminology +- **Log analysis tool**: Build a real-time log search system with regex support and time-based filtering +- **Customer support portal**: Create a help desk with intelligent FAQ search and ticket similarity matching + +The combination of Neon’s serverless architecture and `pg_search` ‘s powerful features gives you the tools to build search experiences that would typically require a dedicated search team and infrastructure. Start simple, scale as needed, and keep your architecture clean. + +--- + +_Neon is a serverless Postgres platform built to help developers ship and scale faster via autoscaling, branching, and instant restores. We have a Free Plan – sign up [here](https://console.neon.tech/signup)._ diff --git a/content/blog/posts/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching.md b/content/blog/posts/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching.md new file mode 100644 index 0000000000..9aec0d1227 --- /dev/null +++ b/content/blog/posts/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching.md @@ -0,0 +1,348 @@ +--- +title: >- + Fullstack Serverless CI/CD in AWS Amplify Hosting with Postgres Database + Branching +description: A Postgres database branch for each SSR app branch +excerpt: >- + This post will guide you through integrating AWS Amplify Hosting CI/CD with + Neon Postgres with a focus on testing your application’s environment or + feature branches using real data enabled by Neon’s database branching. This + architecture automates your workflow and ensures each ne... +date: '2024-03-06T23:50:49' +updatedOn: '2025-04-08T23:00:20' +category: community +categories: + - community +authors: + - stephen-siegert +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Fullstack Serverless CI/CD in AWS Amplify Hosting with Postgres Database + Branching - Neon + description: A Postgres database branch for each SSR app branch + keywords: [] + noindex: false + ogTitle: >- + Fullstack Serverless CI/CD in AWS Amplify Hosting with Postgres Database + Branching - Neon + ogDescription: >- + This post will guide you through integrating AWS Amplify Hosting CI/CD with + Neon Postgres with a focus on testing your application’s environment or + feature branches using real data enabled by Neon’s database branching. This + architecture automates your workflow and ensures each new feature or + environment branch in your Amplify application has a corresponding, isolated + database. […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/cover.jpg) + +This post will guide you through integrating [AWS Amplify Hosting](https://aws.amazon.com/amplify/hosting/) CI/CD with Neon Postgres with a focus on testing your application’s environment or feature branches using real data enabled by Neon’s [database branching](https://neon.tech/docs/introduction/branching). This architecture automates your workflow and ensures each new feature or environment branch in your Amplify application has a corresponding, isolated database. This is especially useful if you’re building and deploying a Server Side Rendering (SSR) application using frameworks like [Next.js](https://nextjs.org/), [Nuxt](https://nuxt.com/), [Astro](https://astro.build/), and [SvelteKit](https://kit.svelte.dev/). + +Because both Neon and Amplify Hosting are serverless, the underlying infrastructure doesn’t require maintenance. Also, when not in use, both services scale down to zero. For Amplify Hosting, this involves serverless compute functions (i.e SSR), while for Neon, it pertains to the Postgres cluster compute. + +## Solution Overview + +In this guide, we cover the deployment process of a serverless full-stack app using Amplify Hosting, leveraging its SSR compute capabilities. SSR enables running code and logic in a server environment, useful for tasks like using routing middleware, interfacing with third-party services, or connecting to a database. + +To follow along, check out the YouTube video below that walks through in depth the process of deploying, and verifying the database branch creation in Neon when deploying a Nuxt SSR app that uses an [Amplify Gen 2 backend](https://docs.amplify.aws/gen2/) with Amplify Auth. While the example features Nuxt for server-side APIs, the methodology applies to any [SSR framework compatible with Amplify Hosting’s function-based compute](https://docs.aws.amazon.com/amplify/latest/userguide/amplify-ssr-framework-support.html). + +For practical implementation, the scripts referenced in this guide (and video) are located in the [Neon Branches with Amplify Hosting CI/CD](https://github.com/siegerts/neon-branches-amplify-cicd) repo on GitHub. + + + +## AWS Amplify Hosting + +AWS Amplify Hosting is a fully managed CI/CD and hosting service that supports static sites, single-page applications (SPAs), and full-stack serverless SSR apps. + +In Amplify Hosting, a typical workflow involves [teams managing app branches that align with various environments](https://docs.aws.amazon.com/amplify/latest/userguide/multi-environments.html), such as development, testing, and production. Each branch, like dev, test, and prod, corresponds to its respective environment and is accessible via a unique URL, which incorporates the first-level subdomain and the branch identifier. For example, `https://..amplifyapp.com/`. + +## Serverless Postgres + +When developing an SSR app in AWS Amplify Hosting and requiring Postgres or an RDBMS, the choices for a serverless relational database are limited. It’s important that your database can handle your application traffic without manual adjustment since serverless SSR fullstack apps can scale out in response to bursts of request traffic. And then scale back down when not in use – similar to how Lambda (and cloud) functions operate. + +Using Amazon RDS involves virtual private cloud (VPC) and architecture configurations, typically requiring a Lambda function or service proxy for database access to bridge the connection from Amplify Hosting to the VPC. Alternatively, directly exposing an RDS database online requires setting up Amazon RDS Proxy, essentially acting as [PgBouncer](https://neon.tech/blog/pgbouncer-the-one-with-prepared-statements), for scaling purposes. + +Amazon DynamoDB (DDB) is another serverless database solution, but it doesn’t offer the relational capabilities inherent to RDBMS like Postgres, such as relational data models, complex joins, or transactions.

Neon is serverless Postgres with [auto-scaling](https://neon.tech/blog/scaling-serverless-postgres), easy setup, [built-in connection pooling](https://neon.tech/docs/connect/connection-pooling), and database branching. Neon’s branching capability allows developers to clone their entire Postgres cluster in seconds, creating isolated environments for testing new features without affecting the primary database branch. For these reasons, it pairs well with the serverless nature of Amplify Hosting and frontend fullstack SSR application architectures. + +## Amplify Hosting CI/CD Integration + +This Amplify Hosting SSR app and Neon Postgres integration involves a custom Bash script (`neon-ci.sh`) [to manage Neon database branches alongside Amplify app branches](https://github.com/siegerts/neon-branches-amplify-cicd/blob/main/neon-ci.sh). This script dynamically updates the .env file’s `DATABASE_URL` environment variable with the database connection string for each branch. This `.env` file is then loaded into the application’s server-side runtime as environment variables. To interact with the Postgres database in serverless or edge contexts, the Neon [Serverless Driver for JavaScript and TypeScript](https://neon.tech/blog/serverless-driver-for-postgres) can be used. Additionally, database migrations can be integrated into the CI/CD pipeline since the connection string is available in the build time environment. + +![CI/CD flow with Neon Postgres and AWS Amplify Hosting](https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/diagram-2-1024x512-3d714910.png) + +#### Here’s how the integration works + +1. A new Git branch is created and pushed in a repo that is configured for fullstack continuous deployments in Amplify Hosting +2. During the CI/CD process, a new Neon Postgres branch will be created using the Neon CLI via a custom bash script +3. This branch’s connection string is injected into the `DATABASE_URL` parameter within a `.env` file. +4. The application is deployed and has access to the `DATABASE_URL` environment variable in server-side compute functions +5. Connect to your database using the Neon serverless driver + +#### In order to set up this flow, you’ll need to complete the following steps + +1. Set up your Amplify app and CI/CD pipeline as per your project requirements +2. Choose the **Amazon Linux: 2023** build image and enable automatic service role creation (if you are not using another role) +3. Copy the [neon-ci.sh script and amplify.yml](https://github.com/siegerts/neon-branches-amplify-cicd) build settings into the root of your app directory +4. Enable branch auto-detection for the app to create a new Amplify app branch for each new Git branch matching the configured pattern +5. Create a [Neon project and your Neon API key](https://console.neon.tech/signup) +6. Create an [SSM **SecureString** parameter](https://neon.tech/blog/deploy-a-serverless-fastapi-app-with-neon-postgres-and-aws-app-runner-at-any-scale#create-a-databaseurl-parameter-in-ssm-parameter-store) for the Neon API key +7. Add the correct policy permissions to the Amplify app Service role for SSM Parameter Store and Amplify from the AWS CLI within the CI/CD build time +8. Modify `neon-ci.sh` invocation in your `amplify.yml` build settings with your specific Neon project ID, database name, and other parameters as needed +9. Push your changes and monitor the Amplify console for the deployment status – the Neon console will show the created branch(es) + +### Configuring Amplify Hosting Build Settings + +To set up build settings in Amplify Hosting, first connect your code repository to AWS Amplify Hosting to activate the CI/CD pipeline. Amplify supports GitHub, Bitbucket, GitLab, or AWS CodeCommit as source control providers. + +#### Create an Amplify app + +![Create an Amplify Hosting Gen 2 app](https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/creat-amplify-app-1024x433-1f0a6783.png) + +#### Confirm the app settings + +Choose the build image for your application (i.e. **Amazon Linux: 2023)** and enable automatic service role creation (if you are not using another role). This role will be updated to allow access to your Neon API key parameter in SSM Parameter Store. + +![Amplify app build settings](https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/amplify-app-build-settings-1024x353-eec6e6a2.png) + +### Update the Amplify Hosting Build Settings + +The app build settings are configured in the Amplify Hosting console _or_ checked in as `amplify.yml` within your code repository. + +This `amplify.yml` file defines the CI/CD pipeline configuration for AWS Amplify Hosting. It is split into backend and frontend stages, with specific commands executed at different phases (**preBuild**, **build**, and **postBuild**) for both the backend and frontend stages. + +Customize the build process by [adjusting the amplify.yml file](https://github.com/siegerts/neon-branches-amplify-cicd/blob/main/amplify.yml), which controls the actions taken during the CI/CD pipeline stages for both frontend and backend components. + +```yaml +version: 1 +backend: + phases: + preBuild: + commands: + - sudo yum -y install jq + - jq --version + - npm i -g neonctl@latest + build: + commands: + - nvm use 18 + - corepack enable + - pnpm install + - | + if [ "${AWS_BRANCH}" = "main" ] || [ "${AWS_BRANCH}" = "dev" ]; then + pnpm amplify pipeline-deploy --branch $AWS_BRANCH --app-id $AWS_APP_ID + else + pnpm amplify generate config --branch main --app-id $AWS_APP_ID + fi + + bash neon-ci.sh create-branch --app-id $AWS_APP_ID --neon-project-id --branch-name $AWS_BRANCH --parent-branch main --api-key-param "" --role-name --database-name --suspend-timeout 0 + cache: + paths: + - $(pnpm store path) +frontend: + phases: + preBuild: + commands: + - nvm use 18 + - corepack enable + - npx --yes nypm i + build: + commands: + - npm run build + + artifacts: + baseDirectory: .amplify-hosting + files: + - "**/* " +``` + +Adjust as needed depending on your app framework requirements, and also make sure to update the `neon-ci.sh` input arguments. The following is a step-by-step breakdown of the sample `amplify.yml` set up: + +**Backend:** + +1. Pre-Build Phase: + 1. Installs [jq for JSON processing](https://jqlang.github.io/jq/) + 2. Installs the `neonctl` CLI tool globally using npm +2. Build Phase: + 1. Sets the Node version using nvm and enables corepack for package manager version management + 2. Based on the branch (main or dev), it deploys the backend Amplify app and then invokes the `neon-ci.sh` script to manage Neon database branches + +**Frontend:** + +1. Sets up Node, installs dependencies, and executes the build process for the frontend + +**Cache Configuration:** + +1. Caches the [pnpm](https://pnpm.io/) store path to enhance the speed of future builds + +### Creating a new Postgres Database in Neon + +Visit the Neon Console, sign up, and create your first project by following the prompts in the UI. Then, [create an API key](https://neon.tech/docs/manage/api-keys#create-an-api-key) for your Neon account. This key should be stored in SSM Parameter Store and accessed by your Amplify build scripts for authentication with Neon for database branching. Whether you’re working with an existing project or starting a new one, this project will host the database linked to your Amplify SSR app. When you create a new branch in the Amplify app, a corresponding database branch is automatically created in your specified Neon project. + +![Get started with Neon Postgres](https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/get-started-with-neon-1024x789-d8b0c6a9.png) + +### Add the Neon CI bash script + +Include the [Neon bash CI script](https://github.com/siegerts/neon-branches-amplify-cicd/blob/main/neon-ci.sh) in your application directory. This will be referenced during the backend build stage of the CI/CD pipeline. This script can be modified, if needed, to account for your specific use case but the primary function is to create a branch for each Amplify app branch and retrieve the connection string. + +The script has the following commands and options: + +```bash +./neon-ci.sh [options] + +Commands +create-branch: + Creates a new Neon database branch. + +options: + --app-id + --neon-project-id + --parent-branch-id + --api-key-param + --role-name + --database-name + --suspend-timeout + +cleanup-branches: + Cleans up Neon database branches that no longer + have corresponding Amplify app branches. + +options: + --app-id + --neon-project-id + --api-key-param +``` + +These commands, specifically `create-branch`, are used during the backend **build** phase of the CI/CD process. + +### Update the Amplify Hosting app backend build role with the correct policy statements + +Update the Amplify Hosting backend build role with the policies below to allow access to SSM Parameter Store and Amplify using the AWS CLI within the CI/CD build environment. The `neon-ci` bash script calls out to SSM and Amplify which will require the service role to have the correct permissions to access. + +An example of this from the bash script: + +```bash +# neon-ci.sh +... + +function set_neon_api_key { + export NEON_API_KEY=$(aws ssm get-parameter --name $NEON_API_KEY_PARAM --with-decryption --query Parameter. Value --output text) + if [[ -z "${NEON_API_KEY}" ]]; then + echo "ERROR: NEON_API_KEY is not set. Exiting." + exit 1 + fi +} + +... +``` + +To update the role, add the below inline policies to the Amplify app backend build role in AWS Identity and Access Management (IAM). Scope these resources as required by your app. + +### For SSM Parameter Store + +```bash +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowAmplifySSMCalls", + "Effect": "Allow", + "Action": [ + "ssm:GetParametersByPath", + "ssm:GetParameters", + "ssm:GetParameter" + ], + "Resource": ["arn:aws:ssm: *:*:parameter/"] + } + ] +} +``` + +### For AWS Amplify access + +```bash +[object Object] +``` + +### Enable branch auto-detection for the app to create a new Amplify app branch + +Update the applications repository settings to [enable branch auto-detection](https://docs.aws.amazon.com/amplify/latest/userguide/pattern-based-feature-branch-deployments.html). Once enabled, a new app branch will automatically create new Amplify app branches when a new Git branch is created and pushed to the repo. + +![Branch auto detection in Amplify Hosting](https://cdn.neonapi.io/public/images/pages/blog/fullstack-serverless-ci-cd-in-aws-amplify-hosting-with-postgres-database-branching/branch-auto-detection-1024x402-f24606b0.png) + +## Post build database branch clean up + +If you want to automate database branch cleanup after each build, then you can include logic to use the clean up branches function in the Neon CI bash script. The `cleanup-branches` function will pull all of your Amplify Hosting app branch references with the aws cli and then compare those with the branch names in your Neon project. + +_**Note:** Make sure to test any destructive branch actions on test apps and branches first before deploying to production CI/CD pipelines._ + +```yaml + ... + +postBuild: + commands: + - # EXAMPLE: only run the cleanup-branches command if you have tested + - | + if! [ "${AWS_BRANCH}" = "main" ] &&! [ "${AWS_BRANCH}" = "dev" ]; then + # bash neon-ci.sh cleanup-branches --app-id $AWS_APP_ID --neon-project-id --api-key-param "" + fi + +... +``` + +## Connecting to Postgres with Nuxt SSR Server Routes + +Now, you can use the Neon Serverless Driver to connect to your database. For example, using Nuxt SSR in Amplify Hosting with [server routes](https://nuxt.com/docs/guide/directory-structure/server#server-routes), you can query your database branch using a similar pattern as below: + +```javascript +// ./server/routes/api/shows.ts + +import { neon } from "@neondatabase/serverless"; +const { databaseUrl } = useRuntimeConfig(); +const sql = neon(databaseUrl!); + +export default defineEventHandler(async (event) => { + + const shows = await sql `SELECT * FROM netflix_shows limit 10`; + + return { + shows, + }; +}); +``` + +The environment variable is populated into Nuxt with `useRuntimeConfig()`. + +```javascript +// nuxt.config.js + +export default defineNuxtConfig({ + devtools: { enabled: true }, + runtimeConfig: { + // The private keys which are only available within server-side + databaseUrl: process.env.DATABASE_URL, + + // Keys within public, will be also exposed to the client-side + public: { + // + }, + }, +}); +``` + +## Best Practices and Considerations + +- Securely manage your Neon API key. Do not commit this key to git. Use a SSM Parameter Store SecureString to store and access your Neon API key securely within your build environment. +- Implement logging within your build scripts to track the creation and deletion of database branches. This will be useful for troubleshooting and auditing your CI/CD process. +- Integrate database migration scripts into your CI/CD pipeline to update the schema of your database branches automatically when necessary. [Drizzle](https://orm.drizzle.team/) and [Prisma](https://www.prisma.io/) can be integrated into your build process for this using the database branch connection string environment variable. + +## Conclusion + +Integrating Neon database branches with Amplify Hosting CI/CD provides a flexible way to automate environment isolation and database creation – in a truly serverless way. + +To get started with incorporating Serverless Postgres into your Amplify Hosting SSR apps, [sign up and try Neon for free](https://console.neon.tech/signup). Follow us on [Twitter](https://twitter.com/neondatabase) and join us in [Discord](https://neon.tech/discord) to share your experiences, suggestions, and challenges. diff --git a/content/blog/posts/funding-a1.md b/content/blog/posts/funding-a1.md new file mode 100644 index 0000000000..4f46f7be8e --- /dev/null +++ b/content/blog/posts/funding-a1.md @@ -0,0 +1,90 @@ +--- +title: Neon doubles funding to $54M +description: Neon completed Series A-1 round +excerpt: >- + Neon just completed our Series A-1 round, raising $30M to more than double our + total funding to $54.3 million. This latest injection of capital means we’re + ready to give developers the best Postgres experience in the cloud. How we’re + capitalizing on this investment With this new... +date: '2022-07-26T13:03:13' +updatedOn: '2023-05-26T09:24:06' +category: company +categories: + - company +authors: + - stas-kelvich +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/funding-a1/cover.jpg' + alt: null +isFeatured: false +seo: + title: Neon doubles funding to $54M - Neon + description: Neon completed Series A-1 round + keywords: [] + noindex: false + ogTitle: Neon doubles funding to $54M - Neon + ogDescription: >- + Neon just completed our Series A-1 round, raising $30M to more than double + our total funding to $54.3 million. This latest injection of capital means + we’re ready to give developers the best Postgres experience in the cloud. + How we’re capitalizing on this investment With this new round of funding, + we’ll grow our engineering team, build out […] + image: 'https://cdn.neonapi.io/public/images/pages/blog/funding-a1/social.jpg' +--- + +Neon just completed our Series A-1 round, raising $30M to more than double our total funding to $54.3 million. This latest injection of capital means we’re ready to give developers the best Postgres experience in the cloud. + +## How we’re capitalizing on this investment (#how-were-capitalizing-on-this-investment) + +With this [new round of funding](https://techcrunch.com/2022/07/26/neon-nabs-30m-to-build-a-scalable-cloud-service-for-postgres-databases/), we’ll grow our engineering team, build out our product roadmap, and bootstrap a GTM team. This capital ensures we can keep expanding a strong developer relations team and continue our never-ending pursuit of giving Neon users the best developer experience. + +As Postgres hackers, and systems, and cloud engineers, we know how critical a good developer experience is to building great products. Every developer needs a database, and in many cases, they’ve made their choice – Postgres. That’s why we’re taking full advantage of the opportunity this funding provides to add features to Neon during our technical preview and find new ways to support our fast-growing community. + +Looking ahead, we’re committed to growing our user base by opening new cloud regions, building team collaboration and other developer-centric features, which will improve developer productivity and create substantial cost savings. + +To ensure we’re positioned for long-term success, we continue to partner with VCs and angel investors who share our mission of simplifying the developer experience. GGV Capital led our latest round, with support from: + +- Founders Fund +- General Catalyst +- Khosla Ventures +- Elad Gil + +Our angel investors include: + +- [Nat Friedman](https://twitter.com/natfriedman) +- [Ajeet Singh](https://twitter.com/ajeets) +- [Guillermo Rauch](https://twitter.com/rauchg) +- [Wes McKinney](https://twitter.com/wesmckinn) +- [Andy Pavlo](https://mobile.twitter.com/andy_pavlo) +- [Søren Brammer Schmidt](https://twitter.com/sorenbs) +- [Mike Ovitz](https://twitter.com/michaelovitz) +- [Amjad Masad](https://twitter.com/amasad) +- [Oleg Rodzinsky](https://twitter.com/olegr) + +## Building our community (#building-our-community) + +We’ve been excited by the response from users during our technical preview. We received a large batch of sign ups after a Hacker News post was published about us prior to our beta launch. We have over 3,700 stars on Github and continue to onboard new users from our invite list daily. In a short period, the developer community has validated our belief that developers want feature-rich, cloud-based services that enhance, instead of stifle, productivity. + +## Building Neon (#building-neon) + +The company started as an incubation at Khosla Ventures, led by our CEO and Partner at Khosla Ventures, Nikita Shamgunov. The founding team had three people; myself, Nikita, and long-time Postgres hacker Heikki Linnakangas. + +In our formative months, we put most of our efforts into nailing down our architecture of separating storage and compute that enables the following DevX features: + +- Serverless: run a compute layer that scales up with traffic changes and down to zero when the database isn’t in use. +- Branching: Instantly create a branch for your test environments for each code deployment in your CI/CD pipeline. +- Time machine: reset your database state to a previous point in time without the typical wait time or costs or simply query as of some time in the past. + +We are certain that no other database service has the full feature set that Neon offers. At the same time Neon stays true to open source, Postgres and provides 100% compatibility with Postgres apps. Investors and partners are equally committed to our aggressive roadmap. This latest funding round gives us a solid foundation to pursue our ambitions to give you the best Postgres experience in the cloud. + +## Join us (#join-us) + +We’re growing our remote, global team with a number of [open roles](https://neon.tech/jobs/) in engineering, product, and operations. + +## Get started (#get-started) + +Request access to the [Neon Technical Preview](https://neon.tech/early-access/) + +Star us on [Github](https://github.com/neondatabase/neon) + +Follow us on [Twitter](https://twitter.com/neondatabase) and [LinkedIn](https://www.linkedin.com/company/neon-inc) diff --git a/content/blog/posts/gdpr-compliance-and-neon.md b/content/blog/posts/gdpr-compliance-and-neon.md new file mode 100644 index 0000000000..dbd06d3b84 --- /dev/null +++ b/content/blog/posts/gdpr-compliance-and-neon.md @@ -0,0 +1,106 @@ +--- +title: 'GDPR Compliance and Neon: Everything You Need to Know' +description: Simplify your path to GDPR compliance with Neon +excerpt: >- + At Neon, we take pride in being GDPR-compliant. We adhere to the regulation’s + strict standards, providing our customers with the tools and confidence to + manage and protect personal data effectively. In this post, we’ll help you + better understand what GDPR entails and how Neon ens... +date: '2025-01-16T18:05:52' +updatedOn: '2025-02-12T11:58:01' +category: company +categories: + - company +authors: + - hayla-westhead +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/gdpr-compliance-and-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: 'GDPR Compliance and Neon: Everything You Need to Know - Neon' + description: >- + At Neon, we take pride in being GDPR-compliant. In this post, we help you + understand what GDPR entails and how Neon ensures compliance. + keywords: [] + noindex: false + ogTitle: 'GDPR Compliance and Neon: Everything You Need to Know - Neon' + ogDescription: >- + At Neon, we take pride in being GDPR-compliant. In this post, we help you + understand what GDPR entails and how Neon ensures compliance. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/gdpr-compliance-and-neon/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/gdpr-compliance-and-neon/neon-gdpr-1024x576-6aeb67cc.jpg) + +At Neon, we take pride in being GDPR-compliant. We adhere to the regulation’s strict standards, providing our customers with the tools and confidence to manage and protect personal data effectively. In this post, we’ll help you better understand what GDPR entails and how Neon ensures compliance. + +## What is GDPR? + +The General Data Protection Regulation (GDPR) is a comprehensive data protection law enacted by the European Union (EU) to regulate how personal data of EU citizens is collected, processed, and stored. Introduced in 2018, it aims to strengthen privacy rights by ensuring individuals have greater control over their personal data while holding organizations accountable for protecting it. + +In addition to providing individuals with rights like data access, rectification, and erasure (commonly known as the “right to be forgotten”), GDPR also emphasizes accountability. Companies are required to demonstrate compliance through measures such as conducting data protection impact assessments (DPIAs), maintaining detailed records of processing activities, and appointing Data Protection Officers (DPOs) in certain cases. + +Organizations that fail to comply with GDPR may face significant penalties, including fines of up to €20 million or 4% of their global annual revenue, whichever is higher. The law applies not only to EU-based companies but also to any organization worldwide that processes the personal data of EU citizens. + +## What classifies as Personal Data under GDPR? + +Under GDPR definitions, **personal data** includes any information relating to an identified or identifiable natural person. This encompasses data that directly or indirectly identifies a person, such as: + +- Name +- Address +- Email address +- Phone number +- IP address +- Online behavior and browsing history + +## How does Neon fit into your GDPR compliance? + +**If you plan on storing Personal Data, such as customer names, email addresses, or other identifiable information in Neon’s platform, we play a crucial role in helping you maintain GDPR compliance.** Neon ensures that all data is processed and secured according to GDPR’s strict standards, providing you with a reliable and secure foundation for managing personal data. + +### Neon as a Subprocessor + +When you use Neon as your database and store Personal Data, we act as a _subprocessor_. This means we process data on behalf of your organization. + +As a subprocessor, Neon is responsible for ensuring that our systems, processes, and policies meet GDPR’s stringent requirements for data security and privacy. A [Data Processing Agreement](https://neon.tech/dpa) is used to legally document both our obligations to meet GDPR compliance. + +## A closer look at Neon’s Data Processing Agreement + +**By becoming a Neon customer, you automatically benefit from a GDPR-compliant DPA embedded in our terms of service or master service agreement.** For added peace of mind, when you become a customer, you can also download and separately sign our DPA at [neon.tech/dpa](https://neon.tech/dpa). This option allows you to maintain a formal, signed agreement for your records or internal compliance needs. + +Neon’s DPA covers the following: + +1. **Data Protection Safeguards**: Commitments to secure the data you store with Neon, including encryption, access controls, and regular security audits. +2. **Support for GDPR Rights**: Assistance in fulfilling GDPR obligations, such as responding to data subject access requests, enabling data deletion under the “right to be forgotten,” and facilitating data portability. +3. **Breach Notification Protocols**: A promise to notify you promptly in the unlikely event of a data breach involving personal data. +4. **Subprocessing Accountability**: Transparency regarding any third-party subprocessors engaged by Neon and ensuring their compliance with GDPR. More information on our subprocessors can be found here [https://neon.tech/subprocessors](https://neon.tech/subprocessors) +5. **Comprehensive GDPR Alignment**: Our DPA ensures that all data processing activities align with GDPR requirements, helping you demonstrate compliance and protect personal data effectively. +6. **Facilitating International Data Transfers**: Neon relies on the **Data Privacy Framework** to enable lawful and secure transfer of personal data. This framework ensures compliance with GDPR’s cross-border data transfer requirements, providing customers with the confidence to store and process data globally while remaining fully compliant. More information can be found in our Privacy Guide [https://neon.tech/privacy-guide](https://neon.tech/privacy-guide) +7. **Security and Accountability**: The DPA includes a legal commitment to ensure robust security measures such as encryption, access controls, and breach notification protocols, ensuring your data is protected at all times. A full and detailed analysis on our security features can be found at [https://neon.tech/docs/security/security-overview](https://neon.tech/docs/security/security-overview) +8. **Clear Roles and Responsibilities**: The DPA clearly defines the roles of Neon as the data processor and our customers as data controllers, ensuring clarity and accountability throughout the data lifecycle. + +## Which security measures does Neon implement to support GDPR compliance? + +At Neon, data security is integral to supporting your GDPR compliance. We implement measures that align with GDPR’s stringent requirements, including: + +1. **Encryption**: All personal data is encrypted both at rest and in transit using industry-standard encryption protocols to ensure its confidentiality and integrity. +2. **Access Controls**: Strict access management policies ensure that only authorized personnel can access personal data. +3. **Monitoring and Auditing**: Continuous security monitoring and regular audits verify compliance with GDPR data protection standards. +4. **Incident Response**: In the unlikely event of a data breach involving personal data, Neon follows a GDPR-compliant notification protocol, informing affected customers promptly, typically within 72 hours. + + +Neon is also **ISO 27701** certified, an international standard for privacy information management systems that supports GDPR compliance. For more details about our certifications, policies, and compliance practices, visit our [Trust Center](https://trust.neon.tech/). To learn more about the security features we implement to protect personal data, visit our [Security Overview](https://neon.tech/docs/security/security-overview). + + +## Simplify your path to GDPR compliance + +By choosing Neon, you simplify your path to GDPR compliance. Our platform is designed with robust data protection measures that align with GDPR standards, reducing the compliance burden on your organization. + +For more information on how Neon supports your compliance efforts, please visit: + +- **Trust Center:** [https://trust.neon.tech/](https://trust.neon.tech/) +- **Privacy Guide:** [https://neon.tech/privacy-guide](https://neon.tech/privacy-guide) +- **Security Overview:** [https://neon.tech/docs/security/security-overview](https://neon.tech/docs/security/security-overview) +- **Data Processing Agreement:** [https://neon.tech/dpa](https://neon.tech/dpa) diff --git a/content/blog/posts/generate-a-cms-based-on-your-neon-postgres-schema.md b/content/blog/posts/generate-a-cms-based-on-your-neon-postgres-schema.md new file mode 100644 index 0000000000..d13ff17617 --- /dev/null +++ b/content/blog/posts/generate-a-cms-based-on-your-neon-postgres-schema.md @@ -0,0 +1,89 @@ +--- +title: Generate a CMS based on your Neon Postgres schema +description: >- + Use Flashboard to turn your Postgres database into a powerful, Notion-style + admin panel +excerpt: >- + You created your app with Neon Postgres and love having all your data in it. + Now you need to manage HTML content, images, and file uploads for your app’s + data, such as products, events, profiles, etc. Your marketing team has to + manage blog posts, landing pages, documentation, FAQ... +date: '2025-05-30T17:39:32' +updatedOn: '2025-05-30T18:39:04' +category: community +categories: + - community +authors: + - felipe-freitag +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/generate-a-cms-based-on-your-neon-postgres-schema/cover.png + alt: Flashbaord and Neon +isFeatured: false +seo: + title: Generate a CMS based on your Neon Postgres schema - Neon + description: >- + Flashboard is an instant CMS for your Postgres database. Connect your Neon + database and get a full admin panel with a Notion-style editor, file + uploads, and more - no setup required. + keywords: [] + noindex: false + ogTitle: Generate a CMS based on your Neon Postgres schema - Neon + ogDescription: >- + Flashboard is an instant CMS for your Postgres database. Connect your Neon + database and get a full admin panel with a Notion-style editor, file + uploads, and more - no setup required. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/generate-a-cms-based-on-your-neon-postgres-schema/social.png +--- + +You created your app with Neon Postgres and love having all your data in it. + +Now you need to manage HTML content, images, and file uploads for your app’s data, such as products, events, profiles, etc. Your marketing team has to manage blog posts, landing pages, documentation, FAQs, and case studies. + +The built-in [Neon Console](https://neon.tech/docs/guides/tables) works nicely for simpler edits. But it falls short when you need to edit HTML or store references to uploads from your storage service. + +A popular alternative is to integrate with a headless CMS. To do so, you must design the CMS schema separately from your app’s database, learn the vendor’s DSL, and integrate with an SDK. It could take you from a few days to a few weeks to get started. From this moment on, whenever you add features to your app, you’ll need to merge data from two sources, keep them in sync, and consider the vendor’s architecture. + +And let’s be honest: the last thing you want is to add another data source and kill the joy of using Postgres for everything ☺️ + +Is there an alternative? What would it cost to create the CMS capabilities yourself? Besides authentication and authorization, you’ll need to: + +- Upload files +- Edit content + +## Edit content + +There are multiple ways to edit content. Maybe you could get away with a markdown editor. But people usually prefer to write in a rich text editor. + +Should you add one to your app? + +Adding an open-source rich text editor to your app could take weeks to months. If a basic implementation works for your use case, it might not take long, but creating a polished UI takes time. + +Before getting in the weeds implementing a rich text editor, it’s worth taking the time to find out exactly which parts you need for your app. Building a full Notion-like editor is much more expensive than a simpler one that only supports headings. + +## Upload files + +If your app already has a storage service, you can use that for your content. + +When you integrate with a storage service, such as an S3-compatible one, your server will communicate with your bucket, store a reference to the files somewhere in your database, and generate temporary signed URLs to download them when needed. That is great for your app’s files. + +But what about rich text? You can’t add abstract storage references in the middle of HTML. + +You need to create another upload strategy. When you add file uploads to your rich text editor, you need to upload those to a public bucket and store a public permanent URL in your HTML. + +You’ll end up with two upload methods. One for your app’s files, uploaded to a protected bucket, and another for rich content, uploaded to a public bucket. + +Implementing an editor and file uploads may be a lot of work, but integrating with a headless CMS is too. Consider what your business needs. The big advantage of the former is that you keep all your data in one place, and you’ll have an easier time growing and maintaining your app. + +We built Flashboard for you to **keep everything in your Neon database** and **get a great CMS UX in minutes.** + +## An instant CMS for Neon Postgres + +[Flashboard](https://www.getflashboard.com/) is an instant CMS for your database. It connects to it and immediately creates a complete admin panel, with a Notion-like HTML editor, uploads, and more. + + + +## Conclusion + +Before integrating a headless CMS, consider the long-term costs of spreading your data among multiple sources. Neon’s Postgres solution is powerful, and the benefits of adding CMS capabilities to your app can outweigh the costs. If you’d rather ship faster and have a CMS for your database without coding one yourself, give Flashboard a try! diff --git a/content/blog/posts/generate-laravel-apps-from-a-prompt.md b/content/blog/posts/generate-laravel-apps-from-a-prompt.md new file mode 100644 index 0000000000..49d7d0a883 --- /dev/null +++ b/content/blog/posts/generate-laravel-apps-from-a-prompt.md @@ -0,0 +1,85 @@ +--- +title: Generate Laravel Apps from a Prompt +description: app.build now supports Laravel via a new open-source template +excerpt: >- + At Laracon 2025, we just demoed a new way to build Laravel apps: by writing a + prompt! app.build is an open-source reference architecture for agent codegen, + able to generate full-stack web apps from natural language. It handles + everything from scaffolding the project to writing te... +date: '2025-07-29T21:22:52' +updatedOn: '2025-07-31T18:17:27' +category: ai +categories: + - ai +authors: + - andre-landgraf +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/generate-laravel-apps-from-a-prompt/cover.png + alt: null +isFeatured: true +seo: + title: Generate Laravel Apps from a Prompt - Neon + description: >- + app.build, an open-source agent that generates full-stack web apps from + natural language, can now build Laravel apps. + keywords: [] + noindex: false + ogTitle: Generate Laravel Apps from a Prompt - Neon + ogDescription: >- + app.build, an open-source agent that generates full-stack web apps from + natural language, can now build Laravel apps. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/generate-laravel-apps-from-a-prompt/cover.png +--- + +At Laracon 2025, we just demoed a new way to build Laravel apps: by writing a prompt! + +[app.build](https://www.app.build/) is an open-source reference architecture for agent codegen, able to generate full-stack web apps from natural language. It handles everything from scaffolding the project to writing tests, deploying to the cloud, and wiring up a Postgres database. It’s all on GitHub: + +[https://github.com/appdotbuild](https://github.com/appdotbuild) + +We started with React/Node templates – and as of today, we’re adding Laravel to the list! + +
+ +Image + +
We also just launched a brand new web UI – check out the open-source code for that
+
+ +## Laravel Meets Codegen + +app.build started as a research project inside [Neon](https://neon.com/) to better support platforms using AI agents and codegen tooling. Tools like Replit already use Neon branches to let agents manage Postgres databases, and app.build helps us go deeper – allowing us to explore how these agents should behave, what kind of infra they need, and how templates shape their output. + + +We’ve shared some of our learnings from building app.build on our blog. If you’re curious, start with [this post covering six principles for agent builders.](https://neon.com/blog/six-principles-for-production-ai-agents) + + +To celebrate Laracon, and with help from the Laravel team, we’ve added a new [Laravel template](https://github.com/appdotbuild/agent/tree/main/agent/laravel_agent/template) to app.build that allows you to generate apps from a prompt. This is a great way to quickly prototype apps, bootstrap a starting point, or see what’s possible with AI agents and Laravel. + +## Try it + +To try it out, install the CLI and use the Laravel template: + +```javascript +npx @app.build/cli --template=laravel +``` + +The CLI will: + +1. Ask for a prompt describing your app +2. Create a private GitHub repo on your behalf +3. Generate your Laravel app with built-in tests +4. Structure the code according to Laravel best practices +5. Output instructions to deploy on Laravel Cloud + +In fact, we used this same flow to build and deploy our [Laracon slide deck](https://github.com/appdotbuilder/laravel-react-slideshow) 🙂 + + + +Try it and tell us what you’re building by tagging [@neondatabase](https://x.com/neondatabase) on X. + +You can also contribute – we’re building app.build under the Apache 2 license, and we welcome external contributions! Feel free to send us a PR: [https://github.com/appdotbuild/agent](https://github.com/appdotbuild/agent) diff --git a/content/blog/posts/get-page-at-lsn.md b/content/blog/posts/get-page-at-lsn.md new file mode 100644 index 0000000000..850923b789 --- /dev/null +++ b/content/blog/posts/get-page-at-lsn.md @@ -0,0 +1,133 @@ +--- +title: Deep dive into Neon storage engine +description: GetPage@LSN +excerpt: >- + Neon is a single writer, multiple readers, and multi-tenant system that runs + in Kubernetes containers. The two cornerstones of Neon’s architecture are: In + a previous article, I talked about architectural decisions and the reasons + behind them. In this article, we take a closer loo... +date: '2023-03-30T12:44:57' +updatedOn: '2026-03-13T15:53:23' +category: engineering +categories: + - engineering +authors: + - heikki-linnakangas +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/get-page-at-lsn/cover.jpg' + alt: Deep dive into Neon storage engine +isFeatured: false +seo: + title: Deep dive into Neon storage engine - Neon + description: GetPage@LSN + keywords: [] + noindex: false + ogTitle: Deep dive into Neon storage engine - Neon + ogDescription: >- + Neon is a single writer, multiple readers, and multi-tenant system that runs + in Kubernetes containers. The two cornerstones of Neon’s architecture are: + In a previous article, I talked about architectural decisions and the + reasons behind them. In this article, we take a closer look at the storage + engine. Separation of Compute and Storage Traditionally, databases […] + image: 'https://cdn.neonapi.io/public/images/pages/blog/get-page-at-lsn/cover.jpg' +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/get-page-at-lsn/deep-dive-into-neon-storage-engine-1024x538-e4149a07.jpg) + +Neon is a single writer, multiple readers, and multi-tenant system that runs in Kubernetes containers. The two cornerstones of Neon’s architecture are: + +1. separation of compute and storage, and +2. non-overwriting storage. + +In a [previous article](https://neon.tech/blog/architecture-decisions-in-neon), I talked about architectural decisions and the reasons behind them. In this article, we take a closer look at the storage engine. + +## Separation of Compute and Storage + +Traditionally, databases like PostgreSQL have a single instance controlling the storage on a single node. This can be a local SSD, an EBS volume on the cloud, or something else. However, this setup becomes complex as you introduce backups, log streaming, and replicas. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/get-page-at-lsn/screenshot-2025-07-11-at-31558percente2percent80percentafpm-1024x564-987acde7.png) + +Neon splits up the system so that only one copy of your database is needed, and it lives in the Neon storage engine. This single storage system can serve the primary, and as many read-only nodes as you want. It also acts as an archive and natively supports time-travel queries and branches, which makes traditional WAL shipping and backups unnecessary. This simplifies the setup quite a lot because now you only have one storage system to worry about. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/get-page-at-lsn/screenshot-2025-07-11-at-31618percente2percent80percentafpm-1024x564-ce0e87ab.png) + +By separating compute and storage, Neon can be serverless. When a database is inactive for more than five minutes, it is shut down, and the compute node is removed. When you connect to it again, a new compute node is spun up in a Kubernetes container. This is quick because it doesn’t need to restore the data; it only needs to connect to the existing storage system. + +The separation of compute and storage also allows for independent scaling. You can add as many read-only nodes as you need and scale out the storage system independently of the compute nodes. The storage is integrated with cloud storage like Amazon S3 or Google Cloud Storage, uploading and downloading files from the cloud storage as needed. + +## Storage Engine: Under the Hood + +Internally, the storage engine is modeled as a key-value store. The key is composed of the relation-id and block number. The value is an 8KB page or a write-ahead-log (WAL) record. There is also metadata to track the sizes of tables and indexes. + +The storage engine is inspired by and shares some similarities with Log-Structured Merge (LSM) trees: + +- Just like LSM trees, the storage is non-overwriting. Files are created and deleted, but they are never updated in place. Immutable files are important to integrate with cloud storage, such as S3, which don’t allow random updates. +- The incoming WAL stream is first buffered and organized in memory. When 1GB of WAL has accumulated, it is written out to a new layer file. +- Old files are merged and compacted by creating new files and deleting old ones. + +Despite the similarities, we could not find an LSM tree implementation that fit our needs directly. The main differences are: + +1. Neon also needs to access the history, not just the latest version of each value, to allow for point-in-time recovery and branching. Many LSM implementations support snapshots, which allow you to see old versions, but snapshots are assumed to be short-lived. They are not designed to keep the history around for weeks or months. +2. The storage engine uploads all files to cloud storage, and downloads them back on demand. +3. As we will see in the later section on non-overwriting storage, we have two kinds of values: image and deltas. + +Let’s first look at the write path and the read path: + +### Write path + +In the write path, PostgreSQL writes a record to the transaction log (write-ahead log) whenever a modification is made. In Neon, this log is streamed to three safekeeper nodes, which provide durability and synchronous replication using Paxos-like consensus algorithm that was explained in a previous [article](https://neon.tech/blog/paxos). + +The safekeeper nodes then stream the logs to the pageservers, where the logs are processed, reorganized, and written to immutable files. Finally, these files are uploaded to cloud storage for long-term storage. + +Cloud storage is useful because it offers virtually bottomless storage, so you never run out of disk space. Read I/O and caching can be scaled by scaling out the pageservers. + +The non-overwriting storage format makes it easy to support copy-on-write branching. When a new branch is created, the branch is empty at first, but the incoming WAL on the branch is stored separately from the parent branch. If you access a part of the database that hasn’t been modified on the branch, the storage system fetches the data from the parent branch instead. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/get-page-at-lsn/screenshot-2025-07-11-at-31644percente2percent80percentafpm-1024x564-f9e091f9.png) + +### Read path + +The read path in the storage system allows immediate access to the most recent data, but also to historical versions of the data, without traditional backups. Every read request from a compute node to the pageserver includes a log sequence number (LSN), and the pageserver returns the data as it was at the requested LSN. + +When a read request comes in, the pageserver finds the last image of the page by the LSN, and any write-ahead-log (WAL) records on top of it, applies the WAL records if needed to reconstruct the page, and returns the materialized page to the caller. We call this function GetPage@LSN. + +The LSN is a monotonically increasing integer that identifies a particular WAL record. It can be thought of as a point in time in the database history – when you perform Point-in-Time Restore, what you really restore to is a particular LSN. The pageserver retains all the WAL in a randomly-accessible format, so it can reconstruct pages as of any LSN, allowing for time travel queries. Primary nodes always request the latest version of each page. However, you can also create a node or a branch at an older LSN, to see the data as it was at that point in time. + +## Non-Overwriting Storage + +The second cornerstone of the Neon storage system is the non-overwriting storage format. This allows Neon to replace traditional backups and work seamlessly with older versions of data. + +Neon eliminates the distinction between active data and backups, treating all data the same in the storage system. Because old data is not modified, whenever new data is digested into the system, all the history stays accessible up to a configurable limit. + +The storage system processes incoming WAL logs, reorganizes them, and writes it out to layer files. The non-overwriting storage is a good fit for cloud storage, as cloud storage doesn’t allow for random updates or reads. + +## Layer Files and Non-Overwriting Storage + +As new data is added to the system, it is organized into layer files. We have two types of layers: delta and image layers. + +- Image layer contains a snapshot of all key-value pairs in a key-range at one LSN +- Delta layer contains all changes in a key and LSN range. If a key wasn’t modified, it’s not stored. Incoming WAL is written out as delta layers. + +Image layers are created in the background to speed up access and to allow garbage collecting old data. + +To reconstruct a page version, GetPage@LSN needs to find the latest image of the page, at or before the requested LSN, and all the WAL records on top of it. The search starts at the given block number and LSN, and walks through the layers collecting the WAL records for the page. The search stops when an image of the page is found. + +To ensure that the search for any particular page doesn’t take too long, the delta and image layers are reshuffled through compaction operations in the background. And eventually, when they are no longer needed, they are garbage collected. + +The below [application](https://pgbench.vercel.app/) illustrates how the storage system manages layer files over time. The workload is pgbench, which is a simple OLTP benchmark tool that comes with PostgreSQL. The X axis represents the key space, and the Y axis represents LSN. We can see three types of objects: + +1. Delta layers (gray rectangles) +2. Image layers (purple lines) +3. Garbage collection horizon (orange line) + +The purple horizontal lines represent image layers. The pgbench benchmark has one append-only table called “history”. You can see a “gap” in the X-axis for that portion of the keyspace, where the delta layers get garbage collected away, and the system keeps only the latest image layers. Suppose you have a cold table or portion of the database, like the “history” table. In that case, the storage system eventually reaches a stable state for that portion where only a set of old image layers is retained. + + + +## Conclusion + +Neon decouples storage and compute, and introduces a new way of thinking about storage. Traditional installations require planning for the primary disk size, number of replicas, backup cadence and retention size and WAL archiving, and time required to restore from backup. + +With Neon, you don’t need to think about those things anymore. Instead, you think in terms of a retention period and branches. diff --git a/content/blog/posts/getting-started-with-claude-skills.md b/content/blog/posts/getting-started-with-claude-skills.md new file mode 100644 index 0000000000..8c0f16cab7 --- /dev/null +++ b/content/blog/posts/getting-started-with-claude-skills.md @@ -0,0 +1,175 @@ +--- +title: Getting Started with Claude Skills +description: Neon’s first set of skills are now available to install +excerpt: >- + When Anthropic introduced Claude Skills, of course we tried it right away. + This post walks through how we built, tested, and published our first set of + Claude Skills for Neon, bundled into a plugin that includes four ready-to-use + Skills and an MCP server integration. What’s a Cla... +date: '2025-10-30T16:43:28' +updatedOn: '2026-01-16T13:30:00' +category: ai +categories: + - ai +authors: + - pedro-figueiredo +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/getting-started-with-claude-skills/cover.jpg + alt: null +isFeatured: true +seo: + title: Getting Started with Claude Skills - Neon + description: >- + A hands-on introduction to Claude Skills: what they are, how they work, and + how you can start incorporating them into your Neon workflows. + keywords: [] + noindex: false + ogTitle: Getting Started with Claude Skills - Neon + ogDescription: >- + A hands-on introduction to Claude Skills: what they are, how they work, and + how you can start incorporating them into your Neon workflows. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/getting-started-with-claude-skills/social.jpg +--- + +Post image + +[When Anthropic introduced Claude Skills](https://www.anthropic.com/news/skills), of course we tried it right away. This post walks through how we built, tested, and published our first set of Claude Skills for Neon, [bundled into a plugin](https://github.com/neondatabase-labs/ai-rules) that includes four ready-to-use Skills and an MCP server integration. + +## What’s a Claude Skill Anyway? + +Truth be told, there’s nothing particularly special about a Claude Skill: it’s just a bundle of markdown files with a bunch of instructions. + +Each skill lives in its own folder, which contains an entrypoint called `SKILL.md`. You describe a workflow in plain language, usually step by step – something like “copy this file, look for references there, paste it over here.” In a few cases, like image or PDF editing, you can also specify which executables or binaries the workflow should use. But underneath, it’s still just markdown. You can look at some examples of skills that Claude Code has built-in in this [repository](https://github.com/anthropics/skills). + +```bash +--- +name: edit-images +description: Use this skill whenever the user asks to modify or transform an image. +tools: [image_editor] +--- + +1. Open the target image file. +2. Apply the requested transformation. +3. Save the modified image to the same directory. +``` + +## The Anatomy of a Claude Skill + +The most interesting aspect of Skills is not their anatomy, but rather the way in which Claude manages them. Instead of fully loading every skill’s workflow, Claude maintains only the name and description of each skill within its global context. When a specific skill is required, Claude then dynamically loads the complete workflow, ensuring the system remains efficient and resources are utilized only as needed. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/getting-started-with-claude-skills/scheme-2-1024x399-69c6ad62.png) + +So if you ask it, “can you edit this image and make it a bit taller?”, it looks at the Skill descriptions, finds one that matches (“use this skill whenever you need to edit an image”), loads the file, and follows the steps inside. + +It’s a simple idea, but powerful. Instead of constantly re-explaining the same workflow, you can define it once and let Claude decide when to apply it. + +| | `skills.md` | MCP server | Custom commands | `CLAUDE.md` / `AGENTS.md` | +| -------------------------------------------------------------------------------------------------------------- | ----------- | ---------- | --------------- | ------------------------- | +| API integration | | ✅ | | | +| Best practices / Global rules
_– Always use absolute imports_
_– Use functional components_ | | | | ✅ | +| Reusable workflows with code execution
_– Create a PR
– Do data analysis
– Format this project_ | ✅ | | ✅ | | + +At Neon, we were already using repeatable workflows (e.g. setting up Drizzle, creating databases, adding best-practice docs) so skills gave us a clean way to package all that into reusable, self-contained guides inside Claude Code. + +## Skills vs. Commands vs. Sub-agents + +When we first started experimenting with Skills, we realized they sit right between custom commands and full agents. If you look at Anthropic’s model, it’s kind of a spectrum: + +```bash + Commands → Skills → Subagents +``` + +![Image](https://cdn.neonapi.io/public/images/pages/blog/getting-started-with-claude-skills/scheme-1-1024x486-948b9cab.png) + +- A command is something like `/summarize` or `/refactor`, where every input and output is tightly controlled by the developer. +- A skill is more like a workflow, a markdown file that tells Claude how to approach a task, but still leaves room for interpretation. +- A subagent is basically another Claude instance spun up to reason independently on a problem, that has its own context window. + +Commands are deterministic: you tell Claude exactly what and when to do something, and it executes that action once. Subagents are the opposite: they’re autonomous and can make decisions on their own. Skills live somewhere in the middle. + +| | `skills.md` | Subagents | +| -------------- | ------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- | +| Model | Inherit | Customizable | +| Context window | Shared with main agent thread
Only the metadata is included by default in the context window | New context window on every invocation | +| Trigger | Model invoked (differs from custom commands) | Model invoked | +| Parallelism | Sequential (it’s just tool calls) | Can run in parallel | +| Use case | Low context size Workflows, template-driven | Independent reasoning, tasks that require a lot of context | + +So when you say “edit this image” or “connect my project to Neon,” Claude checks if it already knows a workflow that matches that description. If it finds one, it pulls in the Skill and executes the steps. That’s what makes skills interesting, they feel structured enough to be predictable, but flexible enough to adapt to context. + +| | `skills.md` | custom commands | Reasoning | +| ------------------------------------------------------ | ----------- | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Automating multi-step workflows | ✅ | | Custom commands don’t allow automating, they need user intervention | +| Bundling scripts/resources | ✅ | | They need multiple files and scripts, custom commands only allow a single file or disperse files across the project | +| Simple modifications (can be defined in a single file) | | ✅ | This can be done in a single file and usually we want them to be a one time thing | +| High risk actions | | ✅ | For “high risk” actions like deploying, it’s better not to allow direct access for the agent and have them manually triggered instead | +| Context-aware, triggered by Claude | ✅ | | Custom commands are not as deeply integrated with the CC session context as `skills.md`, since these are invoked by the model and have the required context for the task | + +## The Neon AI Rules Plugin + + +If you’re new to Neon: it’s a serverless Postgres platform with instant provisioning, autoscaling, and branching - plus a generous Free Plan. Neon powers tens of thousands of databases for developers and AI agents, with over 40,000 new databases created every day. [Take a look.](https://neon.com/) + + +Once we got comfortable with how skills worked, we wanted to see what it would look like to actually bundle them into something reusable – that’s how the Neon AI Rules plugin came together: + +[https://github.com/neondatabase-labs/ai-rules](https://github.com/neondatabase-labs/ai-rules) + +The idea was to create a small marketplace for Claude Code where we could group multiple Neon plugins, starting with just one. Inside that plugin, we included four Claude skills plus an MCP server to handle API interactions. + +```bash +/plugin marketplace add neondatabase-labs/ai-rules +``` + +The marketplace acts like a container. You can add other things to it later: + +- Commands, for quick single-step actions +- Subagents, for more autonomous tasks +- Hooks, to integrate with Claude Code itself + +For now, we kept it minimal and published a single plugin called Neon. It’s already available in Claude Code, and once you install it, you get everything preloaded – the four Neon Skills and a Neon MCP server. + +```bash +/plugin install neon-plugin@neon +``` + +The MCP (Model Context Protocol) part is important. It’s what gives Claude runtime access to Neon’s APIs, things like checking project info, creating new databases, or validating a schema connection. Every tool inside Claude Code can now talk directly to Neon through that MCP interface. + +## Installing the Neon Plugin in Claude Code + +Getting the Neon plugin running inside Claude Code is pretty straightforward. It follows the same flow you’d use for any other marketplace plugin. You open Claude Code, go through the Quick Start, and add the Neon Marketplace. From there, just install the Neon plugin and restart Claude Code. That’s it. + +Once it’s up, Claude automatically detects the Neon MCP connection, so every tool inside Claude Code can talk to Neon. You can even ask Claude, “what skills do you have access to?”, and it will list the Neon ones. All installation steps (and details on how the marketplace works) are in the [AI Rules README](https://github.com/neondatabase-labs/ai-rules#readme). + +## The Four Neon Skills + +We plan to keep expanding this, but by now, there’s four skills bundled into the plugin: + +### neon-drizzle + +This Skill explains how to connect Drizzle ORM to a Neon database. It walks through different flows depending on what you’re doing: setting up a new project, connecting an existing one, or updating a schema. The steps handle scaffolding, schema creation, and connection setup automatically. You can just ask Claude Code for something like “Integrate Neon with Drizzle” and Claude knows what to do. + +### neon-serverless + +This one focuses on integrating Neon’s serverless driver. It teaches Claude how to set up the connection string, configure environment variables, and test queries. It’s all about connecting your app to Neon’s compute/storage-separated architecture with minimal boilerplate. + +### neon-toolkit + +This is a set of workflows around Neon’s Management API – creating and managing databases, provisioning projects, or fetching connection URLs. It’s the skill you’d want if you’re building automation on top of Neon or provisioning resources dynamically inside Claude Code. + +### add-neon-knowledge + +This one covers Neon’s best practices and docs. It helps Claude include relevant documentation snippets or usage patterns in its responses. It’s the “Neon brain”, a way to give Claude contextual knowledge about how Neon is typically used, directly from our documentation and internal examples. + +## Wrap Up + +Over time, the plan is to expand the marketplace with more plugins, subagents, and specialized workflows that cover more of Neon’s developer stack. [Explore the full setup and code](https://github.com/neondatabase-labs/ai-rules) and reach us in Discord if you have any questions. + +--- + +_More good stuff:_ + +- _[Check out this Claude Code Cheatsheet](https://neon.com/blog/our-claude-code-cheatsheet)_ +- _And if you need a Postgres database for your personal projects, side gig, or startup, [give Neon a try](https://console.neon.tech/signup)_ diff --git a/content/blog/posts/handle-pii-staging-databases.md b/content/blog/posts/handle-pii-staging-databases.md new file mode 100644 index 0000000000..6593df8de9 --- /dev/null +++ b/content/blog/posts/handle-pii-staging-databases.md @@ -0,0 +1,350 @@ +--- +title: How to Handle PII in Staging Databases Without Losing Realistic Data +description: Protecting user data while preserving realistic conditions +excerpt: >- + If you’ve got real data, you’ve got a real problem. And that problem has a + name – PII. Suppose your production database contains names, addresses, + emails, and phone numbers. In fintech, you might also have credit card numbers + and transaction histories. In healthtech, medical reco... +date: '2025-11-17T19:18:17' +updatedOn: '2025-11-17T19:18:18' +category: workflows +categories: + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/handle-pii-staging-databases/cover.jpg + alt: null +isFeatured: false +seo: + title: How to Handle PII in Staging Databases Without Losing Realistic Data - Neon + description: >- + A guide to managing PII in staging via masking scripts, synthetic data, + replication, and Neon's anonymized branches. + keywords: [] + noindex: false + ogTitle: How to Handle PII in Staging Databases Without Losing Realistic Data - Neon + ogDescription: >- + A guide to managing PII in staging via masking scripts, synthetic data, + replication, and Neon's anonymized branches. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/handle-pii-staging-databases/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/handle-pii-staging-databases/neon-pii-1-1024x576-a959cf64.jpg) + +If you’ve got real data, you’ve got a real problem. And that problem has a name – PII. + +Suppose your production database contains names, addresses, emails, and phone numbers. In fintech, you might also have credit card numbers and transaction histories. In healthtech, medical records. In edtech, student data. To comply with data regulations and audits, you need to handle that information carefully, which often means restricting access to production. + +This leads to a practical operational headache for developers (on top of the safety and compliance work): you can’t safely “copy” production data into staging or test environments. Developers need staging databases that behave like production, with the same volumes, edge cases, and messy real-world patterns, but PII makes it impossible to use real data directly, so other solutions must be explored. The question isn’t just how to protect PII but **how to keep staging reliable and easy to manage without dedicating half your engineering time to data plumbing.** + +## Approach 1: Manual exports and anonymization scripts + +
+

Our take: This approach is fine for one-off exports, prototypes, or very small projects with stable schemas – but it won’t take you very far. As your data model evolves or your team grows, manual exports turn into ongoing maintenance work and won’t hold up as a reliable staging strategy.

+
+ +The simplest solution you can consider involves exporting production data to CSV files, running Python scripts to mask PII fields, and then loading the anonymized data into staging. + +```python +def anonymize_user_data(input_file, output_file): + """ + Anonymize a CSV file containing user PII + Expected columns: id, name, email, phone, ssn, created_at + """ + with open(input_file, 'r') as infile, open(output_file, 'w', newline='') as outfile: + reader = csv.DictReader(infile) + fieldnames = reader.fieldnames + writer = csv.DictWriter(outfile, fieldnames=fieldnames) + writer.writeheader() + + for row in reader: + row ['name'] = anonymize_name(row ['name']) + row ['email'] = hash_email(row ['email']) + row ['phone'] = anonymize_phone(row ['phone']) + row ['ssn'] = anonymize_ssn(row ['ssn']) + writer.writerow(row) +``` + +This approach masks sensitive fields while preserving the data structure. Email addresses are hashed deterministically, ensuring that foreign key relationships remain intact. Names get replaced with generic placeholders. Phone numbers and SSNs get randomized but maintain valid formats. + +```bash +// input + +1,Alice Johnson,alice@company.com,415-123-4567,123-45-6789,2023-01-15 +2,Bob Smith,bob@company.com,415-987-6543,987-65-4321,2023-02-20 +3,Carol White,carol@company.com,415-555-1234,456-78-9012,2023-03-10 + +// output + +1,Chris Lee,user_2353b240@example.com,555-1811,***-**-2404,2023-01-15 +2,John Doe,user_217195a2@example.com,555-8657,***-**-6404,2023-02-20 +3,Jane Smith,user_f95f10b8@example.com,555-5896,***-**-4110,2023-03-10 +``` + +The appeal of this method lies in its immediacy – you can implement it in an afternoon with basic Python knowledge. It requires no infrastructure changes and gives you complete control over the anonymization logic. For small projects with stable schemas, it works fine. + +The problems emerge over time. When someone adds a secondary_email field to the users table, your script doesn’t know about it. That field gets exported with real data unless you remember to update the anonymization script. Schema changes break your exports. Foreign key relationships can get corrupted if you’re not careful about consistent hashing. You’re manually running these scripts every time you need fresh staging data, which becomes a maintenance burden. + +## Approach 2: Data masking libraries + +
+

Our take: Masking libraries are a clear upgrade over hand-rolled scripts: more realistic data, fewer one-off functions, and better consistency. But they still leave you maintaining configs, running exports manually, and updating rules every time the schema changes. They reduce friction, but they don’t eliminate the operational burden.

+
+ +A more robust version of manual scripts uses libraries like [Faker](https://faker.readthedocs.io/en/master/) to generate realistic fake data with consistent masking rules. Instead of writing custom anonymization functions, you configure field mappings that describe which fields contain PII and how to mask them: + +```json + "field_mappings": { + "user_email": "email", + "customer_name": "name", + "billing_address": "address", + "shipping_address": "address", + "contact_phone": "phone", + "ssn": "ssn", + "credit_card_number": "credit_card", + "birth_date": "date_of_birth", + "ip_addr": "ip_address" + }, + "tables": { + "users": [ + "name", + "email", + "phone", + "ssn" + ], + "orders": [ + "shipping_address", + "billing_address" + ], + "payments": [ + "credit_card_number" + ], + "sessions": [ + "ip_addr" + ] +``` + +The masking happens through a configurable class that uses Faker’s generators to create consistent replacements: + +```python +class PIIMasker: + def __init__(self, config_file=None): + self.fake = Faker() + self.cache = {} # Cache for consistent masking + + self.masking_rules = { + "email": self._mask_email, + "name": self._mask_name, + "phone": self._mask_phone, + "address": self._mask_address + } + + def _mask_email(self, value): + if value in self.cache: + return self.cache [value] + + seed = self._get_seed_from_value(value) + self.fake.seed_instance(seed) + masked = self.fake.email() + self.cache [value] = masked + return masked +``` + +This approach improves on manual scripts in several ways. The fake data looks realistic. The same input always produces the same output, which preserves referential integrity across tables. Configuration lives in a JSON file rather than scattered through code. You can reuse the masking logic across different export jobs. + +```bash +Original: {'id': 1, 'name': 'Alice Johnson', 'email': 'alice@company.com', 'phone': '415-123-4567', 'ssn': '123-45-6789', 'created_at': '2023-01-15'} +Masked: {'id': 1, 'name': 'Craig Baker', 'email': 'cartersteven@example.net', 'phone': '370.223.8887', 'ssn': '033-50-2167', 'created_at': '2023-01-15'} +``` + +But you still face the core problems of manual exports: + +- Someone needs to update the configuration file when the schema changes +- New PII fields won’t get masked automatically unless you explicitly add them to the config +- Export and load processes remain manual + +## Approach 3: Synthetic data generation + +
+

Our take:  Synthetic data is great for dev environments, CI, demos, or cases where production data simply can’t be used. It’s safe, predictable, and easy to regenerate. But it takes real work to maintain, and it inevitably drifts from the messy, uneven patterns of real production traffic, and your staging environment will lose realism over time. 

+
+ +Some teams skip masking production data entirely and generate synthetic datasets from scratch. This approach models business logic and realistic patterns to create fake data that resembles production without containing any real PII: + +```python +class SyntheticDataGenerator: + def __init__(self, seed=42): + self.fake = Faker() + Faker.seed(seed) + + # Define realistic distributions + self.user_segments = { + "power_user": {"order_frequency": 0.3, "avg_order_value": 500}, + "regular": {"order_frequency": 0.15, "avg_order_value": 150}, + "occasional": {"order_frequency": 0.05, "avg_order_value": 80} + } + + def generate_users(self, count=100): + users = [] + for i in range(1, count + 1): + segment = random.choices( + list(self.user_segments.keys()), + weights=[20, 50, 30], + k=1 + ) [0] + + first_name = self.fake.first_name() + last_name = self.fake.last_name() + + user = { + "id": i, + "first_name": first_name, + "last_name": last_name, + "email": f"{first_name.lower()}.{last_name.lower()}@example.com", + "user_segment": segment, + "created_at": self.fake.date_time_between(start_date="-2y") + } + users.append(user) + return users +``` + +Synthetic data offers complete safety. There’s zero risk of PII exposure because no real data exists in the pipeline. The data is reproducible through seeded random generation, which helps with automated testing. You can scale the dataset to any size or deliberately create edge cases for specific tests. + +The limitation is realism. Synthetic data won’t match the quirks and patterns of production data: + +- Users behave in unexpected ways that synthetic generators can’t predict. For example, real users might abandon shopping carts in specific patterns, access the site at unusual times, or create edge cases, such as having 500 items in a wishlist. +- You might miss bugs that only appear with specific data distributions from real usage. A performance issue that surfaces when 5% of users have over 100 orders won’t show up if your synthetic generator gives everyone 2-10 orders. +- Real data has messiness that’s hard to replicate. Production databases contain inconsistent formatting, unexpected nulls, legacy data from old schema versions, and corrupted records that somehow slipped through validation. +- Performance characteristics differ because synthetic data often has cleaner patterns than messy production data. Real databases have uneven distributions, hot spots, and skewed indexes that affect query performance in ways synthetic data won’t reveal. + +## Approach 4: Database replication with masking layers + +
+

Our take: This is a powerful solution but it’s not easy to implement – it’s very heavy. Extensions like PostgreSQL Anonymizer make it far more manageable than rolling your own masking functions, but you’re stuck managing replication pipelines, extra database instances, and masking logic. The next approach shows how to get the same benefits with far less work.

+
+ +Another route is to jump into database replication with masking applied at the database level. In this approach, production data continuously replicates to staging through change data capture or logical replication. Applications connect to masked views rather than raw tables: + +```sql +-- Masking function +CREATE FUNCTION mask_email(email TEXT) RETURNS TEXT AS $$ +BEGIN + RETURN 'user_' || SUBSTRING(MD5(email) FROM 1 FOR 8) || '@example.com'; +END; +$$ LANGUAGE plpgsql IMMUTABLE; + +-- Masked view +CREATE VIEW users AS +SELECT + id, + mask_name(name) AS name, + mask_email(email) AS email, + mask_phone(phone) AS phone, + created_at +FROM staging_users; +``` + +The architecture looks like this. Production contains real PII. A replication process ([AWS DMS](https://aws.amazon.com/dms/), [Debezium](https://debezium.io/), [Postgres logical replication](https://www.postgresql.org/docs/current/logical-replication.html)) continuously syncs data to staging raw tables. Masked views sit on top of those tables, applying SQL functions that transform PII on every query. Applications connect to the views and never see raw PII. + +This solves several problems that plague manual approaches. Schema changes replicate automatically. There’s no manual export step. The masking happens at the database level, where it can’t be bypassed. Queries against staging always return masked data with no risk of someone accidentally querying raw tables. + +The trade-offs shift from maintenance burden to infrastructure complexity and cost. You need multiple database instances, a replication setup, and expertise to tune the masking functions for performance. Replication lag can cause issues where staging data is minutes behind production. + +### Production-grade masking with PostgreSQL Anonymizer + +Building custom masking functions and replication pipelines is effective, but there are tools available to help you do precisely this, as it’s so important. + +[PostgreSQL Anonymizer](https://postgresql-anonymizer.readthedocs.io/) is a production-ready extension that provides declarative PII masking. Instead of writing custom SQL functions for each field type, you declare masking rules using security labels, and the extension handles the rest: + +```sql +-- Install extension +CREATE EXTENSION anon CASCADE; +SELECT anon.init(); + +-- Declare masking rules (declarative!) +SECURITY LABEL FOR anon ON COLUMN users.email + IS 'MASKED WITH FUNCTION anon.fake_email()'; + +SECURITY LABEL FOR anon ON COLUMN users.ssn + IS 'MASKED WITH FUNCTION anon.partial(ssn, 2, ''XXX-XX-'', 4)'; + +SECURITY LABEL FOR anon ON COLUMN users.salary + IS 'MASKED WITH FUNCTION anon.noise(salary, 0.2)'; +``` + +The extension comes with over 60 built-in masking functions for common PII types: + +- `anon.fake_email()` +- `anon.fake_phone()` +- `anon.partial()` for showing only the last 4 digits +- `anon.shuffle()` to maintain distributions while breaking linkages +- `anon.noise()` to add randomness to numeric values. + +You can check [k-anonymity](https://en.wikipedia.org/wiki/K-anonymity) with `anon.k_anonymity('users', ARRAY['city', 'zip_code'])` to ensure your data meets privacy thresholds. + +PostgreSQL Anonymizer supports three masking strategies: + +- **Dynamic masking** applies transformations in real-time when queries run, leaving the original data intact. This is ideal for staging environments where you want to keep production data fresh without exposing PII. +- **Anonymous dumps** let you create masked database copies with pg_dump –anon-dump. This is perfect for spinning up new staging environments from scratch. +- **Static masking** permanently replaces data in place. You’d only use this if you really need to destroy the original data. + +The declarative approach makes maintenance easier than custom functions. When you add a new column containing PII, you add one security label rather than writing new masking functions and updating views. The extension handles consistency automatically. All masking occurs at the database level, so applications can’t bypass it, even if they attempt to do so. + +## Branching with user-defined masking + +
+

Our take: Being able to create anonymized database branches gives you all the realism of production data and the safety of database-level masking with far less work. Once your masking rules are defined, creating a staging branch becomes a near-zero-effort operation that produces an exact copy of production with all sensitive fields masked, and maintaining staging is just as simple.

+
+ +The approaches we’ve covered so far treat staging as a separate database that you must provision, populate, refresh, and clean up on your own. If you’re using [Neon](https://neon.com/), you can take a different approach, built around a concept many application developers already understand from Git – [branches](https://neon.com/docs/introduction/branching) – and applying the PostgreSQL Anonymizer philosophy presented above. + +### Branches in Neon + +A branch in Neon acts as a lightweight copy of your database at a specific point in time. Think of it like this: + +- A branch starts as an exact clone of another branch (e.g. production) +- It shares underlying storage until you modify data ([copy-on-write](https://neon.com/storage)) +- It can have its own compute (Postgres instance) +- You can create [branches for staging, testing, previews, feature environments](https://neon.com/branching)… +- They automatically scale to zero + +Instead of running multiple full-sized databases, you can spin up environments almost instantly by branching from an existing one. This foundation gives us a cleaner primitive for staging – branch from production whenever you need fresh data (or update your existing staging branch). But we need something more: the ability to mask PII when creating that branch. + +### How anonymized branches work + +Neon supports [creating an anonymized branch](https://neon.com/blog/branching-environments-anonymized-pii) – a special kind of branch where you define masking rules, and those rules are applied during branch creation to produce a fully masked copy of the production data. Under the hood, this feature is powered by PostgreSQL Anonymizer and uses the same declarative masking philosophy described in Approach 4, but without requiring you to build or maintain anything. + + +Neon does **not** detect PII automatically, and it does not mask anything without explicit user-defined rules. This is intentional for both compliance and security reasons. (Our roadmap includes improved masking suggestions, but these suggestions will still be user-initiated.) + + +The workflow looks like this: + +- **Production holds your real operational data.** Your primary branch contains your live database, untouched and unmasked. +- **You explicitly define the masking rules.** Before an anonymized branch can be created, you must define: + - which columns contain PII + - how each column should be masked (fake email, partial mask, noise, nulling, etc.) + - whether masking needs to be deterministic or random + - whether these rules should apply to all future anonymized branches +- **You create an anonymized branch from production.** When you create an anonymized branch, + - Neon clones the production branch + - Applies your masking rules using static masking _(dynamic masking on the roadmap)_ + - Produces a fully anonymized copy + - Leaves the original production data untouched +- **You branch off the anonymized branch for other non-prod environments.** Once you have a masked staging branch, you can create additional branches from it. All downstream branches inherit the masked data, not the original PII. So you get + - prod → real data + - staging → masked branch + - dev/preview/QA → masked descendants +- **Refreshing staging is just recreating the branch.** To update staging with fresher production dat, recreate the anonymized branch or reset it from production. This avoids all the manual steps of exports, scripts, replication pipelines, etc. + +## Balancing realism, complexity, cost, and safety in staging + +There’s no perfect, one-size-fits-all solution for handling PII in staging environments. Every approach comes with trade-offs across realism, complexity, cost, and long-term maintenance. The right choice depends on your team size, compliance needs, and how often your data model changes. Many teams combine multiple strategies – for example, synthetic data for everyday development and masked production data for pre-release staging. + +If you’re using Neon, [anonymized branches](https://neon.com/blog/branching-environments-anonymized-pii) give you a very simple way to keep staging environments realistic and low-maintenance by masking data during branch creation. It’s a practical alternative to running your own pipelines or scripting your own exports. + +**You can experiment with the anonymized branches [workflow](https://neon.com/docs/workflows/data-anonymization#common-workflow) via our** [Free Plan](https://console.neon.tech/signup). diff --git a/content/blog/posts/handling-auth-in-a-staging-environment.md b/content/blog/posts/handling-auth-in-a-staging-environment.md new file mode 100644 index 0000000000..f57e4245e0 --- /dev/null +++ b/content/blog/posts/handling-auth-in-a-staging-environment.md @@ -0,0 +1,281 @@ +--- +title: Handling Auth in a Staging Environment +description: 'Branch your database, branch your auth' +excerpt: >- + You just opened a PR that adds a new user role to your app. Your staging + database has the schema changes. Your preview deploy is live. But when QA + tries to test it, they hit a wall: the test users in your auth system still + have the old roles. Someone needs to manually update them... +date: '2026-01-10T00:39:46' +updatedOn: '2026-01-10T00:39:48' +category: app-platform +categories: + - app-platform + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/handling-auth-in-a-staging-environment/cover.jpg + alt: null +isFeatured: false +seo: + title: Handling Auth in a Staging Environment - Neon + description: >- + Authentication often breaks in staging and CI/CD. Learn how Neon Auth lets + auth branch with your database for production-like testing. + keywords: [] + noindex: false + ogTitle: Handling Auth in a Staging Environment - Neon + ogDescription: >- + Authentication often breaks in staging and CI/CD. Learn how Neon Auth lets + auth branch with your database for production-like testing. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/handling-auth-in-a-staging-environment/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/handling-auth-in-a-staging-environment/neon-handling-auth-1-1024x576-7ea5a0f9.jpg) + +You just opened a PR that adds a new user role to your app. Your staging database has the schema changes. Your preview deploy is live. But when QA tries to test it, they hit a wall: the test users in your auth system still have the old roles. Someone needs to manually update them. Again. + +Or worse: your staging environment shares the same auth instance as production. Now you’re testing role changes with real user accounts, hoping nothing breaks. + +Sound familiar? + +## The auth blind spot in CI/CD + +We’ve gotten really good at automating deployments. GitHub Actions spins up preview environments. Vercel deploys on every push. Your database migrations run automatically. But somehow, authentication, the system that controls who can do what in your app is still stuck in manual mode. + +Friction points abound when you need to test auth-related changes: + +1. **Shared auth**: You use Auth0, Clerk, or Supabase Auth across all environments. Your staging app points to a “staging” auth tenant, but it’s completely disconnected from your staging database. When you branch your database to test a new feature, your auth state stays behind in the shared tenant. +2. **Manual setup**: Every time someone needs to test a feature involving users, roles, or permissions, someone has to: + - Create test accounts manually + - Assign the right roles in the auth dashboard + - Share credentials with the team (in Slack, 1Password, orally) + - Clean them up later (probably never) +3. **Data inconsistency**: Your staging database has 100 users. Your auth system has 47 users. Three of them have emails that match. Nobody knows which test user is supposed to be which database record anymore. + +This is the state of auth in 2025: we branch our code, we branch our databases, we automate our deployments. But auth? Auth is still a singleton service floating somewhere in the cloud, disconnected from everything else. + +## Why this happens: the architecture mismatch + +Most auth services were designed to be centralized identity providers. One auth instance, many applications. This made sense when you had one production app and maybe one staging environment. + +But modern development doesn’t work that way anymore. We create: + +- Preview environments for every PR +- Development branches for every feature +- Staging environments that mirror production +- Test environments for CI/CD pipelines + +Each environment gets its own database. But they all share the same auth instance. + +``` +Preview Deploy #47 +├── Database: ep-preview-47.aws.neon.tech ← isolated +├── App Code: preview-47.vercel.app ← isolated +└── Auth: app.auth-provider.com ← shared with everyone + +Preview Deploy #48 +├── Database: ep-preview-48.aws.neon.tech ← isolated +├── App Code: preview-48.vercel.app ← isolated +└── Auth: app.auth-provider.com ← same instance, different mess +``` + +Your database and your code are ephemeral, but your auth system is permanent. That mismatch is where all the problems live. + +## Branching auth with your database + +At [Neon](https://neon.com/), [we rebuilt our auth](https://neon.com/blog/neon-auth-branchable-identity-in-your-database) from the ground up with a different premise: authentication data is just data. And if you can branch your database, you should be able to branch your auth. + +Here’s how it works. + +### 1. Auth lives in your database + +[Neon Auth](https://neon.com/docs/auth/overview) stores all authentication data – users, sessions, OAuth tokens – directly in your Postgres database, in a dedicated neon_auth schema. + +```sql +-- Your auth data is just Postgres tables +SELECT id, email, role, "createdAt" +FROM neon_auth.user +WHERE email LIKE '%@yourcompany.com'; +``` + +This saves you from syncing the state with a separate service and remove the guesswork of if your database and your auth system agree on who user abc-123 is. They’re the same data store. + +### 2. Branching copies everything + +When you create a database branch, the [entire neon_auth schema branches](https://neon.com/docs/auth/branching-authentication) with it. Every user, every role, every session, and every OAuth configuration – copied at the point of branching. + +```bash +# Create a branch to test the new role system +neon branches create --name test-admin-role --parent main +``` + +``` +Production (main) Branch (test-admin-role) +├── neon_auth.user → ├── neon_auth.user (copied) +├── neon_auth.session → ├── neon_auth.session (copied) +├── ... → ├── ... (copied) +└── Your app data → └── Your app data (copied) +``` + +After branching, they’re completely isolated. Changes to test users in your branch don’t affect production. New roles you create in testing don’t leak into the production auth system. + +Each branch gets its own auth endpoint, for example: + +| Branch type | Endpoint | +| ----------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| Production | [https://ep-main-123.neonauth.us-east-2.aws.neon.tech/neondb/auth](https://ep-main-123.neonauth.us-east-2.aws.neon.tech/neondb/auth) | +| Test | [https://ep-test-admin-role.neonauth.us-east-2.aws.neon.tech/neondb/auth](https://ep-test-admin-role.neonauth.us-east-2.aws.neon.tech/neondb/auth) | + +## What does this look like in practice? + +Say you are tasked with adding a new “moderator” role to the application that can approve posts but can’t delete users. + +So, your current workflow of testing such a new release might look like this: + +``` +Push code to staging + +[Manual] Run database migrations in staging + +[Manual] Open external auth provider dashboard in another tab + +[Manual] Create a test user named “test-moderator-1” + +[Manual] Manually assign the new “moderator” role + +[Manual] Share credentials with QA in Slack + +[Manual] QA tests the feature + +Find a bug, fix it, push again + +[Manual] Database migrations run again, data might change + +[Manual] Auth state? Still has the old test user from step 4 + +[Manual] Update the test user’s state to match + +Test and repeat the cycle again + +Test passes? Merge. Delete branch. +``` + +But with Neon Auth, the workflow would change to being more automated: + +``` +Push code with neon.branch=true (an example flag) in your CI config to trigger branching workflows + +[Automatic] Database branch created automatically with migrations applied + +[Automatic] Auth branches automatically with it + +[Automatic] Your test suite creates a test user via the SDK + +[Automatic] Test runs with the exact production-like setup + +Test fails? Push a fix. New branch. Try again. + +Test passes? Merge. Delete branch. + +Done +``` + + +You might have noticed the mention of an SDK in the steps above. We're referring to the [Neon JavaScript SDK](https://neon.com/docs/reference/javascript-sdk) (`@neondatabase/neon-js`), which gives you [Neon Auth](https://neon.com/docs/auth/overview) together with a [PostgREST-compatible API.](https://neon.com/docs/data-api/get-started) + + +### Testing with production-like data + +Your production database has 10,000 users with various roles: admin, editor, viewer. You need to test a new permission system. + +Create a branch from production. You immediately have 10,000 test users with all their existing roles and permissions. Your staging environment now has the same distribution of user types as production. + +Instead of testing against artificial users like “testuser1” with idealized roles, you can now validate your changes using a complete copy of your actual production data. This means your test environment includes the same usernames, roles, and real-world data variations as production, allowing you to catch edge cases and bugs that would be impossible to spot with simple, unrealistic test accounts. + +```sql +-- In your test branch, query actual user distribution +SELECT role, COUNT(*) +FROM neon_auth.user +GROUP BY role; + +-- Results match production exactly: +-- admin: 3 +-- editor: 45 +-- viewer: 9,952 +``` + +When your auth system branches with your database, your staging environments finally become true replicas of production. The above enables you to test: + +- Role changes without affecting production users +- OAuth provider additions without risking existing logins +- Permission system rewrites against production-scale user counts +- Password reset flows with real tokens +- Multi-factor auth setup with actual verification codes + +When testing is finished, you [simply delete the branch](https://neon.com/docs/reference/cli-branches#delete). All 10,000 test users that were copied from production are instantly and completely removed and requires no manual cleanup. This means there’s no leftover test data, and more importantly, no sensitive information or personally identifiable information (PII) lingering in a staging environment that someone might forget about. + +Your test data is tightly scoped to the lifetime of the branch, ensuring a secure and clean environment every time you develop and test new features. + +## How it works under the hood + + + +Neon Auth is built on [Better Auth](https://www.better-auth.com/), but hosted and managed by Neon. When you enable auth on a Neon project: + +1. The neon_auth schema is created in your database +2. Better Auth’s tables are provisioned (user, account, session, verification, etc.) +3. An auth endpoint is created for your branch: `https://ep-.neonauth..aws.neon.tech/neondb/auth` +4. You configure settings in the Neon Console (OAuth providers, Email provider, Domains, etc.) +5. Your app uses the Neon SDK to interact with auth + +```typescript +// File: src/auth.ts +import { createAuthClient } from '@neondatabase/neon-js/auth'; + +// Each branch has its own auth endpoint +const authClient = createAuthClient(process.env.NEON_AUTH_URL); + +// File: src/App.tsx +import { NeonAuthUIProvider, AuthView } from '@neondatabase/neon-js/auth/react/ui'; + +import { authClient } from './auth'; +export default function App() { + return ( + + + + ); +} +``` + +When you branch the database, the neon_auth schema is copied with copy-on-write semantics, i.e., it is the same as your application data. The branch gets a new auth endpoint. Your preview environment points to the new endpoint. Everything is isolated. + +Because auth lives entirely inside your Postgres database, there’s no need to interact with third-party auth APIs, set up webhooks to synchronize user state, or perform manual imports of user records. When you create a branch, Postgres simply copies all your data including users, sessions, and configuration (using its native snapshot and [copy-on-write](https://neon.com/blog/instantly-copy-tb-size-datasets-the-magic-of-copy-on-write) mechanisms). This approach eliminates external dependencies and keeps everything in sync automatically. + +## Get started with Neon Auth (for free) + +Neon Auth is available now for all Neon projects on AWS regions, [including the Free plan](https://neon.com/pricing): + +1. Follow our [quick start guide](https://neon.com/docs/auth/overview) to enable Neon Auth in your project and integrate it with your app using the Neon SDK. We have framework-specific guides for Next.js, React, and TanStack Router. +2. Before launching, review the [Auth production checklist](https://neon.com/docs/auth/production-checklist) +3. You can use our [GitHub Action](https://neon.com/docs/changelog/2026-01-09#github-action-support-for-neon-auth-and-data-api) to provision a branch with its own auth endpoint, point your preview environment to it, and test with production-like data + +```yaml +- name: Create Neon Branch + uses: neondatabase/create-branch-action@v6 + id: create-branch + with: + project_id: ${{ vars.NEON_PROJECT_ID }} + branch_name: feature-branch + api_key: ${{ secrets.NEON_API_KEY }} + get_auth_url: true + get_data_api_url: true +- name: Use outputs + run: | + echo "Auth URL: ${{ steps.create-branch.outputs.auth_url }}" + echo "Data API URL: ${{ steps.create-branch.outputs.data_api_url }}" +``` diff --git a/content/blog/posts/handling-time-variant-dags-with-constraints-in-postgres.md b/content/blog/posts/handling-time-variant-dags-with-constraints-in-postgres.md new file mode 100644 index 0000000000..8f272109ff --- /dev/null +++ b/content/blog/posts/handling-time-variant-dags-with-constraints-in-postgres.md @@ -0,0 +1,264 @@ +--- +title: Handling Time-Variant DAGs with Constraints in Postgres +description: Using Postgres to manage fleet relationships in real-time +excerpt: >- + Managing dynamically changing directed acyclic graphs (DAGs) with constraints + in Postgres enables robust tracking of relationships that evolve over time. At + traconiq, this pattern is central to how we manage our vehicle fleet, where + trucks and trailers attach and detach constantl... +date: '2025-10-20T16:34:49' +updatedOn: '2025-10-20T17:11:33' +category: postgres +categories: + - postgres + - community +authors: + - thorsten-ries +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/handling-time-variant-dags-with-constraints-in-postgres/cover.jpg + alt: null +isFeatured: true +seo: + title: Handling Time-Variant DAGs with Constraints in Postgres - Neon + description: >- + Learn how traconiq uses Postgres on Neon to manage real-time fleet + relationships, prevent graph cycles, and resolve data conflicts with SQL. + keywords: [] + noindex: false + ogTitle: Handling Time-Variant DAGs with Constraints in Postgres - Neon + ogDescription: >- + Learn how traconiq uses Postgres on Neon to manage real-time fleet + relationships, prevent graph cycles, and resolve data conflicts with SQL. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/handling-time-variant-dags-with-constraints-in-postgres/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/handling-time-variant-dags-with-constraints-in-postgres/neon-handling-time-1024x576-a27f46d0.jpg) + +Managing dynamically changing directed acyclic graphs (DAGs) with constraints in Postgres enables robust tracking of relationships that evolve over time. At [traconiq](https://traconiq.ch/de/traconiq-ihre-zentrale-plattform-fuer-den-gesamten-fuhrpark/), this pattern is central to how we manage our vehicle fleet, where trucks and trailers attach and detach constantly, and where data can come from multiple systems with different levels of authority. + +We use [Neon](https://neon.com/) serverless Postgres as our operational database layer. Over time, we developed a schema and logic pattern that allows us to: + +- Enforce DAG integrity (no cycles, single incoming edges) +- Handle time-variant relationships +- Resolve conflicting data from multiple sources automatically + +In this post, we’ll share how we implemented this in Postgres. **The full code is available open source at [traconiq/dags-neon](https://github.com/traconiq/dags-neon).** + +## The Use Case: Tracking Fleet Attachments + + +traconiq manages large-scale fleet operations and real-time logistics data on Neon. Learn more about their use case (and why they picked Neon) [here](https://neon.com/blog/why-traconiq-migrated-from-aws-rds-to-neon). + + +In our fleet-management platform, every vehicle acts as a node in a graph, and each attachment (where one vehicle pulls another) is represented as a directed edge from the pulling vehicle to the pulled vehicle. This setup enables us to analyze chains of vehicles, resolve conflicting attachment data, and maintain a time-aware history of all fleet configurations. + +Our requirements are simple to describe but complex to enforce: + +- **Multiple outgoing edges per vehicle:** a truck can pull several trailers at once. +- **Only one incoming edge per vehicle per time interval:** each trailer can only be attached to one pulling vehicle at a time. +- **No cycles anywhere in the graph:** a vehicle cannot (directly or indirectly) end up pulling itself. +- **Conflicting information must be resolved by priority:** data comes from different systems: manual input, GPS, or automated sensors. Each has a defined authority level. + +Here’s how we model this in Postgres. + +## Implementing Time-Variant DAGs in Postgres + +### Schema design + +To represent vehicle attachment history precisely, each edge in the graph stores not only its source and target nodes, but also its validity interval and a priority level used for conflict resolution. + +The resulting table, `dags.temporal_edges`, captures the time-varying structure of the fleet and enforces basic data integrity directly at the database layer: + +```sql +CREATE TABLE dags.temporal_edges ( + id serial PRIMARY KEY, + source text NOT NULL, -- pulling vehicle id + target text NOT NULL, -- pulled vehicle id + valid_from timestamptz NOT NULL, -- start of attachment + valid_to timestamptz, -- end of attachment, null = ongoing + priority int NOT NULL, -- source authority for conflict resolution + constraint vehicle2vehicle_ck + check (source <> target), + constraint vehicle2vehicle_valid_ck + check ((valid_from < valid_to) OR (valid_to IS NULL)) +); +``` + +This schema leverages Postgres’s native temporal types and constraint system, allowing us to define evolving DAGs with minimal application-level logic. + +The basic constraints prevent impossible states: + +- a vehicle cannot attach to itself (`source <> target`), and +- validity intervals must make sense (`valid_from` must precede `valid_to`, or `valid_to` can be `NULL` for ongoing connections). + +These guarantees ensure that every edge always represents a valid and logically consistent attachment. + +For background reading on data types and constraints, see the [Postgres documentation on temporal types and constraints](https://www.postgresql.org/docs/current/datatype-datetime.html). + +### Enforcing graph constraints with deferred triggers + +To maintain consistency across the entire graph, two key conditions must always hold true: + +1. **No cycles**: inserting or updating edges must never create circular dependencies. +2. **Only one incoming edge per node at any time:** a vehicle can only be pulled by one other vehicle during any overlapping time interval. + +Both constraints are enforced using deferred constraint triggers. These triggers run automatically after each insert or update, but only at transaction commit, allowing multi-step updates or batch inserts to proceed safely before validation. + +```sql +-- Trigger to detect and prevent cycles in the graph +CREATE CONSTRAINT TRIGGER trigger_detect_cycle +AFTER INSERT OR UPDATE ON temporal_edges +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE PROCEDURE detect_cycle(); + +-- Trigger to enforce the single incoming edge per node constraint +CREATE CONSTRAINT TRIGGER trigger_check_single_incoming +AFTER INSERT OR UPDATE ON temporal_edges +DEFERRABLE INITIALLY DEFERRED +FOR EACH ROW +EXECUTE PROCEDURE check_single_incoming_edge(); +``` + +These triggers call the corresponding PL/pgSQL functions, `detect_cycle()` and `check_single_incoming_edge()`, which raise exceptions if violations are detected. The deferred execution model ensures that even complex transactions involving multiple inserts or updates will only commit if all constraints remain valid. + +This approach provides both data integrity and transactional flexibility. This is essential for managing evolving DAGs in a live fleet system, where many updates may happen in parallel or from different sources. + +To simplify the process of adding new edges while respecting priorities and validity intervals, we provide a helper function: + +```sql +add_edge(source varchar, target varchar, valid_from timestamptz, valid_to timestamptz, priority integer) +``` + +This function automatically adjusts existing intervals when overlaps occur, either truncating, splitting, or deleting edges as needed, depending on their assigned priorities. + +[The full function implementations are available in the repo.](https://github.com/traconiq/dags-neon) + +### Adding edges and resolving conflicts + +Once the constraints and triggers are in place, we can safely insert new edges while relying on Postgres to enforce data consistency and authority rules. Let’s look at concrete examples of how the system resolves overlapping or conflicting attachments. + +#### Example 1: Adding a higher-priority edge + +Suppose our table currently contains one confirmed attachment: + +| source | target | valid_from | valid_to | priority | +| ------ | ------ | ------------------- | ------------------- | -------- | +| V001 | V002 | 2025-10-06 08:00:00 | 2025-10-06 12:00:00 | 50 | + +Now, we attempt to add a new edge with higher authority: + +| source | target | valid_from | valid_to | priority | +| ------ | ------ | ------------------- | ------------------- | -------- | +| V003 | V002 | 2025-10-06 10:00:00 | 2025-10-06 13:00:00 | 70 | + +- Conflict window: Between 10:00 and 12:00, both edges claim V002 as a target. +- Resolution: The new edge (priority 70) overrides the previous one (priority 50) for the overlapping period. + +And the system adjusts the existing records automatically: + +| source | target | valid_from | valid_to | priority | +| ------ | ------ | ------------------- | ------------------- | -------- | +| V001 | V002 | 2025-10-06 08:00:00 | 2025-10-06 10:00:00 | 50 | +| V003 | V002 | 2025-10-06 10:00:00 | 2025-10-06 13:00:00 | 70 | + +
The `add_edge()` function truncated the earlier record, ensuring no overlap while preserving full temporal history. + +#### Example 2: Adding a lower-priority edge + +Now suppose a new edge arrives with lower authority: + +| source | target | valid_from | valid_to | priority | +| ------ | ------ | ------------------- | ------------------- | -------- | +| V004 | V002 | 2025-10-06 11:00:00 | 2025-10-06 14:00:00 | 30 | + +- Conflict window: Between 11:00 and 13:00, this overlaps with the higher-priority edge (V003 → V002) +- Resolution: The lower-priority edge is truncated, only taking effect after 13:00. + +The final state ensures that the “one incoming edge per node” rule is always respected, with higher-priority data taking precedence: + +| source | target | valid_from | valid_to | priority | +| ------ | ------ | ------------------- | ------------------- | -------- | +| V001 | V002 | 2025-10-06 08:00:00 | 2025-10-06 10:00:00 | 50 | +| V003 | V002 | 2025-10-06 10:00:00 | 2025-10-06 13:00:00 | 70 | +| V004 | V002 | 2025-10-06 13:00:00 | 2025-10-06 14:00:00 | 30 | + +### Querying and traversing the graph + +Once data integrity and conflict resolution are handled, querying the DAG becomes straightforward. Postgres’ SQL syntax and recursive CTEs make it possible to explore both current and historical attachment states directly within the database. + +Below are a few example queries from our implementation: + +#### Find current attachments at a given time + +This query returns all active edges for the specified time window, giving a snapshot of the fleet’s structure at that moment: + +```sql +SELECT source, target +FROM dags.temporal_edges +WHERE valid_from <= '2025-10-06 11:00:00+00' + AND (valid_to IS NULL OR valid_to > '2025-10-06 11:00:00+00'); +``` + +#### Find what a vehicle is pulling + +To list all vehicles being pulled by a specific truck at a given time (e.g. for monitoring live operations): + +```sql +SELECT target +FROM dags.temporal_edges +WHERE source = 'V004' + AND valid_from <= '2025-10-06 11:30:00+00' + AND (valid_to IS NULL OR valid_to > '2025-10-06 11:30:00+00'); +``` + +#### Detect illegal multiple attachments + +As a safety check, this query identifies any vehicles that erroneously have more than one incoming edge (which should never happen if the triggers are active): + +```sql +SELECT target, COUNT(*) +FROM dags.temporal_edges +WHERE valid_from <= '2025-10-07 12:00:00+00' + AND (valid_to IS NULL OR valid_to > '2025-10-07 12:00:00+00') +GROUP BY target +HAVING COUNT(*) > 1; +``` + +#### Recursively explore the fleet DAG + +A recursive CTE can be used to traverse the entire DAG starting from a pulling vehicle. This query collects all downstream attachments, direct and indirect, active at a given time. It’s a great example of how SQL recursion and temporal filtering can combine to represent dynamic graphs in Postgres: + +```sql +WITH RECURSIVE fleet_dag AS ( + -- Start with the given pulling vehicle at the reference time + SELECT source, target + FROM dags.temporal_edges + WHERE source = 'V001' + AND valid_from <= '2025-10-08 12:00:00+00' + AND (valid_to IS NULL OR valid_to > '2025-10-08 12:00:00+00') + + UNION ALL + + -- Recursively find children attached to each target vehicle in the chain + SELECT e.source, e.target + FROM dags.temporal_edges e + JOIN fleet_dag fd ON e.source = fd.target + WHERE e.valid_from <= '2025-10-08 12:00:00+00' + AND (e.valid_to IS NULL OR e.valid_to > '2025-10-08 12:00:00+00') +) +SELECT * FROM fleet_dag; +``` + +## Why This Works in Postgres (and Neon) + +Modeling time-variant DAGs directly in Postgres provides a number of advantages for systems like ours: + +- Temporal tracking allows complete historical and real-time visibility into vehicle relationships, enabling both live monitoring and retrospective analysis +- PL/pgSQL logic inside functions like `add_edge()` automates conflict resolution and interval updates while maintaining full transactional integrity +- Postgres’ built-in features like range types, recursive queries, and deferred constraints offer the tools needed to manage acyclic, time-varying graphs efficiently at scale +- [Neon’s architecture](https://neon.com/docs/get-started/why-neon) makes this practical in production: serverless compute scales down during idle periods and back up during bursts, while branching supports safe testing of constraint logic or schema evolution without impacting production data
+ +This combination of Postgres features and Neon’s infra provides a clean, reliable pattern for modeling evolving relationships, enforcing data consistency, and maintaining performance as datasets grow. **If you have a similar use case want to explore or adapt this implementation, the complete code example is available here: [https://github.com/traconiq/dags-neon](https://github.com/traconiq/dags-neon)** diff --git a/content/blog/posts/hello-world.md b/content/blog/posts/hello-world.md new file mode 100644 index 0000000000..a3b312f801 --- /dev/null +++ b/content/blog/posts/hello-world.md @@ -0,0 +1,92 @@ +--- +title: "SELECT ’Hello, World’" +description: Serverless Postgres built for the cloud +excerpt: >- + We have just launched Neon to provide you with the best Postgres experience in + the cloud. You can sign up to our waitlist right now and experience serverless + Postgres enabled by the separation of storage and compute. The service is + still gated by the waitlist as we are onboarding... +date: "2022-06-15T11:47:00" +updatedOn: "2025-09-08T10:17:45" +category: company +categories: + - company +authors: + - nikita-shamgunov +cover: + image: "https://cdn.neonapi.io/public/images/pages/blog/hello-world/cover.jpg" + alt: null +isFeatured: false +seo: + title: "SELECT ’Hello, World’ - Neon" + description: Serverless Postgres built for the cloud + keywords: [] + noindex: false + ogTitle: "SELECT ’Hello, World’ - Neon" + ogDescription: >- + We have just launched Neon to provide you with the best Postgres experience + in the cloud. You can sign up to our waitlist right now and experience + serverless Postgres enabled by the separation of storage and compute. The + service is still gated by the waitlist as we are onboarding more and more + users every day, and we […] + image: "https://cdn.neonapi.io/public/images/pages/blog/hello-world/social.jpg" +--- + +We have just launched Neon to provide you with the best Postgres experience in the cloud. You can [sign up to our waitlist](https://neon.tech/early-access/) right now and experience serverless Postgres enabled by the separation of storage and compute. The service is still gated by the waitlist as we are onboarding more and more users every day, and we expect to open it up for everyone soon. + +Our website [showed up on HackerNews](https://news.ycombinator.com/item?id=31536827) a few weeks ago, sooner than we expected and before we officially launched the company. While there was a great deal of discussion there, it’s important for us to talk in detail about the motivation for the project. + +## Why Neon? (#why-neon) + +As we are looking at the world it’s clear to us that database workloads are shifting into the cloud — no one wants to manage a database themselves. We are a team of Postgres hackers, systems and cloud engineers and we are all huge fans of Postgres. We believe that in the ever-changing technology stack Postgres is here to stay. Just like Linux operating system or Git version control Postgres is the default choice for a relational database system. + +That’s why lots of platforms such as AWS, Azure, Google Cloud Platform, Digital Ocean, Heroku, as well as newcomers like Fly.io (we are big fans) offer Postgres as a service. + +## Built for the Cloud (#built-for-the-cloud) + +If you look at the architecture of Postgres deployments, be that self-hosted, managed or cloud, often it looks like this: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/hello-world/typicalpostgressetups-8153773d.png) + +As the diagram illustrates, in order to get more throughput or storage, a user has to migrate to bigger host machines, and the service provider needs to explicitly manage the migration procedure to avoid downtimes. Due to the monolith architecture of Postgres, you end up overprovisioning either storage or compute which also has a direct implication on the cost of running the service in the cloud. Another problem is that since you have to handle failover you are paying triple for storage and compute. Amazon EBS volumes provide you with yet another level of redundancy, but they also charge for it. EBS volumes are very expensive, they throttle your throughput unless you pay for provisioned IOPs in which case your costs balloon even further. + +We realized that a modern Postgres service can be designed differently in order to be cheaper and more efficient in cloud environments, but it will require some real systems engineering. We call this approach **separation of storage and compute**. It allows us to architect the service around performance, reliability, manageability, and cost. Cost is particularly important when you design a system for the cloud. Any cloud service has an infrastructure bill that it has to pass on to the end user. If you don’t account for cost at the architecture level running a service can get very expensive. That’s why when you build for the cloud you have to make the cost of running the service an important design consideration on par with manageability, reliability, and performance. One of the immediate implications of designing for cost was to never use EBS volumes and use a combination of local storage and S3 instead. Local storage for hot and S3 for cold data. + +We built our distributed, multi-tenant storage from the ground up. It integrates into Postgres without the need of forking Postgres itself (it does require small changes in the engine that we are aiming to commit upstream). You can read more on the details of this architecture in our next posts. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/hello-world/neonarchitecture-1024x576-55c8625c.png) + +Each user gets a dedicated Postgres that we run in a container, while the data is safely stored in our multi-tenant storage system written in Rust. Storage consists of two services: Safekeepers and Pageservers. Safekeepers implement a consensus protocol. The combination of Safekeepers and S3 provide the system of record. Pageservers serve database pages with low latency and provide “scratch space” for updates. Pageservers are not part of the system of record – you can lose all the pageservers and won’t lose any data. + +## Architecture Benefits (#architecture-benefits) + +This architecture provides a surprising number of benefits. Compared to the “naive” approach the most obvious are cost and elasticity of storage (bottomless). Compute and storage are separated, so you don’t overprovision either. Storage is backed up by a reliable and cost effective S3, so we do not have to triple the cost of storing the data. Safekeepers only have to store a small window of the most recent data which is not yet saved in S3. Moreover, Safekeepers and Pageservers can be shared between users, so the end user does not pay triple for the compute either. Backups and restores are integrated seamlessly as they are all built into the storage architecture. And most importantly this design allows us to deliver on a better developer experience (DevX), with elasticity, branching, and time machine. All these are enabled by our ability to transparently serve a Postgres instance any version of any page from our storage system. + +## Serverless (#serverless) + +Since storage is separate, compute, which is a Postgres process, becomes stateless (barring the buffer cache). This allows dynamically rescheduling compute and moving it from one node to another. And this opens up the possibility to run a compute layer that scales in response to changes in traffic, including the scale down to 0 when the database is not in use. You don’t need to specify the size of the compute instance. You push a button and get Postgres in under 3 seconds. Your only interface to it is a connection string and we will handle the rest. + +Here is an example of spinning up Postgres in 3 seconds. + +
+ +
+ +## Branching (#branching) + +Today everyone who is following modern app development practices needs to easily create dev, stage, and test environments. Modern developer workflow includes GitLab CI/CD, GitHub actions and other various CI/CD tools. Developers send pull requests and have platforms like [Vercel](https://vercel.com/) to compile, test, and deploy their code. Databases today don’t support this workflow very well and the key missing feature for this is branching – the ability to create a branch of the whole database similar to how Git does it. + +Neon allows instantly branching your Postgres database to support a modern development workflow. You can create a branch for your test environments for every code deployment in your CI/CD pipeline. You can test migrations against a recent production snapshot without affecting the production. Branches are created instantly and implemented using the “copy on write” technique. In the current version, you have to contact us separately if you want to try branching after receiving an invitation. We will open it up for everyone as we build the UI support. + +## Open Source (#open-source) + +We are not the first to offer separation of storage and compute for Postgres. AWS Aurora is probably the most famous example, however it is proprietary and tied to AWS’s internal infrastructure. + +We think we have an opportunity to define the standard for cloud Postgres. We carefully designed our storage focusing on cloud independence, performance, manageability, DevX, and cost. We chose the most permissive open source license: Apache 2.0 and invited the world to participate. You can already build and run your own fully-featured instance of Neon. + +## Hello, World (#hello-world) + +Please welcome Neon to the world. Follow @neondatabase on [Twitter](https://twitter.com/Neondatabase) or [GitHub](https://github.com/neondatabase/) and be on the lookout for exciting new features we will be continuously shipping. diff --git a/content/blog/posts/higher-level-crud-abstraction-for-postgres-rls.md b/content/blog/posts/higher-level-crud-abstraction-for-postgres-rls.md new file mode 100644 index 0000000000..3f6d9831cc --- /dev/null +++ b/content/blog/posts/higher-level-crud-abstraction-for-postgres-rls.md @@ -0,0 +1,122 @@ +--- +title: Higher-level CRUD Abstraction for Postgres RLS +description: Simplify row-level security via a concise API +excerpt: >- + Postgres Row-Level Security is notoriously difficult to comprehend and use. + The policy access model for a todos table in a Todo List app can be declared + as per the following table: Operation Using(applies to data being returned) + Wich Check(applies to data being written) Select au... +date: '2024-11-13T17:09:49' +updatedOn: '2025-09-11T18:56:13' +category: postgres +categories: + - postgres +authors: + - david-gomes +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/higher-level-crud-abstraction-for-postgres-rls/cover.jpg + alt: null +isFeatured: false +seo: + title: Higher-level CRUD Abstraction for Postgres RLS - Neon + description: Simplify row-level security via a concise API + keywords: [] + noindex: false + ogTitle: Higher-level CRUD Abstraction for Postgres RLS - Neon + ogDescription: >- + Postgres Row-Level Security is notoriously difficult to comprehend and use. + The policy access model for a todos table in a Todo List app can be declared + as per the following table: Operation Using(applies to data being returned) + Wich Check(applies to data being written) Select auth.user_id() = user_id + Insert – auth.user_id() = user_id Update auth.user_id() = […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/higher-level-crud-abstraction-for-postgres-rls/social.jpg +--- + +Post image + + +We’ve moved the functionality previously known as Neon RLS / Neon Authorize into the Neon Data API. You can [read more about the Data API here](https://neon.com/docs/data-api/get-started) and start using it in your projects today. + + +[Postgres Row-Level Security](https://neon.tech/postgresql/postgresql-administration/postgresql-row-level-security) is notoriously difficult to comprehend and use. The policy access model for a `todos` table in a Todo List app can be declared as per the following table: + +| Operation | Using
(applies to data being returned) | Wich Check
(applies to data being written) | +| --------- | ------------------------------------------- | ----------------------------------------------- | +| Select | `auth.user_id() = user_id` | | +| Insert | - | `auth.user_id() = user_id` | +| Update | `auth.user_id() = user_id` | `auth.user_id() = user_id` | +| Delete | - | `auth.user_id() = user_id` | + +And the SQL code would look like this: + +```sql +CREATE TABLE IF NOT EXISTS "todos" ( + "id" bigint PRIMARY KEY, + "user_id" text DEFAULT (auth.user_id()) NOT NULL, + "task" text NOT NULL, + "is_complete" boolean DEFAULT false NOT NULL, + "inserted_at" timestamp with time zone DEFAULT now() NOT NULL +); + +ALTER TABLE "todos" ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "create todos" ON "todos" AS PERMISSIVE FOR INSERT TO "authenticated" WITH CHECK ((select auth.user_id() = user_id)); + +CREATE POLICY "view todos" ON "todos" AS PERMISSIVE FOR SELECT TO "authenticated" USING ((select auth.user_id() = user_id)); + +CREATE POLICY "crud-authenticated-policy-update" ON "todos" AS PERMISSIVE FOR UPDATE TO "authenticated" USING ((select auth.user_id() = "todos"."user_id")) WITH CHECK ((select auth.user_id() = "todos"."user_id")); + +CREATE POLICY "delete todos" ON "todos" AS PERMISSIVE FOR DELETE TO "authenticated" USING ((select auth.user_id() = user_id)); +``` + +This is an extremely low-level API. And for most applications and data models, it will get very repetitive. So, **we’ve designed a more concise API that’s specifically designed for CRUD apps.** + +This is what it looks like: + +```typescript +import { crudPolicy, authenticatedRole, authUid } from “drizzle-orm/neon”; + +export const todos = pgTable( + "todos", + { + id: bigint().primaryKey(), + userId: text() + .notNull() + .default(sql `(auth.user_id())`), + task: text().notNull(), + isComplete: boolean().notNull().default(false), + insertedAt: timestamp({ withTimezone: true }) + .defaultNow() + .notNull(), + }, + (table) => [ + crudPolicy({ + role: authenticatedRole, + read: authUid(table.userId), + modify: authUid(table.userId), + }), + ], +); +``` + +The `crudPolicy` function is implemented [here](https://github.com/drizzle-team/drizzle-orm/blob/83daf2d5cf023112de878bc2249ee2c41a2a5b1b/drizzle-orm/src/neon/rls.ts#L15). And these are its inputs: + +- role: The PostgreSQL role(s) to apply the policy to. Can be a single `PgRole` instance or an array of `PgRole` instances or role names. +- read: The SQL expression or boolean value that defines the read policy. Set to `true` to allow all reads, `false` to deny all reads, or provide a custom SQL expression. Set to `null` to prevent the policy from being generated. +- modify: The SQL expression or boolean value that defines the modify (insert, update, delete) policies. Set to `true` to allow all modifications, `false` to deny all modifications, or provide a custom SQL expression. Set to `null` to prevent policies from being generated. + +And it returns an array of PostgreSQL policy definitions, one for each data operation (select, insert, update, delete). + +The `authUid` function is specific to [pg_session_jwt](https://github.com/neondatabase/pg_session_jwt), but its implementation is very simple. It simply allows you to connect `auth.user_id` to a column in your table. + +```typescript +export const authUid = (userIdColumn: AnyPgColumn) => sql`(select auth.user_id() = ${userIdColumn})`; +``` + +This API is a much higher-level abstraction on top of Postgres RLS, which will hopefully be helpful to **anyone** using Postgres RLS (on Neon, or not, with pg_session_jwt or not). + +--- + +**Coming soon:** If you’re curious about this API and what it looks like in a more complete data model, we’ll be publising a blog post with a reference RLS implemenration for a social media network. diff --git a/content/blog/posts/hipaa-best-practices-for-developers.md b/content/blog/posts/hipaa-best-practices-for-developers.md new file mode 100644 index 0000000000..1ed4e236bb --- /dev/null +++ b/content/blog/posts/hipaa-best-practices-for-developers.md @@ -0,0 +1,247 @@ +--- +title: >- + Handling Protected Health Information Under HIPAA: Best Practices for + Developers +description: You’re in HIPAA territory. Here’s how not to blow It +excerpt: >- + You’re building a healthtech app. Maybe you’ve decided to take on the insane + complexity of electronic health records, or maybe you’re building an app for + doctors to communicate with patients, or perhaps you’re creating a platform + for managing clinical trials. Whatever your specif... +date: '2025-06-13T15:36:17' +updatedOn: '2025-09-30T13:26:44' +category: workflows +categories: + - workflows +authors: + - andrew-tate +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/hipaa-best-practices-for-developers/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + Handling Protected Health Information Under HIPAA: Best Practices for + Developers - Neon + description: >- + Get some tips on how to navigate HIPAA as a developer. Including best + practices for storing, de-identifying, and securing PHI in your app. + keywords: [] + noindex: false + ogTitle: >- + Handling Protected Health Information Under HIPAA: Best Practices for + Developers - Neon + ogDescription: >- + Get some tips on how to navigate HIPAA as a developer. Including best + practices for storing, de-identifying, and securing PHI in your app. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/hipaa-best-practices-for-developers/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/hipaa-best-practices-for-developers/neon-hipaa-1024x576-5757ec55.jpg) + + +To enable HIPAA compliance and get a BAA, follow the instructions in [Neon HIPAA compliance](https://neon.com/docs/security/hipaa) documentation. + + +You’re building a healthtech app. Maybe you’ve decided to take on the insane complexity of electronic health records, or maybe you’re building an app for doctors to communicate with patients, or perhaps you’re creating a platform for managing clinical trials. + +Whatever your specific use case, if you’re handling any health information that could identify a patient, you’re in HIPAA territory, and the stakes couldn’t be higher. A single data breach or compliance failure can [result in millions of dollars in fines](https://www.freshpaint.io/blog/ignorance-is-no-longer-an-excuse-a-timeline-of-events-around-tracking-technologies-in-healthcare), a destroyed reputation, and, most importantly, a violation of patient trust. + +[Neon is now HIPAA-compliant](https://neon.com/blog/hipaa). That means you can securely store and process Protected Health Information (PHI) in a Neon Postgres database backed by our security framework that includes encryption at rest and in transit, role-based access controls, and continuous audit logging. + +But, to quote [DBA StackExchange](https://dba.stackexchange.com/questions/345901/best-way-to-achieve-hipaa-compliance-in-a-postgressql-for-patient-data), _“The database is just part of it.”_ Developers, DBAs, data analysts, and data engineers—anyone who might touch sensitive health data—needs to understand HIPAA compliance. Here, we aim to provide you with the background on HIPAA and PHI, so you have the knowledge and ability to handle data proficiently, securely, and in compliance with HIPAA regulations, which is what distinguishes compliant applications from costly violations. + +## What is PHI? + +[Protected Health Information (PHI)](https://www.hhs.gov/hipaa/for-professionals/privacy/laws-regulations/index.html#what) is any _“individually identifiable health information”_ that relates to a person’s health status, provision of healthcare, or payment for healthcare, when held by a covered entity or its associate. In practice, this means that if a piece of health data can be linked to a specific person, it is PHI and is regulated under U.S. HIPAA law. + +So, a record like this: + +- Name: John Smith +- DOB: 5/12/1985 +- Diagnosis: Diabetes +- Treatment date: 3/15/2023 +- Facility: Chicago General Hospital + +Contains multiple pieces of PHI. Not just the medical diagnosis, but also the name, birth date, treatment date, and facility name. Even seemingly innocent data can become PHI when linked to health information, such as a ZIP code combined with a prescription refill date, an email address in a therapy appointment system, or an IP address logged during a telehealth visit. + +This starts to give a clue on why everyone in the data chain needs to understand PHI: + +- Is the developer using the IP address for debugging API requests? That IP becomes PHI when logged alongside patient portal activity. +- Is the DBA running a performance query that includes patient names in the execution plan? Those query logs now contain PHI. +- Is the data analyst creating a dashboard showing appointment patterns by ZIP code? If those ZIPs are too granular (like 90210), they could identify specific patients. +- Is the data engineer building an ETL pipeline that temporarily stores complete patient records in a staging table? That staging environment needs the same HIPAA protections as production. + +When you are building for healthcare, trivial development practices become compliance nightmares—like using production data to debug issues, storing patient emails in application logs, or including real names in test fixtures. + +### What are HIPAA Identifiers? + +HIPAA defines 18 specific identifiers that, when connected to health data, make it PHI: + +| HIPAA identifier | Description / Example | +| ----------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Name** | Any patient name (full name or parts of it). | +| **Geographic data** | All subdivisions smaller than a state (street address, city, county, precinct, ZIP)._HIPAA Safe Harbor allows using the first 3 digits of ZIP code_ **_only_** _if the population in that area is >20,000; otherwise use `000`_. | +| **Dates (except year)** | Any dates related to an individual (birth date, admission/discharge dates, date of death, etc.), and **any age over 89** (must be aggregated as 90 or older). | +| **Telephone numbers** | Any phone number (mobile, home, etc.) | +| **Fax numbers** |
Quick adjustments for both scaling up and down | +| **Email addresses** | Personal email addresses. | +| **Social Security numbers** | U.S. SSNs. | +|
**Medical record numbers** | Patient record identifiers. | +| **Health plan beneficiary numbers** | Insurance or health plan IDs. | +| **Account numbers** | Any account numbers (e.g., billing account). | +| **Certificate/license numbers** | Professional license or certificate numbers. | +| **Vehicle identifiers** | Vehicle IDs and serials, including license plate numbers. | +| **Device identifiers** | Device IDs/serial numbers (e.g., implant serials). | +| **Web URLs** | Web resource locators identifying a person (personal URLs). | +| **IP addresses** | Internet Protocol addresses linked to an individual. | +| **Biometric IDs** | Biometric identifiers (fingerprints, voice prints, retinal scans, etc.) | +| **Full-face photos** | Full-face photographs and any comparable images of a person. | +| **Unique identifying features** | _Any other_ unique characteristic, code, or identifier that could be tied to a person. | + +If health information contains any of the above identifiers (even initials or partial data), it is considered “identified” (and thus PHI). PHI is not just medical test results or diagnoses. It includes any personal detail linked to health data. For example, a lab result tagged only with a patient’s initials and birthdate is still PHI because those are identifying elements. Developers must assume that any combination of health info with personal identifiers is PHI and handle it with the highest care. + +## De-Identifying PHI: Safe Harbor vs. Expert Determination + +When building healthcare applications, you’ll often need to de-identify data for analytics, testing, or research purposes. HIPAA provides [two methods](https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html) to transform PHI into non-PHI data that can be used more freely: + +### Safe Harbor Method: The Developer-Friendly Checklist + +Safe Harbor is the “just remove these 18 things” approach. It’s prescriptive and straightforward to implement programmatically, making it perfect for most development scenarios. + +How it works: Remove all 18 HIPAA identifiers from your dataset. If nothing remains that could identify a person, the data is de-identified. + +```python +# Before Safe Harbor (PHI) +patient_record = { + "name": "John Smith", + "dob": "1985-05-12", + "diagnosis": "Diabetes", + "zip_code": "90210", + "admission_date": "2023-03-15" +} + +# After Safe Harbor (De-identified) +de_identified_record = { + "patient_id": "HASH_7f3a9b2c", # Random identifier + "birth_year": 1985, # Only year retained + "diagnosis": "Diabetes", + "zip_code": "902**", # First 3 digits only + "admission_year": 2023 # Only year retained +} +``` + +- Pros: Easy to implement, verify, and audit. You can write a script that strips these fields. +- Cons: Data loses specificity—no dates beyond year, limited geographic info, which may reduce analytical value. + +### Expert Determination: The Data Science Approach + +Expert Determination uses statistical methods to ensure re-identification risk is “minimal” while preserving more data utility. + +How it works: A qualified expert applies techniques like: + +- Date shifting (offsetting all dates by a random number of days) +- Geographic aggregation (county instead of ZIP) +- Age bucketing (45-50 instead of exact age) +- K-anonymity (ensuring each record matches at least K others) + +```python +# Before Expert Determination (PHI) +patient_record = { + "age": 45, + "diagnosis_date": "2023-03-15", + "zip_code": "90210", + "rare_condition": "Progeria" +} + +# After Expert Determination (De-identified with more utility) +expert_de_identified = { + "age_group": "40-50", + "days_since_diagnosis": 287, # Relative date + "region": "Los Angeles County", + "condition_category": "Rare genetic disorder" +} +``` + +The expert certifies that combinations of these quasi-identifiers appear in multiple patients, preventing unique identification. + +- Pros: Retains more analytical value—you can still see temporal patterns and geographic trends. +- Cons: Requires statistical expertise, formal risk analysis, and documentation. + +### Practical Tips for Developers + +1. Default to Safe Harbor for test data, development environments, and basic analytics. It’s foolproof if implemented correctly. +2. Watch out for free text. A note saying “45-year-old engineer at Tesla” contains identifiers. Free text fields need special handling or exclusion. +3. Consider Limited Datasets as a middle ground. These retain some identifiers (like dates) but remove direct identifiers (names, contacts). They’re still PHI but useful for research under data use agreements. +4. Automate where possible. Build de-identification into your data pipelines: + +```sql +-- Example: Creating a de-identified view +CREATE VIEW patients_deidentified AS +SELECT + MD5(patient_id) as patient_hash, + EXTRACT(YEAR FROM birth_date) as birth_year, + LEFT(zip_code, 3) || '**' as zip_prefix, + diagnosis_code +FROM patients; +``` + +5\. Remember: De-identification isn’t just deletion. A 103-year-old with a rare disease on a specific date is still identifiable even without a name. Think about combinations of data points.

_When in doubt, strip it out._ It’s better to have less detailed, fully de-identified data than to accidentally leak PHI through an obscure combination of quasi-identifiers. + +## Best Practices for Handling PHI in Development + +Consider these non-negotiable requirements, not optional guidelines. + +### Encryption & Storage + +- **Always encrypt PHI at rest and in transit**. Use AES-256 for data at rest and TLS 1.2+ for all network communications. Never store encryption keys in source code—use a dedicated key management service like AWS KMS or HashiCorp Vault. [Neon does this for you](https://neon.com/docs/security/hipaa). +- **Never use real PHI in dev/test environments**. Use synthetic or properly de-identified data for development and testing. If production data is necessary for debugging, require formal approval, use it in an isolated environment, and wipe it immediately after use. +- **Secure all backups and archives**. Encrypt database dumps and backup files before storage, and treat them with the same security controls as production data. When decommissioning storage devices that contain PHI, use secure wiping tools or physical destruction methods. + +### Access Control & Authentication + +- **Enforce unique user IDs and strong authentication**. Never allow shared accounts for PHI access—every user needs a unique, auditable identity. Implement multi-factor authentication (MFA) for all systems handling PHI, especially patient portals and admin interfaces. +- **Implement role-based access control (RBAC) with the principle of least privilege**. Users should only access the minimum PHI necessary for their job function. A lab technician entering results doesn’t need access to complete patient histories, and a billing clerk doesn’t require clinical notes. [Neon does this for you](https://neon.com/docs/security/hipaa). +- **Set aggressive session timeouts**. PHI applications should automatically log out users after short periods of inactivity. Use secure session management with HttpOnly and Secure flags on cookies, and never expose session IDs in URLs. + +### API & Integration Security + +- **Never put PHI in URLs or query parameters**. URLs end up in server logs, browser history, and monitoring tools—use POST requests with PHI in the request body instead. Always use opaque identifiers (UUIDs) rather than revealing patient names or record numbers in API paths. +- **Require authentication for all PHI endpoints**. No public API endpoints should ever return PHI—use OAuth 2.0, API keys with specific scopes, or signed tokens. Validate authorization on the server side for every request; never trust client-side checks alone. +- **Vet all third-party integrations for HIPAA compliance**. Any external service that touches PHI needs a Business Associate Agreement (BAA) and proven security controls. (e.g., [Neon has a BAA to conform with HIPAA compliance](https://neon.com/docs/security/hipaa)). This includes seemingly innocuous services such as email providers, SMS gateways, error tracking, and analytics tools. + +### Logging & Monitoring + +- **Log all PHI access but not PHI content**. Record who accessed what record and when, but avoid logging the actual medical data—log “User X viewed Patient #123” not “User X viewed John Doe’s HIV results.” Treat audit logs as sensitive data with restricted access and long retention periods (six years or more). +- **Actively monitor for suspicious patterns**. Set up alerts for unusual access patterns, such as bulk record downloads, after-hours access, or employees viewing records outside their department. Many insider breaches involve snooping on family members or celebrities—your monitoring should catch this. +- **Never send PHI to external logging services**. Avoid accidentally exposing PHI through error logs, debug output, or monitoring tools. If you must log errors involving PHI, mask or hash sensitive values before they reach any third-party service. + +Neon uses [PGAudit](https://www.pgaudit.org/) for [extensive, HIPAA-compliant audit logging](https://neon.com/blog/postgres-logging-vs-pgaudit). + +### Development Practices + +- **Implement de-identification early in your pipeline**. Build Safe Harbor de-identification into your data export tools—it’s easier to strip the 18 identifiers programmatically than to handle PHI everywhere. For test data generation, use this de-identified production data or fully synthetic datasets. +- **Sanitize all free text fields**. Clinical notes and comments often contain hidden identifiers like “45-year-old Tesla engineer” that violate de-identification. Either exclude free text from de-identified datasets or use NLP tools to detect and redact identifying information. +- **Plan for incident response from day one**. Build features to quickly revoke API keys, deactivate user accounts, and generate compliance reports. You should be able to answer “who accessed what and when” within minutes, not hours, when investigating a potential breach. + +### Common Pitfalls to Avoid + +- **Watch out for metadata leaks**. Patient photos contain PHI in EXIF data, file names might include patient names, and browser caching can expose sensitive data on shared computers. Set Cache-Control: no-store headers and sanitize all file metadata. +- **Don’t forget about client-side security**. Mobile apps storing PHI need device-level encryption and PIN/biometric protection. Web apps should clear sensitive data from memory when not in use and avoid storing PHI in localStorage or browser databases. +- **Remove all debug endpoints before production**. That /dumpAllRecords test route or admin backdoor will eventually be found by attackers. Utilize environment-specific configurations and automated checks to prevent debug features from reaching production systems. + +Implementing these practices isn’t just about avoiding fines; it’s about maintaining the trust patients place in you when they share their most intimate health information. + +## Building with Trust + +Developing with PHI is challenging. It should be. This is your users’ most sensitive data. You need to treat every piece of health information as if it were your medical record—because someday, it might be. + +Many common mistakes come from treating health data “like any other data”. The technical requirements we’ve covered—encryption, access controls, audit logging, secure APIs—are table stakes. What matters more is building a security-first culture where every developer understands that a single leaked database query, an accidentally logged parameter, or a carelessly configured backup can destroy patient trust and your company’s reputation. + +Your users are trusting you with information they might not even share with family members. Honor that trust by making security and privacy non-negotiable requirements, not nice-to-have features. In healthcare technology, there’s no such thing as being too careful with PHI. + +--- + +_[Neon](https://neon.com/) is the serverless Postgres platform built for developers, with autoscaling, braching, instant restores and (of course) HIPAA compliance. [Get started with our Free Plan](https://console.neon.tech/signup) and reach out to us for any questions or assistance._ diff --git a/content/blog/posts/hipaa-multitenancy-b2b-saas.md b/content/blog/posts/hipaa-multitenancy-b2b-saas.md new file mode 100644 index 0000000000..bc10a303ec --- /dev/null +++ b/content/blog/posts/hipaa-multitenancy-b2b-saas.md @@ -0,0 +1,167 @@ +--- +title: 'How Neon Solves HIPAA Compliance, Multi-Tenancy, and Scaling for B2B SaaS' +description: Ensure strict data isolation without the management overhead +excerpt: >- + If you’re running a B2B SaaS company that operates across multiple regions and + needs to stay HIPAA-compliant, managing your Postgres setup can be a real + headache. You need to keep customer data isolated while avoiding performance + issues caused by shared resources, all while ensur... +date: '2025-03-18T02:38:30' +updatedOn: '2025-09-30T13:26:40' +category: workflows +categories: + - workflows +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/hipaa-multitenancy-b2b-saas/cover.png + alt: null +isFeatured: false +seo: + title: >- + How Neon Solves HIPAA Compliance, Multi-Tenancy, and Scaling for B2B SaaS - + Neon + description: >- + If you're operating across multiple regions and need to stay + HIPAA-compliant, managing your Postgres setup can be a headache. Neon can + help. + keywords: [] + noindex: false + ogTitle: >- + How Neon Solves HIPAA Compliance, Multi-Tenancy, and Scaling for B2B SaaS - + Neon + ogDescription: >- + If you're operating across multiple regions and need to stay + HIPAA-compliant, managing your Postgres setup can be a headache. Neon can + help. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/hipaa-multitenancy-b2b-saas/cover.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/hipaa-multitenancy-b2b-saas/neon-1000-projects-1024x576-50ec85c2.png) + + +To enable HIPAA compliance and get a BAA, follow the instructions in [Neon HIPAA compliance](https://neon.com/docs/security/hipaa) documentation. + + +If you’re running a B2B SaaS company that operates across multiple regions and needs to stay HIPAA-compliant, managing your Postgres setup can be a real headache. You need to keep customer data isolated while avoiding performance issues caused by shared resources, all while ensuring your architecture can scale as you onboard more customers. + +Managing a multi-tenant design like this with traditionally managed Postgres solutions like AWS RDS is challenging. If you’re using RDS, chances are you’re hosting all your customers’ data in the same instance. Over time, that instance grows larger and larger, turning into a major bottleneck for your team as maintenance and scaling become increasingly difficult. + +Good news: Neon can help. Instead of cramming multiple customers into a single database, Neon makes it easy to give each customer their own region-specific, dedicated project. This ensures simpler compliance management, better performance, and smooth scalability up to [hundreds of thousands of customers.](https://neon.tech/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases) + +## The Challenges of Multi-Tenant Architectures in B2B SaaS + +Many SaaS companies rely on a shared database model where all customers are tenants within the same instance. While this setup works for some, it comes with significant challenges. Companies that come to us are often struggling with these three problems: + +### Compliance risks + +When multiple customers share a database, ensuring strict data isolation, something that’s crucial for HIPAA and other regulatory requirements, becomes much harder. This challenge is even greater for companies operating in multiple regions (e.g., the U.S. and Europe) and dealing with highly regulated industries, where laws like GDPR and HIPAA mandate strict control over where and how data is stored. + +**How is this often solved in AWS RDS?**
Many companies try to mitigate this by manually partitioning data within a shared database, implementing row-level security, or creating separate schemas for each customer. Others spin up separate RDS instances for regions. + +**The problems with this solution:** + +- RLS and schema-based isolation still share the same database. Any misconfiguration or bug can expose one customer’s data to another. This setup requires expert DBA maintenance at scale. +- Spinning up multiple RDS instances is expensive and requires extensive infrastructure management. Each instance must be provisioned, monitored, and backed up separately. +- Data residency requirements become difficult to enforce without strict per-region database isolation. + +### Noisy neighbor bottlenecks + +In a shared database model, all customers are competing for the same resources. If one customer runs a particularly heavy query or batch job, it can slow down performance for everyone else. This is known as the “noisy neighbor” problem, where a single tenant’s usage impacts the experience of others. + +**How is this often solved in RDS?** + +This is often mitigated by caling up the RDS instance to handle increased load. Sometimes, rate limits are set per customers to prevent excessive resource consumption. Read replicas could also be used to distribute some of the workload. + +**The problems with this solution:** + +- Vertical becomes increasingly cost-prohibitive and inefficient. +- Query limits can frustrate customers who temporarily need more resources, leading to a poor user experience. +- Read replicas help with read-heavy workloads but don’t solve the problem of write-heavy applications or background jobs affecting primary database performance. + +### Complex database management + +When all customer data is mixed within a single database, managing backups, restores, and audits becomes complicated. If one customer needs a point-in-time recovery, the entire database might need to be restored, affecting all tenants. Auditing access and tracking down data issues can also become a challenge when everything is intermingled. + +**How is this often solved in RDS?** + +Some companies work around these issues by tagging customer-specific data for easier filtering in logs and audits, implementing logical backups and partial restores using custom scripts. + +**The problems with this solution:** + +- Restoring a single customer’s data often requires restoring a much larger database, potentially causing disruption for all tenants. +- Schema-based separation still shares the same underlying infrastructure, meaning any database-wide issue (e.g., corruption, downtime, or security misconfiguration) affects all customers. + +## How Neon’s One-Project-Per-Customer Approach Fixes This + +Instead of putting all customers in a single database, Neon lets you spin up [a separate project for each customer](https://neon.tech/use-cases/database-per-tenant). Each project has its own dedicated Postgres instance, which means: + +### Better compliance and security + +Neon’s isolation model makes it much easier to stay compliant with HIPAA and other regulations. Since each customer operates in their own dedicated environment, there’s no risk of accidental cross-tenant data access. Audits also become simpler—there’s no need to sift through mixed data, as each customer’s records are neatly contained within their own project. Neon’s [built-in encryption and security features](https://neon.tech/security) further align with compliance needs right from the start. + +### No more noisy neighbors + +With Neon’s approach, customers don’t compete for database resources, so a single tenant’s heavy workload won’t impact others. Performance remains consistent. Each customer scales independently based on their needs, ensuring efficient scaling and predictable uptime, free from shared resource contention. + +### Scaling globally is straightforward + +For companies expanding globally, Neon’s architecture makes it easy to comply with regional data laws. Customer data can be deployed in specific locations as needed, ensuring adherence to local regulations. Scaling is easy: when new customers come on board, a new project can be spun up instantly in the region of choice without disrupting existing ones. This also optimizes costs by allowing resources to be allocated based on actual customer usage rather than over-provisioning a single shared database. + +### Per-customer data management and restores + +Since each customer gets their own project, managing backups, restores, and upgrades is far simpler than in a shared database model. If a customer needs a restore, [their data can be instantly recovered](https://neon.tech/blog/recover-large-postgres-databases) independently without affecting anyone else. When a customer offboards, their data can be securely removed without impacting other tenants, ensuring clean and efficient data lifecycle management. + +## Why Managing Thousands of Projects is Easy with the Neon API + +Managing thousands of projects might sound overwhelming, but this is where Neon’s unique API makes a huge difference. Unlike traditional managed databases, where provisioning and managing instances at scale is complex and expensive, [Neon is built from the ground up for programmatic deployment at scale.](https://neon.tech/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases) + +Here’s what makes Neon’s API stand out: + +### Automated project and database provisioning + +With Neon’s API, you can automatically create, configure, and manage thousands of Postgres databases without manual intervention. Each Neon project corresponds to a fully isolated database environment, allowing you to deploy region-specific databases for customers instantly. + +Example: If you onboard a new customer, you can programmatically create a new Neon project in their preferred region, ensuring compliance with data residency laws like HIPAA and GDPR. + +### Enforcing per-project resource quotas + +One of the biggest challenges of managing multi-tenant environments is controlling resource consumption across customers. With Neon, you can enforce strict quotas at the project level: + +- Set max compute uptime per billing cycle +- Restrict max CPU usage to prevent individual customers from overloading your infrastructure +- Cap data storage and transfer limits to align with different pricing plans + +### Fine-tuned compute scaling for cost efficiency + +Neon’s API provides granular control over compute settings, allowing you to fine-tune how each database scales. You can set auto-scaling limits by defining the minimum and maximum vCPU per project. Autosuspend (scale-to-zero) timeouts can be configured to automatically pause idle databases, reducing unnecessary compute costs. Additionally, resource utilization can be optimized based on customer tier, e.g. allowing lower-tier databases to scale down aggressively. + +### Fleet-wide monitoring & usage tracking + +For SaaS platforms managing thousands of isolated databases/projects, monitoring real-time usage is crucial. Neon’s API provides detailed consumption tracking across all projects: + +- Track total active compute time across all databases +- Monitor total CPU and storage usage to optimize scaling decisions +- Get real-time insights into data transfer and write activity + +## How Does This Compare to AWS RDS Postgres? + +As we saw at the beginning of the post, if you’re using AWS RDS for Postgres, you’re likely managing a large instance with multiple databases inside it. While this works at a small scale, it quickly becomes a challenge as you grow: + +- **Scaling is manual and expensive**. You’ll have to scale up your entire instance, even if only one customer needs more resources. +- **Compliance is trickier**. Achieving full data isolation in RDS requires extra DBA work, especially across regions. +- **Noisy neighbor issues persist**. Unless you’re dedicating an entire RDS instance to each customer (which gets expensive and unmanageable fast), performance can be unpredictable. + +With Neon, you get built-in isolation from the start, and scaling happens on a per-customer basis. Instead of dealing with rigid instances, you get a flexible, serverless approach that adjusts based on actual demand—and you can manage your entire fleet via the API. + + +[OpusFlow switched from AWS RDS to Neon](https://neon.tech/blog/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers) to manage tenant isolation without the need for a dedicated team. + + +## Try it + +For B2B SaaS companies dealing with HIPAA compliance and scaling across regions, Neon’s one-project-per-customer model offers a huge advantage over traditional multi-tenant setups in AWS RDS. You get better security, more consistent performance, and easier management, all while keeping costs under control. + +[Sign up for Neon today and get $100 in credits](https://fyi.neon.tech/credits). Once you’re ready to set up a proper PoC, [contact us](https://neon.tech/contact-sales)—our team of Postgres experts will help you get started and assist with your evaluation. diff --git a/content/blog/posts/hipaa.md b/content/blog/posts/hipaa.md new file mode 100644 index 0000000000..52e86ef06c --- /dev/null +++ b/content/blog/posts/hipaa.md @@ -0,0 +1,69 @@ +--- +title: Neon is HIPAA Compliant +description: Store and process PHI confidently on Neon +excerpt: >- + Neon has completed its HIPAA compliance audit, adding to our security + achievements: SOC 2 Type 2, ISO 27001, ISO 27701, GDPR, and CCPA. If your + company needs a HIPAA-compliant database, Neon can now securely store + Protected Health Information (PHI). What is HIPAA Compliance? The... +date: '2025-03-13T16:17:42' +updatedOn: '2025-09-30T13:26:49' +category: product +categories: + - product + - company +authors: + - busra-demir +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/hipaa/cover.jpg' + alt: null +isFeatured: true +seo: + title: Neon is HIPAA Compliant - Neon + description: >- + Neon has completed its HIPAA compliance audit, adding to our security + achievements: SOC 2 Type 2, ISO 27001, ISO 27701, GDPR, and CCPA. + keywords: [] + noindex: false + ogTitle: Neon is HIPAA Compliant - Neon + ogDescription: >- + Neon has completed its HIPAA compliance audit, adding to our security + achievements: SOC 2 Type 2, ISO 27001, ISO 27701, GDPR, and CCPA. + image: 'https://cdn.neonapi.io/public/images/pages/blog/hipaa/social.jpg' +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/hipaa/neon-hipaa-1024x576-f3b56a41.jpg) + +Neon has completed its HIPAA compliance audit, adding to our [security achievements](https://neon.tech/security): SOC 2 Type 2, ISO 27001, ISO 27701, GDPR, and CCPA. If your company needs a HIPAA-compliant database, Neon can now securely store Protected Health Information (PHI). + + +To enable HIPAA compliance and get a BAA, follow the instructions in [Neon HIPAA compliance](https://neon.com/docs/security/hipaa) documentation. + + +## What is HIPAA Compliance? + +The Health Insurance Portability and Accountability Act (HIPAA) sets national security and privacy standards for handling PHI. Compliance requires strict encryption, access controls, continuous monitoring, and incident response to keep sensitive health data secure. + +By meeting these standards, healthcare organizations, SaaS platforms, and other regulated businesses can now store and process PHI confidently on Neon. + +## How Neon Protects PHI + +Security is built into every layer of Neon’s platform to protect PHI at every stage. Our HIPAA safeguards include: + +- **Encryption & Access Controls:** PHI is encrypted in transit and at rest using strong cryptographic standards. Role-based access control (RBAC) ensures only authorized users can access it, with full audit logging to track access. +- **Incident Response & Breach Notification:** Our incident response process aligns with HIPAA’s Breach Notification rules, with designated security personnel reviewing and managing potential PHI incidents. If a breach occurs, we notify affected customers within five business days of discovery, providing full details and remediation steps. +- **Audit Logs & Monitoring:** Continuous security monitoring detects unauthorized access attempts, and detailed audit logs provide full visibility into PHI-related activity. +- **Shorter Inactivity Timeout:** Console sessions automatically time out faster, reducing the risk of unauthorized access from unattended or stale sessions. +- **Employee Training & Security Awareness:** All employees handling PHI receive mandatory HIPAA training, reinforcing security best practices and compliance requirements. +- **Third-Party Security:** Any [subcontractors](https://neon.tech/hipaa-contractors) handling PHI must meet HIPAA standards, and we sign Business Associate Agreements (BAAs) with relevant vendors to ensure compliance. +- **Customer Responsibilities:** HIPAA compliance is a shared responsibility. While we provide the necessary safeguards, customers must ensure their configurations and data handling practices align with HIPAA requirements. PHI must only be stored in data rows and not in logs, schema descriptions, or metadata. + +HIPAA compliance isn’t the finish line for us. We’re always improving our security to stay ahead of threats. + +## What’s Next? + +Achieving HIPAA compliance is a significant milestone in our ongoing commitment to data security. We continue to uphold our existing certifications, including SOC 2 Type 2, ISO 27001, and ISO 27701, and align with GDPR and CCPA requirements. + +**Next up: PCI-DSS compliance in Q2.** This will strengthen our security even further and help us scale for enterprise customers. + +All HIPAA customers must sign a Business Associate Agreement (BAA) with Neon. For guidance on setting up a HIPAA-compliant environment in Neon, reach out to our team at hipaa@neon.tech. diff --git a/content/blog/posts/honcathon-get-your-goose-on.md b/content/blog/posts/honcathon-get-your-goose-on.md new file mode 100644 index 0000000000..0b2e2df672 --- /dev/null +++ b/content/blog/posts/honcathon-get-your-goose-on.md @@ -0,0 +1,58 @@ +--- +title: Honcathon – Get Your Goose On +description: >- + October has Hacktoberfest, and December has Advent of Code. But what about + November? Say hello to Honcathon. +excerpt: >- + October has Hacktoberfest, and December has Advent of Code. But what about + November? In some countries, it’s known for turkey on the table, but it’s also + the month when, by its end, the last geese migrate from the north to the + south. This November, we’re celebrating the Goose in... +date: '2024-11-07T13:57:18' +updatedOn: '2024-11-15T00:42:22' +category: community +categories: + - community +authors: + - taraneh-dohmer +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/honcathon-get-your-goose-on/cover.jpg + alt: null +isFeatured: false +seo: + title: Honcathon - Get Your Goose On - Neon + description: >- + October has Hacktoberfest, and December has Advent of Code. But what about + November? Say hello to Honcathon. + keywords: [] + noindex: false + ogTitle: Honcathon - Get Your Goose On - Neon + ogDescription: >- + October has Hacktoberfest, and December has Advent of Code. But what about + November? Say hello to Honcathon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/honcathon-get-your-goose-on/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/honcathon-get-your-goose-on/cover.jpg) + +October has Hacktoberfest, and December has Advent of Code. But what about November? In some countries, it’s known for turkey on the table, but it’s also the month when, by its end, the last geese migrate from the north to the south. + +This November, we’re celebrating the Goose in spirit with our first [virtual Honcathon!](https://honc.dev/honcathon) + +## What is HONC? + +[HONC](https://honc.dev/) is a stack that accelerates web development, featuring [Hono](https://hono.dev/) as a web framework, [Drizzle](https://orm.drizzle.team/) as an ORM, [Neon](https://neon.tech/) as a database, and [Cloudflare Workers](https://workers.cloudflare.com/). With these tools, getting started on web projects has never been faster. Additionally, [Fiberplane](https://fiberplane.com/) helps to debug and test your APIs, making development even smoother. + +This month, Fiberplane is partnering with these tools to bring a touch of feathered fun to your IDE. + +## How It Works + +There are four different categories for building applications, and participants can choose any of them. The winner of each category will receive a €500 Amazon voucher. Be sure [to register](https://honc.dev/honcathon), as we have some exciting activities planned throughout the Honcathon! + +## Ready to get started? + +You’ll have until **December 15th** to submit your project. Don’t worry—building your project is designed to take just a half-day to a day of work, making it perfect to fit into your schedule. + +This November, get your goose on with HONC! diff --git a/content/blog/posts/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand.md b/content/blog/posts/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand.md new file mode 100644 index 0000000000..6fb2ebb129 --- /dev/null +++ b/content/blog/posts/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand.md @@ -0,0 +1,97 @@ +--- +title: How 222 uses Neon to handle their frequent spikes in demand +description: Neon scales their database automatically when traffic increases +excerpt: >- + “Our database traffic peaks at nights and on weekends when thousands of our + members are attending experiences. Building on a database that preemptively + autoscales allows us to regularly handle these traffic spikes.” Lex Nasser, + Founding Engineer at 222 Life in a tech-enabled worl... +date: '2024-06-25T17:23:21' +updatedOn: '2024-06-25T17:23:24' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand/cover.jpg + alt: null +isFeatured: false +seo: + title: How 222 uses Neon to handle their frequent spikes in demand - Neon + description: >- + Thanks to autoscaling, their compute scales up and down dynamically in + response to load. No manual work, no overpaying. + keywords: [] + noindex: false + ogTitle: How 222 uses Neon to handle their frequent spikes in demand - Neon + ogDescription: >- + Thanks to autoscaling, their compute scales up and down dynamically in + response to load. No manual work, no overpaying. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand/neon-222place-2-1024x576-617649d9.jpg) + +
+

“Our database traffic peaks at nights and on weekends when thousands of our members are attending experiences. Building on a database that preemptively autoscales allows us to regularly handle these traffic spikes.”

+Lex Nasser, Founding Engineer at 222 +
+ +Life in a tech-enabled world has become perfectly convenient, but we have to admit that it can also get isolating. Getting through an entire workday of meetings from your couch, with two meals hand-delivered by a Postmates driver, is just a step away from WALL-E. Sometimes, you feel like getting out of the house and actually meeting people, but occasions that enable casual encounters are becoming exceedingly rare. + +The team at [222](https://222.place/) (YC W23) is building a fun platform to fix this. They organize social events in a completely original way: occasionally, members receive curated invites to experiences in their city. Imagine getting an invite for puppy yoga in the park on a Saturday afternoon, or a dinner and ceramics class after work on a Tuesday. 222 takes care of inviting other members who are compatible with your interests, so you can share the experience with them. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-222-uses-neon-to-handle-their-frequent-spikes-in-demand/screenshot-2024-06-21-at-65239percente2percent80percentafpm-b9f11ddd.png) + +To make this happen, 222 has had to build out systems to enable members to RSVP (either alone, with a +1, or in groups), guide them through their experience in real-time, and gather high-fidelity follow-up data to refine their members’ curation preferences. These are already quite a few product requirements to maintain, and they’re a small team: the last thing they want is to put any more time than necessary into their database… Which led them to Neon. + +## Scaling their backend: From Airtable to Postgres + +222 started as a project run entirely by SMS. Experience invitations were sent via text messages, and responses were managed the same way. This backend was powered by Airtable, which was a low-lift and convenient option for the project at that stage. + +Once 222 started growing, it became clear that the team had to find a proper relational database, especially as they prepared for the launch of their iOS app. They began exploring various options: + +- Considering their small team size, self-hosting was ruled out—any time spent on database maintenance was time taken away from product iteration. +- Once the team decided to use a managed database, they first considered Planetscale, but preferred Postgres over MySQL, specifically due to its support for user-defined data types and materialized views. +- The choice then narrowed down to Neon and Supabase. The team spun up experimental databases on both platforms to compare their functionality. +- In the end, Neon stood out for its unique serverless features, which proved especially handy—particularly its robust autoscaling and copy-on-write branching. + +Let’s dig a bit deeper into why these Neon features were so beneficial: + +## The benefits of Neon for applications with variable access pattern + +
+

“Neon autoscaling kicks in when we have bursts of traffic. When we launched our iOS app in February, we charted on the App Store; Neon’s managed database experience made it effortless to handle this traffic.”

+Lex Nasser, Founding Engineer at 222 +
+ +222 experiences mostly occur on weekends or evenings, meaning that their database encounters particularly high traffic regularly in concentrated timeframes. What really saves them time and gives their team confidence is a database that can scale down during off-peak times, and scale up to manage their influxes of usage during these experiences. + +When traffic spikes, which also happens when experience invites are sent out, Neon scales database resources automatically; when traffic slows down, Neon’s resources scale back down. All without a second glance from the engineering team. + +## Using database branching for schema migrations + +
+

“Branching is an invaluable feature for us. Knowing that we’re not going to ruin the database when iterating on our abstractions gives us peace of mind and lets us move so much faster.”

+Lex Nasser, Founding Engineer at 222 +
+ +Another favorite feature of the 222 team is Neon’s branching. In Neon, you can instantly create database branches that include data and schema; [this is immensely useful for setting up development and test environments.](https://neon.tech/flow) + +222’s engineering team has built out tooling to routinely spin up database branches for various testing purposes, such as schema migrations. This tooling, built on Neon’s REST API and the open-source [Goose](https://github.com/pressly/goose) library, allows the team to run schema migrations on an exact copy of their production database without touching their production data. They can validate their migrations in both the up-and down-directions, seeing the exact schema changes in a sandboxed environment. + +This process significantly reduces the anxiety of potentially running a bad query that could disrupt the database, especially when making comprehensive changes to their data models. And, although they haven’t needed to use it yet:), if something goes wrong, Neon’s [point-in-time restore](https://neon.tech/blog/point-in-time-recovery-in-postgres) can be used to instantaneously revert bad actions. + +## Simplify your Postgres experience: give Neon a go + +
+

“Neon takes care of the small things that one can miss when you have a small team looking to ship fast. Even trivialities, like requiring database connections to use SSL, are all built-in by default. This allows us to focus on what we want to focus on, which is improving our product, and let the database stuff handle itself.”

+Lex Nasser, Founding Engineer at 222 +
+ +**Running Postgres in production doesn’t have to be that hard**. Test it yourself: it takes 2 seconds to get started in Neon – [create a Free account and start exploring the platform](https://console.neon.tech/signup). + +At the moment, 222 curates experiences for members in Los Angeles and New York City. Apply to become a member at [222.place](https://222.place/?utm_source=neon) _✨_ diff --git a/content/blog/posts/how-axess-intelligence-simplified-their-backend-with-neons-data-api.md b/content/blog/posts/how-axess-intelligence-simplified-their-backend-with-neons-data-api.md new file mode 100644 index 0000000000..d4ff55f727 --- /dev/null +++ b/content/blog/posts/how-axess-intelligence-simplified-their-backend-with-neons-data-api.md @@ -0,0 +1,71 @@ +--- +title: How Axess Intelligence Simplified Their Backend With Neon’s Data API +description: 'Building a fast, branch-based backend without writing CRUD endpoints' +excerpt: >- + “We didn’t want to spend unnecessary time writing CRUD logic or maintaining a + backend. We just connected Better Auth and the Data API and started building. + The best part: every Neon branch gets its own Data API URL” Mouaz Anan, + Engineer at Axess Intelligence Axess Intelligence is... +date: '2025-11-20T17:49:48' +updatedOn: '2026-01-02T17:39:18' +category: app-platform +categories: + - app-platform + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-axess-intelligence-simplified-their-backend-with-neons-data-api/cover.jpg + alt: null +isFeatured: false +seo: + title: How Axess Intelligence Simplified Their Backend With Neon’s Data API - Neon + description: >- + How a data-driven rewards app built a fast, low-maintenance backend using + Neon’s Data API and branching, no CRUD endpoints required. + keywords: [] + noindex: false + ogTitle: How Axess Intelligence Simplified Their Backend With Neon’s Data API - Neon + ogDescription: >- + How a data-driven rewards app built a fast, low-maintenance backend using + Neon’s Data API and branching, no CRUD endpoints required. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-axess-intelligence-simplified-their-backend-with-neons-data-api/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-axess-intelligence-simplified-their-backend-with-neons-data-api/neon-axess-intelligence-1-1024x576-d1eed9f9.jpg) + +
+

“We didn’t want to spend unnecessary time writing CRUD logic or maintaining a backend. We just connected Better Auth and the Data API and started building. The best part: every Neon branch gets its own Data API URL”



Mouaz Anan, Engineer at Axess Intelligence

+
+ +[Axess Intelligence](https://www.axessintelligence.com/) is a platform that gives GTM teams real-time visibility into their competitors’ CRM strategies. By capturing panel data across markets, channels and customer segments, it provides a clearer view of the competitive landscape, helping companies to spot trends, identify opportunities and stay ahead. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-axess-intelligence-simplified-their-backend-with-neons-data-api/screenshot-2025-11-20-at-94122-am-1024x524-509e1fbb.png) + +## Building an Efficient, Low-Maintenance Full-Stack App + +The team uses [Neon](https://neon.com/) as the database for multiple parts of their platform. One of these is a mobile rewards app that needs a secure, reliable data layer without the overhead of building and maintaining a traditional backend full of CRUD endpoints. + +To build this fast with minimal engineering resources and maintenance, the team implemented this setup: + +- **API-first data access via** [Neon’s Data API.](https://neon.com/docs/data-api/get-started) Neon’s PostgREST-compatible Data API handles the bulk of database operations (create, read, update, delete), returning structured JSON to the client. +- **Authentication handled by** [Better Auth](https://www.better-auth.com/), which issues the JWTs used to call the Data API. +- **Branch-based environments using Neon** [branching](https://neon.com/docs/introduction/branching).Branches are used to create development and feature environments, each with its own unique Data API URL. This makes it easy to automate environment setup and ensures every build targets the right database state safely. + +## Benefits of the Neon + Better Auth + Data API Stack + +By combining Neon’s Data API, Better Auth, and branching, Axess Intelligence built a fast, low-maintenance backend that fits their mobile-first workflow. Why this setup works well for them: + +- **No backend boilerplate.** The Data API replaces hundreds of lines of repetitive CRUD logic. Standard HTTP requests map directly to SQL operations and return JSON results instantly. +- **Postgres power, API simplicity.** The team interacts with a real Postgres database, not an abstraction. They get full SQL performance and reliability with the convenience of an API-first interface. +- **Seamless authentication flow.** Better Auth handles identity and token issuance, while Neon validates each request using signed JWTs. The separation keeps authentication independent from data logic and easy to maintain. +- **Branch-aware environments.** Each Neon branch automatically gets its own Data API URL. The team maintains a production branch, a development branch, and feature-specific branches for testing new ideas. This makes it easy to experiment safely without touching production data, while keeping every environment consistent and ready to merge when changes are validated. +- **Ready for automation.** Because branches and API endpoints are created programmatically, the entire setup can be integrated into CI/CD pipelines or app builds, letting new environments spin up and down automatically. + +## Wrap Up + +Axess Intelligence’s experience shows how a lean, focused team can ship complex data-driven features without a traditional backend. Try a similar setup for your own app using the [Neon Free Plan](https://console.neon.tech/signup): create a database, connect the Data API, and start building without writing a single endpoint. + +**A big thank-you to the team at Axess Intelligence for sharing their stack and insights. Explore their [features](https://www.axessintelligence.com/features) and request a demo** [here](https://www.axessintelligence.com/demo). diff --git a/content/blog/posts/how-cedalio-uses-neon-for-an-efficient-development-workflow.md b/content/blog/posts/how-cedalio-uses-neon-for-an-efficient-development-workflow.md new file mode 100644 index 0000000000..d4562aec02 --- /dev/null +++ b/content/blog/posts/how-cedalio-uses-neon-for-an-efficient-development-workflow.md @@ -0,0 +1,103 @@ +--- +title: How Cedalio uses Neon for an efficient development workflow +description: They choose Neon over RDS because of its superior developer experience +excerpt: >- + We love Postgres, and for us, being able to use Postgres database branches + that can be integrated with our CI/CD pipeline is a killer feature. The + ability to spawn databases that can scale down to zero is also incredibly + helpful. This model fits well with our one database per cus... +date: '2024-05-29T19:38:07' +updatedOn: '2024-05-29T22:41:54' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-cedalio-uses-neon-for-an-efficient-development-workflow/cover.jpg + alt: null +isFeatured: false +seo: + title: How Cedalio uses Neon for an efficient development workflow - Neon + description: >- + They choose Neon over RDS because of its superior developer experience with + scale to zero, database branching, and a user-friendly API. + keywords: [] + noindex: false + ogTitle: How Cedalio uses Neon for an efficient development workflow - Neon + ogDescription: >- + They choose Neon over RDS because of its superior developer experience with + scale to zero, database branching, and a user-friendly API. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-cedalio-uses-neon-for-an-efficient-development-workflow/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-cedalio-uses-neon-for-an-efficient-development-workflow/neon-cedalio-1-1024x576-cec779f0.jpg) + +
+

We love Postgres, and for us, being able to use Postgres database branches that can be integrated with our CI/CD pipeline is a killer feature. The ability to spawn databases that can scale down to zero is also incredibly helpful. This model fits well with our one database per customer architecture.

+Guido Marucci, co-founder at Cedalio +
+ +[Cedalio](https://cedalio.com/) (YC23) is a data platform that focuses on sustainability and transparency by leveraging AI and smart contracts. It offers a robust solution for companies needing to streamline their Environmental, Social, and Governance (ESG) reporting and data verification processes, automating the entire experience and making the data untamperable. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-cedalio-uses-neon-for-an-efficient-development-workflow/cedalio-platform-1024x650-5b96e39b.png) + +## Making sustainability information auditable by default + +[Greenwashing](https://www.un.org/en/climatechange/science/climate-issues/greenwashing#:~:text=By%20misleading%20the%20public%20to,some%20more%20obvious%20than%20others.), the practice of providing misleading information about the environmental benefits of a product or service, is becoming increasingly common — companies need to find a user-friendly way to certify their data so they can confidently support their sustainability claims. + +Cedalio is building a platform to help companies do this, and they’re doing it by leveraging two cutting-edge technologies: AI and smart contracts. + +- Most companies still rely on manual data gathering for audits, which is a very time-consuming process. Cedalio improves operational efficiency by automating data extraction using AI, centralizing information for immediate reporting through dashboards and control panels, and managing suppliers with automated tracking. +- The Cedalio platform also ensures data transparency by organizing and structuring information from various sources while preserving the origin as evidence. Companies can store data on a blockchain, providing an auditable record of each piece of data. Every change made to the stored information is tracked, identifying who made the change and what was changed. + +## Neon vs RDS: the quest for a developer-friendly Postgres + +When they first started building their platform, Cedalio used DynamoDB as their database – but the team was a fan of Postgres, and as soon as they realized that Postgres would be enough to handle their load, they decided to transition. + +Cedalio wanted to use a managed Postgres that was easy to use and implement within their stack. They first considered RDS, but the developer experience was not quite there. The AWS console and APIs were quite convoluted and required extensive setup and configuration to achieve even basic tasks. Not ideal for a startup that needs to quickly iterate. + +After trying out Neon, the team at Cedalio much preferred Neon’s intuitive API and user interface, which offered a big improvement in developer experience compared to RDS. Neon also offered game-changing features for accelerating development workflows and reducing costs of non-prod environments: database branching and scale to zero. + +## Adopting database branching workflows via GitHub Actions and Fly.io + +Cedalio **[uses Neon branches to automate the setup of isolated environments for each PR](https://neon.tech/flow): when a developer creates a PR, GitHub Actions trigger the creation of a new database branch in Neon and deploy a corresponding application instance on Fly.io.** This isolated setup allows the Cedalio engineers to test changes without affecting the production database and without doing any manual work. + +Once the PR is approved and merged, GitHub Actions automates the deployment of changes to the production environment and cleans up the temporary database branch and application instance. This workflow accelerates the development cycle while reducing manual steps. + +An important benefit of using Neon for this workflow is **[scale to zero](https://neon.tech/blog/why-you-want-a-database-that-scales-to-zero),** which **makes these non-production database branches very affordable.** Since these temporary environments are often only needed during active development and testing, the ability to scale down to zero resources when not in use significantly reduces costs. Cedalio can maintain multiple database branches for different PRs without worrying about the bill. + +## Ensuring data isolation with a DB-per-customer design + +Another important capability of Neon is how [**it facilitates one-database-per-customer architectures via its flexible and user-friendly API**.](https://neon.tech/blog/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers) This is important for Cedalio, as they need this architecture to ensure complete data isolation and enhance security – something important given the company’s background in blockchain and user-owned data. + +By using dedicated databases, Cedalio’s developers also avoid the potential errors associated with (for example) filtering data for different clients. This design also makes it easier to implement and test features, ensuring that changes in one customer’s environment do not inadvertently affect others. + +## The Cedalio platform architecture + +Now, let’s take a closer look at the [Cedalio platform](https://docs.cedalio.com/): + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-cedalio-uses-neon-for-an-efficient-development-workflow/cedalio-architecture-1024x591-70b96f5a.png) + +The centralized hub is the Studio, where users (primarily non-developers) interact with various data sources. It is designed to facilitate the management of sustainability and provenance information through an intuitive interface. Users can initiate data collection processes using custom forms, integrate APIs for automatic data imports, and leverage AI assistants for processing various documents. + +- The `AI Data Processor Module` extracts relevant information from uploaded utility invoices, with plans to expand to certificates, purchase orders, and more complex documents. +- The `Data Validation & Heuristic Module` runs heuristics and validations on extracted information to ensure accuracy and consistency across documents. +- The `Auditable Storage Module` Manages storage of processed information, tracking changes and enabling public auditing via blockchain, currently supporting electricity, gas, and water utilities. +- The `Audit View` provides access to data points via the Document AI API or a link generating an HTML page, displaying the entire change history and original document for auditing purposes. +- + +Cedalio’s backend design has undergone significant design changes recently. The new architecture leverages the Postgres WAL to ensure that all operations within the database are captured and secured: + +- The Studio hooks into Postgres WAL to capture all operations executed in the database. +- All operations are bundled together, encrypted, and optionally persisted into decentralized storage solutions like IPFS or Arweave. +- The encrypted bundle of operations is hashed, and this hash is committed to a smart contract. +- The smart contract mirrors the client database and is owned by a wallet controlled by the client. This setup provides an independent verification mechanism for clients and third parties to ensure that the database’s final state has not been tampered with. + +By persisting WAL operations and database checkpoints in decentralized storage, Cedalio ensures that data recovery can occur independently of Cedalio’s infrastructure. Clients also have the flexibility to choose other storage methods for backups, such as S3 or their internal data services, enhancing the reliability and accessibility of their data. + +## Wrap up + +If you want to learn more about Cedalio, you can book yourself some time with the team [here](https://calendly.com/lucianareznik/30min?month=2024-05). And if you’re also on the hunt for a developer-friendly Postgres, [give Neon a go](https://console.neon.tech/signup) — you can get started with our [free tier,](https://neon.tech/pricing) and take it from there as you scale. diff --git a/content/blog/posts/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora.md b/content/blog/posts/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora.md new file mode 100644 index 0000000000..cb28ecf2f0 --- /dev/null +++ b/content/blog/posts/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora.md @@ -0,0 +1,111 @@ +--- +title: How Dispatch speeds up development with Neon while keeping workloads on Aurora +description: They use Neon branches for their ephemeral environments +excerpt: >- + “Neon’s branching paradigm has been great for us. It lets us create isolated + environments without having to move huge amounts of data around. This has + lightened the load on our ops team, now it’s effortless to spin up entire + environments.”Jonathan Reyes, Principal Engineer at Dis... +date: '2024-10-14T15:31:55' +updatedOn: '2024-12-24T18:34:00' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + How Dispatch speeds up development with Neon while keeping workloads on + Aurora - Neon + description: >- + Learn how Dispatch combines Neon with Aurora Global to accelerate + development by using Neon branches as ephemeral environments. + keywords: [] + noindex: false + ogTitle: >- + How Dispatch speeds up development with Neon while keeping workloads on + Aurora - Neon + ogDescription: >- + Learn how Dispatch combines Neon with Aurora Global to accelerate + development by using Neon branches as ephemeral environments. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-dispatch-speeds-up-development-with-neon-while-keeping-workloads-on-aurora/neon-dispatch-1-1-1024x576-d7b49904.jpg) + +
+

“Neon’s branching paradigm has been great for us. It lets us create isolated environments without having to move huge amounts of data around. This has lightened the load on our ops team, now it’s effortless to spin up entire environments.”



Jonathan Reyes, Principal Engineer at Dispatch

+
+ +[Dispatch](https://www.dispatchit.com/) is a technology company that simplifies [last-mile delivery]() for businesses by streamlining the process of tracking deliveries, coordinating drivers, and communicating with customers. The Dispatch platform solves the tricky challenge of efficiently managing the final stage of delivery in many different locations at once, which is usually the most complex and expensive piece of the puzzle for large enterprises. + +Founded in 2016, Dispatch operates in over 75 U.S. markets, providing on-demand delivery solutions via its network of independent contractor drivers and courier partners​. + +
+ +Image + +
Explore their success stories
+
+ +## Adopting serverless to alleviate overprovisioning and accelerate development + +Dispatch runs its database infrastructure on AWS Aurora Global. While Aurora offers multi-region availability and handles traffic relatively well, it’s not the most cost-efficient solution to handle their spiky traffic patterns. + +During peak hours in the morning, when most orders come in, **Dispatch had to overprovision Aurora by 10x their average usage.** This overprovisioning was necessary because Aurora only includes a single writer, and the writer database would often become the single point of failure. + +
+

“We had to overprovision Aurora to handle our spiky traffic, and even then, the writer database would get overwhelmed. We provision 10x more than we need on average to keep things running smoothly”



Jonathan Reyes, Principal Engineer at Dispatch

+
+ +Dispatch is looking to Neon’s serverless writer endpoints to help alleviate this pain point. At the same time, they’re  accelerating their SDLC workflow while reducing cots by using Neon branches to create their development environments. + +More on this below: + +## How Dispatch uses Neon for their ephemeral environments + +The Dispatch team is transitioning from a monolithic to a microservices architecture. As part of this transition, they wanted to create a more agile workflow that would allow them to replicate a portion of their workload data (excluding PII) for testing in ephemeral environments. + +
+

“Getting realistic data into our verification environments was largely unfeasible, it was time-consuming, expensive, and a beast to maintain. You need to process hefty backups, transfer costs stack up, and there’s a lot of manual oversight required just to move that data”



Jonathan Reyes, Principal Engineer at Dispatch

+
+ +**To improve their development experience, Dispatch started using Neon as their ephemeral environment database.** They didn’t immediately migrate workloads off Aurora—a huge undertaking—but began experimenting with Neon as a more agile solution for their various workloads. + +Here’s a summary of how their deployment looks now: + +### Scheduled syncing of environments via AWS DMS + +The Dispatch team synchronizes their development environments with a safe, SOC 2-compliant subset of various workloads on a scheduled basis. To do this, they use [AWS Database Migration Service (DMS)](https://aws.amazon.com/dms/?p=ft&c=mg&z=3) to replicate data from their Aurora cluster into a main branch in Neon. + +This scheduled sync gives them an up-to-date snapshot of their data, which is then used for testing and development. As they move toward more advanced workflows, they are considering transitioning to streaming replication to enable real-time data syncing, especially for scenarios like shadow environments​. + +### Neon branches for each environment + +Neon’s [branching model](https://neon.tech/flow) allows Dispatch to create isolated environments for testing without having to manually “copy” large datasets into new instances. Developers or QA team members can quickly spin up a new environment by running a simple command with their internal tooling. This environment includes a safe, sanitized, subset of data and is available immediately. + +These branches are isolated—any changes or destructive tests are confined to the branch. For instance, feature teams that need access to real-world data for A/B testing or debugging can easily create a branch and conduct tests without worrying about affecting any other environments. + +### Managing branches via Kubernetes + +Dispatch has integrated Neon’s branching into their Kubernetes infrastructure. They’ve built a custom Neon operator for Kubernetes that automatically creates branches whenever a new ephemeral environment is spun up. + +These ephemeral environments are a key part of Dispatch’s development process. They can quickly spin up entire stacks for testing, which include isolated instances of Neon’s databases. This isolation allows them to perform resource-heavy operations without putting additional load on other critical workloads. + +### Neon’s read replicas for ETL and analytics + +In addition to their primary workflow, Dispatch uses Neon’s [read replicas](https://neon.tech/docs/introduction/read-replicas) (read-only endpoints) to make their data pipelines more efficient. Previously, they ran both critical workloads and ETL jobs on the readers of their Aurora Global cluster, which caused frequent issues. Now, they create read-only replicas in Neon for specific use cases. For example, team members who need reports can run live data queries in tools like Metabase without having to wait for ETL pipelines to finish. + +## If you’re running Postgres in AWS, give Neon a try + +**It may not be the right time to move your workload off your current database, but you can still benefit from Neon’s DX for your dev/test environments.** If you haven’t used Neon before, [start by creating a free account](https://console.neon.tech/signup) and spin up a few branches. + + +If you're looking for a detailed comparison of Neon vs Aurora Serverless v2, check out [neon.tech/aurora](https://neon.tech/aurora). + diff --git a/content/blog/posts/how-magic-circle-scaled-up-to-2m-games-with-cloudfare-and-neon.md b/content/blog/posts/how-magic-circle-scaled-up-to-2m-games-with-cloudfare-and-neon.md new file mode 100644 index 0000000000..d0c294d322 --- /dev/null +++ b/content/blog/posts/how-magic-circle-scaled-up-to-2m-games-with-cloudfare-and-neon.md @@ -0,0 +1,129 @@ +--- +title: How Magic Circle Scaled Up To 2M Games With Cloudflare and Neon +description: >- + The team ships new iterations daily supported by their serverless infra. The + cherry on top: Neon branching +excerpt: >- + “We’re a small team, but we’re scaling quickly and doing a lot. We’re shipping + multiple times a day— to do that, we need to test stuff quickly and merge to + main very quickly as well. Neon branches are a game changer for this.” (Avi + Romanoff, Founder at Magic Circle) Magic Circle... +date: "2024-12-12T18:28:46" +updatedOn: "2024-12-12T21:03:34" +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-magic-circle-scaled-up-to-2m-games-with-cloudfare-and-neon/cover.jpg + alt: null +isFeatured: false +seo: + title: How Magic Circle Scaled Up To 2M Games With Cloudflare and Neon - Neon + description: >- + The Magic Circle team ships new iterations of their games daily supported by + their serverless infra running on Cloudflare and Neon. + keywords: [] + noindex: false + ogTitle: How Magic Circle Scaled Up To 2M Games With Cloudflare and Neon - Neon + ogDescription: >- + The Magic Circle team ships new iterations of their games daily supported by + their serverless infra running on Cloudflare and Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-magic-circle-scaled-up-to-2m-games-with-cloudfare-and-neon/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-magic-circle-scaled-up-to-2m-games-with-cloudfare-and-neon/neon-magic-circle-1-1024x576-a2795d67.jpg) + +
+

“We’re a small team, but we’re scaling quickly and doing a lot. We’re shipping multiple times a day— to do that, we need to test stuff quickly and merge to main very quickly as well. Neon branches are a game changer for this.”

(Avi Romanoff, Founder at Magic Circle)

+
+ +[Magic Circle](https://magiccircle.studio/) is a new game studio that helps you connect with friends online through a collection of fun multiplayer games, available on the web and inside Discord. They’re building a relaxed, wonderfully silly low-pressure environment where friend groups hang out online and take a break from their busy schedules. + +
+ +
+ +Last July, Magic Circle won [Discord’s “Best Chill/Party Game” competition](https://discord.com/blog/discord-app-pitches-2024-category-winners#heading-5). Their games are now accessible from any Discord server, private messages, or even group video calls—perfect for when you and your team need a break: + +
+ +Image + +
Magic Circle also has a Discord server (duh). Join it here
+
+ +## Scaling to millions of games with an infra bill below $500 + +
+

“I’ve been building multiplayer games for a long time, and the database question has always been an interesting one. Traffic is often spikey, so in an ideal world, you’d have many small servers that spin up and down based on demand—but the challenge is creating a database that matches that setup, yet still acts as a single source of truth. The solution for us has been Neon and Cloudflare Durable Objects.”

(Avi Romanoff, Founder at Magic Circle) 

+
+ +Magic Circle’s entire infrastructure is designed to be lightweight and cost-efficient: a serverless architecture was a no-brainer. **By combining Cloudflare and Neon, Magic Circle facilitates millions of game sessions and requests at a fraction of the cost of traditional setups.** + +
+ +Image + +
Source
+
+ +This is how they do it: + +- [Cloudflare Workers](https://workers.cloudflare.com/) **for game logic.** These edge workers minimize latency for players across 30+ countries by processing requests and responses closer to users. +- [Cloudflare Durable Objects](https://developers.cloudflare.com/durable-objects/) **for session management.** Durable Objects are a cornerstone of Magic Circle’s architecture. They handle state management for multiplayer sessions, allowing each game session to act effectively as a “tiny server”. +- [Routing](https://developers.cloudflare.com/workers/configuration/routing/routes/) **for versioning**. Cloudflare also plays a role in routing players to the correct version of the game. This allows different groups to play on different versions simultaneously, essential for Magic Circle’s fast development cycles (more about this below). +- [Neon](https://neon.tech/home) **as the relational database.** Neon is the serverless Postgres database storing core player data like accounts, game progress, stats, and in-game currency. Gameplay logic is managed at the edge, and Neon handles all persistent info. + +## Bringing fast software lifecycles to gaming + +
+

“While games typically ship on multi-week release cycles, we want to ship as fast as possible — so we built a pretty cool in-house game platform on top of Cloudflare and Neon that lets us deploy multiple times a day, in minutes, with zero downtime.”

(Avi Romanoff, Founder at Magic Circle)

+
+ +**Something unique about Magic Circle’s development approach is how they bring a rapid iteration strategy—typically seen in web development—to the world of multiplayer gaming.** Their workflow enables them to ship updates multiple times a day, with [Neon branches](https://neon.tech/docs/introduction/branching) serving as a cornerstone of this process. + +In Neon, users can instantly create database branches, which act as fully isolated copies of the database, including both data and schema. These branches are widely used for [creating development and testing environments](https://neon.tech/use-cases/dev-test) that are ephemeral and cost-effective, allowing teams to iterate quickly on features, fix bugs, or test database schema changes. + +This is exactly how Magic Circle uses branching. Not only do they iterate multiple times a day, but they also frequently ship entirely new games in under a week. This process involves updating game logic, introducing new UI elements, and modifying database structures to support new gameplay mechanics. Such speed would be nearly impossible with a traditional managed database setup tied to a monolithic server, but Neon brings them a more modern workflow where the database is no longer a bottleneck. + +## How Magic Circle uses Neon branches to ship faster + +
+

“We have a bot that comments on every pull request on GitHub every time there’s a new PR branch. We get a totally isolated copy to test code changes even when they include database migrations. We can test all changes in real data and ensure that by the time we actually merge the PR to main, things really work” 

(Avi Romanoff, Founder at Magic Circle)

+
+ +Let’s take a closer look at Magic Circle’s branching workflows. + +**1/ A Neon branch is created per every PR using the GitHub Integration.** + +- Each pull request in GitHub automatically triggers the creation of a dedicated Neon branch via a GitHub Action. +- Any schema migrations included in the PR are executed against this branch, ensuring they work as intended without disrupting production. +- A GitHub bot comments on the PR with the branch details. + +**2/ End-to-end testing is done against a Neon branch first.** + +- Neon branches are fully integrated into Magic Circle’s infrastructure, enabling end-to-end testing in environments that mimic production. +- The testing setup includes the PR code, live game servers, and the isolated Neon branch, ensuring thorough validation of new features and database changes. +- This allows the team to validate multiplayer features and gameplay mechanics before merging to prod, ensuring smooth operation across all components without the hiccups associated with testing on seed data. + +**3/ Every developer gets their own Neon branch as an isolated environment.** + +- Each developer in the team also has access to a dedicated Neon branch for their dev needs. +- These branches are linked to personal development environments using [Cloudflare Tunnel,](https://developers.cloudflare.com/cloudflare-one/connections/connect-networks/) allowing developers to run fully isolated versions of the game with a live database backend. +- Developers can test their changes without interfering with others, and each dev environment can be personalized while ensuring that every branch mirrors the production database. + +**4/ Rollbacks are executed fast via branches.** + +- Even if this is not needed often (🤞) Neon’s [Branch Restore](https://neon.tech/docs/guides/branch-restore) makes rollbacks straightforward and reliable without downtime. +- In the very rare case when a problematic commit is indeed merged to production, Magic Circle creates a new Neon branch from a specific point in time to revert to a known good state. +- This process is ready immediately, minimizing potential downtime—something essential in gaming. + +## Try it yourself + +Database branching workflows might seem a bit outlandish at first, but once you try them, [you’ll be converted](https://neon.tech/blog/from-days-to-minutes-how-neo-tax-accelerated-their-development-lifecycle). With Neon’s Free Plan, you can create up to 10 branches per project to get a feel for it—sign up [here](https://console.neon.tech/signup). It only takes seconds and you don’t need a credit card. + +Once you’re set up, don’t forget to play Magic Circle with your friends [in Discord](https://discord.com/application-directory/1227719606223765687) or on the web at [magiccircle.gg](https://magiccircle.gg/). To learn more about them and their company, [give this podcast a listen](https://a16z.com/podcast/next-gen-gaming-ai-souls-real-time-culture-personalized-avatars/). diff --git a/content/blog/posts/how-mindvalley-minimizes-time-to-launch-with-neon-branches.md b/content/blog/posts/how-mindvalley-minimizes-time-to-launch-with-neon-branches.md new file mode 100644 index 0000000000..f0acc1f234 --- /dev/null +++ b/content/blog/posts/how-mindvalley-minimizes-time-to-launch-with-neon-branches.md @@ -0,0 +1,101 @@ +--- +title: How Mindvalley Minimizes Time-To-Launch With Neon Branches +description: Their engineering team is not blocked anymore by the database +excerpt: >- + “Developers already face significant delays when working on a PR—running CI + tests, ensuring everything is ready for preview, it all adds up. Time to + launch is crucial for us: when we tried Neon and saw that spinning up a new + branch takes seconds, we were blown away” (Alex Co, Hea... +date: '2024-10-17T16:24:16' +updatedOn: '2024-10-17T16:35:59' +category: community +categories: + - community + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-mindvalley-minimizes-time-to-launch-with-neon-branches/cover.jpg + alt: null +isFeatured: false +seo: + title: How Mindvalley Minimizes Time-To-Launch With Neon Branches - Neon + description: >- + Mindvalley takes advantage of Neon’s faster DX for development and testing + while keeping their production databases untouched in CloudSQL. + keywords: [] + noindex: false + ogTitle: How Mindvalley Minimizes Time-To-Launch With Neon Branches - Neon + ogDescription: >- + Mindvalley takes advantage of Neon’s faster DX for development and testing + while keeping their production databases untouched in CloudSQL. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-mindvalley-minimizes-time-to-launch-with-neon-branches/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-mindvalley-minimizes-time-to-launch-with-neon-branches/neon-mindvalley-1-1024x576-5bcefab5.jpg) + +
+

“Developers already face significant delays when working on a PR—running CI tests, ensuring everything is ready for preview, it all adds up. Time to launch is crucial for us: when we tried Neon and saw that spinning up a new branch takes seconds, we were blown away” (Alex Co, Head of Platform Engineering at Mindvalley)

+
+ +[Mindvalley](https://www.mindvalley.com/entry-point?otag=mv.com_qaap_VWO3616_V1_desktop) is a global platform that offers a unique approach to personal growth and transformation. With a reach extending to over 11 million lives, it provides courses and content that cover a wide range of topics, from health and fitness to spiritual growth and mental well-being. + +## How Mindvalley uses Neon + +Mindvalley primarily uses Google CloudSQL for their production databases, but they recently started using Neon to speed up their non-production workflows. Previously, everything—including development and testing—was handled in CloudSQL, which introduced inefficiencies that slowed the team down (more details later). + +Mindvalley now takes advantage of Neon’s faster DX for development and testing while keeping their production databases untouched. The key feature driving this boost in developer velocity is [branching](https://neon.tech/docs/introduction/branching), which allows their developers to spin up new previews in less than a minute. [Automatically creating branches with every PR](https://neon.tech/flow#preview-environment-workflow) has drastically reduced waiting times, speeding up their development cycle. + +## Their database branching workflow + +Let’s take a closer look at how Mindvalley uses branching: + +### Nightly data syncs via Github Actions (Neon Twin) + +Mindvalley has built a [Neon Twin](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon) to automate the nightly synchronization between their production database in CloudSQL and a main development branch in Neon. Through GitHub Actions, a nightly dump from CloudSQL is restored in the main branch in Neon without developers having to perform any manual work. + +### Child branches for independent development environments + +From this main branch, hundreds of ephemeral environments can be immediately spun up without requiring any additional data syncs. These child environments feel like perfect “copies” of the main branch, containing all the latest data and schema. Each developer gets their own child branch and can work independently without disrupting others. Developers can [easily propagate updates](https://neon.tech/docs/guides/reset-from-parent) from the master branch to their own environments. + +### Integration with CI/CD for end-to-end testing + +All of this happens through Mindvalley’s in-house CI/CD automation system, which is fully integrated with the Neon API. Whenever developers launch a PR, the automation system automatically creates a new database branch in Neon. Neon’s [connection pooling](https://neon.tech/docs/connect/connection-pooling) ensures there are no issues with too many connections going into the branches. Once the PR is closed, the database branch is deleted. + +## The problem with dev/test on other managed databases: A closer look + +As we said earlier in the post, Mindvalley uses CloudSQL as their main database, but using it also for dev and testing as well proved to be not the best in terms of velocity and efficiency overall. Here’s some of the issues they regularly encountered: + +### Concurrency and state issues + +In a team of 100 developers, using shared database instances for dev/test quickly became a bottleneck, as developers needed to work on different features simultaneously. Testing environments couldn’t often be reused—credentials or test data would change, making it difficult to rerun tests without manually resetting or recreating accounts. Developers had to create new accounts or environments every time they wanted to conduct tests, which meant more work for the operations team. + +### End-to-end testing delays + +For their end-to-end tests, Mindvalley’s engineers (like most devs) wanted the ability to quickly run tests, discard the test data, and revert the environment to its previous state. Their traditional approach in CloudSQL involved manually exporting the database to create backups and importing those backups into new instances for testing. But this process took hours. And much of the operation was obscured by the complexity of the infrastructure—developers aren’t always DBAs. + +### Database seeding maintenance + +Maintaining seed files for local development was another pain point. Each time a new field, column, or schema was introduced in production, seed data needed to be manually updated in all the non-prod environments. + +This problem with seeding is also why Mindvalley chose Neon over Supabase. Like their previous setup, Supabase also required reliance on a seed file to populate all the non-prod environments, since Supabase branches don’t “replicate” data. This didn’t solve the seed file maintenance issue Mindvalley was already facing. Supabase branches also took longer to be ready compared to Neon, making it less effective for speeding up the development lifecycle. + +## Build your own Neon Twin for dev/test + +
+

“We are using the Neon Twin workflow. We just install the GitHub action and it takes care of the rest. Developers may not know how to dump and restore well, but they know how to run a GitHub Action. It’s amazing” (Alex Co, Head of Platform Engineering at Mindvalley)

+
+ +To keep their data in sync with their main setup in CloudSQL and Neon, Mindvalley set up a [Neon Twin](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon). A Neon Twin is a synchronized copy of your testing dataset hosted on Neon, while your main production environment stays elsewhere. + +By scheduling nightly dump/restores via GitHub actions, your developers can access a fresh copy of the dataset every day without manual intervention; this sync is done to the main development branch in Neon, from which many independent child branches can be immediately derived to run tests and building features. You only need to sync your data once, for hundreds of dev environments. + +On a glimpse, the Neon Twin workflow looks like this: + +1. You set up your [Neon account](https://console.neon.tech/signup) and create a project to host your non-production databases. This project will be the home of your Neon Twin. In this project, you create a main branch that will receive the daily dataset updates, and from which all the child branches (for each independent environment) will be derived. +2. To keep the data in the main branch in sync, you automate the dump/restore via GitHub Actions ([we’ve built this action for you](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon#create-the-workflow-file)). +3. Once you’ve tested changes in the Neon environment, you can deploy them back to production in your main database. You can track schema changes in Neon via Prisma or SQLfiles—[we tell you how.](https://neon.tech/blog/neon-twin-deploy-workflow) + +If you’d like to try it out, follow the steps [here](https://neon.tech/blog/optimizing-dev-environments-in-aws-rds-with-neon-postgres-part-ii-using-github-actions-to-mirror-rds-in-neon). [Neon was a Free plan](https://neon.tech/pricing), so you can get started without committing to anything. diff --git a/content/blog/posts/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers.md b/content/blog/posts/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers.md new file mode 100644 index 0000000000..ecb784cb35 --- /dev/null +++ b/content/blog/posts/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers.md @@ -0,0 +1,115 @@ +--- +title: How OpusFlow achieves tenant isolation in Postgres without managing servers +description: Orchestrating hundreds of Postgres databases is effortless with Neon +excerpt: >- + “Our customers require their data to live in an isolated database, but + implementing this in RDS was cumbersome and expensive. We switched over to + Neon to reduce costs and operational overhead” Joey Teunissen, CTO at OpusFlow + The demand for solar panels and sustainable energy inst... +date: '2024-02-22T17:28:50' +updatedOn: '2024-04-17T17:41:52' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers/cover.jpg + alt: null +isFeatured: false +seo: + title: >- + How OpusFlow achieves tenant isolation in Postgres without managing servers + - Neon + description: >- + Architectures demanding one database per customer can get expensive and hard + to manage in RDS, but they're effortless in Neon. + keywords: [] + noindex: false + ogTitle: >- + How OpusFlow achieves tenant isolation in Postgres without managing servers + - Neon + ogDescription: >- + Architectures demanding one database per customer can get expensive and hard + to manage in RDS, but they're effortless in Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-opusflow-achieves-tenant-isolation-in-postgres-without-managing-servers/neon-opusflow-1024x576-5b9577e9.jpg) + +
+

“Our customers require their data to live in an isolated database, but implementing this in RDS was cumbersome and expensive. We switched over to Neon to reduce costs and operational overhead”

+Joey Teunissen, CTO at OpusFlow +
+ +The demand for solar panels and sustainable energy installations keeps increasing. That’s where [OpusFlow](https://opusflow.io/) comes into play, making life easier for the installers on the front lines. First starting as a specialized software agency in the Netherlands around 2019, OpusFlow has quickly grown into a leading ERP tool that simplifies sustainable energy installations. + +“With OpusFlow, you can just draw solar panels on the top of a roof, and we’ll do the rest, including calculations to see what kind of metal and products you need for the installation,” said Joey Teunissen, CTO at OpusFlow. “We help the solar installer send the quotation to their customers, actually install these panels on the roofs, and streamline the entire process end-to-end”. + +## Multi-tenancy in Amazon RDS: a complex and costly design + +Due to the nature of its business, OpusFlow handles sensitive data for its European customers, meaning that it needs to ensure the maximum level of data isolation. In Postgres terms, this means implementing an architecture with one database per customer. + +This architecture doesn’t only assure that data from different customers can never mingle with each other, but it also allows them to create different encryption keys for different databases, scale them independently, run restores for specific customers without affecting operations, and upgrade them at different times if necessary. + +When the OpusFlow team tried to implement this design in Amazon RDS, they soon discovered that it was not going to be as easy as it seemed. First, they tried implementing one big RDS instance hosting many databases. However, this level of isolation was not quite cutting it—for example, all the different databases were still competing for the same resources, something that introduced many problematic edge cases. + +An alternative option was implementing a single-tenancy design, hosting every database into a separate RDS instance. This solution solves the isolation problems, but it creates others—namely, it adds significant overhead both operationally and around costs. Managing a large RDS fleet of hundreds or thousands of instances requires dedicated engineering resources; as Joey put it, **“RDS becomes a bottleneck if you don’t have full-time DevOps dedicated to it”**. + +A fleet of RDS instances also requires big budgets. It implies overprovisioning many times over, which multiplies the bill. + +## The alternative: a fleet of Postgres in Neon + +
+

“In Neon, we use one Neon project per customer for full isolation. For development, testing, and point-in-time restores, we use branches. They don’t cost much and they make everything easier”

+Joey Teunissen, CTO at OpusFlow +
+ +To make things simpler, OpusFlow decided to switch to Neon. + +Today, they use Neon as their main database. They implement a single tenancy system: every OpusFlow customer has their own isolated Postgres database living in a separate Neon project. A backend built on Node.js handles the requests to all databases, also with full isolation, handling authorizations at the database level. + +This setup is seamless to manage in Neon. Since Neon is serverless Postgres, there’s no need to manage servers. There’s no need to allocate storage and compute endpoints are ephemeral and serverless. [Every customer database gets scaled independently](https://neon.tech/docs/introduction/autoscaling), consuming only the compute resources they need. When inactive, compute endpoints go idle automatically. This reduces the monthly bill. + +Besides, Neon supports [database branching](https://neon.tech/docs/introduction/branching) at a project level. Branches act as immediately available, isolated database copies with up-to-date production data. They work using a copy-on-write mechanism, meaning that data isn’t duplicated across branches. For example, OpusFlow leverages branching to offer instantaneous [point-in-time restores](https://neon.tech/docs/guides/branch-restore) to their customers. Branching allows the immediate recovery to previous database states in case of accidental data corruption at no extra cost, being an affordable and user-friendly alternative to maintaining expensive replicas or waiting for restores from backup. + +
+Image +
In a single tenancy design in Neon, every customer gets its own Postgres database within a Neon project. Due to its serverless nature, managing a fleet of Neon projects is simple and affordable. Instantaneous PITR can be run in isolation.
+
+ +OpusFlow also has dedicated Neon projects for testing and development, [where each developer also gets their own branch](https://www.youtube.com/watch?v=zD1_AzzLRH0). This setup mirrors the structure of their Git workflow, ensuring that every new feature or bug fix is developed in an isolated environment. This approach not only streamlines the development process by allowing individual testing without affecting others but also reduces the dependency on local machines, increasing safety. + +
+Image +
OpusFlow also uses dedicated Neon projects for development and testing, where every developer gets its own branch. Branches work on copy-on-write, so they don’t imply extra storage costs unless they diverge from their parent. Only active compute time is billed. 



+
+ +## The OpusFlow serverless stack + +
+

“Our stack is almost 100% serverless. As we are in the sustainable industry, we’d like to be sustainable as well when it comes to our service—when we don’t use things, we turn them off. We like that concept”

+Joey Teunissen, CTO at OpusFlow +
+ +
+Image +
The OpusFlow serverless architecture
+
+ +OpusFlow’s serverless tech stack is built around TypeScript. At the user interface level, they use Next.js, with Clerk handling secure authentication flows. The backend is powered by Hasura, which layers a dynamic GraphQL API on top of Postgres databases, ensuring real-time data interactions. Each tenant enjoys a distinct Postgres database within their own Neon project, ensuring data isolation and integrity. + +For serverless functions, AWS Lambda takes care of executing event-driven, custom TypeScript logic without provisioning or managing servers. This aligns with the infrastructure’s ability to respond flexibly to varying load. Data is mirrored in cold storage, and the architecture is complemented by a suite of automation that orchestrates CI/CD, monitoring, and other operational tasks. + +## Give Neon a try + +
+

“Neon is very easy to use. You create an account and a project, you get a database string, and that’s that. It’s still the Postgres that you’re used to”

+Joey Teunissen, CTO at OpusFlow +
+ +If the idea of an easy to orchestrate, single-tenancy design for Postgres sounds interesting, [request an Enterprise free trial.](https://neon.tech/enterprise#request-trial) You’ll get temporary full access to Neon for your business without resource limitations, so you can create many projects and get a feel for the experience. + +Thank you to [OpusFlow](https://opusflow.io/) for sharing their story! Best of luck as you continue to expand your business across Europe and overseas. Neon will be with you every step of the way. diff --git a/content/blog/posts/how-psql-pg-neon-tech-works.md b/content/blog/posts/how-psql-pg-neon-tech-works.md new file mode 100644 index 0000000000..2b87554ccb --- /dev/null +++ b/content/blog/posts/how-psql-pg-neon-tech-works.md @@ -0,0 +1,108 @@ +--- +title: How Does psql -h pg.neon.tech Work? +description: >- + Many of us here at Neon are big fans of our “passwordless auth” feature. Let’s + find out how it works! +excerpt: >- + One of the things that got me really excited and curious about Neon, long + before I joined the company, was the psql -h pg.neon.tech command. I’ve + recently come to find out that other employees here went through the same + “wow” experience even before they joined the company. So, I’... +date: '2024-08-14T14:22:47' +updatedOn: '2024-08-14T14:22:48' +category: engineering +categories: + - engineering +authors: + - david-gomes +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-psql-pg-neon-tech-works/cover.jpg + alt: null +isFeatured: false +seo: + title: How Does psql -h pg.neon.tech Work? - Neon + description: >- + Many of us here at Neon are big fans of our “passwordless auth” feature. + Let’s find out how it works! + keywords: [] + noindex: false + ogTitle: How Does psql -h pg.neon.tech Work? - Neon + ogDescription: >- + One of the things that got me really excited and curious about Neon, long + before I joined the company, was the psql -h pg.neon.tech command. I’ve + recently come to find out that other employees here went through the same + “wow” experience even before they joined the company. So, I’ve decided to + write a little bit […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-psql-pg-neon-tech-works/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-psql-pg-neon-tech-works/neon-psql-550eec13.jpg) + +One of the things that got me really excited and curious about Neon, long before I joined the company, was the `psql -h pg.neon.tech` command. I’ve recently come to find out that other employees here went through the same “wow” experience even before they joined the company. So, I’ve decided to write a little bit about [how to use it](https://neon.tech/docs/connect/passwordless-connect), how it works and why it’s so useful. + +When you run `psql -h pg.neon.tech` in your CLI, you get a URL that you can click on: + +![Image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXdZP4irPADDgqA7AcwgW-rzK4m7nyqtOB5dhcGPbUmwkCpFeN3ARINE3eGD9jw2hTVB2r_e35cUmlMYNmEBJvyflfWzJfFvy0sHBPsKszZUZkeDKEsiYvy_QTIbrMLXxkS54bv-ftW_tOrLuUlySp4M2Gtk?key=8VwwnKkSiU0ZCKUZnA-siA) + +And then by clicking on that URL, you are taken into the Neon console, where you can choose which Neon project you’d like to connect to. If your project only has one endpoint, then the flow is really quick, but otherwise you have to specify which endpoint you want to connect to. + +![Image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXcTeC4JGYgN4RKAFCgAv4NRFqOrPeKzB0Ku_ZV6dkLeCQXTp1_B4muRcAe09rbDruIFl-MKl5vAs7tnzS9T7f3yntBH-v6zztWDzPCzzHrePxP3vTtI5U5CxmfD8BcXl7l6pdRKM1FGcDl3o3yCNWvoqSc?key=8VwwnKkSiU0ZCKUZnA-siA) + +![Image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXdcZM1yQOBSlb-IsDC3pgk1CKTRhHLibWhOkOQFj1N_CfQdwGzRdPtASTLE00Qk1dCcsc3nJKO5QjM9yV6-YzN-KkwCxsj8E4gpdood_5llBClrQaqxHGBJxjh8wmvmAJla-3t_C0ljNPyUzhK9Yk7kPOBh?key=8VwwnKkSiU0ZCKUZnA-siA) + +![Image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXdjH6klAZgqXkuwaM9lTVEbwlmDVEVxuy-AKmz1_c2RG8g0_WAy_EukWUPnXpHSN0lCthDthdWpXNuQRFsRmWKC1RfKZCsVfLihixvF8GlJ3I2-oFZs-QFaIHwlwAE1Gq2oovDHbVYvkiXbtJS0Ib1gpg-E?key=8VwwnKkSiU0ZCKUZnA-siA) + +And then you’re in! + +![Image](https://lh7-rt.googleusercontent.com/docsz/AD_4nXcFDWPgpe3jQYbrmA4LOb-ovl4eatAj6gmochxDO0PFVMaERojccgF_JEUzbe13Lyen2Mlk9kDbGVqb0lSuvDvu0mH8Q53BFjDGDbhKi4o7zhw5SlYvL5yhvYxyFmT3uWzhj-ZEvn8-gwxprJFVaHgCkYMD?key=8VwwnKkSiU0ZCKUZnA-siA) + +## How does it work? + +Once I joined Neon, I decided to try and figure out how this feature works under the hood. And it all starts with what’s running at `pg.neon.tech`. Is it a real Postgres instance? No, it’s not — it’s just a server that speaks Postgres protocol that we call **"Link Proxy"**. + +**But how can this Postgres server make the client print that “Welcome to Neon!” message?** + +So, the Postgres protocol has a “[message flow](https://www.postgresql.org/docs/current/protocol-flow.html)” that can be used for many different things. One of the message types that the server can respond with to the “Start-up” message from the client is “NoticeResponse”. As per the docs, this is what it should be used for: + +
+

A warning message has been issued. The frontend should display the message but continue listening for ReadyForQuery or ErrorResponse.

+
+ +This is the message type that our server uses to print the “Welcome to Neon!” message. It’s a bit of a hack but it works! + +And **because Neon is open source**, you can actually read the code [here](https://github.com/neondatabase/neon/blob/507f1a5bdd4a168e589550e7c1bb5ac6de41643f/proxy/src/auth/backend/link.rs#L85): + +```rust +let greeting = hello_message(link_uri, &psql_session_id); + +// Give user a URL to spawn a new database. +info!(parent: &span, "sending the auth URL to the user"); + +client + .write_message_noflush(&Be::AuthenticationOk)? + .write_message_noflush(&Be::CLIENT_ENCODING)? + .write_message(&Be::NoticeResponse(&greeting)) + .await?; +``` + +So, now that we’ve figured out how to print the initial message, we can see that it includes a special URL that the user can visit in order to authenticate to a Neon project/compute endpoint. **What’s special about this URL that the “Link Proxy” generates?** + +Just like most “database as a service” products, Neon has a control plane <-> data plane architecture, and this separation of concerns provides many benefits for us. The **control plane** is responsible for orchestrating the data plane and managing metadata around organizations, user accounts, preferences, billing, etc. On the other hand, the **data plane** is essentially the Postgres compute and storage. The “**Link Proxy**” service is actually running in the **data plane**, and it generates a URL that takes the user to a UI powered by the **control plane**, where they can choose the Neon project/endpoint that they wish to connect to. + +Then, the **control plane** connects to the “**Link Proxy**” service and feeds to it the necessary connection details for the request ID that was in the URL. This, in turn, causes the Link Proxy to initiate a connection to the correct IP with the appropriate credentials. The “**Link Proxy**” also forwards all the connection’s traffic (i.e., messages to and from Postgres) to the “psql” client. + +That’s it! + +Confusing? That’s okay, we’ve got a diagram that should help clear things up: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-psql-pg-neon-tech-works/psql-h-pg-neon-tech-diagram-14f15216.png) + +## Why is this command so interesting? + +This command is extremely useful for developers playing around with more than one Neon project, or multiple [compute endpoints](https://neon.tech/docs/manage/endpoints) inside the same Neon project. Most of us are not keeping track of the URLs or credentials for all the endpoint/databases that we’re connecting to in our development workflow. So, it’s simply much easier to just type `psql -h pg.neon.tech` (or even better, to have a `pgneon` alias in your CLI for this!). + +Finally, both the regular Neon Proxy as well as the “Link Proxy” described in this article are [open source](https://github.com/neondatabase/neon/tree/507f1a5bdd4a168e589550e7c1bb5ac6de41643f/proxy). So, feel free to poke around in the codebase and to ask us any questions you might have on its architecture [on our Discord server](https://neon.tech/discord). + +We’re also hiring—take a look at our [open engineering positions](https://neon.tech/careers) and help us shape the future of AI and Postgres. diff --git a/content/blog/posts/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand.md b/content/blog/posts/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand.md new file mode 100644 index 0000000000..efe32b1a11 --- /dev/null +++ b/content/blog/posts/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand.md @@ -0,0 +1,87 @@ +--- +title: How Recrowd uses Neon autoscaling to meet fluctuating demand +description: >- + Autoscaling ensures optimal performance when traffic peaks, avoiding the need + for overprovisioning or manual resizes +excerpt: >- + “We use Neon for its great autoscaling capabilities. It allowed us to solve a + huge problem with simple steps” Pieralberto Colombo, CTO at Recrowd Recrowd is + Italy’s leading crowdfunding platform, empowering small investors to + participate in real estate projects with just a few cl... +date: '2024-03-01T16:35:34' +updatedOn: '2024-03-01T16:35:38' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand/cover.jpg + alt: null +isFeatured: false +seo: + title: How Recrowd uses Neon autoscaling to meet fluctuating demand - Neon + description: >- + To assure good database performance during peak traffic, it's often needed + to overprovision or to manually resize instances—not in Neon. + keywords: [] + noindex: false + ogTitle: How Recrowd uses Neon autoscaling to meet fluctuating demand - Neon + ogDescription: >- + To assure good database performance during peak traffic, it's often needed + to overprovision or to manually resize instances—not in Neon. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand/social.jpeg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-recrowd-uses-neon-autoscaling-to-meet-fluctuating-demand/neon-recrowd-1024x576-253a7d50.jpg) + +
+

“We use Neon for its great autoscaling capabilities. It allowed us to solve a huge problem with simple steps”

+Pieralberto Colombo, CTO at Recrowd +
+ +[Recrowd](https://www.recrowd.com/) is Italy’s leading crowdfunding platform, empowering small investors to participate in real estate projects with just a few clicks. They’re committed to transparency and security for its investors, ensuring quality and reliability, and offering investment opportunities from as low as €250. + +## The scalability dilemma + +
+

“When we were using MySQL in Azure, we had to manually upgrade the database during the days of peak traffic and downgrade later in the day, which caused a couple of minutes of downtime. It was also a huge waste of time for our team”

+Pieralberto Colombo, CTO at Recrowd +
+ +Like it is the case for many applications, Recrowd’s traffic patterns are variable. Right when a new investment project opens, something that typically happens a few times per month, the traffic in Recrowd’s website skyrockets, as thousands of investors request to invest in the new opportunity. These days, the requests to the database are orders of magnitude above average. But it’s precisely at this moment when the database needs to perform at its best to assure a great customer experience. + +This is a very common situation that traditional managed databases don’t solve well. When you are required to provision compute instances in advance, you’re essentially forced to make a bet on the level of resources you’ll need to handle your peak traffic. But if your use case involves periods of high traffic followed by longer periods of low activity, you’re faced with a dilemma: + +- You can overprovision—in other words, you can choose to always have enough capacity to handle peak demand. This ensures you’re never caught short during high-traffic events, but it’s an expensive approach. Most of the time, you’ll be paying for resources you don’t use. +- Your second option is to manually upgrade and downgrade compute capacity. This is more cost-conscious; you would scale your resources up just before you anticipate a surge in traffic, and scale down afterward. The problem with this approach is that it requires monitoring and manual work. You have to predict and execute the scaling up at the right moment. Also, this manual intervention is not instantaneous—most systems take some time to resize, which may cause downtime. + +## Scaling up and down automatically: meeting fluctuating demand with Neon + +
+

“Thanks to autoscaling, we no longer have to worry about performance during our openings or wasting time resizing instances”

+Pieralberto Colombo, CTO at Recrowd +
+ +The Recrowd team decided to move away from the scalability dilemma by switching to Neon. Being serverless Postgres, Neon comes with [autoscaling](https://neon.tech/docs/introduction/autoscaling), a feature that dynamically adjusts compute resources to match current demand without manual intervention. This makes it seamless to manage traffic peaks without the manual overhead, downtime, or cost inefficiencies. + +From a user’s perspective, the process is straightforward and requires minimal intervention: + +- Neon continuously monitors the database’s workload. It automatically detects when the current demand exceeds the capacity of the allocated resources, signaling the need for scaling up. +- Once a need for scaling is detected, Neon dynamically adjusts the compute resources, which become available almost instantly. When demand diminishes, resources are scaled down. +- This elastic scaling happens with zero downtime. Applications remain accessible, and database operations continue without interruption. + +Users can set [autoscaling limits](https://neon.tech/docs/manage/endpoints#compute-size-and-autoscaling-configuration) to control how aggressively or conservatively resources are scaled. + +## Give it a try + +
+

“We’re excited for what’s coming up this year at Recrowd. Neon will be our database partner”

+Pieralberto Colombo, CTO at Recrowd +
+ +If you also have variable traffic, [request a Neon Enterprise trial](https://neon.tech/enterprise#request-trial) and experiment with autoscaling completely for free for 30 days. + +Best of luck to Recrowd as they venture into new territories and continue to disrupt the real estate investment space! diff --git a/content/blog/posts/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases.md b/content/blog/posts/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases.md new file mode 100644 index 0000000000..7b58b72047 --- /dev/null +++ b/content/blog/posts/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases.md @@ -0,0 +1,107 @@ +--- +title: How Retool uses Retool (and the Neon API) to manage 300K+ Postgres databases +description: Retool manages a massive database fleet with only one engineer +excerpt: >- + Neon is a serverless Postgres database with a robust API. Partners like Retool + choose Neon to easily offer managed databases to their end-users, simplifying + the management of massive database fleets while optimizing costs. Thanks to + Neon, Retool is able to manage over 300k Postgr... +date: "2024-03-29T18:03:27" +updatedOn: "2026-03-13T15:57:15" +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases/cover.jpg + alt: How Retool manages 300K+ databases +isFeatured: false +seo: + title: >- + How Retool uses Retool (and the Neon API) to manage 300K+ Postgres databases + - Neon + description: >- + RetoolDB is powered by Neon, which streamlines the management of massive + database fleets while keeping costs in check. + keywords: [] + noindex: false + ogTitle: >- + How Retool uses Retool (and the Neon API) to manage 300K+ Postgres databases + - Neon + ogDescription: >- + RetoolDB is powered by Neon, which streamlines the management of massive + database fleets while keeping costs in check. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases/cover.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases/how-retool-manages-300k-databases-1024x538-c7fa744a.jpg) + +**Neon is a serverless Postgres database with a robust API. [Partners](https://neon.tech/partners) like Retool choose Neon to easily offer managed databases to their end-users, simplifying the management of massive database fleets while optimizing costs. Thanks to Neon, Retool is able to manage over 300k Postgres databases with just one engineer!** + +The [Retool](https://retool.com/) platform makes it easy for developers to create internal apps for their teams and businesses. Retool handles much of the boilerplate code and UI design automatically, empowering developers to deliver effective business software up to 10x faster. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases/web-apps-2-b490d24f.gif) + +Retool’s drag-and-drop interface allows users to quickly assemble UIs from a set of pre-built components such as tables, buttons, and forms, supporting custom code for the specific functionalities of each app. The platform makes it easy to connect these UI components to nearly any API, streamlining the development process. + +This smooth experience empowers teams to build a wide range of internal tools. A few examples of what you can build with Retool: + +- [DevOps dashboards:](https://retool.com/templates/dev-ops-dashboard) to visualize and analyze software development, testing, deployment, and monitoring for faster and more efficient software delivery. +- [Inventory management dashboards:](https://retool.com/use-case/inventory-management-dashboard) to stay organized by tracking inventory, including what’s in stock, adding new SKUs, monitoring the status of orders, and placing new orders. +- [Support ticketing systems:](https://retool.com/templates/support-ticketing-system) to manage and track customer support tickets, streamlining the process of addressing and resolving them. + +Explore the [Retool templates](https://retool.com/templates) for more. + +## Adding a database to the Retool platform + +To further improve the user experience, the Retool team decided to [host a database](https://retool.com/products/database) into their platform. This would allow developers to jump straight into building their internal tools, without worrying about having to spin up and manage database instances separately. + +To make this happen, the engineers identified some key requirements for the implementation. The objective was to ensure the database integration was not only seamless to the end user, but also operationally sustainable and cost-efficient for the team: + +**This system had to be managed with minimal engineering overhead.** + +The fleet had to be easily scalable up to hundreds of thousands of databases without a proportional increase in the management burden. The best way to do this was by offering dedicated Postgres URLs to every end user; this isolated design simplifies the management of the database fleet, as it allows to maintain a clear mapping between customers and their respective databases. + +**The Retool team wanted to automatically suspend databases that were not in use.** + +By suspending idle instances, Retool would reduce the costs associated with running server resources that are not being actively used. This is particularly important for a platform potentially managing a massive number of customer databases with each database on its own instance. + +## Hello, RetoolDB (powered by Neon) + +
+

“We’ve been able to automate virtually all database management tasks via the Neon API. This saved us a tremendous amount of time and engineering effort. The scale-to-zero functionality of Neon allows us to offer dedicated databases to our customers without worrying about the cost of idle resources”

+Himanshu Bhandoh, Software Engineer at Retool +
+ +The Retool team ended up partnering with Neon to power [RetoolDB](https://docs.retool.com/data-sources/quickstarts/retool-database). The Neon API allowed Retool to integrate and automate all database management, streamlining workflows and reducing manual work. Retool creates one [Neon project](https://neon.tech/docs/manage/projects#manage-projects-with-the-neon-api) per every end user, each project with one Postgres database. + +Due to their serverless nature, Neon databases [scale to zero](https://neon.tech/docs/introduction/auto-suspend) automatically when inactive. This saves Retool the costs of idle instances, which is crucial for managing resources cost-effectively across a large fleet. This streamlined architecture enables Retool to manage **+300,000 database projects** with minimal engineering overhead—the fleet is currently managed by one engineer. + +## The beauty of dogfooding: how Retool uses Retool to manage RetoolDB + +Retool oversees virtually all database management tasks directly from an internal tool, which they built (of course) using Retool. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases/screenshot-2024-03-28-at-73756percente2percent80percentafpm-1024x511-7349eb7a.png) + +Let’s peek behind the scenes: + +- When a new Retool user signs up, they’re automatically assigned a pre-created, unclaimed database from a pool of Neon projects. +- Whenever a database is allocated from the pool, the system seamlessly triggers the creation of a new Neon project, maintaining a consistent supply. +- The backbone of this system is an internal Postgres database that tracks the status of each Neon database (claimed or unclaimed) along with user assignment details. +- The team monitors the entire fleet through a custom dashboard. This not only offers real-time insights but also allows for direct actions, such as adjusting storage quotas for individual databases. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-retool-uses-retool-and-the-neon-api-to-manage-300k-postgres-databases/ezgifcom-animated-gif-maker-1024x510-6c2ac64a.gif) + +Sometimes, even complex problems can have elegant solutions (and it feels great when they do!). [Start exploring Retool](https://login.retool.com/auth/signup?source=navbarcta) and discover how to build your own internal tools that connect to your databases and APIs. + +## Offer managed Postgres to your users: partner with Neon + +
+

“The support from the Neon team has been great. Whenever we’ve raised issues, we’ve received fast, effective responses. Their dedication to improving their service reassures us that we have a reliable partner in Neon”

+Himanshu Bhandoh, Software Engineer at Retool +
+ +By partnering with Neon, Retool has been able to incorporate a Postgres offering into their developer platform, reducing time-to-value for their customers. If you’d also like to offer Postgres to your end users, [consider partnering with us](https://neon.tech/partners). Providing your customers with fully managed Postgres has never been easier. diff --git a/content/blog/posts/how-specific-provisions-thousands-of-databases-for-coding-agents-using-neon.md b/content/blog/posts/how-specific-provisions-thousands-of-databases-for-coding-agents-using-neon.md new file mode 100644 index 0000000000..8baf00a7bb --- /dev/null +++ b/content/blog/posts/how-specific-provisions-thousands-of-databases-for-coding-agents-using-neon.md @@ -0,0 +1,108 @@ +--- +title: How Specific Provisions Thousands of Databases for Coding Agents Using Neon +description: Specific.dev is a cloud platform built for coding agents. +excerpt: >- + “I’m genuinely surprised by how well it handles that scale. You can create + tons of databases and they’re available immediately. You can branch out + immediately. All of those things make it really nice for agent-managed infra.” + Iman Radjavi, Co-founder, Specific.dev What Specific b... +date: '2026-03-18T19:19:23' +updatedOn: '2026-03-27T14:05:17' +category: case-study +categories: + - case-study +authors: + - andy-hattemer +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-specific-provisions-thousands-of-databases-for-coding-agents-using-neon/cover.png + alt: null +isFeatured: false +seo: + title: >- + How Specific Provisions Thousands of Databases for Coding Agents Using Neon + - Neon + description: Specific.dev is a cloud platform built for coding agents. + keywords: [] + noindex: false + ogTitle: >- + How Specific Provisions Thousands of Databases for Coding Agents Using Neon + - Neon + ogDescription: >- + “I’m genuinely surprised by how well it handles that scale. You can create + tons of databases and they’re available immediately. You can branch out + immediately. All of those things make it really nice for agent-managed + infra.” Iman Radjavi, Co-founder, Specific.dev What Specific builds Specific + (YC F25) is a cloud platform designed for coding agents. With […] + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-specific-provisions-thousands-of-databases-for-coding-agents-using-neon/cover.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-specific-provisions-thousands-of-databases-for-coding-agents-using-neon/image-6-1024x538-16243d5e.png) + +
+

“I’m genuinely surprised by how well it handles that scale. You can create tons of databases and they’re available immediately. You can branch out immediately. All of those things make it really nice for agent-managed infra.”

+Iman Radjavi, Co-founder, Specific.dev +
+ +## What Specific builds + +[Specific](https://specific.dev/) (YC F25) is a cloud platform designed for coding agents. With Specific, agents write code and configure the infrastructure to run it, from databases and object storage to caching and backend services, all defined in a single config file and deployed with one CLI command. + +--- + +## The problem they are solving + +Before building Specific, Iman Radjavi and his co-founder saw the same pattern repeat across developers: they’d start a project with Vercel for the frontend, reach for Supabase when they needed a database, then add Railway for backend workers. Three platforms, none of which knew about each other. + +Coding agents made this worse. An agent asked to deploy a frontend would naturally recommend Vercel. Asked to add a database, it would recommend Supabase. Each recommendation was reasonable in isolation; together, they created a fragmented stack that was hard to run locally, hard to debug, and hard to hand off to the next agent in the loop. + +The trigger for Specific came from a platform customer: [Natively](https://natively.dev/), a prompt-to-mobile-app builder. Natively’s AI agent was generating mobile app backends for end users. Their previous database provider required maintaining a large block of custom documentation just to get the agent to interact with it correctly, and it still didn’t work reliably. They needed something that worked with their agent from the start, not something retrofitted to work with agents later. + +That’s the design constraint Specific was built around: **infrastructure that coding agents can define, provision, and reason about without human intervention at every step.** + +--- + +## Why Neon + +When Specific needed a database layer, two things made Neon the obvious choice. + +The first was **scale-to-zero**. Every project on Specific’s platform can have a database. Because of how easy and fast creation is, most apps built by agents get abandoned. Paying for idle compute across thousands of databases wasn’t viable. Neon’s serverless architecture scales to zero automatically, keeping costs down without any manual intervention. + +The second was **branching**. Specific’s platform model requires isolating each app’s data from the start. Specific uses branching to create instant preview environments that are isolated from the main database. This allows their users to test and make changes with real data but in isolation. + +Iman noted that Neon’s agent plan also factored in. For a product built around agent workflows, having a pricing plan designed for that use case mattered. + +--- + +## How they use Neon + +When a Specific project needs a database, a new Neon project is provisioned automatically via the Neon API. The user never configures a database connection manually. The coding agent defines the infrastructure it needs in a config file, and Specific wires the Neon database URL into the service as an environment variable. By design, the agent never has direct access to the connection string. Instead, Specific gives the agent the context and guardrails to safely handle deployments and updates. + +The local development flow mirrors production. When a developer runs specific dev, Specific spins up the full stack locally, including the database. When they run specific deploy, the same config goes to Specific’s cloud. No port conflicts to sort out, no environment variables to wire by hand. + +Preview environments for every pull request use Neon branching to give the user full-stack application previews where data reflects what is in production but changes are isolated. + +Specific is now managing over 10,000 Neon databases. New projects provision within a second. + +--- + +## Results + +- **Thousands of Neon databases provisioned every month** by specific.dev users. +- Database creation that is fast enough for the agent provisioning workflow, with no manual steps between “user starts a project” and “database is ready” +- Scale-to-zero keeps costs viable for a high volume and wide range of database workloads + +Iman’s framing: “I’m always surprised by how easy it is to just create a ton of databases. They’re created immediately. You can branch out immediately.” + +--- + +## What’s next + +Specific is in public beta and focused on the self-serve AI engineer experience. The immediate roadmap includes feeding deployed service logs back into the agent context, so users can tell their coding agent that something is behaving poorly and let it investigate without leaving the terminal. Full-stack branching, including object storage alongside databases, is also on the roadmap. + +--- + +If you’re building an agent platform and need a database layer that provisions on demand, scales to zero, and supports branching across thousands of projects, [start with Neon](https://neon.tech/). If you’d like to share your own story, find us on [Discord](https://neon.tech/discord). + +Thank you to Iman Radjavi and the Specific team for sharing how they built this. diff --git a/content/blog/posts/how-supergood-unlocked-their-postgres-developer-productivity.md b/content/blog/posts/how-supergood-unlocked-their-postgres-developer-productivity.md new file mode 100644 index 0000000000..4cf38124a8 --- /dev/null +++ b/content/blog/posts/how-supergood-unlocked-their-postgres-developer-productivity.md @@ -0,0 +1,110 @@ +--- +title: How Supergood unlocked their Postgres developer productivity +description: No more waiting for slow-moving infra or stressing over test data +excerpt: >- + “Neon allows us to develop much faster than we’ve even been used to. In + traditional setups, you’re just trying to get the data that is representative + of real customers—but now, when we build a feature, we’re actually testing it + on real data in a matter of minutes. We can get feat... +date: '2024-02-20T18:01:16' +updatedOn: '2024-05-14T20:03:19' +category: case-study +categories: + - case-study +authors: + - carlota-soto +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-supergood-unlocked-their-postgres-developer-productivity/cover.jpg + alt: null +isFeatured: false +seo: + title: How Supergood unlocked their Postgres developer productivity - Neon + description: >- + Neon allows Supergood to ship faster than ever by removing the delays caused + by slow-moving servers and database copies. + keywords: [] + noindex: false + ogTitle: How Supergood unlocked their Postgres developer productivity - Neon + ogDescription: >- + Neon allows Supergood to ship faster than ever by removing the delays caused + by slow-moving servers and database copies. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-supergood-unlocked-their-postgres-developer-productivity/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-supergood-unlocked-their-postgres-developer-productivity/neon-supergood-1024x576-5214749c.jpg) + +
+

“Neon allows us to develop much faster than we’ve even been used to. In traditional setups, you’re just trying to get the data that is representative of real customers—but now, when we build a feature, we’re actually testing it on real data in a matter of minutes. We can get features shipped way faster”

+Alex Klarfeld, CEO and co-founder of Supergood.ai +
+ +If you’ve felt frustrated with your third party APIs, you’re not alone. Before starting [Supergood](https://supergood.ai/), [Alex Klarfeld](https://www.linkedin.com/in/alex-klarfeld-a2706848/) was a co-founder in [Divvy Homes](https://www.divvyhomes.com/), a market leader in the proptech industry. While focusing on the platform during the day, he (forcefully) held a second job at night: dealing with the dozens of API vendors they were integrating with. + +Third-party APIs often failed in ways that traditional observability tools couldn’t detect, and it was impossible to write code to handle every conceivable way that an API could fail. Monthly invoices from these vendors would come back way higher than expected, and auditing usage was far from straightforward. + +[Supergood](https://supergood.ai/) was born to bring transparency to dealing with third-party APIs. “It’s essentially one line of code that you drop into your code base and makes sure all your third-party APIs don’t take down your business or break your bank”, said Alex, now CEO and co-founder of Supergood. It alerts you to anomalies like subtly changing response payloads, unexpected value fields, or slow response times before they affect your customers, and helps you understand the costs associated with API vendors. + +## Building Supergood using Neon, Redis, and TimescaleDB + +Under the hood, Supergood is powered by Neon as their mission-critical database. Supergood collects a constant stream of API metrics and ingests them into Neon, using Redis as a cache and Google Pub/Sub to streamline the high data volume. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-supergood-unlocked-their-postgres-developer-productivity/screenshot-2024-02-20-at-10436percente2percent80percentafpm-1024x522-f962ad19.png) + +Supergood splits their architecture into staging and production environments, with both environments having their isolated databases, Redis caches, workers running scheduled jobs, and a data worker and server handling API calls.

To optimize performance for time series data, Supergood [enables the TimescaleDB extension in Neon.](https://neon.tech/docs/extensions/pg-extensions) Supergood’s time series data are stored in hypertables, which are Postgres tables with automatic partitioning. + +## How database branching allowed Supergood to ship faster + +Before Neon, the team at Supergood was using the Timescale platform, but they were not impressed by the experience. They decided to switch to Neon, and they were amazed by how much their developer productivity improved. + + + +One Neon feature was behind this productivity boost: database branching. Being able to use Postgres branches increased Supergood’s engineering velocity in three ways: + +### Development and staging databases are immediately available + +
+

“We use one branch per team member and another dedicated to our staging environment. We even launched a new data science project, and we’re offloading our heavier tasks to a separate branch. Our data scientists can run heavy queries without worrying about interfering with production”

+Alex Klarfeld, CEO and co-founder of Supergood.ai +
+ +Neon branches use a copy-on-write mechanism under the hood, meaning that they don’t require to spin up a new storage instance with a separate copy of the production data. Branches are ready immediately after you create them, _with all your production data._ + +Instead of spinning up new instances and waiting for them to be available, something that might take many minutes or even hours if your database is somewhat large, you can start working right away. Branches allows you to create development, staging, and testing environments in seconds with full security—without leaving your VPC and [with additional security features like IP Allow](https://neon.tech/docs/introduction/ip-allow). + +Supergood first realized this potential when onboarding their different team members to Neon. “Onboarding engineers became so easy as soon as we started using Neon. I just spin up a branch of Postgres for them and they can get onboard on the platform in 2 seconds”, said Alex. Soon, every developer was using their own branch. Their staging environment also runs on a dedicated branch. + +Now, Supergood is even using a branch for data science, allowing data scientists to experiment and run heavy queries on their own branch. Branches in Neon have resource isolation, there’s no danger to the production database. + +And since Neon branches don’t require their own copy of storage, and they use ephemeral compute endpoints that scale to zero when inactive, they’re also very affordable. “The Neon bill is lower than I expect it to be every time”, said Alex. + +### Instead of testing locally, you can test in a database branch + +
+

“Balancing development in local environments when you have actual production data with PII [Personal Identifiable Information] is tricky. In previous companies, we had to put a lot of effort into getting a synthetic dataset resembling production data that we could reliably run tests with in Docker or local Postgres. We don’t have to do that now – we just test in a Neon branch, with a perfect copy of production data without leaving our VPC”

+Alex Klarfeld, CEO and co-founder of Supergood.ai +
+ +Another advantage of branches that might not be evident at first is that they’re a great candidate to substitute local testing. For security and compliance reasons, sensitive information cannot be used in local databases, forcing developers to create synthetic datasets for testing. But in databases, edge cases are everything: what might work locally, might not work in prod. + +When using branches for testing, none of these worries are necessary. You’re never leaving your cloud, so security is covered, and you’re actually testing in production data. Everything is done and ready to ship in a matter of minutes. + +This also avoids the need to move data outside of your private, secure network onto local machines, which could risk exposure of PII. It enhances security and compliance by ensuring sensitive data is not transferred to less secure environments. + +### Read replicas are ephemeral and affordable + +Neon’s serverless architecture with decoupled compute and storage gave Supergood one more advantage. In Neon, Supergood can spin up many read replicas, and they’re available immediately – just like branches. They only add to their compute bill when they’re running, and they imply no extra storage charges. They’re a great way to offload the primary database from heavy read queries, improving performance. + +For Supergood, this experience with read replicas was a great improvement compared to other Postgres products, where adding read replicas takes time and doubles your bill. + +## “Try Neon: it’s objectively better” + +
+

“Once you start using branching, it feels crazy that you were doing Postgres any other way. Now I’m spoiled by Neon! Even if your architecture is not serverless, it doesn’t matter. Just try Neon, it’s objectively better”

+Alex Klarfeld, CEO and co-founder of Supergood.ai +
+ +If you’re building on Postgres and this picked your curiosity, [run some experiments in our free tier](https://console.neon.tech/realms/prod-realm/protocol/openid-connect/registrations?client_id=neon-console&redirect_uri=https%3A%2F%2Fconsole.neon.tech%2Fauth%2Fkeycloak%2Fcallback&response_type=code&scope=openid+profile+email&state=nOF8hvjg6cniLoGNTMeymg%3D%3D%2C%2C%2C). + +Neon makes Postgres _just work_, and Supergood is ready to make your third-party APIs _just work_ as well. If you have some vendor horror stories, [reach out to Alex on Twitter](https://twitter.com/AlexKlarfeld) – the Supergood team wants to hear them all! [You can also sign up for their private beta](https://supergood.ai/contact) and get an early sneak peek of what they’re building. diff --git a/content/blog/posts/how-to-add-a-postgres-database-to-your-replit-agent-project.md b/content/blog/posts/how-to-add-a-postgres-database-to-your-replit-agent-project.md new file mode 100644 index 0000000000..8e2aa38048 --- /dev/null +++ b/content/blog/posts/how-to-add-a-postgres-database-to-your-replit-agent-project.md @@ -0,0 +1,186 @@ +--- +title: How to Add a Postgres Database to Your Replit Agent Project +description: Ensure your applications can save and retrieve data across sessions +excerpt: >- + If you’ve used Replit Agent, you already know it can configure your + development environment, build apps, and deploy them to the cloud—all without + needing to install anything yourself. But what you might not know is that + Agent can configure databases for you, ensuring your applica... +date: "2024-12-20T01:53:02" +updatedOn: "2025-01-21T15:37:56" +category: community +categories: + - community + - ai +authors: + - matt-palmer +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/cover.jpg + alt: null +isFeatured: true +seo: + title: How to Add a Postgres Database to Your Replit Agent Project - Neon + description: >- + You can tell Replit Agent to add configured Postgres databases and schemas + to your app for data persistence. Learn how. + keywords: [] + noindex: false + ogTitle: How to Add a Postgres Database to Your Replit Agent Project - Neon + ogDescription: >- + You can tell Replit Agent to add configured Postgres databases and schemas + to your app for data persistence. Learn how. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/social.jpg +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/neon-postgres-to-replit-1-1024x576-ff7bfaa3.jpg) + +If you’ve used [Replit Agent](https://docs.replit.com/replitai/agent), you already know it can configure your development environment, build apps, and deploy them to the cloud—all without needing to install anything yourself. + +But what you _might_ not know is that Agent can configure databases for you, ensuring your application can store user data and access it later. In other words, it adds data persistence to your apps. + +## Persistent data makes functional apps + +Data persistence is the backbone of most applications: it ensures that information created or modified by users—like preferences, progress, or scores—remains accessible even after the app is closed or refreshed. + +Without persistent storage, apps would lose all their data between sessions, making them far less functional. + +For instance, imagine you built a classic game of snake. Without a database to store high scores, the moment you refresh the page, all your progress would go. + +Similarly, consider a habit tracker app. If it couldn’t save your habits beyond one browser session, it wouldn’t be a very useful tool for maintaining long-term habits. + +## Replit Agent knows how to handle persistence + +One of the great features of Replit Agent is that it detects when an app requires data persistence and automatically adds a Neon Postgres database to support it. + +Take persistence in a habit tracker—without it, users would lose their progress! Agent is smart and typically incorporates a database by default, for functional apps right from the start. + +But what if Replit Agent doesn’t add a database in the first iteration of your project? + +## How to add a database if Agent doesn’t automatically + +Let’s return to our Snake example. + +Imagine you ask Replit Agent to build a nostalgic Snake game, reminiscent of the Nokia classic: + +
+ +
+ +In the initial version, our game works and keeps track of the score. However, when we dig deeper, we see that the high score is being stored locally (in the browser). So our app is not very complete: + +- Other players wouldn’t be able to see or compete on a shared leaderboard +- Refreshing the browser or using an incognito tab would wipe the high scores entirely + +So, how can we prompt Replit Agent to add a database for data persistence? + +It’s simpler than you might think—**just ask**. + +Once your app reaches its MVP stage or Agent finishes the initial iteration, it’s often best to start a new chat. This clears the context and ensures Agent focuses on the specific feature you want to add: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-44548percente2percent80percentafpm-1024x764-4cf1116b.png) + +In a new chat, here’s an example prompt you could use: + +
+

Let’s implement a leaderboard so that we can keep track of all the scores. The high score on the app should reflect the number 1 spot on the leaderboard. Users should be able to enter their names (up to 3 characters). 

+
+ +By being very descriptive about the user experience you’re envisioning—play the game, get a leaderboard, enter a name—you set Agent up to success to implement the feature properly. + +After submitting the prompt, Replit Agent will think for a bit, and then propose a plan: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-44751percente2percent80percentafpm-1-1024x764-f65a96a3.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-44907percente2percent80percentafpm-1024x764-ecdacb83.png) + +The plan includes: + +- **Backend changes.** The Agent will create a leaderboard table in a Postgres database. +- **Defining a database schema**. The Agent will define how the leaderboard data will be stored in the new Postgres database, including fields for player names, scores, and timestamps. +- **Frontend changes**. The Agent will update the UI to display the leaderboard and allow players to input their names. + +Once you approve, Agent will start working: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-45346percente2percent80percentafpm-1-1024x764-3e5cea6a.png) + +Once Agent starts, its first step will be configuring the database. First, it’ll generate a `models.p` y file: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-45809percente2percent80percentafpm-967x1024-2c374baf.png) + +In this file, Agent is defining the database schema using SQLAlchemy for the leaderboard feature: + +- `LeaderboardEntry` class: Represents the structure of the database table + - `id`: An auto-incrementing integer acting as the primary key + - `player_name`: A string column limited to 3 characters for storing player names (classic arcade-style) + - `score`: An integer column to store player scores + - `created_at`: A timestamp (using `datetime.utcnow`) to record when the score was submitted +- `to_dict`: Converts database entries into a dictionary format, making it easier to work with in the app + +Next, Agent works in the `app.py` file, which serves as the bridge between the database and the backend logic of the app: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-54335percente2percent80percentafpm-959x1024-33ae2920.png) + +This file, + +- Sets the `SQLALCHEMY_DATABASE_URI` to the `DATABASE_URL`, connecting the app to the Postgres database. +- Disables modification tracking for SQLAlchemy (for performance optimization) +- The database is initialized with the app context (`db.init_app(ap` p)) + +Lastly, it’s time to work the frontend updates: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-54157percente2percent80percentafpm-966x1024-a6b25afd.png) + +The Agent is adding the following features to the the `game.js` file: + +- `playerName`: A variable to store the player’s input (3-character name) +- `nameInput`: Tracks whether the input overlay is active +- `leaderboard`: An array to store the leaderboard entries fetched from the database +- `fetchLeaderboard()`: A function call to retrieve the leaderboard data from the backend + +Once the backend and frontend updates are complete, your Snake game will include a fully functional leaderboard, powered by Postgres: + +
+ +
+ +If we open the app in a new browser tab and refresh, all the high scores will now remain intact. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/ad4nxfdtgfskwosd7k7dijay-n7jh7jg9zl1eoyakz3mjsinxmax46o8edddkhopisgakbls-ncn1o7a7mipowq-tydnhfbj1ezciwevnbzfjfnu3osdl0doozrgzxnwcljo5oc8cu9w-b436e980.png) + +## Inspecting the database and schema + +We can verify the stored data by navigating to the Postgres tool: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-50803percente2percent80percentafpm-1024x844-a0a122a3.png) + +We’ll see two tables, with the table named `leaderboard_entry` including fields for player names, scores, and timestamps: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-51002percente2percent80percentafpm-1024x844-169e9bb5.png) + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-add-a-postgres-database-to-your-replit-agent-project/screenshot-2024-12-19-at-51130percente2percent80percentafpm-1024x554-0c5a6501.png) + +And just like that, we have a Postgres database backing our application. + +If you’d like to see a demo of the complete workflow, check out this video: + + + +## Adding a Postgres database takes only a prompt + +The key takeaway here is this: + +
+

Replit Agent is smart enough to recognize when your app needs persistent storage and will often add a Postgres database automatically.

+
+ +But if Agent doesn’t pick it up in the first iteration—or if you want to add a feature that explicitly requires persistent data—you can guide Agent with a simple, natural language prompt. + +Just describe what you want to achieve, and Agent will take care of the rest. + +--- + +_Under the hood, Replit Agent creates Postgres databases via_ [Neon](https://neon.tech/home)_._ + +_Explore why Neon is a great database backed for AI Agents_ [here](https://neon.tech/use-cases/ai-agents)_, and_ [reach out to the team](https://neon.tech/contact-sales) _if you’d like to learn more._ diff --git a/content/blog/posts/how-to-be-ai-fancy.md b/content/blog/posts/how-to-be-ai-fancy.md new file mode 100644 index 0000000000..90e4784c3b --- /dev/null +++ b/content/blog/posts/how-to-be-ai-fancy.md @@ -0,0 +1,158 @@ +--- +title: How to be AI-fancy +description: Improving our SQL editor with (you guessed it) AI features +excerpt: "Despite the fact that Postgres has been around for almost 30 years, Neon has always been an innovative company supporting experimental ideas. So, obviously, we couldn’t miss the AI fever. \U0001F642 We started our first experiments with AI a little more than a year ago, but only recently..." +date: '2024-11-04T16:48:02' +updatedOn: '2024-11-04T16:48:05' +category: ai +categories: + - ai +authors: + - eduard-dyckman +cover: + image: 'https://cdn.neonapi.io/public/images/pages/blog/how-to-be-ai-fancy/cover.jpg' + alt: null +isFeatured: false +seo: + title: How to be AI-fancy - Neon + description: >- + We've shipped some AI features that improve our SQL Editor, like SQL text + generation, query name generation, and error fixing. + keywords: [] + noindex: false + ogTitle: How to be AI-fancy - Neon + ogDescription: >- + We've shipped some AI features that improve our SQL Editor, like SQL text + generation, query name generation, and error fixing. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-be-ai-fancy/social.jpg +--- + +![Post image](https://cdn.neonapi.io/public/images/pages/blog/how-to-be-ai-fancy/how-to-be-ai-fancy-1-1024x576-6fffd498.jpg) + +Despite the fact that Postgres has been around for almost 30 years, Neon has always been an innovative company supporting experimental ideas. So, obviously, we couldn’t miss the AI fever. 🙂 + +We started our first experiments with AI a little more than a year ago, but only recently have we focused on bringing AI-driven features into our product. Let’s take a look at them, and later discuss how we implemented them. + +## AI features in our SQL editor + +We are happy to bring you our first AI features that will empower the SQL playground experience. They work in tandem to assist with SQL writing. + +### SQL generation + +This has got to be the most exciting and complex feature. By pressing the ✨ button or using Cmd/Ctrl+Shift+M, can enter a natural language prompt describing the SQL you want to execute, and Neon will generate the query text for you. After that, you can change the generated query as you like or just execute it as is. + + + +If you are not sure whether the generated query is safe to run towards your actual database, you can simply [create a branch](https://neon.tech/docs/manage/branches#create-a-branch) and run it there. To make the generation sensible, we extract your database schema and pass it to the LLM along with the prompt. So, if you ask to find data that is not present in the database, you are expected to get a descriptive error response. + +This feature heavily relies on LLM capabilities, so we tried multiple models (more about that in the next section), and none of them were perfect. So it’s possible that a generated query will not work exactly as you expect, that’s why, for now, we’ve stopped at the generation, and we leave the execution to users. + +### Query name generation + +A second feature: Now, the query history block will contain meaningful names for your queries. We take the executed query and generate a name for it by asking the LLM model to predict the best-fitting name. + +![](https://cdn.neonapi.io/public/images/pages/blog/how-to-be-ai-fancy/ad4nxcqpcljbjch5g4zzvgex0uibllajxe25ytnlbyvljtc6j5wdjiaxvjmefwuzu6t01g4yez9z39nvy6rzasiueh-7zkc4xubd27s6gjlgl7rylakohk60-ubxgekuepnklofla0o2gem0pzno0eyqady-fcd1cb12.png) + +### Error fixing + +It’s common to make a typo or confuse some parts of your database when you write a complex query. Or maybe you have a wider park of different databases and don’t exactly remember all Postgres operators and functions. So, in some cases, the execution may result in an error. + +![](https://cdn.neonapi.io/public/images/pages/blog/how-to-be-ai-fancy/ad4nxd-sqqbkbffnkrxpenyv-fym6ebwh2kwyqprdzwphplimxrabwbpgb6qywyyodki4s9sd2hdm9efm4kh8quakmams2jlrcbovctdztmgtx0un4015v7g-4q5myxjhqtjeoqczcoow0saz0nqbre4cnbw-e2952013.png) + +Now, we provide a single button that will take your query and Postgres errors, pass it to the LLM and ask it to fix the query. After that, we will stream the fixed query into the editor box. This can work nicely along the SQL generation: if the generated SQL contains some errors, this step can potentially fix them. This feature also adds the database schema to the request context. + + + +## Why not an AI assistant? + +For users that are already familiar with AI-driven features in similar products, it may seem unnatural that we didn’t choose the “AI assistant” path – this is, a sidebar with a chat where you can ask questions and get answers. + +There are two main reasons for this decision: + +1. **It’s complex to make it useful.** If you are chatting with an assistant bot, you expect it to understand your questions and the current context well. So, if you are asking about the current query, it should understand that and take it into consideration. If you are referencing some table, it should query the database scheme and use that information. To achieve this, the process ideally should be multi-stage: the first steps should help figure out the context, and the last steps would prompt actual results, with an optional follow-up to refine the answer. This is a lot of engineering effort. +2. **Tailored experience.** If we choose the path of an AI assistant we would have much less control over the user experience, basically it narrows down to a simple dialog and set of predefined actions. With our current solution, we provide actionable elements next to their respective contexts, which improves user comfort and boosts discoverability. Also, it’s much easier to do experiments or debugging this way. + +## TIL building AI features – or what did I carry as a valuable experience + +### Don’t make a guessing choice + +If you’ve listened to some of [Uncle Bob’s rants](https://www.youtube.com/watch?v=O4iEr7bElUA), you may favor this idea: if you don’t know what to choose – don’t, until you do know. So, I didn’t know which model would perform best, so I chose a good old [strategy](https://refactoring.guru/design-patterns/strategy) pattern. It’s actually very simple and comfortable to implement in any language with interfaces, like typescript, go, java, etc. Here is how it looks in our typescript code: + +```typescript +export interface AIPrompter { + prompt( + /*user prompt*/ + message: string, + /*system prompt*/ + system: string, + ): Promise<() => AsyncGenerator>; +} +``` + +So, currently, our strategy code should implement a single prompt method, which can be provided by almost any model and provider. Based on certain conditions we could pick a concrete implementation of this interface like OpenAIGpt4o or AWSBedrockClaude3_5. After getting some working code to our production under a feature flag we would conduct several experiments with different providers. + +For now, we’ve stopped on [Claude 3.5 Sonnet](https://www.anthropic.com/news/claude-3-5-sonnet) by Anthropic. But we can easily swap it for some other AWS Bedrock or OpenAI-provided model. + +### Infra for AI + +There are several types of AI providers and options out there. The first two categories I want to touch are self-hosted and API providers. + +It is no secret that hosting a decently performing model costs a lot. So, unless you expect a pretty significant traffic, it’s better to avoid this route. In my opinion, it starts to pay off on a pretty large scale. + +Out of **API providers**, I highlight two groups: + +- Cloud infrastructure integrated providers like AWS Bedrock, Azure AI Services, GCP AI and ML services. +- AI-specializing companies providers like OpenAI, Anthropic, Groq, Meta AI, etc. + +If you are doing a smaller project or experiment, then going to direct AI providers would probably be the best option. Most of them are easy to set up, they have simple, convenient APIs, usage tracking and budget limiting. + +If you are a bigger company or a cloud company and most parts of your system are already hosted in some cloud provider like AWS, Azure, GCP, etc., then consider using their solutions since they are better tied to your existing infrastructure and you can extend your existing infra to provide role access, budgeting, logging, usage tracking. + +The biggest downside is that you are limited by whatever your cloud provider gives you. For example, you won’t be able to use GPT models in AWS Bedrock. Also, in our case, we need to share the user’s database schema with an AI provider, so we would need a separate user’s consent to do that unless we are using Bedrock, which resides in AWS, the same place we store the user data in. + +### HTTP can stream + +Most of the AI providers support streaming response. So, a question arises: how do you stream that response to a browser? With the arrival of WebSockets, it’s easy to forget, but the HTTP response can be streamed! So you don’t really need anything special to return a streaming response from your API. + +Example in node js: + +```typescript +async function performResponse(response: http.ServerResponse, aiStream: Stream) { + response.writeHead(200); + for await (const message of aiStream) { + const { usage, choices } = message; + const [choice] = choices; + if (choice?.delta?.content) { + response.write(choice.delta.content); + } + } + response.end(); +} +``` + +Now in the frontend code we can use Fetch API to receive the stream: + +```typescript +const { body } = await fetch('/api/ai'); +const reader = body.getReader(); +while (true) { + const chunk = await reader.read(); + if (chunk.value) { + // ... do something with the streamed response + } + if (chunk.done) { + break; + } +} +``` + +## Wrap up + +If you are a [Neon user](https://console.neon.tech/signup), I encourage you to try out these new features and [tell us](https://discord.com/channels/1176467419317940276/1176788564890112042) how they work for you! + +If you are a developer, I hope these ideas can be useful for your own projects. You should definitely be enthusiastic about trying out LLMs to boost your product: it’s simple and cheap enough to experiment with. diff --git a/content/blog/posts/how-to-build-a-full-stack-ai-agent.md b/content/blog/posts/how-to-build-a-full-stack-ai-agent.md new file mode 100644 index 0000000000..518cb69755 --- /dev/null +++ b/content/blog/posts/how-to-build-a-full-stack-ai-agent.md @@ -0,0 +1,279 @@ +--- +title: How to Build a Full-Stack AI Agent +description: >- + A complete open-source example using Vercel, Freestyle, Assistant UI, Mastra, + Drizzle, and the Neon stack +excerpt: >- + More and more teams are using Neon to power vibe coding platforms, so we + decided to build one too – not as our + billion-dollar-vibe-coding-startup-side-gig but as a public, open-source + template you can use as a starting point to learn how to build codegen agents + yourself. We calle... +date: '2025-10-29T18:40:53' +updatedOn: '2025-10-29T18:46:52' +category: ai +categories: + - ai +authors: + - andre-landgraf +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-full-stack-ai-agent/cover.jpg + alt: null +isFeatured: true +seo: + title: How to Build a Full-Stack AI Agent - Neon + description: >- + Learn how to build your own full-stack AI agent from scratch, with + open-source code and architectural breakdown. + keywords: [] + noindex: false + ogTitle: How to Build a Full-Stack AI Agent - Neon + ogDescription: >- + Learn how to build your own full-stack AI agent from scratch, with + open-source code and architectural breakdown. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-full-stack-ai-agent/social.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-full-stack-ai-agent/neon-template-1024x576-879a51ec.jpg) + +**More and more teams are using Neon to power vibe coding platforms, so we decided to build one too – not as our _billion-dollar-vibe-coding-startup-side-gig_ but as a public, open-source template you can use as a starting point to learn how to build codegen agents yourself.** + +
We called the agent Aileen, and all the code lives here: + +[github.com/andrelandgraf/aileen](https://github.com/andrelandgraf/aileen?utm_source=chatgpt.com) + +You can also try it out yourself [here](https://aileen-blue.vercel.app/). Here’s how it looks in action: + + + +Let’s walk you through how it works. + +## What Aileen (The Agent) Can Do + +Aileen is an AI chat app and codegen platform that ties together everything you need to prompt (aka. vibe code) web applications front to back. It can + +- **Generate complete web apps from a prompt.** You describe what you want to build and Aileen scaffolds it for you with the right files, dependencies, and setup. +- **Provision a Postgres database automatically.** Every app Aileen generates comes with its own database with schema and connection already configured. +- **Add authentication.** Each generated app also includes a working auth layer, so users can sign up, log in, and interact securely without extra setup. +- **Track project versions.** Aileen saves your generated apps and their versions so you can revisit previous iterations or roll back to an earlier state. Notably, your database always stays in sync with the selected project version. +- **Run and preview apps instantly.** You can test each generated project right from the UI or visit the preview URL directly. Even a VS Code for the Web instance of the development server is accessible through the UI. +- **Keep context between generations.** Conversations with Aileen aren’t one-off: the agent remembers your previous instructions and builds on them, enabling iterative creation. +- **Scale down when idle.** Because each app uses a serverless Neon database, unused projects automatically pause, reducing resource usage to zero when you’re not building. + +
+Image +
Aileen UI. Note the versions list and the Create checkpoint button
+
+ +## The Architecture + +**The stack is intentionally modular,** so you can swap parts without rewriting the world. At a high level, there’s a core platform that coordinates users, projects, and conversations, and a per-app provisioning flow that spins up isolated resources for each generated project. + +### Neon (Postgres for the control plane) + +[Neon](https://neon.com/) stores the platform’s own data: users, projects, projects versions…This is your source of truth for “what exists,” “who owns it,” and “what version is current.” Because it’s Postgres, you also get transactional safety for writes during multi-step generation flows. + +### Neon Auth (powered by Stack Auth) + +[Neon Auth](https://neon.com/docs/neon-auth/overview) handles sign-up/sign-in, session management, and user identity. The app checks this session on every request (server and API routes) and uses the authenticated user ID as the join key across projects, versions, and persisted chat. Defaults are compatible with RLS so you can enforce per-row access if you extend the schema. + +### Assistant UI (chat + persistence) + +[Assistant UI Cloud](https://www.assistant-ui.com/) renders the chat interface, streams model outputs, and keeps message state coherent across turns. All conversation data is persisted so threads survive refreshes, can be reloaded across sessions, and remain queryable for retrieval or long-term “memory.” + +### Mastra (agent orchestration + hosting) + +[Mastra](https://mastra.ai/) is the framework that executes tool calls and the multi-step agent loop. When a user asks for a new app or a change, the codegen-agent iterates on the prompt and decides what tools to call to fullfill the request (e.g. query or inspect the Neon database, read and write files on the development server, or search the web). Mastra Cloud is used to run the Mastra agent in a long-running environment that streams responses to the core platform’s Assistant UI chat. + +You can also explore the Mastra Playground, which lists all available agents (in Aileen’s case, just one) and the tools they have registered. It’s a great debugging companion and lets you run the same agent logic locally before deploying it to Mastra Cloud. + +
+Image +
Mastra Playground, with a list of all currently available tools
+
+ +### Vercel (app hosting & background tasks) + +The core platform runs as a Next.js application deployed on [Vercel](https://vercel.com/). In addition to hosting the app, Vercel is also used to queue and execute longer-running background tasks. Creating a new codegen project (including provisioning a development server) can exceed the execution limits of serverless functions, so these operations are handled through Vercel Workflows. The [Workflow Development Kit (WDK)](https://vercel.com/blog/introducing-workflow) makes it straightforward to build step functions that reliably orchestrate those tasks on Vercel. 👀 + +### Freestyle (development servers & Git integration) + +[Freestyle.sh](https://www.freestyle.sh/) powers Aileen’s development environments. It gives the platform high-level primitives to: + +- create new Git repositories for every generated app +- spin up isolated development sandboxes with filesystem and process access for the agent, and
deploy apps to a lightweight, serverless runtime + +Freestyle also includes a full MCP server, which exposes filesystem and process commands directly to the agent. In theory, Aileen could use it out of the box; in practice, the implementation relies on custom Freestyle tools for tighter control and easier debugging… And, honestly, just for fun 🙂 + +### Per-app provisioning + +Each user-created project is isolated so one project’s experiments don’t interfere with another’s: + +- **Dedicated Neon Postgres database (with** [snapshots](https://neon.com/docs/ai/ai-database-versioning)**).** On “create project,” the platform calls Neon’s API to provision a Neon project for that app. Each project’s data is isolated at the database level, and snapshots are used to capture version states over time. They are tied to the matching Git commit for consistent rollback and version history. +- **Auth configuration (via** [Neon Auth](https://neon.com/docs/neon-auth/overview)**).** Neon Auth brings authentication and user management natively to your Neon Postgres database. Each code-generated app is automatically set up to work with Neon Auth so that the codegen agent can immediately start using user authentication utilities. +- [Freestyle.sh](https://freestyle.sh) **dev server with git integration.** Freestyle provides high-level primitives to create a new Git repository for each generated app, provision development sandboxes with filesystem and process access for the agent, and handle production deployments to a serverless environment. + +## How it Works: End-to-End Flow + +We can break things down into two stages: project setup (handled by the control plane) and project iteration (driven by the agent). + +### Project setup (control plane) + +When a new project is created, the control plane coordinates a full environment setup before the agent starts coding, with source repo, database, auth, and dev server. + +
+Image +
Here’s how the control-plane, meta db Neon project looks like in the Neon console
+
+ +1. **Create Neon project (with Neon Auth)** The control plane calls the [Neon API](https://neon.com/docs/reference/api-reference) to create a new database inside a shared multi-tenant Neon organization. It also initializes [Neon Auth](https://neon.com/docs/neon-auth/overview) for the project, generating client and server keys and registering callback URLs. +2. **Create new git repo.** A new repository is created on [Freestyle.sh](https://freestyle.sh). This will host all generated code and subsequent commits. The repository is based on [a starter Next.js template](https://github.com/andrelandgraf/neon-freestyle-template) that is set up to serve as a good starting point for the agent. +3. **Request dev server (Freestyle).** The control plane sends a request to Freestyle to spin up a development server linked to the repo. This server will later run CLI commands for code scaffolding and preview builds. +4. **Save credentials and secrets.** The resulting DATABASE_URL, Neon Auth keys, and other environment variables are stored securely in the meta database (the control-plane Postgres on Neon). +5. **Retrieve initial commit.** Once Freestyle finishes scaffolding, the system retrieves the initial Git commit hash from the dev server using CLI commands. +6. **Create initial snapshot (Neon).** The control plane calls Neon again to create a [snapshot](https://api-docs.neon.tech/reference/createsnapshot) of the project database, capturing its initial state before the agent begins work. +7. **Save version metadata.** The snapshot ID and Git commit hash are stored together as version 1 of the project in the control-plane database. + +### Project iteration (agent loop) + +Once setup is complete, the agent steps in to plan and execute changes. Each agent execution starts with a prompt sent from the frontend (Assistant UI components) to Mastra Cloud. The agent has access to a toolkit composed of MCP servers (context7 for docs retrieval, Neon for database management) and several custom tools for environment variable management and interacting with the Freestyle development server. + +1. **Prompt.** The user describes what to build or change (“Add a notes table,” “Deploy a new route,” etc) +2. **Agent reasoning.** The agent can take up to 50 steps per task, invoking tools, reading context, and iterating toward a solution. +3. **Tool calls.** The agent uses: + - Custom tools to interact with the Freestyle development server’s filesystem and execute bash commands (ls, read_file, write_file, exec) + - [Neon MCP tools](https://github.com/neondatabase/mcp-server-neon) to inspect the database, run SQL queries, and otherwise manage the project as it sees fit + - A custom commit and push tool that uses Freestyle’s git service to commit and push the changes and then create a new project version – more about that in the next step + - Environment-variable management tools to update the development servers’ environment variables (delete, list, add), later stored in the `project_secret` table. The `project_secrets` table stores all environment variables (like Neon Auth keys and database URLs) and versions them alongside each app release._PS, Be aware hey’re stored in plain text for now; it’s a demo, after all 🙂_ + - Context7 MCP server for optional documentation retrieval, letting the agent fetch external references on demand. +4. **Completion, commit and push.** Once the agent declares the task done, the system prompt instructs it to finalize the iteration by committing changes (calling the commit and push tool). The tool performs: + - `git commit` and `git push` on the Freestyle repo + - Retrieves the new Git commit hash + - Calls Neon to create a new snapshot of the project database + - Stores both identifiers (commit + snapshot) as the next project version in the `project_versions` table + - Persists the latest environment variables in the `project_secrets` table +5. **Version tracking.** Each version pairs code and data: the Git commit represents code changes, and the Neon snapshot represents the exact matching database state. This guarantees you can always roll back or branch from any version with full fidelity. + +
+Image +
Here’s how the code-generated projects look like in the Neon console
+
+ +## Magic Features + +### Checkpointing & versioning via Neon Snapshots + +Aileen treats each version of an app as a [lightweight checkpoint for both code and data](https://neon.com/blog/checkpoints-for-agents-with-neon-snapshots). Every time the agent commits new code, a corresponding Neon snapshot captures the exact state of the project’s database (schema + data) at that moment. The Git commit hash and Neon snapshot ID are stored together, forming a perfectly synced record of that version. + +You can easily jump between versions or fully revert back in case the agent messes up or you decide to discard made changes. This dual versioning model (Git for code, Neon for data) makes it safe to iterate, experiment, and recover from mistakes without losing alignment between your app and its database. + +Here’s the create-snapshot API call: + +```typescript + const res = await fetch( + `${this.baseUrl}/projects/${neonProjectId}/branches/${prodBranch.id}/snapshot`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + Accept: "application/json", + }, + body: JSON.stringify({ + timestamp: options.timestamp?? new Date().toISOString(), + name: options.name, + }), + }, + ); +``` + +The restore-snapshot API call: + +```typescript + const res = await fetch( + `${this.baseUrl}/projects/${neonProjectId}/snapshots/${snapshotId}/restore`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.apiKey}`, + Accept: "application/json", + }, + body: JSON.stringify({ + name: `before_restore_${Date.now()}`, + finalize_restore: true, + target_branch_id: targetBranchId, + }), + }, + ); +``` + +And the create-snapshot workflow: + +```typescript +export async function createManualCheckpoint( + projectId: string, + repoId: string, + neonProjectId: string, + currentDevVersionId: string, + secrets: Record, + assistantMessageId: string | null, +) { + "use workflow"; + const [currentCommitHash, snapshotId] = await Promise.all([ + getCurrentCommitHash(repoId, secrets), + createCheckpointSnapshot(neonProjectId), + ]); + const checkpointVersion = await createCheckpointVersion( + projectId, + currentCommitHash, + snapshotId, + assistantMessageId, + ); + await Promise.all([ + copyProjectSecrets(currentDevVersionId, checkpointVersion.id), + setCurrentDevVersion(projectId, checkpointVersion.id), + ]); + return { success: true, versionId: checkpointVersion.id }; +} +``` + +### Schema migrations with Drizzle + +The Neon MCP server offers migration tools, but for Aileen, we decided to use Drizzle for managing migrations. Whenever the agent makes changes to the Drizzle schema, it is tasked to then generate the migration files and apply them using the drizzle-kit generate and drizzle-kit migrate commands. This makes migrations much easier to replay as they are part of the version controlled code base. + +Here’s part of the system prompt: + +```bash +Drizzle ORM (for schema management): +- Define and modify database schemas in Drizzle schema files +- Use Drizzle in the application code for type-safe queries +- Run schema changes via package.json scripts using the freestyle-exec tool: + - Generate migrations: \`cd /template && npm run db:generate\` (background: false - run in foreground to inspect output) + - Run migrations: \`cd /template && npm run db:migrate\` (background: false - run in foreground to inspect output) +- Never hardcode database credentials - use environment variables +``` + +### Persistence of chat state + +Messages, tool runs, and planning notes are displayed with [Assistant UI](https://www.assistant-ui.com/) and persisted with Assistant UI Cloud so the agent can build context acrosss22 and sessions. That enables “continue where we left off,” auditability of what the agent decided, and retrieval of prior instructions when creating new versions. Referencing message identifiers with project version metadata also makes it easy to correlate a conversation turn with the version it produced. + +### Versioned project secrets + +Each generated project includes a schema for managing and versioning environment variables. Every `project_version` entry references its own set of secrets (database URL, Neon Auth keys, etc.) stored in the `project_secrets` table. This ensures that even if credentials or environment settings change later, historical versions remain reproducible. This is a simple implementation, but demonstrates how agents can manage environment state alongside code and data. + +### Scale-to-zero by default + +Every generated app gets its own serverless Postgres on Neon, which idles automatically when unused and wakes on demand. That’s a perfect fit for codegen/agent workloads, where bursts of activity are followed by long quiet periods. You can run lots of projects concurrently without carrying idle database costs. + +## Wrapping Up + +**Use Aileen as your reference template for building your agent.** You can fork the repo, run it locally, or adapt it to your own setup [: github.com/andrelandgraf/aileen](https://github.com/andrelandgraf/aileen?utm_source=chatgpt.com) + +If you have any questions, you can find us in [Discord](https://discord.com/invite/92vNTzKDGp)! + + +If you're building your own agent platform and need a backend, [take a look at Neon's Agent Plan.](https://neon.com/use-cases/ai-agents) You can get special pricing, resource limits, and assistance to get your platform up and running. + diff --git a/content/blog/posts/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon.md b/content/blog/posts/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon.md new file mode 100644 index 0000000000..172f37065f --- /dev/null +++ b/content/blog/posts/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon.md @@ -0,0 +1,1931 @@ +--- +title: >- + How to Build a Secure Project Management Platform with Next.js, Clerk, and + Neon +description: >- + Learn a security-first approach to building web applications by building a + secure project management platform with Next.js +excerpt: >- + This article was first published in the Clerk blog. Around 30,000 websites and + applications are hacked every day*, and the developer is often to blame. The + vast majority of breaches occur due to misconfiguration rather than an actual + vulnerability. This could be due to exposed da... +date: '2025-02-22T01:41:21' +updatedOn: '2025-02-22T01:41:23' +category: community +categories: + - community +authors: + - brian-morrison +cover: + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/cover.png + alt: null +isFeatured: false +seo: + title: >- + How to Build a Secure Project Management Platform with Next.js, Clerk, and + Neon - Neon + description: >- + Learn a security-first approach to building web applications by building a + secure project management platform with Next.js. + keywords: [] + noindex: false + ogTitle: >- + How to Build a Secure Project Management Platform with Next.js, Clerk, and + Neon - Neon + ogDescription: >- + Learn a security-first approach to building web applications by building a + secure project management platform with Next.js. + image: >- + https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/cover.png +--- + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/screenshot-2025-02-21-at-41311percente2percent80percentafpm-1024x530-f71ee7ee.png) + +_This article was first published [in the Clerk blog](https://clerk.com/blog/build-secure-project-management-nextjs#setting-up-the-database)._ + +Around 30,000 websites and applications are hacked every day\*, and the developer is often to blame. + +The vast majority of breaches occur due to misconfiguration rather than an actual vulnerability. This could be due to exposed database credentials, unprotected API routes, or data operations without the proper authorization checks just to name a few. It’s important to ensure that your application is configured in a way that prevents attackers from gaining unauthorized access to user data. + +**In this article, you’ll learn how to build a project management web application while considering security best practices throughout.** **We’ll cover the process of building [Kozi](https://github.com/bmorrisondev/kozi/tree/main) – a collaborative project and knowledge management tool.** + +Throughout the series, the following features will be implemented: + +- Create organizations to invite others to manage projects as a team. +- A rich, collaborative text editor for project and task notes. +- A system to comment on projects, tasks, and notes. +- Automatic RAG functionality for all notes and uploaded files. +- Invite users from outside your organization to collaborate on individual tasks. +- Be notified when events occur on tasks you subscribe to, or you are mentioned in comments or notes. + +## What makes this a “secure” project management system? +Data security is considered throughout this guide by using the following techniques: + +### Clerk and the Next.js middleware +[Clerk](https://clerk.com) is a user management platform designed to get authentication into your application as quick as possible by providing a complete suite of user management tools as well as drop-in UI components. Behind the scenes, Clerk creates fast expiring tokens upon user sign-in that are sent to your server with each request, where Clerk also verifies the identify of the user. + +Clerk integrates with Next.js middleware to ensure every request to the application is evaluated before it reaches its destination. In the section where the middleware is configured, we instruct the middleware to protect any route starting with `/app` so that only authenticated users may access them. This means that before any functions are executed (on the client or server), the user will need to be authenticated. + +### Server actions +In this project, server actions are the primary method of interacting with the data in the database. Direct access to the database should always happen on the server and NEVER on the client where tech-savvy users can gain access to the database credentials. Since all functions that access the database are built with server actions, they do not execute client-side. + +It’s important to note that calling these server actions should only ever be performed from protected routes. When a Next.js client component executes a server action, an HTTP POST request of form data is submitted to the current path with a unique identifier of the action for Next.js to route the data internally. + +This means that calling a server function from an anonymous route might result in anonymous users getting access to the data. This potential vulnerability is addressed in the next section. + +### Database requests +Protecting access to the functions is only one consideration. Each request will have an accompanying user identifier which can be used to determine the user making that request. This identifier is stored alongside the records the user creates, allowing each request for data to ONLY return the data associated with that user. + +When making data modifications, the requesting user ID is cross-referenced with the records being modified or deleted so that one user cannot affect another user’s data. + +The combination of protecting access to the routes, being mindful of calling server actions, and cross-referencing database queries with the user making the request ensures that the data within the application is secure and only accessible to those who have access to it. + +## How to follow along +Kozi is an open-source project, with each article in the series having corresponding start and end branches. This makes it easy to jump in at any point to get hands-on experience with the concepts outlined in each piece, as well as a point of reference if you simply want to see the completed code. Here are links to the specific branches: + +- `article-2-start` +- [`article-2-end`](https://github.com/bmorrisondev/kozi/tree/article-2-end) + +You should have a basic understanding of Next.js and React as well. + +### Launching the project +Once the branch above is cloned, open the project in your editor or terminal and run the following command to start up the application: + +```javascript +npm install +npm run dev +``` + +Open your browser and navigate to the URL displayed in the terminal to access Kozi. At the bottom right of the screen, you should see Clerk is running in keyless mode. Click the button to claim your keys and associate this instance to your Clerk account. If you don’t have an account, you’ll be prompted to create one. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/image-1024x663-bce6424a.png) + +You are now ready to start building out the core functionality of Kozi! + +## Setting up the database +To store structured data, you’ll be using a serverless instance of Postgress provided by [Neon](https://neon.tech/home). Start by [creating a Free Neon account](https://console.neon.tech/signup) if you don’t have one. Create a new database and copy the connection string as shown below. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/image-1-1024x663-654e22b3.png) + +Create a new file in your local project named `.env.local` and paste the following snippet, replacing the placeholder for your specific Neon database connection string. + +```bash +DATABASE_URL= +``` + +### Configuring Prisma +Prisma is used as the ORM to access and manipulate data in the database, as well as apply schema changes to the database as the data needs are updated. Open the project in your IDE and start by creating the schema file at `prisma/schema.prisma`. Paste in the following code: + +```typescript +generator client { + provider = "prisma-client-js" +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") +} + +model Project { + id String @id @default(cuid()) + name String + description String? + owner_id String + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + is_archived Boolean @default(false) +} + +model Task { + id String @id @default(cuid()) + title String + description String? + owner_id String + is_completed Boolean @default(false) + created_at DateTime @default(now()) + updated_at DateTime @updatedAt + project_id String? +} +``` + + +We’re using the `owner_id` column instead of `user_id` since this application will be updated to support teams and organizations in a future entry. + + +Next, create the `src/lib/db.ts` file and paste in the following code which will be used throughout the application to create a connection to the database: + +```typescript +import { PrismaClient } from '@prisma/client' + +const globalForPrisma = globalThis as unknown as { + prisma: PrismaClient | undefined +} + +export const prisma = globalForPrisma.prisma?? new PrismaClient() + +if (process.env.NODE_ENV!== 'production') globalForPrisma.prisma = prisma +``` + +To sync the schema changes to Neon, run the following command in the terminal: + +```typescript +npx prisma db push +``` + +If you open the database in the Neon console and navigate to the Tables menu item, you should see the `projects` and `tasks` tables shown. + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/image-2-1024x663-f61a9e90.png) + +Finally, since it is not best practice to use the Prisma client in any client-side components, you’ll want a file to store interfaces so that TypeScript can recognize the structure of your objects when passing them between components. + +Create the `src/app/app/models.ts` file and paste in the following: + +```typescript +export interface Task { + id: string + title: string + description?: string | null + is_completed: boolean + created_at: Date + updated_at: Date + project_id?: string | null + owner_id: string +} + +export interface Project { + name: string + id: string + description: string | null + owner_id: string + created_at: Date + updated_at: Date + is_archived: boolean +} +``` + +## Configure /app as a protected route with Clerk +Clerk’s middleware uses a helper function called `createRouteMatcher` that lets you define a list of routes to protect. This includes any pages, server actions, or API handlers stored in the matching folders of the project. + +All of the core functionality of the application will be stored in the `/app` route, so update `src/middleware.ts` to use the `createRouteMatcher` to protect everything in that folder: + +```typescript +import { clerkMiddleware, createRouteMatcher } from '@clerk/nextjs/server' + +const isProtectedRoute = createRouteMatcher(['/app(.*)']) + +export default clerkMiddleware(async (auth, req) => { + if (isProtectedRoute(req)) await auth.protect() +}) + +export const config = { + matcher: [ + // Skip Next.js internals and all static files, unless found in search params + '/((?!_next|[^?] *\.(?:html?|css|js(?!on)|jpe?g|webp|png|gif|svg|ttf|woff2?|ico|csv|docx?|xlsx?|zip|webmanifest)).*)', + // Always run for API routes + '/(api|trpc)(.*)', + ], +} +``` + +The `/app` route will use a different layout from the landing page, which will contain a collapsible sidebar that contains the `` (a Clerk UI component that lets users manage their profile and sign out), an inbox for tasks, and a list of projects that tasks can be created in. + +Start by creating the `src/app/app/components/Sidebar.tsx` file to render the elements of the sidebar: + +```typescript +'use client' + +import { cn } from '@/lib/utils' +import { ChevronRightIcon, ChevronLeftIcon, InboxIcon } from 'lucide-react' +import React from 'react' +import Link from 'next/link' +import { UserButton } from '@clerk/nextjs' + +function Sidebar() { + const [isCollapsed, setIsCollapsed] = React.useState(false) + + return ( +
+ +
+ ) +} + +export default Sidebar +``` + +Now create `src/app/app/layout.tsx` to render the sidebar with the pages in the `/app` route: + +```typescript +import * as React from 'react' +import Sidebar from './components/Sidebar' + +export default function AppLayout({ children }: { children: React. ReactNode }) { + return ( +
+ +
{children}
+
+ ) +} +``` + +Next, create `src/app/app/page.tsx` which is just a simple page that renders some text to make sure the `/app` route works as expected: + +```typescript +import React from 'react' +import { auth } from '@clerk/nextjs/server' +import { redirect } from 'next/navigation' + +export default async function AppHome() { + const { userId } = await auth() + + if (!userId) { + return redirect('/sign-in') + } + + return
Inbox
+} +``` + +Open the application in your browser and test out the changes by navigating to the `/app` which should automatically redirect you to the `/sign-in` route where you can create an account and make sure `/app` only works when authenticated. + + + +## Working with tasks +At the core of every project is a list of tasks, so now we’ll configure the ability to create and work with tasks in the default Inbox list. Several components will be used to provide the following application structure. The following image shows how these components will be used: + +![Image](https://cdn.neonapi.io/public/images/pages/blog/how-to-build-a-secure-project-management-platform-with-next-js-clerk-and-neon/image-3-1024x666-eacfd5ff.png) + +These are all client components so they will need corresponding server actions so they can interact with the database securely. Create the `src/app/app/actions.ts` file and paste in the following code: + +```typescript +'use server' + +import { auth } from '@clerk/nextjs/server' +import { prisma } from '@/lib/db' +import { revalidatePath } from 'next/cache' + +export async function createTask(formData: FormData) { + const { userId } = await auth() + if (!userId) { + throw new Error('Unauthorized') + } + + const title = formData.get('title') as string + if (!title?.trim()) { + throw new Error('Title is required') + } + + await prisma.task.create({ + data: { + title: title.trim(), + owner_id: userId, + project_id: null, + }, + }) + + revalidatePath('/app') +} + +export async function toggleTask(taskId: string) { + const { userId } = await auth() + if (!userId) { + throw new Error('Unauthorized') + } + + const task = await prisma.task.findUnique({ + where: { id: taskId }, + }) + + if (!task || task.owner_id!== userId) { + throw new Error('Task not found or unauthorized') + } + + await prisma.task.update({ + where: { id: taskId }, + data: { is_completed:!task.is_completed }, + }) + + revalidatePath('/app') +} +``` + +We’re going to start with the `` component which renders the field where users can create tasks. Create the `src/app/app/components/CreateTaskInput.tsx` file and paste in the following: + +```typescript +'use client' + +import { useState } from 'react' +import { Input } from '@/components/ui/input' +import { Button } from '@/components/ui/button' +import { createTask } from '@/app/app/actions' +import { PlusIcon } from 'lucide-react' + +export default function CreateTaskInput() { + const [title, setTitle] = useState('') + const [isSubmitting, setIsSubmitting] = useState(false) + + const handleSubmit = async (e: React. FormEvent) => { + e.preventDefault() + + // Don't create a task if the title is empty + if (!title.trim()) return + + try { + setIsSubmitting(true) + const formData = new FormData() + formData.append('title', title) + await createTask(formData) + setTitle('') + } finally { + setIsSubmitting(false) + } + } + + return ( +
+
+
+
+
+ setTitle(e.target.value)} + placeholder="Add a task..." + className="flex-1 border-0 bg-transparent text-gray-900 shadow-none focus-visible:ring-0 focus-visible:ring-offset-0 dark:text-gray-100" + /> + +
+
+
+ ) +} +``` + +Next, we’ll move on to ``, which will display the name of the task and allow users to toggle it using a checkbox, as is standard in task-centric applications. Create the `src/app/app/components/TaskCard.tsx` file and paste in the following: + +```typescript +'use client' + +import React from 'react' +import { toggleTask } from '../actions' +import { cn } from '@/lib/utils' +import { Task } from '@prisma/client' + +interface Props { + task: Task +} + +export default function TaskCard({ task }: Props) { + return ( +
+
+
+ {/*Checkbox*/} + + {/*Task details*/} +
+

+ {task.title} +

+ + {task.description && ( +

+ {task.description} +

+ )} +
+
+
+
+ ) +} +``` + +Finally, create the `` component to render the list of tasks and the input to create new ones. Create the `src/app/app/components/TaskList.tsx` file and paste in the following: + +```typescript +'use client' + +import React from 'react' +import TaskCard from './TaskCard' +import CreateTaskInput from './CreateTaskInput' +import { Task } from '@prisma/client' + +interface Props { + title: string + tasks: Task [] +} + +export default function TaskList({ title, tasks }: Props) { + return ( +
+

{title}

+ +
+
+ {tasks.length === 0? ( +

No tasks

+ ): ( + tasks.map((task) => ) + )} +
+
+
+ +
+
+ ) +} +``` + +With all of our components created, update the `src/app/app/page.tsx` to match the following code which uses the components created above, as well as queries the database for all tasks on load: + +```typescript +import React from 'react' +import { auth } from '@clerk/nextjs/server' +import { redirect } from 'next/navigation' +import { prisma } from '@/lib/db' +import TaskList from './components/TaskList' + +export default async function AppHome() { + const { userId } = await auth() + + if (!userId) { + return redirect('/sign-in') + } + + // Get the user's inbox tasks + const tasks = await prisma.task.findMany({ + where: { + owner_id: userId, + project_id: null, + }, + orderBy: { + created_at: 'desc', + }, + }) + + return ( +
+ +
+ ) +} +``` + +If you access the application again, you can now create tasks in your inbox and complete them. + + + +### Editing and deleting tasks +Now that you can create tasks, the next step is to set up a modal so clicking the task (outside of the checkbox) will display the modal and allow you to change the name of the task and set a description if needed. + +As a design decision, this modal does not include a save button but rather debounces any edits for 1 second to create an experience where users can quickly save values and avoid another click. The modal will also create a menu in the header which allows you to delete the task. + +Start by appending the following code to `src/app/app/actions.ts`: + +```typescript +export async function updateTask(formData: FormData) { + const { userId } = await auth() + if (!userId) { + throw new Error('Unauthorized') + } + + const id = formData.get('id') as string + const title = formData.get('title') as string + const description = formData.get('description') as string + + if (!id ||!title?.trim()) { + throw new Error('Invalid input') + } + + const task = await prisma.task.findUnique({ + where: { id }, + }) + + if (!task || task.owner_id!== userId) { + throw new Error('Task not found or unauthorized') + } + + await prisma.task.update({ + where: { id }, + data: { + title: title.trim(), + description: description?.trim() || null, + }, + }) + + revalidatePath('/app') +} + +export async function deleteTask(taskId: string) { + const { userId } = await auth() + if (!userId) { + throw new Error('Unauthorized') + } + + // Delete the task + await prisma.task.delete({ + where: { + id: taskId, + owner_id: userId, // Ensure the task belongs to the user + }, + }) + + revalidatePath('/app') +} +``` + +Next, create the `src/app/app/components/EditTaskModal.tsx` and paste in the following: + +```typescript +'use client' + +import { useEffect, useState } from 'react' +import { useDebouncedCallback } from 'use-debounce' +import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Textarea } from '@/components/ui/textarea' +import { useRouter } from 'next/navigation' +import { updateTask, toggleTask, deleteTask } from '../actions' +import { Folder, MoreVertical, Trash2, X } from 'lucide-react' +import { Button } from '@/components/ui/button' +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from '@/components/ui/dropdown-menu' +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from '@/components/ui/alert-dialog' +import { Task } from '../models' + +interface Props { + task: Task + open: boolean + onOpenChange: (open: boolean) => void + projectName?: string +} + +export default function EditTaskModal({ + task: initialTask, + open, + onOpenChange, + projectName, +}: Props) { + const [task, setTask] = useState(initialTask) + const [title, setTitle] = useState(task.title) + const [description, setDescription] = useState(task.description || '') + const [isSubmitting, setIsSubmitting] = useState(false) + const [showDeleteConfirm, setShowDeleteConfirm] = useState(false) + const router = useRouter() + + // Reset form when modal opens + useEffect(() => { + if (open) { + setTask(initialTask) + setTitle(initialTask.title) + setDescription(initialTask.description || '') + } + }, [open, initialTask]) + + const saveChanges = useDebouncedCallback(async () => { + if (!title.trim()) return + + try { + setIsSubmitting(true) + const formData = new FormData() + formData.append('id', task.id) + formData.append('title', title.trim()) + formData.append('description', description.trim()) + await updateTask(formData) + router.refresh() + } finally { + setIsSubmitting(false) + } + }, 1000) + + async function onToggleCompleted() { + const newIsCompleted =!task.is_completed + setTask((prev) => ({ ...prev, is_completed: newIsCompleted })) + try { + await toggleTask(task.id) + } catch (error) { + // Revert on error + setTask((prev) => ({ ...prev, is_completed:!newIsCompleted })) + } + } + + function titleRef(el: HTMLTextAreaElement | null) { + if (el) { + el.style.height = '2.5rem' // Set initial height + const scrollHeight = el.scrollHeight + const minHeight = 40 // 2.5rem in pixels + el.style.height = `${Math.max(scrollHeight, minHeight)}px` + } + } + + function onTitleChange(e: React. ChangeEvent) { + setTitle(e.target.value) + saveChanges() + + // Auto-adjust height after value changes + const el = e.target + el.style.height = '2.5rem' // Reset to minimum height + const scrollHeight = el.scrollHeight + const minHeight = 40 // 2.5rem in pixels + el.style.height = `${Math.max(scrollHeight, minHeight)}px` + } + + function onDescriptionChange(e: React. ChangeEvent) { + setDescription(e.target.value) + saveChanges() + } + + async function handleDelete() { + try { + await deleteTask(task.id) + onOpenChange(false) + router.refresh() + } catch (error) { + console.error('Failed to delete task:', error) + } + } + + return ( + <> + {/*The edit task modal*/} + + + +
+ + {projectName?? 'Inbox'} + +
+ + + + + + setShowDeleteConfirm(true)} + className="text-red-600 dark:text-red-400" + > + + Delete Task + + + + +
+
+ +
+
+
+ +
+ +