ENG-822 Reactivate automated generation of dbTypes (#453)

* Create a migrate script that does migrations, and optionally genotypes
* Create a setup tasks that includes genenv and migrations (and hence genTypes). Use in dev and build.
* separate lint of schema and functions, allow lint:fix for all subtasks.
* documentation: task dependencies

Author: maparent

packages/database/README.md

@@ -43,12 +43,12 @@ We follow the Supabase [Declarative Database Schema](https://supabase.com/docs/g
     3. See if there would be a migration to apply with `supabase db diff`
 5. If applying the new schema fails, repeat step 4
 6. If you are satisfied with the migration, create a migration file with `npm run dbdiff:save some_meaningful_migration_name`
-    1. If all goes well, there should be a new file named `supabase/migration/2..._some_meaningful_migration_name.sql` which you should `git add`.
-7. `turbo build`, which will do the following:
+    1. If all goes well, there should be a new file named `supabase/migrations/2..._some_meaningful_migration_name.sql` which you should `git add`.
+7. `npm run migrate`, which will do the following:
     1. Start Supabase
     2. Apply the new migration locally
     3. Regenerate the types file with `supabase gen types typescript --local > src/dbTypes.ts`
-    4. Copy it where appropriate
+    4. Note that you can reapply an idempotent migration with `npm run migrate -- --reapply <migration timestamp>`. This will tell supabase to reapply the migration as if it were new; it may fail if your migration is not idempotent. It allows you to iterate on a migration. If your migration is not idempotent, you should revert changes in the supabase studio before reapplying.
 8. You can start using your changes again `turbo dev`
 9. When your migration is pushed in a branch, Supabase will create a branch instance. Note there is a small cost to this, so we do not want those branches to linger.
     The branch will be also created without data. (Seed data could be added to `.../supabase/seed.sql`)
@@ -65,7 +65,7 @@ If schema changes are deployed to `main` by another developer while you work on
     2. If your migration is not idempotent (which you'll notice in stage 4), you may have to revert some of your migration's changes in the local database, using the SQL editor in the studio, or `psql`.
     3. If all else fails, you can reset your local database with `supabase db reset --local`
 3. If you have an ongoing migration file, the timestamp at the start of the name should come after the latest new migration. Rename (or `git mv`) as needed.
-4. Apply `turbo build` again, so the incoming migrations are applied, and then your working migration. You may have to fix the schema and migration to take the changes into account.
+4. Out-of-order incoming migrations may be applied with `npm run migrate -- --include-all`. You may have to fix the schema and reapply your own migration to take the changes into account.
 
 ### Testing the backend
 

packages/database/doc/tasks.dot

@@ -0,0 +1,86 @@
+DiGraph G {
+
+/* package-level tasks */
+
+node [color=orange]
+
+dev
+setup
+serve
+stop
+check_types
+check_schema
+lint
+build
+test
+genenv
+dbdiff
+dbdiff_save
+deploy
+deploy_functions
+migrate
+gentypes
+
+/* supabase functions */
+node [color=purple]
+
+sup_db_diff_save
+sup_db_diff
+sup_functions
+sup_start
+sup_stop
+migrations_up
+generate_schema
+
+
+
+/* scripts */
+node [color=blue; shape=rectangle]
+
+s_createEnv
+s_genTypes
+s_migrate
+s_lint_schemas
+s_lint_functions
+s_deploy
+s_serve
+
+/* Other scripts */
+node [color=black]
+sqruff
+deno_lint
+tsc
+cucumber_js
+
+/* links */
+
+    migrate -> s_migrate -> gentypes -> s_genTypes
+    s_migrate -> migrations_up
+    dev -> setup
+    dev -> serve
+    setup -> genenv
+    setup -> migrate
+    serve -> s_serve
+    s_serve -> sup_start
+    s_serve -> sup_functions
+    s_createEnv -> sup_start
+    stop -> sup_stop
+    check_types -> tsc
+    check_schema -> s_lint_schemas
+    check_schema -> dbdiff
+    lint -> eslint
+    lint -> s_lint_functions -> deno_lint
+    lint -> s_lint_schemas -> sqruff
+    test -> build
+    test -> cucumber_js
+    genenv -> s_createEnv
+    dbdiff -> sup_stop
+    dbdiff -> sup_db_diff
+    dbdiff_save -> sup_db_diff_save
+    deploy -> s_deploy
+    deploy_functions -> s_deploy
+    build -> setup
+    build -> tsc
+    s_genTypes -> generate_schema
+    s_migrate -> sup_start
+}

packages/database/package.json

@@ -22,22 +22,23 @@
   },
   "scripts": {
     "init": "supabase login",
-    "dev": "tsx scripts/dev.ts",
-    "serve": "supabase start && tsx scripts/createEnv.mts && supabase functions serve",
+    "setup": "npm run genenv && npm run migrate",
+    "dev": "npm run setup && npm run serve",
+    "build": "npm run setup && tsc",
+    "serve": "tsx scripts/serve.ts",
     "stop": "supabase stop",
     "check-types": "tsc --noEmit --skipLibCheck",
-    "check-schema": "tsx scripts/lint.ts && supabase stop && npm run dbdiff",
-    "lint": "eslint . && tsx scripts/lint.ts",
-    "lint:fix": "tsx scripts/lint.ts -f",
-    "build": "tsx scripts/createEnv.mts && tsc",
-    "build-schema": "tsx scripts/build.ts && tsx scripts/createEnv.mts -- local",
-    "test": "tsc && cucumber-js",
+    "check-schema": "tsx scripts/lintSchemas.ts && supabase stop && npm run dbdiff",
+    "lint": "eslint . && tsx scripts/lintSchemas.ts && tsx scripts/lintFunctions.ts",
+    "lint:fix": "eslint --fix . && tsx scripts/lintSchemas.ts -f && tsx scripts/lintFunctions.ts",
+    "migrate": "tsx scripts/migrate.ts",
+    "test": "npm run build && cucumber-js",
     "genenv": "tsx scripts/createEnv.mts",
-    "gentypes:production": "supabase start && supabase gen types typescript --project-id \"$SUPABASE_PROJECT_ID\" --schema public > src/dbTypes.ts",
+    "gentypes": "tsx scripts/genTypes.ts",
     "dbdiff": "supabase stop && supabase db diff",
     "dbdiff:save": "supabase stop && supabase db diff -f",
     "deploy": "tsx scripts/deploy.ts",
-    "deploy:functions": "tsx scripts/lint.ts -f"
+    "deploy:functions": "tsx scripts/deploy.ts -f"
   },
   "dependencies": {
     "@repo/utils": "*",

packages/database/scripts/build.ts packages/database/scripts/genTypes.tspackages/database/scripts/build.ts renamed to packages/database/scripts/genTypes.ts

@@ -13,7 +13,6 @@ if (process.env.HOME !== "/vercel") {
       process.exit(0);
     }
     execSync("supabase start", { cwd: projectRoot, stdio: "inherit" });
-    execSync("supabase migrations up", { cwd: projectRoot, stdio: "inherit" });
     const stdout = execSync(
       "supabase gen types typescript --local --schema public",
       { encoding: "utf8", cwd: projectRoot },

packages/database/scripts/lint.ts

@@ -0,0 +1,50 @@
+import { exec, spawnSync } from "node:child_process";
+import { readdir, Dirent } from "node:fs";
+import { join, dirname } from "path";
+
+const __dirname = dirname(__filename);
+const projectRoot = join(__dirname, "..");
+
+const main = () => {
+  try {
+    let denoError = false;
+    exec("which deno", (err) => {
+      if (err) {
+        console.error("Could not find deno, you may want to install it.");
+        // Do not fail yet
+      } else {
+        const fnDir = join(projectRoot, "supabase", "functions");
+        const fix = process.argv.length == 3 && process.argv[2] == "-f";
+        readdir(fnDir, { withFileTypes: true }, (err, files: Dirent[]) => {
+          if (err) {
+            console.error("error:", err);
+            return;
+          }
+          const dirs = files.filter((f) => f.isDirectory()).map((d) => d.name);
+          for (const dir of dirs) {
+            const args = ["lint"];
+            if (fix) args.push("--fix");
+            args.push("index.ts");
+            const res = spawnSync("deno", args, { cwd: join(fnDir, dir) });
+            const cleanedStderr = (res.stderr?.toString() ?? "")
+              .replace("Checked 1 file", "")
+              .trim();
+            const out = (res.stdout?.toString() ?? "") + cleanedStderr;
+            if (res.status !== 0 || cleanedStderr.length > 0) {
+              console.error(`deno errors for ${dir}:\n${out}`);
+              denoError = true;
+            } else {
+              console.log(`deno checked types of ${dir}`);
+            }
+          }
+          if (denoError) process.exit(1);
+        });
+      }
+    });
+    // exit handled after linting completes
+  } catch (error) {
+    console.error("error:", error);
+    process.exit(1);
+  }
+};
+if (__filename === process.argv[1]) main();

packages/database/scripts/lintFunctions.ts

@@ -0,0 +1,28 @@
+import { exec } from "node:child_process";
+
+const main = () => {
+  try {
+    exec("sqruff --version", (err) => {
+      if (err) {
+        console.error("Could not find sqruff, you may want to install it.");
+        // Do not fail yet
+      } else {
+        const command =
+          process.argv.length == 3 && process.argv[2] == "-f" ? "fix" : "lint";
+        exec(
+          `sqruff ${command} supabase/schemas`,
+          {},
+          (err, stdout, stderr) => {
+            console.log(`${stdout}`);
+            if (stderr) console.log(`${stderr}`);
+            process.exit(err ? err.code : 0);
+          },
+        );
+      }
+    });
+  } catch (error) {
+    console.error("error:", error);
+    process.exit(1);
+  }
+};
+if (__filename === process.argv[1]) main();