Skip to content

Commit f41eeab

Browse files
Merge pull request #1278 from planetscale/import-d1
Add pscale import d1 for Cloudflare D1 offline migration
2 parents fd3b766 + be9edcb commit f41eeab

81 files changed

Lines changed: 12457 additions & 13 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

AGENTS.md

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,23 @@ Error: one JSON object on stdout with `status: "error"`, `error`, and `next_step
157157

158158
Destructive SQL without `--force`: `status: "action_required"`, `query_kind: "destructive"`, `issues`, and `next_steps` (includes `--force` retry command).
159159

160+
## Imports (Cloudflare D1)
161+
162+
`pscale import d1` migrates a Cloudflare D1 (SQLite) export into a PlanetScale Postgres branch. Every subcommand supports `--format json` and returns `status`, `issues`, and `next_steps`; stateful steps return a `migration_id` — pass it back with `--migration-id` to resume.
163+
164+
```bash
165+
pscale import d1 doctor --format json # check prerequisites (pgloader, psql)
166+
pscale import d1 lint --input <file> --format json # pre-import checks; errors block import
167+
pscale import d1 start <database> --org <org> --input <file> --dry-run --format json # plan + migration ID, no writes
168+
pscale import d1 start <database> --org <org> --input <file> --format json # run the import
169+
pscale import d1 verify <database> --org <org> --migration-id <id> --sqlite <file> --format json
170+
pscale import d1 complete <database> --org <org> --migration-id <id> --format json
171+
```
172+
173+
Branch is an optional second positional (defaults to the default branch). `status --migration-id <id>` shows saved migration state; `convert-schema --input <file>` converts schema only.
174+
175+
**`start` in JSON mode does not prompt.** The confirmation prompt is human-format only; with `--format json`, `start` loads data immediately. Run `start --dry-run` first, show the user the plan, and only run the real `start` after they approve.
176+
160177
## API passthrough
161178

162179
```bash

go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ require (
2121
github.com/google/go-cmp v0.7.0
2222
github.com/hashicorp/go-cleanhttp v0.5.2
2323
github.com/hashicorp/go-version v1.8.0
24+
github.com/jackc/pgx/v5 v5.8.0
2425
github.com/lensesio/tableprinter v0.0.0-20201125135848-89e81fc956e7
2526
github.com/lib/pq v1.12.0
2627
github.com/mark3labs/mcp-go v0.46.0
@@ -78,6 +79,9 @@ require (
7879
github.com/google/uuid v1.6.0 // indirect
7980
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
8081
github.com/inconshreveable/mousetrap v1.1.0 // indirect
82+
github.com/jackc/pgpassfile v1.0.0 // indirect
83+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
84+
github.com/jackc/puddle/v2 v2.2.2 // indirect
8185
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 // indirect
8286
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
8387
github.com/klauspost/compress v1.18.2 // indirect
@@ -115,7 +119,6 @@ require (
115119
google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
116120
google.golang.org/grpc v1.79.3 // indirect
117121
google.golang.org/protobuf v1.36.10 // indirect
118-
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
119122
gopkg.in/yaml.v3 v3.0.1 // indirect
120123
)
121124

go.sum

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,14 @@ github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u
112112
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
113113
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
114114
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
115+
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
116+
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
117+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
118+
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
119+
github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo=
120+
github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw=
121+
github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
122+
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
115123
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23 h1:M8exrBzuhWcU6aoHJlHWPe4qFjVKzkMGRal78f5jRRU=
116124
github.com/kataras/tablewriter v0.0.0-20180708051242-e063d29b7c23/go.mod h1:kBSna6b0/RzsOcOZf515vAXwSsXYusl2U7SA0XP09yI=
117125
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
@@ -120,7 +128,6 @@ github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uq
120128
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
121129
github.com/klauspost/connect-compress/v2 v2.1.0 h1:8fM8QrVeHT69e5VVSh4yjDaQASYIvOp2uMZq7nVLj2U=
122130
github.com/klauspost/connect-compress/v2 v2.1.0/go.mod h1:Ayurh2wscMMx3AwdGGVL+ylSR5316WfApREDgsqHyH8=
123-
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
124131
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
125132
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
126133
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
@@ -218,6 +225,7 @@ github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjb
218225
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
219226
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
220227
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
228+
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
221229
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
222230
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
223231
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=

internal/cmd/importcmd/d1.go

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
package importcmd
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/spf13/cobra"
7+
8+
ps "github.com/planetscale/planetscale-go/planetscale"
9+
10+
"github.com/planetscale/cli/internal/cmdutil"
11+
"github.com/planetscale/cli/internal/import/d1"
12+
"github.com/planetscale/cli/internal/printer"
13+
)
14+
15+
const defaultD1Branch = "main"
16+
17+
var d1DatabaseBranchArgs = cobra.RangeArgs(1, 2)
18+
19+
func parseDatabaseBranch(args []string) (database, branch string) {
20+
database = args[0]
21+
branch = defaultD1Branch
22+
if len(args) > 1 {
23+
branch = args[1]
24+
}
25+
return database, branch
26+
}
27+
28+
func d1Org(ch *cmdutil.Helper) string {
29+
return ch.Config.Organization
30+
}
31+
32+
func writeD1(ch *cmdutil.Helper, resp d1.Response) error {
33+
if resp.Status == "error" {
34+
switch ch.Printer.Format() {
35+
case printer.JSON:
36+
if err := ch.Printer.PrintJSON(resp); err != nil {
37+
return err
38+
}
39+
case printer.Human:
40+
d1.PrintHumanResponse(ch.Printer, resp)
41+
default:
42+
return fmt.Errorf(`import d1 does not support output format %q (use human or json)`, ch.Printer.Format())
43+
}
44+
return d1CommandError(resp)
45+
}
46+
47+
switch ch.Printer.Format() {
48+
case printer.JSON:
49+
return ch.Printer.PrintJSON(resp)
50+
case printer.Human:
51+
d1.PrintHumanResponse(ch.Printer, resp)
52+
return nil
53+
default:
54+
return fmt.Errorf(`import d1 does not support output format %q (use human or json)`, ch.Printer.Format())
55+
}
56+
}
57+
58+
func d1CommandError(resp d1.Response) error {
59+
msg := "import d1 command failed"
60+
if resp.Error != nil {
61+
msg = resp.Error.Message
62+
if resp.Error.Remediation != "" {
63+
msg += "\n" + resp.Error.Remediation
64+
}
65+
}
66+
return &cmdutil.Error{
67+
Msg: msg,
68+
ExitCode: cmdutil.ActionRequestedExitCode,
69+
Handled: true,
70+
}
71+
}
72+
73+
func d1NotifyAPI(client *ps.Client) d1.NotifyAPIConfig {
74+
return d1.NotifyAPIConfig{Client: client}
75+
}
76+
77+
func importTableCount(prepared *d1.ImportPrepareResult) int {
78+
if prepared == nil || prepared.Plan == nil {
79+
return 0
80+
}
81+
return countDataTables(prepared.Plan.Tables)
82+
}
83+
84+
func verifyTableCount(org, database, branch, migrationID, inputPath string) int {
85+
path := inputPath
86+
if path == "" && migrationID != "" {
87+
if state, err := d1.LoadState(org, database, branch, migrationID); err == nil {
88+
path = state.InputPath
89+
}
90+
}
91+
if path == "" {
92+
return 0
93+
}
94+
tables, err := d1.ParseDump(path)
95+
if err != nil {
96+
return 0
97+
}
98+
n := 0
99+
for _, t := range tables {
100+
if !d1.IsORMMetadataTable(t.Name) {
101+
n++
102+
}
103+
}
104+
return n
105+
}
106+
107+
func countDataTables(tables []d1.TablePlan) int {
108+
n := 0
109+
for _, table := range tables {
110+
if !d1.IsORMMetadataTable(table.Name) {
111+
n++
112+
}
113+
}
114+
return n
115+
}
116+
117+
// D1Cmd returns the import d1 subcommand group.
118+
func D1Cmd(ch *cmdutil.Helper) *cobra.Command {
119+
cmd := &cobra.Command{
120+
Use: "d1 <command>",
121+
Short: "Import Cloudflare D1 into PlanetScale Postgres",
122+
Long: `Offline import from Cloudflare D1 (SQLite) to PlanetScale Postgres.
123+
124+
Export your D1 database with wrangler (wrangler d1 export <name> --remote --output ./d1-export.sql),
125+
lint the dump, then start the import (use --dry-run to preview).
126+
All commands support --format json for machine-readable output.
127+
128+
Branch-scoped commands use the same positional form as other PlanetScale CLI commands:
129+
pscale import d1 start <database> [branch] --input ./d1-export.sql
130+
Org comes from your pscale config (pscale org).`,
131+
}
132+
133+
cmd.AddCommand(d1DoctorCmd(ch))
134+
cmd.AddCommand(d1LintCmd(ch))
135+
cmd.AddCommand(d1ConvertSchemaCmd(ch))
136+
cmd.AddCommand(d1StartCmd(ch))
137+
cmd.AddCommand(d1VerifyCmd(ch))
138+
cmd.AddCommand(d1StatusCmd(ch))
139+
cmd.AddCommand(d1CompleteCmd(ch))
140+
141+
return cmd
142+
}
Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package importcmd
2+
3+
import (
4+
"github.com/spf13/cobra"
5+
6+
"github.com/planetscale/cli/internal/cmdutil"
7+
"github.com/planetscale/cli/internal/import/d1"
8+
"github.com/planetscale/cli/internal/printer"
9+
)
10+
11+
func d1CompleteCmd(ch *cmdutil.Helper) *cobra.Command {
12+
var flags struct {
13+
migrationID string
14+
force bool
15+
}
16+
17+
cmd := &cobra.Command{
18+
Use: "complete <database> [branch]",
19+
Aliases: []string{"teardown"},
20+
Short: "Mark a D1 migration as complete in local state",
21+
Args: d1DatabaseBranchArgs,
22+
Example: ` pscale import d1 complete mydb --migration-id abc123
23+
pscale import d1 complete mydb --migration-id abc123 --format json`,
24+
RunE: func(cmd *cobra.Command, args []string) error {
25+
database, branch := parseDatabaseBranch(args)
26+
if !flags.force && ch.Printer.Format() == printer.Human {
27+
if err := ch.Printer.ConfirmCommand(flags.migrationID, "import d1 complete", "complete"); err != nil {
28+
return err
29+
}
30+
}
31+
client, err := ch.Client()
32+
if err != nil {
33+
return writeD1(ch, d1.ErrorResponse("complete", err))
34+
}
35+
resp, err := d1.CompleteResponse(d1Org(ch), database, branch, flags.migrationID)
36+
if err != nil {
37+
return writeD1(ch, d1.ErrorResponse("complete", err))
38+
}
39+
if err := d1.Complete(d1Org(ch), database, branch, flags.migrationID, d1NotifyAPI(client)); err != nil {
40+
return writeD1(ch, d1.ErrorResponse("complete", err))
41+
}
42+
return writeD1(ch, resp)
43+
},
44+
}
45+
46+
cmd.Flags().StringVar(&flags.migrationID, "migration-id", "", "Migration ID")
47+
cmd.Flags().BoolVar(&flags.force, "force", false, "Skip confirmation prompt")
48+
cmd.MarkFlagRequired("migration-id")
49+
return cmd
50+
}
Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package importcmd
2+
3+
import (
4+
"bytes"
5+
"io"
6+
"net/http"
7+
"net/http/httptest"
8+
"strings"
9+
"testing"
10+
11+
ps "github.com/planetscale/planetscale-go/planetscale"
12+
13+
"github.com/planetscale/cli/internal/cmdutil"
14+
"github.com/planetscale/cli/internal/config"
15+
"github.com/planetscale/cli/internal/import/d1"
16+
"github.com/planetscale/cli/internal/printer"
17+
)
18+
19+
func TestD1CompleteCmd(t *testing.T) {
20+
t.Setenv("PSCALE_TEST_MODE", "1")
21+
22+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
23+
w.WriteHeader(http.StatusAccepted)
24+
}))
25+
defer srv.Close()
26+
27+
client, err := ps.NewClient(
28+
ps.WithBaseURL(srv.URL),
29+
ps.WithAccessToken("token"),
30+
)
31+
if err != nil {
32+
t.Fatalf("NewClient: %v", err)
33+
}
34+
35+
const migrationID = "completecmd123"
36+
fixture := d1FixturePath(t)
37+
if err := d1.SavePlan(&d1.PlanResult{
38+
MigrationID: migrationID,
39+
Org: "acme",
40+
Database: "mydb",
41+
Branch: "main",
42+
InputPath: fixture,
43+
}); err != nil {
44+
t.Fatalf("SavePlan: %v", err)
45+
}
46+
if err := d1.SetMigrationPhase("acme", "mydb", "main", migrationID, d1.PhaseVerified); err != nil {
47+
t.Fatalf("SetMigrationPhase: %v", err)
48+
}
49+
50+
var buf bytes.Buffer
51+
format := printer.JSON
52+
p := printer.NewPrinter(&format)
53+
p.SetResourceOutput(&buf)
54+
55+
ch := &cmdutil.Helper{
56+
Printer: p,
57+
Config: &config.Config{Organization: "acme"},
58+
Client: func() (*ps.Client, error) {
59+
return client, nil
60+
},
61+
}
62+
63+
cmd := d1CompleteCmd(ch)
64+
cmd.SetArgs([]string{"mydb", "--migration-id", migrationID, "--force"})
65+
cmd.SetOut(io.Discard)
66+
cmd.SetErr(io.Discard)
67+
if err := cmd.Execute(); err != nil {
68+
t.Fatalf("execute: %v", err)
69+
}
70+
71+
assertJSONField(t, &buf, "command", "complete")
72+
assertJSONField(t, &buf, "status", "ok")
73+
assertJSONField(t, &buf, "migration_id", migrationID)
74+
if !strings.Contains(buf.String(), "reminder") {
75+
t.Fatalf("expected reminder in complete JSON output:\n%s", buf.String())
76+
}
77+
if !strings.Contains(buf.String(), "next_steps") {
78+
t.Fatalf("expected next_steps in complete JSON output:\n%s", buf.String())
79+
}
80+
}
81+
82+
func TestD1CompleteCmdRequiresMigrationID(t *testing.T) {
83+
ch, _ := newD1TestHelper(t)
84+
85+
cmd := d1CompleteCmd(ch)
86+
if err := executeD1Cmd(t, cmd, "mydb"); err == nil {
87+
t.Fatal("expected error when --migration-id is missing")
88+
}
89+
}

0 commit comments

Comments
 (0)