-
Notifications
You must be signed in to change notification settings - Fork 178
Expand file tree
/
Copy pathsqlite-import.ts
More file actions
228 lines (194 loc) · 6.97 KB
/
Copy pathsqlite-import.ts
File metadata and controls
228 lines (194 loc) · 6.97 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
import { Database } from "bun:sqlite";
import { Data } from "effect";
import { randomBytes } from "node:crypto";
import { existsSync, mkdirSync, renameSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
/* oxlint-disable executor/no-json-parse, executor/no-switch-statement, executor/no-try-catch-or-throw -- boundary: one-shot legacy SQLite importer normalizes unknown rows and wraps native sqlite failures */
import { type AnyColumn, type AnyTable, type FumaTables } from "@executor-js/sdk";
type SqliteRow = Record<string, unknown>;
type ImportFumaDb = Readonly<{
createMany: (table: string, rows: SqliteRow[]) => Promise<unknown>;
transaction: <A>(run: (db: ImportFumaDb) => Promise<A>) => Promise<A>;
}>;
export class LocalSqliteImportError extends Data.TaggedError("LocalSqliteImportError")<{
readonly message: string;
readonly sqlitePath: string;
readonly table?: string;
readonly cause: unknown;
}> {}
export interface LocalSqliteImportOptions {
readonly sqlitePath: string;
readonly markerPath: string;
readonly target: ImportFumaDb;
readonly tables: FumaTables;
readonly scopeId: string;
}
export interface LocalSqliteImportResult {
readonly imported: boolean;
readonly importedRows: number;
readonly importedTables: readonly string[];
readonly backupPath?: string;
}
const quoteIdent = (value: string): string => `"${value.replaceAll('"', '""')}"`;
const sqliteStringLiteral = (value: string): string => `'${value.replaceAll("'", "''")}'`;
const tableExists = (sqlite: Database, tableName: string): boolean => {
const row = sqlite
.query<{ name: string }, [string]>(
"SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?",
)
.get(tableName);
return row !== null;
};
const sqliteColumnNames = (sqlite: Database, tableName: string): ReadonlySet<string> => {
const rows = sqlite
.query<{ name: string }, []>(`PRAGMA table_info(${sqliteStringLiteral(tableName)})`)
.all();
return new Set(rows.map((row) => row.name));
};
const readRows = (sqlite: Database, tableName: string): readonly SqliteRow[] =>
sqlite.query<SqliteRow, []>(`SELECT * FROM ${quoteIdent(tableName)}`).all();
const parseJson = (value: string): unknown => {
try {
return JSON.parse(value);
} catch {
return value;
}
};
const toBigInt = (value: unknown): unknown => {
if (typeof value === "bigint") return value;
if (typeof value === "number" && Number.isFinite(value)) return BigInt(value);
if (typeof value === "string" && value.trim().length > 0) return BigInt(value);
return value;
};
const toDate = (value: unknown): unknown => {
if (value instanceof Date) return value;
if (typeof value === "number") return new Date(value);
if (typeof value === "string") {
const trimmed = value.trim();
if (/^-?\d+$/.test(trimmed)) return new Date(Number(trimmed));
return new Date(trimmed);
}
return value;
};
const toBool = (value: unknown): unknown => {
if (typeof value === "boolean") return value;
if (typeof value === "number") return value !== 0;
if (typeof value === "string") return value === "1" || value.toLowerCase() === "true";
return value;
};
const defaultColumnValue = (input: {
readonly tableKey: string;
readonly columnKey: string;
readonly row: SqliteRow;
readonly scopeId: string;
}): unknown => {
if (input.columnKey === "scope_id") return input.scopeId;
if (input.tableKey === "blob" && input.columnKey === "id") {
const namespace = input.row.namespace;
const key = input.row.key;
if (typeof namespace === "string" && typeof key === "string") {
return JSON.stringify([namespace, key]);
}
}
return undefined;
};
const normalizeColumnValue = (value: unknown, column: AnyColumn): unknown => {
if (value === undefined || value === null) return value;
switch (column.type) {
case "bool":
return toBool(value);
case "bigint":
return toBigInt(value);
case "date":
case "timestamp":
return toDate(value);
case "json":
return typeof value === "string" ? parseJson(value) : value;
default:
return value;
}
};
const toFumaRow = (input: {
readonly tableKey: string;
readonly table: AnyTable;
readonly sqliteColumns: ReadonlySet<string>;
readonly row: SqliteRow;
readonly scopeId: string;
}): SqliteRow => {
const out: SqliteRow = {};
for (const [columnKey, column] of Object.entries(input.table.columns)) {
if (columnKey === "row_id") continue;
const sqlName = column.names.sql;
const rawValue = input.sqliteColumns.has(sqlName)
? input.row[sqlName]
: defaultColumnValue({
tableKey: input.tableKey,
columnKey,
row: input.row,
scopeId: input.scopeId,
});
const value = normalizeColumnValue(rawValue, column);
if (value !== undefined) out[columnKey] = value;
}
return out;
};
const moveImportedSqliteAside = (sqlitePath: string): string => {
const backupPath = `${sqlitePath}.imported-${Date.now()}-${randomBytes(4).toString("hex")}`;
renameSync(sqlitePath, backupPath);
for (const suffix of ["-wal", "-shm"]) {
const source = `${sqlitePath}${suffix}`;
if (existsSync(source)) renameSync(source, `${backupPath}${suffix}`);
}
return backupPath;
};
export const importSqliteDataToFuma = async (
options: LocalSqliteImportOptions,
): Promise<LocalSqliteImportResult> => {
if (!existsSync(options.sqlitePath) || existsSync(options.markerPath)) {
return { imported: false, importedRows: 0, importedTables: [] };
}
let sqlite: Database | null = null;
try {
sqlite = new Database(options.sqlitePath, { readonly: true });
const importedTables: string[] = [];
let importedRows = 0;
await options.target.transaction(async (db) => {
for (const [tableKey, table] of Object.entries(options.tables)) {
const tableName = table.names.sql;
if (!tableExists(sqlite!, tableName)) continue;
const sqliteColumns = sqliteColumnNames(sqlite!, tableName);
const rows = readRows(sqlite!, tableName).map((row) =>
toFumaRow({
tableKey,
table,
sqliteColumns,
row,
scopeId: options.scopeId,
}),
);
if (rows.length === 0) continue;
await db.createMany(tableKey, rows);
importedTables.push(tableKey);
importedRows += rows.length;
}
});
sqlite.close();
sqlite = null;
mkdirSync(dirname(options.markerPath), { recursive: true });
writeFileSync(
options.markerPath,
`${JSON.stringify({ importedAt: new Date().toISOString(), importedRows, importedTables })}\n`,
{ flag: "w" },
);
const backupPath = moveImportedSqliteAside(options.sqlitePath);
return { imported: true, importedRows, importedTables, backupPath };
} catch (cause) {
throw new LocalSqliteImportError({
message: `Failed to import local SQLite data from ${options.sqlitePath}`,
sqlitePath: options.sqlitePath,
cause,
});
} finally {
sqlite?.close();
}
};