improvement(tables): order export + snapshot rows by order_key so the CSV matches the grid under fractional ordering

Author: TheodoreSpeaks

apps/sim/lib/table/export-runner.test.ts

@@ -93,7 +93,7 @@ describe('runTableExport', () => {
       return Promise.resolve(handle)
     })
     mockSelectExportRowPage.mockResolvedValue([
-      { id: 'r1', data: { col_name: 'Ada' }, position: 0 },
+      { id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' },
     ])
   })
 

apps/sim/lib/table/export-runner.ts

@@ -78,7 +78,7 @@ export async function runTableExport(payload: TableExportPayload): Promise<void>
 
     let exported = 0
     let firstJsonRow = true
-    let after: { position: number; id: string } | null = null
+    let after: { orderKey: string; id: string } | null = null
     while (true) {
       // Ownership gate before every page: a canceled job stops within one batch.
       const owns = await updateJobProgress(tableId, exported, jobId)
@@ -103,7 +103,7 @@ export async function runTableExport(payload: TableExportPayload): Promise<void>
 
       exported += page.length
       const last = page[page.length - 1]
-      after = { position: last.position, id: last.id }
+      after = { orderKey: last.orderKey, id: last.id }
       if (page.length < EXPORT_BATCH_SIZE) break
     }
     if (format === 'json') await handle.write(']')

apps/sim/lib/table/jobs/service.ts

@@ -207,35 +207,35 @@ export async function getJobProgress(tableId: string, jobId: string): Promise<nu
 }
 
 /**
- * One keyset page of rows for the export worker, ordered by `(position, id)`. Keyset (not
- * OFFSET) keeps each page O(page) — offset paging re-scans every prior row per page, which is
- * O(N²) across a large export. `(position, id)` is total (position exists on every row; id breaks
- * ties) and served by the `(table_id, position)` index; under fractional ordering a manually
- * reordered table may export in near-grid rather than exact grid order — the right trade for a
- * bulk dump. The delete-job visibility mask applies, like every user-facing read.
+ * One keyset page of rows for the export worker, ordered by `(order_key, id)` — the same
+ * authoritative visual order the grid (`queryRows`) uses, so exports and snapshots match what the
+ * user sees even after manual reorders. Keyset (not OFFSET) keeps each page O(page); `order_key` is
+ * present on every row (always assigned on insert, backfilled for legacy rows) with `id` as the
+ * tiebreaker, and the `(table_id, order_key, id)` index serves it. The delete-job visibility mask
+ * applies, like every user-facing read.
  */
 export async function selectExportRowPage(
   table: TableDefinition,
-  after: { position: number; id: string } | null,
+  after: { orderKey: string; id: string } | null,
   limit: number
-): Promise<Array<{ id: string; data: RowData; position: number }>> {
+): Promise<Array<{ id: string; data: RowData; orderKey: string }>> {
   const deleteMask = await pendingDeleteMask(table)
   const rows = await db
-    .select({ id: userTableRows.id, data: userTableRows.data, position: userTableRows.position })
+    .select({ id: userTableRows.id, data: userTableRows.data, orderKey: userTableRows.orderKey })
     .from(userTableRows)
     .where(
       and(
         eq(userTableRows.tableId, table.id),
         eq(userTableRows.workspaceId, table.workspaceId),
         deleteMask,
         after
-          ? sql`(${userTableRows.position}, ${userTableRows.id}) > (${after.position}, ${after.id})`
+          ? sql`(${userTableRows.orderKey}, ${userTableRows.id}) > (${after.orderKey}, ${after.id})`
           : undefined
       )
     )
-    .orderBy(asc(userTableRows.position), asc(userTableRows.id))
+    .orderBy(asc(userTableRows.orderKey), asc(userTableRows.id))
     .limit(limit)
-  return rows as Array<{ id: string; data: RowData; position: number }>
+  return rows as Array<{ id: string; data: RowData; orderKey: string }>
 }
 
 /** How long a terminal export stays listable (and re-downloadable from the tray). */

apps/sim/lib/table/snapshot-cache.test.ts

@@ -54,7 +54,7 @@ describe('getOrCreateTableSnapshot', () => {
     lastHandle = null
     mockDeleteFile.mockResolvedValue(undefined)
     mockSelectExportRowPage.mockResolvedValueOnce([
-      { id: 'r1', data: { col_name: 'Ada' }, position: 0 },
+      { id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' },
     ])
     mockSelectExportRowPage.mockResolvedValue([])
     mockCreateMultipartUpload.mockImplementation(({ key }: { key: string }) => {
@@ -151,9 +151,9 @@ describe('getOrCreateTableSnapshot', () => {
     // second materialize needs its own page sequence
     mockSelectExportRowPage.mockReset()
     mockSelectExportRowPage
-      .mockResolvedValueOnce([{ id: 'r1', data: { col_name: 'Ada' }, position: 0 }])
+      .mockResolvedValueOnce([{ id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' }])
       .mockResolvedValueOnce([])
-      .mockResolvedValueOnce([{ id: 'r1', data: { col_name: 'Ada' }, position: 0 }])
+      .mockResolvedValueOnce([{ id: 'r1', data: { col_name: 'Ada' }, orderKey: 'a0' }])
       .mockResolvedValueOnce([])
 
     const ref = await getOrCreateTableSnapshot(table, 'req')
@@ -175,7 +175,7 @@ describe('getOrCreateTableSnapshot', () => {
     mockSelectExportRowPage.mockReset()
     // A full batch of wide rows on every page → the materialize loop keeps paging until the running
     // byte count crosses the cap, then aborts. Peak memory stays at one page (~MBs), not the cap.
-    const wideRow = { id: 'r', data: { col_name: 'x'.repeat(1000) }, position: 0 }
+    const wideRow = { id: 'r', data: { col_name: 'x'.repeat(1000) }, orderKey: 'a0' }
     const fullPage = Array.from({ length: 10000 }, () => wideRow)
     mockSelectExportRowPage.mockResolvedValue(fullPage)
 

apps/sim/lib/table/snapshot-cache.ts

@@ -103,7 +103,7 @@ async function materialize(table: TableDefinition, key: string): Promise<number>
     bytes += Buffer.byteLength(header)
     await handle.write(header)
 
-    let after: { position: number; id: string } | null = null
+    let after: { orderKey: string; id: string } | null = null
     while (true) {
       const page = await selectExportRowPage(table, after, SNAPSHOT_BATCH_SIZE)
       if (page.length === 0) break
@@ -116,7 +116,7 @@ async function materialize(table: TableDefinition, key: string): Promise<number>
       await handle.write(chunk)
 
       const last = page[page.length - 1]
-      after = { position: last.position, id: last.id }
+      after = { orderKey: last.orderKey, id: last.id }
       if (page.length < SNAPSHOT_BATCH_SIZE) break
     }