key all streams and remove keyBy (#144)

Author: samwillis

.changeset/orange-laws-watch.md

@@ -0,0 +1,7 @@
+---
+"@tanstack/react-db": patch
+"@tanstack/vue-db": patch
+"@tanstack/db": patch
+---
+
+the `keyBy` query operator has been removed, keying withing the query pipeline is now automatic

docs/index.md

@@ -279,7 +279,6 @@ const Todos = () => {
       .where('@completed', '=', false)
       .orderBy({'@created_at': 'asc'})
       .select('@id', '@text')
-      .keyBy('@id')
   )
 
   return <List items={ todos } />
@@ -302,7 +301,6 @@ const Todos = () => {
       })
       .where('@lists.active', '=', true)
       .select(`@todos.id`, `@todos.title`, `@lists.name`)
-      .keyBy('@id')
   )
 
   return <List items={ todos } />
@@ -534,7 +532,6 @@ const Todos = () => {
       })
       .where('@l.active', '=', true)
       .select('@t.id', '@t.text', '@t.status', '@l.name')
-      .keyBy('@id')
   )
 
   // Handle local writes by sending them to your API.

docs/overview.md

@@ -283,7 +283,6 @@ const Todos = () => {
       .where('@completed', '=', false)
       .orderBy({'@created_at': 'asc'})
       .select('@id', '@text')
-      .keyBy('@id')
   )
 
   return <List items={ todos } />
@@ -306,7 +305,6 @@ const Todos = () => {
       })
       .where('@lists.active', '=', true)
       .select(`@todos.id`, `@todos.title`, `@lists.name`)
-      .keyBy('@id')
   )
 
   return <List items={ todos } />
@@ -538,7 +536,6 @@ const Todos = () => {
       })
       .where('@l.active', '=', true)
       .select('@t.id', '@t.text', '@t.status', '@l.name')
-      .keyBy('@id')
   )
 
   // Handle local writes by sending them to your API.

examples/react/todo/src/App.tsx

@@ -286,15 +286,13 @@ export default function App() {
   const { data: todos } = useLiveQuery((q) =>
     q
       .from({ todoCollection: todoCollection as Collection<UpdateTodo> })
-      .keyBy(`@id`)
       .orderBy(`@created_at`)
       .select(`@id`, `@created_at`, `@text`, `@completed`)
   )
 
   const { data: configData } = useLiveQuery((q) =>
     q
       .from({ configCollection: configCollection as Collection<UpdateConfig> })
-      .keyBy(`@id`)
       .select(`@id`, `@key`, `@value`)
   )
 

packages/db/src/query/compiled-query.ts

@@ -112,18 +112,23 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
     return this.resultCollection
   }
 
-  private sendChangesToInput(inputKey: string, changes: Array<ChangeMessage>) {
+  private sendChangesToInput(
+    inputKey: string,
+    changes: Array<ChangeMessage>,
+    getId: (item: ChangeMessage[`value`]) => any
+  ) {
     const input = this.inputs[inputKey]!
     const multiSetArray: MultiSetArray<unknown> = []
     for (const change of changes) {
+      const key = getId(change.value)
       if (change.type === `insert`) {
-        multiSetArray.push([change.value, 1])
+        multiSetArray.push([[key, change.value], 1])
       } else if (change.type === `update`) {
-        multiSetArray.push([change.previousValue, -1])
-        multiSetArray.push([change.value, 1])
+        multiSetArray.push([[key, change.previousValue], -1])
+        multiSetArray.push([[key, change.value], 1])
       } else {
         // change.type === `delete`
-        multiSetArray.push([change.value, -1])
+        multiSetArray.push([[key, change.value], -1])
       }
     }
     input.sendData(this.version, new MultiSet(multiSetArray))
@@ -157,7 +162,11 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
 
     batch(() => {
       Object.entries(this.inputCollections).forEach(([key, collection]) => {
-        this.sendChangesToInput(key, collection.currentStateAsChanges())
+        this.sendChangesToInput(
+          key,
+          collection.currentStateAsChanges(),
+          collection.config.getId
+        )
       })
       this.incrementVersion()
       this.sendFrontierToAllInputs()
@@ -168,7 +177,11 @@ export class CompiledQuery<TResults extends object = Record<string, unknown>> {
       fn: () => {
         batch(() => {
           Object.entries(this.inputCollections).forEach(([key, collection]) => {
-            this.sendChangesToInput(key, collection.derivedChanges.state)
+            this.sendChangesToInput(
+              key,
+              collection.derivedChanges.state,
+              collection.config.getId
+            )
           })
           this.incrementVersion()
           this.sendFrontierToAllInputs()

packages/db/src/query/evaluators.ts

@@ -1,4 +1,4 @@
-import { evaluateOperandOnNestedRow } from "./extractors.js"
+import { evaluateOperandOnNamespacedRow } from "./extractors.js"
 import { compareValues, convertLikeToRegex, isValueInArray } from "./utils.js"
 import type {
   Comparator,
@@ -7,21 +7,22 @@ import type {
   LogicalOperator,
   SimpleCondition,
 } from "./schema.js"
+import type { NamespacedRow } from "../types.js"
 
 /**
  * Evaluates a condition against a nested row structure
  */
-export function evaluateConditionOnNestedRow(
-  nestedRow: Record<string, unknown>,
+export function evaluateConditionOnNamespacedRow(
+  namespacedRow: NamespacedRow,
   condition: Condition,
   mainTableAlias?: string,
   joinedTableAlias?: string
 ): boolean {
   // Handle simple conditions with exactly 3 elements
   if (condition.length === 3 && !Array.isArray(condition[0])) {
     const [left, comparator, right] = condition as SimpleCondition
-    return evaluateSimpleConditionOnNestedRow(
-      nestedRow,
+    return evaluateSimpleConditionOnNamespacedRow(
+      namespacedRow,
       left,
       comparator,
       right,
@@ -38,8 +39,8 @@ export function evaluateConditionOnNestedRow(
     ![`and`, `or`].includes(condition[1] as string)
   ) {
     // Start with the first condition (first 3 elements)
-    let result = evaluateSimpleConditionOnNestedRow(
-      nestedRow,
+    let result = evaluateSimpleConditionOnNamespacedRow(
+      namespacedRow,
       condition[0],
       condition[1] as Comparator,
       condition[2],
@@ -53,8 +54,8 @@ export function evaluateConditionOnNestedRow(
 
       // Make sure we have a complete condition to evaluate
       if (i + 3 <= condition.length) {
-        const nextResult = evaluateSimpleConditionOnNestedRow(
-          nestedRow,
+        const nextResult = evaluateSimpleConditionOnNamespacedRow(
+          namespacedRow,
           condition[i + 1],
           condition[i + 2] as Comparator,
           condition[i + 3],
@@ -78,8 +79,8 @@ export function evaluateConditionOnNestedRow(
   // Handle nested composite conditions where the first element is an array
   if (condition.length > 0 && Array.isArray(condition[0])) {
     // Start with the first condition
-    let result = evaluateConditionOnNestedRow(
-      nestedRow,
+    let result = evaluateConditionOnNamespacedRow(
+      namespacedRow,
       condition[0] as Condition,
       mainTableAlias,
       joinedTableAlias
@@ -96,8 +97,8 @@ export function evaluateConditionOnNestedRow(
       if (operator === `and`) {
         result =
           result &&
-          evaluateConditionOnNestedRow(
-            nestedRow,
+          evaluateConditionOnNamespacedRow(
+            namespacedRow,
             nextCondition,
             mainTableAlias,
             joinedTableAlias
@@ -106,8 +107,8 @@ export function evaluateConditionOnNestedRow(
         // logicalOp === `or`
         result =
           result ||
-          evaluateConditionOnNestedRow(
-            nestedRow,
+          evaluateConditionOnNamespacedRow(
+            namespacedRow,
             nextCondition,
             mainTableAlias,
             joinedTableAlias
@@ -125,23 +126,23 @@ export function evaluateConditionOnNestedRow(
 /**
  * Evaluates a simple condition against a nested row structure
  */
-export function evaluateSimpleConditionOnNestedRow(
-  nestedRow: Record<string, unknown>,
+export function evaluateSimpleConditionOnNamespacedRow(
+  namespacedRow: Record<string, unknown>,
   left: ConditionOperand,
   comparator: Comparator,
   right: ConditionOperand,
   mainTableAlias?: string,
   joinedTableAlias?: string
 ): boolean {
-  const leftValue = evaluateOperandOnNestedRow(
-    nestedRow,
+  const leftValue = evaluateOperandOnNamespacedRow(
+    namespacedRow,
     left,
     mainTableAlias,
     joinedTableAlias
   )
 
-  const rightValue = evaluateOperandOnNestedRow(
-    nestedRow,
+  const rightValue = evaluateOperandOnNamespacedRow(
+    namespacedRow,
     right,
     mainTableAlias,
     joinedTableAlias

packages/db/src/query/extractors.ts

@@ -3,14 +3,14 @@ import type { AllowedFunctionName, ConditionOperand } from "./schema.js"
 
 /**
  * Extracts a value from a nested row structure
- * @param nestedRow The nested row structure
+ * @param namespacedRow The nested row structure
  * @param columnRef The column reference (may include table.column format)
  * @param mainTableAlias The main table alias to check first for columns without table reference
  * @param joinedTableAlias The joined table alias to check second for columns without table reference
  * @returns The extracted value or undefined if not found
  */
-export function extractValueFromNestedRow(
-  nestedRow: Record<string, unknown>,
+export function extractValueFromNamespacedRow(
+  namespacedRow: Record<string, unknown>,
   columnRef: string,
   mainTableAlias?: string,
   joinedTableAlias?: string
@@ -20,7 +20,7 @@ export function extractValueFromNestedRow(
     const [tableAlias, colName] = columnRef.split(`.`) as [string, string]
 
     // Get the table data
-    const tableData = nestedRow[tableAlias] as
+    const tableData = namespacedRow[tableAlias] as
       | Record<string, unknown>
       | null
       | undefined
@@ -34,16 +34,19 @@ export function extractValueFromNestedRow(
     return value
   } else {
     // If no table is specified, first try to find in the main table if provided
-    if (mainTableAlias && nestedRow[mainTableAlias]) {
-      const mainTableData = nestedRow[mainTableAlias] as Record<string, unknown>
+    if (mainTableAlias && namespacedRow[mainTableAlias]) {
+      const mainTableData = namespacedRow[mainTableAlias] as Record<
+        string,
+        unknown
+      >
       if (typeof mainTableData === `object` && columnRef in mainTableData) {
         return mainTableData[columnRef]
       }
     }
 
     // Then try the joined table if provided
-    if (joinedTableAlias && nestedRow[joinedTableAlias]) {
-      const joinedTableData = nestedRow[joinedTableAlias] as Record<
+    if (joinedTableAlias && namespacedRow[joinedTableAlias]) {
+      const joinedTableData = namespacedRow[joinedTableAlias] as Record<
         string,
         unknown
       >
@@ -53,7 +56,7 @@ export function extractValueFromNestedRow(
     }
 
     // If not found in main or joined table, try to find the column in any table
-    for (const [_tableAlias, tableData] of Object.entries(nestedRow)) {
+    for (const [_tableAlias, tableData] of Object.entries(namespacedRow)) {
       if (
         tableData &&
         typeof tableData === `object` &&
@@ -69,17 +72,17 @@ export function extractValueFromNestedRow(
 /**
  * Evaluates an operand against a nested row structure
  */
-export function evaluateOperandOnNestedRow(
-  nestedRow: Record<string, unknown>,
+export function evaluateOperandOnNamespacedRow(
+  namespacedRow: Record<string, unknown>,
   operand: ConditionOperand,
   mainTableAlias?: string,
   joinedTableAlias?: string
 ): unknown {
   // Handle column references
   if (typeof operand === `string` && operand.startsWith(`@`)) {
     const columnRef = operand.substring(1)
-    return extractValueFromNestedRow(
-      nestedRow,
+    return extractValueFromNamespacedRow(
+      namespacedRow,
       columnRef,
       mainTableAlias,
       joinedTableAlias
@@ -92,17 +95,17 @@ export function evaluateOperandOnNestedRow(
 
     if (typeof colRef === `string`) {
       // First try to extract from nested row structure
-      const nestedValue = extractValueFromNestedRow(
-        nestedRow,
+      const nestedValue = extractValueFromNamespacedRow(
+        namespacedRow,
         colRef,
         mainTableAlias,
         joinedTableAlias
       )
 
       // If not found in nested structure, check if it's a direct property of the row
       // This is important for HAVING clauses that reference aggregated values
-      if (nestedValue === undefined && colRef in nestedRow) {
-        return nestedRow[colRef]
+      if (nestedValue === undefined && colRef in namespacedRow) {
+        return namespacedRow[colRef]
       }
 
       return nestedValue
@@ -121,15 +124,15 @@ export function evaluateOperandOnNestedRow(
     // If the arguments are a reference or another expression, evaluate them first
     const evaluatedArgs = Array.isArray(args)
       ? args.map((arg) =>
-          evaluateOperandOnNestedRow(
-            nestedRow,
+          evaluateOperandOnNamespacedRow(
+            namespacedRow,
             arg as ConditionOperand,
             mainTableAlias,
             joinedTableAlias
           )
         )
-      : evaluateOperandOnNestedRow(
-          nestedRow,
+      : evaluateOperandOnNamespacedRow(
+          namespacedRow,
           args as ConditionOperand,
           mainTableAlias,
           joinedTableAlias