From aa0c963aba91fd5d52ded34b6c18b9268f8854f3 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 26 Nov 2025 15:50:28 +0100 Subject: [PATCH 01/35] feat: table schema are defined by value types --- .../internal-representation-parsing.ts | 28 +++--- .../value-representation-validity.ts | 7 ++ .../visitors/sql-column-type-visitor.ts | 6 ++ .../sql-value-representation-visitor.ts | 8 ++ .../src/lib/table-interpreter-executor.ts | 93 ++++++++++--------- .../src/grammar/expression.langium | 1 + .../lib/ast/expressions/evaluation-context.ts | 4 + .../internal-value-representation.ts | 6 ++ .../src/lib/ast/expressions/type-inference.ts | 4 + .../typed-object/block-type-wrapper.ts | 4 + .../wrappers/value-type/primitive/index.ts | 1 + .../primitive-value-type-provider.ts | 3 + .../value-type-definition-value-type.ts | 32 +++++++ .../lib/ast/wrappers/value-type/value-type.ts | 2 + .../property-assignment.ts | 12 +-- .../builtin-block-types/TableInterpreter.jv | 39 +++++--- 16 files changed, 175 insertions(+), 75 deletions(-) create mode 100644 libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts diff --git a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts index 3e8d2a093..464da9f32 100644 --- a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts +++ b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts @@ -86,23 +86,23 @@ class InternalRepresentationParserVisitor extends ValueTypeVisitor< return value; } - visitBoolean(vt: BooleanValuetype): boolean | InvalidValue { + override visitBoolean(vt: BooleanValuetype): boolean | InvalidValue { return vt.fromString(this.applyTrimOptions(this.value)); } - visitDecimal(vt: DecimalValuetype): number | InvalidValue { + override visitDecimal(vt: DecimalValuetype): number | InvalidValue { return vt.fromString(this.applyTrimOptions(this.value)); } - visitInteger(vt: IntegerValuetype): number | InvalidValue { + override visitInteger(vt: IntegerValuetype): number | InvalidValue { return vt.fromString(this.applyTrimOptions(this.value)); } - visitText(vt: TextValuetype): string { + override visitText(vt: TextValuetype): string { return vt.fromString(this.value); } - visitAtomicValueType( + override visitAtomicValueType( valueType: AtomicValueType, ): InternalValidValueRepresentation | InvalidValue { const containedTypes = valueType.getContainedTypes(); @@ -116,29 +116,35 @@ class InternalRepresentationParserVisitor extends ValueTypeVisitor< return containedType.acceptVisitor(this); } - visitCellRange(): InvalidValue { + override visitCellRange(): InvalidValue { return new InvalidValue(`Cannot parse cell ranges into internal values`); } - visitCollection(): InvalidValue { + override visitCollection(): InvalidValue { return new InvalidValue(`Cannot parse collections into internal values`); } - visitConstraint(): InvalidValue { + override visitConstraint(): InvalidValue { return new InvalidValue(`Cannot parse constraints into internal values`); } - visitRegex(): InvalidValue { + override visitRegex(): InvalidValue { return new InvalidValue(`Cannot parse regex into internal values`); } - visitTransform(): InvalidValue { + override visitTransform(): InvalidValue { return new InvalidValue(`Cannot parse transforms into internal values`); } - visitValuetypeAssignment(): InvalidValue { + override visitValuetypeAssignment(): InvalidValue { return new InvalidValue( `Cannot parse valuetype assignments into internal values`, ); } + + override visitValuetypeDefinition(): InvalidValue { + return new InvalidValue( + `Cannot parse valuetype definitions into internal values`, + ); + } } diff --git a/libs/execution/src/lib/types/value-types/value-representation-validity.ts b/libs/execution/src/lib/types/value-types/value-representation-validity.ts index 326f7e37c..eadda4006 100644 --- a/libs/execution/src/lib/types/value-types/value-representation-validity.ts +++ b/libs/execution/src/lib/types/value-types/value-representation-validity.ts @@ -21,6 +21,7 @@ import { type ValueType, ValueTypeVisitor, type ValuetypeAssignmentValuetype, + type ValuetypeDefinitionValuetype, isConstraintDefinition, } from '@jvalue/jayvee-language-server'; @@ -109,6 +110,12 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { return this.isValidForPrimitiveValuetype(valueType); } + override visitValuetypeDefinition( + valueType: ValuetypeDefinitionValuetype, + ): boolean { + return this.isValidForPrimitiveValuetype(valueType); + } + override visitCollection(valueType: CollectionValueType): boolean { return this.isValidForPrimitiveValuetype(valueType); } diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts index ea9037bfa..f47a92e3e 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts @@ -64,6 +64,12 @@ export class SQLColumnTypeVisitor extends ValueTypeVisitor { ); } + override visitValuetypeDefinition(): string { + throw new Error( + 'No visit implementation given for valueType definitions. Cannot be the type of a column.', + ); + } + override visitCollection(): string { throw new Error( 'No visit implementation given for collections. Cannot be the type of a column.', diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts index c64dd34e2..9b6bdba78 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts @@ -114,6 +114,14 @@ export class SQLValueRepresentationVisitor extends ValueTypeVisitor< ); } + override visitValuetypeDefinition(): ( + value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + ) => string { + throw new Error( + 'No visit implementation given for valueType definitions. Cannot be the type of a column.', + ); + } + override visitCollection(): ( value: InternalValidValueRepresentation | InternalErrorValueRepresentation, ) => string { diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index cb21cb2c7..5d7da20e1 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -17,6 +17,7 @@ import { parseValueToInternalRepresentation, } from '@jvalue/jayvee-execution'; import { + AtomicValueType, CellIndex, ERROR_TYPEGUARD, IOType, @@ -25,15 +26,16 @@ import { InvalidValue, MissingValue, type ValueType, - type ValuetypeAssignment, + ValueTypeProperty, internalValueToString, + isAtomicValueType, } from '@jvalue/jayvee-language-server'; export interface ColumnDefinitionEntry { sheetColumnIndex: number; columnName: string; valueType: ValueType; - astNode: ValuetypeAssignment; + astNode: ValueTypeProperty; } @implementsStatic() @@ -56,11 +58,9 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< 'header', context.valueTypeProvider.Primitives.Boolean, ); - const columnDefinitions = context.getPropertyValue( + const columnsValueTypeDefinition = context.getPropertyValue( 'columns', - context.valueTypeProvider.createCollectionValueTypeOf( - context.valueTypeProvider.Primitives.ValuetypeAssignment, - ), + context.valueTypeProvider.Primitives.ValuetypeDefinition, ); const skipLeadingWhitespace = context.getPropertyValue( 'skipLeadingWhitespace', @@ -71,6 +71,15 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< context.valueTypeProvider.Primitives.Boolean, ); + const columnsValueType = context.wrapperFactories.ValueType.wrap( + columnsValueTypeDefinition, + ); + + assert( + isAtomicValueType(columnsValueType), + 'This must have been checked earlier at the validation step', + ); + let columnEntries: ColumnDefinitionEntry[]; if (header) { @@ -86,16 +95,17 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< const headerRow = inputSheet.getHeaderRow(); columnEntries = this.deriveColumnDefinitionEntriesFromHeader( - columnDefinitions, + columnsValueType, headerRow, context, ); } else { - if (inputSheet.getNumberOfColumns() < columnDefinitions.length) { + if ( + inputSheet.getNumberOfColumns() < + columnsValueType.getProperties().length + ) { return R.err({ - message: `There are ${ - columnDefinitions.length - } column definitions but the input sheet only has ${inputSheet.getNumberOfColumns()} columns`, + message: `The value type ${columnsValueType.getName()} has ${columnsValueType.getProperties().length} properties, but the input sheet only has ${inputSheet.getNumberOfColumns()} columns`, diagnostic: { node: context.getOrFailProperty('columns'), }, @@ -103,7 +113,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< } columnEntries = this.deriveColumnDefinitionEntriesWithoutHeader( - columnDefinitions, + columnsValueType, context, ); } @@ -233,56 +243,53 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< } private deriveColumnDefinitionEntriesWithoutHeader( - columnDefinitions: ValuetypeAssignment[], + columnsValueType: AtomicValueType, context: ExecutionContext, ): ColumnDefinitionEntry[] { - return columnDefinitions.map( - (columnDefinition, columnDefinitionIndex) => { - const columnValuetype = context.wrapperFactories.ValueType.wrap( - columnDefinition.type, - ); - assert(columnValuetype !== undefined); - return { - sheetColumnIndex: columnDefinitionIndex, - columnName: columnDefinition.name, - valueType: columnValuetype, - astNode: columnDefinition, - }; - }, - ); + return columnsValueType.getProperties().map((property, propertyIndex) => { + const columnValuetype = context.wrapperFactories.ValueType.wrap( + property.valueType, + ); + assert(columnValuetype !== undefined); + return { + sheetColumnIndex: propertyIndex, + columnName: property.name, + valueType: columnValuetype, + astNode: property, + }; + }); } private deriveColumnDefinitionEntriesFromHeader( - columnDefinitions: ValuetypeAssignment[], + columnsValueType: AtomicValueType, headerRow: string[], context: ExecutionContext, ): ColumnDefinitionEntry[] { context.logger.logDebug(`Matching header with provided column names`); - const columnEntries: ColumnDefinitionEntry[] = []; - for (const columnDefinition of columnDefinitions) { + return columnsValueType.getProperties().flatMap((property) => { const indexOfMatchingHeader = headerRow.findIndex( - (headerColumnName) => headerColumnName === columnDefinition.name, + (headerColumnName) => headerColumnName === property.name, ); if (indexOfMatchingHeader === -1) { context.logger.logDebug( - `Omitting column "${columnDefinition.name}" as the name was not found in the header`, + `Omitting column "${property.name}" as the name was not found in the header`, ); - continue; + return []; } const columnValuetype = context.wrapperFactories.ValueType.wrap( - columnDefinition.type, + property.valueType, ); assert(columnValuetype !== undefined); - columnEntries.push({ - sheetColumnIndex: indexOfMatchingHeader, - columnName: columnDefinition.name, - valueType: columnValuetype, - astNode: columnDefinition, - }); - } - - return columnEntries; + return [ + { + sheetColumnIndex: indexOfMatchingHeader, + columnName: property.name, + valueType: columnValuetype, + astNode: property, + }, + ]; + }); } } diff --git a/libs/language-server/src/grammar/expression.langium b/libs/language-server/src/grammar/expression.langium index 4e8bb5c6e..8b0accc13 100644 --- a/libs/language-server/src/grammar/expression.langium +++ b/libs/language-server/src/grammar/expression.langium @@ -88,6 +88,7 @@ Referencable: | BlockTypeProperty | TransformDefinition | TransformPortDefinition + | CustomValuetypeDefinition | ValueTypeProperty; NestedPropertyAccess: diff --git a/libs/language-server/src/lib/ast/expressions/evaluation-context.ts b/libs/language-server/src/lib/ast/expressions/evaluation-context.ts index bdadd9efc..819b6e692 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluation-context.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluation-context.ts @@ -21,6 +21,7 @@ import { isTransformPortDefinition, isValueKeywordLiteral, isValueTypeProperty, + isValuetypeDefinition, } from '../generated/ast'; import { type ValueTypeProvider } from '../wrappers'; import { type ValueType } from '../wrappers/value-type/value-type'; @@ -138,6 +139,9 @@ export class EvaluationContext { ) ); } + if (isValuetypeDefinition(dereferenced)) { + return dereferenced; + } assertUnreachable(dereferenced); } diff --git a/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts b/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts index d2e93d2a7..255b9196c 100644 --- a/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts +++ b/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts @@ -13,11 +13,13 @@ import { type ConstraintDefinition, type TransformDefinition, type ValuetypeAssignment, + type ValuetypeDefinition, isBlockTypeProperty, isCellRangeLiteral, isConstraintDefinition, isTransformDefinition, isValuetypeAssignment, + isValuetypeDefinition, } from '../generated/ast'; import { type WrapperFactoryProvider } from '../wrappers'; @@ -81,6 +83,7 @@ export type AtomicInternalValidValueRepresentation = | CellRangeLiteral | ConstraintDefinition | ValuetypeAssignment + | ValuetypeDefinition | BlockTypeProperty | TransformDefinition; @@ -153,6 +156,9 @@ export function internalValueToString( if (isValuetypeAssignment(valueRepresentation)) { return valueRepresentation.name; } + if (isValuetypeDefinition(valueRepresentation)) { + return valueRepresentation.name; + } if (isTransformDefinition(valueRepresentation)) { return valueRepresentation.name; } diff --git a/libs/language-server/src/lib/ast/expressions/type-inference.ts b/libs/language-server/src/lib/ast/expressions/type-inference.ts index 422892bdc..a4b42fcbd 100644 --- a/libs/language-server/src/lib/ast/expressions/type-inference.ts +++ b/libs/language-server/src/lib/ast/expressions/type-inference.ts @@ -38,6 +38,7 @@ import { isValueLiteral, isValueTypeProperty, isValuetypeAssignmentLiteral, + isValuetypeDefinition, } from '../generated/ast'; import { getNextAstNodeContainer } from '../model-util'; import { @@ -342,6 +343,9 @@ function inferTypeFromReferenceLiteral( if (isTransformDefinition(referenced)) { return valueTypeProvider.Primitives.Transform; } + if (isValuetypeDefinition(referenced)) { + return valueTypeProvider.Primitives.ValuetypeDefinition; + } if ( isTransformPortDefinition(referenced) || isBlockTypeProperty(referenced) || diff --git a/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts b/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts index 4e6614eca..e6279f9a6 100644 --- a/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts +++ b/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts @@ -120,6 +120,10 @@ export class BlockTypeWrapper extends TypedObjectWrapper { + if (property.valueType.reference.ref === undefined) { + console.log(property.valueType.reference.error); + process.exit(1); + } return property.valueType.reference.ref === undefined; }) ) { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts index c0158b62b..cc74c3f40 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts @@ -20,6 +20,7 @@ export { type IntegerValuetype } from './integer-value-type'; export { type RegexValuetype } from './regex-value-type'; export { type TextValuetype } from './text-value-type'; export { type ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; +export { type ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; export { type TransformValuetype } from './transform-value-type'; export { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts index a4d90f2ab..3563b78a0 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts @@ -17,6 +17,7 @@ import { RegexValuetype } from './regex-value-type'; import { TextValuetype } from './text-value-type'; import { TransformValuetype } from './transform-value-type'; import { ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; +import { ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; /** * Should be created as singleton due to the equality comparison of primitive value types. @@ -43,6 +44,7 @@ export class PrimitiveValueTypeProvider { CellRange = new CellRangeValuetype(); Constraint = new ConstraintValuetype(); ValuetypeAssignment = new ValuetypeAssignmentValuetype(); + ValuetypeDefinition = new ValuetypeDefinitionValuetype(); Transform = new TransformValuetype(); @@ -56,6 +58,7 @@ export class PrimitiveValueTypeProvider { this.CellRange, this.Constraint, this.ValuetypeAssignment, + this.ValuetypeDefinition, this.Transform, ]; } diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts new file mode 100644 index 000000000..17a09711e --- /dev/null +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type InternalValidValueRepresentation } from '../../../expressions/internal-value-representation'; +import { + type ValuetypeDefinition as AstValuetypeDefinition, + isValuetypeDefinition as isAstValuetypeDefinition, +} from '../../../generated/ast'; +import { type ValueTypeVisitor } from '../value-type'; + +import { PrimitiveValueType } from './primitive-value-type'; + +export class ValuetypeDefinitionValuetype extends PrimitiveValueType { + acceptVisitor(visitor: ValueTypeVisitor): R { + return visitor.visitValuetypeDefinition(this); + } + + override isAllowedAsRuntimeParameter(): boolean { + return false; + } + + override getName(): 'ValuetypeDefinition' { + return 'ValuetypeDefinition'; + } + + override isInternalValidValueRepresentation( + operandValue: InternalValidValueRepresentation, + ): operandValue is AstValuetypeDefinition { + return isAstValuetypeDefinition(operandValue); + } +} diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts index d2b1015cd..dab5cf62a 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts @@ -9,6 +9,7 @@ import { import { type AtomicValueType } from './atomic-value-type'; import { + type ValuetypeDefinitionValuetype, type BooleanValuetype, type CellRangeValuetype, type CollectionValueType, @@ -85,6 +86,7 @@ export abstract class ValueTypeVisitor { abstract visitRegex(valueType: RegexValuetype): R; abstract visitConstraint(valueType: ConstraintValuetype): R; abstract visitValuetypeAssignment(valueType: ValuetypeAssignmentValuetype): R; + abstract visitValuetypeDefinition(valueType: ValuetypeDefinitionValuetype): R; abstract visitCollection( valueType: CollectionValueType | EmptyCollectionValueType, ): R; diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts index 68c92736e..266f72ec6 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts @@ -13,7 +13,6 @@ import { isRowWrapper, } from '../../../ast'; import { type JayveeValidationProps } from '../../validation-registry'; -import { checkUniqueNames } from '../../validation-util'; export function checkBlockTypeSpecificProperties( property: PropertyAssignment, @@ -308,15 +307,16 @@ function checkTableInterpreterProperty( property, props.evaluationContext, props.wrapperFactories, - props.valueTypeProvider.createCollectionValueTypeOf( - props.valueTypeProvider.Primitives.ValuetypeAssignment, - ), + props.valueTypeProvider.Primitives.ValuetypeDefinition, ); if (ERROR_TYPEGUARD(valueTypeAssignments)) { + props.validationContext.accept( + 'error', + "The property 'columns' must have a value that is a reference to a value type definition", + { node: property }, + ); return; } - - checkUniqueNames(valueTypeAssignments, props.validationContext, 'column'); } } diff --git a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv index 0d60cd70f..2eadea048 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv @@ -3,26 +3,33 @@ // SPDX-License-Identifier: AGPL-3.0-only /** -* Interprets a `Sheet` as a `Table`. In case a header row is present in the sheet, its names can be matched with the provided column names. Otherwise, the provided column names are assigned in order. +* Interprets a `Sheet` as a `Table`. In case a header row is present in the +* sheet, its names can be matched with the provided column names. Otherwise, the +* provided column names are assigned in order. +* +* @example Interprets a `Sheet` about cars with a topmost header row and +* interprets it as a `Table` by assigning a value type property to each column. +* The column names are matched to the header, so the order of the type +* assignments does not matter. +* valuetype Car { +* property name oftype text; +* property mpg oftype decimal; +* property cyl oftype integer; +* } * -* @example Interprets a `Sheet` about cars with a topmost header row and interprets it as a `Table` by assigning a primitive value type to each column. The column names are matched to the header, so the order of the type assignments does not matter. * block CarsTableInterpreter oftype TableInterpreter { * header: true; -* columns: [ -* "name" oftype text, -* "mpg" oftype decimal, -* "cyl" oftype integer, -* ]; +* columns: Car; * } * -* @example Interprets a `Sheet` about cars without a topmost header row and interprets it as a `Table` by sequentially assigning a name and a primitive value type to each column of the sheet. Note that the order of columns matters here. The first column (column `A`) will be named "name", the second column (column `B`) will be named "mpg" etc. +* @example Interprets a `Sheet` about cars without a topmost header row and +* interprets it as a `Table` by sequentially assigning a name and a value type +* property to each column of the sheet. Note that the order of columns matters +* here. The first column (column `A`) will be named "name", the second column +* (column `B`) will be named "mpg" etc. * block CarsTableInterpreter oftype TableInterpreter { * header: false; -* columns: [ -* "name" oftype text, -* "mpg" oftype decimal, -* "cyl" oftype integer, -* ]; +* columns: Car; * } */ publish builtin blocktype TableInterpreter { @@ -35,9 +42,11 @@ publish builtin blocktype TableInterpreter { property header oftype boolean: true; /** - * Collection of value type assignments. Uses column names (potentially matched with the header or by sequence depending on the `header` property) to assign a primitive value type to each column. + * Collection of value type assignments. Assigns a value type property to each + * column (potentially matched with the header or by sequence depending on the + * `header` property). */ - property columns oftype Collection; + property columns oftype ValuetypeDefinition; /** * Whether to ignore whitespace before values. Does not apply to `text` cells From 5bd64114077c1fa113e7d4bede335f564a305345 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 26 Nov 2025 15:50:53 +0100 Subject: [PATCH 02/35] test: adapt tests --- .../property-assignment.spec.ts | 22 ------------------- .../invalid-non-unique-column-names.jv | 20 ----------------- .../table-interpreter/valid-correct-table.jv | 8 +++---- 3 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts index 9302bb750..99b43b2a9 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts @@ -326,28 +326,6 @@ describe('Validation of block type specific properties', () => { }); describe('TableInterpreter block type', () => { - it('should diagnose error on non unique column names', async () => { - const text = readJvTestAsset( - 'property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv', - ); - - await parseAndValidatePropertyAssignment(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(2); - expect(validationAcceptorMock).toHaveBeenNthCalledWith( - 1, - 'error', - 'The column name "name" needs to be unique.', - expect.any(Object), - ); - expect(validationAcceptorMock).toHaveBeenNthCalledWith( - 2, - 'error', - 'The column name "name" needs to be unique.', - expect.any(Object), - ); - }); - it('should diagnose no error', async () => { const text = readJvTestAsset( 'property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv', diff --git a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv b/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv deleted file mode 100644 index 6cea9044f..000000000 --- a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline Pipeline { - block Test oftype TableInterpreter { - columns: [ - "name" oftype text, - "name" oftype integer, - ]; - } - - block TestExtractor oftype TestSheetExtractor { - } - - block TestLoader oftype TestTableLoader { - } - - TestExtractor -> Test -> TestLoader; -} diff --git a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv b/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv index 06cc9f9d9..37dfe7bec 100644 --- a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv +++ b/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv @@ -3,11 +3,11 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline Pipeline { + valuetype NaturalNumber { + property number oftype integer; + } block Test oftype TableInterpreter { - columns: [ - "name" oftype text, - "version" oftype integer, - ]; + columns: NaturalNumber; } block TestExtractor oftype TestSheetExtractor { From bf02f71586c9c8a07541cfd03f3795bcff907455 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 26 Nov 2025 16:26:33 +0100 Subject: [PATCH 03/35] test: adapt execution tests --- .../lib/table-interpreter-executor.spec.ts | 42 ++++++------------- .../invalid-empty-columns-with-header.jv | 22 ++++++++++ ...> invalid-empty-columns-without-header.jv} | 6 ++- .../valid-empty-columns-with-header.jv | 5 ++- .../valid-with-capitalized-header.jv | 12 +++--- .../valid-with-header.jv | 12 +++--- .../valid-without-header-without-trim.jv | 12 +++--- .../valid-without-header.jv | 12 +++--- .../valid-wrong-value-type-with-header.jv | 12 +++--- .../valid-wrong-value-type-without-header.jv | 12 +++--- 10 files changed, 86 insertions(+), 61 deletions(-) create mode 100644 libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv rename libs/extensions/tabular/exec/test/assets/table-interpreter-executor/{valid-empty-columns-without-header.jv => invalid-empty-columns-without-header.jv} (85%) diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts index 84939a6b2..361e40577 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts @@ -116,21 +116,13 @@ describe('Validation of TableInterpreterExecutor', () => { } }); - it('should diagnose empty table on empty column parameter', async () => { - const text = readJvTestAsset('valid-empty-columns-with-header.jv'); + it('should diagnose error on empty column value type', async () => { + const text = readJvTestAsset('invalid-empty-columns-with-header.jv'); - const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(0); - expect(result.right.getNumberOfRows()).toEqual(0); - } + const document = await parse(text, { validation: true }); + expect( + document.parseResult.parserErrors.map((error) => error.message), + ).toContainEqual("Expecting token of type 'property' but found `}`."); }); it('should diagnose empty table on wrong header case', async () => { @@ -262,21 +254,13 @@ describe('Validation of TableInterpreterExecutor', () => { } }); - it('should diagnose empty table on empty column parameter', async () => { - const text = readJvTestAsset('valid-empty-columns-without-header.jv'); + it('should diagnose error on empty column parameter', async () => { + const text = readJvTestAsset('invalid-empty-columns-without-header.jv'); - const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(0); - expect(result.right.getNumberOfRows()).toEqual(0); - } + const document = await parse(text, { validation: true }); + expect( + document.parseResult.parserErrors.map((error) => error.message), + ).toContainEqual("Expecting token of type 'property' but found `}`."); }); it('should diagnose error on empty sheet', async () => { @@ -291,7 +275,7 @@ describe('Validation of TableInterpreterExecutor', () => { expect(R.isOk(result)).toEqual(false); if (R.isErr(result)) { expect(result.left.message).toEqual( - 'There are 3 column definitions but the input sheet only has 0 columns', + 'The value type TestValueType has 3 properties, but the input sheet only has 0 columns', ); } }); diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv new file mode 100644 index 000000000..5810d24db --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + valuetype TestValueType { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: TestValueType; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-without-header.jv similarity index 85% rename from libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv rename to libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-without-header.jv index b6e423edc..b6c088d00 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-without-header.jv @@ -7,9 +7,13 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + + valuetype TestValueType { + } + block TestBlock oftype TableInterpreter { header: false; - columns: []; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv index 7e46a44c6..5810d24db 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv @@ -7,9 +7,12 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + } + block TestBlock oftype TableInterpreter { header: true; - columns: []; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv index 3d9929495..19d138c08 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv @@ -7,13 +7,15 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property Index oftype integer; + property Name oftype text; + property Flag oftype boolean; + } + block TestBlock oftype TableInterpreter { header: true; - columns: [ - "Index" oftype integer, - "Name" oftype text, - "Flag" oftype boolean - ]; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv index cd368186b..0f37661ea 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv @@ -7,13 +7,15 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype boolean; + } + block TestBlock oftype TableInterpreter { header: true; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype boolean - ]; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv index ec1fc6b7e..a6cc632d7 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv @@ -6,13 +6,15 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype boolean; + } + block TestBlock oftype TableInterpreter { header: false; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype boolean - ]; + columns: TestValueType; skipLeadingWhitespace: false; skipTrailingWhitespace: false; } diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv index 6fa8f5c1d..f0a969bcc 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv @@ -7,13 +7,15 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype boolean; + } + block TestBlock oftype TableInterpreter { header: false; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype boolean - ]; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv index f8ec03446..9dfb35257 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv @@ -7,13 +7,15 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype integer; + } + block TestBlock oftype TableInterpreter { header: true; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype integer - ]; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv index ebc269171..fb5cd57a3 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv @@ -7,13 +7,15 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype integer; + } + block TestBlock oftype TableInterpreter { header: false; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype integer - ]; + columns: TestValueType; } block TestLoader oftype TestTableLoader { From cacafa5a6e26cb4ca4fedad0e3c398e776e1d9c9 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Thu, 27 Nov 2025 10:08:25 +0100 Subject: [PATCH 04/35] tests: adapt more tests --- example/cars.jv | 91 ++++++++++--------- .../test/assets/graph/composite-block.jv | 8 +- .../test/assets/graph/two-pipelines.jv | 30 +++--- .../valid-builtin-and-composite-blocks.jv | 8 +- 4 files changed, 73 insertions(+), 64 deletions(-) diff --git a/example/cars.jv b/example/cars.jv index 644cea639..68122ceb8 100644 --- a/example/cars.jv +++ b/example/cars.jv @@ -7,14 +7,12 @@ // - Understand the core concepts pipeline, block, and pipe // - Understand the general structure of a pipeline -// 1. This Jayvee model describes a pipeline -// from a CSV file in the web -// to a SQLite file sink. +// 1. This Jayvee model describes a pipeline from a CSV file in the web to a +// SQLite file sink. pipeline CarsPipeline { - // 2. We describe the structure of the pipeline, - // usually at the top of the pipeline. - // by connecting blocks via pipes. + // 2. We describe the structure of the pipeline, usually at the top of the + // pipeline, by connecting blocks via pipes. // 3. Syntax of a pipe // connecting the block CarsExtractor @@ -22,8 +20,8 @@ pipeline CarsPipeline { CarsExtractor -> CarsTextFileInterpreter; - // 4. The output of the preceding block is hereby used - // as input for the succeeding block. + // 4. The output of the preceding block is hereby used as input for the + // succeeding block. // 5. Pipes can be further chained, // leading to an overview of the pipeline. @@ -34,12 +32,12 @@ pipeline CarsPipeline { -> CarsLoader; - // 6. Below the pipes, we usually define the blocks - // that are connected by the pipes. + // 6. Below the pipes, we usually define the blocks that are connected by the + // pipes. // 7. Blocks instantiate a block type by using the oftype keyword. - // The block type defines the available properties that the block - // can use to specify the intended behavior of the block + // The block type defines the available properties that can be used to specify + // the intended behavior of the block. block CarsExtractor oftype HttpExtractor { // 8. Properties are assigned to concrete values. @@ -47,12 +45,13 @@ pipeline CarsPipeline { url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv"; } - // 9. The HttpExtractor requires no input and produces a binary file as output. - // This file has to be interpreted, e.g., as text file. + // 9. The HttpExtractor requires no input and produces a binary file as + // output. This file has to be interpreted, e.g., as text file. block CarsTextFileInterpreter oftype TextFileInterpreter { } // 10. Next, we interpret the text file as sheet. - // A sheet only contains text cells and is useful for manipulating the shape of data before assigning more strict value types to cells. + // A sheet only contains text cells and is useful for manipulating the shape + // of data before assigning more strict value types to cells. block CarsCSVInterpreter oftype CSVInterpreter { enclosing: '"'; } @@ -60,48 +59,52 @@ pipeline CarsPipeline { // 11. We can write into cells of a sheet using the CellWriter block type. block NameHeaderWriter oftype CellWriter { // 12. We utilize a syntax similar to spreadsheet programs. - // Cell ranges can be described using the keywords "cell", "row", "column", or "range" that indicate which - // cells are selected for the write action. + // Cell ranges can be described using the keywords "cell", "row", "column", + // or "range" that indicate which cells are selected for the write action. at: cell A1; - // 13. For each cell we selected with the "at" property above, - // we can specify what value shall be written into the cell. + // 13. For each cell we selected with the "at" property above, we can + // specify what value shall be written into the cell. write: [ "name" ]; } - // 14. As a next step, we interpret the sheet as a table by adding structure. - // We define a value type per column that specifies the data type of the column. - // Rows that include values that are not valid according to the their value types are dropped automatically. + // 14. Next, we define the schema of our table. + // For this, we use a value type where each property corresponds to a column + // in the table. The column will have the same valuetype as the property. + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + + // 15. As a next step, we interpret the sheet as a table, using the valuetype + // defined above. Rows that include values that are not valid according to the + // their value types are dropped automatically. block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer - ]; + columns: Car; } - // 15. As a last step, we load the table into a sink, - // here into a sqlite file. - // The structural information of the table is used - // to generate the correct table. + // 16. As a last step, we load the table into a sink, here into a sqlite file. + // The structural information of the table is used to generate the correct + // table. block CarsLoader oftype SQLiteLoader { table: "Cars"; file: "./cars.sqlite"; } - // 16. Congratulations! - // You can now use the sink for your data analysis, app, - // or whatever you want to do with the cleaned data. -} \ No newline at end of file + // 17. Congratulations! + // You can now use the sink for your data analysis, app, or whatever you want + // to do with the cleaned data. +} diff --git a/libs/interpreter-lib/test/assets/graph/composite-block.jv b/libs/interpreter-lib/test/assets/graph/composite-block.jv index d1727c192..4e16bd891 100644 --- a/libs/interpreter-lib/test/assets/graph/composite-block.jv +++ b/libs/interpreter-lib/test/assets/graph/composite-block.jv @@ -15,11 +15,13 @@ pipeline CarsPipeline { enclosing: '"'; } + valuetype CarName { + property name oftype text; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - ]; + columns: CarName; } transform copy { diff --git a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv index 24844e45e..480e15812 100644 --- a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv +++ b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv @@ -31,22 +31,24 @@ pipeline CarsPipeline { ]; } + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer - ]; + columns: Car; } block CarsLoader oftype SQLiteLoader { diff --git a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv index 3595e7b4e..1538a399d 100644 --- a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv +++ b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv @@ -28,11 +28,13 @@ pipeline CarsPipeline { ]; } + valuetype CarName { + property name oftype text; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - ]; + columns: CarName; } transform copy { From 1d32525301926bd754ffbfb4d7bce7d54f5eb918 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Fri, 2 Jan 2026 11:28:34 +0100 Subject: [PATCH 05/35] fix: correct dfs for type cycle search --- .../lib/ast/wrappers/value-type/abstract-value-type.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts index 21a1e9380..87316910c 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts @@ -49,13 +49,15 @@ export abstract class AbstractValueType< if (cycleDetected) { return -1; } - visited.push(this); const idx = this.doGetContainedTypes().findIndex( (containedType) => - containedType.getIndexOfFirstPropertyInATypeCycle(visited) !== - undefined, + containedType.getIndexOfFirstPropertyInATypeCycle([ + ...visited, + this, + ]) !== undefined, ); + return idx !== -1 ? idx : undefined; } } From b754e0970097d9998f7fd6a093e1ba282dc7e82e Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Fri, 2 Jan 2026 14:00:34 +0100 Subject: [PATCH 06/35] docs: update description of the columns property --- .../src/stdlib/builtin-block-types/TableInterpreter.jv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv index 2eadea048..7c5648eb2 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv @@ -42,9 +42,9 @@ publish builtin blocktype TableInterpreter { property header oftype boolean: true; /** - * Collection of value type assignments. Assigns a value type property to each - * column (potentially matched with the header or by sequence depending on the - * `header` property). + * Reference to a valuetype that defines the table schema. + * Each of the valuetype's properties symbolizes a column, the property's name + * is the column's name and the property's valuetype is the column's valuetype. */ property columns oftype ValuetypeDefinition; From 57a4bb1cd17ba9d83e25f5f9b4dff7f03818becd Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Fri, 2 Jan 2026 15:36:29 +0100 Subject: [PATCH 07/35] feat: constraints on tables via value types --- .../lib/constraints/constraint-executor.ts | 44 +++---- .../src/lib/types/io-types/table.spec.ts | 4 +- .../execution/src/lib/types/io-types/table.ts | 123 +++++++++++++----- .../value-representation-validity.ts | 34 +++-- .../test/utils/test-infrastructure-util.ts | 6 +- .../src/lib/table-interpreter-executor.ts | 32 ++++- .../src/lib/table-transformer-executor.ts | 8 ++ libs/extensions/tabular/exec/test/util.ts | 18 ++- 8 files changed, 184 insertions(+), 85 deletions(-) diff --git a/libs/execution/src/lib/constraints/constraint-executor.ts b/libs/execution/src/lib/constraints/constraint-executor.ts index b5a3f9aad..e0b914b5f 100644 --- a/libs/execution/src/lib/constraints/constraint-executor.ts +++ b/libs/execution/src/lib/constraints/constraint-executor.ts @@ -11,34 +11,43 @@ import { ERROR_TYPEGUARD, type InternalErrorValueRepresentation, type InternalValidValueRepresentation, - type ValueTypeProperty, type ValueTypeConstraintInlineDefinition, evaluateExpression, + isConstraintDefinition, } from '@jvalue/jayvee-language-server'; import { type ExecutionContext } from '../execution-context'; -export class ConstraintExecutor< - T extends ConstraintDefinition | ValueTypeConstraintInlineDefinition, -> implements AstNodeWrapper +export class ConstraintExecutor + implements + AstNodeWrapper { - constructor(public readonly astNode: T) {} + constructor( + public readonly astNode: + | ConstraintDefinition + | ValueTypeConstraintInlineDefinition, + ) {} + + get name(): string { + return this.astNode.name; + } isValid( - value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + values: Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, context: ExecutionContext, - properties: T extends ValueTypeConstraintInlineDefinition - ? ValueTypeProperty[] - : void, ): boolean { const expression = this.astNode.expression; - if (properties === undefined) { + if (isConstraintDefinition(this.astNode)) { + const value = values.get('value'); + assert(value !== undefined); context.evaluationContext.setValueForValueKeyword(value); } else { - const assignmentForTypeSystem: ValueTypeProperty[] = properties; - for (const property of assignmentForTypeSystem) { - context.evaluationContext.setValueForReference(property.name, value); + for (const [name, value] of values) { + context.evaluationContext.setValueForReference(name, value); } } @@ -57,15 +66,6 @@ export class ConstraintExecutor< ), ); - if (properties === undefined) { - context.evaluationContext.deleteValueForValueKeyword(); - } else { - const assignment_for_type_system: ValueTypeProperty[] = properties; - for (const property of assignment_for_type_system) { - context.evaluationContext.deleteValueForReference(property.name); - } - } - return result; } } diff --git a/libs/execution/src/lib/types/io-types/table.spec.ts b/libs/execution/src/lib/types/io-types/table.spec.ts index 0575e26c5..2b9d1669c 100644 --- a/libs/execution/src/lib/types/io-types/table.spec.ts +++ b/libs/execution/src/lib/types/io-types/table.spec.ts @@ -4,14 +4,14 @@ import { ValueTypeProvider } from '@jvalue/jayvee-language-server'; -import { Table } from './table'; +import { Table, type TableColumn } from './table'; describe('Table', () => { let table: Table; let valueTypeProvider: ValueTypeProvider; beforeEach(() => { - table = new Table(); + table = new Table(0, new Map(), []); valueTypeProvider = new ValueTypeProvider(); }); diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index 3c1deae53..870ffad3b 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -7,6 +7,7 @@ import { strict as assert } from 'assert'; import { ERROR_TYPEGUARD, + InvalidValue, IOType, type InternalErrorValueRepresentation, type InternalValidValueRepresentation, @@ -22,6 +23,9 @@ import { type IOTypeImplementation, type IoTypeVisitor, } from './io-type-implementation'; +import { ConstraintExecutor } from '../../constraints'; +import { type ExecutionContext } from '../../execution-context'; +import { assertUnreachable } from 'langium'; export interface TableColumn< T extends InternalValidValueRepresentation = InternalValidValueRepresentation, @@ -30,7 +34,7 @@ export interface TableColumn< valueType: ValueType; } -export type TableRow = Record< +export type TableRow = Map< string, InternalValidValueRepresentation | InternalErrorValueRepresentation >; @@ -42,12 +46,14 @@ export type TableRow = Record< export class Table implements IOTypeImplementation { public readonly ioType = IOType.TABLE; - private numberOfRows = 0; - - private columns = new Map(); - - public constructor(numberOfRows = 0) { - this.numberOfRows = numberOfRows; + public constructor( + private numberOfRows: number, + private columns: Map, + private constraints: ConstraintExecutor[], + ) { + assert(this.numberOfRows !== undefined); + assert(this.columns !== undefined); + assert(this.constraints !== undefined); } addColumn(name: string, column: TableColumn): void { @@ -60,32 +66,44 @@ export class Table implements IOTypeImplementation { * NOTE: This method will only add the row if the table has at least one column! * @param row data of this row for each column */ - addRow(row: TableRow): void { - const rowLength = Object.keys(row).length; + addRow( + row: Record< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, + ): void; + addRow(row: TableRow): void; + addRow( + row: + | TableRow + | Record< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, + ): void { + const rowLength = row instanceof Map ? row.size : Object.keys(row).length; assert( rowLength === this.columns.size, `Added row has the wrong dimension (expected: ${this.columns.size}, actual: ${rowLength})`, ); - if (rowLength === 0) { - return; + + if (rowLength > 0) { + this.numberOfRows++; } - assert( - Object.keys(row).every((x) => this.hasColumn(x)), - 'Added row does not fit the columns in the table', - ); - Object.entries(row).forEach(([columnName, value]) => { + const rowValues = + row instanceof Map ? [...row.entries()] : Object.entries(row); + + for (const [columnName, cellValue] of rowValues) { const column = this.columns.get(columnName); - assert(column !== undefined); + assert(column !== undefined, 'All added rows fit columns in the table'); assert( - ERROR_TYPEGUARD(value) || - column.valueType.isInternalValidValueRepresentation(value), + ERROR_TYPEGUARD(cellValue) || + column.valueType.isInternalValidValueRepresentation(cellValue), ); - column.values.push(value); - }); - - this.numberOfRows++; + column.values.push(cellValue); + } } getNumberOfRows(): number { @@ -108,12 +126,7 @@ export class Table implements IOTypeImplementation { return this.columns.get(name); } - getRow( - rowId: number, - ): Map< - string, - InternalValidValueRepresentation | InternalErrorValueRepresentation - > { + getRow(rowId: number): TableRow { const numberOfRows = this.getNumberOfRows(); if (rowId >= numberOfRows) { throw new Error( @@ -133,6 +146,45 @@ export class Table implements IOTypeImplementation { return row; } + private setRowInvalid(rowIdx: number, message: string): void { + for (const [columnName, column] of this.columns.entries()) { + column.values[rowIdx] = new InvalidValue(message); + this.columns.set(columnName, column); + } + } + + findUnfullfilledRows( + onInvalidRow: ( + constraint: ConstraintExecutor, + rowIndex: number, + row: TableRow, + ) => 'markInvalid' | 'ignore', + executionContext: ExecutionContext, + ): void { + for (let rowIdx = 0; rowIdx < this.numberOfRows; rowIdx++) { + const row = this.getRow(rowIdx); + + for (const constraint of this.constraints) { + if (constraint.isValid(row, executionContext)) { + continue; + } + const invalidHandling = onInvalidRow(constraint, rowIdx, row); + switch (invalidHandling) { + case 'markInvalid': { + this.setRowInvalid(rowIdx, `Invalid constraint ${constraint.name}`); + break; + } + case 'ignore': { + break; + } + default: { + assertUnreachable(invalidHandling); + } + } + } + } + } + static generateDropTableStatement(tableName: string): string { return `DROP TABLE IF EXISTS "${tableName}";`; } @@ -186,16 +238,19 @@ export class Table implements IOTypeImplementation { } clone(): Table { - const cloned = new Table(); - cloned.numberOfRows = this.numberOfRows; - [...this.columns.entries()].forEach(([columnName, column]) => { - cloned.addColumn(columnName, { + const copiedColumns = new Map(); + [...this.columns.entries()].map(([columnName, column]) => { + copiedColumns.set(columnName, { values: cloneInternalValue(column.values), valueType: column.valueType, }); }); - return cloned; + const copiedConstraints = this.constraints.map( + (constraint) => new ConstraintExecutor(constraint.astNode), + ); + + return new Table(this.numberOfRows, copiedColumns, copiedConstraints); } acceptVisitor(visitor: IoTypeVisitor): R { diff --git a/libs/execution/src/lib/types/value-types/value-representation-validity.ts b/libs/execution/src/lib/types/value-types/value-representation-validity.ts index eadda4006..c6b87d8d9 100644 --- a/libs/execution/src/lib/types/value-types/value-representation-validity.ts +++ b/libs/execution/src/lib/types/value-types/value-representation-validity.ts @@ -22,7 +22,7 @@ import { ValueTypeVisitor, type ValuetypeAssignmentValuetype, type ValuetypeDefinitionValuetype, - isConstraintDefinition, + type InternalErrorValueRepresentation, } from '@jvalue/jayvee-language-server'; import { ConstraintExecutor } from '../../constraints'; @@ -33,13 +33,27 @@ export function isValidValueRepresentation( valueType: ValueType, context: ExecutionContext, ): boolean { - const visitor = new ValueRepresentationValidityVisitor(value, context); + const values = new Map(); + values.set('value', value); + const visitor = new ValueRepresentationValidityVisitor(values, context); return valueType.acceptVisitor(visitor); } +export function allValueRepresentationsValid( + values: Map, + atomicValueType: AtomicValueType, + context: ExecutionContext, +): boolean { + const visitor = new ValueRepresentationValidityVisitor(values, context); + return atomicValueType.acceptVisitor(visitor); +} + class ValueRepresentationValidityVisitor extends ValueTypeVisitor { constructor( - private value: InternalValidValueRepresentation, + private values: Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, private context: ExecutionContext, ) { super(); @@ -58,13 +72,9 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { for (const constraint of valueType.getConstraints()) { this.context.enterNode(constraint); - const valueFulfilledConstraint = isConstraintDefinition(constraint) - ? new ConstraintExecutor(constraint).isValid(this.value, this.context) - : new ConstraintExecutor(constraint).isValid( - this.value, - this.context, - valueType.getProperties(), - ); + const valueFulfilledConstraint = new ConstraintExecutor( + constraint, + ).isValid(this.values, this.context); this.context.exitNode(constraint); @@ -125,6 +135,8 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { } private isValidForPrimitiveValuetype(valueType: PrimitiveValueType): boolean { - return valueType.isInternalValidValueRepresentation(this.value); + const value = this.values.get('value'); + assert(value !== undefined); + return valueType.isInternalValidValueRepresentation(value); } } diff --git a/libs/execution/test/utils/test-infrastructure-util.ts b/libs/execution/test/utils/test-infrastructure-util.ts index 9dd9febd8..729b300db 100644 --- a/libs/execution/test/utils/test-infrastructure-util.ts +++ b/libs/execution/test/utils/test-infrastructure-util.ts @@ -89,7 +89,7 @@ export function constructTable( columns: TableColumnDefinition[], numberOfRows: number, ): Table { - const table = new Table(numberOfRows); - columns.forEach((col) => table.addColumn(col.columnName, col.column)); - return table; + const columnMap = new Map(); + columns.forEach((col) => columnMap.set(col.columnName, col.column)); + return new Table(numberOfRows, columnMap, []); } diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index 5d7da20e1..ada4a24e9 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -129,6 +129,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< inputSheet, header, columnEntries, + columnsValueType, skipLeadingWhitespace, skipTrailingWhitespace, context, @@ -143,20 +144,25 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< sheet: Sheet, header: boolean, columnEntries: ColumnDefinitionEntry[], + columnsValueType: AtomicValueType, skipLeadingWhitespace: boolean, skipTrailingWhitespace: boolean, context: ExecutionContext, ): Table { - const table = new Table(); - - // add columns + const columns = new Map(); columnEntries.forEach((columnEntry) => { - table.addColumn(columnEntry.columnName, { + columns.set(columnEntry.columnName, { values: [], valueType: columnEntry.valueType, }); }); + const constraints = columnsValueType + .getConstraints() + .map((constraint) => new R.ConstraintExecutor(constraint)); + + const table = new Table(0, columns, constraints); + // add rows sheet.iterateRows((sheetRow, sheetRowIndex) => { if (header && sheetRowIndex === 0) { @@ -173,6 +179,15 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< ); table.addRow(tableRow); }); + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + table.findUnfullfilledRows((constraint, rowIdx, _row) => { + context.logger.logErr( + `Invalid constraint ${constraint.name} on row ${rowIdx}`, + ); + return 'markInvalid'; + }, context); + return table; } @@ -184,7 +199,10 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< skipTrailingWhitespace: boolean, context: ExecutionContext, ): R.TableRow { - const tableRow: R.TableRow = {}; + const tableRow: R.TableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); columnEntries.forEach((columnEntry) => { const valueType = columnEntry.valueType; const sheetColumnIndex = columnEntry.sheetColumnIndex; @@ -209,10 +227,10 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< ); } - tableRow[columnEntry.columnName] = parsedValue; + tableRow.set(columnEntry.columnName, parsedValue); }); - assert(Object.keys(tableRow).length === columnEntries.length); + assert(tableRow.size === columnEntries.length); return tableRow; } diff --git a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts index d272711c2..a8443e7a3 100644 --- a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts @@ -97,6 +97,14 @@ export class TableTransformerExecutor extends AbstractBlockExecutor< outputColumnName, ); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + outputTable.findUnfullfilledRows((constraint, rowIdx, _row) => { + context.logger.logErr( + `Invalid constraint ${constraint.name} on row ${rowIdx}`, + ); + return 'markInvalid'; + }, context); + return R.ok(outputTable); } diff --git a/libs/extensions/tabular/exec/test/util.ts b/libs/extensions/tabular/exec/test/util.ts index b0b65c306..ba2a1ea8c 100644 --- a/libs/extensions/tabular/exec/test/util.ts +++ b/libs/extensions/tabular/exec/test/util.ts @@ -7,6 +7,7 @@ import path from 'node:path'; import { Table, type TableRow, + type TableColumn, Workbook, parseValueToInternalRepresentation, } from '@jvalue/jayvee-execution'; @@ -71,15 +72,17 @@ export async function createTableFromLocalExcelFile( await workBookFromFile.xlsx.readFile(path.resolve(__dirname, fileName)); const workSheet = workBookFromFile.worksheets[0] as exceljs.Worksheet; - const table = new Table(); + const columns = new Map(); columnDefinitions.forEach((columnDefinition) => { - table.addColumn(columnDefinition.columnName, { + columns.set(columnDefinition.columnName, { values: [], valueType: columnDefinition.valueType, }); }); + const table = new Table(0, columns, []); + workSheet.eachRow((row) => { const tableRow = constructTableRow(row, columnDefinitions); table.addRow(tableRow); @@ -91,7 +94,10 @@ function constructTableRow( row: exceljs.Row, columnDefinitions: ReducedColumnDefinitionEntry[], ): TableRow { - const tableRow: TableRow = {}; + const tableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); row.eachCell( { includeEmpty: true }, @@ -102,9 +108,9 @@ function constructTableRow( const value = cell.text; const valueType = columnDefinition.valueType; - tableRow[columnDefinition.columnName] = parseAndValidatePrimitiveValue( - value, - valueType, + tableRow.set( + columnDefinition.columnName, + parseAndValidatePrimitiveValue(value, valueType), ); }, ); From 41468a743c631f92bb56d9542c80be5bbf85a7dc Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Sat, 3 Jan 2026 09:29:09 +0100 Subject: [PATCH 08/35] refactor: return undefined instead of throwing an exception from `getRow()` --- libs/execution/src/lib/debugging/debug-log-visitor.ts | 3 +++ libs/execution/src/lib/types/io-types/table.ts | 9 ++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/libs/execution/src/lib/debugging/debug-log-visitor.ts b/libs/execution/src/lib/debugging/debug-log-visitor.ts index 2b083c39e..396d035b9 100644 --- a/libs/execution/src/lib/debugging/debug-log-visitor.ts +++ b/libs/execution/src/lib/debugging/debug-log-visitor.ts @@ -20,6 +20,8 @@ import { type Table } from '../types/io-types/table'; import { findLineBounds } from '../util/string-util'; import { type DebugGranularity } from './debug-configuration'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; export class DebugLogVisitor implements IoTypeVisitor { private readonly PEEK_NUMBER_OF_WORKBOOKS = 5; @@ -57,6 +59,7 @@ export class DebugLogVisitor implements IoTypeVisitor { } const row = table.getRow(i); + assert(row !== undefined); const rowData = [...row.values()] .map((cell) => { if (ERROR_TYPEGUARD(cell)) { diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index 870ffad3b..a6aa15a16 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: AGPL-3.0-only // eslint-disable-next-line unicorn/prefer-node-protocol -import { strict as assert } from 'assert'; +import assert from 'assert'; import { ERROR_TYPEGUARD, @@ -126,12 +126,10 @@ export class Table implements IOTypeImplementation { return this.columns.get(name); } - getRow(rowId: number): TableRow { + getRow(rowId: number): TableRow | undefined { const numberOfRows = this.getNumberOfRows(); if (rowId >= numberOfRows) { - throw new Error( - `Trying to access table row ${rowId} (of ${numberOfRows} rows)`, - ); + return undefined; } const row = new Map< @@ -163,6 +161,7 @@ export class Table implements IOTypeImplementation { ): void { for (let rowIdx = 0; rowIdx < this.numberOfRows; rowIdx++) { const row = this.getRow(rowIdx); + assert(row !== undefined); for (const constraint of this.constraints) { if (constraint.isValid(row, executionContext)) { From 11e5e1c30771e6c5c2538c94334e38b2b0966130 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Sat, 3 Jan 2026 11:38:31 +0100 Subject: [PATCH 09/35] feat: table row creation syntax --- .../src/grammar/expression.langium | 7 +++++- .../ast/expressions/evaluate-expression.ts | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/libs/language-server/src/grammar/expression.langium b/libs/language-server/src/grammar/expression.langium index 8b0accc13..b20548a36 100644 --- a/libs/language-server/src/grammar/expression.langium +++ b/libs/language-server/src/grammar/expression.langium @@ -17,7 +17,6 @@ IfExpression infers Expression: ReplaceExpression ({infer TernaryExpression.first=current} operator='if' second=ReplaceExpression 'else' third=ReplaceExpression)*; -// The nesting of the following rules implies the precedence of the operators: ReplaceExpression infers Expression: BinaryExpression ({infer TernaryExpression.first=current} operator='replace' second=BinaryExpression 'with' third=BinaryExpression)*; @@ -53,6 +52,7 @@ ValueLiteral: | CellRangeLiteral | ValuetypeAssignmentLiteral | CollectionLiteral + | TableRowLiteral | ErrorLiteral; TextLiteral: @@ -71,6 +71,11 @@ RegexLiteral: CollectionLiteral: '[' (values+=(Expression) (',' values+=(Expression))*)? ','? ']'; +TableRowLiteral: + '{' (cells+=(TableCellLiteral) (',' cells+=(TableCellLiteral))*)? ','? '}'; +TableCellLiteral: + name=ID ':' expression=Expression; + ErrorLiteral: error= "invalid" | "missing"; diff --git a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts index 338ae12c3..2fc23a5f3 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts @@ -22,6 +22,7 @@ import { isFreeVariableLiteral, isRegexLiteral, isRuntimeParameterLiteral, + isTableRowLiteral, isTernaryExpression, isUnaryExpression, isValueLiteral, @@ -172,6 +173,28 @@ function evaluateValueLiteral( } return evaluatedCollection; } + if (isTableRowLiteral(expression)) { + const tableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); + + for (const cell of expression.cells) { + const cellValue = evaluateExpression( + cell.expression, + evaluationContext, + wrapperFactories, + validationContext, + strategy, + ); + if (ERROR_TYPEGUARD(cellValue)) { + return cellValue; + } + + tableRow.set(cell.name, cellValue); + } + return tableRow; + } if (isCellRangeLiteral(expression)) { if (!wrapperFactories.CellRange.canWrap(expression)) { return new InvalidValue( From 0e616bbf6552ea937f88048bd58913daa4de3956 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Sat, 3 Jan 2026 11:39:53 +0100 Subject: [PATCH 10/35] feat: internal representation for table rows --- .../internal-value-representation.ts | 59 +++++++++++++++---- .../src/lib/ast/expressions/typeguards.ts | 13 ++++ 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts b/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts index 255b9196c..bb13226ff 100644 --- a/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts +++ b/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts @@ -23,7 +23,11 @@ import { } from '../generated/ast'; import { type WrapperFactoryProvider } from '../wrappers'; -import { COLLECTION_TYPEGUARD, ERROR_TYPEGUARD } from './typeguards'; +import { + COLLECTION_TYPEGUARD, + ERROR_TYPEGUARD, + TABLEROW_TYPEGUARD, +} from './typeguards'; // INFO: `ErroneousValue` extends `Error` in order to make use of the `stack` // property. @@ -71,8 +75,14 @@ export class MissingValue extends ErroneousValue { export type InternalErrorValueRepresentation = InvalidValue | MissingValue; +export type TableRow = Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation +>; + export type InternalValidValueRepresentation = | AtomicInternalValidValueRepresentation + | TableRow | (InternalValidValueRepresentation | InternalErrorValueRepresentation)[]; export type AtomicInternalValidValueRepresentation = @@ -87,6 +97,20 @@ export type AtomicInternalValidValueRepresentation = | BlockTypeProperty | TransformDefinition; +function internalValueToStringRecursionHelper( + value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + wrapperFactories?: WrapperFactoryProvider, +): string { + if (isCellRangeLiteral(value)) { + assert(wrapperFactories !== undefined); + return internalValueToString(value, wrapperFactories); + } + if (ERROR_TYPEGUARD(value)) { + return value.name; + } + return internalValueToString(value); +} + export type InternalValidValueRepresentationTypeguard< T extends InternalValidValueRepresentation, > = (value: unknown) => value is T; @@ -109,21 +133,23 @@ export function internalValueToString( return ( '[ ' + valueRepresentation - .map((value) => { - if (isCellRangeLiteral(value)) { - assert(wrapperFactories !== undefined); - return internalValueToString(value, wrapperFactories); - } - if (ERROR_TYPEGUARD(value)) { - return value.name; - } - return internalValueToString(value); - }) + .map((value) => + internalValueToStringRecursionHelper(value, wrapperFactories), + ) .join(', ') + ' ]' ); } + if (valueRepresentation instanceof Map) { + return `{ ${[...valueRepresentation.entries()] + .map( + ([columnName, cellValue]) => + `${columnName} : ${internalValueToStringRecursionHelper(cellValue, wrapperFactories)}`, + ) + .join(', ')} }`; + } + if (typeof valueRepresentation === 'boolean') { return String(valueRepresentation); } @@ -175,6 +201,17 @@ export function cloneInternalValue< return valueRepresentation.map(cloneInternalValue) as T; } + if (TABLEROW_TYPEGUARD(valueRepresentation)) { + const clonedMap = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); + for (const [columnName, cellValue] of valueRepresentation.entries()) { + clonedMap.set(columnName, cellValue); + } + return clonedMap as T; + } + if (ERROR_TYPEGUARD(valueRepresentation)) { return valueRepresentation.clone() as T; } diff --git a/libs/language-server/src/lib/ast/expressions/typeguards.ts b/libs/language-server/src/lib/ast/expressions/typeguards.ts index 9a1598007..f3e59fa0a 100644 --- a/libs/language-server/src/lib/ast/expressions/typeguards.ts +++ b/libs/language-server/src/lib/ast/expressions/typeguards.ts @@ -16,6 +16,7 @@ import { } from '../generated/ast'; import { + type TableRow, type AtomicInternalValidValueRepresentation, type InternalErrorValueRepresentation, type InternalValidValueRepresentation, @@ -28,6 +29,7 @@ export const INTERNAL_VALID_VALUE_REPRESENTATION_TYPEGUARD: InternalValidValueRe InternalValidValueRepresentation > = (value): value is InternalValidValueRepresentation => ATOMIC_INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(value) || + TABLEROW_TYPEGUARD(value) || COLLECTION_TYPEGUARD(value); export const ATOMIC_INTERNAL_VALUE_REPRESENTATION_TYPEGUARD: InternalValidValueRepresentationTypeguard< @@ -76,6 +78,17 @@ export const TRANSFORMDEFINITION_TYPEGUARD: InternalValidValueRepresentationType TransformDefinition > = (value) => isTransformDefinition(value); +export const TABLEROW_TYPEGUARD: InternalValidValueRepresentationTypeguard< + TableRow +> = (value): value is TableRow => + value instanceof Map && + [...value.entries()].every( + ([key, value]) => + STRING_TYPEGUARD(key) && + (INTERNAL_VALID_VALUE_REPRESENTATION_TYPEGUARD(value) || + ERROR_TYPEGUARD(value)), + ); + export const COLLECTION_TYPEGUARD: InternalValidValueRepresentationTypeguard< (InternalValidValueRepresentation | InternalErrorValueRepresentation)[] > = (value) => From 5268475bdd2c55a8514375b324cd6ab53368f88c Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Sat, 3 Jan 2026 11:40:50 +0100 Subject: [PATCH 11/35] feat: value type for table rows --- .../internal-representation-parsing.ts | 4 ++ .../value-representation-validity.ts | 5 ++ .../visitors/sql-column-type-visitor.ts | 6 ++ .../sql-value-representation-visitor.ts | 8 +++ .../src/lib/ast/expressions/type-inference.ts | 32 ++++++++++ .../wrappers/value-type/primitive/index.ts | 3 +- .../primitive-value-type-provider.ts | 5 ++ .../primitive/table-row-value-type.ts | 58 +++++++++++++++++++ .../lib/ast/wrappers/value-type/value-type.ts | 2 + 9 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts diff --git a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts index 464da9f32..3d15112a9 100644 --- a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts +++ b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts @@ -124,6 +124,10 @@ class InternalRepresentationParserVisitor extends ValueTypeVisitor< return new InvalidValue(`Cannot parse collections into internal values`); } + override visitTableRow(): InvalidValue { + return new InvalidValue(`Cannot parse table rows into internal values`); + } + override visitConstraint(): InvalidValue { return new InvalidValue(`Cannot parse constraints into internal values`); } diff --git a/libs/execution/src/lib/types/value-types/value-representation-validity.ts b/libs/execution/src/lib/types/value-types/value-representation-validity.ts index c6b87d8d9..f918acf11 100644 --- a/libs/execution/src/lib/types/value-types/value-representation-validity.ts +++ b/libs/execution/src/lib/types/value-types/value-representation-validity.ts @@ -6,6 +6,7 @@ import { strict as assert } from 'assert'; import { + type TableRowValueType, type AtomicValueType, type BooleanValuetype, type CellRangeValuetype, @@ -130,6 +131,10 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { return this.isValidForPrimitiveValuetype(valueType); } + override visitTableRow(valueType: TableRowValueType): boolean { + return this.isValidForPrimitiveValuetype(valueType); + } + override visitTransform(valueType: TransformValuetype): boolean { return this.isValidForPrimitiveValuetype(valueType); } diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts index f47a92e3e..3c06bdd87 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts @@ -76,6 +76,12 @@ export class SQLColumnTypeVisitor extends ValueTypeVisitor { ); } + override visitTableRow(): string { + throw new Error( + 'No visit implementation given for table rows. Cannot be the type of a column.', + ); + } + override visitTransform(): string { throw new Error( 'No visit implementation given for transforms. Cannot be the type of a column.', diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts index 9b6bdba78..f7ad91bb9 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts @@ -130,6 +130,14 @@ export class SQLValueRepresentationVisitor extends ValueTypeVisitor< ); } + override visitTableRow(): ( + value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + ) => string { + throw new Error( + 'No visit implementation given for table rows. Cannot be the type of a column.', + ); + } + override visitTransform(): ( value: InternalValidValueRepresentation | InternalErrorValueRepresentation, ) => string { diff --git a/libs/language-server/src/lib/ast/expressions/type-inference.ts b/libs/language-server/src/lib/ast/expressions/type-inference.ts index a4b42fcbd..3e5d69ce3 100644 --- a/libs/language-server/src/lib/ast/expressions/type-inference.ts +++ b/libs/language-server/src/lib/ast/expressions/type-inference.ts @@ -29,6 +29,7 @@ import { isNumericLiteral, isReferenceLiteral, isRegexLiteral, + isTableRowLiteral, isTernaryExpression, isTextLiteral, isTransformDefinition, @@ -39,10 +40,12 @@ import { isValueTypeProperty, isValuetypeAssignmentLiteral, isValuetypeDefinition, + type TableRowLiteral, } from '../generated/ast'; import { getNextAstNodeContainer } from '../model-util'; import { isAtomicValueType, + type TableRowValueType, type ValueType, type ValueTypeProvider, type WrapperFactoryProvider, @@ -186,6 +189,13 @@ function inferTypeFromExpressionLiteral( valueTypeProvider, wrapperFactories, ); + } else if (isTableRowLiteral(expression)) { + return inferTableRowType( + expression, + validationContext, + valueTypeProvider, + wrapperFactories, + ); } else if (isErrorLiteral(expression)) { return undefined; } @@ -297,6 +307,28 @@ function inferCollectionElementTypes( return undefined; } +function inferTableRowType( + tableRow: TableRowLiteral, + validationContext: ValidationContext, + valueTypeProvider: ValueTypeProvider, + wrapperFactories: WrapperFactoryProvider, +): TableRowValueType | undefined { + const cellValueTypes = new Map(); + for (const cell of tableRow.cells) { + const cellValueType = inferExpressionType( + cell.expression, + validationContext, + valueTypeProvider, + wrapperFactories, + ); + if (cellValueType === undefined) { + return undefined; + } + cellValueTypes.set(cell.name, cellValueType); + } + return valueTypeProvider.createTableRowValueTypeOf(cellValueTypes); +} + function inferTypeFromValueKeyword( expression: ValueKeywordLiteral, validationContext: ValidationContext, diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts index cc74c3f40..6afc09a45 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts @@ -18,10 +18,11 @@ export { type ConstraintValuetype } from './constraint-value-type'; export { type DecimalValuetype } from './decimal-value-type'; export { type IntegerValuetype } from './integer-value-type'; export { type RegexValuetype } from './regex-value-type'; +export { type TableRowValueType } from './table-row-value-type'; export { type TextValuetype } from './text-value-type'; +export { type TransformValuetype } from './transform-value-type'; export { type ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; export { type ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; -export { type TransformValuetype } from './transform-value-type'; export { ValueTypeProvider, diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts index 3563b78a0..7b2e23d4c 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts @@ -18,6 +18,7 @@ import { TextValuetype } from './text-value-type'; import { TransformValuetype } from './transform-value-type'; import { ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; import { ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; +import { TableRowValueType } from './table-row-value-type'; /** * Should be created as singleton due to the equality comparison of primitive value types. @@ -32,6 +33,10 @@ export class ValueTypeProvider { ): CollectionValueType { return new CollectionValueType(input); } + + createTableRowValueTypeOf(input: Map): TableRowValueType { + return new TableRowValueType(input); + } } export class PrimitiveValueTypeProvider { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts new file mode 100644 index 000000000..9f6cb59ab --- /dev/null +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts @@ -0,0 +1,58 @@ +// SPDX-FileCopyrightText: 2026 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { + type TableRow, + type InternalValidValueRepresentation, +} from '../../../expressions/internal-value-representation'; +import { type ValueType, type ValueTypeVisitor } from '../value-type'; + +import { PrimitiveValueType } from './primitive-value-type'; +import { TABLEROW_TYPEGUARD } from '../../../expressions'; + +export class TableRowValueType extends PrimitiveValueType { + constructor(private schema: Map) { + super(); + } + + acceptVisitor(visitor: ValueTypeVisitor): R { + return visitor.visitTableRow(this); + } + + override isAllowedAsRuntimeParameter(): boolean { + return false; + } + + override getName(): 'TableRow' { + return 'TableRow'; + } + + private equalSchema(otherSchema: Map): boolean { + if (this.schema.size !== otherSchema.size) { + return false; + } + for (const [columnName, cellValue] of this.schema) { + const otherCellValue = otherSchema.get(columnName); + if (cellValue !== otherCellValue) { + return false; + } + if (otherCellValue === undefined && !otherSchema.has(columnName)) { + return false; + } + } + return true; + } + + override equals(target: ValueType): boolean { + return ( + target instanceof TableRowValueType && this.equalSchema(target.schema) + ); + } + + override isInternalValidValueRepresentation( + operandValue: InternalValidValueRepresentation, + ): operandValue is TableRow { + return TABLEROW_TYPEGUARD(operandValue); + } +} diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts index dab5cf62a..1e2743e86 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts @@ -13,6 +13,7 @@ import { type BooleanValuetype, type CellRangeValuetype, type CollectionValueType, + type TableRowValueType, type ConstraintValuetype, type DecimalValuetype, type EmptyCollectionValueType, @@ -90,6 +91,7 @@ export abstract class ValueTypeVisitor { abstract visitCollection( valueType: CollectionValueType | EmptyCollectionValueType, ): R; + abstract visitTableRow(valueType: TableRowValueType): R; abstract visitTransform(valueType: TransformValuetype): R; abstract visitAtomicValueType(valueType: AtomicValueType): R; From fc37f356c54c9ca310c93421d10d055a98e47e3a Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:00:27 +0100 Subject: [PATCH 12/35] feat: `cellInColumn` operator evaluator --- .../src/grammar/expression.langium | 3 +- .../lib/ast/expressions/evaluation-context.ts | 10 ++ .../cell-in-column-operator-evaluator.ts | 106 ++++++++++++++++++ .../lib/ast/expressions/operator-registry.ts | 4 + .../cell-in-column-operator-type-computer.ts | 56 +++++++++ 5 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts create mode 100644 libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts diff --git a/libs/language-server/src/grammar/expression.langium b/libs/language-server/src/grammar/expression.langium index b20548a36..aa490cf54 100644 --- a/libs/language-server/src/grammar/expression.langium +++ b/libs/language-server/src/grammar/expression.langium @@ -22,7 +22,8 @@ ReplaceExpression infers Expression: second=BinaryExpression 'with' third=BinaryExpression)*; infix BinaryExpression on PrimaryExpression: - 'pow' | 'root' // Higher precedence + 'cellInColumn' // Higher precedence + > 'pow' | 'root' > '*' | '/' | '%' > '+' | '-' > '<' | '<=' | '>' | '>=' diff --git a/libs/language-server/src/lib/ast/expressions/evaluation-context.ts b/libs/language-server/src/lib/ast/expressions/evaluation-context.ts index 819b6e692..72cacf414 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluation-context.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluation-context.ts @@ -46,6 +46,8 @@ export class EvaluationContext { | InternalValidValueRepresentation | InternalErrorValueRepresentation = NO_KEYWORD_ERROR; + private headerRow: string[] | undefined = undefined; + constructor( public readonly runtimeParameterProvider: RuntimeParameterProvider, public readonly operatorRegistry: OperatorEvaluatorRegistry, @@ -171,4 +173,12 @@ export class EvaluationContext { | InternalErrorValueRepresentation { return this.valueKeywordValue; } + + getColumnIndex(columnName: string): number | undefined { + const columnIdx = this.headerRow?.findIndex((cN) => cN === columnName); + return columnIdx === -1 ? undefined : columnIdx; + } + setHeaderRow(headerRow: string[]): void { + this.headerRow = headerRow; + } } diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts new file mode 100644 index 000000000..cd2612d6e --- /dev/null +++ b/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type BinaryExpression } from '../../generated/ast'; +import { type OperatorEvaluator } from '../operator-evaluator'; +import { type EvaluationContext } from '../evaluation-context'; +import { type WrapperFactoryProvider } from '../../wrappers'; +import { EvaluationStrategy } from '../evaluation-strategy'; +import { type ValidationContext } from '../../../validation'; +import { + InvalidValue, + MissingValue, + type InternalErrorValueRepresentation, +} from '../internal-value-representation'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; +import { evaluateExpression } from '../evaluate-expression'; +import { + COLLECTION_TYPEGUARD, + ERROR_TYPEGUARD, + INVALID_TYPEGUARD, + MISSING_TYPEGUARD, + NUMBER_TYPEGUARD, + STRING_TYPEGUARD, +} from '../typeguards'; + +export class CellInColumnOperatorEvaluator + implements OperatorEvaluator +{ + public readonly operator = 'cellInColumn' as const; + + evaluate( + expression: BinaryExpression, + evaluationContext: EvaluationContext, + wrapperFactories: WrapperFactoryProvider, + strategy: EvaluationStrategy, + validationContext: ValidationContext | undefined, + ): string | InternalErrorValueRepresentation { + assert(expression.operator === this.operator); + + const sheetRowValues = evaluateExpression( + expression.left, + evaluationContext, + wrapperFactories, + validationContext, + strategy, + ); + + if ( + strategy === EvaluationStrategy.LAZY && + ERROR_TYPEGUARD(sheetRowValues) + ) { + return sheetRowValues; + } + + const accessor = evaluateExpression( + expression.right, + evaluationContext, + wrapperFactories, + validationContext, + strategy, + ); + if (INVALID_TYPEGUARD(sheetRowValues)) { + return sheetRowValues; + } + if (INVALID_TYPEGUARD(accessor)) { + return accessor; + } + if (MISSING_TYPEGUARD(sheetRowValues)) { + return sheetRowValues; + } + if (MISSING_TYPEGUARD(accessor)) { + return accessor; + } + + assert(COLLECTION_TYPEGUARD(sheetRowValues)); + assert(sheetRowValues.every((value) => STRING_TYPEGUARD(value))); + + assert(STRING_TYPEGUARD(accessor) || NUMBER_TYPEGUARD(accessor)); + + const index = + typeof accessor === 'string' + ? evaluationContext.getColumnIndex(accessor) + : accessor; + if (index === undefined) { + validationContext?.accept( + 'error', + `Could not find index for column "${accessor}"`, + { + node: expression.right, + }, + ); + return new MissingValue(`Could not find index for column ${accessor}`); + } + + if (index >= sheetRowValues.length) { + return new InvalidValue( + `Cannot column ${index} in a sheet with only ${sheetRowValues.length} columns`, + ); + } + const value = sheetRowValues.at(index); + assert(value !== undefined); + return value; + } +} diff --git a/libs/language-server/src/lib/ast/expressions/operator-registry.ts b/libs/language-server/src/lib/ast/expressions/operator-registry.ts index 16db0d190..d541c18b6 100644 --- a/libs/language-server/src/lib/ast/expressions/operator-registry.ts +++ b/libs/language-server/src/lib/ast/expressions/operator-registry.ts @@ -15,6 +15,7 @@ import { import { AdditionOperatorEvaluator } from './evaluators/addition-operator-evaluator'; import { AndOperatorEvaluator } from './evaluators/and-operator-evaluator'; import { CeilOperatorEvaluator } from './evaluators/ceil-operator-evaluator'; +import { CellInColumnOperatorEvaluator } from './evaluators/cell-in-column-operator-evaluator'; import { DivisionOperatorEvaluator } from './evaluators/division-operator-evaluator'; import { EqualityOperatorEvaluator } from './evaluators/equality-operator-evaluator'; import { FloorOperatorEvaluator } from './evaluators/floor-operator-evaluator'; @@ -60,6 +61,7 @@ import { type UnaryExpressionOperator, } from './operator-types'; import { BasicArithmeticOperatorTypeComputer } from './type-computers/basic-arithmetic-operator-type-computer'; +import { CellInColumnOperatorTypeComputer } from './type-computers/cell-in-column-operator-type-computer'; import { DivisionOperatorTypeComputer } from './type-computers/division-operator-type-computer'; import { EqualityOperatorTypeComputer } from './type-computers/equality-operator-type-computer'; import { ExponentialOperatorTypeComputer } from './type-computers/exponential-operator-type-computer'; @@ -117,6 +119,7 @@ export class DefaultOperatorEvaluatorRegistry lengthof: new LengthofOperatorEvaluator(), }; binary = { + cellInColumn: new CellInColumnOperatorEvaluator(), pow: new PowOperatorEvaluator(), root: new RootOperatorEvaluator(), '*': new MultiplicationOperatorEvaluator(), @@ -164,6 +167,7 @@ export class DefaultOperatorTypeComputerRegistry lengthof: new LengthofOperatorTypeComputer(this.valueTypeProvider), }; binary = { + cellInColumn: new CellInColumnOperatorTypeComputer(this.valueTypeProvider), pow: new ExponentialOperatorTypeComputer(this.valueTypeProvider), root: new ExponentialOperatorTypeComputer(this.valueTypeProvider), '*': new BasicArithmeticOperatorTypeComputer(this.valueTypeProvider), diff --git a/libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts b/libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts new file mode 100644 index 000000000..95408757b --- /dev/null +++ b/libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type ValidationContext } from '../../../validation/validation-context'; +import { type BinaryExpression } from '../../generated/ast'; +import { + type ValueType, + type ValueTypeProvider, + isCollectionValueType, +} from '../../wrappers/value-type'; +import { type BinaryOperatorTypeComputer } from '../operator-type-computer'; + +export class CellInColumnOperatorTypeComputer + implements BinaryOperatorTypeComputer +{ + constructor(protected readonly valueTypeProvider: ValueTypeProvider) {} + + computeType( + leftOperandType: ValueType, + rightOperandType: ValueType, + expression: BinaryExpression, + context: ValidationContext | undefined, + ): ValueType | undefined { + if ( + !isCollectionValueType( + leftOperandType, + this.valueTypeProvider.Primitives.Text, + ) + ) { + context?.accept( + 'error', + `Operator does not support type ${leftOperandType.getName()}`, + { + node: expression.left, + }, + ); + return undefined; + } + + if ( + !rightOperandType.equals(this.valueTypeProvider.Primitives.Text) && + !rightOperandType.equals(this.valueTypeProvider.Primitives.Integer) + ) { + context?.accept( + 'error', + `Operator does not support type ${rightOperandType.getName()}`, + { + node: expression.right, + }, + ); + return undefined; + } + return this.valueTypeProvider.Primitives.Text; + } +} From 5218be83920ea1f4dfa3c57b6a63598157feee7b Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:03:32 +0100 Subject: [PATCH 13/35] fix: table row evaluation always returns a table row. Even if an error occurs --- .../src/lib/ast/expressions/evaluate-expression.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts index 2fc23a5f3..aa11f329d 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts @@ -187,10 +187,6 @@ function evaluateValueLiteral( validationContext, strategy, ); - if (ERROR_TYPEGUARD(cellValue)) { - return cellValue; - } - tableRow.set(cell.name, cellValue); } return tableRow; From 80a3ab15751636e0b3dbbb62d82d5ff38319ace3 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:05:41 +0100 Subject: [PATCH 14/35] fix: find the actual type of the table row expression without using intermediate `TableRow` --- .../src/lib/ast/expressions/type-inference.ts | 57 +++++++++++++++++-- .../wrappers/value-type/atomic-value-type.ts | 12 ++++ .../abstract-collection-value-type.ts | 4 ++ 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/libs/language-server/src/lib/ast/expressions/type-inference.ts b/libs/language-server/src/lib/ast/expressions/type-inference.ts index 3e5d69ce3..54239edcb 100644 --- a/libs/language-server/src/lib/ast/expressions/type-inference.ts +++ b/libs/language-server/src/lib/ast/expressions/type-inference.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { assertUnreachable } from 'langium'; +import { assertUnreachable, AstUtils, type AstNode } from 'langium'; // eslint-disable-next-line unicorn/prefer-node-protocol import assert from 'assert'; @@ -44,8 +44,8 @@ import { } from '../generated/ast'; import { getNextAstNodeContainer } from '../model-util'; import { + type AtomicValueType, isAtomicValueType, - type TableRowValueType, type ValueType, type ValueTypeProvider, type WrapperFactoryProvider, @@ -307,13 +307,57 @@ function inferCollectionElementTypes( return undefined; } +export function equalSchemas( + a: Map, + b: Map, +): boolean { + if (a.size !== b.size) { + return false; + } + for (const [columnName, cellValue] of a) { + assert(cellValue !== undefined); + const otherCellValue = b.get(columnName); + if (cellValue !== otherCellValue) { + return false; + } + } + return true; +} + +function findValueTypeWithSameSchema( + schema: Map, + node: AstNode, + wrapperFactories: WrapperFactoryProvider, +): AtomicValueType | undefined { + const root = AstUtils.findRootNode(node); + const valueTypesWithSameSchema = AstUtils.streamAllContents(root) + .filter((astNode) => isValuetypeDefinition(astNode)) + .map((valueTypeDefinition) => { + const valueType = wrapperFactories.ValueType.wrap(valueTypeDefinition); + assert(valueType !== undefined); + return valueType; + }) + .filter( + (valueType) => + isAtomicValueType(valueType) && + equalSchemas(schema, valueType.getSchema()), + ) + .map((valueType) => { + assert(isAtomicValueType(valueType)); + return valueType; + }) + .toArray(); + + return onlyElementOrUndefined(valueTypesWithSameSchema); +} + function inferTableRowType( tableRow: TableRowLiteral, validationContext: ValidationContext, valueTypeProvider: ValueTypeProvider, wrapperFactories: WrapperFactoryProvider, -): TableRowValueType | undefined { - const cellValueTypes = new Map(); +): AtomicValueType | undefined { + const schema = new Map(); for (const cell of tableRow.cells) { const cellValueType = inferExpressionType( cell.expression, @@ -324,9 +368,10 @@ function inferTableRowType( if (cellValueType === undefined) { return undefined; } - cellValueTypes.set(cell.name, cellValueType); + schema.set(cell.name, cellValueType); } - return valueTypeProvider.createTableRowValueTypeOf(cellValueTypes); + + return findValueTypeWithSameSchema(schema, tableRow, wrapperFactories); } function inferTypeFromValueKeyword( diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts index 9a8725908..b697638a2 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts @@ -46,6 +46,18 @@ export class AtomicValueType return this.astNode?.properties; } + getSchema(): Map { + const schema = new Map(); + for (const property of this.getProperties()) { + const valueType = this.wrapperFactories.ValueType.wrap( + property.valueType, + ); + assert(valueType !== undefined); + schema.set(property.name, valueType); + } + return schema; + } + private getNestedProperty( nestedPropertyAccess: NestedPropertyAccess, ): ValueTypeProperty | undefined { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts index b4ea4d4b5..468c38901 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts @@ -14,4 +14,8 @@ export abstract class AbstractCollectionValueType< override isAllowedAsRuntimeParameter(): boolean { return false; } + + override isReferenceableByUser(): boolean { + return true; + } } From f5f629e77ff13d939c3795adac5b77f6db0854a9 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:26:48 +0100 Subject: [PATCH 15/35] fix: validate transforms used in TableInterpreter --- .../block-type-specific/property-body.ts | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts index f86a56837..4b373a609 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts @@ -3,7 +3,11 @@ // SPDX-License-Identifier: AGPL-3.0-only import { + type AtomicValueType, ERROR_TYPEGUARD, + isAtomicValueType, + isTableRowLiteral, + type TransformDefinition, type PropertyBody, evaluatePropertyValue, } from '../../../ast'; @@ -20,6 +24,8 @@ export function checkBlockTypeSpecificPropertyBody( return checkCellWriterPropertyBody(propertyBody, props); case 'TableTransformer': return checkTableTransformerPropertyBody(propertyBody, props); + case 'TableInterpreter': + return checkTableInterpreterPropertyBody(propertyBody, props); default: } } @@ -123,6 +129,98 @@ function checkTableTransformerPropertyBody( checkInputColumnsMatchTransformationPorts(propertyBody, props); } +function checkTableInterpreterPropertyBody( + propertyBody: PropertyBody, + props: JayveeValidationProps, +): void { + const parseWithProperty = propertyBody.properties.find( + (property) => property.name == 'parseWith', + ); + if (parseWithProperty === undefined) { + return; + } + + const transform = evaluatePropertyValue( + parseWithProperty, + props.evaluationContext, + props.wrapperFactories, + props.valueTypeProvider.Primitives.Transform, + ); + + if (ERROR_TYPEGUARD(transform)) { + return; + } + + const columnsProperty = propertyBody.properties.find( + (property) => property.name == 'columns', + ); + if (columnsProperty === undefined) { + return; + } + + const columns = evaluatePropertyValue( + columnsProperty, + props.evaluationContext, + props.wrapperFactories, + props.valueTypeProvider.Primitives.ValuetypeDefinition, + ); + if (ERROR_TYPEGUARD(columns)) { + return; + } + + const columnsValueType = props.wrapperFactories.ValueType.wrap(columns); + if (columnsValueType === undefined) { + return; + } + if (!isAtomicValueType(columnsValueType)) { + props.validationContext.accept( + 'error', + 'This value type must be user created and reflect the columns that should be in the table', + { node: columnsProperty.value }, + ); + return; + } + + checkParseWithTransform(transform, columnsValueType, props); +} + +function checkParseWithTransform( + transform: TransformDefinition, + columnsValueType: AtomicValueType, + props: JayveeValidationProps, +): void { + for (const port of transform.body.ports) { + if (port.kind !== 'to') { + continue; + } + + const portValueType = props.wrapperFactories.ValueType.wrap(port.valueType); + if (portValueType === undefined) { + continue; + } + + if (!portValueType.equals(columnsValueType)) { + props.validationContext.accept( + 'error', + `Expected value type ${columnsValueType.getName()}, but found ${portValueType.getName()}`, + { node: port.valueType }, + ); + } + } + + for (const outputAssignment of transform.body.outputAssignments) { + if (!isTableRowLiteral(outputAssignment.expression)) { + props.validationContext.accept( + 'error', + 'Transforms used in TableInterpreter blocks must have one output assignmet using a table row expression', + { + node: outputAssignment.expression, + }, + ); + } + } +} + function checkInputColumnsMatchTransformationPorts( propertyBody: PropertyBody, props: JayveeValidationProps, From 9552e743f944cf5b48ce025b77ac572667b3c30c Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:28:55 +0100 Subject: [PATCH 16/35] fix: validate transforms that use TableRowLiteral expressions --- .../lib/validation/checks/transform-body.ts | 103 ++++++++++++++++++ .../checks/transform-output-assigment.ts | 78 ++++++++++++- 2 files changed, 178 insertions(+), 3 deletions(-) diff --git a/libs/language-server/src/lib/validation/checks/transform-body.ts b/libs/language-server/src/lib/validation/checks/transform-body.ts index 6cbc34404..04769e8e1 100644 --- a/libs/language-server/src/lib/validation/checks/transform-body.ts +++ b/libs/language-server/src/lib/validation/checks/transform-body.ts @@ -7,6 +7,7 @@ */ import { + isTableRowLiteral, type TransformBody, type TransformPortDefinition, isTransformPortDefinition, @@ -18,6 +19,10 @@ import { extractReferenceLiterals, validateTransformOutputAssignment, } from './transform-output-assigment'; +import { onlyElementOrUndefined } from '../../util'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; +import { isAtomicValueType } from '../../ast'; export function validateTransformBody( transformBody: TransformBody, @@ -34,6 +39,9 @@ export function validateTransformBody( checkAreInputsUsed(transformBody, props); + checkInputForTableRowTransform(transformBody, props); + checkOutputForTableRowTransform(transformBody, props); + for (const property of transformBody.outputAssignments) { validateTransformOutputAssignment(property, props); } @@ -108,6 +116,101 @@ function checkSingleOutputPort( } } +function checkInputForTableRowTransform( + transformBody: TransformBody, + props: JayveeValidationProps, +): void { + const inputs = transformBody.ports?.filter((x) => x.kind === 'from'); + const outputAssignments = transformBody?.outputAssignments; + if (inputs === undefined || outputAssignments === undefined) { + return undefined; + } + + const isTableRowTransform = outputAssignments.some((outputAssignment) => + isTableRowLiteral(outputAssignment), + ); + if (!isTableRowTransform) { + return; + } + + const textCollection = props.valueTypeProvider.createCollectionValueTypeOf( + props.valueTypeProvider.Primitives.Text, + ); + + const input = onlyElementOrUndefined(inputs); + const inputValueType = props.wrapperFactories.ValueType.wrap( + input?.valueType, + ); + assert(input !== undefined); + + if (inputValueType === undefined) { + props.validationContext.accept( + 'error', + 'Transforms with a table row expression must have exactly one input', + { + node: transformBody, + }, + ); + return; + } + + if (!inputValueType.equals(textCollection)) { + props.validationContext.accept( + 'error', + 'This input must be of type `Collection', + { + node: input.valueType, + }, + ); + return; + } +} + +function checkOutputForTableRowTransform( + transformBody: TransformBody, + props: JayveeValidationProps, +): void { + const outputs = transformBody.ports?.filter((x) => x.kind === 'to'); + const outputAssignments = transformBody?.outputAssignments; + if (outputs === undefined || outputAssignments === undefined) { + return undefined; + } + + const isTableRowTransform = outputAssignments.some((outputAssignment) => + isTableRowLiteral(outputAssignment), + ); + if (!isTableRowTransform) { + return; + } + + const output = onlyElementOrUndefined(outputs); + const outputValueType = props.wrapperFactories.ValueType.wrap( + output?.valueType, + ); + assert(output !== undefined); + if (outputValueType === undefined) { + props.validationContext.accept( + 'error', + 'Transforms with a table row expression must have exactly one ouptut', + { + node: transformBody, + }, + ); + return; + } + + if (!isAtomicValueType(outputValueType)) { + props.validationContext.accept( + 'error', + 'This must be a user created value type', + { + node: output.valueType, + }, + ); + return; + } +} + function checkAreInputsUsed( transformBody: TransformBody, props: JayveeValidationProps, diff --git a/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts b/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts index deb2816b0..30cf299db 100644 --- a/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts +++ b/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts @@ -16,12 +16,16 @@ import { isBinaryExpression, isExpressionLiteral, isReferenceLiteral, + isTableRowLiteral, isTernaryExpression, isTransformPortDefinition, isUnaryExpression, } from '../../ast/generated/ast'; import { type JayveeValidationProps } from '../validation-registry'; import { checkExpressionSimplification } from '../validation-util'; +import { isAtomicValueType } from '../../ast'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; export function validateTransformOutputAssignment( outputAssignment: TransformOutputAssignment, @@ -51,15 +55,83 @@ function checkOutputValueTyping( props.valueTypeProvider, props.wrapperFactories, ); - if (inferredType === undefined) { - return; - } const expectedType = props.wrapperFactories.ValueType.wrap(outputType); if (expectedType === undefined) { return; } + if ( + isTableRowLiteral(assignmentExpression) && + !isAtomicValueType(expectedType) + ) { + props.validationContext.accept( + 'error', + `The output type of a Table Row transform must be user created`, + { node: outputType }, + ); + return; + } + + if (inferredType === undefined && isTableRowLiteral(assignmentExpression)) { + assert(isAtomicValueType(expectedType)); + + const schema = expectedType.getSchema(); + for (const [columnName, columnValueType] of schema) { + const cell = assignmentExpression.cells.find( + (cell) => cell.name == columnName, + ); + if (cell === undefined) { + props.validationContext.accept( + 'error', + `The table row expression must contain an assignment for the property ${columnName}`, + { node: assignmentExpression }, + ); + return; + } + const cellValueType = inferExpressionType( + cell.expression, + props.validationContext, + props.valueTypeProvider, + props.wrapperFactories, + ); + if (cellValueType === undefined) { + props.validationContext.accept('error', `Could not infer value type`, { + node: cell.expression, + }); + return; + } + if ( + !cellValueType.isConvertibleTo(columnValueType) && + !columnValueType.isConvertibleTo(cellValueType) + ) { + props.validationContext.accept( + 'error', + `${cellValueType.getName()} is not convertible to ${columnValueType.getName()}`, + { + node: cell.expression, + }, + ); + } + } + for (const cell of assignmentExpression.cells) { + if (!schema.has(cell.name)) { + props.validationContext.accept( + 'error', + `${expectedType.getName()} does not have any properties with name ${cell.name}`, + { + node: cell, + }, + ); + } + } + return; + } + + if (inferredType === undefined) { + return; + } + if (!inferredType.isConvertibleTo(expectedType)) { props.validationContext.accept( 'error', From 6dcfc431bf1db144ff4546a70d26f2a63be04b9e Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:29:55 +0100 Subject: [PATCH 17/35] fix: allow referencing multi property value types from transforms with table row expressions --- .../validation/checks/value-type-reference.ts | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/libs/language-server/src/lib/validation/checks/value-type-reference.ts b/libs/language-server/src/lib/validation/checks/value-type-reference.ts index 9a4e48d14..c8070ac8d 100644 --- a/libs/language-server/src/lib/validation/checks/value-type-reference.ts +++ b/libs/language-server/src/lib/validation/checks/value-type-reference.ts @@ -14,6 +14,7 @@ export function validateValueTypeReference( ): void { checkGenericsMatchDefinition(valueTypeRef, props); checkIsValueTypeReferenceable(valueTypeRef, props); + checkIsValueTypeInTableParsingTransform(valueTypeRef, props); } function checkGenericsMatchDefinition( @@ -25,16 +26,6 @@ function checkGenericsMatchDefinition( return; } - const numberOfProperties = valueTypeDefinition.properties?.length; - if (numberOfProperties > 1) { - props.validationContext.accept( - 'error', - 'The referenced value type has more than one property. ' + - 'This is unsupported for now', - { node: valueTypeRef }, - ); - } - const requiredGenerics = valueTypeDefinition.genericDefinition?.generics?.length ?? 0; @@ -79,3 +70,27 @@ function checkIsValueTypeReferenceable( }, ); } + +function checkIsValueTypeInTableParsingTransform( + valueTypeRef: ValueTypeReference, + props: JayveeValidationProps, +) { + const valueTypeDefinition = valueTypeRef.reference?.ref; + if (valueTypeDefinition === undefined) { + return; + } + const parent = valueTypeRef.$container; + if (parent?.$type === 'TransformPortDefinition') { + return; + } + + const numberOfProperties = valueTypeDefinition.properties?.length; + if (numberOfProperties > 1) { + props.validationContext.accept( + 'error', + `The referenced value type has more than one property. ` + + 'This is unsupported for now', + { node: valueTypeRef }, + ); + } +} From 06a41e0591a1882cb3d43ac12f540081b57b6bd8 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:31:10 +0100 Subject: [PATCH 18/35] feat!: add parseWith property and remove obsolete properties --- .../stdlib/builtin-block-types/TableInterpreter.jv | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv index 7c5648eb2..54826c958 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv @@ -48,13 +48,8 @@ publish builtin blocktype TableInterpreter { */ property columns oftype ValuetypeDefinition; - /** - * Whether to ignore whitespace before values. Does not apply to `text` cells - */ - property skipLeadingWhitespace oftype boolean: true; - - /** - * Whether to ignore whitespace after values. Does not apply to `text` cells - */ - property skipTrailingWhitespace oftype boolean: true; + /** + * This transform to create each table row from the sheet row + */ + property parseWith oftype Transform; } From ff85374fb6bed828a3d332d8d3d651ffc434a00c Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:32:57 +0100 Subject: [PATCH 19/35] misc: `popHeaderRow` replaces `getHeaderRow` and `getColumns` return value is no longer readonly --- libs/execution/src/lib/types/io-types/sheet.ts | 9 ++------- libs/execution/src/lib/types/io-types/table.ts | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/libs/execution/src/lib/types/io-types/sheet.ts b/libs/execution/src/lib/types/io-types/sheet.ts index 70989b1e8..bd1f84a8c 100644 --- a/libs/execution/src/lib/types/io-types/sheet.ts +++ b/libs/execution/src/lib/types/io-types/sheet.ts @@ -45,13 +45,8 @@ export class Sheet implements IOTypeImplementation { return this.numberOfColumns; } - getHeaderRow(): string[] { - assert( - this.getNumberOfRows() > 0, - 'The sheet is expected to be non-empty and have a header row', - ); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return this.data[0]!; + popHeaderRow(): string[] | undefined { + return this.data.shift(); } iterateRows(callbackFn: (row: string[], rowIndex: number) => void) { diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index a6aa15a16..93ec929b2 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -118,7 +118,7 @@ export class Table implements IOTypeImplementation { return this.columns.has(name); } - getColumns(): ReadonlyMap { + getColumns(): Map { return this.columns; } From db18ecbd89864d7c0ac5d3ec960ec85b07ed5573 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:34:52 +0100 Subject: [PATCH 20/35] fix: adapt tests to new `popHeaderRow` --- .../exec/src/lib/cell-range-selector-executor.spec.ts | 2 -- .../tabular/exec/src/lib/cell-writer-executor.spec.ts | 2 -- .../tabular/exec/src/lib/column-deleter-executor.spec.ts | 3 --- .../tabular/exec/src/lib/row-deleter-executor.spec.ts | 6 +++--- .../tabular/exec/src/lib/sheet-picker-executor.spec.ts | 1 - 5 files changed, 3 insertions(+), 11 deletions(-) diff --git a/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts index 1f03f6fc9..9f9334781 100644 --- a/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts @@ -96,7 +96,6 @@ describe('Validation of CellRangeSelectorExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], @@ -121,7 +120,6 @@ describe('Validation of CellRangeSelectorExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(2); - expect(result.right.getHeaderRow()).toEqual(['', 'Test', 'true']); expect(result.right.getData()).toEqual([ ['', 'Test', 'true'], ['', 'Test', 'false'], diff --git a/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts index 7f2c77a49..396b68078 100644 --- a/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts @@ -96,7 +96,6 @@ describe('Validation of CellWriterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['16', 'Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['16', 'Test', 'true'], @@ -121,7 +120,6 @@ describe('Validation of CellWriterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['16', 'Test2', '']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['16', 'Test2', ''], diff --git a/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts index ca80bda84..191b57226 100644 --- a/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts @@ -96,7 +96,6 @@ describe('Validation of ColumnDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(2); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['Test', 'true'], @@ -121,7 +120,6 @@ describe('Validation of ColumnDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(1); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['Test']); expect(result.right.getData()).toEqual( expect.arrayContaining([['Test'], ['Test'], ['Test']]), ); @@ -159,7 +157,6 @@ describe('Validation of ColumnDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(2); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['Test', 'true'], diff --git a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts index 44d9106e6..7dd9cb114 100644 --- a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts @@ -96,7 +96,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(15); - expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([['0', 'Test', 'true']]), ); @@ -117,7 +117,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(14); - expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], @@ -158,7 +158,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(15); - expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([['0', 'Test', 'true']]), ); diff --git a/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts index d656e161e..8e0a02100 100644 --- a/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts @@ -93,7 +93,6 @@ describe('Validation of SheetPickerExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], From a50931f13666d70481975aa4f5bd2b7e0d37a812 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:40:57 +0100 Subject: [PATCH 21/35] fix: rework `TableInterpreterExecutor`. Most notable changes: - Remove the logic deriving column definitions with/without header - The columns property is the definitive source of truth - The header row is used to map sheet-column-names to sheet-column-indices --- .../lib/table-interpreter-executor.spec.ts | 65 +---- .../src/lib/table-interpreter-executor.ts | 273 +++++++----------- 2 files changed, 105 insertions(+), 233 deletions(-) diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts index 361e40577..987e920f6 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts @@ -137,7 +137,7 @@ describe('Validation of TableInterpreterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(0); } }); @@ -245,7 +245,10 @@ describe('Validation of TableInterpreterExecutor', () => { values: expect.arrayContaining([true, false]) as boolean[], }), ); - for (const value of result.right.getRow(0).values()) { + const row = result.right.getRow(0); + expect(row).toBeDefined(); + assert(row !== undefined); + for (const value of row.values()) { if (value === 'name') { continue; } @@ -302,63 +305,5 @@ describe('Validation of TableInterpreterExecutor', () => { } } }); - - it('should skip leading and trailing whitespace on numeric columns but not text columns', async () => { - const text = readJvTestAsset('valid-without-header.jv'); - - const testWorkbook = await readTestWorkbook('test-with-whitespace.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - assert(R.isOk(result)); - - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(3); - expect(result.right.getNumberOfRows()).toEqual(3); - - expect([...result.right.getColumns().keys()]).toStrictEqual([ - 'index', - 'name', - 'flag', - ]); - - const row = result.right.getRow(0); - const index = row.get('index'); - expect(index).toBe(0); - const name = row.get('name'); - expect(name).toBe(' text with leading whitespace'); - - for (let rowIdx = 1; rowIdx < result.right.getNumberOfRows(); rowIdx++) { - const row = result.right.getRow(rowIdx); - const index = row.get('index'); - expect(index).toBe(rowIdx); - } - }); - - it('should not skip leading or trailing whitespace if the relevant block properties are false', async () => { - const text = readJvTestAsset('valid-without-header-without-trim.jv'); - - const testWorkbook = await readTestWorkbook('test-with-whitespace.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(3); - expect(result.right.getNumberOfRows()).toEqual(3); - const indexColumn = result.right.getColumn('index')?.values; - expect(indexColumn).toBeDefined(); - assert(indexColumn !== undefined); - indexColumn.forEach((cell) => - expect(cell).toBeInstanceOf(InvalidValue), - ); - } - }); }); }); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index ada4a24e9..1500ca13e 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -8,36 +8,30 @@ import { strict as assert } from 'assert'; import * as R from '@jvalue/jayvee-execution'; import { AbstractBlockExecutor, + TransformExecutor, type BlockExecutorClass, type ExecutionContext, type Sheet, Table, implementsStatic, - isValidValueRepresentation, - parseValueToInternalRepresentation, } from '@jvalue/jayvee-execution'; import { AtomicValueType, - CellIndex, ERROR_TYPEGUARD, - IOType, - InternalErrorValueRepresentation, - type InternalValidValueRepresentation, - InvalidValue, - MissingValue, - type ValueType, - ValueTypeProperty, + evaluateExpression, internalValueToString, + InvalidValue, + IOType, + isTableRowLiteral, + MISSING_TYPEGUARD, + TableRowLiteral, isAtomicValueType, + MissingValue, + onlyElementOrUndefined, + TABLEROW_TYPEGUARD, + ValueType, } from '@jvalue/jayvee-language-server'; -export interface ColumnDefinitionEntry { - sheetColumnIndex: number; - columnName: string; - valueType: ValueType; - astNode: ValueTypeProperty; -} - @implementsStatic() export class TableInterpreterExecutor extends AbstractBlockExecutor< IOType.SHEET, @@ -62,13 +56,9 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< 'columns', context.valueTypeProvider.Primitives.ValuetypeDefinition, ); - const skipLeadingWhitespace = context.getPropertyValue( - 'skipLeadingWhitespace', - context.valueTypeProvider.Primitives.Boolean, - ); - const skipTrailingWhitespace = context.getPropertyValue( - 'skipTrailingWhitespace', - context.valueTypeProvider.Primitives.Boolean, + const parseWith = context.getPropertyValue( + 'parseWith', + context.valueTypeProvider.Primitives.Transform, ); const columnsValueType = context.wrapperFactories.ValueType.wrap( @@ -80,10 +70,12 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< 'This must have been checked earlier at the validation step', ); - let columnEntries: ColumnDefinitionEntry[]; + const schema = columnsValueType.getSchema(); + let headerRow: string[] | undefined = undefined; if (header) { - if (inputSheet.getNumberOfRows() < 1) { + headerRow = inputSheet.popHeaderRow(); + if (headerRow === undefined) { return R.err({ message: 'The input sheet is empty and thus has no header', diagnostic: { @@ -91,14 +83,15 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< }, }); } - - const headerRow = inputSheet.getHeaderRow(); - - columnEntries = this.deriveColumnDefinitionEntriesFromHeader( - columnsValueType, - headerRow, - context, - ); + if (headerRow.length !== schema.size) { + return R.err({ + message: + 'The length of the header does not fit the columns value type', + diagnostic: { + node: context.getOrFailProperty('header'), + }, + }); + } } else { if ( inputSheet.getNumberOfColumns() < @@ -111,27 +104,19 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< }, }); } - - columnEntries = this.deriveColumnDefinitionEntriesWithoutHeader( - columnsValueType, - context, - ); } - const numberOfTableRows = header - ? inputSheet.getNumberOfRows() - 1 - : inputSheet.getNumberOfRows(); context.logger.logDebug( - `Validating ${numberOfTableRows} row(s) according to the column types`, + `Validating ${inputSheet.getNumberOfRows()} row(s) according to the column types`, ); + const parseRowTransform = new TransformExecutor(parseWith, context); + const resultingTable = this.constructAndValidateTable( inputSheet, - header, - columnEntries, + headerRow, columnsValueType, - skipLeadingWhitespace, - skipTrailingWhitespace, + parseRowTransform, context, ); context.logger.logDebug( @@ -142,20 +127,32 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< private constructAndValidateTable( sheet: Sheet, - header: boolean, - columnEntries: ColumnDefinitionEntry[], + headerRow: string[] | undefined, columnsValueType: AtomicValueType, - skipLeadingWhitespace: boolean, - skipTrailingWhitespace: boolean, + parseRowTransform: TransformExecutor, context: ExecutionContext, ): Table { const columns = new Map(); - columnEntries.forEach((columnEntry) => { - columns.set(columnEntry.columnName, { + for (const [columnName, columnValueType] of columnsValueType + .getSchema() + .entries()) { + columns.set(columnName, { values: [], - valueType: columnEntry.valueType, + valueType: columnValueType, }); - }); + } + if (headerRow !== undefined) { + context.evaluationContext.setHeaderRow(headerRow); + } + + const sheetRowReferenceName = onlyElementOrUndefined( + parseRowTransform.getInputDetails(), + )?.port.name; + assert(sheetRowReferenceName !== undefined); + + const parseRowExpression = + parseRowTransform.getOutputAssignment().expression; + assert(isTableRowLiteral(parseRowExpression)); const constraints = columnsValueType .getConstraints() @@ -164,20 +161,19 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< const table = new Table(0, columns, constraints); // add rows - sheet.iterateRows((sheetRow, sheetRowIndex) => { - if (header && sheetRowIndex === 0) { - return; - } - + sheet.iterateRows((sheetRow) => { const tableRow = this.constructAndValidateTableRow( + sheetRowReferenceName, sheetRow, - sheetRowIndex, - columnEntries, - skipLeadingWhitespace, - skipTrailingWhitespace, + columnsValueType.getSchema(), + parseRowExpression, context, ); - table.addRow(tableRow); + if (MISSING_TYPEGUARD(tableRow)) { + context.logger.logDebug(tableRow.toString()); + } else { + table.addRow(tableRow); + } }); // eslint-disable-next-line @typescript-eslint/no-unused-vars @@ -192,122 +188,53 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< } private constructAndValidateTableRow( + sheetRowReferenceName: string, sheetRow: string[], - sheetRowIndex: number, - columnEntries: ColumnDefinitionEntry[], - skipLeadingWhitespace: boolean, - skipTrailingWhitespace: boolean, + schema: Map, + parseRowExpression: TableRowLiteral, context: ExecutionContext, - ): R.TableRow { - const tableRow: R.TableRow = new Map< - string, - InternalValidValueRepresentation | InternalErrorValueRepresentation - >(); - columnEntries.forEach((columnEntry) => { - const valueType = columnEntry.valueType; - const sheetColumnIndex = columnEntry.sheetColumnIndex; - const value = sheetRow[sheetColumnIndex]; - - const parsedValue = - value !== undefined - ? this.parseAndValidateValue( - value, - valueType, - skipLeadingWhitespace, - skipTrailingWhitespace, - context, - ) - : new MissingValue( - `The sheet row did not contain a value at index ${sheetColumnIndex}`, - ); - if (ERROR_TYPEGUARD(parsedValue)) { - const currentCellIndex = new CellIndex(sheetColumnIndex, sheetRowIndex); - context.logger.logDebug( - `Invalid value at cell ${currentCellIndex.toString()}: "${value}" does not match the type ${columnEntry.valueType.getName()}`, - ); - } - - tableRow.set(columnEntry.columnName, parsedValue); - }); - - assert(tableRow.size === columnEntries.length); - return tableRow; - } + ): R.TableRow | MissingValue { + context.evaluationContext.setValueForReference( + sheetRowReferenceName, + sheetRow, + ); - private parseAndValidateValue( - value: string, - valueType: ValueType, - skipLeadingWhitespace: boolean, - skipTrailingWhitespace: boolean, - context: ExecutionContext, - ): InternalValidValueRepresentation | InternalErrorValueRepresentation { - const parsedValue = parseValueToInternalRepresentation(value, valueType, { - skipLeadingWhitespace, - skipTrailingWhitespace, - }); + const tableRow = evaluateExpression( + parseRowExpression, + context.evaluationContext, + context.wrapperFactories, + ); + assert(TABLEROW_TYPEGUARD(tableRow)); - if ( - !ERROR_TYPEGUARD(parsedValue) && - !isValidValueRepresentation(parsedValue, valueType, context) - ) { - return new InvalidValue( - `The following value was not valid for valuetype ${valueType.getName()}: ${internalValueToString( - parsedValue, - context.wrapperFactories, - )}`, - ); + let missingCount = 0; + for (const value of tableRow.values()) { + if (MISSING_TYPEGUARD(value)) { + context.logger.logDebug(value.toString()); + missingCount += 1; + } + } + if (missingCount === tableRow.size) { + return new MissingValue('All values in row were missing. Discarding row'); } - return parsedValue; - } - - private deriveColumnDefinitionEntriesWithoutHeader( - columnsValueType: AtomicValueType, - context: ExecutionContext, - ): ColumnDefinitionEntry[] { - return columnsValueType.getProperties().map((property, propertyIndex) => { - const columnValuetype = context.wrapperFactories.ValueType.wrap( - property.valueType, - ); - assert(columnValuetype !== undefined); - return { - sheetColumnIndex: propertyIndex, - columnName: property.name, - valueType: columnValuetype, - astNode: property, - }; - }); - } - - private deriveColumnDefinitionEntriesFromHeader( - columnsValueType: AtomicValueType, - headerRow: string[], - context: ExecutionContext, - ): ColumnDefinitionEntry[] { - context.logger.logDebug(`Matching header with provided column names`); - return columnsValueType.getProperties().flatMap((property) => { - const indexOfMatchingHeader = headerRow.findIndex( - (headerColumnName) => headerColumnName === property.name, - ); - if (indexOfMatchingHeader === -1) { - context.logger.logDebug( - `Omitting column "${property.name}" as the name was not found in the header`, + assert(tableRow.size === schema.size); + for (const [columnName, cellValue] of tableRow) { + const entry = [...schema.entries()].find(([cN]) => cN === columnName); + assert(entry !== undefined); + const [, columnValueType] = entry; + if ( + !ERROR_TYPEGUARD(cellValue) && + !columnValueType.isInternalValidValueRepresentation(cellValue) + ) { + tableRow.set( + columnName, + new InvalidValue( + `cell value ${internalValueToString(cellValue)} is not a valid value for ${columnValueType.getName()}`, + ), ); - return []; } - const columnValuetype = context.wrapperFactories.ValueType.wrap( - property.valueType, - ); - assert(columnValuetype !== undefined); + } - return [ - { - sheetColumnIndex: indexOfMatchingHeader, - columnName: property.name, - valueType: columnValuetype, - astNode: property, - }, - ]; - }); + return tableRow; } } From c4c7c4c3f3bdc3d9f0ed907794a7e660975346a6 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:43:17 +0100 Subject: [PATCH 22/35] fix: don't depend on removed `ColumnDefinitionEntry` --- libs/extensions/tabular/exec/test/util.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/libs/extensions/tabular/exec/test/util.ts b/libs/extensions/tabular/exec/test/util.ts index ba2a1ea8c..96725b91c 100644 --- a/libs/extensions/tabular/exec/test/util.ts +++ b/libs/extensions/tabular/exec/test/util.ts @@ -18,8 +18,6 @@ import { } from '@jvalue/jayvee-language-server'; import * as exceljs from 'exceljs'; -import { type ColumnDefinitionEntry } from '../src/lib/table-interpreter-executor'; - export async function createWorkbookFromLocalExcelFile( fileName: string, ): Promise { @@ -51,10 +49,11 @@ export async function createWorkbookFromLocalExcelFile( return workbook; } -export type ReducedColumnDefinitionEntry = Pick< - ColumnDefinitionEntry, - 'sheetColumnIndex' | 'columnName' | 'valueType' ->; +export type ReducedColumnDefinitionEntry = { + sheetColumnIndex: number; + columnName: string; + valueType: ValueType; +}; /** * Creates a Table from the first sheet of the excel file pointed to by {@link fileName} From fe25d4d0e554925e458bc8308344e9e0df463992 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:44:38 +0100 Subject: [PATCH 23/35] fix: adapt test --- libs/interpreter-lib/src/interpreter.spec.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/interpreter-lib/src/interpreter.spec.ts b/libs/interpreter-lib/src/interpreter.spec.ts index 4043ddc84..3859d1bdb 100644 --- a/libs/interpreter-lib/src/interpreter.spec.ts +++ b/libs/interpreter-lib/src/interpreter.spec.ts @@ -200,7 +200,8 @@ describe('Interpreter', () => { assert(input != null); assert(input instanceof Table); - expect(input.getNumberOfColumns()).toBe(1); + const initial_columns = 12; + expect(input.getNumberOfColumns()).toBe(initial_columns); expect(input.getColumn('name')?.values).toStrictEqual(EXPECTED_NAMES); expect(isOk(output)).toBe(true); @@ -210,7 +211,7 @@ describe('Interpreter', () => { assert(out != null); assert(out instanceof Table); - expect(out.getNumberOfColumns()).toBe(2); + expect(out.getNumberOfColumns()).toBe(initial_columns + 1); expect(out.getColumn('name')?.values).toStrictEqual(EXPECTED_NAMES); expect(out.getColumn('nameCopy')?.values).toStrictEqual( EXPECTED_NAMES, From ce37b1f249fb5ed7172e1d5afdf6bacf0e99d3d0 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:45:48 +0100 Subject: [PATCH 24/35] fix: adapt jayvee test assets --- .../valid-empty-columns-with-header.jv | 9 +++++++++ .../valid-with-capitalized-header.jv | 8 ++++++++ .../valid-with-header.jv | 8 ++++++++ .../valid-without-header-without-trim.jv | 19 +++++++++++++------ .../valid-without-header.jv | 9 +++++++++ .../valid-wrong-value-type-with-header.jv | 8 ++++++++ .../valid-wrong-value-type-without-header.jv | 9 +++++++++ 7 files changed, 64 insertions(+), 6 deletions(-) diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv index 5810d24db..d3c3189cd 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv @@ -10,9 +10,18 @@ pipeline TestPipeline { valuetype TestValueType { } + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { }; + } + block TestBlock oftype TableInterpreter { header: true; columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv index 19d138c08..08fc9db23 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv @@ -13,9 +13,17 @@ pipeline TestPipeline { property Flag oftype boolean; } + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { Index: asInteger (r cellInColumn "Index"), Name: r cellInColumn "Name", Flag: asBoolean (r cellInColumn "Flag") }; + } + block TestBlock oftype TableInterpreter { header: true; columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv index 0f37661ea..4d46a854d 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv @@ -13,9 +13,17 @@ pipeline TestPipeline { property flag oftype boolean; } + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn "index"), name: r cellInColumn "name", flag: asBoolean (r cellInColumn "flag") }; + } + block TestBlock oftype TableInterpreter { header: true; columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv index a6cc632d7..37d9b00f3 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv @@ -12,12 +12,19 @@ pipeline TestPipeline { property flag oftype boolean; } - block TestBlock oftype TableInterpreter { - header: false; - columns: TestValueType; - skipLeadingWhitespace: false; - skipTrailingWhitespace: false; - } + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn "index"), name: r cellInColumn "name", + flag: asBoolean (r cellInColumn "flag"), }; + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: TestValueType; + parseWith: Parser; + } block TestLoader oftype TestTableLoader { } diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv index f0a969bcc..9925fd6a0 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv @@ -13,9 +13,18 @@ pipeline TestPipeline { property flag oftype boolean; } + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn 0), name: r cellInColumn 1, flag: + asBoolean (r cellInColumn 2) }; + } + block TestBlock oftype TableInterpreter { header: false; columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv index 9dfb35257..1c480af72 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv @@ -13,9 +13,17 @@ pipeline TestPipeline { property flag oftype integer; } + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn "index"), name: r cellInColumn "name", flag: asBoolean (r cellInColumn "flag") }; + } + block TestBlock oftype TableInterpreter { header: true; columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv index fb5cd57a3..20d65adf8 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv @@ -13,9 +13,18 @@ pipeline TestPipeline { property flag oftype integer; } + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn 0), name: r cellInColumn 1, flag: + asBoolean (r cellInColumn 2) }; + } + block TestBlock oftype TableInterpreter { header: false; columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { From 734cf6f9aaac55323732d9ee8eb32b96a170c8a2 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:46:29 +0100 Subject: [PATCH 25/35] fix: adapt jayvee test assets --- .../test/assets/graph/composite-block.jv | 7 ++ .../test/assets/graph/two-pipelines.jv | 75 ++++++++++++++----- .../valid-builtin-and-composite-blocks.jv | 42 +++++++++-- 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/libs/interpreter-lib/test/assets/graph/composite-block.jv b/libs/interpreter-lib/test/assets/graph/composite-block.jv index 4e16bd891..8ba0df960 100644 --- a/libs/interpreter-lib/test/assets/graph/composite-block.jv +++ b/libs/interpreter-lib/test/assets/graph/composite-block.jv @@ -19,9 +19,16 @@ pipeline CarsPipeline { property name oftype text; } + transform parser { + from r oftype Collection; + to y oftype CarName; + y: { name: asText (r cellInColumn "name") }; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; columns: CarName; + parseWith: parser; } transform copy { diff --git a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv index 480e15812..cd9c9f41e 100644 --- a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv +++ b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv @@ -46,9 +46,17 @@ pipeline CarsPipeline { property carb oftype integer; } + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { name: asText (r cellInColumn "name"), mpg: asDecimal (r cellInColumn "mpg"), cyl: asInteger (r cellInColumn 2), disp: asDecimal (r cellInColumn 3), hp: asInteger (r cellInColumn "hp"), drat: asDecimal (r cellInColumn "drat"), wt: asDecimal (r cellInColumn "wt"), qsec: asDecimal (r cellInColumn "qsec"), vs: asInteger (r cellInColumn "vs"), am: asInteger (r cellInColumn "am"), gear: asInteger (r cellInColumn "gear"), carb: asInteger (r cellInColumn "carb") }; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; columns: Car; + parseWith: CarParser; } block CarsLoader oftype SQLiteLoader { @@ -85,27 +93,56 @@ pipeline ElectricVehiclesPipeline { block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { } + valuetype ElectricVehicle { + property vin oftype VehicleIdentificationNumber10; + property county oftype text; + property city oftype text; + property state oftype text; + property postal oftype text; + property modelYear oftype integer; + property make oftype text; + property model oftype text; + property evType oftype text; + property cafvEligibility oftype text; + property electricRange oftype integer; + property baseMSRP oftype integer; + property legislativeDistrict oftype text; + property dolID oftype integer; + property location oftype text; + property utility oftype text; + property censusTract oftype text; + } + + transform ElectricVehicleParser { + from r oftype Collection; + to ev oftype ElectricVehicle; + + ev: { + vin: r cellInColumn "VIN (1-10)", + county: asText (r cellInColumn "County"), + city: asText (r cellInColumn "City"), + state: asText (r cellInColumn "State"), + postal: asText (r cellInColumn "Postal Code"), + modelYear: asInteger (r cellInColumn "Model Year"), + make: asText (r cellInColumn "Make"), + model: asText (r cellInColumn "Model"), + evType: asText (r cellInColumn "Electric Vehicle Type"), + cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"), + electricRange: asInteger (r cellInColumn "Electric Range"), + baseMSRP: asInteger (r cellInColumn "Base MSRP"), + legislativeDistrict: asText (r cellInColumn "Legislative District"), + dolID: asInteger (r cellInColumn "DOL Vehicle ID"), + location: asText (r cellInColumn "Vehicle Location"), + utility: asText (r cellInColumn "Electric Utility"), + censusTract: asText (r cellInColumn "2020 Census Tract"), + }; + } + + block ElectricVehiclesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "VIN (1-10)" oftype VehicleIdentificationNumber10, - "County" oftype text, - "City" oftype text, - "State" oftype text, - "Postal Code" oftype text, - "Model Year" oftype integer, - "Make" oftype text, - "Model" oftype text, - "Electric Vehicle Type" oftype text, - "Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text, - "Electric Range" oftype integer, - "Base MSRP" oftype integer, - "Legislative District" oftype text, - "DOL Vehicle ID" oftype integer, - "Vehicle Location" oftype text, - "Electric Utility" oftype text, - "2020 Census Tract" oftype text, - ]; + columns: ElectricVehicle; + parseWith: ElectricVehicleParser; } block ElectricRangeTransformer oftype TableTransformer { diff --git a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv index 1538a399d..115a62157 100644 --- a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv +++ b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv @@ -28,14 +28,46 @@ pipeline CarsPipeline { ]; } - valuetype CarName { + valuetype Car { property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; } - block CarsTableInterpreter oftype TableInterpreter { - header: true; - columns: CarName; - } + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + + block CarsTableInterpreter oftype TableInterpreter { + header: true; + columns: Car; + parseWith: CarParser; + } transform copy { from s oftype text; From 4aaec4db7acff92227dc10f1ec606e11da328f42 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:46:46 +0100 Subject: [PATCH 26/35] fix: adapt jayvee test assets --- apps/interpreter/test/assets/broken-model.jv | 53 ++++++++++++++------ 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/apps/interpreter/test/assets/broken-model.jv b/apps/interpreter/test/assets/broken-model.jv index 447eab7c2..ff20db91d 100644 --- a/apps/interpreter/test/assets/broken-model.jv +++ b/apps/interpreter/test/assets/broken-model.jv @@ -28,22 +28,45 @@ pipeline CarsPipeline { write: ["name"]; } + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer - ]; + columns: Car; + parseWith: CarParser; } block CarsLoader oftype SQLiteLoader { @@ -51,4 +74,4 @@ pipeline CarsPipeline { file: "./cars.sqlite"; } -} \ No newline at end of file +} From df931730cab609950a508a8d25cb4f5d1c9be30d Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:46:58 +0100 Subject: [PATCH 27/35] fix: adapt mobility domain from stdlib --- .../domain/mobility/GTFSAgencyInterpreter.jv | 26 ++++++++--- .../mobility/GTFSCalendarDatesInterpreter.jv | 25 +++++++--- .../mobility/GTFSCalendarInterpreter.jv | 46 +++++++++++++------ .../mobility/GTFSFareAttributesInterpreter.jv | 34 ++++++++++---- .../mobility/GTFSFareRulesInterpreter.jv | 29 +++++++++--- .../mobility/GTFSFrequenciesInterpreter.jv | 26 ++++++++--- .../domain/mobility/GTFSRoutesInterpreter.jv | 41 ++++++++++++----- .../domain/mobility/GTFSShapesInterpreter.jv | 29 +++++++++--- .../mobility/GTFSStopTimesInterpreter.jv | 41 ++++++++++++----- .../domain/mobility/GTFSStopsInterpreter.jv | 35 ++++++++++---- .../domain/mobility/GTFSTripsInterpreter.jv | 35 ++++++++++---- 11 files changed, 273 insertions(+), 94 deletions(-) diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv index afafbe0b3..7a9fafe83 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv @@ -2,6 +2,24 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Agency { + property agencyId oftype text; // Conditional columns are considered required for now + property agencyName oftype text; + property agencyUrl oftype GTFSUrl; + property agencyTimezone oftype text; +} +transform AgencyParser { + from r oftype Collection; + to agency oftype Agency; + + agency: { + agencyId: r cellInColumn "agency_id", + agencyName: r cellInColumn "agency_name", + agencyUrl: r cellInColumn "agency_url", + agencyTimezone: r cellInColumn "agency_timezone", + }; +} + /** * A GTFSAgencyInterpreter interprets a agency.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#agencytxt @@ -27,11 +45,7 @@ publish composite blocktype GTFSAgencyInterpreter { block AgencyTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "agency_id" oftype text, // Conditional columns are considered required for now - "agency_name" oftype text, - "agency_url" oftype GTFSUrl, - "agency_timezone" oftype text - ]; + columns: Agency; + parseWith: AgencyParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv index 2e25dcb8d..d1b10a125 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv @@ -2,6 +2,23 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype CalendarDates { + property serviceId oftype text; // Conditional columns are considered required for now + property date oftype GTFSDate; + property exceptionType oftype GTFSEnumOneOrTwo; // 1 - Service has been added for the specified date + // 2 - Service has been removed for the specified date. +} +transform CalendarDatesParser { + from r oftype Collection; + to dates oftype CalendarDates; + + dates: { + serviceId: r cellInColumn "service_id", + date: r cellInColumn "date", + exceptionType: r cellInColumn "exception_type", + }; +} + /** * A GTFSCalendarDatesInterpreter interprets a calendar_dates.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#calendar_datestxt @@ -27,11 +44,7 @@ publish composite blocktype GTFSCalendarDatesInterpreter { block CalendarDatesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "service_id" oftype text, - "date" oftype GTFSDate, - "exception_type" oftype GTFSEnumOneOrTwo // 1 - Service has been added for the specified date - // 2 - Service has been removed for the specified date. - ]; + columns: CalendarDates; + parseWith: CalendarDatesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv index a6176a102..29614539f 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv @@ -2,6 +2,37 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Calendar { + property serviceId oftype text; + property monday oftype GTFSEnumTwo; // 1 - Service is available for all Mondays in the date range. + // 0 - Service is not available for Mondays in the date range. + property tuesday oftype GTFSEnumTwo; + property wednesday oftype GTFSEnumTwo; + property thursday oftype GTFSEnumTwo; + property friday oftype GTFSEnumTwo; + property saturday oftype GTFSEnumTwo; + property sunday oftype GTFSEnumTwo; + property startDate oftype GTFSDate; + property endDate oftype GTFSDate; +} +transform CalendarParser { + from r oftype Collection; + to calendar oftype Calendar; + + calendar: { + serviceId: r cellInColumn 0, + monday: r cellInColumn 1, + tuesday: r cellInColumn 2, + wednesday: r cellInColumn 3, + thursday: r cellInColumn 4, + friday: r cellInColumn 5, + saturday: r cellInColumn 6, + sunday: r cellInColumn 7, + startDate: r cellInColumn 8, + endDate: r cellInColumn 9, + }; +} + /** * A GTFSCalendarInterpreter interprets a calendar.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#calendartxt @@ -27,18 +58,7 @@ publish composite blocktype GTFSCalendarInterpreter { block CalendarTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "service_id" oftype text, - "monday" oftype GTFSEnumTwo, // 1 - Service is available for all Mondays in the date range. - // 0 - Service is not available for Mondays in the date range. - "tuesday" oftype GTFSEnumTwo, - "wednesday" oftype GTFSEnumTwo, - "thursday" oftype GTFSEnumTwo, - "friday" oftype GTFSEnumTwo, - "saturday" oftype GTFSEnumTwo, - "sunday" oftype GTFSEnumTwo, - "start_date" oftype GTFSDate, - "end_date" oftype GTFSDate - ]; + columns: Calendar; + parseWith: CalendarParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv index abfba17f9..7b8f0d995 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv @@ -2,6 +2,29 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype FareAttributes { + property fareId oftype text; + property price oftype GTFSNonNegativeDecimal; + property currencyType oftype GTFSCurrency; + property paymentMethod oftype GTFSEnumTwo; // 0 - Fare is paid on board. + // 1 - Fare must be paid before boarding. + property transfers oftype text; // Is required but can be empty (?!) so has to be modelled as text... + property transferDuration oftype text; +} +transform FareAttributesParser { + from r oftype Collection; + to fareAttributes oftype FareAttributes; + + fareAttributes: { + fareId: r cellInColumn 0, + price: r cellInColumn 1, + currencyType: r cellInColumn 2, + paymentMethod: r cellInColumn 3, + transfers: r cellInColumn 4, + transferDuration: r cellInColumn 5, + }; +} + /** * A GTFSFareAttributesInterpreter interprets a fare_attributes.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#fare_attributestxt @@ -27,14 +50,7 @@ publish composite blocktype GTFSFareAttributesInterpreter { block FareAttributesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "fare_id" oftype text, - "price" oftype GTFSNonNegativeDecimal, - "currency_type" oftype GTFSCurrency, - "payment_method" oftype GTFSEnumTwo, // 0 - Fare is paid on board. - // 1 - Fare must be paid before boarding. - "transfers" oftype text, // Is required but can be empty (?!) so has to be modelled as text... - "transfer_duration" oftype text - ]; + columns: FareAttributes; + parseWith: FareAttributesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv index ce42b3d5f..d2e709a8e 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv @@ -2,6 +2,26 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype FareRules { + property fareId oftype text; + property routeId oftype text; + property originId oftype text; + property destinationId oftype text; + property containsId oftype text; +} +transform FareRulesParser { + from r oftype Collection; + to fareRules oftype FareRules; + + fareAttributes: { + fareId: r cellInColumn 0, + routeId: r cellInColumn 1, + originId: r cellInColumn 2, + destinationId: r cellInColumn 3, + containsId: r cellInColumn 4, + }; +} + /** * A GTFSFareRulesInterpreter interprets a fare_rules.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#fare_rulestxt @@ -27,12 +47,7 @@ publish composite blocktype GTFSFareRulesInterpreter { block FareRulesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "fare_id" oftype text, - "route_id" oftype text, - "origin_id" oftype text, - "destination_id" oftype text, - "contains_id" oftype text - ]; + columns: FareRules; + parseWith: FareRulesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv index a851292bf..d12e6c087 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv @@ -2,6 +2,24 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Frequencies { + property tripId oftype text; + property startTime oftype GTFSTime; + property endTime oftype GTFSTime; + property headwaySecs oftype GTFSNonNegativeInteger; +} +transform FrequenciesParser { + from r oftype Collection; + to frequencies oftype Frequencies; + + frequencies: { + tripId: r cellInColumn 0, + startTime: r cellInColumn 1, + endTime: r cellInColumn 2, + headwaySecs: r cellInColumn 3, + }; +} + /** * A GTFSFrequenciesInterpreter interprets a frequencies.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#frequenciestxt @@ -27,11 +45,7 @@ publish composite blocktype GTFSFrequenciesInterpreter { block FrequenciesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "trip_id" oftype text, - "start_time" oftype GTFSTime, - "end_time" oftype GTFSTime, - "headway_secs" oftype GTFSNonNegativeInteger - ]; + columns: Frequencies; + parseWith: FrequenciesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv index 36cc6227e..2d5a6b187 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv @@ -2,6 +2,34 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Route { + property routeId oftype text; + property agencyId oftype text; + property routeShortName oftype text; + property routeLongName oftype text; + property routeDesc oftype text; + property routeType oftype integer; // Technically is an enum from 0 - 12 + property routeUrl oftype text; + property routeColor oftype text; + property routeTextColor oftype text; +} +transform RouteParser { + from r oftype Collection; + to route oftype Route; + + route: { + routeId: r cellInColumn 0, + agencyId: r cellInColumn 1, + routeShortName: r cellInColumn 2, + routeLongName: r cellInColumn 3, + routeDesc: r cellInColumn 4, + routeType: r cellInColumn 5, + routeUrl: r cellInColumn 6, + routeColor: r cellInColumn 7, + routeTextColor: r cellInColumn 8, + }; +} + /** * A GTFSRoutesInterpreter interprets a routes.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#routestxt @@ -27,16 +55,7 @@ publish composite blocktype GTFSRoutesInterpreter { block RoutesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "route_id" oftype text, - "agency_id" oftype text, - "route_short_name" oftype text, - "route_long_name" oftype text, - "route_desc" oftype text, - "route_type" oftype integer, // Technically is an enum from 0 - 12 - "route_url" oftype text, - "route_color" oftype text, - "route_text_color" oftype text - ]; + columns: Route; + parseWith: RouteParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv index de2382829..2f0ce1e64 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv @@ -2,6 +2,26 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Shape { + property shapeId oftype text; + property shapePtLat oftype GTFSLatitude; + property shapePtLon oftype GTFSLongitude; + property shapePtSequence oftype GTFSNonNegativeInteger; + property shapeDistTraveled oftype text; +} +transform ShapeParser { + from r oftype Collection; + to shape oftype Shape; + + shape: { + shapeId: r cellInColumn 0, + shapePtLat: r cellInColumn 1, + shapePtLon: r cellInColumn 2, + shapePtSequence: r cellInColumn 3, + shapeDistTraveled: r cellInColumn 4, + }; +} + /** * A GTFSShapesInterpreter interprets a shapes.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#shapestxt @@ -27,12 +47,7 @@ publish composite blocktype GTFSShapesInterpreter { block ShapesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "shape_id" oftype text, - "shape_pt_lat" oftype GTFSLatitude, - "shape_pt_lon" oftype GTFSLongitude, - "shape_pt_sequence" oftype GTFSNonNegativeInteger, - "shape_dist_traveled" oftype text - ]; + columns: Shape; + parseWith: ShapeParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv index 289e61522..a0e894cad 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv @@ -2,6 +2,34 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype StopTime { + property tripId oftype text; + property arrivalId oftype GTFSTime; + property departureTime oftype GTFSTime; + property stopId oftype text; + property stopSequence oftype GTFSNonNegativeInteger; + property stopHeadsign oftype text; + property pickupType oftype text; + property dropOffTime oftype text; + property shapeDistTraveled oftype text; +} +transform StopTimeParser { + from r oftype Collection; + to stopTime oftype StopTime; + + stopTime: { + tripId: r cellInColumn 0, + arrivalId: r cellInColumn 1, + departureTime: r cellInColumn 2, + stopId: r cellInColumn 3, + stopSequence: r cellInColumn 4, + stopHeadsign: r cellInColumn 5, + pickupType: r cellInColumn 6, + dropOffTime: r cellInColumn 7, + shapeDistTraveled: r cellInColumn 8, + }; +} + /** * A GTFSStopTimesInterpreter interprets a stop_times.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#stop_timestxt @@ -27,16 +55,7 @@ publish composite blocktype GTFSStopTimesInterpreter { block StopTimesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "trip_id" oftype text, - "arrival_time" oftype GTFSTime, - "departure_time" oftype GTFSTime, - "stop_id" oftype text, - "stop_sequence" oftype GTFSNonNegativeInteger, - "stop_headsign" oftype text, - "pickup_type" oftype text, - "drop_off_time" oftype text, - "shape_dist_traveled" oftype text - ]; + columns: StopTime; + parseWith: StopTimeParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv index 9e9ea8323..ec37aa87f 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv @@ -2,6 +2,30 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Stop { + property stopId oftype text; + property stopName oftype text; + property stopDesc oftype text; + property stopLat oftype GTFSLatitude; + property stopLon oftype GTFSLongitude; + property zoneId oftype text; + property stopUrl oftype text; +} +transform StopParser { + from r oftype Collection; + to stop oftype Stop; + + stop: { + stopId: r cellInColumn 0, + stopName: r cellInColumn 1, + stopDesc: r cellInColumn 2, + stopLat: r cellInColumn 3, + stopLon: r cellInColumn 4, + zoneId: r cellInColumn 5, + stopUrl: r cellInColumn 6, + }; +} + /** * A GTFSStopsInterpreter interprets a stops.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#stopstxt @@ -27,14 +51,7 @@ publish composite blocktype GTFSStopsInterpreter { block StopsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "stop_id" oftype text, - "stop_name" oftype text, - "stop_desc" oftype text, - "stop_lat" oftype GTFSLatitude, - "stop_lon" oftype GTFSLongitude, - "zone_id" oftype text, - "stop_url" oftype text - ]; + columns: Stop; + parseWith: StopParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv index 11998662b..549a12b27 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv @@ -2,6 +2,30 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Trip { + property routeId oftype text; + property serviceId oftype text; + property tripId oftype text; + property tripHeadsign oftype text; + property directionId oftype text; + property blockId oftype text; + property shapeId oftype text; +} +transform TripParser { + from r oftype Collection; + to trip oftype Trip; + + trip: { + routeId: r cellInColumn 0, + serviceId: r cellInColumn 1, + tripId: r cellInColumn 2, + tripHeadsign: r cellInColumn 3, + directionId: r cellInColumn 4, + blockId: r cellInColumn 5, + shapeId: r cellInColumn 6, + }; +} + /** * A GTFSTripsInterpreter interprets a trips.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#tripstxt @@ -27,14 +51,7 @@ publish composite blocktype GTFSTripsInterpreter { block TripsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "route_id" oftype text, - "service_id" oftype text, - "trip_id" oftype text, - "trip_headsign" oftype text, - "direction_id" oftype text, - "block_id" oftype text, - "shape_id" oftype text - ]; + columns: Trip; + parseWith: TripParser; } } From 2eaa21515201581baf7b8aa86e9dfa48d39cabd8 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:49:01 +0100 Subject: [PATCH 28/35] fix: adapt examples --- example/cars.jv | 21 ++++++ example/electric-vehicles.jv | 74 ++++++++++++++------- example/gtfs-rt.jv | 124 +++++++++++++++++++++++++---------- 3 files changed, 161 insertions(+), 58 deletions(-) diff --git a/example/cars.jv b/example/cars.jv index 68122ceb8..4b28eefa4 100644 --- a/example/cars.jv +++ b/example/cars.jv @@ -88,12 +88,33 @@ pipeline CarsPipeline { property carb oftype integer; } + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + // 15. As a next step, we interpret the sheet as a table, using the valuetype // defined above. Rows that include values that are not valid according to the // their value types are dropped automatically. block CarsTableInterpreter oftype TableInterpreter { header: true; columns: Car; + parseWith: CarParser; } // 16. As a last step, we load the table into a sink, here into a sqlite file. diff --git a/example/electric-vehicles.jv b/example/electric-vehicles.jv index 6aa1163e3..91b05ff75 100644 --- a/example/electric-vehicles.jv +++ b/example/electric-vehicles.jv @@ -49,30 +49,56 @@ pipeline ElectricVehiclesPipeline { block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { } + valuetype ElectricVehicle { + property vin oftype VehicleIdentificationNumber10; + property county oftype text; + property city oftype text; + property state oftype UsStateCode; + property postal oftype text; + property modelYear oftype integer; + property make oftype text; + property model oftype text; + property evType oftype text; + property cafvEligibility oftype text; + property electricRange oftype integer; + property baseMSRP oftype integer; + property legislativeDistrict oftype text; + property dolID oftype integer; + property location oftype text; + property utility oftype text; + property censusTract oftype text; + } + + transform ElectricVehicleParser { + from r oftype Collection; + to ev oftype ElectricVehicle; + + ev: { + vin: r cellInColumn "VIN (1-10)", + county: asText (r cellInColumn "County"), + city: asText (r cellInColumn "City"), + state: asText (r cellInColumn "State"), + postal: asText (r cellInColumn "Postal Code"), + modelYear: asInteger (r cellInColumn "Model Year"), + make: asText (r cellInColumn "Make"), + model: asText (r cellInColumn "Model"), + evType: asText (r cellInColumn "Electric Vehicle Type"), + cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"), + electricRange: asInteger (r cellInColumn "Electric Range"), + baseMSRP: asInteger (r cellInColumn "Base MSRP"), + legislativeDistrict: asText (r cellInColumn "LegislativeDistrict"), + dolID: asInteger (r cellInColumn "DOL Vehicle ID"), + location: asText (r cellInColumn "Vehicle Location"), + utility: asText (r cellInColumn "Electric Utility"), + censusTract: asText (r cellInColumn "2020 Census Tract"), + }; + } + + block ElectricVehiclesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - // 4. Here, a user-deifned value type is used to describe this column. - // The capital letter indicates that the value type is not built-in - // by convention. The value type itself is defined further below. - "VIN (1-10)" oftype VehicleIdentificationNumber10, - "County" oftype text, - "City" oftype text, - "State" oftype UsStateCode, // We can just use the element as if it was defined in this file. - "Postal Code" oftype text, - "Model Year" oftype integer, - "Make" oftype text, - "Model" oftype text, - "Electric Vehicle Type" oftype text, - "Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text, - "Electric Range" oftype integer, - "Base MSRP" oftype integer, - "Legislative District" oftype text, - "DOL Vehicle ID" oftype integer, - "Vehicle Location" oftype text, - "Electric Utility" oftype text, - "2020 Census Tract" oftype text, - ]; + columns: ElectricVehicle; + parseWith: ElectricVehicleParser; } // 5. This block describes the application of a transform function @@ -81,9 +107,9 @@ pipeline ElectricVehiclesPipeline { // by the "use" property. block ElectricRangeTransformer oftype TableTransformer { inputColumns: [ - "Electric Range" + "electricRange" ]; - outputColumn: "Electric Range (km)"; + outputColumn: "electricRange (km)"; uses: MilesToKilometers; } diff --git a/example/gtfs-rt.jv b/example/gtfs-rt.jv index f1891d348..faa496a8d 100644 --- a/example/gtfs-rt.jv +++ b/example/gtfs-rt.jv @@ -58,49 +58,105 @@ pipeline GtfsRTSimplePipeline { } // 5. Next, we interpret the sheets as tables + valuetype TripUpdate { + property headerGtfsRealtimeVersion oftype text; + property headerTimestamp oftype text; + property headerIncrementality oftype text; + property entityId oftype text; + property entityTripUpdateTripTripId oftype text; + property entityTripUpdateTripRouteId oftype text; + property entityTripUpdateStopTimeUpdateStopSequence oftype text; + property entityTripUpdateStopTimeUpdateStopId oftype text; + property entityTripUpdateStopTimeUpdateArrivalTime oftype text; + property entityTripUpdateStopTimeUpdateDepartureTime oftype text; + } + transform TripUpdateParser { + from r oftype Collection; + to tripUpdate oftype TripUpdate; + + tripUpdate: { + headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version", + headerTimestamp: r cellInColumn "header.timestamp", + headerIncrementality: r cellInColumn "header.incrementality", + entityId: r cellInColumn "entity.id", + entityTripUpdateTripTripId: r cellInColumn "entity.trip_update.trip.trip_id", + entityTripUpdateTripRouteId: r cellInColumn "entity.trip_update.trip.route_id", + entityTripUpdateStopTimeUpdateStopSequence: r cellInColumn "entity.trip_update.stop_time_update.stop_sequence", + entityTripUpdateStopTimeUpdateStopId: r cellInColumn "entity.trip_update.stop_time_update.stop_id", + entityTripUpdateStopTimeUpdateArrivalTime: r cellInColumn "entity.trip_update.stop_time_update.arrival.time", + entityTripUpdateStopTimeUpdateDepartureTime: r cellInColumn "entity.trip_update.stop_time_update.departure.time", + }; + } block TripUpdateTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "header.gtfs_realtime_version" oftype text, - "header.timestamp" oftype text, - "header.incrementality" oftype text, - "entity.id" oftype text, - "entity.trip_update.trip.trip_id" oftype text, - "entity.trip_update.trip.route_id" oftype text, - "entity.trip_update.stop_time_update.stop_sequence" oftype text, - "entity.trip_update.stop_time_update.stop_id" oftype text, - "entity.trip_update.stop_time_update.arrival.time" oftype text, - "entity.trip_update.stop_time_update.departure.time" oftype text, - ]; + columns: TripUpdate; + parseWith: TripUpdateParser; } + + valuetype VehiclePosition { + property headerGtfsRealtimeVersion oftype text; + property headerTimestamp oftype text; + property headerIncrementality oftype text; + property entityId oftype text; + property entityVehiclePositionVehicleDescriptorId oftype text; + property entityVehiclePositionTripTripId oftype text; + property entityVehiclePositionTripRouteId oftype text; + property entityVehiclePositionPositionLatitude oftype text; + property entityVehiclePositionPositionLongitude oftype text; + property entityVehiclePositionTimestamp oftype text; + } + transform VehiclePositionParser { + from r oftype Collection; + to vehiclePosition oftype VehiclePosition; + + vehiclePosition: { + headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version", + headerTimestamp: r cellInColumn "header.timestamp", + headerIncrementality: r cellInColumn "header.incrementality", + entityId: r cellInColumn "entity.id", + entityVehiclePositionVehicleDescriptorId: r cellInColumn "entity.vehicle_position.vehicle_descriptor.id", + entityVehiclePositionTripTripId: r cellInColumn "entity.vehicle_position.trip.trip_id", + entityVehiclePositionTripRouteId: r cellInColumn "entity.vehicle_position.trip.route_id", + entityVehiclePositionPositionLatitude: r cellInColumn "entity.vehicle_position.position.latitude", + entityVehiclePositionPositionLongitude: r cellInColumn "entity.vehicle_position.position.longitude", + entityVehiclePositionTimestamp: r cellInColumn "entity.vehicle_position.timestamp", + }; + } block VehiclePositionTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "header.gtfs_realtime_version" oftype text, - "header.timestamp" oftype text, - "header.incrementality" oftype text, - "entity.id" oftype text, - "entity.vehicle_position.vehicle_descriptor.id" oftype text, - "entity.vehicle_position.trip.trip_id" oftype text, - "entity.vehicle_position.trip.route_id" oftype text, - "entity.vehicle_position.position.latitude" oftype text, - "entity.vehicle_position.position.longitude" oftype text, - "entity.vehicle_position.timestamp" oftype text - ]; + columns: VehiclePosition; + parseWith: VehiclePositionParser; } + + valuetype Alert { + property headerGtfsRealtimeVersion oftype text; + property headerTimestamp oftype text; + property headerIncrementality oftype text; + property entityId oftype text; + property entityAlertInformedEntityRouteId oftype text; + property entityAlertHeaderText oftype text; + property entityAlertDescriptionText oftype text; + } + transform AlertParser { + from r oftype Collection; + to alert oftype Alert; + + alert: { + headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version", + headerTimestamp: r cellInColumn "header.timestamp", + headerIncrementality: r cellInColumn "header.incrementality", + entityId: r cellInColumn "entity.id", + entityAlertInformedEntityRouteId: r cellInColumn "entity.alert.informed_entity.route_id", + entityAlertHeaderText: r cellInColumn "entity.alert.header_text", + entityAlertDescriptionText: r cellInColumn "entity.alert.description_text", + }; + } block AlertTableInterpreter oftype TableInterpreter { header: true; - columns: [ - 'header.gtfs_realtime_version' oftype text, - 'header.timestamp' oftype text, - 'header.incrementality' oftype text, - 'entity.id' oftype text, - 'entity.alert.informed_entity.route_id' oftype text, - 'entity.alert.header_text' oftype text, - 'entity.alert.description_text' oftype text, - ]; + columns: Alert; + parseWith: AlertParser; } // 6. Last, we load the tables into the same SQLite file. @@ -124,4 +180,4 @@ pipeline GtfsRTSimplePipeline { file: "./gtfs.sqlite"; dropTable: false; } -} \ No newline at end of file +} From 9735c5aaecd3f57b451c078848a87a3076e114b3 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Wed, 7 Jan 2026 09:56:13 +0100 Subject: [PATCH 29/35] fix: remove unused `TableRowValueType` --- .../internal-representation-parsing.ts | 4 -- .../value-representation-validity.ts | 5 -- .../visitors/sql-column-type-visitor.ts | 6 -- .../sql-value-representation-visitor.ts | 8 --- .../wrappers/value-type/primitive/index.ts | 1 - .../primitive-value-type-provider.ts | 5 -- .../primitive/table-row-value-type.ts | 58 ------------------- .../lib/ast/wrappers/value-type/value-type.ts | 2 - 8 files changed, 89 deletions(-) delete mode 100644 libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts diff --git a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts index 3d15112a9..464da9f32 100644 --- a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts +++ b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts @@ -124,10 +124,6 @@ class InternalRepresentationParserVisitor extends ValueTypeVisitor< return new InvalidValue(`Cannot parse collections into internal values`); } - override visitTableRow(): InvalidValue { - return new InvalidValue(`Cannot parse table rows into internal values`); - } - override visitConstraint(): InvalidValue { return new InvalidValue(`Cannot parse constraints into internal values`); } diff --git a/libs/execution/src/lib/types/value-types/value-representation-validity.ts b/libs/execution/src/lib/types/value-types/value-representation-validity.ts index f918acf11..c6b87d8d9 100644 --- a/libs/execution/src/lib/types/value-types/value-representation-validity.ts +++ b/libs/execution/src/lib/types/value-types/value-representation-validity.ts @@ -6,7 +6,6 @@ import { strict as assert } from 'assert'; import { - type TableRowValueType, type AtomicValueType, type BooleanValuetype, type CellRangeValuetype, @@ -131,10 +130,6 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { return this.isValidForPrimitiveValuetype(valueType); } - override visitTableRow(valueType: TableRowValueType): boolean { - return this.isValidForPrimitiveValuetype(valueType); - } - override visitTransform(valueType: TransformValuetype): boolean { return this.isValidForPrimitiveValuetype(valueType); } diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts index 3c06bdd87..f47a92e3e 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts @@ -76,12 +76,6 @@ export class SQLColumnTypeVisitor extends ValueTypeVisitor { ); } - override visitTableRow(): string { - throw new Error( - 'No visit implementation given for table rows. Cannot be the type of a column.', - ); - } - override visitTransform(): string { throw new Error( 'No visit implementation given for transforms. Cannot be the type of a column.', diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts index f7ad91bb9..9b6bdba78 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts @@ -130,14 +130,6 @@ export class SQLValueRepresentationVisitor extends ValueTypeVisitor< ); } - override visitTableRow(): ( - value: InternalValidValueRepresentation | InternalErrorValueRepresentation, - ) => string { - throw new Error( - 'No visit implementation given for table rows. Cannot be the type of a column.', - ); - } - override visitTransform(): ( value: InternalValidValueRepresentation | InternalErrorValueRepresentation, ) => string { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts index 6afc09a45..103301814 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts @@ -18,7 +18,6 @@ export { type ConstraintValuetype } from './constraint-value-type'; export { type DecimalValuetype } from './decimal-value-type'; export { type IntegerValuetype } from './integer-value-type'; export { type RegexValuetype } from './regex-value-type'; -export { type TableRowValueType } from './table-row-value-type'; export { type TextValuetype } from './text-value-type'; export { type TransformValuetype } from './transform-value-type'; export { type ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts index 7b2e23d4c..3563b78a0 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts @@ -18,7 +18,6 @@ import { TextValuetype } from './text-value-type'; import { TransformValuetype } from './transform-value-type'; import { ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; import { ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; -import { TableRowValueType } from './table-row-value-type'; /** * Should be created as singleton due to the equality comparison of primitive value types. @@ -33,10 +32,6 @@ export class ValueTypeProvider { ): CollectionValueType { return new CollectionValueType(input); } - - createTableRowValueTypeOf(input: Map): TableRowValueType { - return new TableRowValueType(input); - } } export class PrimitiveValueTypeProvider { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts deleted file mode 100644 index 9f6cb59ab..000000000 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/table-row-value-type.ts +++ /dev/null @@ -1,58 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -import { - type TableRow, - type InternalValidValueRepresentation, -} from '../../../expressions/internal-value-representation'; -import { type ValueType, type ValueTypeVisitor } from '../value-type'; - -import { PrimitiveValueType } from './primitive-value-type'; -import { TABLEROW_TYPEGUARD } from '../../../expressions'; - -export class TableRowValueType extends PrimitiveValueType { - constructor(private schema: Map) { - super(); - } - - acceptVisitor(visitor: ValueTypeVisitor): R { - return visitor.visitTableRow(this); - } - - override isAllowedAsRuntimeParameter(): boolean { - return false; - } - - override getName(): 'TableRow' { - return 'TableRow'; - } - - private equalSchema(otherSchema: Map): boolean { - if (this.schema.size !== otherSchema.size) { - return false; - } - for (const [columnName, cellValue] of this.schema) { - const otherCellValue = otherSchema.get(columnName); - if (cellValue !== otherCellValue) { - return false; - } - if (otherCellValue === undefined && !otherSchema.has(columnName)) { - return false; - } - } - return true; - } - - override equals(target: ValueType): boolean { - return ( - target instanceof TableRowValueType && this.equalSchema(target.schema) - ); - } - - override isInternalValidValueRepresentation( - operandValue: InternalValidValueRepresentation, - ): operandValue is TableRow { - return TABLEROW_TYPEGUARD(operandValue); - } -} diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts index 1e2743e86..dab5cf62a 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts @@ -13,7 +13,6 @@ import { type BooleanValuetype, type CellRangeValuetype, type CollectionValueType, - type TableRowValueType, type ConstraintValuetype, type DecimalValuetype, type EmptyCollectionValueType, @@ -91,7 +90,6 @@ export abstract class ValueTypeVisitor { abstract visitCollection( valueType: CollectionValueType | EmptyCollectionValueType, ): R; - abstract visitTableRow(valueType: TableRowValueType): R; abstract visitTransform(valueType: TransformValuetype): R; abstract visitAtomicValueType(valueType: AtomicValueType): R; From cb9e077913984482b80d10d01e53a18b04151079 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Mon, 12 Jan 2026 10:44:59 +0100 Subject: [PATCH 30/35] refactor: review nitpicks Co-authored-by: Georg Schwarz --- libs/execution/src/lib/types/io-types/table.ts | 10 +++++----- .../exec/src/lib/table-interpreter-executor.ts | 2 +- .../exec/src/lib/table-transformer-executor.ts | 2 +- .../test/assets/graph/two-pipelines.jv | 15 ++++++++++++++- .../wrappers/typed-object/block-type-wrapper.ts | 10 +++------- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index 93ec929b2..ce8544730 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -49,11 +49,11 @@ export class Table implements IOTypeImplementation { public constructor( private numberOfRows: number, private columns: Map, - private constraints: ConstraintExecutor[], + private constraintExecutors: ConstraintExecutor[], ) { assert(this.numberOfRows !== undefined); assert(this.columns !== undefined); - assert(this.constraints !== undefined); + assert(this.constraintExecutors !== undefined); } addColumn(name: string, column: TableColumn): void { @@ -151,7 +151,7 @@ export class Table implements IOTypeImplementation { } } - findUnfullfilledRows( + forEachUnfulfilledRow( onInvalidRow: ( constraint: ConstraintExecutor, rowIndex: number, @@ -163,7 +163,7 @@ export class Table implements IOTypeImplementation { const row = this.getRow(rowIdx); assert(row !== undefined); - for (const constraint of this.constraints) { + for (const constraint of this.constraintExecutors) { if (constraint.isValid(row, executionContext)) { continue; } @@ -245,7 +245,7 @@ export class Table implements IOTypeImplementation { }); }); - const copiedConstraints = this.constraints.map( + const copiedConstraints = this.constraintExecutors.map( (constraint) => new ConstraintExecutor(constraint.astNode), ); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index 1500ca13e..c92cbe1f4 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -177,7 +177,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< }); // eslint-disable-next-line @typescript-eslint/no-unused-vars - table.findUnfullfilledRows((constraint, rowIdx, _row) => { + table.forEachUnfulfilledRow((constraint, rowIdx, _row) => { context.logger.logErr( `Invalid constraint ${constraint.name} on row ${rowIdx}`, ); diff --git a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts index a8443e7a3..0c67e1dc7 100644 --- a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts @@ -98,7 +98,7 @@ export class TableTransformerExecutor extends AbstractBlockExecutor< ); // eslint-disable-next-line @typescript-eslint/no-unused-vars - outputTable.findUnfullfilledRows((constraint, rowIdx, _row) => { + outputTable.forEachUnfulfilledRow((constraint, rowIdx, _row) => { context.logger.logErr( `Invalid constraint ${constraint.name} on row ${rowIdx}`, ); diff --git a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv index cd9c9f41e..c31ce3229 100644 --- a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv +++ b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv @@ -50,7 +50,20 @@ pipeline CarsPipeline { from r oftype Collection; to car oftype Car; - car: { name: asText (r cellInColumn "name"), mpg: asDecimal (r cellInColumn "mpg"), cyl: asInteger (r cellInColumn 2), disp: asDecimal (r cellInColumn 3), hp: asInteger (r cellInColumn "hp"), drat: asDecimal (r cellInColumn "drat"), wt: asDecimal (r cellInColumn "wt"), qsec: asDecimal (r cellInColumn "qsec"), vs: asInteger (r cellInColumn "vs"), am: asInteger (r cellInColumn "am"), gear: asInteger (r cellInColumn "gear"), carb: asInteger (r cellInColumn "carb") }; + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; } block CarsTableInterpreter oftype TableInterpreter { diff --git a/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts b/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts index e6279f9a6..b813b6ec9 100644 --- a/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts +++ b/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts @@ -119,13 +119,9 @@ export class BlockTypeWrapper extends TypedObjectWrapper { - if (property.valueType.reference.ref === undefined) { - console.log(property.valueType.reference.error); - process.exit(1); - } - return property.valueType.reference.ref === undefined; - }) + blockTypeDefinition.properties.some( + (property) => property.valueType.reference.ref === undefined, + ) ) { return false; } From a227a7a5fbc11c4afde1123c8a6277e46fffae7c Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Mon, 12 Jan 2026 11:12:35 +0100 Subject: [PATCH 31/35] refactor: replace method signature only used by tests with a wrapper --- .../src/lib/types/io-types/table.spec.ts | 37 +++++++++++++++---- .../execution/src/lib/types/io-types/table.ts | 28 +++----------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/libs/execution/src/lib/types/io-types/table.spec.ts b/libs/execution/src/lib/types/io-types/table.spec.ts index 2b9d1669c..c02bd34ff 100644 --- a/libs/execution/src/lib/types/io-types/table.spec.ts +++ b/libs/execution/src/lib/types/io-types/table.spec.ts @@ -2,10 +2,33 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { ValueTypeProvider } from '@jvalue/jayvee-language-server'; +import { + type InternalErrorValueRepresentation, + type InternalValidValueRepresentation, + ValueTypeProvider, +} from '@jvalue/jayvee-language-server'; import { Table, type TableColumn } from './table'; +function addRowWrapper( + table: Table, + row: Record< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, +): void { + const tableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); + + for (const [columnName, cellValue] of Object.entries(row)) { + tableRow.set(columnName, cellValue); + } + + return table.addRow(tableRow); +} + describe('Table', () => { let table: Table; let valueTypeProvider: ValueTypeProvider; @@ -36,9 +59,9 @@ describe('Table', () => { valueType: valueTypeProvider.Primitives.Text, values: [], }); - table.addRow({ a: 'a1' }); - table.addRow({ a: 'a2' }); - table.addRow({ a: 'a3' }); + addRowWrapper(table, { a: 'a1' }); + addRowWrapper(table, { a: 'a2' }); + addRowWrapper(table, { a: 'a3' }); table.addColumn('b', { valueType: valueTypeProvider.Primitives.Text, values: ['b1', 'b2', 'b3'], @@ -52,9 +75,9 @@ describe('Table', () => { valueType: valueTypeProvider.Primitives.Text, values: [], }); - table.addRow({ a: 'a1' }); - table.addRow({ a: 'a2' }); - table.addRow({ a: 'a3' }); + addRowWrapper(table, { a: 'a1' }); + addRowWrapper(table, { a: 'a2' }); + addRowWrapper(table, { a: 'a3' }); expect(table.getNumberOfRows()).toBe(3); }); diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index ce8544730..f2a79b9a8 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -66,35 +66,17 @@ export class Table implements IOTypeImplementation { * NOTE: This method will only add the row if the table has at least one column! * @param row data of this row for each column */ - addRow( - row: Record< - string, - InternalValidValueRepresentation | InternalErrorValueRepresentation - >, - ): void; - addRow(row: TableRow): void; - addRow( - row: - | TableRow - | Record< - string, - InternalValidValueRepresentation | InternalErrorValueRepresentation - >, - ): void { - const rowLength = row instanceof Map ? row.size : Object.keys(row).length; + addRow(row: TableRow): void { assert( - rowLength === this.columns.size, - `Added row has the wrong dimension (expected: ${this.columns.size}, actual: ${rowLength})`, + row.size === this.columns.size, + `Added row has the wrong dimension (expected: ${this.columns.size}, actual: ${row.size})`, ); - if (rowLength > 0) { + if (row.size > 0) { this.numberOfRows++; } - const rowValues = - row instanceof Map ? [...row.entries()] : Object.entries(row); - - for (const [columnName, cellValue] of rowValues) { + for (const [columnName, cellValue] of row.entries()) { const column = this.columns.get(columnName); assert(column !== undefined, 'All added rows fit columns in the table'); From bb7887d3e6033f5b63dba68931d036a4043367f2 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Mon, 12 Jan 2026 15:14:04 +0100 Subject: [PATCH 32/35] refactor: remove confusing convienience methods Co-authored-by: Georg Schwarz --- libs/execution/src/lib/types/io-types/sheet.ts | 8 -------- .../exec/src/lib/row-deleter-executor.spec.ts | 6 +++--- .../exec/src/lib/table-interpreter-executor.ts | 15 ++++++++------- 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/libs/execution/src/lib/types/io-types/sheet.ts b/libs/execution/src/lib/types/io-types/sheet.ts index bd1f84a8c..099a87537 100644 --- a/libs/execution/src/lib/types/io-types/sheet.ts +++ b/libs/execution/src/lib/types/io-types/sheet.ts @@ -45,14 +45,6 @@ export class Sheet implements IOTypeImplementation { return this.numberOfColumns; } - popHeaderRow(): string[] | undefined { - return this.data.shift(); - } - - iterateRows(callbackFn: (row: string[], rowIndex: number) => void) { - this.data.forEach(callbackFn); - } - clone(): Sheet { return new Sheet(structuredClone(this.data)); } diff --git a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts index 7dd9cb114..f3e2401f5 100644 --- a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts @@ -96,7 +96,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(15); - expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()[0]).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([['0', 'Test', 'true']]), ); @@ -117,7 +117,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(14); - expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()[0]).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], @@ -158,7 +158,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(15); - expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()[0]).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([['0', 'Test', 'true']]), ); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index c92cbe1f4..5b7d1988b 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -72,9 +72,10 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< const schema = columnsValueType.getSchema(); + const sheetData = inputSheet.getData().slice(); let headerRow: string[] | undefined = undefined; if (header) { - headerRow = inputSheet.popHeaderRow(); + headerRow = sheetData.shift()?.slice(); if (headerRow === undefined) { return R.err({ message: 'The input sheet is empty and thus has no header', @@ -113,7 +114,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< const parseRowTransform = new TransformExecutor(parseWith, context); const resultingTable = this.constructAndValidateTable( - inputSheet, + sheetData, headerRow, columnsValueType, parseRowTransform, @@ -126,7 +127,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< } private constructAndValidateTable( - sheet: Sheet, + sheetData: (readonly string[])[], headerRow: string[] | undefined, columnsValueType: AtomicValueType, parseRowTransform: TransformExecutor, @@ -161,7 +162,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< const table = new Table(0, columns, constraints); // add rows - sheet.iterateRows((sheetRow) => { + for (const sheetRow of sheetData) { const tableRow = this.constructAndValidateTableRow( sheetRowReferenceName, sheetRow, @@ -174,7 +175,7 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< } else { table.addRow(tableRow); } - }); + } // eslint-disable-next-line @typescript-eslint/no-unused-vars table.forEachUnfulfilledRow((constraint, rowIdx, _row) => { @@ -189,14 +190,14 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< private constructAndValidateTableRow( sheetRowReferenceName: string, - sheetRow: string[], + sheetRow: readonly string[], schema: Map, parseRowExpression: TableRowLiteral, context: ExecutionContext, ): R.TableRow | MissingValue { context.evaluationContext.setValueForReference( sheetRowReferenceName, - sheetRow, + sheetRow.slice(), ); const tableRow = evaluateExpression( From 740016638fbe901704cad52d108aed51f0f091ef Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Thu, 15 Jan 2026 15:11:58 +0100 Subject: [PATCH 33/35] chore: remove obsolete test file --- .../valid-without-header-without-trim.jv | 34 ------------------- 1 file changed, 34 deletions(-) delete mode 100644 libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv deleted file mode 100644 index 37d9b00f3..000000000 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv +++ /dev/null @@ -1,34 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline TestPipeline { - - block TestExtractor oftype TestSheetExtractor { } - - valuetype TestValueType { - property index oftype integer; - property name oftype text; - property flag oftype boolean; - } - - transform Parser { - from r oftype Collection; - to t oftype TestValueType; - - t: { index: asInteger (r cellInColumn "index"), name: r cellInColumn "name", - flag: asBoolean (r cellInColumn "flag"), }; - } - - block TestBlock oftype TableInterpreter { - header: false; - columns: TestValueType; - parseWith: Parser; - } - - block TestLoader oftype TestTableLoader { } - - TestExtractor - -> TestBlock - -> TestLoader; -} From cf21ffce8a75ccb46a592566b017a007e5463230 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Thu, 15 Jan 2026 15:14:07 +0100 Subject: [PATCH 34/35] fix: incorrect output assignment Co-authored-by: Copilot --- .../src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv index d2e709a8e..3f7c3af98 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv @@ -13,7 +13,7 @@ transform FareRulesParser { from r oftype Collection; to fareRules oftype FareRules; - fareAttributes: { + fareRules: { fareId: r cellInColumn 0, routeId: r cellInColumn 1, originId: r cellInColumn 2, From 99ab83fe91ac7be8efa90f06d48daa92f8b51537 Mon Sep 17 00:00:00 2001 From: Jonas Zeltner Date: Thu, 15 Jan 2026 15:18:25 +0100 Subject: [PATCH 35/35] chore: typos Co-authored-by: Copilot --- .../expressions/evaluators/cell-in-column-operator-evaluator.ts | 2 +- .../lib/validation/checks/block-type-specific/property-body.ts | 2 +- .../language-server/src/lib/validation/checks/transform-body.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts index cd2612d6e..237c687db 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts @@ -96,7 +96,7 @@ export class CellInColumnOperatorEvaluator if (index >= sheetRowValues.length) { return new InvalidValue( - `Cannot column ${index} in a sheet with only ${sheetRowValues.length} columns`, + `Cannot access column ${index} in a sheet with only ${sheetRowValues.length} columns`, ); } const value = sheetRowValues.at(index); diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts index 4b373a609..5468ef552 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts @@ -212,7 +212,7 @@ function checkParseWithTransform( if (!isTableRowLiteral(outputAssignment.expression)) { props.validationContext.accept( 'error', - 'Transforms used in TableInterpreter blocks must have one output assignmet using a table row expression', + 'Transforms used in TableInterpreter blocks must have one output assignment using a table row expression', { node: outputAssignment.expression, }, diff --git a/libs/language-server/src/lib/validation/checks/transform-body.ts b/libs/language-server/src/lib/validation/checks/transform-body.ts index 04769e8e1..d52eff947 100644 --- a/libs/language-server/src/lib/validation/checks/transform-body.ts +++ b/libs/language-server/src/lib/validation/checks/transform-body.ts @@ -191,7 +191,7 @@ function checkOutputForTableRowTransform( if (outputValueType === undefined) { props.validationContext.accept( 'error', - 'Transforms with a table row expression must have exactly one ouptut', + 'Transforms with a table row expression must have exactly one output', { node: transformBody, },