diff --git a/apps/interpreter/test/assets/broken-model.jv b/apps/interpreter/test/assets/broken-model.jv index 447eab7c2..ff20db91d 100644 --- a/apps/interpreter/test/assets/broken-model.jv +++ b/apps/interpreter/test/assets/broken-model.jv @@ -28,22 +28,45 @@ pipeline CarsPipeline { write: ["name"]; } + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer - ]; + columns: Car; + parseWith: CarParser; } block CarsLoader oftype SQLiteLoader { @@ -51,4 +74,4 @@ pipeline CarsPipeline { file: "./cars.sqlite"; } -} \ No newline at end of file +} diff --git a/example/cars.jv b/example/cars.jv index 644cea639..4b28eefa4 100644 --- a/example/cars.jv +++ b/example/cars.jv @@ -7,14 +7,12 @@ // - Understand the core concepts pipeline, block, and pipe // - Understand the general structure of a pipeline -// 1. This Jayvee model describes a pipeline -// from a CSV file in the web -// to a SQLite file sink. +// 1. This Jayvee model describes a pipeline from a CSV file in the web to a +// SQLite file sink. pipeline CarsPipeline { - // 2. We describe the structure of the pipeline, - // usually at the top of the pipeline. - // by connecting blocks via pipes. + // 2. We describe the structure of the pipeline, usually at the top of the + // pipeline, by connecting blocks via pipes. // 3. Syntax of a pipe // connecting the block CarsExtractor @@ -22,8 +20,8 @@ pipeline CarsPipeline { CarsExtractor -> CarsTextFileInterpreter; - // 4. The output of the preceding block is hereby used - // as input for the succeeding block. + // 4. The output of the preceding block is hereby used as input for the + // succeeding block. // 5. Pipes can be further chained, // leading to an overview of the pipeline. @@ -34,12 +32,12 @@ pipeline CarsPipeline { -> CarsLoader; - // 6. Below the pipes, we usually define the blocks - // that are connected by the pipes. + // 6. Below the pipes, we usually define the blocks that are connected by the + // pipes. // 7. Blocks instantiate a block type by using the oftype keyword. - // The block type defines the available properties that the block - // can use to specify the intended behavior of the block + // The block type defines the available properties that can be used to specify + // the intended behavior of the block. block CarsExtractor oftype HttpExtractor { // 8. Properties are assigned to concrete values. @@ -47,12 +45,13 @@ pipeline CarsPipeline { url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv"; } - // 9. The HttpExtractor requires no input and produces a binary file as output. - // This file has to be interpreted, e.g., as text file. + // 9. The HttpExtractor requires no input and produces a binary file as + // output. This file has to be interpreted, e.g., as text file. block CarsTextFileInterpreter oftype TextFileInterpreter { } // 10. Next, we interpret the text file as sheet. - // A sheet only contains text cells and is useful for manipulating the shape of data before assigning more strict value types to cells. + // A sheet only contains text cells and is useful for manipulating the shape + // of data before assigning more strict value types to cells. block CarsCSVInterpreter oftype CSVInterpreter { enclosing: '"'; } @@ -60,48 +59,73 @@ pipeline CarsPipeline { // 11. We can write into cells of a sheet using the CellWriter block type. block NameHeaderWriter oftype CellWriter { // 12. We utilize a syntax similar to spreadsheet programs. - // Cell ranges can be described using the keywords "cell", "row", "column", or "range" that indicate which - // cells are selected for the write action. + // Cell ranges can be described using the keywords "cell", "row", "column", + // or "range" that indicate which cells are selected for the write action. at: cell A1; - // 13. For each cell we selected with the "at" property above, - // we can specify what value shall be written into the cell. + // 13. For each cell we selected with the "at" property above, we can + // specify what value shall be written into the cell. write: [ "name" ]; } - // 14. As a next step, we interpret the sheet as a table by adding structure. - // We define a value type per column that specifies the data type of the column. - // Rows that include values that are not valid according to the their value types are dropped automatically. + // 14. Next, we define the schema of our table. + // For this, we use a value type where each property corresponds to a column + // in the table. The column will have the same valuetype as the property. + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + + // 15. As a next step, we interpret the sheet as a table, using the valuetype + // defined above. Rows that include values that are not valid according to the + // their value types are dropped automatically. block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer - ]; + columns: Car; + parseWith: CarParser; } - // 15. As a last step, we load the table into a sink, - // here into a sqlite file. - // The structural information of the table is used - // to generate the correct table. + // 16. As a last step, we load the table into a sink, here into a sqlite file. + // The structural information of the table is used to generate the correct + // table. block CarsLoader oftype SQLiteLoader { table: "Cars"; file: "./cars.sqlite"; } - // 16. Congratulations! - // You can now use the sink for your data analysis, app, - // or whatever you want to do with the cleaned data. -} \ No newline at end of file + // 17. Congratulations! + // You can now use the sink for your data analysis, app, or whatever you want + // to do with the cleaned data. +} diff --git a/example/electric-vehicles.jv b/example/electric-vehicles.jv index 6aa1163e3..91b05ff75 100644 --- a/example/electric-vehicles.jv +++ b/example/electric-vehicles.jv @@ -49,30 +49,56 @@ pipeline ElectricVehiclesPipeline { block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { } + valuetype ElectricVehicle { + property vin oftype VehicleIdentificationNumber10; + property county oftype text; + property city oftype text; + property state oftype UsStateCode; + property postal oftype text; + property modelYear oftype integer; + property make oftype text; + property model oftype text; + property evType oftype text; + property cafvEligibility oftype text; + property electricRange oftype integer; + property baseMSRP oftype integer; + property legislativeDistrict oftype text; + property dolID oftype integer; + property location oftype text; + property utility oftype text; + property censusTract oftype text; + } + + transform ElectricVehicleParser { + from r oftype Collection; + to ev oftype ElectricVehicle; + + ev: { + vin: r cellInColumn "VIN (1-10)", + county: asText (r cellInColumn "County"), + city: asText (r cellInColumn "City"), + state: asText (r cellInColumn "State"), + postal: asText (r cellInColumn "Postal Code"), + modelYear: asInteger (r cellInColumn "Model Year"), + make: asText (r cellInColumn "Make"), + model: asText (r cellInColumn "Model"), + evType: asText (r cellInColumn "Electric Vehicle Type"), + cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"), + electricRange: asInteger (r cellInColumn "Electric Range"), + baseMSRP: asInteger (r cellInColumn "Base MSRP"), + legislativeDistrict: asText (r cellInColumn "LegislativeDistrict"), + dolID: asInteger (r cellInColumn "DOL Vehicle ID"), + location: asText (r cellInColumn "Vehicle Location"), + utility: asText (r cellInColumn "Electric Utility"), + censusTract: asText (r cellInColumn "2020 Census Tract"), + }; + } + + block ElectricVehiclesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - // 4. Here, a user-deifned value type is used to describe this column. - // The capital letter indicates that the value type is not built-in - // by convention. The value type itself is defined further below. - "VIN (1-10)" oftype VehicleIdentificationNumber10, - "County" oftype text, - "City" oftype text, - "State" oftype UsStateCode, // We can just use the element as if it was defined in this file. - "Postal Code" oftype text, - "Model Year" oftype integer, - "Make" oftype text, - "Model" oftype text, - "Electric Vehicle Type" oftype text, - "Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text, - "Electric Range" oftype integer, - "Base MSRP" oftype integer, - "Legislative District" oftype text, - "DOL Vehicle ID" oftype integer, - "Vehicle Location" oftype text, - "Electric Utility" oftype text, - "2020 Census Tract" oftype text, - ]; + columns: ElectricVehicle; + parseWith: ElectricVehicleParser; } // 5. This block describes the application of a transform function @@ -81,9 +107,9 @@ pipeline ElectricVehiclesPipeline { // by the "use" property. block ElectricRangeTransformer oftype TableTransformer { inputColumns: [ - "Electric Range" + "electricRange" ]; - outputColumn: "Electric Range (km)"; + outputColumn: "electricRange (km)"; uses: MilesToKilometers; } diff --git a/example/gtfs-rt.jv b/example/gtfs-rt.jv index f1891d348..faa496a8d 100644 --- a/example/gtfs-rt.jv +++ b/example/gtfs-rt.jv @@ -58,49 +58,105 @@ pipeline GtfsRTSimplePipeline { } // 5. Next, we interpret the sheets as tables + valuetype TripUpdate { + property headerGtfsRealtimeVersion oftype text; + property headerTimestamp oftype text; + property headerIncrementality oftype text; + property entityId oftype text; + property entityTripUpdateTripTripId oftype text; + property entityTripUpdateTripRouteId oftype text; + property entityTripUpdateStopTimeUpdateStopSequence oftype text; + property entityTripUpdateStopTimeUpdateStopId oftype text; + property entityTripUpdateStopTimeUpdateArrivalTime oftype text; + property entityTripUpdateStopTimeUpdateDepartureTime oftype text; + } + transform TripUpdateParser { + from r oftype Collection; + to tripUpdate oftype TripUpdate; + + tripUpdate: { + headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version", + headerTimestamp: r cellInColumn "header.timestamp", + headerIncrementality: r cellInColumn "header.incrementality", + entityId: r cellInColumn "entity.id", + entityTripUpdateTripTripId: r cellInColumn "entity.trip_update.trip.trip_id", + entityTripUpdateTripRouteId: r cellInColumn "entity.trip_update.trip.route_id", + entityTripUpdateStopTimeUpdateStopSequence: r cellInColumn "entity.trip_update.stop_time_update.stop_sequence", + entityTripUpdateStopTimeUpdateStopId: r cellInColumn "entity.trip_update.stop_time_update.stop_id", + entityTripUpdateStopTimeUpdateArrivalTime: r cellInColumn "entity.trip_update.stop_time_update.arrival.time", + entityTripUpdateStopTimeUpdateDepartureTime: r cellInColumn "entity.trip_update.stop_time_update.departure.time", + }; + } block TripUpdateTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "header.gtfs_realtime_version" oftype text, - "header.timestamp" oftype text, - "header.incrementality" oftype text, - "entity.id" oftype text, - "entity.trip_update.trip.trip_id" oftype text, - "entity.trip_update.trip.route_id" oftype text, - "entity.trip_update.stop_time_update.stop_sequence" oftype text, - "entity.trip_update.stop_time_update.stop_id" oftype text, - "entity.trip_update.stop_time_update.arrival.time" oftype text, - "entity.trip_update.stop_time_update.departure.time" oftype text, - ]; + columns: TripUpdate; + parseWith: TripUpdateParser; } + + valuetype VehiclePosition { + property headerGtfsRealtimeVersion oftype text; + property headerTimestamp oftype text; + property headerIncrementality oftype text; + property entityId oftype text; + property entityVehiclePositionVehicleDescriptorId oftype text; + property entityVehiclePositionTripTripId oftype text; + property entityVehiclePositionTripRouteId oftype text; + property entityVehiclePositionPositionLatitude oftype text; + property entityVehiclePositionPositionLongitude oftype text; + property entityVehiclePositionTimestamp oftype text; + } + transform VehiclePositionParser { + from r oftype Collection; + to vehiclePosition oftype VehiclePosition; + + vehiclePosition: { + headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version", + headerTimestamp: r cellInColumn "header.timestamp", + headerIncrementality: r cellInColumn "header.incrementality", + entityId: r cellInColumn "entity.id", + entityVehiclePositionVehicleDescriptorId: r cellInColumn "entity.vehicle_position.vehicle_descriptor.id", + entityVehiclePositionTripTripId: r cellInColumn "entity.vehicle_position.trip.trip_id", + entityVehiclePositionTripRouteId: r cellInColumn "entity.vehicle_position.trip.route_id", + entityVehiclePositionPositionLatitude: r cellInColumn "entity.vehicle_position.position.latitude", + entityVehiclePositionPositionLongitude: r cellInColumn "entity.vehicle_position.position.longitude", + entityVehiclePositionTimestamp: r cellInColumn "entity.vehicle_position.timestamp", + }; + } block VehiclePositionTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "header.gtfs_realtime_version" oftype text, - "header.timestamp" oftype text, - "header.incrementality" oftype text, - "entity.id" oftype text, - "entity.vehicle_position.vehicle_descriptor.id" oftype text, - "entity.vehicle_position.trip.trip_id" oftype text, - "entity.vehicle_position.trip.route_id" oftype text, - "entity.vehicle_position.position.latitude" oftype text, - "entity.vehicle_position.position.longitude" oftype text, - "entity.vehicle_position.timestamp" oftype text - ]; + columns: VehiclePosition; + parseWith: VehiclePositionParser; } + + valuetype Alert { + property headerGtfsRealtimeVersion oftype text; + property headerTimestamp oftype text; + property headerIncrementality oftype text; + property entityId oftype text; + property entityAlertInformedEntityRouteId oftype text; + property entityAlertHeaderText oftype text; + property entityAlertDescriptionText oftype text; + } + transform AlertParser { + from r oftype Collection; + to alert oftype Alert; + + alert: { + headerGtfsRealtimeVersion: r cellInColumn "header.gtfs_realtime_version", + headerTimestamp: r cellInColumn "header.timestamp", + headerIncrementality: r cellInColumn "header.incrementality", + entityId: r cellInColumn "entity.id", + entityAlertInformedEntityRouteId: r cellInColumn "entity.alert.informed_entity.route_id", + entityAlertHeaderText: r cellInColumn "entity.alert.header_text", + entityAlertDescriptionText: r cellInColumn "entity.alert.description_text", + }; + } block AlertTableInterpreter oftype TableInterpreter { header: true; - columns: [ - 'header.gtfs_realtime_version' oftype text, - 'header.timestamp' oftype text, - 'header.incrementality' oftype text, - 'entity.id' oftype text, - 'entity.alert.informed_entity.route_id' oftype text, - 'entity.alert.header_text' oftype text, - 'entity.alert.description_text' oftype text, - ]; + columns: Alert; + parseWith: AlertParser; } // 6. Last, we load the tables into the same SQLite file. @@ -124,4 +180,4 @@ pipeline GtfsRTSimplePipeline { file: "./gtfs.sqlite"; dropTable: false; } -} \ No newline at end of file +} diff --git a/libs/execution/src/lib/constraints/constraint-executor.ts b/libs/execution/src/lib/constraints/constraint-executor.ts index b5a3f9aad..e0b914b5f 100644 --- a/libs/execution/src/lib/constraints/constraint-executor.ts +++ b/libs/execution/src/lib/constraints/constraint-executor.ts @@ -11,34 +11,43 @@ import { ERROR_TYPEGUARD, type InternalErrorValueRepresentation, type InternalValidValueRepresentation, - type ValueTypeProperty, type ValueTypeConstraintInlineDefinition, evaluateExpression, + isConstraintDefinition, } from '@jvalue/jayvee-language-server'; import { type ExecutionContext } from '../execution-context'; -export class ConstraintExecutor< - T extends ConstraintDefinition | ValueTypeConstraintInlineDefinition, -> implements AstNodeWrapper +export class ConstraintExecutor + implements + AstNodeWrapper { - constructor(public readonly astNode: T) {} + constructor( + public readonly astNode: + | ConstraintDefinition + | ValueTypeConstraintInlineDefinition, + ) {} + + get name(): string { + return this.astNode.name; + } isValid( - value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + values: Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, context: ExecutionContext, - properties: T extends ValueTypeConstraintInlineDefinition - ? ValueTypeProperty[] - : void, ): boolean { const expression = this.astNode.expression; - if (properties === undefined) { + if (isConstraintDefinition(this.astNode)) { + const value = values.get('value'); + assert(value !== undefined); context.evaluationContext.setValueForValueKeyword(value); } else { - const assignmentForTypeSystem: ValueTypeProperty[] = properties; - for (const property of assignmentForTypeSystem) { - context.evaluationContext.setValueForReference(property.name, value); + for (const [name, value] of values) { + context.evaluationContext.setValueForReference(name, value); } } @@ -57,15 +66,6 @@ export class ConstraintExecutor< ), ); - if (properties === undefined) { - context.evaluationContext.deleteValueForValueKeyword(); - } else { - const assignment_for_type_system: ValueTypeProperty[] = properties; - for (const property of assignment_for_type_system) { - context.evaluationContext.deleteValueForReference(property.name); - } - } - return result; } } diff --git a/libs/execution/src/lib/debugging/debug-log-visitor.ts b/libs/execution/src/lib/debugging/debug-log-visitor.ts index 2b083c39e..396d035b9 100644 --- a/libs/execution/src/lib/debugging/debug-log-visitor.ts +++ b/libs/execution/src/lib/debugging/debug-log-visitor.ts @@ -20,6 +20,8 @@ import { type Table } from '../types/io-types/table'; import { findLineBounds } from '../util/string-util'; import { type DebugGranularity } from './debug-configuration'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; export class DebugLogVisitor implements IoTypeVisitor { private readonly PEEK_NUMBER_OF_WORKBOOKS = 5; @@ -57,6 +59,7 @@ export class DebugLogVisitor implements IoTypeVisitor { } const row = table.getRow(i); + assert(row !== undefined); const rowData = [...row.values()] .map((cell) => { if (ERROR_TYPEGUARD(cell)) { diff --git a/libs/execution/src/lib/types/io-types/sheet.ts b/libs/execution/src/lib/types/io-types/sheet.ts index 70989b1e8..099a87537 100644 --- a/libs/execution/src/lib/types/io-types/sheet.ts +++ b/libs/execution/src/lib/types/io-types/sheet.ts @@ -45,19 +45,6 @@ export class Sheet implements IOTypeImplementation { return this.numberOfColumns; } - getHeaderRow(): string[] { - assert( - this.getNumberOfRows() > 0, - 'The sheet is expected to be non-empty and have a header row', - ); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - return this.data[0]!; - } - - iterateRows(callbackFn: (row: string[], rowIndex: number) => void) { - this.data.forEach(callbackFn); - } - clone(): Sheet { return new Sheet(structuredClone(this.data)); } diff --git a/libs/execution/src/lib/types/io-types/table.spec.ts b/libs/execution/src/lib/types/io-types/table.spec.ts index 0575e26c5..c02bd34ff 100644 --- a/libs/execution/src/lib/types/io-types/table.spec.ts +++ b/libs/execution/src/lib/types/io-types/table.spec.ts @@ -2,16 +2,39 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { ValueTypeProvider } from '@jvalue/jayvee-language-server'; +import { + type InternalErrorValueRepresentation, + type InternalValidValueRepresentation, + ValueTypeProvider, +} from '@jvalue/jayvee-language-server'; -import { Table } from './table'; +import { Table, type TableColumn } from './table'; + +function addRowWrapper( + table: Table, + row: Record< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, +): void { + const tableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); + + for (const [columnName, cellValue] of Object.entries(row)) { + tableRow.set(columnName, cellValue); + } + + return table.addRow(tableRow); +} describe('Table', () => { let table: Table; let valueTypeProvider: ValueTypeProvider; beforeEach(() => { - table = new Table(); + table = new Table(0, new Map(), []); valueTypeProvider = new ValueTypeProvider(); }); @@ -36,9 +59,9 @@ describe('Table', () => { valueType: valueTypeProvider.Primitives.Text, values: [], }); - table.addRow({ a: 'a1' }); - table.addRow({ a: 'a2' }); - table.addRow({ a: 'a3' }); + addRowWrapper(table, { a: 'a1' }); + addRowWrapper(table, { a: 'a2' }); + addRowWrapper(table, { a: 'a3' }); table.addColumn('b', { valueType: valueTypeProvider.Primitives.Text, values: ['b1', 'b2', 'b3'], @@ -52,9 +75,9 @@ describe('Table', () => { valueType: valueTypeProvider.Primitives.Text, values: [], }); - table.addRow({ a: 'a1' }); - table.addRow({ a: 'a2' }); - table.addRow({ a: 'a3' }); + addRowWrapper(table, { a: 'a1' }); + addRowWrapper(table, { a: 'a2' }); + addRowWrapper(table, { a: 'a3' }); expect(table.getNumberOfRows()).toBe(3); }); diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index 3c1deae53..f2a79b9a8 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -3,10 +3,11 @@ // SPDX-License-Identifier: AGPL-3.0-only // eslint-disable-next-line unicorn/prefer-node-protocol -import { strict as assert } from 'assert'; +import assert from 'assert'; import { ERROR_TYPEGUARD, + InvalidValue, IOType, type InternalErrorValueRepresentation, type InternalValidValueRepresentation, @@ -22,6 +23,9 @@ import { type IOTypeImplementation, type IoTypeVisitor, } from './io-type-implementation'; +import { ConstraintExecutor } from '../../constraints'; +import { type ExecutionContext } from '../../execution-context'; +import { assertUnreachable } from 'langium'; export interface TableColumn< T extends InternalValidValueRepresentation = InternalValidValueRepresentation, @@ -30,7 +34,7 @@ export interface TableColumn< valueType: ValueType; } -export type TableRow = Record< +export type TableRow = Map< string, InternalValidValueRepresentation | InternalErrorValueRepresentation >; @@ -42,12 +46,14 @@ export type TableRow = Record< export class Table implements IOTypeImplementation { public readonly ioType = IOType.TABLE; - private numberOfRows = 0; - - private columns = new Map(); - - public constructor(numberOfRows = 0) { - this.numberOfRows = numberOfRows; + public constructor( + private numberOfRows: number, + private columns: Map, + private constraintExecutors: ConstraintExecutor[], + ) { + assert(this.numberOfRows !== undefined); + assert(this.columns !== undefined); + assert(this.constraintExecutors !== undefined); } addColumn(name: string, column: TableColumn): void { @@ -61,31 +67,25 @@ export class Table implements IOTypeImplementation { * @param row data of this row for each column */ addRow(row: TableRow): void { - const rowLength = Object.keys(row).length; assert( - rowLength === this.columns.size, - `Added row has the wrong dimension (expected: ${this.columns.size}, actual: ${rowLength})`, + row.size === this.columns.size, + `Added row has the wrong dimension (expected: ${this.columns.size}, actual: ${row.size})`, ); - if (rowLength === 0) { - return; + + if (row.size > 0) { + this.numberOfRows++; } - assert( - Object.keys(row).every((x) => this.hasColumn(x)), - 'Added row does not fit the columns in the table', - ); - Object.entries(row).forEach(([columnName, value]) => { + for (const [columnName, cellValue] of row.entries()) { const column = this.columns.get(columnName); - assert(column !== undefined); + assert(column !== undefined, 'All added rows fit columns in the table'); assert( - ERROR_TYPEGUARD(value) || - column.valueType.isInternalValidValueRepresentation(value), + ERROR_TYPEGUARD(cellValue) || + column.valueType.isInternalValidValueRepresentation(cellValue), ); - column.values.push(value); - }); - - this.numberOfRows++; + column.values.push(cellValue); + } } getNumberOfRows(): number { @@ -100,7 +100,7 @@ export class Table implements IOTypeImplementation { return this.columns.has(name); } - getColumns(): ReadonlyMap { + getColumns(): Map { return this.columns; } @@ -108,17 +108,10 @@ export class Table implements IOTypeImplementation { return this.columns.get(name); } - getRow( - rowId: number, - ): Map< - string, - InternalValidValueRepresentation | InternalErrorValueRepresentation - > { + getRow(rowId: number): TableRow | undefined { const numberOfRows = this.getNumberOfRows(); if (rowId >= numberOfRows) { - throw new Error( - `Trying to access table row ${rowId} (of ${numberOfRows} rows)`, - ); + return undefined; } const row = new Map< @@ -133,6 +126,46 @@ export class Table implements IOTypeImplementation { return row; } + private setRowInvalid(rowIdx: number, message: string): void { + for (const [columnName, column] of this.columns.entries()) { + column.values[rowIdx] = new InvalidValue(message); + this.columns.set(columnName, column); + } + } + + forEachUnfulfilledRow( + onInvalidRow: ( + constraint: ConstraintExecutor, + rowIndex: number, + row: TableRow, + ) => 'markInvalid' | 'ignore', + executionContext: ExecutionContext, + ): void { + for (let rowIdx = 0; rowIdx < this.numberOfRows; rowIdx++) { + const row = this.getRow(rowIdx); + assert(row !== undefined); + + for (const constraint of this.constraintExecutors) { + if (constraint.isValid(row, executionContext)) { + continue; + } + const invalidHandling = onInvalidRow(constraint, rowIdx, row); + switch (invalidHandling) { + case 'markInvalid': { + this.setRowInvalid(rowIdx, `Invalid constraint ${constraint.name}`); + break; + } + case 'ignore': { + break; + } + default: { + assertUnreachable(invalidHandling); + } + } + } + } + } + static generateDropTableStatement(tableName: string): string { return `DROP TABLE IF EXISTS "${tableName}";`; } @@ -186,16 +219,19 @@ export class Table implements IOTypeImplementation { } clone(): Table { - const cloned = new Table(); - cloned.numberOfRows = this.numberOfRows; - [...this.columns.entries()].forEach(([columnName, column]) => { - cloned.addColumn(columnName, { + const copiedColumns = new Map(); + [...this.columns.entries()].map(([columnName, column]) => { + copiedColumns.set(columnName, { values: cloneInternalValue(column.values), valueType: column.valueType, }); }); - return cloned; + const copiedConstraints = this.constraintExecutors.map( + (constraint) => new ConstraintExecutor(constraint.astNode), + ); + + return new Table(this.numberOfRows, copiedColumns, copiedConstraints); } acceptVisitor(visitor: IoTypeVisitor): R { diff --git a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts index 3e8d2a093..464da9f32 100644 --- a/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts +++ b/libs/execution/src/lib/types/value-types/internal-representation-parsing.ts @@ -86,23 +86,23 @@ class InternalRepresentationParserVisitor extends ValueTypeVisitor< return value; } - visitBoolean(vt: BooleanValuetype): boolean | InvalidValue { + override visitBoolean(vt: BooleanValuetype): boolean | InvalidValue { return vt.fromString(this.applyTrimOptions(this.value)); } - visitDecimal(vt: DecimalValuetype): number | InvalidValue { + override visitDecimal(vt: DecimalValuetype): number | InvalidValue { return vt.fromString(this.applyTrimOptions(this.value)); } - visitInteger(vt: IntegerValuetype): number | InvalidValue { + override visitInteger(vt: IntegerValuetype): number | InvalidValue { return vt.fromString(this.applyTrimOptions(this.value)); } - visitText(vt: TextValuetype): string { + override visitText(vt: TextValuetype): string { return vt.fromString(this.value); } - visitAtomicValueType( + override visitAtomicValueType( valueType: AtomicValueType, ): InternalValidValueRepresentation | InvalidValue { const containedTypes = valueType.getContainedTypes(); @@ -116,29 +116,35 @@ class InternalRepresentationParserVisitor extends ValueTypeVisitor< return containedType.acceptVisitor(this); } - visitCellRange(): InvalidValue { + override visitCellRange(): InvalidValue { return new InvalidValue(`Cannot parse cell ranges into internal values`); } - visitCollection(): InvalidValue { + override visitCollection(): InvalidValue { return new InvalidValue(`Cannot parse collections into internal values`); } - visitConstraint(): InvalidValue { + override visitConstraint(): InvalidValue { return new InvalidValue(`Cannot parse constraints into internal values`); } - visitRegex(): InvalidValue { + override visitRegex(): InvalidValue { return new InvalidValue(`Cannot parse regex into internal values`); } - visitTransform(): InvalidValue { + override visitTransform(): InvalidValue { return new InvalidValue(`Cannot parse transforms into internal values`); } - visitValuetypeAssignment(): InvalidValue { + override visitValuetypeAssignment(): InvalidValue { return new InvalidValue( `Cannot parse valuetype assignments into internal values`, ); } + + override visitValuetypeDefinition(): InvalidValue { + return new InvalidValue( + `Cannot parse valuetype definitions into internal values`, + ); + } } diff --git a/libs/execution/src/lib/types/value-types/value-representation-validity.ts b/libs/execution/src/lib/types/value-types/value-representation-validity.ts index 326f7e37c..c6b87d8d9 100644 --- a/libs/execution/src/lib/types/value-types/value-representation-validity.ts +++ b/libs/execution/src/lib/types/value-types/value-representation-validity.ts @@ -21,7 +21,8 @@ import { type ValueType, ValueTypeVisitor, type ValuetypeAssignmentValuetype, - isConstraintDefinition, + type ValuetypeDefinitionValuetype, + type InternalErrorValueRepresentation, } from '@jvalue/jayvee-language-server'; import { ConstraintExecutor } from '../../constraints'; @@ -32,13 +33,27 @@ export function isValidValueRepresentation( valueType: ValueType, context: ExecutionContext, ): boolean { - const visitor = new ValueRepresentationValidityVisitor(value, context); + const values = new Map(); + values.set('value', value); + const visitor = new ValueRepresentationValidityVisitor(values, context); return valueType.acceptVisitor(visitor); } +export function allValueRepresentationsValid( + values: Map, + atomicValueType: AtomicValueType, + context: ExecutionContext, +): boolean { + const visitor = new ValueRepresentationValidityVisitor(values, context); + return atomicValueType.acceptVisitor(visitor); +} + class ValueRepresentationValidityVisitor extends ValueTypeVisitor { constructor( - private value: InternalValidValueRepresentation, + private values: Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >, private context: ExecutionContext, ) { super(); @@ -57,13 +72,9 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { for (const constraint of valueType.getConstraints()) { this.context.enterNode(constraint); - const valueFulfilledConstraint = isConstraintDefinition(constraint) - ? new ConstraintExecutor(constraint).isValid(this.value, this.context) - : new ConstraintExecutor(constraint).isValid( - this.value, - this.context, - valueType.getProperties(), - ); + const valueFulfilledConstraint = new ConstraintExecutor( + constraint, + ).isValid(this.values, this.context); this.context.exitNode(constraint); @@ -109,6 +120,12 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { return this.isValidForPrimitiveValuetype(valueType); } + override visitValuetypeDefinition( + valueType: ValuetypeDefinitionValuetype, + ): boolean { + return this.isValidForPrimitiveValuetype(valueType); + } + override visitCollection(valueType: CollectionValueType): boolean { return this.isValidForPrimitiveValuetype(valueType); } @@ -118,6 +135,8 @@ class ValueRepresentationValidityVisitor extends ValueTypeVisitor { } private isValidForPrimitiveValuetype(valueType: PrimitiveValueType): boolean { - return valueType.isInternalValidValueRepresentation(this.value); + const value = this.values.get('value'); + assert(value !== undefined); + return valueType.isInternalValidValueRepresentation(value); } } diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts index ea9037bfa..f47a92e3e 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-column-type-visitor.ts @@ -64,6 +64,12 @@ export class SQLColumnTypeVisitor extends ValueTypeVisitor { ); } + override visitValuetypeDefinition(): string { + throw new Error( + 'No visit implementation given for valueType definitions. Cannot be the type of a column.', + ); + } + override visitCollection(): string { throw new Error( 'No visit implementation given for collections. Cannot be the type of a column.', diff --git a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts index c64dd34e2..9b6bdba78 100644 --- a/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts +++ b/libs/execution/src/lib/types/value-types/visitors/sql-value-representation-visitor.ts @@ -114,6 +114,14 @@ export class SQLValueRepresentationVisitor extends ValueTypeVisitor< ); } + override visitValuetypeDefinition(): ( + value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + ) => string { + throw new Error( + 'No visit implementation given for valueType definitions. Cannot be the type of a column.', + ); + } + override visitCollection(): ( value: InternalValidValueRepresentation | InternalErrorValueRepresentation, ) => string { diff --git a/libs/execution/test/utils/test-infrastructure-util.ts b/libs/execution/test/utils/test-infrastructure-util.ts index 9dd9febd8..729b300db 100644 --- a/libs/execution/test/utils/test-infrastructure-util.ts +++ b/libs/execution/test/utils/test-infrastructure-util.ts @@ -89,7 +89,7 @@ export function constructTable( columns: TableColumnDefinition[], numberOfRows: number, ): Table { - const table = new Table(numberOfRows); - columns.forEach((col) => table.addColumn(col.columnName, col.column)); - return table; + const columnMap = new Map(); + columns.forEach((col) => columnMap.set(col.columnName, col.column)); + return new Table(numberOfRows, columnMap, []); } diff --git a/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts index 1f03f6fc9..9f9334781 100644 --- a/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts @@ -96,7 +96,6 @@ describe('Validation of CellRangeSelectorExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], @@ -121,7 +120,6 @@ describe('Validation of CellRangeSelectorExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(2); - expect(result.right.getHeaderRow()).toEqual(['', 'Test', 'true']); expect(result.right.getData()).toEqual([ ['', 'Test', 'true'], ['', 'Test', 'false'], diff --git a/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts index 7f2c77a49..396b68078 100644 --- a/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts @@ -96,7 +96,6 @@ describe('Validation of CellWriterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['16', 'Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['16', 'Test', 'true'], @@ -121,7 +120,6 @@ describe('Validation of CellWriterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['16', 'Test2', '']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['16', 'Test2', ''], diff --git a/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts index ca80bda84..191b57226 100644 --- a/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts @@ -96,7 +96,6 @@ describe('Validation of ColumnDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(2); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['Test', 'true'], @@ -121,7 +120,6 @@ describe('Validation of ColumnDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(1); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['Test']); expect(result.right.getData()).toEqual( expect.arrayContaining([['Test'], ['Test'], ['Test']]), ); @@ -159,7 +157,6 @@ describe('Validation of ColumnDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(2); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['Test', 'true'], diff --git a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts index 44d9106e6..f3e2401f5 100644 --- a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts @@ -96,7 +96,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(15); - expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()[0]).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([['0', 'Test', 'true']]), ); @@ -117,7 +117,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(14); - expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()[0]).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], @@ -158,7 +158,7 @@ describe('Validation of RowDeleterExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(15); - expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()[0]).toEqual(['1', 'Test', 'false']); expect(result.right.getData()).not.toEqual( expect.arrayContaining([['0', 'Test', 'true']]), ); diff --git a/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts index d656e161e..8e0a02100 100644 --- a/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts @@ -93,7 +93,6 @@ describe('Validation of SheetPickerExecutor', () => { expect(result.right.ioType).toEqual(IOType.SHEET); expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(16); - expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); expect(result.right.getData()).toEqual( expect.arrayContaining([ ['0', 'Test', 'true'], diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts index 84939a6b2..987e920f6 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts @@ -116,21 +116,13 @@ describe('Validation of TableInterpreterExecutor', () => { } }); - it('should diagnose empty table on empty column parameter', async () => { - const text = readJvTestAsset('valid-empty-columns-with-header.jv'); + it('should diagnose error on empty column value type', async () => { + const text = readJvTestAsset('invalid-empty-columns-with-header.jv'); - const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(0); - expect(result.right.getNumberOfRows()).toEqual(0); - } + const document = await parse(text, { validation: true }); + expect( + document.parseResult.parserErrors.map((error) => error.message), + ).toContainEqual("Expecting token of type 'property' but found `}`."); }); it('should diagnose empty table on wrong header case', async () => { @@ -145,7 +137,7 @@ describe('Validation of TableInterpreterExecutor', () => { expect(R.isErr(result)).toEqual(false); if (R.isOk(result)) { expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfColumns()).toEqual(3); expect(result.right.getNumberOfRows()).toEqual(0); } }); @@ -253,7 +245,10 @@ describe('Validation of TableInterpreterExecutor', () => { values: expect.arrayContaining([true, false]) as boolean[], }), ); - for (const value of result.right.getRow(0).values()) { + const row = result.right.getRow(0); + expect(row).toBeDefined(); + assert(row !== undefined); + for (const value of row.values()) { if (value === 'name') { continue; } @@ -262,21 +257,13 @@ describe('Validation of TableInterpreterExecutor', () => { } }); - it('should diagnose empty table on empty column parameter', async () => { - const text = readJvTestAsset('valid-empty-columns-without-header.jv'); + it('should diagnose error on empty column parameter', async () => { + const text = readJvTestAsset('invalid-empty-columns-without-header.jv'); - const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(0); - expect(result.right.getNumberOfRows()).toEqual(0); - } + const document = await parse(text, { validation: true }); + expect( + document.parseResult.parserErrors.map((error) => error.message), + ).toContainEqual("Expecting token of type 'property' but found `}`."); }); it('should diagnose error on empty sheet', async () => { @@ -291,7 +278,7 @@ describe('Validation of TableInterpreterExecutor', () => { expect(R.isOk(result)).toEqual(false); if (R.isErr(result)) { expect(result.left.message).toEqual( - 'There are 3 column definitions but the input sheet only has 0 columns', + 'The value type TestValueType has 3 properties, but the input sheet only has 0 columns', ); } }); @@ -318,63 +305,5 @@ describe('Validation of TableInterpreterExecutor', () => { } } }); - - it('should skip leading and trailing whitespace on numeric columns but not text columns', async () => { - const text = readJvTestAsset('valid-without-header.jv'); - - const testWorkbook = await readTestWorkbook('test-with-whitespace.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - assert(R.isOk(result)); - - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(3); - expect(result.right.getNumberOfRows()).toEqual(3); - - expect([...result.right.getColumns().keys()]).toStrictEqual([ - 'index', - 'name', - 'flag', - ]); - - const row = result.right.getRow(0); - const index = row.get('index'); - expect(index).toBe(0); - const name = row.get('name'); - expect(name).toBe(' text with leading whitespace'); - - for (let rowIdx = 1; rowIdx < result.right.getNumberOfRows(); rowIdx++) { - const row = result.right.getRow(rowIdx); - const index = row.get('index'); - expect(index).toBe(rowIdx); - } - }); - - it('should not skip leading or trailing whitespace if the relevant block properties are false', async () => { - const text = readJvTestAsset('valid-without-header-without-trim.jv'); - - const testWorkbook = await readTestWorkbook('test-with-whitespace.xlsx'); - const result = await parseAndExecuteExecutor( - text, - testWorkbook.getSheetByName('Sheet1') as R.Sheet, - ); - - expect(R.isErr(result)).toEqual(false); - if (R.isOk(result)) { - expect(result.right.ioType).toEqual(IOType.TABLE); - expect(result.right.getNumberOfColumns()).toEqual(3); - expect(result.right.getNumberOfRows()).toEqual(3); - const indexColumn = result.right.getColumn('index')?.values; - expect(indexColumn).toBeDefined(); - assert(indexColumn !== undefined); - indexColumn.forEach((cell) => - expect(cell).toBeInstanceOf(InvalidValue), - ); - } - }); }); }); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index cb21cb2c7..5b7d1988b 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -8,34 +8,30 @@ import { strict as assert } from 'assert'; import * as R from '@jvalue/jayvee-execution'; import { AbstractBlockExecutor, + TransformExecutor, type BlockExecutorClass, type ExecutionContext, type Sheet, Table, implementsStatic, - isValidValueRepresentation, - parseValueToInternalRepresentation, } from '@jvalue/jayvee-execution'; import { - CellIndex, + AtomicValueType, ERROR_TYPEGUARD, - IOType, - InternalErrorValueRepresentation, - type InternalValidValueRepresentation, + evaluateExpression, + internalValueToString, InvalidValue, + IOType, + isTableRowLiteral, + MISSING_TYPEGUARD, + TableRowLiteral, + isAtomicValueType, MissingValue, - type ValueType, - type ValuetypeAssignment, - internalValueToString, + onlyElementOrUndefined, + TABLEROW_TYPEGUARD, + ValueType, } from '@jvalue/jayvee-language-server'; -export interface ColumnDefinitionEntry { - sheetColumnIndex: number; - columnName: string; - valueType: ValueType; - astNode: ValuetypeAssignment; -} - @implementsStatic() export class TableInterpreterExecutor extends AbstractBlockExecutor< IOType.SHEET, @@ -56,25 +52,31 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< 'header', context.valueTypeProvider.Primitives.Boolean, ); - const columnDefinitions = context.getPropertyValue( + const columnsValueTypeDefinition = context.getPropertyValue( 'columns', - context.valueTypeProvider.createCollectionValueTypeOf( - context.valueTypeProvider.Primitives.ValuetypeAssignment, - ), + context.valueTypeProvider.Primitives.ValuetypeDefinition, ); - const skipLeadingWhitespace = context.getPropertyValue( - 'skipLeadingWhitespace', - context.valueTypeProvider.Primitives.Boolean, + const parseWith = context.getPropertyValue( + 'parseWith', + context.valueTypeProvider.Primitives.Transform, ); - const skipTrailingWhitespace = context.getPropertyValue( - 'skipTrailingWhitespace', - context.valueTypeProvider.Primitives.Boolean, + + const columnsValueType = context.wrapperFactories.ValueType.wrap( + columnsValueTypeDefinition, + ); + + assert( + isAtomicValueType(columnsValueType), + 'This must have been checked earlier at the validation step', ); - let columnEntries: ColumnDefinitionEntry[]; + const schema = columnsValueType.getSchema(); + const sheetData = inputSheet.getData().slice(); + let headerRow: string[] | undefined = undefined; if (header) { - if (inputSheet.getNumberOfRows() < 1) { + headerRow = sheetData.shift()?.slice(); + if (headerRow === undefined) { return R.err({ message: 'The input sheet is empty and thus has no header', diagnostic: { @@ -82,45 +84,40 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< }, }); } - - const headerRow = inputSheet.getHeaderRow(); - - columnEntries = this.deriveColumnDefinitionEntriesFromHeader( - columnDefinitions, - headerRow, - context, - ); + if (headerRow.length !== schema.size) { + return R.err({ + message: + 'The length of the header does not fit the columns value type', + diagnostic: { + node: context.getOrFailProperty('header'), + }, + }); + } } else { - if (inputSheet.getNumberOfColumns() < columnDefinitions.length) { + if ( + inputSheet.getNumberOfColumns() < + columnsValueType.getProperties().length + ) { return R.err({ - message: `There are ${ - columnDefinitions.length - } column definitions but the input sheet only has ${inputSheet.getNumberOfColumns()} columns`, + message: `The value type ${columnsValueType.getName()} has ${columnsValueType.getProperties().length} properties, but the input sheet only has ${inputSheet.getNumberOfColumns()} columns`, diagnostic: { node: context.getOrFailProperty('columns'), }, }); } - - columnEntries = this.deriveColumnDefinitionEntriesWithoutHeader( - columnDefinitions, - context, - ); } - const numberOfTableRows = header - ? inputSheet.getNumberOfRows() - 1 - : inputSheet.getNumberOfRows(); context.logger.logDebug( - `Validating ${numberOfTableRows} row(s) according to the column types`, + `Validating ${inputSheet.getNumberOfRows()} row(s) according to the column types`, ); + const parseRowTransform = new TransformExecutor(parseWith, context); + const resultingTable = this.constructAndValidateTable( - inputSheet, - header, - columnEntries, - skipLeadingWhitespace, - skipTrailingWhitespace, + sheetData, + headerRow, + columnsValueType, + parseRowTransform, context, ); context.logger.logDebug( @@ -130,159 +127,115 @@ export class TableInterpreterExecutor extends AbstractBlockExecutor< } private constructAndValidateTable( - sheet: Sheet, - header: boolean, - columnEntries: ColumnDefinitionEntry[], - skipLeadingWhitespace: boolean, - skipTrailingWhitespace: boolean, + sheetData: (readonly string[])[], + headerRow: string[] | undefined, + columnsValueType: AtomicValueType, + parseRowTransform: TransformExecutor, context: ExecutionContext, ): Table { - const table = new Table(); - - // add columns - columnEntries.forEach((columnEntry) => { - table.addColumn(columnEntry.columnName, { + const columns = new Map(); + for (const [columnName, columnValueType] of columnsValueType + .getSchema() + .entries()) { + columns.set(columnName, { values: [], - valueType: columnEntry.valueType, + valueType: columnValueType, }); - }); + } + if (headerRow !== undefined) { + context.evaluationContext.setHeaderRow(headerRow); + } - // add rows - sheet.iterateRows((sheetRow, sheetRowIndex) => { - if (header && sheetRowIndex === 0) { - return; - } + const sheetRowReferenceName = onlyElementOrUndefined( + parseRowTransform.getInputDetails(), + )?.port.name; + assert(sheetRowReferenceName !== undefined); + + const parseRowExpression = + parseRowTransform.getOutputAssignment().expression; + assert(isTableRowLiteral(parseRowExpression)); + + const constraints = columnsValueType + .getConstraints() + .map((constraint) => new R.ConstraintExecutor(constraint)); + const table = new Table(0, columns, constraints); + + // add rows + for (const sheetRow of sheetData) { const tableRow = this.constructAndValidateTableRow( + sheetRowReferenceName, sheetRow, - sheetRowIndex, - columnEntries, - skipLeadingWhitespace, - skipTrailingWhitespace, + columnsValueType.getSchema(), + parseRowExpression, context, ); - table.addRow(tableRow); - }); - return table; - } - - private constructAndValidateTableRow( - sheetRow: string[], - sheetRowIndex: number, - columnEntries: ColumnDefinitionEntry[], - skipLeadingWhitespace: boolean, - skipTrailingWhitespace: boolean, - context: ExecutionContext, - ): R.TableRow { - const tableRow: R.TableRow = {}; - columnEntries.forEach((columnEntry) => { - const valueType = columnEntry.valueType; - const sheetColumnIndex = columnEntry.sheetColumnIndex; - const value = sheetRow[sheetColumnIndex]; - - const parsedValue = - value !== undefined - ? this.parseAndValidateValue( - value, - valueType, - skipLeadingWhitespace, - skipTrailingWhitespace, - context, - ) - : new MissingValue( - `The sheet row did not contain a value at index ${sheetColumnIndex}`, - ); - if (ERROR_TYPEGUARD(parsedValue)) { - const currentCellIndex = new CellIndex(sheetColumnIndex, sheetRowIndex); - context.logger.logDebug( - `Invalid value at cell ${currentCellIndex.toString()}: "${value}" does not match the type ${columnEntry.valueType.getName()}`, - ); + if (MISSING_TYPEGUARD(tableRow)) { + context.logger.logDebug(tableRow.toString()); + } else { + table.addRow(tableRow); } + } - tableRow[columnEntry.columnName] = parsedValue; - }); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + table.forEachUnfulfilledRow((constraint, rowIdx, _row) => { + context.logger.logErr( + `Invalid constraint ${constraint.name} on row ${rowIdx}`, + ); + return 'markInvalid'; + }, context); - assert(Object.keys(tableRow).length === columnEntries.length); - return tableRow; + return table; } - private parseAndValidateValue( - value: string, - valueType: ValueType, - skipLeadingWhitespace: boolean, - skipTrailingWhitespace: boolean, + private constructAndValidateTableRow( + sheetRowReferenceName: string, + sheetRow: readonly string[], + schema: Map, + parseRowExpression: TableRowLiteral, context: ExecutionContext, - ): InternalValidValueRepresentation | InternalErrorValueRepresentation { - const parsedValue = parseValueToInternalRepresentation(value, valueType, { - skipLeadingWhitespace, - skipTrailingWhitespace, - }); - - if ( - !ERROR_TYPEGUARD(parsedValue) && - !isValidValueRepresentation(parsedValue, valueType, context) - ) { - return new InvalidValue( - `The following value was not valid for valuetype ${valueType.getName()}: ${internalValueToString( - parsedValue, - context.wrapperFactories, - )}`, - ); - } - return parsedValue; - } + ): R.TableRow | MissingValue { + context.evaluationContext.setValueForReference( + sheetRowReferenceName, + sheetRow.slice(), + ); - private deriveColumnDefinitionEntriesWithoutHeader( - columnDefinitions: ValuetypeAssignment[], - context: ExecutionContext, - ): ColumnDefinitionEntry[] { - return columnDefinitions.map( - (columnDefinition, columnDefinitionIndex) => { - const columnValuetype = context.wrapperFactories.ValueType.wrap( - columnDefinition.type, - ); - assert(columnValuetype !== undefined); - return { - sheetColumnIndex: columnDefinitionIndex, - columnName: columnDefinition.name, - valueType: columnValuetype, - astNode: columnDefinition, - }; - }, + const tableRow = evaluateExpression( + parseRowExpression, + context.evaluationContext, + context.wrapperFactories, ); - } + assert(TABLEROW_TYPEGUARD(tableRow)); - private deriveColumnDefinitionEntriesFromHeader( - columnDefinitions: ValuetypeAssignment[], - headerRow: string[], - context: ExecutionContext, - ): ColumnDefinitionEntry[] { - context.logger.logDebug(`Matching header with provided column names`); + let missingCount = 0; + for (const value of tableRow.values()) { + if (MISSING_TYPEGUARD(value)) { + context.logger.logDebug(value.toString()); + missingCount += 1; + } + } + if (missingCount === tableRow.size) { + return new MissingValue('All values in row were missing. Discarding row'); + } - const columnEntries: ColumnDefinitionEntry[] = []; - for (const columnDefinition of columnDefinitions) { - const indexOfMatchingHeader = headerRow.findIndex( - (headerColumnName) => headerColumnName === columnDefinition.name, - ); - if (indexOfMatchingHeader === -1) { - context.logger.logDebug( - `Omitting column "${columnDefinition.name}" as the name was not found in the header`, + assert(tableRow.size === schema.size); + for (const [columnName, cellValue] of tableRow) { + const entry = [...schema.entries()].find(([cN]) => cN === columnName); + assert(entry !== undefined); + const [, columnValueType] = entry; + if ( + !ERROR_TYPEGUARD(cellValue) && + !columnValueType.isInternalValidValueRepresentation(cellValue) + ) { + tableRow.set( + columnName, + new InvalidValue( + `cell value ${internalValueToString(cellValue)} is not a valid value for ${columnValueType.getName()}`, + ), ); - continue; } - const columnValuetype = context.wrapperFactories.ValueType.wrap( - columnDefinition.type, - ); - assert(columnValuetype !== undefined); - - columnEntries.push({ - sheetColumnIndex: indexOfMatchingHeader, - columnName: columnDefinition.name, - valueType: columnValuetype, - astNode: columnDefinition, - }); } - return columnEntries; + return tableRow; } } diff --git a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts index d272711c2..0c67e1dc7 100644 --- a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.ts @@ -97,6 +97,14 @@ export class TableTransformerExecutor extends AbstractBlockExecutor< outputColumnName, ); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + outputTable.forEachUnfulfilledRow((constraint, rowIdx, _row) => { + context.logger.logErr( + `Invalid constraint ${constraint.name} on row ${rowIdx}`, + ); + return 'markInvalid'; + }, context); + return R.ok(outputTable); } diff --git a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv similarity index 55% rename from libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv rename to libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv index 6cea9044f..5810d24db 100644 --- a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-with-header.jv @@ -2,19 +2,21 @@ // // SPDX-License-Identifier: AGPL-3.0-only -pipeline Pipeline { - block Test oftype TableInterpreter { - columns: [ - "name" oftype text, - "name" oftype integer, - ]; - } +pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: TestValueType; + } + block TestLoader oftype TestTableLoader { } - TestExtractor -> Test -> TestLoader; + TestExtractor -> TestBlock -> TestLoader; } diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-without-header.jv similarity index 85% rename from libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv rename to libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-without-header.jv index b6e423edc..b6c088d00 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/invalid-empty-columns-without-header.jv @@ -7,9 +7,13 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + + valuetype TestValueType { + } + block TestBlock oftype TableInterpreter { header: false; - columns: []; + columns: TestValueType; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv index 7e46a44c6..d3c3189cd 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv @@ -7,9 +7,21 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + } + + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { }; + } + block TestBlock oftype TableInterpreter { header: true; - columns: []; + columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv index 3d9929495..08fc9db23 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv @@ -7,13 +7,23 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property Index oftype integer; + property Name oftype text; + property Flag oftype boolean; + } + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { Index: asInteger (r cellInColumn "Index"), Name: r cellInColumn "Name", Flag: asBoolean (r cellInColumn "Flag") }; + } + block TestBlock oftype TableInterpreter { header: true; - columns: [ - "Index" oftype integer, - "Name" oftype text, - "Flag" oftype boolean - ]; + columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv index cd368186b..4d46a854d 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv @@ -7,13 +7,23 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype boolean; + } + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn "index"), name: r cellInColumn "name", flag: asBoolean (r cellInColumn "flag") }; + } + block TestBlock oftype TableInterpreter { header: true; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype boolean - ]; + columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv deleted file mode 100644 index ec1fc6b7e..000000000 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header-without-trim.jv +++ /dev/null @@ -1,25 +0,0 @@ -// SPDX-FileCopyrightText: 2025 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline TestPipeline { - - block TestExtractor oftype TestSheetExtractor { } - - block TestBlock oftype TableInterpreter { - header: false; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype boolean - ]; - skipLeadingWhitespace: false; - skipTrailingWhitespace: false; - } - - block TestLoader oftype TestTableLoader { } - - TestExtractor - -> TestBlock - -> TestLoader; -} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv index 6fa8f5c1d..9925fd6a0 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv @@ -7,13 +7,24 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype boolean; + } + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn 0), name: r cellInColumn 1, flag: + asBoolean (r cellInColumn 2) }; + } + block TestBlock oftype TableInterpreter { header: false; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype boolean - ]; + columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv index f8ec03446..1c480af72 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-with-header.jv @@ -7,13 +7,23 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype integer; + } + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn "index"), name: r cellInColumn "name", flag: asBoolean (r cellInColumn "flag") }; + } + block TestBlock oftype TableInterpreter { header: true; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype integer - ]; + columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv index ebc269171..20d65adf8 100644 --- a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-value-type-without-header.jv @@ -7,13 +7,24 @@ pipeline TestPipeline { block TestExtractor oftype TestSheetExtractor { } + valuetype TestValueType { + property index oftype integer; + property name oftype text; + property flag oftype integer; + } + + transform Parser { + from r oftype Collection; + to t oftype TestValueType; + + t: { index: asInteger (r cellInColumn 0), name: r cellInColumn 1, flag: + asBoolean (r cellInColumn 2) }; + } + block TestBlock oftype TableInterpreter { header: false; - columns: [ - "index" oftype integer, - "name" oftype text, - "flag" oftype integer - ]; + columns: TestValueType; + parseWith: Parser; } block TestLoader oftype TestTableLoader { diff --git a/libs/extensions/tabular/exec/test/util.ts b/libs/extensions/tabular/exec/test/util.ts index b0b65c306..96725b91c 100644 --- a/libs/extensions/tabular/exec/test/util.ts +++ b/libs/extensions/tabular/exec/test/util.ts @@ -7,6 +7,7 @@ import path from 'node:path'; import { Table, type TableRow, + type TableColumn, Workbook, parseValueToInternalRepresentation, } from '@jvalue/jayvee-execution'; @@ -17,8 +18,6 @@ import { } from '@jvalue/jayvee-language-server'; import * as exceljs from 'exceljs'; -import { type ColumnDefinitionEntry } from '../src/lib/table-interpreter-executor'; - export async function createWorkbookFromLocalExcelFile( fileName: string, ): Promise { @@ -50,10 +49,11 @@ export async function createWorkbookFromLocalExcelFile( return workbook; } -export type ReducedColumnDefinitionEntry = Pick< - ColumnDefinitionEntry, - 'sheetColumnIndex' | 'columnName' | 'valueType' ->; +export type ReducedColumnDefinitionEntry = { + sheetColumnIndex: number; + columnName: string; + valueType: ValueType; +}; /** * Creates a Table from the first sheet of the excel file pointed to by {@link fileName} @@ -71,15 +71,17 @@ export async function createTableFromLocalExcelFile( await workBookFromFile.xlsx.readFile(path.resolve(__dirname, fileName)); const workSheet = workBookFromFile.worksheets[0] as exceljs.Worksheet; - const table = new Table(); + const columns = new Map(); columnDefinitions.forEach((columnDefinition) => { - table.addColumn(columnDefinition.columnName, { + columns.set(columnDefinition.columnName, { values: [], valueType: columnDefinition.valueType, }); }); + const table = new Table(0, columns, []); + workSheet.eachRow((row) => { const tableRow = constructTableRow(row, columnDefinitions); table.addRow(tableRow); @@ -91,7 +93,10 @@ function constructTableRow( row: exceljs.Row, columnDefinitions: ReducedColumnDefinitionEntry[], ): TableRow { - const tableRow: TableRow = {}; + const tableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); row.eachCell( { includeEmpty: true }, @@ -102,9 +107,9 @@ function constructTableRow( const value = cell.text; const valueType = columnDefinition.valueType; - tableRow[columnDefinition.columnName] = parseAndValidatePrimitiveValue( - value, - valueType, + tableRow.set( + columnDefinition.columnName, + parseAndValidatePrimitiveValue(value, valueType), ); }, ); diff --git a/libs/interpreter-lib/src/interpreter.spec.ts b/libs/interpreter-lib/src/interpreter.spec.ts index 4043ddc84..3859d1bdb 100644 --- a/libs/interpreter-lib/src/interpreter.spec.ts +++ b/libs/interpreter-lib/src/interpreter.spec.ts @@ -200,7 +200,8 @@ describe('Interpreter', () => { assert(input != null); assert(input instanceof Table); - expect(input.getNumberOfColumns()).toBe(1); + const initial_columns = 12; + expect(input.getNumberOfColumns()).toBe(initial_columns); expect(input.getColumn('name')?.values).toStrictEqual(EXPECTED_NAMES); expect(isOk(output)).toBe(true); @@ -210,7 +211,7 @@ describe('Interpreter', () => { assert(out != null); assert(out instanceof Table); - expect(out.getNumberOfColumns()).toBe(2); + expect(out.getNumberOfColumns()).toBe(initial_columns + 1); expect(out.getColumn('name')?.values).toStrictEqual(EXPECTED_NAMES); expect(out.getColumn('nameCopy')?.values).toStrictEqual( EXPECTED_NAMES, diff --git a/libs/interpreter-lib/test/assets/graph/composite-block.jv b/libs/interpreter-lib/test/assets/graph/composite-block.jv index d1727c192..8ba0df960 100644 --- a/libs/interpreter-lib/test/assets/graph/composite-block.jv +++ b/libs/interpreter-lib/test/assets/graph/composite-block.jv @@ -15,11 +15,20 @@ pipeline CarsPipeline { enclosing: '"'; } + valuetype CarName { + property name oftype text; + } + + transform parser { + from r oftype Collection; + to y oftype CarName; + y: { name: asText (r cellInColumn "name") }; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - ]; + columns: CarName; + parseWith: parser; } transform copy { diff --git a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv index 24844e45e..c31ce3229 100644 --- a/libs/interpreter-lib/test/assets/graph/two-pipelines.jv +++ b/libs/interpreter-lib/test/assets/graph/two-pipelines.jv @@ -31,22 +31,45 @@ pipeline CarsPipeline { ]; } + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + block CarsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "name" oftype text, - "mpg" oftype decimal, - "cyl" oftype integer, - "disp" oftype decimal, - "hp" oftype integer, - "drat" oftype decimal, - "wt" oftype decimal, - "qsec" oftype decimal, - "vs" oftype integer, - "am" oftype integer, - "gear" oftype integer, - "carb" oftype integer - ]; + columns: Car; + parseWith: CarParser; } block CarsLoader oftype SQLiteLoader { @@ -83,27 +106,56 @@ pipeline ElectricVehiclesPipeline { block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { } + valuetype ElectricVehicle { + property vin oftype VehicleIdentificationNumber10; + property county oftype text; + property city oftype text; + property state oftype text; + property postal oftype text; + property modelYear oftype integer; + property make oftype text; + property model oftype text; + property evType oftype text; + property cafvEligibility oftype text; + property electricRange oftype integer; + property baseMSRP oftype integer; + property legislativeDistrict oftype text; + property dolID oftype integer; + property location oftype text; + property utility oftype text; + property censusTract oftype text; + } + + transform ElectricVehicleParser { + from r oftype Collection; + to ev oftype ElectricVehicle; + + ev: { + vin: r cellInColumn "VIN (1-10)", + county: asText (r cellInColumn "County"), + city: asText (r cellInColumn "City"), + state: asText (r cellInColumn "State"), + postal: asText (r cellInColumn "Postal Code"), + modelYear: asInteger (r cellInColumn "Model Year"), + make: asText (r cellInColumn "Make"), + model: asText (r cellInColumn "Model"), + evType: asText (r cellInColumn "Electric Vehicle Type"), + cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"), + electricRange: asInteger (r cellInColumn "Electric Range"), + baseMSRP: asInteger (r cellInColumn "Base MSRP"), + legislativeDistrict: asText (r cellInColumn "Legislative District"), + dolID: asInteger (r cellInColumn "DOL Vehicle ID"), + location: asText (r cellInColumn "Vehicle Location"), + utility: asText (r cellInColumn "Electric Utility"), + censusTract: asText (r cellInColumn "2020 Census Tract"), + }; + } + + block ElectricVehiclesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "VIN (1-10)" oftype VehicleIdentificationNumber10, - "County" oftype text, - "City" oftype text, - "State" oftype text, - "Postal Code" oftype text, - "Model Year" oftype integer, - "Make" oftype text, - "Model" oftype text, - "Electric Vehicle Type" oftype text, - "Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text, - "Electric Range" oftype integer, - "Base MSRP" oftype integer, - "Legislative District" oftype text, - "DOL Vehicle ID" oftype integer, - "Vehicle Location" oftype text, - "Electric Utility" oftype text, - "2020 Census Tract" oftype text, - ]; + columns: ElectricVehicle; + parseWith: ElectricVehicleParser; } block ElectricRangeTransformer oftype TableTransformer { diff --git a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv index 3595e7b4e..115a62157 100644 --- a/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv +++ b/libs/interpreter-lib/test/assets/hooks/valid-builtin-and-composite-blocks.jv @@ -28,12 +28,46 @@ pipeline CarsPipeline { ]; } - block CarsTableInterpreter oftype TableInterpreter { - header: true; - columns: [ - "name" oftype text, - ]; - } + valuetype Car { + property name oftype text; + property mpg oftype decimal; + property cyl oftype integer; + property disp oftype decimal; + property hp oftype integer; + property drat oftype decimal; + property wt oftype decimal; + property qsec oftype decimal; + property vs oftype integer; + property am oftype integer; + property gear oftype integer; + property carb oftype integer; + } + + transform CarParser { + from r oftype Collection; + to car oftype Car; + + car: { + name: asText (r cellInColumn "name"), + mpg: asDecimal (r cellInColumn "mpg"), + cyl: asInteger (r cellInColumn 2), + disp: asDecimal (r cellInColumn 3), + hp: asInteger (r cellInColumn "hp"), + drat: asDecimal (r cellInColumn "drat"), + wt: asDecimal (r cellInColumn "wt"), + qsec: asDecimal (r cellInColumn "qsec"), + vs: asInteger (r cellInColumn "vs"), + am: asInteger (r cellInColumn "am"), + gear: asInteger (r cellInColumn "gear"), + carb: asInteger (r cellInColumn "carb") + }; + } + + block CarsTableInterpreter oftype TableInterpreter { + header: true; + columns: Car; + parseWith: CarParser; + } transform copy { from s oftype text; diff --git a/libs/language-server/src/grammar/expression.langium b/libs/language-server/src/grammar/expression.langium index 4e8bb5c6e..aa490cf54 100644 --- a/libs/language-server/src/grammar/expression.langium +++ b/libs/language-server/src/grammar/expression.langium @@ -17,13 +17,13 @@ IfExpression infers Expression: ReplaceExpression ({infer TernaryExpression.first=current} operator='if' second=ReplaceExpression 'else' third=ReplaceExpression)*; -// The nesting of the following rules implies the precedence of the operators: ReplaceExpression infers Expression: BinaryExpression ({infer TernaryExpression.first=current} operator='replace' second=BinaryExpression 'with' third=BinaryExpression)*; infix BinaryExpression on PrimaryExpression: - 'pow' | 'root' // Higher precedence + 'cellInColumn' // Higher precedence + > 'pow' | 'root' > '*' | '/' | '%' > '+' | '-' > '<' | '<=' | '>' | '>=' @@ -53,6 +53,7 @@ ValueLiteral: | CellRangeLiteral | ValuetypeAssignmentLiteral | CollectionLiteral + | TableRowLiteral | ErrorLiteral; TextLiteral: @@ -71,6 +72,11 @@ RegexLiteral: CollectionLiteral: '[' (values+=(Expression) (',' values+=(Expression))*)? ','? ']'; +TableRowLiteral: + '{' (cells+=(TableCellLiteral) (',' cells+=(TableCellLiteral))*)? ','? '}'; +TableCellLiteral: + name=ID ':' expression=Expression; + ErrorLiteral: error= "invalid" | "missing"; @@ -88,6 +94,7 @@ Referencable: | BlockTypeProperty | TransformDefinition | TransformPortDefinition + | CustomValuetypeDefinition | ValueTypeProperty; NestedPropertyAccess: diff --git a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts index 338ae12c3..aa11f329d 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluate-expression.ts @@ -22,6 +22,7 @@ import { isFreeVariableLiteral, isRegexLiteral, isRuntimeParameterLiteral, + isTableRowLiteral, isTernaryExpression, isUnaryExpression, isValueLiteral, @@ -172,6 +173,24 @@ function evaluateValueLiteral( } return evaluatedCollection; } + if (isTableRowLiteral(expression)) { + const tableRow = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); + + for (const cell of expression.cells) { + const cellValue = evaluateExpression( + cell.expression, + evaluationContext, + wrapperFactories, + validationContext, + strategy, + ); + tableRow.set(cell.name, cellValue); + } + return tableRow; + } if (isCellRangeLiteral(expression)) { if (!wrapperFactories.CellRange.canWrap(expression)) { return new InvalidValue( diff --git a/libs/language-server/src/lib/ast/expressions/evaluation-context.ts b/libs/language-server/src/lib/ast/expressions/evaluation-context.ts index bdadd9efc..72cacf414 100644 --- a/libs/language-server/src/lib/ast/expressions/evaluation-context.ts +++ b/libs/language-server/src/lib/ast/expressions/evaluation-context.ts @@ -21,6 +21,7 @@ import { isTransformPortDefinition, isValueKeywordLiteral, isValueTypeProperty, + isValuetypeDefinition, } from '../generated/ast'; import { type ValueTypeProvider } from '../wrappers'; import { type ValueType } from '../wrappers/value-type/value-type'; @@ -45,6 +46,8 @@ export class EvaluationContext { | InternalValidValueRepresentation | InternalErrorValueRepresentation = NO_KEYWORD_ERROR; + private headerRow: string[] | undefined = undefined; + constructor( public readonly runtimeParameterProvider: RuntimeParameterProvider, public readonly operatorRegistry: OperatorEvaluatorRegistry, @@ -138,6 +141,9 @@ export class EvaluationContext { ) ); } + if (isValuetypeDefinition(dereferenced)) { + return dereferenced; + } assertUnreachable(dereferenced); } @@ -167,4 +173,12 @@ export class EvaluationContext { | InternalErrorValueRepresentation { return this.valueKeywordValue; } + + getColumnIndex(columnName: string): number | undefined { + const columnIdx = this.headerRow?.findIndex((cN) => cN === columnName); + return columnIdx === -1 ? undefined : columnIdx; + } + setHeaderRow(headerRow: string[]): void { + this.headerRow = headerRow; + } } diff --git a/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts b/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts new file mode 100644 index 000000000..237c687db --- /dev/null +++ b/libs/language-server/src/lib/ast/expressions/evaluators/cell-in-column-operator-evaluator.ts @@ -0,0 +1,106 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type BinaryExpression } from '../../generated/ast'; +import { type OperatorEvaluator } from '../operator-evaluator'; +import { type EvaluationContext } from '../evaluation-context'; +import { type WrapperFactoryProvider } from '../../wrappers'; +import { EvaluationStrategy } from '../evaluation-strategy'; +import { type ValidationContext } from '../../../validation'; +import { + InvalidValue, + MissingValue, + type InternalErrorValueRepresentation, +} from '../internal-value-representation'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; +import { evaluateExpression } from '../evaluate-expression'; +import { + COLLECTION_TYPEGUARD, + ERROR_TYPEGUARD, + INVALID_TYPEGUARD, + MISSING_TYPEGUARD, + NUMBER_TYPEGUARD, + STRING_TYPEGUARD, +} from '../typeguards'; + +export class CellInColumnOperatorEvaluator + implements OperatorEvaluator +{ + public readonly operator = 'cellInColumn' as const; + + evaluate( + expression: BinaryExpression, + evaluationContext: EvaluationContext, + wrapperFactories: WrapperFactoryProvider, + strategy: EvaluationStrategy, + validationContext: ValidationContext | undefined, + ): string | InternalErrorValueRepresentation { + assert(expression.operator === this.operator); + + const sheetRowValues = evaluateExpression( + expression.left, + evaluationContext, + wrapperFactories, + validationContext, + strategy, + ); + + if ( + strategy === EvaluationStrategy.LAZY && + ERROR_TYPEGUARD(sheetRowValues) + ) { + return sheetRowValues; + } + + const accessor = evaluateExpression( + expression.right, + evaluationContext, + wrapperFactories, + validationContext, + strategy, + ); + if (INVALID_TYPEGUARD(sheetRowValues)) { + return sheetRowValues; + } + if (INVALID_TYPEGUARD(accessor)) { + return accessor; + } + if (MISSING_TYPEGUARD(sheetRowValues)) { + return sheetRowValues; + } + if (MISSING_TYPEGUARD(accessor)) { + return accessor; + } + + assert(COLLECTION_TYPEGUARD(sheetRowValues)); + assert(sheetRowValues.every((value) => STRING_TYPEGUARD(value))); + + assert(STRING_TYPEGUARD(accessor) || NUMBER_TYPEGUARD(accessor)); + + const index = + typeof accessor === 'string' + ? evaluationContext.getColumnIndex(accessor) + : accessor; + if (index === undefined) { + validationContext?.accept( + 'error', + `Could not find index for column "${accessor}"`, + { + node: expression.right, + }, + ); + return new MissingValue(`Could not find index for column ${accessor}`); + } + + if (index >= sheetRowValues.length) { + return new InvalidValue( + `Cannot access column ${index} in a sheet with only ${sheetRowValues.length} columns`, + ); + } + const value = sheetRowValues.at(index); + assert(value !== undefined); + return value; + } +} diff --git a/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts b/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts index d2e93d2a7..bb13226ff 100644 --- a/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts +++ b/libs/language-server/src/lib/ast/expressions/internal-value-representation.ts @@ -13,15 +13,21 @@ import { type ConstraintDefinition, type TransformDefinition, type ValuetypeAssignment, + type ValuetypeDefinition, isBlockTypeProperty, isCellRangeLiteral, isConstraintDefinition, isTransformDefinition, isValuetypeAssignment, + isValuetypeDefinition, } from '../generated/ast'; import { type WrapperFactoryProvider } from '../wrappers'; -import { COLLECTION_TYPEGUARD, ERROR_TYPEGUARD } from './typeguards'; +import { + COLLECTION_TYPEGUARD, + ERROR_TYPEGUARD, + TABLEROW_TYPEGUARD, +} from './typeguards'; // INFO: `ErroneousValue` extends `Error` in order to make use of the `stack` // property. @@ -69,8 +75,14 @@ export class MissingValue extends ErroneousValue { export type InternalErrorValueRepresentation = InvalidValue | MissingValue; +export type TableRow = Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation +>; + export type InternalValidValueRepresentation = | AtomicInternalValidValueRepresentation + | TableRow | (InternalValidValueRepresentation | InternalErrorValueRepresentation)[]; export type AtomicInternalValidValueRepresentation = @@ -81,9 +93,24 @@ export type AtomicInternalValidValueRepresentation = | CellRangeLiteral | ConstraintDefinition | ValuetypeAssignment + | ValuetypeDefinition | BlockTypeProperty | TransformDefinition; +function internalValueToStringRecursionHelper( + value: InternalValidValueRepresentation | InternalErrorValueRepresentation, + wrapperFactories?: WrapperFactoryProvider, +): string { + if (isCellRangeLiteral(value)) { + assert(wrapperFactories !== undefined); + return internalValueToString(value, wrapperFactories); + } + if (ERROR_TYPEGUARD(value)) { + return value.name; + } + return internalValueToString(value); +} + export type InternalValidValueRepresentationTypeguard< T extends InternalValidValueRepresentation, > = (value: unknown) => value is T; @@ -106,21 +133,23 @@ export function internalValueToString( return ( '[ ' + valueRepresentation - .map((value) => { - if (isCellRangeLiteral(value)) { - assert(wrapperFactories !== undefined); - return internalValueToString(value, wrapperFactories); - } - if (ERROR_TYPEGUARD(value)) { - return value.name; - } - return internalValueToString(value); - }) + .map((value) => + internalValueToStringRecursionHelper(value, wrapperFactories), + ) .join(', ') + ' ]' ); } + if (valueRepresentation instanceof Map) { + return `{ ${[...valueRepresentation.entries()] + .map( + ([columnName, cellValue]) => + `${columnName} : ${internalValueToStringRecursionHelper(cellValue, wrapperFactories)}`, + ) + .join(', ')} }`; + } + if (typeof valueRepresentation === 'boolean') { return String(valueRepresentation); } @@ -153,6 +182,9 @@ export function internalValueToString( if (isValuetypeAssignment(valueRepresentation)) { return valueRepresentation.name; } + if (isValuetypeDefinition(valueRepresentation)) { + return valueRepresentation.name; + } if (isTransformDefinition(valueRepresentation)) { return valueRepresentation.name; } @@ -169,6 +201,17 @@ export function cloneInternalValue< return valueRepresentation.map(cloneInternalValue) as T; } + if (TABLEROW_TYPEGUARD(valueRepresentation)) { + const clonedMap = new Map< + string, + InternalValidValueRepresentation | InternalErrorValueRepresentation + >(); + for (const [columnName, cellValue] of valueRepresentation.entries()) { + clonedMap.set(columnName, cellValue); + } + return clonedMap as T; + } + if (ERROR_TYPEGUARD(valueRepresentation)) { return valueRepresentation.clone() as T; } diff --git a/libs/language-server/src/lib/ast/expressions/operator-registry.ts b/libs/language-server/src/lib/ast/expressions/operator-registry.ts index 16db0d190..d541c18b6 100644 --- a/libs/language-server/src/lib/ast/expressions/operator-registry.ts +++ b/libs/language-server/src/lib/ast/expressions/operator-registry.ts @@ -15,6 +15,7 @@ import { import { AdditionOperatorEvaluator } from './evaluators/addition-operator-evaluator'; import { AndOperatorEvaluator } from './evaluators/and-operator-evaluator'; import { CeilOperatorEvaluator } from './evaluators/ceil-operator-evaluator'; +import { CellInColumnOperatorEvaluator } from './evaluators/cell-in-column-operator-evaluator'; import { DivisionOperatorEvaluator } from './evaluators/division-operator-evaluator'; import { EqualityOperatorEvaluator } from './evaluators/equality-operator-evaluator'; import { FloorOperatorEvaluator } from './evaluators/floor-operator-evaluator'; @@ -60,6 +61,7 @@ import { type UnaryExpressionOperator, } from './operator-types'; import { BasicArithmeticOperatorTypeComputer } from './type-computers/basic-arithmetic-operator-type-computer'; +import { CellInColumnOperatorTypeComputer } from './type-computers/cell-in-column-operator-type-computer'; import { DivisionOperatorTypeComputer } from './type-computers/division-operator-type-computer'; import { EqualityOperatorTypeComputer } from './type-computers/equality-operator-type-computer'; import { ExponentialOperatorTypeComputer } from './type-computers/exponential-operator-type-computer'; @@ -117,6 +119,7 @@ export class DefaultOperatorEvaluatorRegistry lengthof: new LengthofOperatorEvaluator(), }; binary = { + cellInColumn: new CellInColumnOperatorEvaluator(), pow: new PowOperatorEvaluator(), root: new RootOperatorEvaluator(), '*': new MultiplicationOperatorEvaluator(), @@ -164,6 +167,7 @@ export class DefaultOperatorTypeComputerRegistry lengthof: new LengthofOperatorTypeComputer(this.valueTypeProvider), }; binary = { + cellInColumn: new CellInColumnOperatorTypeComputer(this.valueTypeProvider), pow: new ExponentialOperatorTypeComputer(this.valueTypeProvider), root: new ExponentialOperatorTypeComputer(this.valueTypeProvider), '*': new BasicArithmeticOperatorTypeComputer(this.valueTypeProvider), diff --git a/libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts b/libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts new file mode 100644 index 000000000..95408757b --- /dev/null +++ b/libs/language-server/src/lib/ast/expressions/type-computers/cell-in-column-operator-type-computer.ts @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type ValidationContext } from '../../../validation/validation-context'; +import { type BinaryExpression } from '../../generated/ast'; +import { + type ValueType, + type ValueTypeProvider, + isCollectionValueType, +} from '../../wrappers/value-type'; +import { type BinaryOperatorTypeComputer } from '../operator-type-computer'; + +export class CellInColumnOperatorTypeComputer + implements BinaryOperatorTypeComputer +{ + constructor(protected readonly valueTypeProvider: ValueTypeProvider) {} + + computeType( + leftOperandType: ValueType, + rightOperandType: ValueType, + expression: BinaryExpression, + context: ValidationContext | undefined, + ): ValueType | undefined { + if ( + !isCollectionValueType( + leftOperandType, + this.valueTypeProvider.Primitives.Text, + ) + ) { + context?.accept( + 'error', + `Operator does not support type ${leftOperandType.getName()}`, + { + node: expression.left, + }, + ); + return undefined; + } + + if ( + !rightOperandType.equals(this.valueTypeProvider.Primitives.Text) && + !rightOperandType.equals(this.valueTypeProvider.Primitives.Integer) + ) { + context?.accept( + 'error', + `Operator does not support type ${rightOperandType.getName()}`, + { + node: expression.right, + }, + ); + return undefined; + } + return this.valueTypeProvider.Primitives.Text; + } +} diff --git a/libs/language-server/src/lib/ast/expressions/type-inference.ts b/libs/language-server/src/lib/ast/expressions/type-inference.ts index 422892bdc..54239edcb 100644 --- a/libs/language-server/src/lib/ast/expressions/type-inference.ts +++ b/libs/language-server/src/lib/ast/expressions/type-inference.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { assertUnreachable } from 'langium'; +import { assertUnreachable, AstUtils, type AstNode } from 'langium'; // eslint-disable-next-line unicorn/prefer-node-protocol import assert from 'assert'; @@ -29,6 +29,7 @@ import { isNumericLiteral, isReferenceLiteral, isRegexLiteral, + isTableRowLiteral, isTernaryExpression, isTextLiteral, isTransformDefinition, @@ -38,9 +39,12 @@ import { isValueLiteral, isValueTypeProperty, isValuetypeAssignmentLiteral, + isValuetypeDefinition, + type TableRowLiteral, } from '../generated/ast'; import { getNextAstNodeContainer } from '../model-util'; import { + type AtomicValueType, isAtomicValueType, type ValueType, type ValueTypeProvider, @@ -185,6 +189,13 @@ function inferTypeFromExpressionLiteral( valueTypeProvider, wrapperFactories, ); + } else if (isTableRowLiteral(expression)) { + return inferTableRowType( + expression, + validationContext, + valueTypeProvider, + wrapperFactories, + ); } else if (isErrorLiteral(expression)) { return undefined; } @@ -296,6 +307,73 @@ function inferCollectionElementTypes( return undefined; } +export function equalSchemas( + a: Map, + b: Map, +): boolean { + if (a.size !== b.size) { + return false; + } + for (const [columnName, cellValue] of a) { + assert(cellValue !== undefined); + const otherCellValue = b.get(columnName); + if (cellValue !== otherCellValue) { + return false; + } + } + return true; +} + +function findValueTypeWithSameSchema( + schema: Map, + node: AstNode, + wrapperFactories: WrapperFactoryProvider, +): AtomicValueType | undefined { + const root = AstUtils.findRootNode(node); + const valueTypesWithSameSchema = AstUtils.streamAllContents(root) + .filter((astNode) => isValuetypeDefinition(astNode)) + .map((valueTypeDefinition) => { + const valueType = wrapperFactories.ValueType.wrap(valueTypeDefinition); + assert(valueType !== undefined); + return valueType; + }) + .filter( + (valueType) => + isAtomicValueType(valueType) && + equalSchemas(schema, valueType.getSchema()), + ) + .map((valueType) => { + assert(isAtomicValueType(valueType)); + return valueType; + }) + .toArray(); + + return onlyElementOrUndefined(valueTypesWithSameSchema); +} + +function inferTableRowType( + tableRow: TableRowLiteral, + validationContext: ValidationContext, + valueTypeProvider: ValueTypeProvider, + wrapperFactories: WrapperFactoryProvider, +): AtomicValueType | undefined { + const schema = new Map(); + for (const cell of tableRow.cells) { + const cellValueType = inferExpressionType( + cell.expression, + validationContext, + valueTypeProvider, + wrapperFactories, + ); + if (cellValueType === undefined) { + return undefined; + } + schema.set(cell.name, cellValueType); + } + + return findValueTypeWithSameSchema(schema, tableRow, wrapperFactories); +} + function inferTypeFromValueKeyword( expression: ValueKeywordLiteral, validationContext: ValidationContext, @@ -342,6 +420,9 @@ function inferTypeFromReferenceLiteral( if (isTransformDefinition(referenced)) { return valueTypeProvider.Primitives.Transform; } + if (isValuetypeDefinition(referenced)) { + return valueTypeProvider.Primitives.ValuetypeDefinition; + } if ( isTransformPortDefinition(referenced) || isBlockTypeProperty(referenced) || diff --git a/libs/language-server/src/lib/ast/expressions/typeguards.ts b/libs/language-server/src/lib/ast/expressions/typeguards.ts index 9a1598007..f3e59fa0a 100644 --- a/libs/language-server/src/lib/ast/expressions/typeguards.ts +++ b/libs/language-server/src/lib/ast/expressions/typeguards.ts @@ -16,6 +16,7 @@ import { } from '../generated/ast'; import { + type TableRow, type AtomicInternalValidValueRepresentation, type InternalErrorValueRepresentation, type InternalValidValueRepresentation, @@ -28,6 +29,7 @@ export const INTERNAL_VALID_VALUE_REPRESENTATION_TYPEGUARD: InternalValidValueRe InternalValidValueRepresentation > = (value): value is InternalValidValueRepresentation => ATOMIC_INTERNAL_VALUE_REPRESENTATION_TYPEGUARD(value) || + TABLEROW_TYPEGUARD(value) || COLLECTION_TYPEGUARD(value); export const ATOMIC_INTERNAL_VALUE_REPRESENTATION_TYPEGUARD: InternalValidValueRepresentationTypeguard< @@ -76,6 +78,17 @@ export const TRANSFORMDEFINITION_TYPEGUARD: InternalValidValueRepresentationType TransformDefinition > = (value) => isTransformDefinition(value); +export const TABLEROW_TYPEGUARD: InternalValidValueRepresentationTypeguard< + TableRow +> = (value): value is TableRow => + value instanceof Map && + [...value.entries()].every( + ([key, value]) => + STRING_TYPEGUARD(key) && + (INTERNAL_VALID_VALUE_REPRESENTATION_TYPEGUARD(value) || + ERROR_TYPEGUARD(value)), + ); + export const COLLECTION_TYPEGUARD: InternalValidValueRepresentationTypeguard< (InternalValidValueRepresentation | InternalErrorValueRepresentation)[] > = (value) => diff --git a/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts b/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts index 4e6614eca..b813b6ec9 100644 --- a/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts +++ b/libs/language-server/src/lib/ast/wrappers/typed-object/block-type-wrapper.ts @@ -119,9 +119,9 @@ export class BlockTypeWrapper extends TypedObjectWrapper { - return property.valueType.reference.ref === undefined; - }) + blockTypeDefinition.properties.some( + (property) => property.valueType.reference.ref === undefined, + ) ) { return false; } diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts index 21a1e9380..87316910c 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/abstract-value-type.ts @@ -49,13 +49,15 @@ export abstract class AbstractValueType< if (cycleDetected) { return -1; } - visited.push(this); const idx = this.doGetContainedTypes().findIndex( (containedType) => - containedType.getIndexOfFirstPropertyInATypeCycle(visited) !== - undefined, + containedType.getIndexOfFirstPropertyInATypeCycle([ + ...visited, + this, + ]) !== undefined, ); + return idx !== -1 ? idx : undefined; } } diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts index 9a8725908..b697638a2 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/atomic-value-type.ts @@ -46,6 +46,18 @@ export class AtomicValueType return this.astNode?.properties; } + getSchema(): Map { + const schema = new Map(); + for (const property of this.getProperties()) { + const valueType = this.wrapperFactories.ValueType.wrap( + property.valueType, + ); + assert(valueType !== undefined); + schema.set(property.name, valueType); + } + return schema; + } + private getNestedProperty( nestedPropertyAccess: NestedPropertyAccess, ): ValueTypeProperty | undefined { diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts index b4ea4d4b5..468c38901 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/collection/abstract-collection-value-type.ts @@ -14,4 +14,8 @@ export abstract class AbstractCollectionValueType< override isAllowedAsRuntimeParameter(): boolean { return false; } + + override isReferenceableByUser(): boolean { + return true; + } } diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts index c0158b62b..103301814 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/index.ts @@ -19,8 +19,9 @@ export { type DecimalValuetype } from './decimal-value-type'; export { type IntegerValuetype } from './integer-value-type'; export { type RegexValuetype } from './regex-value-type'; export { type TextValuetype } from './text-value-type'; -export { type ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; export { type TransformValuetype } from './transform-value-type'; +export { type ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; +export { type ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; export { ValueTypeProvider, diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts index a4d90f2ab..3563b78a0 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/primitive-value-type-provider.ts @@ -17,6 +17,7 @@ import { RegexValuetype } from './regex-value-type'; import { TextValuetype } from './text-value-type'; import { TransformValuetype } from './transform-value-type'; import { ValuetypeAssignmentValuetype } from './value-type-assignment-value-type'; +import { ValuetypeDefinitionValuetype } from './value-type-definition-value-type'; /** * Should be created as singleton due to the equality comparison of primitive value types. @@ -43,6 +44,7 @@ export class PrimitiveValueTypeProvider { CellRange = new CellRangeValuetype(); Constraint = new ConstraintValuetype(); ValuetypeAssignment = new ValuetypeAssignmentValuetype(); + ValuetypeDefinition = new ValuetypeDefinitionValuetype(); Transform = new TransformValuetype(); @@ -56,6 +58,7 @@ export class PrimitiveValueTypeProvider { this.CellRange, this.Constraint, this.ValuetypeAssignment, + this.ValuetypeDefinition, this.Transform, ]; } diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts new file mode 100644 index 000000000..17a09711e --- /dev/null +++ b/libs/language-server/src/lib/ast/wrappers/value-type/primitive/value-type-definition-value-type.ts @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { type InternalValidValueRepresentation } from '../../../expressions/internal-value-representation'; +import { + type ValuetypeDefinition as AstValuetypeDefinition, + isValuetypeDefinition as isAstValuetypeDefinition, +} from '../../../generated/ast'; +import { type ValueTypeVisitor } from '../value-type'; + +import { PrimitiveValueType } from './primitive-value-type'; + +export class ValuetypeDefinitionValuetype extends PrimitiveValueType { + acceptVisitor(visitor: ValueTypeVisitor): R { + return visitor.visitValuetypeDefinition(this); + } + + override isAllowedAsRuntimeParameter(): boolean { + return false; + } + + override getName(): 'ValuetypeDefinition' { + return 'ValuetypeDefinition'; + } + + override isInternalValidValueRepresentation( + operandValue: InternalValidValueRepresentation, + ): operandValue is AstValuetypeDefinition { + return isAstValuetypeDefinition(operandValue); + } +} diff --git a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts index d2b1015cd..dab5cf62a 100644 --- a/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts +++ b/libs/language-server/src/lib/ast/wrappers/value-type/value-type.ts @@ -9,6 +9,7 @@ import { import { type AtomicValueType } from './atomic-value-type'; import { + type ValuetypeDefinitionValuetype, type BooleanValuetype, type CellRangeValuetype, type CollectionValueType, @@ -85,6 +86,7 @@ export abstract class ValueTypeVisitor { abstract visitRegex(valueType: RegexValuetype): R; abstract visitConstraint(valueType: ConstraintValuetype): R; abstract visitValuetypeAssignment(valueType: ValuetypeAssignmentValuetype): R; + abstract visitValuetypeDefinition(valueType: ValuetypeDefinitionValuetype): R; abstract visitCollection( valueType: CollectionValueType | EmptyCollectionValueType, ): R; diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts index 9302bb750..99b43b2a9 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.spec.ts @@ -326,28 +326,6 @@ describe('Validation of block type specific properties', () => { }); describe('TableInterpreter block type', () => { - it('should diagnose error on non unique column names', async () => { - const text = readJvTestAsset( - 'property-assignment/block-type-specific/table-interpreter/invalid-non-unique-column-names.jv', - ); - - await parseAndValidatePropertyAssignment(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(2); - expect(validationAcceptorMock).toHaveBeenNthCalledWith( - 1, - 'error', - 'The column name "name" needs to be unique.', - expect.any(Object), - ); - expect(validationAcceptorMock).toHaveBeenNthCalledWith( - 2, - 'error', - 'The column name "name" needs to be unique.', - expect.any(Object), - ); - }); - it('should diagnose no error', async () => { const text = readJvTestAsset( 'property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv', diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts index 68c92736e..266f72ec6 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-assignment.ts @@ -13,7 +13,6 @@ import { isRowWrapper, } from '../../../ast'; import { type JayveeValidationProps } from '../../validation-registry'; -import { checkUniqueNames } from '../../validation-util'; export function checkBlockTypeSpecificProperties( property: PropertyAssignment, @@ -308,15 +307,16 @@ function checkTableInterpreterProperty( property, props.evaluationContext, props.wrapperFactories, - props.valueTypeProvider.createCollectionValueTypeOf( - props.valueTypeProvider.Primitives.ValuetypeAssignment, - ), + props.valueTypeProvider.Primitives.ValuetypeDefinition, ); if (ERROR_TYPEGUARD(valueTypeAssignments)) { + props.validationContext.accept( + 'error', + "The property 'columns' must have a value that is a reference to a value type definition", + { node: property }, + ); return; } - - checkUniqueNames(valueTypeAssignments, props.validationContext, 'column'); } } diff --git a/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts b/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts index f86a56837..5468ef552 100644 --- a/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts +++ b/libs/language-server/src/lib/validation/checks/block-type-specific/property-body.ts @@ -3,7 +3,11 @@ // SPDX-License-Identifier: AGPL-3.0-only import { + type AtomicValueType, ERROR_TYPEGUARD, + isAtomicValueType, + isTableRowLiteral, + type TransformDefinition, type PropertyBody, evaluatePropertyValue, } from '../../../ast'; @@ -20,6 +24,8 @@ export function checkBlockTypeSpecificPropertyBody( return checkCellWriterPropertyBody(propertyBody, props); case 'TableTransformer': return checkTableTransformerPropertyBody(propertyBody, props); + case 'TableInterpreter': + return checkTableInterpreterPropertyBody(propertyBody, props); default: } } @@ -123,6 +129,98 @@ function checkTableTransformerPropertyBody( checkInputColumnsMatchTransformationPorts(propertyBody, props); } +function checkTableInterpreterPropertyBody( + propertyBody: PropertyBody, + props: JayveeValidationProps, +): void { + const parseWithProperty = propertyBody.properties.find( + (property) => property.name == 'parseWith', + ); + if (parseWithProperty === undefined) { + return; + } + + const transform = evaluatePropertyValue( + parseWithProperty, + props.evaluationContext, + props.wrapperFactories, + props.valueTypeProvider.Primitives.Transform, + ); + + if (ERROR_TYPEGUARD(transform)) { + return; + } + + const columnsProperty = propertyBody.properties.find( + (property) => property.name == 'columns', + ); + if (columnsProperty === undefined) { + return; + } + + const columns = evaluatePropertyValue( + columnsProperty, + props.evaluationContext, + props.wrapperFactories, + props.valueTypeProvider.Primitives.ValuetypeDefinition, + ); + if (ERROR_TYPEGUARD(columns)) { + return; + } + + const columnsValueType = props.wrapperFactories.ValueType.wrap(columns); + if (columnsValueType === undefined) { + return; + } + if (!isAtomicValueType(columnsValueType)) { + props.validationContext.accept( + 'error', + 'This value type must be user created and reflect the columns that should be in the table', + { node: columnsProperty.value }, + ); + return; + } + + checkParseWithTransform(transform, columnsValueType, props); +} + +function checkParseWithTransform( + transform: TransformDefinition, + columnsValueType: AtomicValueType, + props: JayveeValidationProps, +): void { + for (const port of transform.body.ports) { + if (port.kind !== 'to') { + continue; + } + + const portValueType = props.wrapperFactories.ValueType.wrap(port.valueType); + if (portValueType === undefined) { + continue; + } + + if (!portValueType.equals(columnsValueType)) { + props.validationContext.accept( + 'error', + `Expected value type ${columnsValueType.getName()}, but found ${portValueType.getName()}`, + { node: port.valueType }, + ); + } + } + + for (const outputAssignment of transform.body.outputAssignments) { + if (!isTableRowLiteral(outputAssignment.expression)) { + props.validationContext.accept( + 'error', + 'Transforms used in TableInterpreter blocks must have one output assignment using a table row expression', + { + node: outputAssignment.expression, + }, + ); + } + } +} + function checkInputColumnsMatchTransformationPorts( propertyBody: PropertyBody, props: JayveeValidationProps, diff --git a/libs/language-server/src/lib/validation/checks/transform-body.ts b/libs/language-server/src/lib/validation/checks/transform-body.ts index 6cbc34404..d52eff947 100644 --- a/libs/language-server/src/lib/validation/checks/transform-body.ts +++ b/libs/language-server/src/lib/validation/checks/transform-body.ts @@ -7,6 +7,7 @@ */ import { + isTableRowLiteral, type TransformBody, type TransformPortDefinition, isTransformPortDefinition, @@ -18,6 +19,10 @@ import { extractReferenceLiterals, validateTransformOutputAssignment, } from './transform-output-assigment'; +import { onlyElementOrUndefined } from '../../util'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; +import { isAtomicValueType } from '../../ast'; export function validateTransformBody( transformBody: TransformBody, @@ -34,6 +39,9 @@ export function validateTransformBody( checkAreInputsUsed(transformBody, props); + checkInputForTableRowTransform(transformBody, props); + checkOutputForTableRowTransform(transformBody, props); + for (const property of transformBody.outputAssignments) { validateTransformOutputAssignment(property, props); } @@ -108,6 +116,101 @@ function checkSingleOutputPort( } } +function checkInputForTableRowTransform( + transformBody: TransformBody, + props: JayveeValidationProps, +): void { + const inputs = transformBody.ports?.filter((x) => x.kind === 'from'); + const outputAssignments = transformBody?.outputAssignments; + if (inputs === undefined || outputAssignments === undefined) { + return undefined; + } + + const isTableRowTransform = outputAssignments.some((outputAssignment) => + isTableRowLiteral(outputAssignment), + ); + if (!isTableRowTransform) { + return; + } + + const textCollection = props.valueTypeProvider.createCollectionValueTypeOf( + props.valueTypeProvider.Primitives.Text, + ); + + const input = onlyElementOrUndefined(inputs); + const inputValueType = props.wrapperFactories.ValueType.wrap( + input?.valueType, + ); + assert(input !== undefined); + + if (inputValueType === undefined) { + props.validationContext.accept( + 'error', + 'Transforms with a table row expression must have exactly one input', + { + node: transformBody, + }, + ); + return; + } + + if (!inputValueType.equals(textCollection)) { + props.validationContext.accept( + 'error', + 'This input must be of type `Collection', + { + node: input.valueType, + }, + ); + return; + } +} + +function checkOutputForTableRowTransform( + transformBody: TransformBody, + props: JayveeValidationProps, +): void { + const outputs = transformBody.ports?.filter((x) => x.kind === 'to'); + const outputAssignments = transformBody?.outputAssignments; + if (outputs === undefined || outputAssignments === undefined) { + return undefined; + } + + const isTableRowTransform = outputAssignments.some((outputAssignment) => + isTableRowLiteral(outputAssignment), + ); + if (!isTableRowTransform) { + return; + } + + const output = onlyElementOrUndefined(outputs); + const outputValueType = props.wrapperFactories.ValueType.wrap( + output?.valueType, + ); + assert(output !== undefined); + if (outputValueType === undefined) { + props.validationContext.accept( + 'error', + 'Transforms with a table row expression must have exactly one output', + { + node: transformBody, + }, + ); + return; + } + + if (!isAtomicValueType(outputValueType)) { + props.validationContext.accept( + 'error', + 'This must be a user created value type', + { + node: output.valueType, + }, + ); + return; + } +} + function checkAreInputsUsed( transformBody: TransformBody, props: JayveeValidationProps, diff --git a/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts b/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts index deb2816b0..30cf299db 100644 --- a/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts +++ b/libs/language-server/src/lib/validation/checks/transform-output-assigment.ts @@ -16,12 +16,16 @@ import { isBinaryExpression, isExpressionLiteral, isReferenceLiteral, + isTableRowLiteral, isTernaryExpression, isTransformPortDefinition, isUnaryExpression, } from '../../ast/generated/ast'; import { type JayveeValidationProps } from '../validation-registry'; import { checkExpressionSimplification } from '../validation-util'; +import { isAtomicValueType } from '../../ast'; +// eslint-disable-next-line unicorn/prefer-node-protocol +import assert from 'assert'; export function validateTransformOutputAssignment( outputAssignment: TransformOutputAssignment, @@ -51,15 +55,83 @@ function checkOutputValueTyping( props.valueTypeProvider, props.wrapperFactories, ); - if (inferredType === undefined) { - return; - } const expectedType = props.wrapperFactories.ValueType.wrap(outputType); if (expectedType === undefined) { return; } + if ( + isTableRowLiteral(assignmentExpression) && + !isAtomicValueType(expectedType) + ) { + props.validationContext.accept( + 'error', + `The output type of a Table Row transform must be user created`, + { node: outputType }, + ); + return; + } + + if (inferredType === undefined && isTableRowLiteral(assignmentExpression)) { + assert(isAtomicValueType(expectedType)); + + const schema = expectedType.getSchema(); + for (const [columnName, columnValueType] of schema) { + const cell = assignmentExpression.cells.find( + (cell) => cell.name == columnName, + ); + if (cell === undefined) { + props.validationContext.accept( + 'error', + `The table row expression must contain an assignment for the property ${columnName}`, + { node: assignmentExpression }, + ); + return; + } + const cellValueType = inferExpressionType( + cell.expression, + props.validationContext, + props.valueTypeProvider, + props.wrapperFactories, + ); + if (cellValueType === undefined) { + props.validationContext.accept('error', `Could not infer value type`, { + node: cell.expression, + }); + return; + } + if ( + !cellValueType.isConvertibleTo(columnValueType) && + !columnValueType.isConvertibleTo(cellValueType) + ) { + props.validationContext.accept( + 'error', + `${cellValueType.getName()} is not convertible to ${columnValueType.getName()}`, + { + node: cell.expression, + }, + ); + } + } + for (const cell of assignmentExpression.cells) { + if (!schema.has(cell.name)) { + props.validationContext.accept( + 'error', + `${expectedType.getName()} does not have any properties with name ${cell.name}`, + { + node: cell, + }, + ); + } + } + return; + } + + if (inferredType === undefined) { + return; + } + if (!inferredType.isConvertibleTo(expectedType)) { props.validationContext.accept( 'error', diff --git a/libs/language-server/src/lib/validation/checks/value-type-reference.ts b/libs/language-server/src/lib/validation/checks/value-type-reference.ts index 9a4e48d14..c8070ac8d 100644 --- a/libs/language-server/src/lib/validation/checks/value-type-reference.ts +++ b/libs/language-server/src/lib/validation/checks/value-type-reference.ts @@ -14,6 +14,7 @@ export function validateValueTypeReference( ): void { checkGenericsMatchDefinition(valueTypeRef, props); checkIsValueTypeReferenceable(valueTypeRef, props); + checkIsValueTypeInTableParsingTransform(valueTypeRef, props); } function checkGenericsMatchDefinition( @@ -25,16 +26,6 @@ function checkGenericsMatchDefinition( return; } - const numberOfProperties = valueTypeDefinition.properties?.length; - if (numberOfProperties > 1) { - props.validationContext.accept( - 'error', - 'The referenced value type has more than one property. ' + - 'This is unsupported for now', - { node: valueTypeRef }, - ); - } - const requiredGenerics = valueTypeDefinition.genericDefinition?.generics?.length ?? 0; @@ -79,3 +70,27 @@ function checkIsValueTypeReferenceable( }, ); } + +function checkIsValueTypeInTableParsingTransform( + valueTypeRef: ValueTypeReference, + props: JayveeValidationProps, +) { + const valueTypeDefinition = valueTypeRef.reference?.ref; + if (valueTypeDefinition === undefined) { + return; + } + const parent = valueTypeRef.$container; + if (parent?.$type === 'TransformPortDefinition') { + return; + } + + const numberOfProperties = valueTypeDefinition.properties?.length; + if (numberOfProperties > 1) { + props.validationContext.accept( + 'error', + `The referenced value type has more than one property. ` + + 'This is unsupported for now', + { node: valueTypeRef }, + ); + } +} diff --git a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv index 0d60cd70f..54826c958 100644 --- a/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv +++ b/libs/language-server/src/stdlib/builtin-block-types/TableInterpreter.jv @@ -3,26 +3,33 @@ // SPDX-License-Identifier: AGPL-3.0-only /** -* Interprets a `Sheet` as a `Table`. In case a header row is present in the sheet, its names can be matched with the provided column names. Otherwise, the provided column names are assigned in order. +* Interprets a `Sheet` as a `Table`. In case a header row is present in the +* sheet, its names can be matched with the provided column names. Otherwise, the +* provided column names are assigned in order. +* +* @example Interprets a `Sheet` about cars with a topmost header row and +* interprets it as a `Table` by assigning a value type property to each column. +* The column names are matched to the header, so the order of the type +* assignments does not matter. +* valuetype Car { +* property name oftype text; +* property mpg oftype decimal; +* property cyl oftype integer; +* } * -* @example Interprets a `Sheet` about cars with a topmost header row and interprets it as a `Table` by assigning a primitive value type to each column. The column names are matched to the header, so the order of the type assignments does not matter. * block CarsTableInterpreter oftype TableInterpreter { * header: true; -* columns: [ -* "name" oftype text, -* "mpg" oftype decimal, -* "cyl" oftype integer, -* ]; +* columns: Car; * } * -* @example Interprets a `Sheet` about cars without a topmost header row and interprets it as a `Table` by sequentially assigning a name and a primitive value type to each column of the sheet. Note that the order of columns matters here. The first column (column `A`) will be named "name", the second column (column `B`) will be named "mpg" etc. +* @example Interprets a `Sheet` about cars without a topmost header row and +* interprets it as a `Table` by sequentially assigning a name and a value type +* property to each column of the sheet. Note that the order of columns matters +* here. The first column (column `A`) will be named "name", the second column +* (column `B`) will be named "mpg" etc. * block CarsTableInterpreter oftype TableInterpreter { * header: false; -* columns: [ -* "name" oftype text, -* "mpg" oftype decimal, -* "cyl" oftype integer, -* ]; +* columns: Car; * } */ publish builtin blocktype TableInterpreter { @@ -35,17 +42,14 @@ publish builtin blocktype TableInterpreter { property header oftype boolean: true; /** - * Collection of value type assignments. Uses column names (potentially matched with the header or by sequence depending on the `header` property) to assign a primitive value type to each column. + * Reference to a valuetype that defines the table schema. + * Each of the valuetype's properties symbolizes a column, the property's name + * is the column's name and the property's valuetype is the column's valuetype. */ - property columns oftype Collection; + property columns oftype ValuetypeDefinition; - /** - * Whether to ignore whitespace before values. Does not apply to `text` cells - */ - property skipLeadingWhitespace oftype boolean: true; - - /** - * Whether to ignore whitespace after values. Does not apply to `text` cells - */ - property skipTrailingWhitespace oftype boolean: true; + /** + * This transform to create each table row from the sheet row + */ + property parseWith oftype Transform; } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv index afafbe0b3..7a9fafe83 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSAgencyInterpreter.jv @@ -2,6 +2,24 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Agency { + property agencyId oftype text; // Conditional columns are considered required for now + property agencyName oftype text; + property agencyUrl oftype GTFSUrl; + property agencyTimezone oftype text; +} +transform AgencyParser { + from r oftype Collection; + to agency oftype Agency; + + agency: { + agencyId: r cellInColumn "agency_id", + agencyName: r cellInColumn "agency_name", + agencyUrl: r cellInColumn "agency_url", + agencyTimezone: r cellInColumn "agency_timezone", + }; +} + /** * A GTFSAgencyInterpreter interprets a agency.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#agencytxt @@ -27,11 +45,7 @@ publish composite blocktype GTFSAgencyInterpreter { block AgencyTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "agency_id" oftype text, // Conditional columns are considered required for now - "agency_name" oftype text, - "agency_url" oftype GTFSUrl, - "agency_timezone" oftype text - ]; + columns: Agency; + parseWith: AgencyParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv index 2e25dcb8d..d1b10a125 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarDatesInterpreter.jv @@ -2,6 +2,23 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype CalendarDates { + property serviceId oftype text; // Conditional columns are considered required for now + property date oftype GTFSDate; + property exceptionType oftype GTFSEnumOneOrTwo; // 1 - Service has been added for the specified date + // 2 - Service has been removed for the specified date. +} +transform CalendarDatesParser { + from r oftype Collection; + to dates oftype CalendarDates; + + dates: { + serviceId: r cellInColumn "service_id", + date: r cellInColumn "date", + exceptionType: r cellInColumn "exception_type", + }; +} + /** * A GTFSCalendarDatesInterpreter interprets a calendar_dates.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#calendar_datestxt @@ -27,11 +44,7 @@ publish composite blocktype GTFSCalendarDatesInterpreter { block CalendarDatesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "service_id" oftype text, - "date" oftype GTFSDate, - "exception_type" oftype GTFSEnumOneOrTwo // 1 - Service has been added for the specified date - // 2 - Service has been removed for the specified date. - ]; + columns: CalendarDates; + parseWith: CalendarDatesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv index a6176a102..29614539f 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSCalendarInterpreter.jv @@ -2,6 +2,37 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Calendar { + property serviceId oftype text; + property monday oftype GTFSEnumTwo; // 1 - Service is available for all Mondays in the date range. + // 0 - Service is not available for Mondays in the date range. + property tuesday oftype GTFSEnumTwo; + property wednesday oftype GTFSEnumTwo; + property thursday oftype GTFSEnumTwo; + property friday oftype GTFSEnumTwo; + property saturday oftype GTFSEnumTwo; + property sunday oftype GTFSEnumTwo; + property startDate oftype GTFSDate; + property endDate oftype GTFSDate; +} +transform CalendarParser { + from r oftype Collection; + to calendar oftype Calendar; + + calendar: { + serviceId: r cellInColumn 0, + monday: r cellInColumn 1, + tuesday: r cellInColumn 2, + wednesday: r cellInColumn 3, + thursday: r cellInColumn 4, + friday: r cellInColumn 5, + saturday: r cellInColumn 6, + sunday: r cellInColumn 7, + startDate: r cellInColumn 8, + endDate: r cellInColumn 9, + }; +} + /** * A GTFSCalendarInterpreter interprets a calendar.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#calendartxt @@ -27,18 +58,7 @@ publish composite blocktype GTFSCalendarInterpreter { block CalendarTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "service_id" oftype text, - "monday" oftype GTFSEnumTwo, // 1 - Service is available for all Mondays in the date range. - // 0 - Service is not available for Mondays in the date range. - "tuesday" oftype GTFSEnumTwo, - "wednesday" oftype GTFSEnumTwo, - "thursday" oftype GTFSEnumTwo, - "friday" oftype GTFSEnumTwo, - "saturday" oftype GTFSEnumTwo, - "sunday" oftype GTFSEnumTwo, - "start_date" oftype GTFSDate, - "end_date" oftype GTFSDate - ]; + columns: Calendar; + parseWith: CalendarParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv index abfba17f9..7b8f0d995 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFareAttributesInterpreter.jv @@ -2,6 +2,29 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype FareAttributes { + property fareId oftype text; + property price oftype GTFSNonNegativeDecimal; + property currencyType oftype GTFSCurrency; + property paymentMethod oftype GTFSEnumTwo; // 0 - Fare is paid on board. + // 1 - Fare must be paid before boarding. + property transfers oftype text; // Is required but can be empty (?!) so has to be modelled as text... + property transferDuration oftype text; +} +transform FareAttributesParser { + from r oftype Collection; + to fareAttributes oftype FareAttributes; + + fareAttributes: { + fareId: r cellInColumn 0, + price: r cellInColumn 1, + currencyType: r cellInColumn 2, + paymentMethod: r cellInColumn 3, + transfers: r cellInColumn 4, + transferDuration: r cellInColumn 5, + }; +} + /** * A GTFSFareAttributesInterpreter interprets a fare_attributes.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#fare_attributestxt @@ -27,14 +50,7 @@ publish composite blocktype GTFSFareAttributesInterpreter { block FareAttributesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "fare_id" oftype text, - "price" oftype GTFSNonNegativeDecimal, - "currency_type" oftype GTFSCurrency, - "payment_method" oftype GTFSEnumTwo, // 0 - Fare is paid on board. - // 1 - Fare must be paid before boarding. - "transfers" oftype text, // Is required but can be empty (?!) so has to be modelled as text... - "transfer_duration" oftype text - ]; + columns: FareAttributes; + parseWith: FareAttributesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv index ce42b3d5f..3f7c3af98 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFareRulesInterpreter.jv @@ -2,6 +2,26 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype FareRules { + property fareId oftype text; + property routeId oftype text; + property originId oftype text; + property destinationId oftype text; + property containsId oftype text; +} +transform FareRulesParser { + from r oftype Collection; + to fareRules oftype FareRules; + + fareRules: { + fareId: r cellInColumn 0, + routeId: r cellInColumn 1, + originId: r cellInColumn 2, + destinationId: r cellInColumn 3, + containsId: r cellInColumn 4, + }; +} + /** * A GTFSFareRulesInterpreter interprets a fare_rules.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#fare_rulestxt @@ -27,12 +47,7 @@ publish composite blocktype GTFSFareRulesInterpreter { block FareRulesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "fare_id" oftype text, - "route_id" oftype text, - "origin_id" oftype text, - "destination_id" oftype text, - "contains_id" oftype text - ]; + columns: FareRules; + parseWith: FareRulesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv index a851292bf..d12e6c087 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSFrequenciesInterpreter.jv @@ -2,6 +2,24 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Frequencies { + property tripId oftype text; + property startTime oftype GTFSTime; + property endTime oftype GTFSTime; + property headwaySecs oftype GTFSNonNegativeInteger; +} +transform FrequenciesParser { + from r oftype Collection; + to frequencies oftype Frequencies; + + frequencies: { + tripId: r cellInColumn 0, + startTime: r cellInColumn 1, + endTime: r cellInColumn 2, + headwaySecs: r cellInColumn 3, + }; +} + /** * A GTFSFrequenciesInterpreter interprets a frequencies.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#frequenciestxt @@ -27,11 +45,7 @@ publish composite blocktype GTFSFrequenciesInterpreter { block FrequenciesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "trip_id" oftype text, - "start_time" oftype GTFSTime, - "end_time" oftype GTFSTime, - "headway_secs" oftype GTFSNonNegativeInteger - ]; + columns: Frequencies; + parseWith: FrequenciesParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv index 36cc6227e..2d5a6b187 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSRoutesInterpreter.jv @@ -2,6 +2,34 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Route { + property routeId oftype text; + property agencyId oftype text; + property routeShortName oftype text; + property routeLongName oftype text; + property routeDesc oftype text; + property routeType oftype integer; // Technically is an enum from 0 - 12 + property routeUrl oftype text; + property routeColor oftype text; + property routeTextColor oftype text; +} +transform RouteParser { + from r oftype Collection; + to route oftype Route; + + route: { + routeId: r cellInColumn 0, + agencyId: r cellInColumn 1, + routeShortName: r cellInColumn 2, + routeLongName: r cellInColumn 3, + routeDesc: r cellInColumn 4, + routeType: r cellInColumn 5, + routeUrl: r cellInColumn 6, + routeColor: r cellInColumn 7, + routeTextColor: r cellInColumn 8, + }; +} + /** * A GTFSRoutesInterpreter interprets a routes.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#routestxt @@ -27,16 +55,7 @@ publish composite blocktype GTFSRoutesInterpreter { block RoutesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "route_id" oftype text, - "agency_id" oftype text, - "route_short_name" oftype text, - "route_long_name" oftype text, - "route_desc" oftype text, - "route_type" oftype integer, // Technically is an enum from 0 - 12 - "route_url" oftype text, - "route_color" oftype text, - "route_text_color" oftype text - ]; + columns: Route; + parseWith: RouteParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv index de2382829..2f0ce1e64 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSShapesInterpreter.jv @@ -2,6 +2,26 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Shape { + property shapeId oftype text; + property shapePtLat oftype GTFSLatitude; + property shapePtLon oftype GTFSLongitude; + property shapePtSequence oftype GTFSNonNegativeInteger; + property shapeDistTraveled oftype text; +} +transform ShapeParser { + from r oftype Collection; + to shape oftype Shape; + + shape: { + shapeId: r cellInColumn 0, + shapePtLat: r cellInColumn 1, + shapePtLon: r cellInColumn 2, + shapePtSequence: r cellInColumn 3, + shapeDistTraveled: r cellInColumn 4, + }; +} + /** * A GTFSShapesInterpreter interprets a shapes.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#shapestxt @@ -27,12 +47,7 @@ publish composite blocktype GTFSShapesInterpreter { block ShapesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "shape_id" oftype text, - "shape_pt_lat" oftype GTFSLatitude, - "shape_pt_lon" oftype GTFSLongitude, - "shape_pt_sequence" oftype GTFSNonNegativeInteger, - "shape_dist_traveled" oftype text - ]; + columns: Shape; + parseWith: ShapeParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv index 289e61522..a0e894cad 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSStopTimesInterpreter.jv @@ -2,6 +2,34 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype StopTime { + property tripId oftype text; + property arrivalId oftype GTFSTime; + property departureTime oftype GTFSTime; + property stopId oftype text; + property stopSequence oftype GTFSNonNegativeInteger; + property stopHeadsign oftype text; + property pickupType oftype text; + property dropOffTime oftype text; + property shapeDistTraveled oftype text; +} +transform StopTimeParser { + from r oftype Collection; + to stopTime oftype StopTime; + + stopTime: { + tripId: r cellInColumn 0, + arrivalId: r cellInColumn 1, + departureTime: r cellInColumn 2, + stopId: r cellInColumn 3, + stopSequence: r cellInColumn 4, + stopHeadsign: r cellInColumn 5, + pickupType: r cellInColumn 6, + dropOffTime: r cellInColumn 7, + shapeDistTraveled: r cellInColumn 8, + }; +} + /** * A GTFSStopTimesInterpreter interprets a stop_times.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#stop_timestxt @@ -27,16 +55,7 @@ publish composite blocktype GTFSStopTimesInterpreter { block StopTimesTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "trip_id" oftype text, - "arrival_time" oftype GTFSTime, - "departure_time" oftype GTFSTime, - "stop_id" oftype text, - "stop_sequence" oftype GTFSNonNegativeInteger, - "stop_headsign" oftype text, - "pickup_type" oftype text, - "drop_off_time" oftype text, - "shape_dist_traveled" oftype text - ]; + columns: StopTime; + parseWith: StopTimeParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv index 9e9ea8323..ec37aa87f 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSStopsInterpreter.jv @@ -2,6 +2,30 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Stop { + property stopId oftype text; + property stopName oftype text; + property stopDesc oftype text; + property stopLat oftype GTFSLatitude; + property stopLon oftype GTFSLongitude; + property zoneId oftype text; + property stopUrl oftype text; +} +transform StopParser { + from r oftype Collection; + to stop oftype Stop; + + stop: { + stopId: r cellInColumn 0, + stopName: r cellInColumn 1, + stopDesc: r cellInColumn 2, + stopLat: r cellInColumn 3, + stopLon: r cellInColumn 4, + zoneId: r cellInColumn 5, + stopUrl: r cellInColumn 6, + }; +} + /** * A GTFSStopsInterpreter interprets a stops.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#stopstxt @@ -27,14 +51,7 @@ publish composite blocktype GTFSStopsInterpreter { block StopsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "stop_id" oftype text, - "stop_name" oftype text, - "stop_desc" oftype text, - "stop_lat" oftype GTFSLatitude, - "stop_lon" oftype GTFSLongitude, - "zone_id" oftype text, - "stop_url" oftype text - ]; + columns: Stop; + parseWith: StopParser; } } diff --git a/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv b/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv index 11998662b..549a12b27 100644 --- a/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv +++ b/libs/language-server/src/stdlib/domain/mobility/GTFSTripsInterpreter.jv @@ -2,6 +2,30 @@ // // SPDX-License-Identifier: AGPL-3.0-only +valuetype Trip { + property routeId oftype text; + property serviceId oftype text; + property tripId oftype text; + property tripHeadsign oftype text; + property directionId oftype text; + property blockId oftype text; + property shapeId oftype text; +} +transform TripParser { + from r oftype Collection; + to trip oftype Trip; + + trip: { + routeId: r cellInColumn 0, + serviceId: r cellInColumn 1, + tripId: r cellInColumn 2, + tripHeadsign: r cellInColumn 3, + directionId: r cellInColumn 4, + blockId: r cellInColumn 5, + shapeId: r cellInColumn 6, + }; +} + /** * A GTFSTripsInterpreter interprets a trips.txt file from an extracted ZIP file according to the GTFS standard * See https://gtfs.org/schedule/reference/#tripstxt @@ -27,14 +51,7 @@ publish composite blocktype GTFSTripsInterpreter { block TripsTableInterpreter oftype TableInterpreter { header: true; - columns: [ - "route_id" oftype text, - "service_id" oftype text, - "trip_id" oftype text, - "trip_headsign" oftype text, - "direction_id" oftype text, - "block_id" oftype text, - "shape_id" oftype text - ]; + columns: Trip; + parseWith: TripParser; } } diff --git a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv b/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv index 06cc9f9d9..37dfe7bec 100644 --- a/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv +++ b/libs/language-server/src/test/assets/property-assignment/block-type-specific/table-interpreter/valid-correct-table.jv @@ -3,11 +3,11 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline Pipeline { + valuetype NaturalNumber { + property number oftype integer; + } block Test oftype TableInterpreter { - columns: [ - "name" oftype text, - "version" oftype integer, - ]; + columns: NaturalNumber; } block TestExtractor oftype TestSheetExtractor {