Skip to content

Commit 2e1aec6

Browse files
jrentlezjrentlez
committed
WIP
1 parent 5268475 commit 2e1aec6

41 files changed

Lines changed: 932 additions & 375 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/interpreter/test/assets/broken-model.jv‎

Lines changed: 38 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,27 +28,50 @@ pipeline CarsPipeline {
2828
write: ["name"];
2929
}
3030

31+
valuetype Car {
32+
property name oftype text;
33+
property mpg oftype decimal;
34+
property cyl oftype integer;
35+
property disp oftype decimal;
36+
property hp oftype integer;
37+
property drat oftype decimal;
38+
property wt oftype decimal;
39+
property qsec oftype decimal;
40+
property vs oftype integer;
41+
property am oftype integer;
42+
property gear oftype integer;
43+
property carb oftype integer;
44+
}
45+
46+
transform CarParser {
47+
from r oftype Collection<text>;
48+
to car oftype Car;
49+
50+
car: {
51+
name: asText (r cellInColumn "name"),
52+
mpg: asDecimal (r cellInColumn "mpg"),
53+
cyl: asInteger (r cellInColumn 2),
54+
disp: asDecimal (r cellInColumn 3),
55+
hp: asInteger (r cellInColumn "hp"),
56+
drat: asDecimal (r cellInColumn "drat"),
57+
wt: asDecimal (r cellInColumn "wt"),
58+
qsec: asDecimal (r cellInColumn "qsec"),
59+
vs: asInteger (r cellInColumn "vs"),
60+
am: asInteger (r cellInColumn "am"),
61+
gear: asInteger (r cellInColumn "gear"),
62+
carb: asInteger (r cellInColumn "carb")
63+
};
64+
}
65+
3166
block CarsTableInterpreter oftype TableInterpreter {
3267
header: true;
33-
columns: [
34-
"name" oftype text,
35-
"mpg" oftype decimal,
36-
"cyl" oftype integer,
37-
"disp" oftype decimal,
38-
"hp" oftype integer,
39-
"drat" oftype decimal,
40-
"wt" oftype decimal,
41-
"qsec" oftype decimal,
42-
"vs" oftype integer,
43-
"am" oftype integer,
44-
"gear" oftype integer,
45-
"carb" oftype integer
46-
];
68+
columns: Car;
69+
parseWith: CarParser;
4770
}
4871

4972
block CarsLoader oftype SQLiteLoader {
5073
table: "Cars";
5174
file: "./cars.sqlite";
5275
}
5376

54-
}
77+
}

‎example/cars.jv‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,12 +88,33 @@ pipeline CarsPipeline {
8888
property carb oftype integer;
8989
}
9090

91+
transform CarParser {
92+
from r oftype Collection<text>;
93+
to car oftype Car;
94+
95+
car: {
96+
name: asText (r cellInColumn "name"),
97+
mpg: asDecimal (r cellInColumn "mpg"),
98+
cyl: asInteger (r cellInColumn 2),
99+
disp: asDecimal (r cellInColumn 3),
100+
hp: asInteger (r cellInColumn "hp"),
101+
drat: asDecimal (r cellInColumn "drat"),
102+
wt: asDecimal (r cellInColumn "wt"),
103+
qsec: asDecimal (r cellInColumn "qsec"),
104+
vs: asInteger (r cellInColumn "vs"),
105+
am: asInteger (r cellInColumn "am"),
106+
gear: asInteger (r cellInColumn "gear"),
107+
carb: asInteger (r cellInColumn "carb")
108+
};
109+
}
110+
91111
// 15. As a next step, we interpret the sheet as a table, using the valuetype
92112
// defined above. Rows that include values that are not valid according to the
93113
// their value types are dropped automatically.
94114
block CarsTableInterpreter oftype TableInterpreter {
95115
header: true;
96116
columns: Car;
117+
parseWith: CarParser;
97118
}
98119

99120
// 16. As a last step, we load the table into a sink, here into a sqlite file.

‎example/electric-vehicles.jv‎

Lines changed: 50 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -49,30 +49,56 @@ pipeline ElectricVehiclesPipeline {
4949

5050
block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { }
5151

52+
valuetype ElectricVehicle {
53+
property vin oftype VehicleIdentificationNumber10;
54+
property county oftype text;
55+
property city oftype text;
56+
property state oftype UsStateCode;
57+
property postal oftype text;
58+
property modelYear oftype integer;
59+
property make oftype text;
60+
property model oftype text;
61+
property evType oftype text;
62+
property cafvEligibility oftype text;
63+
property electricRange oftype integer;
64+
property baseMSRP oftype integer;
65+
property legislativeDistrict oftype text;
66+
property dolID oftype integer;
67+
property location oftype text;
68+
property utility oftype text;
69+
property censusTract oftype text;
70+
}
71+
72+
transform ElectricVehicleParser {
73+
from r oftype Collection<text>;
74+
to ev oftype ElectricVehicle;
75+
76+
ev: {
77+
vin: r cellInColumn "VIN (1-10)",
78+
county: asText (r cellInColumn "County"),
79+
city: asText (r cellInColumn "City"),
80+
state: asText (r cellInColumn "State"),
81+
postal: asText (r cellInColumn "Postal Code"),
82+
modelYear: asInteger (r cellInColumn "Model Year"),
83+
make: asText (r cellInColumn "Make"),
84+
model: asText (r cellInColumn "Model"),
85+
evType: asText (r cellInColumn "Electric Vehicle Type"),
86+
cafvEligibility: asText (r cellInColumn "Clean Alternative Fuel Vehicle (CAFV) Eligibility"),
87+
electricRange: asInteger (r cellInColumn "Electric Range"),
88+
baseMSRP: asInteger (r cellInColumn "Base MSRP"),
89+
legislativeDistrict: asText (r cellInColumn "LegislativeDistrict"),
90+
dolID: asInteger (r cellInColumn "DOL Vehicle ID"),
91+
location: asText (r cellInColumn "Vehicle Location"),
92+
utility: asText (r cellInColumn "Electric Utility"),
93+
censusTract: asText (r cellInColumn "2020 Census Tract"),
94+
};
95+
}
96+
97+
5298
block ElectricVehiclesTableInterpreter oftype TableInterpreter {
5399
header: true;
54-
columns: [
55-
// 4. Here, a user-deifned value type is used to describe this column.
56-
// The capital letter indicates that the value type is not built-in
57-
// by convention. The value type itself is defined further below.
58-
"VIN (1-10)" oftype VehicleIdentificationNumber10,
59-
"County" oftype text,
60-
"City" oftype text,
61-
"State" oftype UsStateCode, // We can just use the element as if it was defined in this file.
62-
"Postal Code" oftype text,
63-
"Model Year" oftype integer,
64-
"Make" oftype text,
65-
"Model" oftype text,
66-
"Electric Vehicle Type" oftype text,
67-
"Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text,
68-
"Electric Range" oftype integer,
69-
"Base MSRP" oftype integer,
70-
"Legislative District" oftype text,
71-
"DOL Vehicle ID" oftype integer,
72-
"Vehicle Location" oftype text,
73-
"Electric Utility" oftype text,
74-
"2020 Census Tract" oftype text,
75-
];
100+
columns: ElectricVehicle;
101+
parseWith: ElectricVehicleParser;
76102
}
77103

78104
// 5. This block describes the application of a transform function
@@ -81,9 +107,9 @@ pipeline ElectricVehiclesPipeline {
81107
// by the "use" property.
82108
block ElectricRangeTransformer oftype TableTransformer {
83109
inputColumns: [
84-
"Electric Range"
110+
"electricRange"
85111
];
86-
outputColumn: "Electric Range (km)";
112+
outputColumn: "electricRange (km)";
87113
uses: MilesToKilometers;
88114
}
89115

‎example/gtfs-rt.jv‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -58,20 +58,39 @@ pipeline GtfsRTSimplePipeline {
5858
}
5959

6060
// 5. Next, we interpret the sheets as tables
61+
valuetype TripUpdate {
62+
property headerGtfsRealtimeVersion oftype text;
63+
property headerTimestamp oftype text;
64+
property headerIncrementality oftype text;
65+
property entityId oftype text;
66+
property entityTripUpdateTripTripId oftype text;
67+
property entityTripUpdateTripRouteId oftype text;
68+
property entityTripUpdateStopTimeUpdateStopSequence oftype text;
69+
property entityTripUpdateStopTimeUpdateStopId oftype text;
70+
property entityTripUpdateStopTimeUpdateArrivalTime oftype text;
71+
property entityTripUpdateStopTimeUpdateDepartureTime oftype text;
72+
}
73+
transform TripUpdateParser {
74+
from r oftype Collection<text>;
75+
to tripUpdate oftype TripUpdate;
76+
77+
tripUpdate: {
78+
headerGtfsRealtimeVersion: r in "header.gtfs_realtime_version",
79+
headerTimestamp: r in "header.timestamp",
80+
headerIncrementality: r in "header.incrementality",
81+
entityId: r in "entity.id",
82+
entityTripUpdateTripTripId: r in "entity.trip_update.trip.trip_id",
83+
entityTripUpdateTripRouteId: r in "entity.trip_update.trip.route_id",
84+
entityTripUpdateStopTimeUpdateStopSequence: r in "entity.trip_update.stop_time_update.stop_sequence",
85+
entityTripUpdateStopTimeUpdateStopId: r in "entity.trip_update.stop_time_update.stop_id",
86+
entityTripUpdateStopTimeUpdateArrivalTime: r in "entity.trip_update.stop_time_update.arrival.time",
87+
entityTripUpdateStopTimeUpdateDepartureTime: r in "entity.trip_update.stop_time_update.departure.time",
88+
};
89+
}
6190
block TripUpdateTableInterpreter oftype TableInterpreter {
6291
header: true;
63-
columns: [
64-
"header.gtfs_realtime_version" oftype text,
65-
"header.timestamp" oftype text,
66-
"header.incrementality" oftype text,
67-
"entity.id" oftype text,
68-
"entity.trip_update.trip.trip_id" oftype text,
69-
"entity.trip_update.trip.route_id" oftype text,
70-
"entity.trip_update.stop_time_update.stop_sequence" oftype text,
71-
"entity.trip_update.stop_time_update.stop_id" oftype text,
72-
"entity.trip_update.stop_time_update.arrival.time" oftype text,
73-
"entity.trip_update.stop_time_update.departure.time" oftype text,
74-
];
92+
columns: TripUpdate;
93+
columns: TripUpdateParser;
7594
}
7695

7796
block VehiclePositionTableInterpreter oftype TableInterpreter {
@@ -124,4 +143,4 @@ pipeline GtfsRTSimplePipeline {
124143
file: "./gtfs.sqlite";
125144
dropTable: false;
126145
}
127-
}
146+
}

‎libs/execution/src/lib/types/io-types/sheet.ts‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,8 @@ export class Sheet implements IOTypeImplementation<IOType.SHEET> {
4545
return this.numberOfColumns;
4646
}
4747

48-
getHeaderRow(): string[] {
49-
assert(
50-
this.getNumberOfRows() > 0,
51-
'The sheet is expected to be non-empty and have a header row',
52-
);
53-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
54-
return this.data[0]!;
48+
popHeaderRow(): string[] | undefined {
49+
return this.data.shift();
5550
}
5651

5752
iterateRows(callbackFn: (row: string[], rowIndex: number) => void) {

‎libs/execution/src/lib/types/io-types/table.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ export class Table implements IOTypeImplementation<IOType.TABLE> {
118118
return this.columns.has(name);
119119
}
120120

121-
getColumns(): ReadonlyMap<string, TableColumn> {
121+
getColumns(): Map<string, TableColumn> {
122122
return this.columns;
123123
}
124124

‎libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ describe('Validation of CellRangeSelectorExecutor', () => {
9696
expect(result.right.ioType).toEqual(IOType.SHEET);
9797
expect(result.right.getNumberOfColumns()).toEqual(3);
9898
expect(result.right.getNumberOfRows()).toEqual(16);
99-
expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']);
10099
expect(result.right.getData()).toEqual(
101100
expect.arrayContaining([
102101
['0', 'Test', 'true'],
@@ -121,7 +120,6 @@ describe('Validation of CellRangeSelectorExecutor', () => {
121120
expect(result.right.ioType).toEqual(IOType.SHEET);
122121
expect(result.right.getNumberOfColumns()).toEqual(3);
123122
expect(result.right.getNumberOfRows()).toEqual(2);
124-
expect(result.right.getHeaderRow()).toEqual(['', 'Test', 'true']);
125123
expect(result.right.getData()).toEqual([
126124
['', 'Test', 'true'],
127125
['', 'Test', 'false'],

‎libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ describe('Validation of CellWriterExecutor', () => {
9696
expect(result.right.ioType).toEqual(IOType.SHEET);
9797
expect(result.right.getNumberOfColumns()).toEqual(3);
9898
expect(result.right.getNumberOfRows()).toEqual(16);
99-
expect(result.right.getHeaderRow()).toEqual(['16', 'Test', 'true']);
10099
expect(result.right.getData()).toEqual(
101100
expect.arrayContaining([
102101
['16', 'Test', 'true'],
@@ -121,7 +120,6 @@ describe('Validation of CellWriterExecutor', () => {
121120
expect(result.right.ioType).toEqual(IOType.SHEET);
122121
expect(result.right.getNumberOfColumns()).toEqual(3);
123122
expect(result.right.getNumberOfRows()).toEqual(16);
124-
expect(result.right.getHeaderRow()).toEqual(['16', 'Test2', '']);
125123
expect(result.right.getData()).toEqual(
126124
expect.arrayContaining([
127125
['16', 'Test2', ''],

‎libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,6 @@ describe('Validation of ColumnDeleterExecutor', () => {
9696
expect(result.right.ioType).toEqual(IOType.SHEET);
9797
expect(result.right.getNumberOfColumns()).toEqual(2);
9898
expect(result.right.getNumberOfRows()).toEqual(16);
99-
expect(result.right.getHeaderRow()).toEqual(['Test', 'true']);
10099
expect(result.right.getData()).toEqual(
101100
expect.arrayContaining([
102101
['Test', 'true'],
@@ -121,7 +120,6 @@ describe('Validation of ColumnDeleterExecutor', () => {
121120
expect(result.right.ioType).toEqual(IOType.SHEET);
122121
expect(result.right.getNumberOfColumns()).toEqual(1);
123122
expect(result.right.getNumberOfRows()).toEqual(16);
124-
expect(result.right.getHeaderRow()).toEqual(['Test']);
125123
expect(result.right.getData()).toEqual(
126124
expect.arrayContaining([['Test'], ['Test'], ['Test']]),
127125
);
@@ -159,7 +157,6 @@ describe('Validation of ColumnDeleterExecutor', () => {
159157
expect(result.right.ioType).toEqual(IOType.SHEET);
160158
expect(result.right.getNumberOfColumns()).toEqual(2);
161159
expect(result.right.getNumberOfRows()).toEqual(16);
162-
expect(result.right.getHeaderRow()).toEqual(['Test', 'true']);
163160
expect(result.right.getData()).toEqual(
164161
expect.arrayContaining([
165162
['Test', 'true'],

‎libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ describe('Validation of RowDeleterExecutor', () => {
9696
expect(result.right.ioType).toEqual(IOType.SHEET);
9797
expect(result.right.getNumberOfColumns()).toEqual(3);
9898
expect(result.right.getNumberOfRows()).toEqual(15);
99-
expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']);
99+
expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']);
100100
expect(result.right.getData()).not.toEqual(
101101
expect.arrayContaining([['0', 'Test', 'true']]),
102102
);
@@ -117,7 +117,7 @@ describe('Validation of RowDeleterExecutor', () => {
117117
expect(result.right.ioType).toEqual(IOType.SHEET);
118118
expect(result.right.getNumberOfColumns()).toEqual(3);
119119
expect(result.right.getNumberOfRows()).toEqual(14);
120-
expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']);
120+
expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']);
121121
expect(result.right.getData()).not.toEqual(
122122
expect.arrayContaining([
123123
['0', 'Test', 'true'],
@@ -158,7 +158,7 @@ describe('Validation of RowDeleterExecutor', () => {
158158
expect(result.right.ioType).toEqual(IOType.SHEET);
159159
expect(result.right.getNumberOfColumns()).toEqual(3);
160160
expect(result.right.getNumberOfRows()).toEqual(15);
161-
expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']);
161+
expect(result.right.popHeaderRow()).toEqual(['1', 'Test', 'false']);
162162
expect(result.right.getData()).not.toEqual(
163163
expect.arrayContaining([['0', 'Test', 'true']]),
164164
);

0 commit comments

Comments
 (0)