Skip to content

Commit 6cb819b

Browse files
norberttechnorberttech
committed
Aligned types definitions with static analysis
1 parent 0ab1f46 commit 6cb819b

60 files changed

Lines changed: 529 additions & 432 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎src/adapter/etl-adapter-parquet/src/Flow/ETL/Adapter/Parquet/SchemaConverter.php‎

Lines changed: 35 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
type_json,
2424
type_list,
2525
type_map,
26+
type_optional,
2627
type_string,
2728
type_structure,
2829
type_time,
@@ -207,6 +208,9 @@ private function parquetToFlowDefinition(Column $column) : Schema\Definition
207208
return struct_schema($column->name(), type_structure($elements), $nullable);
208209
}
209210

211+
/**
212+
* @return Type<mixed>
213+
*/
210214
private function parquetToFlowType(Column $column) : Type
211215
{
212216
if ($column instanceof FlatColumn) {
@@ -215,38 +219,45 @@ private function parquetToFlowType(Column $column) : Type
215219
$nullable = $column->repetition() === ParquetSchema\Repetition::OPTIONAL;
216220

217221
if ($logicalType === null) {
218-
return match ($column->type()) {
222+
$type = match ($column->type()) {
219223
ParquetSchema\PhysicalType::INT32 => match ($column->convertedType()) {
220-
ParquetSchema\ConvertedType::DATE => type_date($nullable),
221-
default => type_int($nullable),
224+
ParquetSchema\ConvertedType::DATE => type_date(),
225+
default => type_int(),
222226
},
223-
ParquetSchema\PhysicalType::INT64 => type_int($nullable),
224-
ParquetSchema\PhysicalType::BOOLEAN => type_boolean($nullable),
225-
ParquetSchema\PhysicalType::DOUBLE => type_float($nullable),
226-
ParquetSchema\PhysicalType::FLOAT => type_float($nullable),
227-
ParquetSchema\PhysicalType::BYTE_ARRAY => type_string($nullable),
227+
ParquetSchema\PhysicalType::INT64 => type_int(),
228+
ParquetSchema\PhysicalType::BOOLEAN => type_boolean(),
229+
ParquetSchema\PhysicalType::DOUBLE => type_float(),
230+
ParquetSchema\PhysicalType::FLOAT => type_float(),
231+
ParquetSchema\PhysicalType::BYTE_ARRAY => type_string(),
228232
default => throw new RuntimeException($column->type()->name . ' is not supported.'),
229233
};
234+
235+
return $nullable ? type_optional($type) : $type;
230236
}
231237

232-
return match ($logicalType->name()) {
233-
ParquetSchema\LogicalType::STRING => type_string($nullable),
234-
ParquetSchema\LogicalType::TIME => type_time($nullable),
235-
ParquetSchema\LogicalType::DATE => type_date($nullable),
236-
ParquetSchema\LogicalType::TIMESTAMP => type_datetime($nullable),
237-
ParquetSchema\LogicalType::UUID => type_uuid($nullable),
238-
ParquetSchema\LogicalType::JSON => type_json($nullable),
239-
ParquetSchema\LogicalType::DECIMAL => type_float($nullable),
240-
ParquetSchema\LogicalType::INTEGER => type_int($nullable),
238+
$type = match ($logicalType->name()) {
239+
ParquetSchema\LogicalType::STRING => type_string(),
240+
ParquetSchema\LogicalType::TIME => type_time(),
241+
ParquetSchema\LogicalType::DATE => type_date(),
242+
ParquetSchema\LogicalType::TIMESTAMP => type_datetime(),
243+
ParquetSchema\LogicalType::UUID => type_uuid(),
244+
ParquetSchema\LogicalType::JSON => type_json(),
245+
ParquetSchema\LogicalType::DECIMAL => type_float(),
246+
ParquetSchema\LogicalType::INTEGER => type_int(),
241247
default => throw new RuntimeException($logicalType->name() . ' is not supported.'),
242248
};
249+
250+
return $nullable ? type_optional($type) : $type;
243251
}
244252

245253
/** @var NestedColumn $column */
246254
$nullable = $column->repetition() === ParquetSchema\Repetition::OPTIONAL;
247255

248256
if ($column->isList()) {
249-
return type_list($this->parquetToFlowType($column->getListElement()), $nullable);
257+
258+
return $nullable
259+
? type_optional(type_list($this->parquetToFlowType($column->getListElement())))
260+
: type_list($this->parquetToFlowType($column->getListElement()));
250261
}
251262

252263
if ($column->isMap()) {
@@ -256,11 +267,9 @@ private function parquetToFlowType(Column $column) : Type
256267
throw new RuntimeException('Flow expects map key type to be string or integer type.');
257268
}
258269

259-
return type_map(
260-
$keyType,
261-
$this->parquetToFlowType($column->getMapValueColumn()),
262-
$nullable
263-
);
270+
return $nullable
271+
? type_optional(type_map($keyType, $this->parquetToFlowType($column->getMapValueColumn())))
272+
: type_map($keyType, $this->parquetToFlowType($column->getMapValueColumn()));
264273
}
265274

266275
$elements = [];
@@ -269,6 +278,8 @@ private function parquetToFlowType(Column $column) : Type
269278
$elements[$structColumn->name()] = $this->parquetToFlowType($structColumn);
270279
}
271280

272-
return type_structure($elements, $nullable);
281+
return $nullable
282+
? type_optional(type_structure($elements))
283+
: type_structure($elements);
273284
}
274285
}

‎src/adapter/etl-adapter-parquet/tests/Flow/ETL/Adapter/Parquet/Tests/Unit/ParquetToFlowSchemaTest.php‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
type_int,
2020
type_list,
2121
type_map,
22+
type_optional,
2223
type_string,
2324
type_structure,
2425
type_uuid,
@@ -79,7 +80,7 @@ public function test_converting_list_to_flow_schema() : void
7980

8081
self::assertEquals(
8182
\Flow\ETL\DSL\schema(
82-
list_schema('list', type_list(type_string(true)), true)
83+
list_schema('list', type_list(type_optional(type_string())), true)
8384
),
8485
$flowSchema,
8586
);
@@ -95,7 +96,7 @@ public function test_converting_map_to_flow_schema() : void
9596

9697
self::assertEquals(
9798
\Flow\ETL\DSL\schema(
98-
map_schema('map', type_map(type_string(), type_int(true)), true)
99+
map_schema('map', type_map(type_string(), type_optional(type_int())), true)
99100
),
100101
$flowSchema,
101102
);
@@ -122,9 +123,9 @@ public function test_converting_struct_to_flow_schema() : void
122123
'struct',
123124
type_structure(
124125
[
125-
'uuid' => type_uuid(true),
126-
'name' => type_string(true),
127-
'active' => type_boolean(true),
126+
'uuid' => type_optional(type_uuid()),
127+
'name' => type_optional(type_string()),
128+
'active' => type_optional(type_boolean()),
128129
],
129130
),
130131
true

‎src/adapter/etl-adapter-xml/src/Flow/ETL/Adapter/XML/RowsNormalizer/EntryNormalizer.php‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ private function listToNode(ListEntry $entry) : XMLNode
7575
{
7676
$node = XMLNode::nestedNode($entry->name());
7777

78-
/** @var ListType $type */
78+
/** @var ListType<mixed> $type */
7979
$type = $entry->type();
8080

8181
$listValue = $entry->value();
@@ -132,7 +132,7 @@ private function mapToNode(MapEntry $entry) : XMLNode
132132
return $node;
133133
}
134134

135-
/** @var MapType $type */
135+
/** @var MapType<array-key, mixed> $type */
136136
$type = $entry->type();
137137

138138
foreach ($mapValue as $key => $value) {
@@ -143,6 +143,9 @@ private function mapToNode(MapEntry $entry) : XMLNode
143143
return $node;
144144
}
145145

146+
/**
147+
* @param StructureEntry<array> $entry
148+
*/
146149
private function structureToNode(StructureEntry $entry) : XMLNode
147150
{
148151
$node = XMLNode::nestedNode($entry->name());
@@ -153,7 +156,7 @@ private function structureToNode(StructureEntry $entry) : XMLNode
153156
return $node;
154157
}
155158

156-
/** @var StructureType $type */
159+
/** @var StructureType<array> $type */
157160
$type = $entry->type();
158161

159162
$structureIterator = new \MultipleIterator(\MultipleIterator::MIT_KEYS_ASSOC);

‎src/adapter/etl-adapter-xml/tests/Flow/ETL/Adapter/XML/Tests/Unit/RowsNormalizer/EntryNormalizer/PHPValueNormalizerTest.php‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
namespace Flow\ETL\Adapter\XML\Tests\Unit\RowsNormalizer\EntryNormalizer;
66

77
use function Flow\ETL\DSL\{type_array, type_boolean, type_datetime, type_float, type_integer, type_json, type_object, type_string};
8+
use function Flow\ETL\DSL\type_optional;
89
use Flow\ETL\Adapter\XML\Abstraction\{XMLAttribute, XMLNode};
910
use Flow\ETL\Adapter\XML\RowsNormalizer\EntryNormalizer\PHPValueNormalizer;
1011
use Flow\ETL\Tests\FlowTestCase;
@@ -76,7 +77,7 @@ public function test_normalizing_integer_type() : void
7677

7778
self::assertEquals(
7879
XMLNode::flatNode('int', ''),
79-
$normalizer->normalize('int', type_integer(true), null)
80+
$normalizer->normalize('int', type_optional(type_integer()), null)
8081
);
8182
}
8283

‎src/cli/tests/Flow/CLI/Tests/Integration/SchemaFormatCommandTest.php‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -178,9 +178,9 @@ public function test_run_schema_format_php() : void
178178
\Flow\ETL\DSL\float_schema("discount", nullable: true, metadata: \Flow\ETL\DSL\schema_metadata()),
179179
\Flow\ETL\DSL\string_schema("email", nullable: false, metadata: \Flow\ETL\DSL\schema_metadata()),
180180
\Flow\ETL\DSL\string_schema("customer", nullable: false, metadata: \Flow\ETL\DSL\schema_metadata()),
181-
\Flow\ETL\DSL\structure_schema("address", type: \Flow\ETL\DSL\type_structure(elements: ["street" => \Flow\ETL\DSL\type_string(nullable: false), "city" => \Flow\ETL\DSL\type_string(nullable: false), "zip" => \Flow\ETL\DSL\type_string(nullable: false), "country" => \Flow\ETL\DSL\type_string(nullable: false)], nullable: false), nullable: false, metadata: \Flow\ETL\DSL\schema_metadata()),
182-
\Flow\ETL\DSL\list_schema("notes", type: \Flow\ETL\DSL\type_list(element: \Flow\ETL\DSL\type_string(nullable: false), nullable: false), metadata: \Flow\ETL\DSL\schema_metadata()),
183-
\Flow\ETL\DSL\list_schema("items", type: \Flow\ETL\DSL\type_list(element: \Flow\ETL\DSL\type_structure(elements: ["sku" => \Flow\ETL\DSL\type_string(nullable: false), "quantity" => \Flow\ETL\DSL\type_integer(nullable: false), "price" => \Flow\ETL\DSL\type_float(nullable: false)], nullable: false), nullable: false), metadata: \Flow\ETL\DSL\schema_metadata()),
181+
\Flow\ETL\DSL\structure_schema("address", type: \Flow\ETL\DSL\type_structure(elements: ["street" => \Flow\ETL\DSL\type_string(), "city" => \Flow\ETL\DSL\type_string(), "zip" => \Flow\ETL\DSL\type_string(), "country" => \Flow\ETL\DSL\type_string()]), nullable: false, metadata: \Flow\ETL\DSL\schema_metadata()),
182+
\Flow\ETL\DSL\list_schema("notes", type: \Flow\ETL\DSL\type_list(element: \Flow\ETL\DSL\type_string()), nullable: false, metadata: \Flow\ETL\DSL\schema_metadata()),
183+
\Flow\ETL\DSL\list_schema("items", type: \Flow\ETL\DSL\type_list(element: \Flow\ETL\DSL\type_structure(elements: ["sku" => \Flow\ETL\DSL\type_string(), "quantity" => \Flow\ETL\DSL\type_integer(), "price" => \Flow\ETL\DSL\type_float()])), nullable: false, metadata: \Flow\ETL\DSL\schema_metadata()),
184184
);
185185

186186
PHP,

‎src/core/etl/src/Flow/ETL/Config.php‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@
88
use Flow\ETL\Config\ConfigBuilder;
99
use Flow\ETL\Config\Sort\SortConfig;
1010
use Flow\ETL\Filesystem\FilesystemStreams;
11-
use Flow\ETL\PHP\Type\Caster;
1211
use Flow\ETL\Pipeline\Optimizer;
1312
use Flow\ETL\Row\EntryFactory;
1413
use Flow\Filesystem\{FilesystemTable};
@@ -55,11 +54,6 @@ public static function default() : self
5554
return self::builder()->build();
5655
}
5756

58-
public function caster() : Caster
59-
{
60-
return $this->caster;
61-
}
62-
6357
public function clock() : ClockInterface
6458
{
6559
return $this->clock;

0 commit comments

Comments
 (0)