Skip to content

Commit 4e69aa7

Browse files
authored
Fixed calculation of definition level in Parquet DremelShredder implementation (#1772)
1 parent 8433616 commit 4e69aa7

2 files changed

Lines changed: 36 additions & 2 deletions

File tree

src/lib/parquet/src/Flow/Parquet/Dremel/DremelShredder.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -178,7 +178,7 @@ private function shredList(NestedColumn $column, ?array $listValue, int $definit
178178
return;
179179
}
180180

181-
if (!$listElementColumn->repetition()?->isRequired()) {
181+
if (!$column->repetition()?->isRequired()) {
182182
$definitionLevel++;
183183
}
184184

src/lib/parquet/tests/Flow/Parquet/Tests/Integration/IO/ListsWritingTest.php

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
use Faker\Factory;
99
use Flow\Parquet\{Consts, Reader, Writer};
1010
use Flow\Parquet\ParquetFile\Schema;
11-
use Flow\Parquet\ParquetFile\Schema\{FlatColumn, ListElement, NestedColumn};
11+
use Flow\Parquet\ParquetFile\Schema\{FlatColumn, ListElement, NestedColumn, Repetition};
1212
use PHPUnit\Framework\TestCase;
1313

1414
final class ListsWritingTest extends TestCase
@@ -226,6 +226,40 @@ public function test_writing_nullable_list_of_structures() : void
226226
);
227227
}
228228

229+
public function test_writing_nullable_list_of_structures_with_required_fields() : void
230+
{
231+
$path = __DIR__ . '/var/test-writer-parquet-test-' . generate_random_string() . '.parquet';
232+
233+
$writer = new Writer();
234+
$schema = Schema::with(
235+
NestedColumn::list(
236+
'list_of_structs',
237+
ListElement::structure(
238+
[
239+
FlatColumn::int32('id', Repetition::REQUIRED),
240+
],
241+
true
242+
),
243+
)
244+
);
245+
246+
$faker = Factory::create();
247+
$inputData = \array_merge(...\array_map(static fn (int $i) : array => [
248+
[
249+
'list_of_structs' => \array_map(static fn ($i) => [
250+
'id' => $faker->numberBetween(0, Consts::PHP_INT32_MAX),
251+
], \range(1, generate_random_int(2, 10))),
252+
],
253+
], \range(1, 10)));
254+
255+
$writer->write($path, $schema, $inputData);
256+
257+
self::assertSame(
258+
$inputData,
259+
\iterator_to_array((new Reader())->read($path)->values())
260+
);
261+
}
262+
229263
public function test_writing_nullable_lists_of_ints() : void
230264
{
231265
$path = __DIR__ . '/var/test-writer-parquet-test-' . generate_random_string() . '.parquet';

0 commit comments

Comments
 (0)