Skip to content

Commit b8f92fa

Browse files
serramatutuserramatutu
committed
Fix tests that were implicitly depending on wrong nullable semantics
1 parent e2daf37 commit b8f92fa

10 files changed

Lines changed: 56 additions & 44 deletions

File tree

‎arrow/array/table_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,7 @@ func TestTableFromRecords(t *testing.T) {
619619

620620
schema := arrow.NewSchema(
621621
[]arrow.Field{
622-
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
622+
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
623623
{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
624624
},
625625
nil,
@@ -793,7 +793,7 @@ func TestTableToString(t *testing.T) {
793793

794794
schema := arrow.NewSchema(
795795
[]arrow.Field{
796-
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
796+
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
797797
{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
798798
},
799799
nil,
@@ -822,7 +822,7 @@ func TestTableToString(t *testing.T) {
822822
expected_str :=
823823
`schema:
824824
fields: 2
825-
- f1-i32: type=int32
825+
- f1-i32: type=int32, nullable
826826
- f2-f64: type=float64
827827
f1-i32: [[1 2 3 4 5 6 7 8 (null) 10], [111 112 113 114 115 116 117 118 119 120]]
828828
f2-f64: [[11 12 13 14 15 16 17 18 19 20], [211 212 213 214 215 216 217 218 219 220]]

‎arrow/array/validate_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ func TestTopLevelValidate(t *testing.T) {
194194

195195
t.Run("ValidateRecord validates all columns", func(t *testing.T) {
196196
validArr := makeStringArrayRaw(t, []int32{0, 3, 6}, "abcdef", 2, 0)
197-
corruptArr := makeStringArrayRaw(t, []int32{0, 5, 3, 5}, "hello", 3, 0)
197+
corruptArr := makeStringArrayRaw(t, []int32{0, 4, 3}, "hello", 2, 0)
198198

199199
schema := arrow.NewSchema([]arrow.Field{
200200
{Name: "ok", Type: arrow.BinaryTypes.String},

‎arrow/compute/vector_sort_test.go‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1373,8 +1373,8 @@ func TestVectorSortIndicesCppRecordBatchParity(t *testing.T) {
13731373

13741374
t.Run("Null", func(t *testing.T) {
13751375
schema := arrow.NewSchema([]arrow.Field{
1376-
{Name: "a", Type: arrow.PrimitiveTypes.Uint8},
1377-
{Name: "b", Type: arrow.PrimitiveTypes.Uint32},
1376+
{Name: "a", Type: arrow.PrimitiveTypes.Uint8, Nullable: true},
1377+
{Name: "b", Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
13781378
}, nil)
13791379
jsonRows := `[
13801380
{"a": null, "b": 5},
@@ -1426,8 +1426,8 @@ func TestVectorSortIndicesCppRecordBatchParity(t *testing.T) {
14261426

14271427
t.Run("NaNAndNull", func(t *testing.T) {
14281428
schema := arrow.NewSchema([]arrow.Field{
1429-
{Name: "a", Type: arrow.PrimitiveTypes.Float32},
1430-
{Name: "b", Type: arrow.PrimitiveTypes.Float64},
1429+
{Name: "a", Type: arrow.PrimitiveTypes.Float32, Nullable: true},
1430+
{Name: "b", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
14311431
}, nil)
14321432
ba := array.NewFloat32Builder(mem)
14331433
defer ba.Release()
@@ -1460,8 +1460,8 @@ func TestVectorSortIndicesCppRecordBatchParity(t *testing.T) {
14601460

14611461
t.Run("Boolean", func(t *testing.T) {
14621462
schema := arrow.NewSchema([]arrow.Field{
1463-
{Name: "a", Type: arrow.FixedWidthTypes.Boolean},
1464-
{Name: "b", Type: arrow.FixedWidthTypes.Boolean},
1463+
{Name: "a", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
1464+
{Name: "b", Type: arrow.FixedWidthTypes.Boolean, Nullable: true},
14651465
}, nil)
14661466
jsonRows := `[
14671467
{"a": true, "b": null},
@@ -1536,7 +1536,7 @@ func TestVectorSortIndicesCppRecordBatchParity(t *testing.T) {
15361536
d256 := &arrow.Decimal256Type{Precision: 4, Scale: 2}
15371537
schema := arrow.NewSchema([]arrow.Field{
15381538
{Name: "a", Type: d128},
1539-
{Name: "b", Type: d256},
1539+
{Name: "b", Type: d256, Nullable: true},
15401540
}, nil)
15411541
jsonRows := `[
15421542
{"a": "12.3", "b": "12.34"},
@@ -1561,8 +1561,8 @@ func TestVectorSortIndicesCppRecordBatchParity(t *testing.T) {
15611561

15621562
t.Run("DuplicateSortKeys", func(t *testing.T) {
15631563
schema := arrow.NewSchema([]arrow.Field{
1564-
{Name: "a", Type: arrow.PrimitiveTypes.Float32},
1565-
{Name: "b", Type: arrow.PrimitiveTypes.Float64},
1564+
{Name: "a", Type: arrow.PrimitiveTypes.Float32, Nullable: true},
1565+
{Name: "b", Type: arrow.PrimitiveTypes.Float64, Nullable: true},
15661566
}, nil)
15671567
ba := array.NewFloat32Builder(mem)
15681568
defer ba.Release()
@@ -1610,8 +1610,8 @@ func TestVectorSortIndicesCppTableParity(t *testing.T) {
16101610
ctx := context.Background()
16111611

16121612
schemaAB := arrow.NewSchema([]arrow.Field{
1613-
{Name: "a", Type: arrow.PrimitiveTypes.Uint8},
1614-
{Name: "b", Type: arrow.PrimitiveTypes.Uint32},
1613+
{Name: "a", Type: arrow.PrimitiveTypes.Uint8, Nullable: true},
1614+
{Name: "b", Type: arrow.PrimitiveTypes.Uint32, Nullable: true},
16151615
}, nil)
16161616

16171617
t.Run("EmptyTable", func(t *testing.T) {
@@ -1667,8 +1667,8 @@ func TestVectorSortIndicesCppTableParity(t *testing.T) {
16671667
t.Run("BinaryLikeTwoChunks", func(t *testing.T) {
16681668
fsb3 := &arrow.FixedSizeBinaryType{ByteWidth: 3}
16691669
s := arrow.NewSchema([]arrow.Field{
1670-
{Name: "a", Type: arrow.BinaryTypes.LargeString},
1671-
{Name: "b", Type: fsb3},
1670+
{Name: "a", Type: arrow.BinaryTypes.LargeString, Nullable: true},
1671+
{Name: "b", Type: fsb3, Nullable: true},
16721672
}, nil)
16731673
buildBatch := func(a []string, b [][]byte, bNulls []bool) arrow.RecordBatch {
16741674
ab := array.NewLargeStringBuilder(mem)

‎arrow/example_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ func Example_structArray() {
261261
pool := memory.NewGoAllocator()
262262

263263
dtype := arrow.StructOf([]arrow.Field{
264-
{Name: "f1", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint8)},
264+
{Name: "f1", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint8), Nullable: true},
265265
{Name: "f2", Type: arrow.PrimitiveTypes.Int32},
266266
}...)
267267

@@ -331,7 +331,7 @@ func Example_structArray() {
331331
// Output:
332332
// NullN() = 1
333333
// Len() = 4
334-
// Type() = struct<f1: list<item: uint8, nullable>, f2: int32>
334+
// Type() = struct<f1: list<item: uint8, nullable> nullable, f2: int32>
335335
// Struct[0] = [[j, o, e], 1]
336336
// Struct[1] = [[], 2]
337337
// Struct[2] = (null)
@@ -463,7 +463,7 @@ func Example_record() {
463463

464464
schema := arrow.NewSchema(
465465
[]arrow.Field{
466-
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
466+
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
467467
{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
468468
},
469469
nil,
@@ -493,7 +493,7 @@ func Example_recordReader() {
493493

494494
schema := arrow.NewSchema(
495495
[]arrow.Field{
496-
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
496+
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
497497
{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
498498
},
499499
nil,
@@ -542,7 +542,7 @@ func Example_table() {
542542

543543
schema := arrow.NewSchema(
544544
[]arrow.Field{
545-
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32},
545+
{Name: "f1-i32", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
546546
{Name: "f2-f64", Type: arrow.PrimitiveTypes.Float64},
547547
},
548548
nil,

‎arrow/extensions/uuid_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ func TestUUIDExtensionBuilder(t *testing.T) {
6262

6363
func TestUUIDExtensionRecordBuilder(t *testing.T) {
6464
schema := arrow.NewSchema([]arrow.Field{
65-
{Name: "uuid", Type: extensions.NewUUIDType()},
65+
{Name: "uuid", Type: extensions.NewUUIDType(), Nullable: true},
6666
}, nil)
6767
builder := array.NewRecordBuilder(memory.DefaultAllocator, schema)
6868
builder.Field(0).(*extensions.UUIDBuilder).Append(testUUID)

‎arrow/internal/arrdata/arrdata.go‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -192,8 +192,8 @@ func makeStructsRecords() []arrow.RecordBatch {
192192
mem := memory.NewGoAllocator()
193193

194194
fields := []arrow.Field{
195-
{Name: "f1", Type: arrow.PrimitiveTypes.Int32},
196-
{Name: "f2", Type: arrow.BinaryTypes.String},
195+
{Name: "f1", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
196+
{Name: "f2", Type: arrow.BinaryTypes.String, Nullable: true},
197197
}
198198
dtype := arrow.StructOf(fields...)
199199
schema := arrow.NewSchema([]arrow.Field{{Name: "struct_nullable", Type: dtype, Nullable: true}}, nil)
@@ -433,8 +433,8 @@ func makeFixedSizeListsRecords() []arrow.RecordBatch {
433433
func makeStringsRecords() []arrow.RecordBatch {
434434
mem := memory.NewGoAllocator()
435435
schema := arrow.NewSchema([]arrow.Field{
436-
{Name: "strings", Type: arrow.BinaryTypes.String},
437-
{Name: "bytes", Type: arrow.BinaryTypes.Binary},
436+
{Name: "strings", Type: arrow.BinaryTypes.String, Nullable: true},
437+
{Name: "bytes", Type: arrow.BinaryTypes.Binary, Nullable: true},
438438
}, nil)
439439

440440
mask := []bool{true, false, false, true, true}
@@ -1108,9 +1108,21 @@ func makeUnionRecords() []arrow.RecordBatch {
11081108
func makeRunEndEncodedRecords() []arrow.RecordBatch {
11091109
mem := memory.NewGoAllocator()
11101110
schema := arrow.NewSchema([]arrow.Field{
1111-
{Name: "ree16", Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int16, arrow.BinaryTypes.String)},
1112-
{Name: "ree32", Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32)},
1113-
{Name: "ree64", Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int64, arrow.BinaryTypes.Binary)},
1111+
{
1112+
Name: "ree16",
1113+
Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int16, arrow.BinaryTypes.String),
1114+
Nullable: true,
1115+
},
1116+
{
1117+
Name: "ree32",
1118+
Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int32, arrow.PrimitiveTypes.Int32),
1119+
Nullable: true,
1120+
},
1121+
{
1122+
Name: "ree64",
1123+
Type: arrow.RunEndEncodedOf(arrow.PrimitiveTypes.Int64, arrow.BinaryTypes.Binary),
1124+
Nullable: true,
1125+
},
11141126
}, nil)
11151127

11161128
schema.Field(1).Type.(*arrow.RunEndEncodedType).ValueNullable = false

‎arrow/internal/arrjson/arrjson_test.go‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -948,15 +948,15 @@ func makeStructsWantJSONs() string {
948948
"isSigned": true,
949949
"bitWidth": 32
950950
},
951-
"nullable": false,
951+
"nullable": true,
952952
"children": []
953953
},
954954
{
955955
"name": "f2",
956956
"type": {
957957
"name": "utf8"
958958
},
959-
"nullable": false,
959+
"nullable": true,
960960
"children": []
961961
}
962962
]
@@ -1957,15 +1957,15 @@ func makeStringsWantJSONs() string {
19571957
"type": {
19581958
"name": "utf8"
19591959
},
1960-
"nullable": false,
1960+
"nullable": true,
19611961
"children": []
19621962
},
19631963
{
19641964
"name": "bytes",
19651965
"type": {
19661966
"name": "binary"
19671967
},
1968-
"nullable": false,
1968+
"nullable": true,
19691969
"children": []
19701970
}
19711971
]
@@ -5765,7 +5765,7 @@ func makeRunEndEncodedWantJSONs() string {
57655765
"type": {
57665766
"name": "runendencoded"
57675767
},
5768-
"nullable": false,
5768+
"nullable": true,
57695769
"children": [
57705770
{
57715771
"name": "run_ends",
@@ -5792,7 +5792,7 @@ func makeRunEndEncodedWantJSONs() string {
57925792
"type": {
57935793
"name": "runendencoded"
57945794
},
5795-
"nullable": false,
5795+
"nullable": true,
57965796
"children": [
57975797
{
57985798
"name": "run_ends",
@@ -5821,7 +5821,7 @@ func makeRunEndEncodedWantJSONs() string {
58215821
"type": {
58225822
"name": "runendencoded"
58235823
},
5824-
"nullable": false,
5824+
"nullable": true,
58255825
"children": [
58265826
{
58275827
"name": "run_ends",

‎arrow/ipc/cmd/arrow-ls/main_test.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ records: 3
5959
name: "structs",
6060
want: `schema:
6161
fields: 1
62-
- struct_nullable: type=struct<f1: int32, f2: utf8>, nullable
62+
- struct_nullable: type=struct<f1: int32 nullable, f2: utf8 nullable>, nullable
6363
records: 2
6464
`,
6565
},
@@ -75,8 +75,8 @@ records: 4
7575
name: "strings",
7676
want: `schema:
7777
fields: 2
78-
- strings: type=utf8
79-
- bytes: type=binary
78+
- strings: type=utf8, nullable
79+
- bytes: type=binary, nullable
8080
records: 3
8181
`,
8282
},
@@ -221,7 +221,7 @@ records: 3
221221
name: "structs",
222222
want: `schema:
223223
fields: 1
224-
- struct_nullable: type=struct<f1: int32, f2: utf8>, nullable
224+
- struct_nullable: type=struct<f1: int32 nullable, f2: utf8 nullable>, nullable
225225
records: 2
226226
`,
227227
},
@@ -230,7 +230,7 @@ records: 2
230230
want: `version: V5
231231
schema:
232232
fields: 1
233-
- struct_nullable: type=struct<f1: int32, f2: utf8>, nullable
233+
- struct_nullable: type=struct<f1: int32 nullable, f2: utf8 nullable>, nullable
234234
records: 2
235235
`,
236236
},

‎arrow/ipc/ipc_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ func TestWriteColumnWithOffset(t *testing.T) {
282282

283283
func TestIPCTable(t *testing.T) {
284284
pool := memory.NewGoAllocator()
285-
schema := arrow.NewSchema([]arrow.Field{{Name: "f1", Type: arrow.PrimitiveTypes.Int32}}, nil)
285+
schema := arrow.NewSchema([]arrow.Field{{Name: "f1", Type: arrow.PrimitiveTypes.Int32, Nullable: true}}, nil)
286286
b := array.NewRecordBuilder(pool, schema)
287287
defer b.Release()
288288
b.Field(0).(*array.Int32Builder).AppendValues([]int32{1, 2, 3, 4}, []bool{true, true, false, true})

‎arrow/ipc/reader_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ func TestReaderCheckedAllocator(t *testing.T) {
100100
func TestMappedReader(t *testing.T) {
101101
pool := memory.NewCheckedAllocator(memory.NewGoAllocator())
102102
defer pool.AssertSize(t, 0)
103-
schema := arrow.NewSchema([]arrow.Field{{Name: "f1", Type: arrow.PrimitiveTypes.Int32}}, nil)
103+
schema := arrow.NewSchema([]arrow.Field{{Name: "f1", Type: arrow.PrimitiveTypes.Int32, Nullable: true}}, nil)
104104
b := array.NewRecordBuilder(pool, schema)
105105
defer b.Release()
106106
b.Field(0).(*array.Int32Builder).AppendValues([]int32{1, 2, 3, 4}, []bool{true, true, false, true})

0 commit comments

Comments
 (0)