Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/CI.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ jobs:
- uses: julia-actions/cache@v2
- uses: julia-actions/julia-buildpkg@v1
- uses: julia-actions/julia-runtest@v1
with:
test_args: '--verbose'
- uses: julia-actions/julia-processcoverage@v1
- uses: codecov/codecov-action@v5
with:
Expand Down
7 changes: 4 additions & 3 deletions Project.toml
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
name = "ASDF"
uuid = "686f71d1-807d-59a4-a860-28280ea06d7b"
authors = ["Erik Schnetter <schnetter@gmail.com>"]
version = "2.0.0"
authors = ["Erik Schnetter <schnetter@gmail.com>"]

[workspace]
projects = ["test", "docs"]

[deps]
BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e"
ChunkCodecLibBlosc = "c6a955be-ab7f-4fbb-b38f-caf93db6b928"
ChunkCodecLibBzip2 = "2b723af9-f480-4e8d-a1e4-4a9f5a906122"
ChunkCodecLibLz4 = "7e9cc85e-5614-42a3-ad86-b78f920b38a5"
Expand All @@ -17,7 +19,6 @@ StridedViews = "4db3bf67-4bd7-4b4e-b153-31dc3fb37143"
YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"

[compat]
BlockArrays = "0.16.43, 1"
ChunkCodecLibBlosc = "0.2.0, 0.3"
ChunkCodecLibBzip2 = "0.2.0, 1.0"
ChunkCodecLibLz4 = "0.2.1, 1.0"
Expand Down
21 changes: 10 additions & 11 deletions src/ASDF.jl
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
module ASDF

using BlockArrays
using CodecXz
using ChunkCodecLibBlosc
using ChunkCodecLibBzip2
using ChunkCodecLibLz4
using ChunkCodecLibZlib
using ChunkCodecLibZstd
using MD5
using PkgVersion
using StridedViews
using YAML
using ChunkCodecLibBlosc: BloscCodec, BloscEncodeOptions
using ChunkCodecLibBzip2: BZ2Codec, BZ2EncodeOptions
using ChunkCodecLibLz4: LZ4FrameCodec, LZ4FrameEncodeOptions
using ChunkCodecLibZlib: ZlibCodec, ZlibEncodeOptions
using ChunkCodecLibZstd: ZstdCodec, ZstdEncodeOptions, decode, encode
using CodecXz: XzCompressor, XzDecompressor
using MD5: md5
using PkgVersion: PkgVersion
using StridedViews: StridedView
using YAML: YAML

################################################################################

Expand Down
3 changes: 3 additions & 0 deletions test/Project.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
[deps]
ASDF = "686f71d1-807d-59a4-a860-28280ea06d7b"
Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
ParallelTestRunner = "d3525ed8-44d0-4b2c-a655-542cee43accc"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6"
118 changes: 20 additions & 98 deletions test/runtests.jl
Original file line number Diff line number Diff line change
@@ -1,103 +1,25 @@
using ParallelTestRunner: runtests, find_tests, parse_args
using ASDF
using Test
using YAML

map_tree(f, x) = f(x)
map_tree(f, vec::AbstractVector) = [map_tree(f, elem) for elem in vec]
map_tree(f, dict::AbstractDict) = Dict(key => map_tree(f, val) for (key, val) in dict)

output(x) = nothing
function output(arr::ASDF.NDArray)
println("source: $(arr.source)")
data = arr[]
println(" type: $(typeof(data))")
return println(" size: $(size(data))")
end

################################################################################

@testset "Read ASDF file" begin
asdf = ASDF.load_file("blue_upchan_gain.00000000.asdf")
println(YAML.write(asdf.metadata))

map_tree(output, asdf.metadata)

buffer = asdf.metadata[0]["buffer"][]
@test eltype(buffer) == Float16
@test size(buffer) == (256,)
@test buffer == fill(1, 256)

dish_index = asdf.metadata[0]["dish_index"][]
@test eltype(dish_index) == Int32
@test size(dish_index) == (3, 2)
@test dish_index == [
-1 -1
42 53
43 54
]
end

################################################################################

@testset "Read ASDF file with chunked arrays" begin
asdf = ASDF.load_file("chunking.asdf")
println(YAML.write(asdf.metadata))

map_tree(output, asdf.metadata)

chunky = asdf.metadata["chunky"][]
@test eltype(chunky) == Float16
@test size(chunky) == (4, 4)
@test chunky == [
11 21 31 41
12 22 32 42
13 23 33 43
14 24 34 44
]
const init_code = quote
using ASDF
using Test
using YAML

map_tree(f, x) = f(x)
map_tree(f, vec::AbstractVector) = [map_tree(f, elem) for elem in vec]
map_tree(f, dict::AbstractDict) = Dict(key => map_tree(f, val) for (key, val) in dict)

output(x) = nothing
function output(arr::ASDF.NDArray)
println("source: $(arr.source)")
data = arr[]
println(" type: $(typeof(data))")
return println(" size: $(size(data))")
end
end

################################################################################

@testset "Write ASDF file" begin
dirname = mktempdir(; cleanup=true)
filename = joinpath(dirname, "output.asdf")

array = Float64[1/(i+j+k-2) for i in 1:50, j in 1:51, k in 1:52]
doc = Dict{Any,Any}(
"data1" => ASDF.NDArrayWrapper([1 2; 3 4]; inline=false),
"data2" => ASDF.NDArrayWrapper([1 2; 3 4]; inline=true),
"group" => Dict{Any,Any}(
"element1" => ASDF.NDArrayWrapper(array; compression=ASDF.C_None),
"element2" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Blosc),
"element3" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Bzip2),
"element4" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Lz4),
"element5" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Xz),
"element6" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Zlib),
"element7" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Zstd),
),
)
ASDF.write_file(filename, doc)
args = parse_args(Base.ARGS)
testsuite = find_tests(@__DIR__)

doc′ = ASDF.load_file(filename)
map_tree(output, doc′.metadata)

data1 = doc["data1"][]
data1′ = doc′.metadata["data1"][]
@test eltype(data1′) == eltype(data1)
@test size(data1′) == size(data1)
@test data1′ == data1

data2 = doc["data2"][]
data2′ = doc′.metadata["data2"][]
@test eltype(data2′) == eltype(data2)
@test size(data2′) == size(data2)
@test data2′ == data2

for n in 1:7
element = doc["group"]["element$n"][]
element′ = doc′.metadata["group"]["element$n"][]
@test eltype(element′) == eltype(element)
@test size(element′) == size(element)
@test element′ == element
end
end
runtests(ASDF, args; testsuite, init_code)
4 changes: 4 additions & 0 deletions test/test-aqua.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
@testset "Aqua" begin
using ASDF, Aqua
Aqua.test_all(ASDF)
end
20 changes: 20 additions & 0 deletions test/test-read.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
@testset "Read ASDF file" begin
asdf = ASDF.load_file("blue_upchan_gain.00000000.asdf")
println(YAML.write(asdf.metadata))

map_tree(output, asdf.metadata)

buffer = asdf.metadata[0]["buffer"][]
@test eltype(buffer) == Float16
@test size(buffer) == (256,)
@test buffer == fill(1, 256)

dish_index = asdf.metadata[0]["dish_index"][]
@test eltype(dish_index) == Int32
@test size(dish_index) == (3, 2)
@test dish_index == [
-1 -1
42 53
43 54
]
end
16 changes: 16 additions & 0 deletions test/test-read_chunked.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
@testset "Read ASDF file with chunked arrays" begin
asdf = ASDF.load_file("chunking.asdf")
println(YAML.write(asdf.metadata))

map_tree(output, asdf.metadata)

chunky = asdf.metadata["chunky"][]
@test eltype(chunky) == Float16
@test size(chunky) == (4, 4)
@test chunky == [
11 21 31 41
12 22 32 42
13 23 33 43
14 24 34 44
]
end
43 changes: 43 additions & 0 deletions test/test-write.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
@testset "Write ASDF file" begin
dirname = mktempdir(; cleanup=true)
filename = joinpath(dirname, "output.asdf")

array = Float64[1/(i+j+k-2) for i in 1:50, j in 1:51, k in 1:52]
doc = Dict{Any,Any}(
"data1" => ASDF.NDArrayWrapper([1 2; 3 4]; inline=false),
"data2" => ASDF.NDArrayWrapper([1 2; 3 4]; inline=true),
"group" => Dict{Any,Any}(
"element1" => ASDF.NDArrayWrapper(array; compression=ASDF.C_None),
"element2" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Blosc),
"element3" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Bzip2),
"element4" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Lz4),
"element5" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Xz),
"element6" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Zlib),
"element7" => ASDF.NDArrayWrapper(array; compression=ASDF.C_Zstd),
),
)
ASDF.write_file(filename, doc)

doc′ = ASDF.load_file(filename)
map_tree(output, doc′.metadata)

data1 = doc["data1"][]
data1′ = doc′.metadata["data1"][]
@test eltype(data1′) == eltype(data1)
@test size(data1′) == size(data1)
@test data1′ == data1

data2 = doc["data2"][]
data2′ = doc′.metadata["data2"][]
@test eltype(data2′) == eltype(data2)
@test size(data2′) == size(data2)
@test data2′ == data2

for n in 1:7
element = doc["group"]["element$n"][]
element′ = doc′.metadata["group"]["element$n"][]
@test eltype(element′) == eltype(element)
@test size(element′) == size(element)
@test element′ == element
end
end