Insert timeit_debug to measure the timing of loading a DimTree

Author: felixcremer

Project.toml

@@ -1,7 +1,7 @@
 name = "SentinelDataSource"
 uuid = "afd1bdb2-e5b9-463c-8ecd-ef9b9458f1b0"
-authors = ["Felix Cremer <fcremer@bgc-jena.mpg.de> and contributors"]
 version = "0.1.0-DEV"
+authors = ["Felix Cremer <fcremer@bgc-jena.mpg.de> and contributors"]
 
 [deps]
 CFTime = "179af706-886a-5703-950a-314cd64e0468"
@@ -10,20 +10,22 @@ DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
 DiskArrayTools = "fcd2136c-9f69-4db6-97e5-f31981721d63"
 DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
+TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
 YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14"
 YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
 Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"
 ZarrDatasets = "519a4cdf-1362-424a-9ea1-b1d782dbb24b"
 
 [compat]
 CFTime = "0.2.1"
-CommonDataModel = "0.3.8"
+CommonDataModel = "0.3.8, 0.4"
 DimensionalData = "0.29.12"
 DiskArrayTools = "0.1.12"
 DiskArrays = "0.4.11"
 Rasters = "0.14.4"
+TimerOutputs = "0.5.29"
 YAXArrayBase = "0.7.5"
-YAXArrays = "0.6.1"
+YAXArrays = "0.7"
 Zarr = "0.9.4"
 ZarrDatasets = "0.1.3"
 julia = "1.6.7"

src/SentinelDataSource.jl

@@ -1,30 +1,38 @@
 module SentinelDataSource
-using DimensionalData: DimTree, DimArray
+using DimensionalData: DimTree, DimArray, DimensionalData
 using Zarr: zopen
 using ZarrDatasets: ZarrDataset
 using CommonDataModel: CommonDataModel as CDM
 using Rasters: Rasters
-
+using TimerOutputs
 export open_tree
 
 open_tree(path::AbstractString) = open_tree(ZarrDataset(path))
 
 function open_tree(dataset::ZarrDataset)
-    @time "stem" stem = DimTree()
-    @time "groups" groupnames = CDM.groupnames(dataset)
-    @time "vars" varnames = CDM.varnames(dataset)
-    @time "dims" alldimnames = nesteddimnames(dataset)
-    @time "forvar" for v in setdiff(varnames, alldimnames)
-        @time "var" var = CDM.variable(dataset, v)
-        @time "vardims" vardims = Rasters._dims(var)
-        @time "meta" metadata_out = Rasters._metadata(var)
-        @time "missing" missingval_out = Rasters._read_missingval_pair(var, metadata_out, Rasters.nokw)
-        @time "mod" mod = Rasters._mod(eltype(var), metadata_out, missingval_out;scaled=true, coerce=true)
-        @time "vardata" vardata = Rasters._maybe_modify(var, mod)
-        @time "set" setindex!(stem, DimArray(vardata, vardims),Symbol(v))
+    @timeit_debug "stem" stem = DimTree()
+    @timeit_debug "groups" groupnames = CDM.groupnames(dataset)
+    @timeit_debug "vars" varnames = CDM.varnames(dataset)
+    @timeit_debug "dims" alldimnames = nesteddimnames(dataset)
+    diffnames = setdiff(varnames, alldimnames)
+    for v in diffnames
+        @timeit_debug "var $v" begin
+        var = CDM.variable(dataset, v)
+        vardims = Rasters._dims(var)
+        metadata_out = Rasters._metadata(var)
+        missingval_out = Rasters._read_missingval_pair(var, metadata_out, Rasters.nokw)
+        mod = Rasters._mod(eltype(var), metadata_out, missingval_out;scaled=true, coerce=true)
+        #=
+        Rasters.FileArray{ZarrDataset}(var, filename;
+                name=v, Rasters.nokw, mod, write=false
+            )
+                =#
+        vardata = Rasters._maybe_modify(var, mod)
+        setindex!(stem, DimArray(vardata, vardims),Symbol(v))
+        end
     end
-    @time "forg" for g in groupnames
-        setindex!(stem,  open_tree(CDM.group(dataset, g)),Symbol(g))
+    for g in groupnames
+        @timeit_debug "forg $g"    setindex!(stem,  open_tree(CDM.group(dataset, g)),Symbol(g))
     end
     stem
 end