Merge pull request #19 from JuliaGeo/fc/fixdataloading

Fix data loading

Author: felixcremer

.gitignore

@@ -1 +1,2 @@
 /Manifest.toml
+test/Manifest.toml

Project.toml

@@ -1,29 +1,25 @@
 name = "SentinelDataSource"
 uuid = "afd1bdb2-e5b9-463c-8ecd-ef9b9458f1b0"
-authors = ["Felix Cremer <fcremer@bgc-jena.mpg.de> and contributors"]
 version = "0.1.0-DEV"
+authors = ["Felix Cremer <fcremer@bgc-jena.mpg.de> and contributors"]
 
 [deps]
 CFTime = "179af706-886a-5703-950a-314cd64e0468"
 CommonDataModel = "1fbeeb36-5f17-413c-809b-666fb144f157"
 DimensionalData = "0703355e-b756-11e9-17c0-8b28908087d0"
-DiskArrayTools = "fcd2136c-9f69-4db6-97e5-f31981721d63"
 DiskArrays = "3c3547ce-8d99-4f5e-a174-61eb10b00ae3"
 Rasters = "a3a2b9e3-a471-40c9-b274-f788e487c689"
-YAXArrayBase = "90b8fcef-0c2d-428d-9c56-5f86629e9d14"
-YAXArrays = "c21b50f5-aa40-41ea-b809-c0f5e47bfa5c"
+TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
 Zarr = "0a941bbe-ad1d-11e8-39d9-ab76183a1d99"
 ZarrDatasets = "519a4cdf-1362-424a-9ea1-b1d782dbb24b"
 
 [compat]
 CFTime = "0.2.1"
-CommonDataModel = "0.3.8"
-DimensionalData = "0.29.12"
-DiskArrayTools = "0.1.12"
+CommonDataModel = "0.3.8, 0.4"
+DimensionalData = "0.29.12, 0.30"
 DiskArrays = "0.4.11"
-Rasters = "0.14.4"
-YAXArrayBase = "0.7.5"
-YAXArrays = "0.6.1"
+Rasters = "0.15"
+TimerOutputs = "0.5.29"
 Zarr = "0.9.4"
 ZarrDatasets = "0.1.3"
 julia = "1.6.7"

docs/src/spectraldiversity.jl

@@ -0,0 +1,47 @@
+using SentinelDataSource
+using YAXArrays
+using Distances
+using DimensionalData
+using AWS
+
+struct AnonymousGCS <:AbstractAWSConfig end
+struct NoCredentials end
+AWS.region(aws::AnonymousGCS) = "" # No region
+AWS.credentials(aws::AnonymousGCS) = NoCredentials() # No credentials
+AWS.check_credentials(c::NoCredentials) = c # Skip credentials check
+AWS.sign!(aws::AnonymousGCS, ::AWS.Request) = nothing # Don't sign request
+function AWS.generate_service_url(aws::AnonymousGCS, service::String, resource::String)
+    service == "s3" || throw(ArgumentError("Can only handle s3 requests to GCS"))
+    awsurl =  string("https://objects.eodc.eu:2222", resource)
+    return awsurl
+end
+AWS.global_aws_config(AnonymousGCS())
+path = "s3://e05ab01a9d56408d82ac32d69a5aae2a:sample-data/tutorial_data/cpm_v253/S2A_MSIL2A_20180601T102021_N0500_R065_T32UPC_20230902T045008.zarr/measurements/reflectance/r20m"
+
+pathbase = "s3://e05ab01a9d56408d82ac32d69a5aae2a:sample-data/eopf-sample-output/"
+zopen(path)
+c = Cube(open_dataset(zopen(path)))
+yax = YAXArray((Dim{:x}(1:500), Dim{:y}(500:-1:1), Dim{:Variables}(1:10)), rand(500,500,10))
+c = yax
+function spectraldiversity(c)
+    dlat = dims(c, :x)
+    dlon = dims(c, :y)
+    dband = dims(c, :Variables)
+    # This needs a better interface for within index space
+    latinterval = MovingIntervals(center=dlat.val, width=3*step(dlat), n=length(dlat)-1, step=step(dlat))
+    loninterval = MovingIntervals(center=dlon.val, width=-3*step(dlon), n=length(dlon)-1, step=-step(dlon))
+    bandinterval = MovingIntervals(:closed, :closed, left=first(dband.val), right=last(dband.val), n=1)
+    meanwindows = windows(c, :y=>loninterval, :Variables => Whole())
+    @show meanwindows[10,10,1]
+    @time xmap(innerdiversity, meanwindows)
+end
+spectraldiversity(c)
+function innerdiversity(xout, arr)
+    winsize =  prod(size(arr)) ÷ size(arr,3)
+    arrreshaped = reshape(arr, (winsize, size(arr,3)))
+    tslist = eachrow(arrreshaped)
+    dists = pairwise(Euclidean(), tslist) .* 2 ./ winsize .^ 4
+    xout .= sum(dists) ./ 2 
+end
+
+spectraldiversity(c)

src/SentinelDataSource.jl

@@ -1,24 +1,38 @@
 module SentinelDataSource
-using DimensionalData: DimTree
+using DimensionalData: DimTree, DimArray, DimensionalData
 using Zarr: zopen
 using ZarrDatasets: ZarrDataset
 using CommonDataModel: CommonDataModel as CDM
-using Rasters:Raster
-
+using Rasters: Rasters
+using TimerOutputs
 export open_tree
 
 open_tree(path::AbstractString) = open_tree(ZarrDataset(path))
 
 function open_tree(dataset::ZarrDataset)
-    stem = DimTree()
-    groupnames = CDM.groupnames(dataset)
-    varnames = CDM.varnames(dataset)
-    alldimnames = nesteddimnames(dataset)
-    for v in setdiff(varnames, alldimnames)
-        setindex!(stem, Raster(CDM.variable(dataset, v), lazy=true),Symbol(v))
+    @timeit_debug "stem" stem = DimTree()
+    @timeit_debug "groups" groupnames = CDM.groupnames(dataset)
+    @timeit_debug "vars" varnames = CDM.varnames(dataset)
+    @timeit_debug "dims" alldimnames = nesteddimnames(dataset)
+    diffnames = setdiff(varnames, alldimnames)
+    for v in diffnames
+        @timeit_debug "var $v" begin
+        var = CDM.variable(dataset, v)
+        vardims = Rasters._dims(var)
+        metadata_out = Rasters._metadata(var)
+        missingval_out = Rasters._read_missingval_pair(var, metadata_out, Rasters.nokw)
+        mod = Rasters._mod(eltype(var), metadata_out, missingval_out;scaled=true, coerce=true)
+        #=
+        Rasters.FileArray{ZarrDataset}(var, filename;
+                name=v, Rasters.nokw, mod, write=false
+            )
+                =#
+        vardata = Rasters._maybe_modify(var, mod)
+        setindex!(stem, DimArray(vardata, vardims),Symbol(v))
+        end
     end
     for g in groupnames
-        setindex!(stem,  open_tree(CDM.group(dataset, g)),Symbol(g))
+        @timeit_debug "forg $g"    setindex!(stem,  open_tree(CDM.group(dataset, g)),Symbol(g))
     end
     stem
 end