Skip to content

Commit 21e7173

Browse files
StephanPreibischStephanPreibisch
authored
Merge pull request #47 from JaneliaSciComp/ome-zarr-xml-export
Add support for exporting/fusing a BDV compatible OME-ZARR
2 parents 3111f62 + 77c1142 commit 21e7173

3 files changed

Lines changed: 124 additions & 91 deletions

File tree

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@
102102
<imglib2-algorithm.version>0.18.0</imglib2-algorithm.version>
103103
<bigdataviewer-core.version>10.6.4</bigdataviewer-core.version>
104104
<spim_data.version>2.3.5</spim_data.version>
105-
<multiview-reconstruction.version>5.4.1</multiview-reconstruction.version>
105+
<multiview-reconstruction.version>6.0.3</multiview-reconstruction.version>
106106
<BigStitcher.version>2.5.0</BigStitcher.version>
107107

108108
<n5-universe.version>2.1.0</n5-universe.version>

‎src/main/java/net/preibisch/bigstitcher/spark/CreateFusionContainer.java‎

Lines changed: 119 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@
66
import java.util.ArrayList;
77
import java.util.Arrays;
88
import java.util.Collection;
9+
import java.util.HashMap;
910
import java.util.List;
11+
import java.util.Map;
1012
import java.util.concurrent.Callable;
1113
import java.util.function.Function;
1214

@@ -43,6 +45,7 @@
4345
import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
4446
import net.preibisch.mvrecon.fiji.spimdata.boundingbox.BoundingBox;
4547
import net.preibisch.mvrecon.fiji.spimdata.imgloaders.OMEZarrAttibutes;
48+
import net.preibisch.mvrecon.fiji.spimdata.imgloaders.AllenOMEZarrLoader.OMEZARREntry;
4649
import net.preibisch.mvrecon.process.export.ExportN5Api;
4750
import net.preibisch.mvrecon.process.interestpointregistration.TransformationTools;
4851
import net.preibisch.mvrecon.process.n5api.N5ApiTools;
@@ -176,12 +179,6 @@ else if ( numTimepoints > numTimepointsXML )
176179
{
177180
this.xmlOutURI = URITools.toURI( xmlOutURIString );
178181
System.out.println( "XML: " + xmlOutURI );
179-
180-
if ( storageType == StorageFormat.ZARR )
181-
{
182-
System.out.println( "BDV project for OME-ZARR not yet supported (but very soon!)" );
183-
return null;
184-
}
185182
}
186183

187184
BoundingBox boundingBox = Import.getBoundingBox( dataGlobal, viewIdsGlobal, boundingBoxName );
@@ -324,86 +321,20 @@ else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.ZARR )
324321
}
325322

326323
// setup datasets and metadata
327-
MultiResolutionLevelInfo[][] mrInfos;
328-
329-
if ( bdv )
330-
{
331-
System.out.println( "Creating BDV compatible container at '" + outPathURI + "' ... " );
332-
333-
if ( storageType == StorageFormat.N5 )
334-
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/N5" );
335-
else
336-
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/HDF5" );
337-
338-
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/OutputXML", xmlOutURI );
339-
340-
final long[] bb = boundingBox.dimensionsAsLongArray();
341-
342-
final ArrayList< ViewSetup > setups = new ArrayList<>();
343-
final ArrayList< TimePoint > tps = new ArrayList<>();
344-
345-
for ( int t = 0; t < numTimepoints; ++t )
346-
tps.add( new TimePoint( t ) );
347-
348-
// extract the resolution of the s0 export
349-
// TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied
350-
// TODO: this is a hack (returns 1,1,1) so the export downsampling pyramid is working
351-
final VoxelDimensions vx = new FinalVoxelDimensions( "micrometer", new double[] { 1, 1, 1 } );// dataGlobal.getSequenceDescription().getViewSetupsOrdered().iterator().next().getVoxelSize();
352-
final double[] resolutionS0 = OMEZarrAttibutes.getResolutionS0( vx, anisotropyFactor, Double.NaN );
353-
354-
System.out.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() );
355-
356-
final VoxelDimensions vxNew = new FinalVoxelDimensions( "micrometer", resolutionS0 );
357-
358-
for ( int c = 0; c < numChannels; ++c )
359-
{
360-
setups.add(
361-
new ViewSetup(
362-
c,
363-
"setup " + c,
364-
new FinalDimensions( bb ),
365-
vxNew,
366-
new Tile( 0 ),
367-
new Channel( c, "Channel " + c ),
368-
new Angle( 0 ),
369-
new Illumination( 0 ) ) );
370-
}
371-
372-
final SpimData2 dataFusion =
373-
SpimData2Tools.createNewSpimDataForFusion( storageType, outPathURI, xmlOutURI, setups, tps );
374-
375-
new XmlIoSpimData2().save( dataFusion, xmlOutURI );
376-
377-
final Collection<ViewDescription> vds = dataFusion.getSequenceDescription().getViewDescriptions().values();
378-
379-
mrInfos = new MultiResolutionLevelInfo[ vds.size() ][];
380-
381-
vds.stream().parallel().forEach( vd ->
382-
{
383-
final int c = vd.getViewSetup().getChannel().getId();
384-
final int t = vd.getTimePointId();
385-
386-
if ( storageType == StorageFormat.N5 )
387-
{
388-
mrInfos[ c + t*c ] = N5ApiTools.setupBdvDatasetsN5(
389-
driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings);
390-
391-
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/N5" );
392-
}
393-
else // HDF5
394-
{
395-
mrInfos[ c + t*numChannels ] = N5ApiTools.setupBdvDatasetsHDF5(
396-
driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings);
397-
}
398-
});
399-
400-
// TODO: set extra attributes to load the state
401-
}
402-
else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export
324+
MultiResolutionLevelInfo[][] mrInfos = null;
325+
326+
// OME-Zarr export
327+
// this code needs refactoring some sort of refactoring. When exporting OME-ZARR, we first create the OME-ZARR container,
328+
// and if it is BDV-XML, we only create the XML in the next if statement. If it is N5/HDF5, there
329+
// is code that creates the N5/HDF5 container and the XML in one if statement. The reason is that
330+
// HDF5/N5 containers with XML may be different that OME-ZARR's; they are always the same no matter
331+
// if it is a BDV project or not
332+
if ( storageType == StorageFormat.ZARR )
403333
{
404334
System.out.println( "Creating 5D OME-ZARR metadata for '" + outPathURI + "' ... " );
405335

406-
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "OME-ZARR" );
336+
if ( !bdv )
337+
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "OME-ZARR" );
407338

408339
final long[] dim3d = boundingBox.dimensionsAsLongArray();
409340

@@ -427,8 +358,10 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export
427358
blockSize5d, //5d
428359
ds ); // 5d
429360

361+
final MultiResolutionLevelInfo[] mrInfo = mrInfos[ 0 ];
362+
430363
final Function<Integer, AffineTransform3D> levelToMipmapTransform =
431-
(level) -> MipmapTransforms.getMipmapTransformDefault( mrInfos[ 0 ][level].absoluteDownsamplingDouble() );
364+
(level) -> MipmapTransforms.getMipmapTransformDefault( mrInfo[level].absoluteDownsamplingDouble() );
432365

433366
// extract the resolution of the s0 export
434367
// TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied
@@ -455,7 +388,107 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export
455388
// final GsonBuilder builder = new GsonBuilder().registerTypeAdapter( CoordinateTransformation.class, new CoordinateTransformationAdapter() );
456389
driverVolumeWriter.setAttribute( "/", "multiscales", meta );
457390
}
458-
else // simple (no bdv project) HDF5/N5 export
391+
392+
if ( bdv )
393+
{
394+
System.out.println( "Creating BDV compatible container at '" + outPathURI + "' ... " );
395+
396+
if ( storageType == StorageFormat.N5 )
397+
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/N5" );
398+
else if ( storageType == StorageFormat.ZARR )
399+
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/OME-ZARR" );
400+
else
401+
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/HDF5" );
402+
403+
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/OutputXML", xmlOutURI );
404+
405+
final long[] bb = boundingBox.dimensionsAsLongArray();
406+
407+
final ArrayList< ViewSetup > setups = new ArrayList<>();
408+
final ArrayList< TimePoint > tps = new ArrayList<>();
409+
410+
for ( int t = 0; t < numTimepoints; ++t )
411+
tps.add( new TimePoint( t ) );
412+
413+
// extract the resolution of the s0 export
414+
// TODO: this is inaccurate, we should actually estimate it from the final transformn that is applied
415+
// TODO: this is a hack (returns 1,1,1) so the export downsampling pyramid is working
416+
final VoxelDimensions vx = new FinalVoxelDimensions( "micrometer", new double[] { 1, 1, 1 } );// dataGlobal.getSequenceDescription().getViewSetupsOrdered().iterator().next().getVoxelSize();
417+
final double[] resolutionS0 = OMEZarrAttibutes.getResolutionS0( vx, anisotropyFactor, Double.NaN );
418+
419+
System.out.println( "Resolution of level 0: " + Util.printCoordinates( resolutionS0 ) + " " + "m" ); //vx.unit() might not be OME-ZARR compatiblevx.unit() );
420+
421+
final VoxelDimensions vxNew = new FinalVoxelDimensions( "micrometer", resolutionS0 );
422+
423+
for ( int c = 0; c < numChannels; ++c )
424+
{
425+
setups.add(
426+
new ViewSetup(
427+
c,
428+
"setup " + c,
429+
new FinalDimensions( bb ),
430+
vxNew,
431+
new Tile( 0 ),
432+
new Channel( c, "Channel " + c ),
433+
new Angle( 0 ),
434+
new Illumination( 0 ) ) );
435+
}
436+
437+
final Map< ViewId, OMEZARREntry > viewIdToPath;
438+
439+
if ( storageType == StorageFormat.ZARR )
440+
{
441+
viewIdToPath = new HashMap<>();
442+
443+
for ( int c = 0; c < numChannels; ++c )
444+
for ( int t = 0; t < numTimepoints; ++t )
445+
{
446+
final OMEZARREntry omeZarrEntry = new OMEZARREntry(
447+
mrInfos[ 0 ][ 0 ].dataset.substring(0, mrInfos[ 0 ][ 0 ].dataset.lastIndexOf( "/" ) ),
448+
new int[] { c, t } );
449+
450+
viewIdToPath.put( new ViewId( t, c ), omeZarrEntry );
451+
}
452+
}
453+
else
454+
{
455+
viewIdToPath = null;
456+
}
457+
458+
final SpimData2 dataFusion =
459+
SpimData2Tools.createNewSpimDataForFusion( storageType, outPathURI, xmlOutURI, viewIdToPath, setups, tps );
460+
461+
new XmlIoSpimData2().save( dataFusion, xmlOutURI );
462+
463+
if ( storageType != StorageFormat.ZARR )
464+
{
465+
final Collection<ViewDescription> vds = dataFusion.getSequenceDescription().getViewDescriptions().values();
466+
467+
mrInfos = new MultiResolutionLevelInfo[ vds.size() ][];
468+
final MultiResolutionLevelInfo myMrInfo[][] = mrInfos;
469+
470+
vds.stream().parallel().forEach( vd ->
471+
{
472+
final int c = vd.getViewSetup().getChannel().getId();
473+
final int t = vd.getTimePointId();
474+
475+
if ( storageType == StorageFormat.N5 )
476+
{
477+
myMrInfo[ c + t*c ] = N5ApiTools.setupBdvDatasetsN5(
478+
driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings);
479+
480+
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/FusionFormat", "BDV/N5" );
481+
}
482+
else // HDF5
483+
{
484+
myMrInfo[ c + t*numChannels ] = N5ApiTools.setupBdvDatasetsHDF5(
485+
driverVolumeWriter, vd, dt, bb, compression, blockSize, downsamplings);
486+
}
487+
});
488+
}
489+
// TODO: set extra attributes to load the state
490+
}
491+
else if ( storageType == StorageFormat.N5 || storageType == StorageFormat.HDF5 ) // simple (no bdv project) HDF5/N5 export
459492
{
460493
mrInfos = new MultiResolutionLevelInfo[ numChannels * numTimepoints ][];
461494

@@ -483,7 +516,6 @@ else if ( storageType == StorageFormat.ZARR ) // OME-Zarr export
483516
}
484517
}
485518

486-
487519
// TODO: set extra attributes to load the state
488520
driverVolumeWriter.setAttribute( "/", "Bigstitcher-Spark/MultiResolutionInfos", mrInfos );
489521

@@ -504,4 +536,4 @@ public static void main(final String... args) throws SpimDataException
504536

505537
System.exit(new CommandLine(new CreateFusionContainer()).execute(args));
506538
}
507-
}
539+
}

‎src/main/java/net/preibisch/bigstitcher/spark/SparkResaveN5.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import net.preibisch.mvrecon.fiji.spimdata.SpimData2;
5454
import net.preibisch.mvrecon.fiji.spimdata.XmlIoSpimData2;
5555
import net.preibisch.mvrecon.fiji.spimdata.imgloaders.AllenOMEZarrLoader;
56+
import net.preibisch.mvrecon.fiji.spimdata.imgloaders.AllenOMEZarrLoader.OMEZARREntry;
5657
import net.preibisch.mvrecon.process.n5api.N5ApiTools;
5758
import net.preibisch.mvrecon.process.n5api.N5ApiTools.MultiResolutionLevelInfo;
5859
import picocli.CommandLine;
@@ -218,7 +219,7 @@ public Void call() throws Exception
218219
viewId,
219220
dataTypes.get( viewId.getViewSetupId() ),
220221
dimensions.get( viewId.getViewSetupId() ),
221-
dataGlobal.getSequenceDescription().getViewDescription( viewId ).getViewSetup().getVoxelSize().dimensionsAsDoubleArray(),
222+
//dataGlobal.getSequenceDescription().getViewDescription( viewId ).getViewSetup().getVoxelSize().dimensionsAsDoubleArray(), // TODO: this is a hack for now
222223
compression,
223224
blockSize,
224225
downsamplings);
@@ -335,12 +336,12 @@ else if ( useN5 )
335336
}
336337
else
337338
{
338-
final Map< ViewId, String > viewIdToPath = new HashMap<>();
339+
final Map< ViewId, OMEZARREntry > viewIdToPath = new HashMap<>();
339340

340341
viewIdToMrInfo.forEach( (viewId, mrInfo ) ->
341342
viewIdToPath.put(
342343
viewId,
343-
mrInfo[ 0 ].dataset.substring(0, mrInfo[ 0 ].dataset.lastIndexOf( "/" ) ) )
344+
new OMEZARREntry( mrInfo[ 0 ].dataset.substring(0, mrInfo[ 0 ].dataset.lastIndexOf( "/" ) ), new int[] { 0, 0 } ) )
344345
);
345346

346347
dataGlobal.getSequenceDescription().setImgLoader(

0 commit comments

Comments
 (0)