Skip to content

Commit dae29aa

Browse files
stefanhahmannclaude
stefanhahmann
and
claude
committed
Add BackendBenchmark for comparing N5 and zarr-java backends
A small main()-driven benchmark that loads the same OME-Zarr dataset through both PyramidBackend implementations and reports load and random-access timings. Useful for evaluating performance trade-offs between the two backends; not run as part of the regular test suite. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 621286d commit dae29aa

1 file changed

Lines changed: 357 additions & 0 deletions

File tree

‎src/test/java/sc/fiji/ome/zarr/pyramid/BackendBenchmark.java‎

Lines changed: 357 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,357 @@
1+
package sc.fiji.ome.zarr.pyramid;
2+
3+
import ch.qos.logback.classic.Level;
4+
import ch.qos.logback.classic.LoggerContext;
5+
import dev.zarr.zarrjava.ZarrException;
6+
import dev.zarr.zarrjava.core.Array;
7+
import dev.zarr.zarrjava.experimental.ome.MultiscaleImage;
8+
import dev.zarr.zarrjava.store.FilesystemStore;
9+
import dev.zarr.zarrjava.store.StoreHandle;
10+
import net.imglib2.Cursor;
11+
import net.imglib2.RandomAccessibleInterval;
12+
import org.scijava.Context;
13+
import org.slf4j.LoggerFactory;
14+
import org.janelia.saalfeldlab.n5.N5Reader;
15+
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
16+
import org.janelia.saalfeldlab.n5.universe.N5DatasetDiscoverer;
17+
import org.janelia.saalfeldlab.n5.universe.N5Factory;
18+
import org.janelia.saalfeldlab.n5.universe.N5TreeNode;
19+
import org.janelia.saalfeldlab.n5.universe.metadata.N5Metadata;
20+
import org.janelia.saalfeldlab.n5.universe.metadata.N5MetadataParser;
21+
import org.janelia.saalfeldlab.n5.universe.metadata.N5SingleScaleMetadata;
22+
import org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v05.OmeNgffV05Metadata;
23+
import sc.fiji.ome.zarr.pyramid.backend.zarrjava.ZarrJavaPyramidBackend;
24+
import sc.fiji.ome.zarr.util.ZarrOnFileSystemUtils;
25+
import sc.fiji.ome.zarr.util.ZarrTestUtils;
26+
27+
import java.io.ByteArrayOutputStream;
28+
import java.io.IOException;
29+
import java.io.PrintStream;
30+
import java.net.URISyntaxException;
31+
import java.nio.file.Path;
32+
import java.nio.file.Paths;
33+
import java.util.Arrays;
34+
import java.util.List;
35+
import java.util.Locale;
36+
import java.util.logging.Handler;
37+
import java.util.logging.LogManager;
38+
import java.util.logging.Logger;
39+
40+
/**
41+
* Simple backend benchmark without extra dependencies.
42+
* Run with:
43+
* mvn -q -DskipTests -Dexec.classpathScope=test -Dexec.mainClass=sc.fiji.ome.zarr.pyramid.BackendBenchmark exec:java
44+
*/
45+
public class BackendBenchmark
46+
{
47+
private static final int WARMUP_ROUNDS = 2;
48+
49+
private static final int MEASURE_ROUNDS = 5;
50+
51+
private static final List< String > DATASETS = Arrays.asList(
52+
"/home/hannes/Documents/projects/scm/data/3.66.9-6.141020_15-41-29.00.ome.zarr/0",
53+
"sc/fiji/ome/zarr/util/2d_testing/ome_zarr_v4_example",
54+
"sc/fiji/ome/zarr/util/2d_testing/ome_zarr_v5_example",
55+
"sc/fiji/ome/zarr/util/5d_testing/5d_dataset_v4.ome.zarr",
56+
"sc/fiji/ome/zarr/util/5d_testing/5d_dataset_v5.ome.zarr"
57+
);
58+
59+
public static void main( final String[] args ) throws Exception
60+
{
61+
disableAllLogs();
62+
System.out.println( "Backend benchmark (times in ms, mean over " + MEASURE_ROUNDS + " rounds)" );
63+
System.out.println( "Warmup rounds: " + WARMUP_ROUNDS );
64+
System.out.println();
65+
System.out.printf( Locale.ROOT, "%-48s %10s %10s %12s %12s %10s %10s %12s %12s%n",
66+
"Dataset", "N5 open", "ZJ open", "PureN5 open", "PureZJ open", "N5 read", "ZJ read", "PureN5 read",
67+
"PureZJ read" );
68+
System.out.println( divider( 146 ) );
69+
70+
for ( final String resource : DATASETS )
71+
{
72+
final Path datasetPath = resourcePath( resource );
73+
final String dataset = datasetPath.toString();
74+
final OpenedReadContexts opened = openReadContexts( dataset );
75+
warmup( dataset );
76+
77+
final double n5Open = measure( () -> benchN5Open( dataset ) );
78+
final double zjOpen = measure( () -> benchZarrJavaOpen( dataset ) );
79+
final double pureN5Open = measure( () -> benchPureN5Open( dataset ) );
80+
final double pureZjOpen = measure( () -> benchPureZarrJavaOpen( dataset ) );
81+
final double n5OpenRead = measure( () -> readWholeImage( opened.n5WrappedLevel0 ) );
82+
final double zjOpenRead = measure( () -> readWholeImage( opened.zjWrappedLevel0 ) );
83+
final double pureN5Read = measure( () -> readWholeImage( opened.pureN5Level0 ) );
84+
final double pureZjRead = measure( () -> readWholePureZarr( opened.pureZjLevel0 ) );
85+
86+
System.out.printf( Locale.ROOT, "%-48s %10.2f %10.2f %12.2f %12.2f %10.2f %10.2f %12.2f %12.2f%n",
87+
shortName( resource ), n5Open, zjOpen, pureN5Open, pureZjOpen, n5OpenRead, zjOpenRead, pureN5Read, pureZjRead );
88+
opened.close();
89+
}
90+
System.exit( 0 );
91+
}
92+
93+
private static void warmup( final String dataset ) throws Exception
94+
{
95+
final OpenedReadContexts opened = openReadContexts( dataset );
96+
for ( int i = 0; i < WARMUP_ROUNDS; i++ )
97+
{
98+
benchN5Open( dataset );
99+
benchZarrJavaOpen( dataset );
100+
benchPureN5Open( dataset );
101+
benchPureZarrJavaOpen( dataset );
102+
readWholeImage( opened.n5WrappedLevel0 );
103+
readWholeImage( opened.zjWrappedLevel0 );
104+
readWholeImage( opened.pureN5Level0 );
105+
readWholePureZarr( opened.pureZjLevel0 );
106+
}
107+
opened.close();
108+
}
109+
110+
private static double measure( final ThrowingRunnable benchmark ) throws Exception
111+
{
112+
long totalNanos = 0L;
113+
for ( int i = 0; i < MEASURE_ROUNDS; i++ )
114+
{
115+
final long t0 = System.nanoTime();
116+
benchmark.run();
117+
totalNanos += ( System.nanoTime() - t0 );
118+
}
119+
return ( totalNanos / ( double ) MEASURE_ROUNDS ) / 1_000_000.0;
120+
}
121+
122+
private static void benchN5Open( final String dataset )
123+
{
124+
try (Context context = new Context())
125+
{
126+
new Pyramidal5DImageDataImpl<>( context, dataset );
127+
}
128+
}
129+
130+
@SuppressWarnings( { "rawtypes", "unchecked" } )
131+
private static void benchZarrJavaOpen( final String dataset )
132+
{
133+
try (Context context = new Context())
134+
{
135+
new Pyramidal5DImageDataImpl( context, new ZarrJavaPyramidBackend( dataset ) );
136+
}
137+
}
138+
139+
private static void benchPureZarrJavaOpen( final String dataset ) throws IOException, ZarrException
140+
{
141+
final MultiscaleImage image = openMultiscaleImage( Paths.get( dataset ) );
142+
image.getMultiscaleNode( 0 );
143+
}
144+
145+
private static void benchPureN5Open( final String dataset ) throws IOException
146+
{
147+
final N5OpenContext ctx = openN5Context( Paths.get( dataset ) );
148+
resolveN5Level0Path( ctx );
149+
}
150+
151+
private static MultiscaleImage openMultiscaleImage( final Path inputPath ) throws IOException, ZarrException
152+
{
153+
final Path rootPath = ZarrOnFileSystemUtils.findRootFolder( inputPath );
154+
final Path zarrRoot = rootPath != null ? rootPath : inputPath;
155+
final FilesystemStore store = new FilesystemStore( zarrRoot );
156+
StoreHandle handle = store.resolve();
157+
if ( rootPath != null && !rootPath.equals( inputPath ) )
158+
{
159+
for ( final String segment : ZarrOnFileSystemUtils.relativePathElements( rootPath, inputPath ) )
160+
handle = handle.resolve( segment );
161+
}
162+
return MultiscaleImage.open( handle );
163+
}
164+
165+
private static N5OpenContext openN5Context( final Path inputPath ) throws IOException
166+
{
167+
final Path rootPath = ZarrOnFileSystemUtils.findRootFolder( inputPath );
168+
if ( rootPath == null )
169+
throw new IOException( "No zarr root for " + inputPath );
170+
final String relativePath = String.join( "/", ZarrOnFileSystemUtils.relativePathElements( rootPath, inputPath ) );
171+
final N5Reader reader = new N5Factory().openReader( rootPath.toUri().toString() );
172+
final N5TreeNode node = new N5TreeNode( relativePath );
173+
final List< N5MetadataParser< ? > > parsers = Arrays.asList(
174+
new org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v03.OmeNgffMetadataParser(),
175+
new org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadataParser(),
176+
new org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v05.OmeNgffV05MetadataParser() );
177+
N5DatasetDiscoverer.parseMetadataShallow( reader, node, parsers, parsers );
178+
final N5Metadata metadata = node.getMetadata();
179+
if ( metadata == null )
180+
throw new IOException( "No NGFF metadata for " + inputPath );
181+
return new N5OpenContext( reader, metadata );
182+
}
183+
184+
private static String resolveN5Level0Path( final N5OpenContext ctx ) throws IOException
185+
{
186+
final N5Metadata metadata = ctx.metadata;
187+
if ( metadata instanceof OmeNgffV05Metadata )
188+
{
189+
final OmeNgffV05Metadata v05 = ( OmeNgffV05Metadata ) metadata;
190+
return v05.multiscales[ 0 ].getChildrenMetadata()[ 0 ].getPath();
191+
}
192+
if ( metadata instanceof org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata )
193+
{
194+
final org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata v04 =
195+
( org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v04.OmeNgffMetadata ) metadata;
196+
return v04.multiscales[ 0 ].getChildrenMetadata()[ 0 ].getPath();
197+
}
198+
if ( metadata instanceof org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v03.OmeNgffMetadata )
199+
{
200+
final org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v03.OmeNgffMetadata v03 =
201+
( org.janelia.saalfeldlab.n5.universe.metadata.ome.ngff.v03.OmeNgffMetadata ) metadata;
202+
final N5SingleScaleMetadata[] children = v03.getMultiscales()[ 0 ].getChildrenMetadata();
203+
return children[ 0 ].getPath();
204+
}
205+
throw new IOException( "Unsupported metadata class: " + metadata.getClass().getName() );
206+
}
207+
208+
private static OpenedReadContexts openReadContexts( final String dataset ) throws Exception
209+
{
210+
final Context n5Context = new Context();
211+
final Pyramidal5DImageDataImpl< ?, ? > n5Wrapped = new Pyramidal5DImageDataImpl<>( n5Context, dataset );
212+
final RandomAccessibleInterval< ? > n5WrappedLevel0 = n5Wrapped.asSources().get( 0 ).getSpimSource().getSource( 0, 0 );
213+
214+
final Context zjContext = new Context();
215+
@SuppressWarnings( { "rawtypes", "unchecked" } )
216+
final Pyramidal5DImageDataImpl< ?, ? > zjWrapped = new Pyramidal5DImageDataImpl( zjContext, new ZarrJavaPyramidBackend( dataset ) );
217+
final RandomAccessibleInterval< ? > zjWrappedLevel0 = zjWrapped.asSources().get( 0 ).getSpimSource().getSource( 0, 0 );
218+
219+
final N5OpenContext n5Pure = openN5Context( Paths.get( dataset ) );
220+
final String level0Path = resolveN5Level0Path( n5Pure );
221+
final RandomAccessibleInterval< ? > pureN5Level0 = N5Utils.open( n5Pure.reader, level0Path );
222+
final MultiscaleImage pureZjImage = openMultiscaleImage( Paths.get( dataset ) );
223+
final Array pureZjLevel0 = pureZjImage.openScaleLevel( 0 );
224+
225+
return new OpenedReadContexts(
226+
n5WrappedLevel0,
227+
zjWrappedLevel0,
228+
pureN5Level0,
229+
pureZjLevel0,
230+
n5Context,
231+
zjContext,
232+
n5Pure.reader );
233+
}
234+
235+
private static void readWholeImage( final RandomAccessibleInterval< ? > img )
236+
{
237+
final Cursor< ? > cursor = net.imglib2.view.Views.flatIterable( img ).cursor();
238+
while ( cursor.hasNext() )
239+
cursor.next();
240+
}
241+
242+
private static void readWholePureZarr( final Array array ) throws ZarrException
243+
{
244+
final long[] origin = new long[ array.metadata().shape.length ];
245+
final long[] shape = array.metadata().shape.clone();
246+
array.read( origin, shape );
247+
}
248+
249+
private static String shortName( final String resource )
250+
{
251+
final int idx = resource.lastIndexOf( '/' );
252+
return idx >= 0 ? resource.substring( idx + 1 ) : resource;
253+
}
254+
255+
private static String divider( final int n )
256+
{
257+
final StringBuilder sb = new StringBuilder( n );
258+
for ( int i = 0; i < n; i++ )
259+
sb.append( '-' );
260+
return sb.toString();
261+
}
262+
263+
private static Path resourcePath( final String resource ) throws URISyntaxException
264+
{
265+
final Path path = Paths.get( resource );
266+
if ( path.isAbsolute() )
267+
return path;
268+
return ZarrTestUtils.resourcePath( resource );
269+
}
270+
271+
private static void disableAllLogs()
272+
{
273+
// Mute System.err to suppress noisy plugin/framework startup logs.
274+
System.setErr( new PrintStream( new ByteArrayOutputStream() ) );
275+
276+
// Force Logback root logger OFF.
277+
final LoggerContext context = ( LoggerContext ) LoggerFactory.getILoggerFactory();
278+
context.getLogger( org.slf4j.Logger.ROOT_LOGGER_NAME ).setLevel( Level.OFF );
279+
280+
final LogManager logManager = LogManager.getLogManager();
281+
final Logger rootLogger = logManager.getLogger( "" );
282+
if ( rootLogger != null )
283+
{
284+
rootLogger.setLevel( java.util.logging.Level.OFF );
285+
for ( final Handler handler : rootLogger.getHandlers() )
286+
handler.setLevel( java.util.logging.Level.OFF );
287+
}
288+
}
289+
290+
@FunctionalInterface
291+
private interface ThrowingRunnable
292+
{
293+
void run() throws Exception;
294+
}
295+
296+
private static final class N5OpenContext
297+
{
298+
private final N5Reader reader;
299+
300+
private final N5Metadata metadata;
301+
302+
private N5OpenContext( final N5Reader reader, final N5Metadata metadata )
303+
{
304+
this.reader = reader;
305+
this.metadata = metadata;
306+
}
307+
}
308+
309+
private static final class OpenedReadContexts implements AutoCloseable
310+
{
311+
private final RandomAccessibleInterval< ? > n5WrappedLevel0;
312+
313+
private final RandomAccessibleInterval< ? > zjWrappedLevel0;
314+
315+
private final RandomAccessibleInterval< ? > pureN5Level0;
316+
317+
private final Array pureZjLevel0;
318+
319+
private final Context n5Context;
320+
321+
private final Context zjContext;
322+
323+
private final N5Reader pureN5Reader;
324+
325+
private OpenedReadContexts(
326+
final RandomAccessibleInterval< ? > n5WrappedLevel0,
327+
final RandomAccessibleInterval< ? > zjWrappedLevel0,
328+
final RandomAccessibleInterval< ? > pureN5Level0,
329+
final Array pureZjLevel0,
330+
final Context n5Context,
331+
final Context zjContext,
332+
final N5Reader pureN5Reader )
333+
{
334+
this.n5WrappedLevel0 = n5WrappedLevel0;
335+
this.zjWrappedLevel0 = zjWrappedLevel0;
336+
this.pureN5Level0 = pureN5Level0;
337+
this.pureZjLevel0 = pureZjLevel0;
338+
this.n5Context = n5Context;
339+
this.zjContext = zjContext;
340+
this.pureN5Reader = pureN5Reader;
341+
}
342+
343+
@Override
344+
public void close()
345+
{
346+
pureN5Reader.close();
347+
try
348+
{
349+
n5Context.dispose();
350+
}
351+
finally
352+
{
353+
zjContext.dispose();
354+
}
355+
}
356+
}
357+
}

0 commit comments

Comments
 (0)