From e148be792bccbd72245774770f03170247fcd323 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Tue, 12 May 2026 09:27:44 +0200 Subject: [PATCH 1/9] Introduced `BufferedChannelReader`, `BufferedChannelWriter`, and `GrowingByteBuffer` for efficient binary I/O, refactored `BinWriter` to adopt buffer-based writing, and updated tests to validate new implementations. Signed-off-by: Leclerc Clement --- .../com/powsybl/commons/binary/BinReader.java | 110 +++++---- .../com/powsybl/commons/binary/BinWriter.java | 175 +++++++------- .../commons/binary/BufferedChannelReader.java | 169 ++++++++++++++ .../commons/binary/BufferedChannelWriter.java | 109 +++++++++ .../commons/binary/GrowingByteBuffer.java | 80 +++++++ .../commons/binary/BinWriterReaderTest.java | 32 +++ .../BufferedChannelReaderWriterTest.java | 219 ++++++++++++++++++ .../com/powsybl/iidm/serde/NetworkSerDe.java | 43 ++-- 8 files changed, 787 insertions(+), 150 deletions(-) create mode 100644 commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java create mode 100644 commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java create mode 100644 commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java create mode 100644 commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java index 5374ae3bef2..c6893f9e32b 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java @@ -11,8 +11,15 @@ import com.powsybl.commons.io.AbstractTreeDataReader; import com.powsybl.commons.io.TreeDataHeader; -import java.io.*; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import static com.powsybl.commons.binary.BinUtil.*; @@ -22,7 +29,7 @@ */ public class BinReader extends AbstractTreeDataReader { - private final DataInputStream dis; + private final BufferedChannelReader in; private final byte[] binaryMagicNumber; private String[] names; @@ -31,9 +38,28 @@ public class BinReader extends AbstractTreeDataReader { private int nextNameIdx = END_NODE; private byte nextType; - public BinReader(InputStream is, byte[] binaryMagicNumber) { + /** Preferred constructor: direct channel access avoids the InputStream re-buffering layer. */ + public BinReader(ReadableByteChannel channel, byte[] binaryMagicNumber) { this.binaryMagicNumber = binaryMagicNumber; - this.dis = new DataInputStream(new BufferedInputStream(Objects.requireNonNull(is))); + this.in = new BufferedChannelReader(Objects.requireNonNull(channel)); + } + + /** Opens a file directly as a byte channel — the fastest path for file inputs. */ + public BinReader(Path path, byte[] binaryMagicNumber) throws IOException { + this(Files.newByteChannel(Objects.requireNonNull(path), StandardOpenOption.READ), binaryMagicNumber); + } + + /** + * Compatibility constructor for callers holding an {@link InputStream}. + * Note: wrapping an {@code InputStream} via {@link Channels#newChannel} gives no perf gain + * over the previous {@code DataInputStream} chain — prefer the {@link ReadableByteChannel} + * or {@link Path} constructors for real I/O speedup. + * + * @deprecated use {@link #BinReader(ReadableByteChannel, byte[])} or {@link #BinReader(Path, byte[])} + */ + @Deprecated(since = "6.10.0") + public BinReader(InputStream is, byte[] binaryMagicNumber) { + this(Channels.newChannel(Objects.requireNonNull(is)), binaryMagicNumber); } @Override @@ -59,7 +85,7 @@ protected String readRootVersion() { } private void readMagicNumber() throws IOException { - byte[] read = dis.readNBytes(binaryMagicNumber.length); + byte[] read = in.readNBytes(binaryMagicNumber.length); if (!Arrays.equals(read, binaryMagicNumber)) { throw new PowsyblException("Unexpected bytes at file start"); } @@ -68,7 +94,7 @@ private void readMagicNumber() throws IOException { @Override protected Map readExtensionVersions() { try { - int nbVersions = dis.readUnsignedShort(); + int nbVersions = in.readUnsignedShort(); Map versions = new HashMap<>(); for (int i = 0; i < nbVersions; i++) { versions.put(readString(), readString()); @@ -80,23 +106,24 @@ protected Map readExtensionVersions() { } private void readNamesDictionary() throws IOException { - int nbEntries = dis.readUnsignedShort(); + int nbEntries = in.readUnsignedShort(); names = new String[nbEntries + 1]; types = new byte[nbEntries + 1]; for (int i = 0; i < nbEntries; i++) { names[i + 1] = readString(); - types[i + 1] = dis.readByte(); + types[i + 1] = in.readByte(); } } private void peekNextEntry() throws IOException { - try { - nextNameIdx = dis.readUnsignedShort(); - if (nextNameIdx != END_NODE) { - nextType = types[nextNameIdx]; - } - } catch (EOFException e) { + int idx = in.tryReadUnsignedShort(); + if (idx == BufferedChannelReader.EOF) { nextNameIdx = END_NODE; + return; + } + nextNameIdx = idx; + if (nextNameIdx != END_NODE) { + nextType = types[nextNameIdx]; } } @@ -120,11 +147,11 @@ private void skipRemainingAttributes() throws IOException { private void skipTypedValue(byte typeTag) throws IOException { switch (typeTag) { - case TYPE_DOUBLE -> dis.skipNBytes(8); - case TYPE_FLOAT, TYPE_INT -> dis.skipNBytes(4); - case TYPE_BOOLEAN -> dis.skipNBytes(1); + case TYPE_DOUBLE -> in.skipNBytes(8); + case TYPE_FLOAT, TYPE_INT -> in.skipNBytes(4); + case TYPE_BOOLEAN -> in.skipNBytes(1); case TYPE_STRING, TYPE_STRING_CONTENT -> skipString(); - case TYPE_ENUM -> dis.skipNBytes(2); + case TYPE_ENUM -> in.skipNBytes(2); case TYPE_INT_ARRAY -> skipIntArray(); case TYPE_STRING_ARRAY -> skipStringArray(); default -> throw new PowsyblException("Binary format: unknown type tag " + typeTag); @@ -132,37 +159,37 @@ private void skipTypedValue(byte typeTag) throws IOException { } private void skipString() throws IOException { - int len = dis.readUnsignedShort(); + int len = in.readUnsignedShort(); if (len != NULL_STRING_SENTINEL) { - dis.skipNBytes(len); + in.skipNBytes(len); } } private void skipIntArray() throws IOException { - int count = dis.readUnsignedShort(); + int count = in.readUnsignedShort(); if (count > 0) { - dis.skipNBytes(4L * count); + in.skipNBytes(4L * count); } } private void skipStringArray() throws IOException { - int count = dis.readUnsignedShort(); + int count = in.readUnsignedShort(); for (int i = 0; i < count; i++) { skipString(); } } private List readIntArrayRaw() throws IOException { - int count = dis.readUnsignedShort(); + int count = in.readUnsignedShort(); List list = new ArrayList<>(count); for (int i = 0; i < count; i++) { - list.add(dis.readInt()); + list.add(in.readInt()); } return list; } private List readStringArrayRaw() throws IOException { - int count = dis.readUnsignedShort(); + int count = in.readUnsignedShort(); List list = new ArrayList<>(count); for (int i = 0; i < count; i++) { list.add(readString()); @@ -172,14 +199,11 @@ private List readStringArrayRaw() throws IOException { private String readString() { try { - int stringNbBytes = dis.readUnsignedShort(); + int stringNbBytes = in.readUnsignedShort(); if (stringNbBytes == NULL_STRING_SENTINEL) { return null; } - byte[] stringBytes = dis.readNBytes(stringNbBytes); - if (stringBytes.length != stringNbBytes) { - throw new PowsyblException("Cannot read the full string, bytes missing: " + (stringNbBytes - stringBytes.length)); - } + byte[] stringBytes = in.readNBytes(stringNbBytes); return new String(stringBytes, StandardCharsets.UTF_8); } catch (IOException e) { throw new UncheckedIOException(e); @@ -192,7 +216,7 @@ public double readDoubleAttribute(String name, double defaultValue) { if (isAttrAbsent(name)) { return defaultValue; } - double val = dis.readDouble(); + double val = in.readDouble(); peekNextEntry(); return val; } catch (IOException e) { @@ -206,7 +230,7 @@ public OptionalDouble readOptionalDoubleAttribute(String name) { if (isAttrAbsent(name)) { return OptionalDouble.empty(); } - OptionalDouble val = OptionalDouble.of(dis.readDouble()); + OptionalDouble val = OptionalDouble.of(in.readDouble()); peekNextEntry(); return val; } catch (IOException e) { @@ -220,7 +244,7 @@ public float readFloatAttribute(String name, float defaultValue) { if (isAttrAbsent(name)) { return defaultValue; } - float val = dis.readFloat(); + float val = in.readFloat(); peekNextEntry(); return val; } catch (IOException e) { @@ -248,7 +272,7 @@ public int readIntAttribute(String name) { throw new PowsyblException("Missing required int attribute: " + name); } try { - int val = dis.readInt(); + int val = in.readInt(); peekNextEntry(); return val; } catch (IOException e) { @@ -262,7 +286,7 @@ public int readIntAttribute(String name, int defaultValue) { if (isAttrAbsent(name)) { return defaultValue; } - int val = dis.readInt(); + int val = in.readInt(); peekNextEntry(); return val; } catch (IOException e) { @@ -276,7 +300,7 @@ public OptionalInt readOptionalIntAttribute(String name) { if (isAttrAbsent(name)) { return OptionalInt.empty(); } - OptionalInt val = OptionalInt.of(dis.readInt()); + OptionalInt val = OptionalInt.of(in.readInt()); peekNextEntry(); return val; } catch (IOException e) { @@ -290,7 +314,7 @@ public boolean readBooleanAttribute(String name) { throw new PowsyblException("Missing required boolean attribute: " + name); } try { - boolean val = dis.readBoolean(); + boolean val = in.readBoolean(); peekNextEntry(); return val; } catch (IOException e) { @@ -304,7 +328,7 @@ public boolean readBooleanAttribute(String name, boolean defaultValue) { if (isAttrAbsent(name)) { return defaultValue; } - boolean val = dis.readBoolean(); + boolean val = in.readBoolean(); peekNextEntry(); return val; } catch (IOException e) { @@ -318,7 +342,7 @@ public Optional readOptionalBooleanAttribute(String name) { if (isAttrAbsent(name)) { return Optional.empty(); } - Optional val = Optional.of(dis.readBoolean()); + Optional val = Optional.of(in.readBoolean()); peekNextEntry(); return val; } catch (IOException e) { @@ -332,7 +356,7 @@ public > T readEnumAttribute(String name, Class clazz, T de if (isAttrAbsent(name)) { return defaultValue; } - int ordinal = dis.readUnsignedShort(); + int ordinal = in.readUnsignedShort(); peekNextEntry(); T[] constants = clazz.getEnumConstants(); return ordinal < constants.length ? constants[ordinal] : defaultValue; @@ -423,7 +447,7 @@ public void readEndNode() { boolean readEndOfStream() { try { - return dis.read() == -1; + return in.isEndOfStream(); } catch (IOException e) { throw new UncheckedIOException(e); } @@ -432,7 +456,7 @@ boolean readEndOfStream() { @Override public void close() { try { - dis.close(); + in.close(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java index 8be88cee67a..a19ef6f8dd0 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java @@ -10,8 +10,15 @@ import com.powsybl.commons.PowsyblException; import com.powsybl.commons.io.AbstractTreeDataWriter; -import java.io.*; +import java.io.IOException; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.nio.channels.Channels; +import java.nio.channels.WritableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import static com.powsybl.commons.binary.BinUtil.*; @@ -22,9 +29,8 @@ public class BinWriter extends AbstractTreeDataWriter { private final String rootVersion; - private final DataOutputStream dos; - private final DataOutputStream tmpDos; - private final ByteArrayOutputStream buffer; + private final BufferedChannelWriter out; + private final GrowingByteBuffer body; private final byte[] binaryMagicNumber; private Map extensionVersions = Collections.emptyMap(); @@ -32,36 +38,57 @@ public class BinWriter extends AbstractTreeDataWriter { private record TypedName(String name, byte type) { } - public BinWriter(OutputStream outputStream, byte[] binaryMagicNumber, String rootVersion) { + /** Preferred constructor: direct channel access avoids the OutputStream re-buffering layer. */ + public BinWriter(WritableByteChannel channel, byte[] binaryMagicNumber, String rootVersion) { this.binaryMagicNumber = Objects.requireNonNull(binaryMagicNumber); this.rootVersion = Objects.requireNonNull(rootVersion); - this.dos = new DataOutputStream(new BufferedOutputStream(Objects.requireNonNull(outputStream))); - this.buffer = new ByteArrayOutputStream(); - this.tmpDos = new DataOutputStream(buffer); + this.out = new BufferedChannelWriter(Objects.requireNonNull(channel)); + this.body = new GrowingByteBuffer(); } - private static void writeIndex(int index, DataOutputStream dataOutputStream) { - try { - dataOutputStream.writeShort(index); - } catch (IOException e) { - throw new UncheckedIOException(e); + /** Opens a file directly as a byte channel — the fastest path for file outputs. */ + public BinWriter(Path path, byte[] binaryMagicNumber, String rootVersion) throws IOException { + this(Files.newByteChannel(Objects.requireNonNull(path), + StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING), + binaryMagicNumber, rootVersion); + } + + /** + * Compatibility constructor for callers holding an {@link OutputStream}. + * Wrapping an {@code OutputStream} via {@link Channels#newChannel} gives no perf gain over + * the previous {@code DataOutputStream} chain — prefer the {@link WritableByteChannel} or + * {@link Path} constructors for real I/O speedup. + * + * @deprecated use {@link #BinWriter(WritableByteChannel, byte[], String)} or {@link #BinWriter(Path, byte[], String)} + */ + @Deprecated(since = "6.10.0") + public BinWriter(OutputStream outputStream, byte[] binaryMagicNumber, String rootVersion) { + this(Channels.newChannel(Objects.requireNonNull(outputStream)), binaryMagicNumber, rootVersion); + } + + private static void writeStringToBody(String value, GrowingByteBuffer buf) { + if (value == null) { + buf.writeShort(NULL_STRING_SENTINEL); + } else { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length >= NULL_STRING_SENTINEL) { + throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); + } + buf.writeShort(bytes.length); + buf.writeBytes(bytes); } } - private static void writeString(String value, DataOutputStream dataOutputStream) { - try { - if (value == null) { - writeIndex(NULL_STRING_SENTINEL, dataOutputStream); - } else { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - if (bytes.length >= NULL_STRING_SENTINEL) { - throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); - } - writeIndex(bytes.length, dataOutputStream); - dataOutputStream.write(bytes); + private static void writeStringToHeader(String value, BufferedChannelWriter w) throws IOException { + if (value == null) { + w.writeShort(NULL_STRING_SENTINEL); + } else { + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length >= NULL_STRING_SENTINEL) { + throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); } - } catch (IOException e) { - throw new UncheckedIOException(e); + w.writeShort(bytes.length); + w.writeBytes(bytes); } } @@ -86,7 +113,7 @@ public void writeStartNode(String namespace, String name) { @Override public void writeEndNode() { - writeIndex(END_NODE, tmpDos); + body.writeShort(END_NODE); } private void writeEntry(String name, byte type) { @@ -100,7 +127,7 @@ private void writeEntry(String name, byte type) { namesIndex.put(key, newIndex); index = newIndex; } - writeIndex(index, tmpDos); + body.writeShort(index); } @Override @@ -111,13 +138,13 @@ public void writeNamespace(String prefix, String namespace) { @Override public void writeNodeContent(String value) { writeEntry("", TYPE_STRING_CONTENT); - writeString(value, tmpDos); + writeStringToBody(value, body); } @Override public void writeStringAttribute(String name, String value) { writeEntry(name, TYPE_STRING); - writeString(value, tmpDos); + writeStringToBody(value, body); } @Override @@ -132,11 +159,7 @@ public void writeDoubleAttribute(String name, double value, double absentValue) return; } writeEntry(name, TYPE_DOUBLE); - try { - tmpDos.writeDouble(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeDouble(value); } @Override @@ -145,21 +168,13 @@ public void writeFloatAttribute(String name, float value) { return; } writeEntry(name, TYPE_FLOAT); - try { - tmpDos.writeFloat(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeFloat(value); } @Override public void writeIntAttribute(String name, int value) { writeEntry(name, TYPE_INT); - try { - tmpDos.writeInt(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeInt(value); } @Override @@ -173,11 +188,7 @@ public void writeIntAttribute(String name, int value, int absentValue) { @Override public void writeBooleanAttribute(String name, boolean value) { writeEntry(name, TYPE_BOOLEAN); - try { - tmpDos.writeBoolean(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeBoolean(value); } @Override @@ -194,74 +205,58 @@ public > void writeEnumAttribute(String name, E value) { return; } writeEntry(name, TYPE_ENUM); - try { - tmpDos.writeShort(value.ordinal()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeShort(value.ordinal()); } @Override public void writeIntArrayAttribute(String name, Collection values) { writeEntry(name, TYPE_INT_ARRAY); - try { - tmpDos.writeShort(values.size()); - for (int v : values) { - tmpDos.writeInt(v); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + body.writeShort(values.size()); + for (int v : values) { + body.writeInt(v); } } @Override public void writeStringArrayAttribute(String name, Collection values) { writeEntry(name, TYPE_STRING_ARRAY); - try { - tmpDos.writeShort(values.size()); - for (String s : values) { - writeString(s, tmpDos); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + body.writeShort(values.size()); + for (String s : values) { + writeStringToBody(s, body); } } @Override public void close() { try { - tmpDos.flush(); writeHeader(); - dos.write(buffer.toByteArray()); - dos.close(); + out.writeFully(body.toReadBuffer()); + out.close(); } catch (IOException e) { throw new UncheckedIOException(e); } } private void writeHeader() throws IOException { - // magic number ("Binary IIDM" in ASCII) - dos.write(binaryMagicNumber); + // magic number + out.writeBytes(binaryMagicNumber); // iidm version - writeString(rootVersion, dos); + writeStringToHeader(rootVersion, out); // extensions versions - writeIndex(extensionVersions.size(), dos); - extensionVersions.forEach((extensionName, extensionVersion) -> { - writeString(extensionName, dos); - writeString(extensionVersion, dos); - }); - - writeIndex(namesIndex.size(), dos); - namesIndex.keySet().forEach(nameTypeKey -> { - writeString(nameTypeKey.name(), dos); - try { - dos.writeByte(nameTypeKey.type()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); + out.writeShort(extensionVersions.size()); + for (var entry : extensionVersions.entrySet()) { + writeStringToHeader(entry.getKey(), out); + writeStringToHeader(entry.getValue(), out); + } + + // names dictionary + out.writeShort(namesIndex.size()); + for (TypedName key : namesIndex.keySet()) { + writeStringToHeader(key.name(), out); + out.writeByte(key.type()); + } } @Override diff --git a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java new file mode 100644 index 00000000000..e74453aff6d --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java @@ -0,0 +1,169 @@ +/** + * Copyright (c) 2025, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import com.powsybl.commons.PowsyblException; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.util.Objects; + +/** + * Buffered reader on top of a {@link ReadableByteChannel}, backed by a direct {@link ByteBuffer}. + * Replaces the {@code DataInputStream + BufferedInputStream} chain to avoid double indirection + * and to leverage JVM intrinsics on aligned multi-byte reads (big-endian, network order). + * + * @author Clement Leclerc {@literal } + */ +final class BufferedChannelReader implements AutoCloseable { + + static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + + /** Sentinel returned by {@link #tryReadUnsignedShort()} on end-of-stream. */ + static final int EOF = -1; + + private final ReadableByteChannel channel; + private final ByteBuffer buffer; + private boolean channelExhausted; + + BufferedChannelReader(ReadableByteChannel channel) { + this(channel, DEFAULT_BUFFER_SIZE); + } + + BufferedChannelReader(ReadableByteChannel channel, int bufferSize) { + this.channel = Objects.requireNonNull(channel); + this.buffer = ByteBuffer.allocateDirect(bufferSize); + this.buffer.flip(); // start empty in read mode + } + + /** Ensures at least {@code n} bytes are available in the buffer, refilling from the channel if needed. */ + private void ensureAvailable(int n) throws IOException { + if (buffer.remaining() >= n) { + return; + } + buffer.compact(); + while (buffer.position() < n) { + int read = channel.read(buffer); + if (read == -1) { + channelExhausted = true; + break; + } + } + buffer.flip(); + if (buffer.remaining() < n) { + throw new PowsyblException("Unexpected end of stream: needed " + n + " bytes, got " + buffer.remaining()); + } + } + + byte readByte() throws IOException { + ensureAvailable(1); + return buffer.get(); + } + + int readUnsignedShort() throws IOException { + ensureAvailable(2); + return Short.toUnsignedInt(buffer.getShort()); + } + + int readInt() throws IOException { + ensureAvailable(4); + return buffer.getInt(); + } + + float readFloat() throws IOException { + ensureAvailable(4); + return buffer.getFloat(); + } + + double readDouble() throws IOException { + ensureAvailable(8); + return buffer.getDouble(); + } + + boolean readBoolean() throws IOException { + return readByte() != 0; + } + + /** Reads exactly {@code n} bytes; throws if the stream ends early. */ + byte[] readNBytes(int n) throws IOException { + byte[] out = new byte[n]; + int filled = 0; + while (filled < n) { + if (!buffer.hasRemaining()) { + ensureAvailable(1); + } + int take = Math.min(buffer.remaining(), n - filled); + buffer.get(out, filled, take); + filled += take; + } + return out; + } + + void skipNBytes(long n) throws IOException { + long remaining = n; + while (remaining > 0) { + if (!buffer.hasRemaining()) { + ensureAvailable(1); + } + int skip = (int) Math.min(buffer.remaining(), remaining); + buffer.position(buffer.position() + skip); + remaining -= skip; + } + } + + /** + * Attempts to read an unsigned short. Returns {@link #EOF} (-1) if the stream is exhausted + * before any byte of the short can be read, instead of throwing. + * Useful for end-of-stream detection in peek-style parsers. + */ + int tryReadUnsignedShort() throws IOException { + if (buffer.remaining() >= 2) { + return Short.toUnsignedInt(buffer.getShort()); + } + buffer.compact(); + while (buffer.position() < 2) { + int read = channel.read(buffer); + if (read == -1) { + channelExhausted = true; + break; + } + } + buffer.flip(); + if (buffer.remaining() < 2) { + if (buffer.remaining() == 0) { + return EOF; + } + throw new PowsyblException("Unexpected end of stream: needed 2 bytes, got " + buffer.remaining()); + } + return Short.toUnsignedInt(buffer.getShort()); + } + + /** Returns true if no more bytes are available in the buffer or the underlying channel. */ + boolean isEndOfStream() throws IOException { + if (buffer.hasRemaining()) { + return false; + } + if (channelExhausted) { + return true; + } + buffer.compact(); + int read = channel.read(buffer); + buffer.flip(); + if (read == -1) { + channelExhausted = true; + return !buffer.hasRemaining(); + } + return false; + } + + @Override + public void close() throws IOException { + channel.close(); + } +} diff --git a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java new file mode 100644 index 00000000000..baa0f122a2f --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java @@ -0,0 +1,109 @@ +/** + * Copyright (c) 2025, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.Objects; + +/** + * Buffered writer on top of a {@link WritableByteChannel}, backed by a direct {@link ByteBuffer}. + * Replaces the {@code DataOutputStream + BufferedOutputStream} chain. + * Payloads larger than the buffer capacity are written directly to the channel after a flush, + * avoiding pointless intermediate copies. + * + * @author Clement Leclerc {@literal } + */ +final class BufferedChannelWriter implements AutoCloseable { + + static final int DEFAULT_BUFFER_SIZE = 256 * 1024; + + private final WritableByteChannel channel; + private final ByteBuffer buffer; + + BufferedChannelWriter(WritableByteChannel channel) { + this(channel, DEFAULT_BUFFER_SIZE); + } + + BufferedChannelWriter(WritableByteChannel channel, int bufferSize) { + this.channel = Objects.requireNonNull(channel); + this.buffer = ByteBuffer.allocateDirect(bufferSize); + } + + private void ensureSpace(int n) throws IOException { + if (buffer.remaining() < n) { + flush(); + } + } + + void writeByte(int b) throws IOException { + ensureSpace(1); + buffer.put((byte) b); + } + + void writeShort(int s) throws IOException { + ensureSpace(2); + buffer.putShort((short) s); + } + + void writeInt(int i) throws IOException { + ensureSpace(4); + buffer.putInt(i); + } + + void writeFloat(float f) throws IOException { + ensureSpace(4); + buffer.putFloat(f); + } + + void writeDouble(double d) throws IOException { + ensureSpace(8); + buffer.putDouble(d); + } + + void writeBoolean(boolean b) throws IOException { + writeByte(b ? 1 : 0); + } + + void writeBytes(byte[] bytes) throws IOException { + if (bytes.length > buffer.capacity()) { + // payload exceeds buffer capacity: flush then write directly + flush(); + ByteBuffer wrap = ByteBuffer.wrap(bytes); + while (wrap.hasRemaining()) { + channel.write(wrap); + } + return; + } + ensureSpace(bytes.length); + buffer.put(bytes); + } + + /** Drains a fully-prepared (already flipped) ByteBuffer to the channel after flushing internal buffer. */ + void writeFully(ByteBuffer src) throws IOException { + flush(); + while (src.hasRemaining()) { + channel.write(src); + } + } + + void flush() throws IOException { + buffer.flip(); + while (buffer.hasRemaining()) { + channel.write(buffer); + } + buffer.clear(); + } + + @Override + public void close() throws IOException { + flush(); + channel.close(); + } +} diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java new file mode 100644 index 00000000000..6a208ec4d83 --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -0,0 +1,80 @@ +/** + * Copyright (c) 2024, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import java.nio.ByteBuffer; + +final class GrowingByteBuffer { + + private static final int DEFAULT_INITIAL_CAPACITY = 16 * 1024; + + private ByteBuffer buffer; + + GrowingByteBuffer() { + this(DEFAULT_INITIAL_CAPACITY); + } + + GrowingByteBuffer(int initialCapacity) { + this.buffer = ByteBuffer.allocate(initialCapacity); + } + + private void ensureSpace(int n) { + if (buffer.remaining() < n) { + int needed = buffer.position() + n; + int newCapacity = buffer.capacity() * 2; + while (newCapacity < needed) { + newCapacity *= 2; + } + ByteBuffer next = ByteBuffer.allocate(newCapacity); + buffer.flip(); + next.put(buffer); + buffer = next; + } + } + + void writeByte(int b) { + ensureSpace(1); + buffer.put((byte) b); + } + + void writeShort(int s) { + ensureSpace(2); + buffer.putShort((short) s); + } + + void writeInt(int i) { + ensureSpace(4); + buffer.putInt(i); + } + + void writeFloat(float f) { + ensureSpace(4); + buffer.putFloat(f); + } + + void writeDouble(double d) { + ensureSpace(8); + buffer.putDouble(d); + } + + void writeBoolean(boolean b) { + writeByte(b ? 1 : 0); + } + + void writeBytes(byte[] bytes) { + ensureSpace(bytes.length); + buffer.put(bytes); + } + + /** Returns a read-only view positioned at 0 with limit at current size, ready for channel.write(). */ + ByteBuffer toReadBuffer() { + ByteBuffer view = buffer.duplicate(); + view.flip(); + return view; + } +} diff --git a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java index 5d387182e38..d5a7b66f4fa 100644 --- a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java +++ b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java @@ -12,6 +12,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.nio.channels.Channels; import java.util.*; import static org.junit.jupiter.api.Assertions.*; @@ -222,6 +223,37 @@ void testAbsentNodeContent() { reader.close(); } + @Test + void streamAndChannelCtorsReadIdentically() { + // Same bytes parsed via the (deprecated) InputStream ctor and the ReadableByteChannel ctor → same result + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BinWriter writer = new BinWriter(baos, MAGIC, ROOT_VERSION)) { + writer.setVersions(Collections.emptyMap()); + writer.writeStartNode(null, "root"); + writer.writeIntAttribute("a", 42); + writer.writeStringAttribute("s", "hello"); + writer.writeDoubleAttribute("d", 3.14); + writer.writeEndNode(); + } catch (Exception e) { + throw new PowsyblException(e); + } + byte[] payload = baos.toByteArray(); + + BinReader streamReader = new BinReader(new ByteArrayInputStream(payload), MAGIC); + streamReader.readHeader(); + BinReader channelReader = new BinReader(Channels.newChannel(new ByteArrayInputStream(payload)), MAGIC); + channelReader.readHeader(); + + assertEquals(streamReader.readIntAttribute("a"), channelReader.readIntAttribute("a")); + assertEquals(streamReader.readStringAttribute("s"), channelReader.readStringAttribute("s")); + assertEquals(streamReader.readDoubleAttribute("d"), channelReader.readDoubleAttribute("d"), 0d); + + streamReader.readEndNode(); + channelReader.readEndNode(); + streamReader.close(); + channelReader.close(); + } + @Test void testInvalidMagicNumber() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); diff --git a/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java new file mode 100644 index 00000000000..2929da3df77 --- /dev/null +++ b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java @@ -0,0 +1,219 @@ +/** + * Copyright (c) 2025, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import com.powsybl.commons.PowsyblException; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Clement Leclerc {@literal } + */ +class BufferedChannelReaderWriterTest { + + private static ReadableByteChannel readerOf(byte[] data) { + return Channels.newChannel(new ByteArrayInputStream(data)); + } + + @Test + void endiannessIsBigEndian() { + // Format compatibility guarantee: ByteBuffer must match DataInputStream's network byte order + assertEquals(ByteOrder.BIG_ENDIAN, ByteBuffer.allocateDirect(8).order()); + } + + @Test + void roundTripAllPrimitives() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + WritableByteChannel out = Channels.newChannel(baos); + try (BufferedChannelWriter w = new BufferedChannelWriter(out)) { + w.writeByte(0x7F); + w.writeShort(0xFEDC); + w.writeInt(0xDEADBEEF); + w.writeFloat(2.71f); + w.writeDouble(3.14159265358979); + w.writeBoolean(true); + w.writeBoolean(false); + w.writeBytes(new byte[] {1, 2, 3, 4, 5}); + } + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()))) { + assertEquals((byte) 0x7F, r.readByte()); + assertEquals(0xFEDC, r.readUnsignedShort()); + assertEquals(0xDEADBEEF, r.readInt()); + assertEquals(2.71f, r.readFloat(), 0f); + assertEquals(3.14159265358979, r.readDouble(), 0d); + assertTrue(r.readBoolean()); + assertFalse(r.readBoolean()); + assertArrayEquals(new byte[] {1, 2, 3, 4, 5}, r.readNBytes(5)); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void parityWithDataStream() throws Exception { + // Same bytes produced as DataOutputStream/DataInputStream → format compatibility + ByteArrayOutputStream dosOut = new ByteArrayOutputStream(); + try (DataOutputStream dos = new DataOutputStream(dosOut)) { + dos.writeByte(0x42); + dos.writeShort(0xABCD); + dos.writeInt(123456789); + dos.writeFloat(1.5f); + dos.writeDouble(-9.875); + dos.writeBoolean(true); + } + + ByteArrayOutputStream channelOut = new ByteArrayOutputStream(); + try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(channelOut))) { + w.writeByte(0x42); + w.writeShort(0xABCD); + w.writeInt(123456789); + w.writeFloat(1.5f); + w.writeDouble(-9.875); + w.writeBoolean(true); + } + + assertArrayEquals(dosOut.toByteArray(), channelOut.toByteArray()); + + // Channel reader reads bytes written by DataOutputStream identically + try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dosOut.toByteArray())); + BufferedChannelReader r = new BufferedChannelReader(readerOf(dosOut.toByteArray()))) { + assertEquals(dis.readByte(), r.readByte()); + assertEquals(dis.readUnsignedShort(), r.readUnsignedShort()); + assertEquals(dis.readInt(), r.readInt()); + assertEquals(dis.readFloat(), r.readFloat(), 0f); + assertEquals(dis.readDouble(), r.readDouble(), 0d); + assertEquals(dis.readBoolean(), r.readBoolean()); + } + } + + @Test + void readsCrossBufferBoundary() throws Exception { + // Small buffer (16 B) forces multiple refills within a single readNBytes / readInt + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 16)) { + for (int i = 0; i < 100; i++) { + w.writeInt(i); + } + } + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 16)) { + for (int i = 0; i < 100; i++) { + assertEquals(i, r.readInt()); + } + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readNBytesAcrossManyRefills() throws Exception { + byte[] payload = new byte[1024]; + for (int i = 0; i < payload.length; i++) { + payload[i] = (byte) (i & 0xFF); + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 32)) { + w.writeBytes(payload); + } + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 32)) { + assertArrayEquals(payload, r.readNBytes(payload.length)); + } + } + + @Test + void writeBytesLargerThanBufferGoesDirect() throws Exception { + byte[] big = new byte[200]; + for (int i = 0; i < big.length; i++) { + big[i] = (byte) i; + } + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 64)) { + w.writeByte(0xAA); + w.writeBytes(big); + w.writeByte(0xBB); + } + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 64)) { + assertEquals((byte) 0xAA, r.readByte()); + assertArrayEquals(big, r.readNBytes(big.length)); + assertEquals((byte) 0xBB, r.readByte()); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void skipNBytes() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 16)) { + for (int i = 0; i < 50; i++) { + w.writeInt(i); + } + } + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 16)) { + r.skipNBytes(4L * 25); + assertEquals(25, r.readInt()); + r.skipNBytes(4L * 24); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void tryReadUnsignedShortReturnsEofWhenEmpty() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[0]))) { + assertEquals(BufferedChannelReader.EOF, r.tryReadUnsignedShort()); + } + } + + @Test + void tryReadUnsignedShortReturnsValueWhenAvailable() throws Exception { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos))) { + w.writeShort(0x1234); + w.writeShort(0xCAFE); + } + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()))) { + assertEquals(0x1234, r.tryReadUnsignedShort()); + assertEquals(0xCAFE, r.tryReadUnsignedShort()); + assertEquals(BufferedChannelReader.EOF, r.tryReadUnsignedShort()); + } + } + + @Test + void tryReadUnsignedShortThrowsOnPartialShort() throws Exception { + // Single trailing byte: cannot read 2-byte short, must throw rather than silently return EOF + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[] {0x42}))) { + assertThrows(PowsyblException.class, r::tryReadUnsignedShort); + } + } + + @Test + void isEndOfStreamWorksWithoutRead() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[0]))) { + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readUnexpectedEofThrows() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[] {0x01, 0x02}))) { + assertThrows(PowsyblException.class, r::readInt); + } + } +} diff --git a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java index fe02e76c4c8..c4b4d4f51cc 100644 --- a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java +++ b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java @@ -47,6 +47,8 @@ import javax.xml.validation.SchemaFactory; import javax.xml.validation.Validator; import java.io.*; +import java.nio.channels.Channels; +import java.nio.channels.Pipe; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -319,7 +321,7 @@ private static JsonWriter createJsonWriter(OutputStream os, ExportOptions option } private static TreeDataWriter createBinWriter(OutputStream os, ExportOptions options) { - return new BinWriter(os, BIIDM_MAGIC_NUMBER, options.getVersion().toString(".")); + return new BinWriter(Channels.newChannel(os), BIIDM_MAGIC_NUMBER, options.getVersion().toString(".")); } private static void writeRootElement(Network n, NetworkSerializerContext context) { @@ -547,13 +549,22 @@ public static Anonymizer write(Network n, ExportOptions options, OutputStream os public static Anonymizer write(Network n, ExportOptions options, OutputStream os, ExtensionsSupplier extensionsSupplier) { try (TreeDataWriter writer = createTreeDataWriter(n, options, os, extensionsSupplier)) { - NetworkSerializerContext context = createContext(n, options, writer); - writer.setVersions(getExtensionVersions(n, options, extensionsSupplier)); - write(n, context, extensionsSupplier); - return context.getAnonymizer(); + return write(n, options, writer, extensionsSupplier); } } + /** Writes a network using a caller-provided {@link TreeDataWriter}. The caller owns its lifecycle. */ + public static Anonymizer write(Network n, ExportOptions options, TreeDataWriter writer) { + return write(n, options, writer, DefaultExtensionsSupplier.getInstance()); + } + + public static Anonymizer write(Network n, ExportOptions options, TreeDataWriter writer, ExtensionsSupplier extensionsSupplier) { + NetworkSerializerContext context = createContext(n, options, writer); + writer.setVersions(getExtensionVersions(n, options, extensionsSupplier)); + write(n, context, extensionsSupplier); + return context.getAnonymizer(); + } + /** * Return true if the given element has to be written in the given network, false otherwise */ @@ -634,7 +645,7 @@ private static TreeDataReader createTreeDataReader(InputStream is, ImportOptions return switch (config.getFormat()) { case XML -> createXmlReader(is, config, extensionsSupplier); case JSON -> createJsonReader(is, config, extensionsSupplier); - case BIN -> new BinReader(is, BIIDM_MAGIC_NUMBER); + case BIN -> new BinReader(Channels.newChannel(is), BIIDM_MAGIC_NUMBER); }; } @@ -1067,22 +1078,20 @@ public static Network copy(Network network, NetworkFactory networkFactory, Execu Objects.requireNonNull(network); Objects.requireNonNull(networkFactory); Objects.requireNonNull(executor); - PipedOutputStream pos = new PipedOutputStream(); - try (InputStream is = new PipedInputStream(pos)) { + try { + Pipe pipe = Pipe.open(); + Pipe.SinkChannel sink = pipe.sink(); + Pipe.SourceChannel source = pipe.source(); executor.execute(() -> { - try { - write(network, new ExportOptions().setFormat(format), pos); + try (OutputStream os = Channels.newOutputStream(sink)) { + write(network, new ExportOptions().setFormat(format), os); } catch (Exception t) { LOGGER.error(t.toString(), t); - } finally { - try { - pos.close(); - } catch (IOException e) { - LOGGER.error(e.toString(), e); - } } }); - return read(is, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); + try (InputStream is = Channels.newInputStream(source)) { + return read(is, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); + } } catch (IOException e) { throw new UncheckedIOException(e); } From cf59f7df3ca68b3125d3042d20ad976180d627bd Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Mon, 18 May 2026 12:02:30 +0200 Subject: [PATCH 2/9] Added support for `Path`-based constructors in `BinWriter` and `BinReader` Refactored `BinWriter` to directly manage headers and buffers without intermediate wrapping. Signed-off-by: Leclerc Clement --- .../com/powsybl/commons/binary/BinReader.java | 342 ++++++------------ .../com/powsybl/commons/binary/BinWriter.java | 121 +++---- .../commons/binary/BufferedChannelReader.java | 108 ++---- .../commons/binary/BufferedChannelWriter.java | 109 ------ .../commons/binary/GrowingByteBuffer.java | 22 +- .../commons/binary/BinWriterReaderTest.java | 51 +-- .../binary/BufferedChannelReaderTest.java | 136 +++++++ .../BufferedChannelReaderWriterTest.java | 219 ----------- .../com/powsybl/iidm/serde/NetworkSerDe.java | 27 ++ 9 files changed, 380 insertions(+), 755 deletions(-) delete mode 100644 commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java create mode 100644 commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java delete mode 100644 commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java index c6893f9e32b..31098f749d7 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java @@ -12,9 +12,7 @@ import com.powsybl.commons.io.TreeDataHeader; import java.io.IOException; -import java.io.InputStream; import java.io.UncheckedIOException; -import java.nio.channels.Channels; import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -26,6 +24,7 @@ /** * @author Florian Dupuy {@literal } + * @author Clement Leclerc {@literal } */ public class BinReader extends AbstractTreeDataReader { @@ -38,74 +37,43 @@ public class BinReader extends AbstractTreeDataReader { private int nextNameIdx = END_NODE; private byte nextType; - /** Preferred constructor: direct channel access avoids the InputStream re-buffering layer. */ public BinReader(ReadableByteChannel channel, byte[] binaryMagicNumber) { - this.binaryMagicNumber = binaryMagicNumber; + this.binaryMagicNumber = Objects.requireNonNull(binaryMagicNumber); this.in = new BufferedChannelReader(Objects.requireNonNull(channel)); } - /** Opens a file directly as a byte channel — the fastest path for file inputs. */ public BinReader(Path path, byte[] binaryMagicNumber) throws IOException { this(Files.newByteChannel(Objects.requireNonNull(path), StandardOpenOption.READ), binaryMagicNumber); } - /** - * Compatibility constructor for callers holding an {@link InputStream}. - * Note: wrapping an {@code InputStream} via {@link Channels#newChannel} gives no perf gain - * over the previous {@code DataInputStream} chain — prefer the {@link ReadableByteChannel} - * or {@link Path} constructors for real I/O speedup. - * - * @deprecated use {@link #BinReader(ReadableByteChannel, byte[])} or {@link #BinReader(Path, byte[])} - */ - @Deprecated(since = "6.10.0") - public BinReader(InputStream is, byte[] binaryMagicNumber) { - this(Channels.newChannel(Objects.requireNonNull(is)), binaryMagicNumber); - } - @Override public TreeDataHeader readHeader() { TreeDataHeader header = super.readHeader(); - try { - readNamesDictionary(); - peekNextEntry(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + readNamesDictionary(); + peekNextEntry(); return header; } @Override protected String readRootVersion() { - try { - readMagicNumber(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return readString(); - } - - private void readMagicNumber() throws IOException { - byte[] read = in.readNBytes(binaryMagicNumber.length); - if (!Arrays.equals(read, binaryMagicNumber)) { + byte[] magic = in.readNBytes(binaryMagicNumber.length); + if (!Arrays.equals(magic, binaryMagicNumber)) { throw new PowsyblException("Unexpected bytes at file start"); } + return readString(); } @Override protected Map readExtensionVersions() { - try { - int nbVersions = in.readUnsignedShort(); - Map versions = new HashMap<>(); - for (int i = 0; i < nbVersions; i++) { - versions.put(readString(), readString()); - } - return versions; - } catch (IOException e) { - throw new UncheckedIOException(e); + int nbVersions = in.readUnsignedShort(); + Map versions = new HashMap<>(); + for (int i = 0; i < nbVersions; i++) { + versions.put(readString(), readString()); } + return versions; } - private void readNamesDictionary() throws IOException { + private void readNamesDictionary() { int nbEntries = in.readUnsignedShort(); names = new String[nbEntries + 1]; types = new byte[nbEntries + 1]; @@ -115,13 +83,12 @@ private void readNamesDictionary() throws IOException { } } - private void peekNextEntry() throws IOException { - int idx = in.tryReadUnsignedShort(); - if (idx == BufferedChannelReader.EOF) { + private void peekNextEntry() { + if (in.isEndOfStream()) { nextNameIdx = END_NODE; return; } - nextNameIdx = idx; + nextNameIdx = in.readUnsignedShort(); if (nextNameIdx != END_NODE) { nextType = types[nextNameIdx]; } @@ -138,14 +105,14 @@ private boolean isAttrAbsent(String name) { return !name.equals(entryName); } - private void skipRemainingAttributes() throws IOException { + private void skipRemainingAttributes() { while (nextNameIdx != END_NODE && nextType != TYPE_OBJECT) { skipTypedValue(nextType); peekNextEntry(); } } - private void skipTypedValue(byte typeTag) throws IOException { + private void skipTypedValue(byte typeTag) { switch (typeTag) { case TYPE_DOUBLE -> in.skipNBytes(8); case TYPE_FLOAT, TYPE_INT -> in.skipNBytes(4); @@ -158,112 +125,73 @@ private void skipTypedValue(byte typeTag) throws IOException { } } - private void skipString() throws IOException { + private void skipString() { int len = in.readUnsignedShort(); if (len != NULL_STRING_SENTINEL) { in.skipNBytes(len); } } - private void skipIntArray() throws IOException { + private void skipIntArray() { int count = in.readUnsignedShort(); if (count > 0) { in.skipNBytes(4L * count); } } - private void skipStringArray() throws IOException { + private void skipStringArray() { int count = in.readUnsignedShort(); for (int i = 0; i < count; i++) { skipString(); } } - private List readIntArrayRaw() throws IOException { - int count = in.readUnsignedShort(); - List list = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - list.add(in.readInt()); - } - return list; - } - - private List readStringArrayRaw() throws IOException { - int count = in.readUnsignedShort(); - List list = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - list.add(readString()); - } - return list; - } - private String readString() { - try { - int stringNbBytes = in.readUnsignedShort(); - if (stringNbBytes == NULL_STRING_SENTINEL) { - return null; - } - byte[] stringBytes = in.readNBytes(stringNbBytes); - return new String(stringBytes, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new UncheckedIOException(e); + int len = in.readUnsignedShort(); + if (len == NULL_STRING_SENTINEL) { + return null; } + return new String(in.readNBytes(len), StandardCharsets.UTF_8); } @Override public double readDoubleAttribute(String name, double defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - double val = in.readDouble(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + double val = in.readDouble(); + peekNextEntry(); + return val; } @Override public OptionalDouble readOptionalDoubleAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return OptionalDouble.empty(); - } - OptionalDouble val = OptionalDouble.of(in.readDouble()); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return OptionalDouble.empty(); } + OptionalDouble val = OptionalDouble.of(in.readDouble()); + peekNextEntry(); + return val; } @Override public float readFloatAttribute(String name, float defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - float val = in.readFloat(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + float val = in.readFloat(); + peekNextEntry(); + return val; } @Override public String readStringAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return null; - } - String val = readString(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return null; } + String val = readString(); + peekNextEntry(); + return val; } @Override @@ -271,41 +199,29 @@ public int readIntAttribute(String name) { if (isAttrAbsent(name)) { throw new PowsyblException("Missing required int attribute: " + name); } - try { - int val = in.readInt(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + int val = in.readInt(); + peekNextEntry(); + return val; } @Override public int readIntAttribute(String name, int defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - int val = in.readInt(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + int val = in.readInt(); + peekNextEntry(); + return val; } @Override public OptionalInt readOptionalIntAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return OptionalInt.empty(); - } - OptionalInt val = OptionalInt.of(in.readInt()); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return OptionalInt.empty(); } + OptionalInt val = OptionalInt.of(in.readInt()); + peekNextEntry(); + return val; } @Override @@ -313,100 +229,80 @@ public boolean readBooleanAttribute(String name) { if (isAttrAbsent(name)) { throw new PowsyblException("Missing required boolean attribute: " + name); } - try { - boolean val = in.readBoolean(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + boolean val = in.readBoolean(); + peekNextEntry(); + return val; } @Override public boolean readBooleanAttribute(String name, boolean defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - boolean val = in.readBoolean(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + boolean val = in.readBoolean(); + peekNextEntry(); + return val; } @Override public Optional readOptionalBooleanAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return Optional.empty(); - } - Optional val = Optional.of(in.readBoolean()); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return Optional.empty(); } + Optional val = Optional.of(in.readBoolean()); + peekNextEntry(); + return val; } @Override public > T readEnumAttribute(String name, Class clazz, T defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - int ordinal = in.readUnsignedShort(); - peekNextEntry(); - T[] constants = clazz.getEnumConstants(); - return ordinal < constants.length ? constants[ordinal] : defaultValue; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + int ordinal = in.readUnsignedShort(); + peekNextEntry(); + T[] constants = clazz.getEnumConstants(); + return ordinal < constants.length ? constants[ordinal] : defaultValue; } @Override public String readContent() { - try { - if (nextNameIdx == END_NODE || nextType != TYPE_STRING_CONTENT) { - readEndNode(); - return null; - } - String val = readString(); - peekNextEntry(); + if (nextNameIdx == END_NODE || nextType != TYPE_STRING_CONTENT) { readEndNode(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + return null; } + String val = readString(); + peekNextEntry(); + readEndNode(); + return val; } @Override public List readIntArrayAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return Collections.emptyList(); - } - List val = readIntArrayRaw(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return Collections.emptyList(); } + int count = in.readUnsignedShort(); + List val = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + val.add(in.readInt()); + } + peekNextEntry(); + return val; } @Override public List readStringArrayAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return Collections.emptyList(); - } - List val = readStringArrayRaw(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return Collections.emptyList(); } + int count = in.readUnsignedShort(); + List val = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + val.add(readString()); + } + peekNextEntry(); + return val; } @Override @@ -416,41 +312,29 @@ public void skipNode() { @Override public void readChildNodes(ChildNodeReader childNodeReader) { - try { - skipRemainingAttributes(); - while (nextNameIdx != END_NODE) { - String nodeName = names[nextNameIdx]; - if (nodeName == null) { - throw new PowsyblException("Cannot read child node: unknown name index " + nextNameIdx); - } - peekNextEntry(); - childNodeReader.onStartNode(nodeName); + skipRemainingAttributes(); + while (nextNameIdx != END_NODE) { + String nodeName = names[nextNameIdx]; + if (nodeName == null) { + throw new PowsyblException("Cannot read child node: unknown name index " + nextNameIdx); } peekNextEntry(); - } catch (IOException e) { - throw new UncheckedIOException(e); + childNodeReader.onStartNode(nodeName); } + peekNextEntry(); } @Override public void readEndNode() { - try { - skipRemainingAttributes(); - if (nextNameIdx != END_NODE) { - throw new PowsyblException("Binary parsing: expected end node but got name index " + nextNameIdx); - } - peekNextEntry(); - } catch (IOException e) { - throw new UncheckedIOException(e); + skipRemainingAttributes(); + if (nextNameIdx != END_NODE) { + throw new PowsyblException("Binary parsing: expected end node but got name index " + nextNameIdx); } + peekNextEntry(); } boolean readEndOfStream() { - try { - return in.isEndOfStream(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + return in.isEndOfStream(); } @Override diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java index a19ef6f8dd0..e060bc3e76b 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java @@ -11,9 +11,8 @@ import com.powsybl.commons.io.AbstractTreeDataWriter; import java.io.IOException; -import java.io.OutputStream; import java.io.UncheckedIOException; -import java.nio.channels.Channels; +import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -25,71 +24,44 @@ /** * @author Florian Dupuy {@literal } + * @author Clement Leclerc {@literal } */ public class BinWriter extends AbstractTreeDataWriter { + private static final int HEADER_INITIAL_CAPACITY = 4 * 1024; + private final String rootVersion; - private final BufferedChannelWriter out; - private final GrowingByteBuffer body; private final byte[] binaryMagicNumber; - private Map extensionVersions = Collections.emptyMap(); - + private final WritableByteChannel channel; + private final GrowingByteBuffer body = new GrowingByteBuffer(); private final Map namesIndex = new LinkedHashMap<>(); + private Map extensionVersions = Collections.emptyMap(); private record TypedName(String name, byte type) { } - /** Preferred constructor: direct channel access avoids the OutputStream re-buffering layer. */ public BinWriter(WritableByteChannel channel, byte[] binaryMagicNumber, String rootVersion) { + this.channel = Objects.requireNonNull(channel); this.binaryMagicNumber = Objects.requireNonNull(binaryMagicNumber); this.rootVersion = Objects.requireNonNull(rootVersion); - this.out = new BufferedChannelWriter(Objects.requireNonNull(channel)); - this.body = new GrowingByteBuffer(); } - /** Opens a file directly as a byte channel — the fastest path for file outputs. */ public BinWriter(Path path, byte[] binaryMagicNumber, String rootVersion) throws IOException { this(Files.newByteChannel(Objects.requireNonNull(path), StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING), binaryMagicNumber, rootVersion); } - /** - * Compatibility constructor for callers holding an {@link OutputStream}. - * Wrapping an {@code OutputStream} via {@link Channels#newChannel} gives no perf gain over - * the previous {@code DataOutputStream} chain — prefer the {@link WritableByteChannel} or - * {@link Path} constructors for real I/O speedup. - * - * @deprecated use {@link #BinWriter(WritableByteChannel, byte[], String)} or {@link #BinWriter(Path, byte[], String)} - */ - @Deprecated(since = "6.10.0") - public BinWriter(OutputStream outputStream, byte[] binaryMagicNumber, String rootVersion) { - this(Channels.newChannel(Objects.requireNonNull(outputStream)), binaryMagicNumber, rootVersion); - } - - private static void writeStringToBody(String value, GrowingByteBuffer buf) { + private static void writeString(String value, GrowingByteBuffer buf) { if (value == null) { buf.writeShort(NULL_STRING_SENTINEL); - } else { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - if (bytes.length >= NULL_STRING_SENTINEL) { - throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); - } - buf.writeShort(bytes.length); - buf.writeBytes(bytes); + return; } - } - - private static void writeStringToHeader(String value, BufferedChannelWriter w) throws IOException { - if (value == null) { - w.writeShort(NULL_STRING_SENTINEL); - } else { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - if (bytes.length >= NULL_STRING_SENTINEL) { - throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); - } - w.writeShort(bytes.length); - w.writeBytes(bytes); + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length >= NULL_STRING_SENTINEL) { + throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); } + buf.writeShort(bytes.length); + buf.writeBytes(bytes); } @Override @@ -102,10 +74,16 @@ public void writeEndNodes() { // nothing to do } + @Override + public void writeNamespace(String prefix, String namespace) { + // nothing to do + } + @Override public void writeStartNode(String namespace, String name) { if (namesIndex.isEmpty()) { - namesIndex.put(new TypedName(name, TYPE_OBJECT), 1); // root element is not a child of another node, hence index is not expected + // root element is not a child of another node, its index is not consumed in the body + namesIndex.put(new TypedName(name, TYPE_OBJECT), 1); } else { writeEntry(name, TYPE_OBJECT); } @@ -130,21 +108,16 @@ private void writeEntry(String name, byte type) { body.writeShort(index); } - @Override - public void writeNamespace(String prefix, String namespace) { - // nothing to do - } - @Override public void writeNodeContent(String value) { writeEntry("", TYPE_STRING_CONTENT); - writeStringToBody(value, body); + writeString(value, body); } @Override public void writeStringAttribute(String name, String value) { writeEntry(name, TYPE_STRING); - writeStringToBody(value, body); + writeString(value, body); } @Override @@ -222,45 +195,47 @@ public void writeStringArrayAttribute(String name, Collection values) { writeEntry(name, TYPE_STRING_ARRAY); body.writeShort(values.size()); for (String s : values) { - writeStringToBody(s, body); + writeString(s, body); } } + @Override + public void setVersions(Map extensionVersions) { + this.extensionVersions = Objects.requireNonNull(extensionVersions); + } + @Override public void close() { - try { - writeHeader(); - out.writeFully(body.toReadBuffer()); - out.close(); + try (channel) { + drain(buildHeader()); + drain(body.toReadBuffer()); } catch (IOException e) { throw new UncheckedIOException(e); } } - private void writeHeader() throws IOException { - // magic number - out.writeBytes(binaryMagicNumber); - - // iidm version - writeStringToHeader(rootVersion, out); + private ByteBuffer buildHeader() { + GrowingByteBuffer header = new GrowingByteBuffer(HEADER_INITIAL_CAPACITY); + header.writeBytes(binaryMagicNumber); + writeString(rootVersion, header); - // extensions versions - out.writeShort(extensionVersions.size()); + header.writeShort(extensionVersions.size()); for (var entry : extensionVersions.entrySet()) { - writeStringToHeader(entry.getKey(), out); - writeStringToHeader(entry.getValue(), out); + writeString(entry.getKey(), header); + writeString(entry.getValue(), header); } - // names dictionary - out.writeShort(namesIndex.size()); + header.writeShort(namesIndex.size()); for (TypedName key : namesIndex.keySet()) { - writeStringToHeader(key.name(), out); - out.writeByte(key.type()); + writeString(key.name(), header); + header.writeByte(key.type()); } + return header.toReadBuffer(); } - @Override - public void setVersions(Map extensionVersions) { - this.extensionVersions = Objects.requireNonNull(extensionVersions); + private void drain(ByteBuffer buf) throws IOException { + while (buf.hasRemaining()) { + channel.write(buf); + } } } diff --git a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java index e74453aff6d..c0bce10bbc9 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2025, RTE (http://www.rte-france.com) + * Copyright (c) 2026, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -10,14 +10,13 @@ import com.powsybl.commons.PowsyblException; import java.io.IOException; +import java.io.UncheckedIOException; import java.nio.ByteBuffer; import java.nio.channels.ReadableByteChannel; import java.util.Objects; /** * Buffered reader on top of a {@link ReadableByteChannel}, backed by a direct {@link ByteBuffer}. - * Replaces the {@code DataInputStream + BufferedInputStream} chain to avoid double indirection - * and to leverage JVM intrinsics on aligned multi-byte reads (big-endian, network order). * * @author Clement Leclerc {@literal } */ @@ -25,9 +24,6 @@ final class BufferedChannelReader implements AutoCloseable { static final int DEFAULT_BUFFER_SIZE = 64 * 1024; - /** Sentinel returned by {@link #tryReadUnsignedShort()} on end-of-stream. */ - static final int EOF = -1; - private final ReadableByteChannel channel; private final ByteBuffer buffer; private boolean channelExhausted; @@ -39,64 +35,69 @@ final class BufferedChannelReader implements AutoCloseable { BufferedChannelReader(ReadableByteChannel channel, int bufferSize) { this.channel = Objects.requireNonNull(channel); this.buffer = ByteBuffer.allocateDirect(bufferSize); - this.buffer.flip(); // start empty in read mode + this.buffer.flip(); } - /** Ensures at least {@code n} bytes are available in the buffer, refilling from the channel if needed. */ - private void ensureAvailable(int n) throws IOException { + /** Pulls bytes from the channel until the buffer holds at least {@code n}, or the channel ends. */ + private int fill(int n) { if (buffer.remaining() >= n) { - return; + return buffer.remaining(); } buffer.compact(); - while (buffer.position() < n) { - int read = channel.read(buffer); - if (read == -1) { - channelExhausted = true; - break; + try { + while (buffer.position() < n && !channelExhausted) { + if (channel.read(buffer) == -1) { + channelExhausted = true; + } } + } catch (IOException e) { + throw new UncheckedIOException(e); } buffer.flip(); - if (buffer.remaining() < n) { + return buffer.remaining(); + } + + private void require(int n) { + if (fill(n) < n) { throw new PowsyblException("Unexpected end of stream: needed " + n + " bytes, got " + buffer.remaining()); } } - byte readByte() throws IOException { - ensureAvailable(1); + byte readByte() { + require(1); return buffer.get(); } - int readUnsignedShort() throws IOException { - ensureAvailable(2); + int readUnsignedShort() { + require(2); return Short.toUnsignedInt(buffer.getShort()); } - int readInt() throws IOException { - ensureAvailable(4); + int readInt() { + require(4); return buffer.getInt(); } - float readFloat() throws IOException { - ensureAvailable(4); + float readFloat() { + require(4); return buffer.getFloat(); } - double readDouble() throws IOException { - ensureAvailable(8); + double readDouble() { + require(8); return buffer.getDouble(); } - boolean readBoolean() throws IOException { + boolean readBoolean() { return readByte() != 0; } - /** Reads exactly {@code n} bytes; throws if the stream ends early. */ - byte[] readNBytes(int n) throws IOException { + byte[] readNBytes(int n) { byte[] out = new byte[n]; int filled = 0; while (filled < n) { if (!buffer.hasRemaining()) { - ensureAvailable(1); + require(1); } int take = Math.min(buffer.remaining(), n - filled); buffer.get(out, filled, take); @@ -105,11 +106,11 @@ byte[] readNBytes(int n) throws IOException { return out; } - void skipNBytes(long n) throws IOException { + void skipNBytes(long n) { long remaining = n; while (remaining > 0) { if (!buffer.hasRemaining()) { - ensureAvailable(1); + require(1); } int skip = (int) Math.min(buffer.remaining(), remaining); buffer.position(buffer.position() + skip); @@ -117,49 +118,12 @@ void skipNBytes(long n) throws IOException { } } - /** - * Attempts to read an unsigned short. Returns {@link #EOF} (-1) if the stream is exhausted - * before any byte of the short can be read, instead of throwing. - * Useful for end-of-stream detection in peek-style parsers. - */ - int tryReadUnsignedShort() throws IOException { - if (buffer.remaining() >= 2) { - return Short.toUnsignedInt(buffer.getShort()); - } - buffer.compact(); - while (buffer.position() < 2) { - int read = channel.read(buffer); - if (read == -1) { - channelExhausted = true; - break; - } - } - buffer.flip(); - if (buffer.remaining() < 2) { - if (buffer.remaining() == 0) { - return EOF; - } - throw new PowsyblException("Unexpected end of stream: needed 2 bytes, got " + buffer.remaining()); - } - return Short.toUnsignedInt(buffer.getShort()); - } - - /** Returns true if no more bytes are available in the buffer or the underlying channel. */ - boolean isEndOfStream() throws IOException { + /** Returns true when no more bytes are available in the buffer or the channel. */ + boolean isEndOfStream() { if (buffer.hasRemaining()) { return false; } - if (channelExhausted) { - return true; - } - buffer.compact(); - int read = channel.read(buffer); - buffer.flip(); - if (read == -1) { - channelExhausted = true; - return !buffer.hasRemaining(); - } - return false; + return fill(1) == 0; } @Override diff --git a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java deleted file mode 100644 index baa0f122a2f..00000000000 --- a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelWriter.java +++ /dev/null @@ -1,109 +0,0 @@ -/** - * Copyright (c) 2025, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.commons.binary; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.WritableByteChannel; -import java.util.Objects; - -/** - * Buffered writer on top of a {@link WritableByteChannel}, backed by a direct {@link ByteBuffer}. - * Replaces the {@code DataOutputStream + BufferedOutputStream} chain. - * Payloads larger than the buffer capacity are written directly to the channel after a flush, - * avoiding pointless intermediate copies. - * - * @author Clement Leclerc {@literal } - */ -final class BufferedChannelWriter implements AutoCloseable { - - static final int DEFAULT_BUFFER_SIZE = 256 * 1024; - - private final WritableByteChannel channel; - private final ByteBuffer buffer; - - BufferedChannelWriter(WritableByteChannel channel) { - this(channel, DEFAULT_BUFFER_SIZE); - } - - BufferedChannelWriter(WritableByteChannel channel, int bufferSize) { - this.channel = Objects.requireNonNull(channel); - this.buffer = ByteBuffer.allocateDirect(bufferSize); - } - - private void ensureSpace(int n) throws IOException { - if (buffer.remaining() < n) { - flush(); - } - } - - void writeByte(int b) throws IOException { - ensureSpace(1); - buffer.put((byte) b); - } - - void writeShort(int s) throws IOException { - ensureSpace(2); - buffer.putShort((short) s); - } - - void writeInt(int i) throws IOException { - ensureSpace(4); - buffer.putInt(i); - } - - void writeFloat(float f) throws IOException { - ensureSpace(4); - buffer.putFloat(f); - } - - void writeDouble(double d) throws IOException { - ensureSpace(8); - buffer.putDouble(d); - } - - void writeBoolean(boolean b) throws IOException { - writeByte(b ? 1 : 0); - } - - void writeBytes(byte[] bytes) throws IOException { - if (bytes.length > buffer.capacity()) { - // payload exceeds buffer capacity: flush then write directly - flush(); - ByteBuffer wrap = ByteBuffer.wrap(bytes); - while (wrap.hasRemaining()) { - channel.write(wrap); - } - return; - } - ensureSpace(bytes.length); - buffer.put(bytes); - } - - /** Drains a fully-prepared (already flipped) ByteBuffer to the channel after flushing internal buffer. */ - void writeFully(ByteBuffer src) throws IOException { - flush(); - while (src.hasRemaining()) { - channel.write(src); - } - } - - void flush() throws IOException { - buffer.flip(); - while (buffer.hasRemaining()) { - channel.write(buffer); - } - buffer.clear(); - } - - @Override - public void close() throws IOException { - flush(); - channel.close(); - } -} diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java index 6a208ec4d83..ed8f8ae7d06 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2024, RTE (http://www.rte-france.com) + * Copyright (c) 2025, RTE (http://www.rte-france.com) * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. @@ -9,9 +9,15 @@ import java.nio.ByteBuffer; +/** + * Growable direct {@link ByteBuffer} used to stage binary payloads in memory before draining + * them to a channel. Capacity doubles on overflow. + * + * @author Clement Leclerc {@literal } + */ final class GrowingByteBuffer { - private static final int DEFAULT_INITIAL_CAPACITY = 16 * 1024; + private static final int DEFAULT_INITIAL_CAPACITY = 32 * 1024 * 1024; private ByteBuffer buffer; @@ -20,17 +26,13 @@ final class GrowingByteBuffer { } GrowingByteBuffer(int initialCapacity) { - this.buffer = ByteBuffer.allocate(initialCapacity); + this.buffer = ByteBuffer.allocateDirect(initialCapacity); } private void ensureSpace(int n) { if (buffer.remaining() < n) { - int needed = buffer.position() + n; - int newCapacity = buffer.capacity() * 2; - while (newCapacity < needed) { - newCapacity *= 2; - } - ByteBuffer next = ByteBuffer.allocate(newCapacity); + int newCapacity = Math.max(buffer.capacity() * 2, buffer.position() + n); + ByteBuffer next = ByteBuffer.allocateDirect(newCapacity); buffer.flip(); next.put(buffer); buffer = next; @@ -71,7 +73,7 @@ void writeBytes(byte[] bytes) { buffer.put(bytes); } - /** Returns a read-only view positioned at 0 with limit at current size, ready for channel.write(). */ + /** Returns a view positioned at 0, limit at current size, ready for {@code channel.write()}. */ ByteBuffer toReadBuffer() { ByteBuffer view = buffer.duplicate(); view.flip(); diff --git a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java index d5a7b66f4fa..5d73347a807 100644 --- a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java +++ b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java @@ -25,10 +25,10 @@ class BinWriterReaderTest { private static final byte[] MAGIC = {0x54, 0x45, 0x53, 0x54}; // "TEST" private static final String ROOT_VERSION = "1.0"; - /** Write a single root node, close the writer, return an initialised reader. */ + /** Writes a single root node, closes the writer, returns an initialised reader. */ private BinReader roundTrip(WriterAction action) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BinWriter writer = new BinWriter(baos, MAGIC, ROOT_VERSION)) { + try (BinWriter writer = new BinWriter(Channels.newChannel(baos), MAGIC, ROOT_VERSION)) { writer.setVersions(Collections.emptyMap()); writer.writeStartNode(null, "root"); action.run(writer); @@ -36,7 +36,7 @@ private BinReader roundTrip(WriterAction action) { } catch (Exception e) { throw new PowsyblException(e); } - BinReader reader = new BinReader(new ByteArrayInputStream(baos.toByteArray()), MAGIC); + BinReader reader = new BinReader(Channels.newChannel(new ByteArrayInputStream(baos.toByteArray())), MAGIC); reader.readHeader(); return reader; } @@ -79,16 +79,14 @@ void testMissingAttributesReturnDefaults() { @Test void testAbsentAttributesReturnDefault() { - // Writer writes only "a" and "c", skipping "b" (optional/default value) + // writer skips "b": reader should return default and not consume the stream BinReader reader = roundTrip(writer -> { writer.writeIntAttribute("a", 1); writer.writeStringAttribute("c", "hello"); }); assertEquals(1, reader.readIntAttribute("a")); - // "b" was not written: next attr is "c", name mismatch → default returned, stream not consumed assertNull(reader.readStringAttribute("b")); - // "c" is still available assertEquals("hello", reader.readStringAttribute("c")); reader.readEndNode(); @@ -153,8 +151,8 @@ void testOptionalTypesRoundTrip() { @Test void testSkipRemainingAttributes() { - // Writes attrs of every type, reads only the first one. - // readEndNode must skip the remaining ones → exercises skipRemainingAttributes + all skipTypedValue branches. + // writes attrs of every type, reads only the first one; + // readEndNode must skip the rest, exercising every branch of skipTypedValue BinReader reader = roundTrip(writer -> { writer.writeIntAttribute("a", 1); writer.writeDoubleAttribute("b", 2.0); @@ -167,7 +165,6 @@ void testSkipRemainingAttributes() { }); assertEquals(1, reader.readIntAttribute("a")); - // All remaining attrs skipped by readEndNode reader.readEndNode(); reader.close(); } @@ -203,7 +200,6 @@ void testSkipNode() { writer.writeEndNode(); }); - // skipNode must recursively skip attrs and children at all depths reader.readChildNodes(nodeName -> reader.skipNode()); assertTrue(reader.readEndOfStream()); reader.close(); @@ -223,41 +219,10 @@ void testAbsentNodeContent() { reader.close(); } - @Test - void streamAndChannelCtorsReadIdentically() { - // Same bytes parsed via the (deprecated) InputStream ctor and the ReadableByteChannel ctor → same result - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BinWriter writer = new BinWriter(baos, MAGIC, ROOT_VERSION)) { - writer.setVersions(Collections.emptyMap()); - writer.writeStartNode(null, "root"); - writer.writeIntAttribute("a", 42); - writer.writeStringAttribute("s", "hello"); - writer.writeDoubleAttribute("d", 3.14); - writer.writeEndNode(); - } catch (Exception e) { - throw new PowsyblException(e); - } - byte[] payload = baos.toByteArray(); - - BinReader streamReader = new BinReader(new ByteArrayInputStream(payload), MAGIC); - streamReader.readHeader(); - BinReader channelReader = new BinReader(Channels.newChannel(new ByteArrayInputStream(payload)), MAGIC); - channelReader.readHeader(); - - assertEquals(streamReader.readIntAttribute("a"), channelReader.readIntAttribute("a")); - assertEquals(streamReader.readStringAttribute("s"), channelReader.readStringAttribute("s")); - assertEquals(streamReader.readDoubleAttribute("d"), channelReader.readDoubleAttribute("d"), 0d); - - streamReader.readEndNode(); - channelReader.readEndNode(); - streamReader.close(); - channelReader.close(); - } - @Test void testInvalidMagicNumber() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BinWriter writer = new BinWriter(baos, MAGIC, ROOT_VERSION)) { + try (BinWriter writer = new BinWriter(Channels.newChannel(baos), MAGIC, ROOT_VERSION)) { writer.setVersions(Collections.emptyMap()); writer.writeStartNode(null, "root"); writer.writeEndNode(); @@ -265,7 +230,7 @@ void testInvalidMagicNumber() { throw new RuntimeException(e); } byte[] wrongMagic = {0x00, 0x00, 0x00, 0x00}; - BinReader reader = new BinReader(new ByteArrayInputStream(baos.toByteArray()), wrongMagic); + BinReader reader = new BinReader(Channels.newChannel(new ByteArrayInputStream(baos.toByteArray())), wrongMagic); assertThrows(PowsyblException.class, reader::readHeader); reader.close(); } diff --git a/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java new file mode 100644 index 00000000000..c2fd891c2e1 --- /dev/null +++ b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2025, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import com.powsybl.commons.PowsyblException; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Clement Leclerc {@literal } + */ +class BufferedChannelReaderTest { + + private static ReadableByteChannel readerOf(byte[] data) { + return Channels.newChannel(new ByteArrayInputStream(data)); + } + + @FunctionalInterface + private interface ByteSource { + void write(DataOutputStream dos) throws IOException; + } + + /** Produces bytes via {@link DataOutputStream} (big-endian) — matches the binary format wire layout. */ + private static byte[] bytes(ByteSource source) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (DataOutputStream dos = new DataOutputStream(baos)) { + source.write(dos); + } + return baos.toByteArray(); + } + + @Test + void endiannessIsBigEndian() { + // format guarantee: ByteBuffer must match DataOutputStream's network byte order + assertEquals(ByteOrder.BIG_ENDIAN, ByteBuffer.allocateDirect(8).order()); + } + + @Test + void readsAllPrimitives() throws Exception { + byte[] data = bytes(dos -> { + dos.writeByte(0x7F); + dos.writeShort(0xFEDC); + dos.writeInt(0xDEADBEEF); + dos.writeFloat(2.71f); + dos.writeDouble(3.14159265358979); + dos.writeBoolean(true); + dos.writeBoolean(false); + dos.write(new byte[] {1, 2, 3, 4, 5}); + }); + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(data))) { + assertEquals((byte) 0x7F, r.readByte()); + assertEquals(0xFEDC, r.readUnsignedShort()); + assertEquals(0xDEADBEEF, r.readInt()); + assertEquals(2.71f, r.readFloat(), 0f); + assertEquals(3.14159265358979, r.readDouble(), 0d); + assertTrue(r.readBoolean()); + assertFalse(r.readBoolean()); + assertArrayEquals(new byte[] {1, 2, 3, 4, 5}, r.readNBytes(5)); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readsCrossBufferBoundary() throws Exception { + // small buffer forces multiple channel refills inside a single readInt + byte[] data = bytes(dos -> { + for (int i = 0; i < 100; i++) { + dos.writeInt(i); + } + }); + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(data), 16)) { + for (int i = 0; i < 100; i++) { + assertEquals(i, r.readInt()); + } + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readNBytesAcrossManyRefills() throws Exception { + byte[] payload = new byte[1024]; + for (int i = 0; i < payload.length; i++) { + payload[i] = (byte) (i & 0xFF); + } + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(payload), 32)) { + assertArrayEquals(payload, r.readNBytes(payload.length)); + } + } + + @Test + void skipNBytes() throws Exception { + byte[] data = bytes(dos -> { + for (int i = 0; i < 50; i++) { + dos.writeInt(i); + } + }); + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(data), 16)) { + r.skipNBytes(4L * 25); + assertEquals(25, r.readInt()); + r.skipNBytes(4L * 24); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void isEndOfStreamWorksWithoutRead() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[0]))) { + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readUnexpectedEofThrows() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[] {0x01, 0x02}))) { + assertThrows(PowsyblException.class, r::readInt); + } + } +} diff --git a/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java deleted file mode 100644 index 2929da3df77..00000000000 --- a/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderWriterTest.java +++ /dev/null @@ -1,219 +0,0 @@ -/** - * Copyright (c) 2025, RTE (http://www.rte-france.com) - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. - * SPDX-License-Identifier: MPL-2.0 - */ -package com.powsybl.commons.binary; - -import com.powsybl.commons.PowsyblException; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.WritableByteChannel; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * @author Clement Leclerc {@literal } - */ -class BufferedChannelReaderWriterTest { - - private static ReadableByteChannel readerOf(byte[] data) { - return Channels.newChannel(new ByteArrayInputStream(data)); - } - - @Test - void endiannessIsBigEndian() { - // Format compatibility guarantee: ByteBuffer must match DataInputStream's network byte order - assertEquals(ByteOrder.BIG_ENDIAN, ByteBuffer.allocateDirect(8).order()); - } - - @Test - void roundTripAllPrimitives() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - WritableByteChannel out = Channels.newChannel(baos); - try (BufferedChannelWriter w = new BufferedChannelWriter(out)) { - w.writeByte(0x7F); - w.writeShort(0xFEDC); - w.writeInt(0xDEADBEEF); - w.writeFloat(2.71f); - w.writeDouble(3.14159265358979); - w.writeBoolean(true); - w.writeBoolean(false); - w.writeBytes(new byte[] {1, 2, 3, 4, 5}); - } - - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()))) { - assertEquals((byte) 0x7F, r.readByte()); - assertEquals(0xFEDC, r.readUnsignedShort()); - assertEquals(0xDEADBEEF, r.readInt()); - assertEquals(2.71f, r.readFloat(), 0f); - assertEquals(3.14159265358979, r.readDouble(), 0d); - assertTrue(r.readBoolean()); - assertFalse(r.readBoolean()); - assertArrayEquals(new byte[] {1, 2, 3, 4, 5}, r.readNBytes(5)); - assertTrue(r.isEndOfStream()); - } - } - - @Test - void parityWithDataStream() throws Exception { - // Same bytes produced as DataOutputStream/DataInputStream → format compatibility - ByteArrayOutputStream dosOut = new ByteArrayOutputStream(); - try (DataOutputStream dos = new DataOutputStream(dosOut)) { - dos.writeByte(0x42); - dos.writeShort(0xABCD); - dos.writeInt(123456789); - dos.writeFloat(1.5f); - dos.writeDouble(-9.875); - dos.writeBoolean(true); - } - - ByteArrayOutputStream channelOut = new ByteArrayOutputStream(); - try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(channelOut))) { - w.writeByte(0x42); - w.writeShort(0xABCD); - w.writeInt(123456789); - w.writeFloat(1.5f); - w.writeDouble(-9.875); - w.writeBoolean(true); - } - - assertArrayEquals(dosOut.toByteArray(), channelOut.toByteArray()); - - // Channel reader reads bytes written by DataOutputStream identically - try (DataInputStream dis = new DataInputStream(new ByteArrayInputStream(dosOut.toByteArray())); - BufferedChannelReader r = new BufferedChannelReader(readerOf(dosOut.toByteArray()))) { - assertEquals(dis.readByte(), r.readByte()); - assertEquals(dis.readUnsignedShort(), r.readUnsignedShort()); - assertEquals(dis.readInt(), r.readInt()); - assertEquals(dis.readFloat(), r.readFloat(), 0f); - assertEquals(dis.readDouble(), r.readDouble(), 0d); - assertEquals(dis.readBoolean(), r.readBoolean()); - } - } - - @Test - void readsCrossBufferBoundary() throws Exception { - // Small buffer (16 B) forces multiple refills within a single readNBytes / readInt - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 16)) { - for (int i = 0; i < 100; i++) { - w.writeInt(i); - } - } - - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 16)) { - for (int i = 0; i < 100; i++) { - assertEquals(i, r.readInt()); - } - assertTrue(r.isEndOfStream()); - } - } - - @Test - void readNBytesAcrossManyRefills() throws Exception { - byte[] payload = new byte[1024]; - for (int i = 0; i < payload.length; i++) { - payload[i] = (byte) (i & 0xFF); - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 32)) { - w.writeBytes(payload); - } - - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 32)) { - assertArrayEquals(payload, r.readNBytes(payload.length)); - } - } - - @Test - void writeBytesLargerThanBufferGoesDirect() throws Exception { - byte[] big = new byte[200]; - for (int i = 0; i < big.length; i++) { - big[i] = (byte) i; - } - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 64)) { - w.writeByte(0xAA); - w.writeBytes(big); - w.writeByte(0xBB); - } - - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 64)) { - assertEquals((byte) 0xAA, r.readByte()); - assertArrayEquals(big, r.readNBytes(big.length)); - assertEquals((byte) 0xBB, r.readByte()); - assertTrue(r.isEndOfStream()); - } - } - - @Test - void skipNBytes() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos), 16)) { - for (int i = 0; i < 50; i++) { - w.writeInt(i); - } - } - - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()), 16)) { - r.skipNBytes(4L * 25); - assertEquals(25, r.readInt()); - r.skipNBytes(4L * 24); - assertTrue(r.isEndOfStream()); - } - } - - @Test - void tryReadUnsignedShortReturnsEofWhenEmpty() throws Exception { - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[0]))) { - assertEquals(BufferedChannelReader.EOF, r.tryReadUnsignedShort()); - } - } - - @Test - void tryReadUnsignedShortReturnsValueWhenAvailable() throws Exception { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BufferedChannelWriter w = new BufferedChannelWriter(Channels.newChannel(baos))) { - w.writeShort(0x1234); - w.writeShort(0xCAFE); - } - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(baos.toByteArray()))) { - assertEquals(0x1234, r.tryReadUnsignedShort()); - assertEquals(0xCAFE, r.tryReadUnsignedShort()); - assertEquals(BufferedChannelReader.EOF, r.tryReadUnsignedShort()); - } - } - - @Test - void tryReadUnsignedShortThrowsOnPartialShort() throws Exception { - // Single trailing byte: cannot read 2-byte short, must throw rather than silently return EOF - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[] {0x42}))) { - assertThrows(PowsyblException.class, r::tryReadUnsignedShort); - } - } - - @Test - void isEndOfStreamWorksWithoutRead() throws Exception { - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[0]))) { - assertTrue(r.isEndOfStream()); - } - } - - @Test - void readUnexpectedEofThrows() throws Exception { - try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[] {0x01, 0x02}))) { - assertThrows(PowsyblException.class, r::readInt); - } - } -} diff --git a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java index c4b4d4f51cc..4bf9932318a 100644 --- a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java +++ b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java @@ -598,6 +598,13 @@ public static Anonymizer write(Network n, OutputStream os) { } public static Anonymizer write(Network n, ExportOptions options, Path xmlFile) { + if (options.getFormat() == TreeDataFormat.BIN) { + try (BinWriter writer = new BinWriter(xmlFile, BIIDM_MAGIC_NUMBER, options.getVersion().toString("."))) { + return write(n, options, writer); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(xmlFile))) { return write(n, options, os); } catch (IOException e) { @@ -953,6 +960,13 @@ public static Network read(Path xmlFile, ImportOptions options) { } public static Network read(Path xmlFile, ImportOptions options, Anonymizer anonymizer, NetworkFactory networkFactory, ReportNode reportNode) { + if (options.getFormat() == TreeDataFormat.BIN) { + try (BinReader reader = new BinReader(xmlFile, BIIDM_MAGIC_NUMBER)) { + return read(reader, options, anonymizer, networkFactory, reportNode); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } try (InputStream is = Files.newInputStream(xmlFile)) { return read(is, options, anonymizer, networkFactory, reportNode); } catch (IOException e) { @@ -1082,6 +1096,19 @@ public static Network copy(Network network, NetworkFactory networkFactory, Execu Pipe pipe = Pipe.open(); Pipe.SinkChannel sink = pipe.sink(); Pipe.SourceChannel source = pipe.source(); + if (format == TreeDataFormat.BIN) { + ExportOptions exportOptions = new ExportOptions().setFormat(format); + executor.execute(() -> { + try (BinWriter writer = new BinWriter(sink, BIIDM_MAGIC_NUMBER, exportOptions.getVersion().toString("."))) { + write(network, exportOptions, writer); + } catch (Exception t) { + LOGGER.error(t.toString(), t); + } + }); + try (BinReader reader = new BinReader(source, BIIDM_MAGIC_NUMBER)) { + return read(reader, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); + } + } executor.execute(() -> { try (OutputStream os = Channels.newOutputStream(sink)) { write(network, new ExportOptions().setFormat(format), os); From 04e9e166eeaf7332c7b8bedce04511ee85371077 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Mon, 18 May 2026 16:48:00 +0200 Subject: [PATCH 3/9] Reduced `DEFAULT_INITIAL_CAPACITY` in `GrowingByteBuffer` Signed-off-by: Leclerc Clement --- .../main/java/com/powsybl/commons/binary/GrowingByteBuffer.java | 2 +- .../java/com/powsybl/commons/binary/BinWriterReaderTest.java | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java index ed8f8ae7d06..8639a4ef76a 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -17,7 +17,7 @@ */ final class GrowingByteBuffer { - private static final int DEFAULT_INITIAL_CAPACITY = 32 * 1024 * 1024; + private static final int DEFAULT_INITIAL_CAPACITY = 1024 * 1024; private ByteBuffer buffer; diff --git a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java index 5d73347a807..47db6c193cf 100644 --- a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java +++ b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java @@ -151,8 +151,6 @@ void testOptionalTypesRoundTrip() { @Test void testSkipRemainingAttributes() { - // writes attrs of every type, reads only the first one; - // readEndNode must skip the rest, exercising every branch of skipTypedValue BinReader reader = roundTrip(writer -> { writer.writeIntAttribute("a", 1); writer.writeDoubleAttribute("b", 2.0); From cc36f0f322b61a56c8cea154f28470a012de7be7 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Tue, 26 May 2026 16:32:01 +0200 Subject: [PATCH 4/9] Added `Files` and `Path` imports to `BinReader` Signed-off-by: Leclerc Clement --- .../src/main/java/com/powsybl/commons/binary/BinReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java index b74bda37f5f..37e2ce4d184 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java @@ -15,6 +15,9 @@ import java.io.UncheckedIOException; import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import static com.powsybl.commons.binary.BinUtil.*; From 58340e40f14c75b19cc228b40d2b4c78b6c198c0 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Wed, 27 May 2026 09:40:31 +0200 Subject: [PATCH 5/9] Buffered streams in NetworkSerDe copy() Signed-off-by: Leclerc Clement --- .../src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java index 4bf9932318a..ba09f44354e 100644 --- a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java +++ b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java @@ -1110,13 +1110,13 @@ public static Network copy(Network network, NetworkFactory networkFactory, Execu } } executor.execute(() -> { - try (OutputStream os = Channels.newOutputStream(sink)) { + try (OutputStream os = new BufferedOutputStream(Channels.newOutputStream(sink))) { write(network, new ExportOptions().setFormat(format), os); } catch (Exception t) { LOGGER.error(t.toString(), t); } }); - try (InputStream is = Channels.newInputStream(source)) { + try (InputStream is = new BufferedInputStream(Channels.newInputStream(source))) { return read(is, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); } } catch (IOException e) { From f8d122ff280128cd8d80878610dbaa9e15e01cc7 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Wed, 27 May 2026 10:33:35 +0200 Subject: [PATCH 6/9] Switched `GrowingByteBuffer` to use heap-based `ByteBuffer` instead of direct allocation for improved performance in small-element writes. Signed-off-by: Leclerc Clement --- .../java/com/powsybl/commons/binary/GrowingByteBuffer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java index 8639a4ef76a..f0f432d75fa 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -10,7 +10,7 @@ import java.nio.ByteBuffer; /** - * Growable direct {@link ByteBuffer} used to stage binary payloads in memory before draining + * Growable heap {@link ByteBuffer} used to stage binary payloads in memory before draining * them to a channel. Capacity doubles on overflow. * * @author Clement Leclerc {@literal } @@ -26,13 +26,13 @@ final class GrowingByteBuffer { } GrowingByteBuffer(int initialCapacity) { - this.buffer = ByteBuffer.allocateDirect(initialCapacity); + this.buffer = ByteBuffer.allocate(initialCapacity); } private void ensureSpace(int n) { if (buffer.remaining() < n) { int newCapacity = Math.max(buffer.capacity() * 2, buffer.position() + n); - ByteBuffer next = ByteBuffer.allocateDirect(newCapacity); + ByteBuffer next = ByteBuffer.allocate(newCapacity); buffer.flip(); next.put(buffer); buffer = next; From 8b492277b37cff7c48fa37e1bd8a643835cba2d7 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Wed, 27 May 2026 13:02:03 +0200 Subject: [PATCH 7/9] Refactored `GrowingByteBuffer` to segmented buffer model and simplified `BinWriter` drain logic for improved memory management and performance. Signed-off-by: Leclerc Clement --- .../com/powsybl/commons/binary/BinWriter.java | 19 ++---- .../commons/binary/GrowingByteBuffer.java | 66 ++++++++++++------- 2 files changed, 47 insertions(+), 38 deletions(-) diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java index e060bc3e76b..7c2c9432da6 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java @@ -12,7 +12,6 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.nio.ByteBuffer; import java.nio.channels.WritableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -28,7 +27,7 @@ */ public class BinWriter extends AbstractTreeDataWriter { - private static final int HEADER_INITIAL_CAPACITY = 4 * 1024; + private static final int HEADER_BLOCK_SIZE = 4 * 1024; private final String rootVersion; private final byte[] binaryMagicNumber; @@ -207,15 +206,15 @@ public void setVersions(Map extensionVersions) { @Override public void close() { try (channel) { - drain(buildHeader()); - drain(body.toReadBuffer()); + buildHeader().drainTo(channel); + body.drainTo(channel); } catch (IOException e) { throw new UncheckedIOException(e); } } - private ByteBuffer buildHeader() { - GrowingByteBuffer header = new GrowingByteBuffer(HEADER_INITIAL_CAPACITY); + private GrowingByteBuffer buildHeader() { + GrowingByteBuffer header = new GrowingByteBuffer(HEADER_BLOCK_SIZE); header.writeBytes(binaryMagicNumber); writeString(rootVersion, header); @@ -230,12 +229,6 @@ private ByteBuffer buildHeader() { writeString(key.name(), header); header.writeByte(key.type()); } - return header.toReadBuffer(); - } - - private void drain(ByteBuffer buf) throws IOException { - while (buf.hasRemaining()) { - channel.write(buf); - } + return header; } } diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java index f0f432d75fa..e1ddff0fd85 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -7,61 +7,66 @@ */ package com.powsybl.commons.binary; +import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; /** - * Growable heap {@link ByteBuffer} used to stage binary payloads in memory before draining - * them to a channel. Capacity doubles on overflow. + * Segmented heap buffer used to stage binary payloads in memory before draining them to a channel. * * @author Clement Leclerc {@literal } */ final class GrowingByteBuffer { - private static final int DEFAULT_INITIAL_CAPACITY = 1024 * 1024; + private static final int BLOCK_SIZE = 1024 * 1024; - private ByteBuffer buffer; + private final int blockSize; + private final List filledBlocks = new ArrayList<>(); + private ByteBuffer current; GrowingByteBuffer() { - this(DEFAULT_INITIAL_CAPACITY); + this(BLOCK_SIZE); } - GrowingByteBuffer(int initialCapacity) { - this.buffer = ByteBuffer.allocate(initialCapacity); + GrowingByteBuffer(int blockSize) { + this.blockSize = blockSize; + this.current = ByteBuffer.allocate(blockSize); } + /** Rolls to a fresh block if the current one cannot hold {@code n} more bytes. */ private void ensureSpace(int n) { - if (buffer.remaining() < n) { - int newCapacity = Math.max(buffer.capacity() * 2, buffer.position() + n); - ByteBuffer next = ByteBuffer.allocate(newCapacity); - buffer.flip(); - next.put(buffer); - buffer = next; + if (current.remaining() < n) { + current.flip(); + filledBlocks.add(current); + current = ByteBuffer.allocate(blockSize); } } void writeByte(int b) { ensureSpace(1); - buffer.put((byte) b); + current.put((byte) b); } void writeShort(int s) { ensureSpace(2); - buffer.putShort((short) s); + current.putShort((short) s); } void writeInt(int i) { ensureSpace(4); - buffer.putInt(i); + current.putInt(i); } void writeFloat(float f) { ensureSpace(4); - buffer.putFloat(f); + current.putFloat(f); } void writeDouble(double d) { ensureSpace(8); - buffer.putDouble(d); + current.putDouble(d); } void writeBoolean(boolean b) { @@ -69,14 +74,25 @@ void writeBoolean(boolean b) { } void writeBytes(byte[] bytes) { - ensureSpace(bytes.length); - buffer.put(bytes); + int offset = 0; + while (offset < bytes.length) { + if (current.remaining() == 0) { + ensureSpace(1); + } + int n = Math.min(current.remaining(), bytes.length - offset); + current.put(bytes, offset, n); + offset += n; + } } - /** Returns a view positioned at 0, limit at current size, ready for {@code channel.write()}. */ - ByteBuffer toReadBuffer() { - ByteBuffer view = buffer.duplicate(); - view.flip(); - return view; + /** Writes every staged byte to the channel in order, blocking until fully drained. */ + void drainTo(WritableByteChannel channel) throws IOException { + current.flip(); + filledBlocks.add(current); + for (ByteBuffer block : filledBlocks) { + while (block.hasRemaining()) { + channel.write(block); + } + } } } From ea3f521212dffe37d534bd177746b9af6af15459 Mon Sep 17 00:00:00 2001 From: Leclerc Clement Date: Wed, 27 May 2026 15:21:56 +0200 Subject: [PATCH 8/9] Reduced `BLOCK_SIZE` in `GrowingByteBuffer` Signed-off-by: Leclerc Clement --- .../main/java/com/powsybl/commons/binary/GrowingByteBuffer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java index e1ddff0fd85..f5ad3c19124 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -20,7 +20,7 @@ */ final class GrowingByteBuffer { - private static final int BLOCK_SIZE = 1024 * 1024; + private static final int BLOCK_SIZE = 64 * 1024; private final int blockSize; private final List filledBlocks = new ArrayList<>(); From e321ba46261540bc1de4820fa2408d949c9085af Mon Sep 17 00:00:00 2001 From: Damien Jeandemange Date: Mon, 1 Jun 2026 10:28:06 +0200 Subject: [PATCH 9/9] Fix CGMES connectivityNodesContainer performance (#3915) Signed-off-by: Damien Jeandemange --- .../powsybl/cgmes/model/AbstractCgmesModel.java | 6 +++--- cgmes/cgmes-model/src/main/resources/CIM16.sparql | 14 +++----------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java index e84bbb0ea97..2514cf0cf07 100644 --- a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java +++ b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java @@ -162,10 +162,10 @@ private Map computeContainers() { Map cs = new HashMap<>(); connectivityNodeContainers().forEach(c -> { String id = c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER); - String voltageLevel = c.getId("VoltageLevel"); - String substation = c.getId(SUBSTATION); String type = c.getId("connectivityNodeContainerType"); - String line = type != null && type.contains("Line") ? id : null; + String voltageLevel = type.contains("VoltageLevel") ? id : c.getId("VoltageLevel"); + String substation = type.contains(SUBSTATION) ? id : c.getId(SUBSTATION); + String line = type.contains("Line") ? id : null; String name = c.get("name"); cs.put(id, new CgmesContainer(voltageLevel, substation, line, name)); }); diff --git a/cgmes/cgmes-model/src/main/resources/CIM16.sparql b/cgmes/cgmes-model/src/main/resources/CIM16.sparql index 0b90fa54c9d..5d04c2d8d1c 100644 --- a/cgmes/cgmes-model/src/main/resources/CIM16.sparql +++ b/cgmes/cgmes-model/src/main/resources/CIM16.sparql @@ -313,19 +313,11 @@ WHERE { OPTIONAL { ?ConnectivityNodeContainer cim:IdentifiedObject.name ?name } VALUES ?connectivityNodeContainerType { cim:VoltageLevel cim:Bay cim:Line cim:Substation} . OPTIONAL { - ?ConnectivityNodeContainer - a cim:VoltageLevel ; - cim:VoltageLevel.Substation ?Substation . - BIND ( ?ConnectivityNodeContainer AS ?VoltageLevel ) + ?ConnectivityNodeContainer cim:VoltageLevel.Substation ?Substation . } OPTIONAL { - ?ConnectivityNodeContainer a cim:Bay ; - cim:Bay.VoltageLevel ?VoltageLevel . - ?VoltageLevel cim:VoltageLevel.Substation ?Substation - } - OPTIONAL { - ?ConnectivityNodeContainer a cim:Substation ; - BIND ( ?ConnectivityNodeContainer AS ?Substation ) + ?ConnectivityNodeContainer cim:Bay.VoltageLevel ?VoltageLevel . + ?VoltageLevel cim:VoltageLevel.Substation ?Substation . } }