diff --git a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java index e84bbb0ea97..2514cf0cf07 100644 --- a/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java +++ b/cgmes/cgmes-model/src/main/java/com/powsybl/cgmes/model/AbstractCgmesModel.java @@ -162,10 +162,10 @@ private Map computeContainers() { Map cs = new HashMap<>(); connectivityNodeContainers().forEach(c -> { String id = c.getId(CgmesNames.CONNECTIVITY_NODE_CONTAINER); - String voltageLevel = c.getId("VoltageLevel"); - String substation = c.getId(SUBSTATION); String type = c.getId("connectivityNodeContainerType"); - String line = type != null && type.contains("Line") ? id : null; + String voltageLevel = type.contains("VoltageLevel") ? id : c.getId("VoltageLevel"); + String substation = type.contains(SUBSTATION) ? id : c.getId(SUBSTATION); + String line = type.contains("Line") ? id : null; String name = c.get("name"); cs.put(id, new CgmesContainer(voltageLevel, substation, line, name)); }); diff --git a/cgmes/cgmes-model/src/main/resources/CIM16.sparql b/cgmes/cgmes-model/src/main/resources/CIM16.sparql index 0b90fa54c9d..5d04c2d8d1c 100644 --- a/cgmes/cgmes-model/src/main/resources/CIM16.sparql +++ b/cgmes/cgmes-model/src/main/resources/CIM16.sparql @@ -313,19 +313,11 @@ WHERE { OPTIONAL { ?ConnectivityNodeContainer cim:IdentifiedObject.name ?name } VALUES ?connectivityNodeContainerType { cim:VoltageLevel cim:Bay cim:Line cim:Substation} . OPTIONAL { - ?ConnectivityNodeContainer - a cim:VoltageLevel ; - cim:VoltageLevel.Substation ?Substation . - BIND ( ?ConnectivityNodeContainer AS ?VoltageLevel ) + ?ConnectivityNodeContainer cim:VoltageLevel.Substation ?Substation . } OPTIONAL { - ?ConnectivityNodeContainer a cim:Bay ; - cim:Bay.VoltageLevel ?VoltageLevel . - ?VoltageLevel cim:VoltageLevel.Substation ?Substation - } - OPTIONAL { - ?ConnectivityNodeContainer a cim:Substation ; - BIND ( ?ConnectivityNodeContainer AS ?Substation ) + ?ConnectivityNodeContainer cim:Bay.VoltageLevel ?VoltageLevel . + ?VoltageLevel cim:VoltageLevel.Substation ?Substation . } } diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java index 599b61e96c0..37e2ce4d184 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinReader.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinReader.java @@ -11,18 +11,24 @@ import com.powsybl.commons.io.AbstractTreeDataReader; import com.powsybl.commons.io.TreeDataHeader; -import java.io.*; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.channels.ReadableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import static com.powsybl.commons.binary.BinUtil.*; /** * @author Florian Dupuy {@literal } + * @author Clement Leclerc {@literal } */ public class BinReader extends AbstractTreeDataReader { - private final DataInputStream dis; + private final BufferedChannelReader in; private final byte[] binaryMagicNumber; private String[] names; @@ -31,72 +37,60 @@ public class BinReader extends AbstractTreeDataReader { private int nextNameIdx = END_NODE; private byte nextType; - public BinReader(InputStream is, byte[] binaryMagicNumber) { - this.binaryMagicNumber = binaryMagicNumber; - this.dis = new DataInputStream(new BufferedInputStream(Objects.requireNonNull(is))); + public BinReader(ReadableByteChannel channel, byte[] binaryMagicNumber) { + this.binaryMagicNumber = Objects.requireNonNull(binaryMagicNumber); + this.in = new BufferedChannelReader(Objects.requireNonNull(channel)); + } + + public BinReader(Path path, byte[] binaryMagicNumber) throws IOException { + this(Files.newByteChannel(Objects.requireNonNull(path), StandardOpenOption.READ), binaryMagicNumber); } @Override public TreeDataHeader readHeader() { TreeDataHeader header = super.readHeader(); - try { - readNamesDictionary(); - peekNextEntry(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + readNamesDictionary(); + peekNextEntry(); return header; } @Override protected String readRootVersion() { - try { - readMagicNumber(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - return readString(); - } - - private void readMagicNumber() throws IOException { - byte[] read = dis.readNBytes(binaryMagicNumber.length); - if (!Arrays.equals(read, binaryMagicNumber)) { + byte[] magic = in.readNBytes(binaryMagicNumber.length); + if (!Arrays.equals(magic, binaryMagicNumber)) { throw new PowsyblException("Unexpected bytes at file start"); } + return readString(); } @Override protected Map readExtensionVersions() { - try { - int nbVersions = dis.readUnsignedShort(); - Map versions = new HashMap<>(); - for (int i = 0; i < nbVersions; i++) { - versions.put(readString(), readString()); - } - return versions; - } catch (IOException e) { - throw new UncheckedIOException(e); + int nbVersions = in.readUnsignedShort(); + Map versions = new HashMap<>(); + for (int i = 0; i < nbVersions; i++) { + versions.put(readString(), readString()); } + return versions; } - private void readNamesDictionary() throws IOException { - int nbEntries = dis.readUnsignedShort(); + private void readNamesDictionary() { + int nbEntries = in.readUnsignedShort(); names = new String[nbEntries + 1]; types = new byte[nbEntries + 1]; for (int i = 0; i < nbEntries; i++) { names[i + 1] = readString(); - types[i + 1] = dis.readByte(); + types[i + 1] = in.readByte(); } } - private void peekNextEntry() throws IOException { - try { - nextNameIdx = dis.readUnsignedShort(); - if (nextNameIdx != END_NODE) { - nextType = types[nextNameIdx]; - } - } catch (EOFException e) { + private void peekNextEntry() { + if (in.isEndOfStream()) { nextNameIdx = END_NODE; + return; + } + nextNameIdx = in.readUnsignedShort(); + if (nextNameIdx != END_NODE) { + nextType = types[nextNameIdx]; } } @@ -111,135 +105,93 @@ private boolean isAttrAbsent(String name) { return !name.equals(entryName); } - private void skipRemainingAttributes() throws IOException { + private void skipRemainingAttributes() { while (nextNameIdx != END_NODE && nextType != TYPE_OBJECT) { skipTypedValue(nextType); peekNextEntry(); } } - private void skipTypedValue(byte typeTag) throws IOException { + private void skipTypedValue(byte typeTag) { switch (typeTag) { - case TYPE_DOUBLE -> dis.skipNBytes(8); - case TYPE_FLOAT, TYPE_INT -> dis.skipNBytes(4); - case TYPE_BOOLEAN -> dis.skipNBytes(1); + case TYPE_DOUBLE -> in.skipNBytes(8); + case TYPE_FLOAT, TYPE_INT -> in.skipNBytes(4); + case TYPE_BOOLEAN -> in.skipNBytes(1); case TYPE_STRING, TYPE_STRING_CONTENT -> skipString(); - case TYPE_ENUM -> dis.skipNBytes(2); + case TYPE_ENUM -> in.skipNBytes(2); case TYPE_INT_ARRAY -> skipIntArray(); case TYPE_STRING_ARRAY -> skipStringArray(); default -> throw new PowsyblException("Binary format: unknown type tag " + typeTag); } } - private void skipString() throws IOException { - int len = dis.readUnsignedShort(); + private void skipString() { + int len = in.readUnsignedShort(); if (len != NULL_STRING_SENTINEL) { - dis.skipNBytes(len); + in.skipNBytes(len); } } - private void skipIntArray() throws IOException { - int count = dis.readUnsignedShort(); + private void skipIntArray() { + int count = in.readUnsignedShort(); if (count > 0) { - dis.skipNBytes(4L * count); + in.skipNBytes(4L * count); } } - private void skipStringArray() throws IOException { - int count = dis.readUnsignedShort(); + private void skipStringArray() { + int count = in.readUnsignedShort(); for (int i = 0; i < count; i++) { skipString(); } } - private List readIntArrayRaw() throws IOException { - int count = dis.readUnsignedShort(); - List list = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - list.add(dis.readInt()); - } - return list; - } - - private List readStringArrayRaw() throws IOException { - int count = dis.readUnsignedShort(); - List list = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - list.add(readString()); - } - return list; - } - private String readString() { - try { - int stringNbBytes = dis.readUnsignedShort(); - if (stringNbBytes == NULL_STRING_SENTINEL) { - return null; - } - byte[] stringBytes = dis.readNBytes(stringNbBytes); - if (stringBytes.length != stringNbBytes) { - throw new PowsyblException("Cannot read the full string, bytes missing: " + (stringNbBytes - stringBytes.length)); - } - return new String(stringBytes, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new UncheckedIOException(e); + int len = in.readUnsignedShort(); + if (len == NULL_STRING_SENTINEL) { + return null; } + return new String(in.readNBytes(len), StandardCharsets.UTF_8); } @Override public double readDoubleAttribute(String name, double defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - double val = dis.readDouble(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + double val = in.readDouble(); + peekNextEntry(); + return val; } @Override public OptionalDouble readOptionalDoubleAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return OptionalDouble.empty(); - } - OptionalDouble val = OptionalDouble.of(dis.readDouble()); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return OptionalDouble.empty(); } + OptionalDouble val = OptionalDouble.of(in.readDouble()); + peekNextEntry(); + return val; } @Override public float readFloatAttribute(String name, float defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - float val = dis.readFloat(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + float val = in.readFloat(); + peekNextEntry(); + return val; } @Override public String readStringAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return null; - } - String val = readString(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return null; } + String val = readString(); + peekNextEntry(); + return val; } @Override @@ -247,41 +199,29 @@ public int readIntAttribute(String name) { if (isAttrAbsent(name)) { throw new PowsyblException("Missing required int attribute: " + name); } - try { - int val = dis.readInt(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + int val = in.readInt(); + peekNextEntry(); + return val; } @Override public int readIntAttribute(String name, int defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - int val = dis.readInt(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + int val = in.readInt(); + peekNextEntry(); + return val; } @Override public OptionalInt readOptionalIntAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return OptionalInt.empty(); - } - OptionalInt val = OptionalInt.of(dis.readInt()); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return OptionalInt.empty(); } + OptionalInt val = OptionalInt.of(in.readInt()); + peekNextEntry(); + return val; } @Override @@ -289,100 +229,80 @@ public boolean readBooleanAttribute(String name) { if (isAttrAbsent(name)) { throw new PowsyblException("Missing required boolean attribute: " + name); } - try { - boolean val = dis.readBoolean(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + boolean val = in.readBoolean(); + peekNextEntry(); + return val; } @Override public boolean readBooleanAttribute(String name, boolean defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - boolean val = dis.readBoolean(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + boolean val = in.readBoolean(); + peekNextEntry(); + return val; } @Override public Optional readOptionalBooleanAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return Optional.empty(); - } - Optional val = Optional.of(dis.readBoolean()); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return Optional.empty(); } + Optional val = Optional.of(in.readBoolean()); + peekNextEntry(); + return val; } @Override public > T readEnumAttribute(String name, Class clazz, T defaultValue) { - try { - if (isAttrAbsent(name)) { - return defaultValue; - } - int ordinal = dis.readUnsignedShort(); - peekNextEntry(); - T[] constants = clazz.getEnumConstants(); - return ordinal < constants.length ? constants[ordinal] : defaultValue; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return defaultValue; } + int ordinal = in.readUnsignedShort(); + peekNextEntry(); + T[] constants = clazz.getEnumConstants(); + return ordinal < constants.length ? constants[ordinal] : defaultValue; } @Override public String readContent() { - try { - if (nextNameIdx == END_NODE || nextType != TYPE_STRING_CONTENT) { - readEndNode(); - return null; - } - String val = readString(); - peekNextEntry(); + if (nextNameIdx == END_NODE || nextType != TYPE_STRING_CONTENT) { readEndNode(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + return null; } + String val = readString(); + peekNextEntry(); + readEndNode(); + return val; } @Override public List readIntArrayAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return Collections.emptyList(); - } - List val = readIntArrayRaw(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return Collections.emptyList(); + } + int count = in.readUnsignedShort(); + List val = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + val.add(in.readInt()); } + peekNextEntry(); + return val; } @Override public List readStringArrayAttribute(String name) { - try { - if (isAttrAbsent(name)) { - return Collections.emptyList(); - } - List val = readStringArrayRaw(); - peekNextEntry(); - return val; - } catch (IOException e) { - throw new UncheckedIOException(e); + if (isAttrAbsent(name)) { + return Collections.emptyList(); + } + int count = in.readUnsignedShort(); + List val = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + val.add(readString()); } + peekNextEntry(); + return val; } @Override @@ -392,39 +312,31 @@ public void skipNode() { @Override public void readChildNodes(ChildNodeReader childNodeReader) { - try { - skipRemainingAttributes(); - while (nextNameIdx != END_NODE) { - String nodeName = names[nextNameIdx]; - if (nodeName == null) { - throw new PowsyblException("Cannot read child node: unknown name index " + nextNameIdx); - } - peekNextEntry(); - childNodeReader.onStartNode(nodeName); + skipRemainingAttributes(); + while (nextNameIdx != END_NODE) { + String nodeName = names[nextNameIdx]; + if (nodeName == null) { + throw new PowsyblException("Cannot read child node: unknown name index " + nextNameIdx); } peekNextEntry(); - } catch (IOException e) { - throw new UncheckedIOException(e); + childNodeReader.onStartNode(nodeName); } + peekNextEntry(); } @Override public void readEndNode() { - try { - skipRemainingAttributes(); - if (nextNameIdx != END_NODE) { - throw new PowsyblException("Binary parsing: expected end node but got name index " + nextNameIdx); - } - peekNextEntry(); - } catch (IOException e) { - throw new UncheckedIOException(e); + skipRemainingAttributes(); + if (nextNameIdx != END_NODE) { + throw new PowsyblException("Binary parsing: expected end node but got name index " + nextNameIdx); } + peekNextEntry(); } @Override public void close() { try { - dis.close(); + in.close(); } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java index 8be88cee67a..7c2c9432da6 100644 --- a/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java +++ b/commons/src/main/java/com/powsybl/commons/binary/BinWriter.java @@ -10,59 +10,57 @@ import com.powsybl.commons.PowsyblException; import com.powsybl.commons.io.AbstractTreeDataWriter; -import java.io.*; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.channels.WritableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.*; import static com.powsybl.commons.binary.BinUtil.*; /** * @author Florian Dupuy {@literal } + * @author Clement Leclerc {@literal } */ public class BinWriter extends AbstractTreeDataWriter { + private static final int HEADER_BLOCK_SIZE = 4 * 1024; + private final String rootVersion; - private final DataOutputStream dos; - private final DataOutputStream tmpDos; - private final ByteArrayOutputStream buffer; private final byte[] binaryMagicNumber; - private Map extensionVersions = Collections.emptyMap(); - + private final WritableByteChannel channel; + private final GrowingByteBuffer body = new GrowingByteBuffer(); private final Map namesIndex = new LinkedHashMap<>(); + private Map extensionVersions = Collections.emptyMap(); private record TypedName(String name, byte type) { } - public BinWriter(OutputStream outputStream, byte[] binaryMagicNumber, String rootVersion) { + public BinWriter(WritableByteChannel channel, byte[] binaryMagicNumber, String rootVersion) { + this.channel = Objects.requireNonNull(channel); this.binaryMagicNumber = Objects.requireNonNull(binaryMagicNumber); this.rootVersion = Objects.requireNonNull(rootVersion); - this.dos = new DataOutputStream(new BufferedOutputStream(Objects.requireNonNull(outputStream))); - this.buffer = new ByteArrayOutputStream(); - this.tmpDos = new DataOutputStream(buffer); } - private static void writeIndex(int index, DataOutputStream dataOutputStream) { - try { - dataOutputStream.writeShort(index); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + public BinWriter(Path path, byte[] binaryMagicNumber, String rootVersion) throws IOException { + this(Files.newByteChannel(Objects.requireNonNull(path), + StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING), + binaryMagicNumber, rootVersion); } - private static void writeString(String value, DataOutputStream dataOutputStream) { - try { - if (value == null) { - writeIndex(NULL_STRING_SENTINEL, dataOutputStream); - } else { - byte[] bytes = value.getBytes(StandardCharsets.UTF_8); - if (bytes.length >= NULL_STRING_SENTINEL) { - throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); - } - writeIndex(bytes.length, dataOutputStream); - dataOutputStream.write(bytes); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + private static void writeString(String value, GrowingByteBuffer buf) { + if (value == null) { + buf.writeShort(NULL_STRING_SENTINEL); + return; } + byte[] bytes = value.getBytes(StandardCharsets.UTF_8); + if (bytes.length >= NULL_STRING_SENTINEL) { + throw new PowsyblException("Binary format: string too long (max " + (NULL_STRING_SENTINEL - 1) + " bytes)"); + } + buf.writeShort(bytes.length); + buf.writeBytes(bytes); } @Override @@ -75,10 +73,16 @@ public void writeEndNodes() { // nothing to do } + @Override + public void writeNamespace(String prefix, String namespace) { + // nothing to do + } + @Override public void writeStartNode(String namespace, String name) { if (namesIndex.isEmpty()) { - namesIndex.put(new TypedName(name, TYPE_OBJECT), 1); // root element is not a child of another node, hence index is not expected + // root element is not a child of another node, its index is not consumed in the body + namesIndex.put(new TypedName(name, TYPE_OBJECT), 1); } else { writeEntry(name, TYPE_OBJECT); } @@ -86,7 +90,7 @@ public void writeStartNode(String namespace, String name) { @Override public void writeEndNode() { - writeIndex(END_NODE, tmpDos); + body.writeShort(END_NODE); } private void writeEntry(String name, byte type) { @@ -100,24 +104,19 @@ private void writeEntry(String name, byte type) { namesIndex.put(key, newIndex); index = newIndex; } - writeIndex(index, tmpDos); - } - - @Override - public void writeNamespace(String prefix, String namespace) { - // nothing to do + body.writeShort(index); } @Override public void writeNodeContent(String value) { writeEntry("", TYPE_STRING_CONTENT); - writeString(value, tmpDos); + writeString(value, body); } @Override public void writeStringAttribute(String name, String value) { writeEntry(name, TYPE_STRING); - writeString(value, tmpDos); + writeString(value, body); } @Override @@ -132,11 +131,7 @@ public void writeDoubleAttribute(String name, double value, double absentValue) return; } writeEntry(name, TYPE_DOUBLE); - try { - tmpDos.writeDouble(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeDouble(value); } @Override @@ -145,21 +140,13 @@ public void writeFloatAttribute(String name, float value) { return; } writeEntry(name, TYPE_FLOAT); - try { - tmpDos.writeFloat(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeFloat(value); } @Override public void writeIntAttribute(String name, int value) { writeEntry(name, TYPE_INT); - try { - tmpDos.writeInt(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeInt(value); } @Override @@ -173,11 +160,7 @@ public void writeIntAttribute(String name, int value, int absentValue) { @Override public void writeBooleanAttribute(String name, boolean value) { writeEntry(name, TYPE_BOOLEAN); - try { - tmpDos.writeBoolean(value); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeBoolean(value); } @Override @@ -194,78 +177,58 @@ public > void writeEnumAttribute(String name, E value) { return; } writeEntry(name, TYPE_ENUM); - try { - tmpDos.writeShort(value.ordinal()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } + body.writeShort(value.ordinal()); } @Override public void writeIntArrayAttribute(String name, Collection values) { writeEntry(name, TYPE_INT_ARRAY); - try { - tmpDos.writeShort(values.size()); - for (int v : values) { - tmpDos.writeInt(v); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + body.writeShort(values.size()); + for (int v : values) { + body.writeInt(v); } } @Override public void writeStringArrayAttribute(String name, Collection values) { writeEntry(name, TYPE_STRING_ARRAY); - try { - tmpDos.writeShort(values.size()); - for (String s : values) { - writeString(s, tmpDos); - } - } catch (IOException e) { - throw new UncheckedIOException(e); + body.writeShort(values.size()); + for (String s : values) { + writeString(s, body); } } + @Override + public void setVersions(Map extensionVersions) { + this.extensionVersions = Objects.requireNonNull(extensionVersions); + } + @Override public void close() { - try { - tmpDos.flush(); - writeHeader(); - dos.write(buffer.toByteArray()); - dos.close(); + try (channel) { + buildHeader().drainTo(channel); + body.drainTo(channel); } catch (IOException e) { throw new UncheckedIOException(e); } } - private void writeHeader() throws IOException { - // magic number ("Binary IIDM" in ASCII) - dos.write(binaryMagicNumber); - - // iidm version - writeString(rootVersion, dos); - - // extensions versions - writeIndex(extensionVersions.size(), dos); - extensionVersions.forEach((extensionName, extensionVersion) -> { - writeString(extensionName, dos); - writeString(extensionVersion, dos); - }); - - writeIndex(namesIndex.size(), dos); - namesIndex.keySet().forEach(nameTypeKey -> { - writeString(nameTypeKey.name(), dos); - try { - dos.writeByte(nameTypeKey.type()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - }); - } + private GrowingByteBuffer buildHeader() { + GrowingByteBuffer header = new GrowingByteBuffer(HEADER_BLOCK_SIZE); + header.writeBytes(binaryMagicNumber); + writeString(rootVersion, header); - @Override - public void setVersions(Map extensionVersions) { - this.extensionVersions = Objects.requireNonNull(extensionVersions); + header.writeShort(extensionVersions.size()); + for (var entry : extensionVersions.entrySet()) { + writeString(entry.getKey(), header); + writeString(entry.getValue(), header); + } + + header.writeShort(namesIndex.size()); + for (TypedName key : namesIndex.keySet()) { + writeString(key.name(), header); + header.writeByte(key.type()); + } + return header; } } diff --git a/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java new file mode 100644 index 00000000000..c0bce10bbc9 --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/binary/BufferedChannelReader.java @@ -0,0 +1,133 @@ +/** + * Copyright (c) 2026, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import com.powsybl.commons.PowsyblException; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.ByteBuffer; +import java.nio.channels.ReadableByteChannel; +import java.util.Objects; + +/** + * Buffered reader on top of a {@link ReadableByteChannel}, backed by a direct {@link ByteBuffer}. + * + * @author Clement Leclerc {@literal } + */ +final class BufferedChannelReader implements AutoCloseable { + + static final int DEFAULT_BUFFER_SIZE = 64 * 1024; + + private final ReadableByteChannel channel; + private final ByteBuffer buffer; + private boolean channelExhausted; + + BufferedChannelReader(ReadableByteChannel channel) { + this(channel, DEFAULT_BUFFER_SIZE); + } + + BufferedChannelReader(ReadableByteChannel channel, int bufferSize) { + this.channel = Objects.requireNonNull(channel); + this.buffer = ByteBuffer.allocateDirect(bufferSize); + this.buffer.flip(); + } + + /** Pulls bytes from the channel until the buffer holds at least {@code n}, or the channel ends. */ + private int fill(int n) { + if (buffer.remaining() >= n) { + return buffer.remaining(); + } + buffer.compact(); + try { + while (buffer.position() < n && !channelExhausted) { + if (channel.read(buffer) == -1) { + channelExhausted = true; + } + } + } catch (IOException e) { + throw new UncheckedIOException(e); + } + buffer.flip(); + return buffer.remaining(); + } + + private void require(int n) { + if (fill(n) < n) { + throw new PowsyblException("Unexpected end of stream: needed " + n + " bytes, got " + buffer.remaining()); + } + } + + byte readByte() { + require(1); + return buffer.get(); + } + + int readUnsignedShort() { + require(2); + return Short.toUnsignedInt(buffer.getShort()); + } + + int readInt() { + require(4); + return buffer.getInt(); + } + + float readFloat() { + require(4); + return buffer.getFloat(); + } + + double readDouble() { + require(8); + return buffer.getDouble(); + } + + boolean readBoolean() { + return readByte() != 0; + } + + byte[] readNBytes(int n) { + byte[] out = new byte[n]; + int filled = 0; + while (filled < n) { + if (!buffer.hasRemaining()) { + require(1); + } + int take = Math.min(buffer.remaining(), n - filled); + buffer.get(out, filled, take); + filled += take; + } + return out; + } + + void skipNBytes(long n) { + long remaining = n; + while (remaining > 0) { + if (!buffer.hasRemaining()) { + require(1); + } + int skip = (int) Math.min(buffer.remaining(), remaining); + buffer.position(buffer.position() + skip); + remaining -= skip; + } + } + + /** Returns true when no more bytes are available in the buffer or the channel. */ + boolean isEndOfStream() { + if (buffer.hasRemaining()) { + return false; + } + return fill(1) == 0; + } + + @Override + public void close() throws IOException { + channel.close(); + } +} diff --git a/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java new file mode 100644 index 00000000000..f5ad3c19124 --- /dev/null +++ b/commons/src/main/java/com/powsybl/commons/binary/GrowingByteBuffer.java @@ -0,0 +1,98 @@ +/** + * Copyright (c) 2025, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.WritableByteChannel; +import java.util.ArrayList; +import java.util.List; + +/** + * Segmented heap buffer used to stage binary payloads in memory before draining them to a channel. + * + * @author Clement Leclerc {@literal } + */ +final class GrowingByteBuffer { + + private static final int BLOCK_SIZE = 64 * 1024; + + private final int blockSize; + private final List filledBlocks = new ArrayList<>(); + private ByteBuffer current; + + GrowingByteBuffer() { + this(BLOCK_SIZE); + } + + GrowingByteBuffer(int blockSize) { + this.blockSize = blockSize; + this.current = ByteBuffer.allocate(blockSize); + } + + /** Rolls to a fresh block if the current one cannot hold {@code n} more bytes. */ + private void ensureSpace(int n) { + if (current.remaining() < n) { + current.flip(); + filledBlocks.add(current); + current = ByteBuffer.allocate(blockSize); + } + } + + void writeByte(int b) { + ensureSpace(1); + current.put((byte) b); + } + + void writeShort(int s) { + ensureSpace(2); + current.putShort((short) s); + } + + void writeInt(int i) { + ensureSpace(4); + current.putInt(i); + } + + void writeFloat(float f) { + ensureSpace(4); + current.putFloat(f); + } + + void writeDouble(double d) { + ensureSpace(8); + current.putDouble(d); + } + + void writeBoolean(boolean b) { + writeByte(b ? 1 : 0); + } + + void writeBytes(byte[] bytes) { + int offset = 0; + while (offset < bytes.length) { + if (current.remaining() == 0) { + ensureSpace(1); + } + int n = Math.min(current.remaining(), bytes.length - offset); + current.put(bytes, offset, n); + offset += n; + } + } + + /** Writes every staged byte to the channel in order, blocking until fully drained. */ + void drainTo(WritableByteChannel channel) throws IOException { + current.flip(); + filledBlocks.add(current); + for (ByteBuffer block : filledBlocks) { + while (block.hasRemaining()) { + channel.write(block); + } + } + } +} diff --git a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java index 095cb0f8083..0c1c22b942c 100644 --- a/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java +++ b/commons/src/test/java/com/powsybl/commons/binary/BinWriterReaderTest.java @@ -12,6 +12,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.nio.channels.Channels; import java.util.*; import static org.junit.jupiter.api.Assertions.*; @@ -24,10 +25,10 @@ class BinWriterReaderTest { private static final byte[] MAGIC = {0x54, 0x45, 0x53, 0x54}; // "TEST" private static final String ROOT_VERSION = "1.0"; - /** Write a single root node, close the writer, return an initialised reader. */ + /** Writes a single root node, closes the writer, returns an initialised reader. */ private BinReader roundTrip(WriterAction action) { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BinWriter writer = new BinWriter(baos, MAGIC, ROOT_VERSION)) { + try (BinWriter writer = new BinWriter(Channels.newChannel(baos), MAGIC, ROOT_VERSION)) { writer.setVersions(Collections.emptyMap()); writer.writeStartNode(null, "root"); action.run(writer); @@ -35,7 +36,7 @@ private BinReader roundTrip(WriterAction action) { } catch (Exception e) { throw new PowsyblException(e); } - BinReader reader = new BinReader(new ByteArrayInputStream(baos.toByteArray()), MAGIC); + BinReader reader = new BinReader(Channels.newChannel(new ByteArrayInputStream(baos.toByteArray())), MAGIC); reader.readHeader(); return reader; } @@ -78,16 +79,14 @@ void testMissingAttributesReturnDefaults() { @Test void testAbsentAttributesReturnDefault() { - // Writer writes only "a" and "c", skipping "b" (optional/default value) + // writer skips "b": reader should return default and not consume the stream BinReader reader = roundTrip(writer -> { writer.writeIntAttribute("a", 1); writer.writeStringAttribute("c", "hello"); }); assertEquals(1, reader.readIntAttribute("a")); - // "b" was not written: next attr is "c", name mismatch → default returned, stream not consumed assertNull(reader.readStringAttribute("b")); - // "c" is still available assertEquals("hello", reader.readStringAttribute("c")); reader.readEndNode(); @@ -152,8 +151,6 @@ void testOptionalTypesRoundTrip() { @Test void testSkipRemainingAttributes() { - // Writes attrs of every type, reads only the first one. - // readEndNode must skip the remaining ones → exercises skipRemainingAttributes + all skipTypedValue branches. BinReader reader = roundTrip(writer -> { writer.writeIntAttribute("a", 1); writer.writeDoubleAttribute("b", 2.0); @@ -166,7 +163,6 @@ void testSkipRemainingAttributes() { }); assertEquals(1, reader.readIntAttribute("a")); - // All remaining attrs skipped by readEndNode reader.readEndNode(); reader.close(); } @@ -234,7 +230,7 @@ void testAbsentNodeContent() { @Test void testInvalidMagicNumber() { ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (BinWriter writer = new BinWriter(baos, MAGIC, ROOT_VERSION)) { + try (BinWriter writer = new BinWriter(Channels.newChannel(baos), MAGIC, ROOT_VERSION)) { writer.setVersions(Collections.emptyMap()); writer.writeStartNode(null, "root"); writer.writeEndNode(); @@ -242,7 +238,7 @@ void testInvalidMagicNumber() { throw new RuntimeException(e); } byte[] wrongMagic = {0x00, 0x00, 0x00, 0x00}; - BinReader reader = new BinReader(new ByteArrayInputStream(baos.toByteArray()), wrongMagic); + BinReader reader = new BinReader(Channels.newChannel(new ByteArrayInputStream(baos.toByteArray())), wrongMagic); assertThrows(PowsyblException.class, reader::readHeader); reader.close(); } diff --git a/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java new file mode 100644 index 00000000000..c2fd891c2e1 --- /dev/null +++ b/commons/src/test/java/com/powsybl/commons/binary/BufferedChannelReaderTest.java @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2025, RTE (http://www.rte-france.com) + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. + * SPDX-License-Identifier: MPL-2.0 + */ +package com.powsybl.commons.binary; + +import com.powsybl.commons.PowsyblException; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author Clement Leclerc {@literal } + */ +class BufferedChannelReaderTest { + + private static ReadableByteChannel readerOf(byte[] data) { + return Channels.newChannel(new ByteArrayInputStream(data)); + } + + @FunctionalInterface + private interface ByteSource { + void write(DataOutputStream dos) throws IOException; + } + + /** Produces bytes via {@link DataOutputStream} (big-endian) — matches the binary format wire layout. */ + private static byte[] bytes(ByteSource source) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (DataOutputStream dos = new DataOutputStream(baos)) { + source.write(dos); + } + return baos.toByteArray(); + } + + @Test + void endiannessIsBigEndian() { + // format guarantee: ByteBuffer must match DataOutputStream's network byte order + assertEquals(ByteOrder.BIG_ENDIAN, ByteBuffer.allocateDirect(8).order()); + } + + @Test + void readsAllPrimitives() throws Exception { + byte[] data = bytes(dos -> { + dos.writeByte(0x7F); + dos.writeShort(0xFEDC); + dos.writeInt(0xDEADBEEF); + dos.writeFloat(2.71f); + dos.writeDouble(3.14159265358979); + dos.writeBoolean(true); + dos.writeBoolean(false); + dos.write(new byte[] {1, 2, 3, 4, 5}); + }); + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(data))) { + assertEquals((byte) 0x7F, r.readByte()); + assertEquals(0xFEDC, r.readUnsignedShort()); + assertEquals(0xDEADBEEF, r.readInt()); + assertEquals(2.71f, r.readFloat(), 0f); + assertEquals(3.14159265358979, r.readDouble(), 0d); + assertTrue(r.readBoolean()); + assertFalse(r.readBoolean()); + assertArrayEquals(new byte[] {1, 2, 3, 4, 5}, r.readNBytes(5)); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readsCrossBufferBoundary() throws Exception { + // small buffer forces multiple channel refills inside a single readInt + byte[] data = bytes(dos -> { + for (int i = 0; i < 100; i++) { + dos.writeInt(i); + } + }); + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(data), 16)) { + for (int i = 0; i < 100; i++) { + assertEquals(i, r.readInt()); + } + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readNBytesAcrossManyRefills() throws Exception { + byte[] payload = new byte[1024]; + for (int i = 0; i < payload.length; i++) { + payload[i] = (byte) (i & 0xFF); + } + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(payload), 32)) { + assertArrayEquals(payload, r.readNBytes(payload.length)); + } + } + + @Test + void skipNBytes() throws Exception { + byte[] data = bytes(dos -> { + for (int i = 0; i < 50; i++) { + dos.writeInt(i); + } + }); + + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(data), 16)) { + r.skipNBytes(4L * 25); + assertEquals(25, r.readInt()); + r.skipNBytes(4L * 24); + assertTrue(r.isEndOfStream()); + } + } + + @Test + void isEndOfStreamWorksWithoutRead() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[0]))) { + assertTrue(r.isEndOfStream()); + } + } + + @Test + void readUnexpectedEofThrows() throws Exception { + try (BufferedChannelReader r = new BufferedChannelReader(readerOf(new byte[] {0x01, 0x02}))) { + assertThrows(PowsyblException.class, r::readInt); + } + } +} diff --git a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java index fe02e76c4c8..ba09f44354e 100644 --- a/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java +++ b/iidm/iidm-serde/src/main/java/com/powsybl/iidm/serde/NetworkSerDe.java @@ -47,6 +47,8 @@ import javax.xml.validation.SchemaFactory; import javax.xml.validation.Validator; import java.io.*; +import java.nio.channels.Channels; +import java.nio.channels.Pipe; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; @@ -319,7 +321,7 @@ private static JsonWriter createJsonWriter(OutputStream os, ExportOptions option } private static TreeDataWriter createBinWriter(OutputStream os, ExportOptions options) { - return new BinWriter(os, BIIDM_MAGIC_NUMBER, options.getVersion().toString(".")); + return new BinWriter(Channels.newChannel(os), BIIDM_MAGIC_NUMBER, options.getVersion().toString(".")); } private static void writeRootElement(Network n, NetworkSerializerContext context) { @@ -547,13 +549,22 @@ public static Anonymizer write(Network n, ExportOptions options, OutputStream os public static Anonymizer write(Network n, ExportOptions options, OutputStream os, ExtensionsSupplier extensionsSupplier) { try (TreeDataWriter writer = createTreeDataWriter(n, options, os, extensionsSupplier)) { - NetworkSerializerContext context = createContext(n, options, writer); - writer.setVersions(getExtensionVersions(n, options, extensionsSupplier)); - write(n, context, extensionsSupplier); - return context.getAnonymizer(); + return write(n, options, writer, extensionsSupplier); } } + /** Writes a network using a caller-provided {@link TreeDataWriter}. The caller owns its lifecycle. */ + public static Anonymizer write(Network n, ExportOptions options, TreeDataWriter writer) { + return write(n, options, writer, DefaultExtensionsSupplier.getInstance()); + } + + public static Anonymizer write(Network n, ExportOptions options, TreeDataWriter writer, ExtensionsSupplier extensionsSupplier) { + NetworkSerializerContext context = createContext(n, options, writer); + writer.setVersions(getExtensionVersions(n, options, extensionsSupplier)); + write(n, context, extensionsSupplier); + return context.getAnonymizer(); + } + /** * Return true if the given element has to be written in the given network, false otherwise */ @@ -587,6 +598,13 @@ public static Anonymizer write(Network n, OutputStream os) { } public static Anonymizer write(Network n, ExportOptions options, Path xmlFile) { + if (options.getFormat() == TreeDataFormat.BIN) { + try (BinWriter writer = new BinWriter(xmlFile, BIIDM_MAGIC_NUMBER, options.getVersion().toString("."))) { + return write(n, options, writer); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } try (OutputStream os = new BufferedOutputStream(Files.newOutputStream(xmlFile))) { return write(n, options, os); } catch (IOException e) { @@ -634,7 +652,7 @@ private static TreeDataReader createTreeDataReader(InputStream is, ImportOptions return switch (config.getFormat()) { case XML -> createXmlReader(is, config, extensionsSupplier); case JSON -> createJsonReader(is, config, extensionsSupplier); - case BIN -> new BinReader(is, BIIDM_MAGIC_NUMBER); + case BIN -> new BinReader(Channels.newChannel(is), BIIDM_MAGIC_NUMBER); }; } @@ -942,6 +960,13 @@ public static Network read(Path xmlFile, ImportOptions options) { } public static Network read(Path xmlFile, ImportOptions options, Anonymizer anonymizer, NetworkFactory networkFactory, ReportNode reportNode) { + if (options.getFormat() == TreeDataFormat.BIN) { + try (BinReader reader = new BinReader(xmlFile, BIIDM_MAGIC_NUMBER)) { + return read(reader, options, anonymizer, networkFactory, reportNode); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } try (InputStream is = Files.newInputStream(xmlFile)) { return read(is, options, anonymizer, networkFactory, reportNode); } catch (IOException e) { @@ -1067,22 +1092,33 @@ public static Network copy(Network network, NetworkFactory networkFactory, Execu Objects.requireNonNull(network); Objects.requireNonNull(networkFactory); Objects.requireNonNull(executor); - PipedOutputStream pos = new PipedOutputStream(); - try (InputStream is = new PipedInputStream(pos)) { + try { + Pipe pipe = Pipe.open(); + Pipe.SinkChannel sink = pipe.sink(); + Pipe.SourceChannel source = pipe.source(); + if (format == TreeDataFormat.BIN) { + ExportOptions exportOptions = new ExportOptions().setFormat(format); + executor.execute(() -> { + try (BinWriter writer = new BinWriter(sink, BIIDM_MAGIC_NUMBER, exportOptions.getVersion().toString("."))) { + write(network, exportOptions, writer); + } catch (Exception t) { + LOGGER.error(t.toString(), t); + } + }); + try (BinReader reader = new BinReader(source, BIIDM_MAGIC_NUMBER)) { + return read(reader, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); + } + } executor.execute(() -> { - try { - write(network, new ExportOptions().setFormat(format), pos); + try (OutputStream os = new BufferedOutputStream(Channels.newOutputStream(sink))) { + write(network, new ExportOptions().setFormat(format), os); } catch (Exception t) { LOGGER.error(t.toString(), t); - } finally { - try { - pos.close(); - } catch (IOException e) { - LOGGER.error(e.toString(), e); - } } }); - return read(is, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); + try (InputStream is = new BufferedInputStream(Channels.newInputStream(source))) { + return read(is, new ImportOptions().setFormat(format), null, networkFactory, ReportNode.NO_OP); + } } catch (IOException e) { throw new UncheckedIOException(e); }