Skip to content

Commit a09fcae

Browse files
franzpoeschelfranzpoeschel
committed
Use UNDEFINED_EXTENT for this instead of empty extent
1 parent 7a60c30 commit a09fcae

8 files changed

Lines changed: 75 additions & 17 deletions

File tree

‎include/openPMD/Dataset.hpp‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,21 @@ class Dataset
5454
*/
5555
JOINED_DIMENSION = std::numeric_limits<std::uint64_t>::max(),
5656
/**
57-
* Some backends (i.e. JSON and TOML in template mode) support the
58-
* creation of dataset with undefined datatype and extent.
59-
* The extent should be given as {UNDEFINED_EXTENT} for that.
57+
* In some use cases, the extent needs not be specified.
58+
* For these, specify Extent{UNDEFINED_EXTENT}.
59+
* Use cases:
60+
*
61+
* 1. Some backends (i.e. JSON and TOML in template mode) support the
62+
* creation of dataset with undefined datatype and extent.
63+
* The extent should be given as {UNDEFINED_EXTENT} for that.
64+
* 2. With openPMD 2.0, the shape of constant components may be omitted
65+
* in writing if it is defined somewhere else as part
66+
* of the same Mesh / Species.
67+
* (https://github.com/openPMD/openPMD-standard/pull/289)
68+
* When reading such datasets, the openPMD-api will try to fill in
69+
* the missing extents, so the extent for constistently-defined
70+
* datasets should ideally not be reported by the read-side API
71+
* as undefined.
6072
*/
6173
UNDEFINED_EXTENT = std::numeric_limits<std::uint64_t>::max() - 1
6274
};
@@ -87,5 +99,8 @@ class Dataset
8799

88100
std::optional<size_t> joinedDimension() const;
89101
static std::optional<size_t> joinedDimension(Extent const &);
102+
103+
bool undefinedExtent() const;
104+
static bool undefinedExtent(Extent const &);
90105
};
91106
} // namespace openPMD

‎include/openPMD/IO/IOTask.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#include "openPMD/ChunkInfo.hpp"
2424
#include "openPMD/Dataset.hpp"
25+
#include "openPMD/Error.hpp"
2526
#include "openPMD/IterationEncoding.hpp"
2627
#include "openPMD/Streaming.hpp"
2728
#include "openPMD/auxiliary/Export.hpp"

‎include/openPMD/RecordComponent.hpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ class RecordComponent : public BaseRecordComponent
158158
* * Shrinking any dimension's extent.
159159
* * Changing the number of dimensions.
160160
*
161+
* The dataset extent may be empty to indicate undefined extents.
162+
*
161163
* Backend support for resizing datasets:
162164
* * JSON: Supported
163165
* * ADIOS2: Supported as of ADIOS2 2.7.0

‎src/Dataset.cpp‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,4 +95,13 @@ std::optional<size_t> Dataset::joinedDimension(Extent const &extent)
9595
}
9696
return res;
9797
}
98+
99+
bool Dataset::undefinedExtent() const
100+
{
101+
return undefinedExtent(extent);
102+
}
103+
bool Dataset::undefinedExtent(Extent const &e)
104+
{
105+
return e.size() == 1 && e.at(0) == Dataset::UNDEFINED_EXTENT;
106+
}
98107
} // namespace openPMD

‎src/IO/ADIOS/ADIOS2IOHandler.cpp‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -833,6 +833,12 @@ void ADIOS2IOHandlerImpl::createDataset(
833833
"only is not possible.");
834834
}
835835

836+
if (Dataset::undefinedExtent(parameters.extent))
837+
{
838+
throw error::OperationUnsupportedInBackend(
839+
"ADIOS2", "No support for Datasets with undefined extent.");
840+
}
841+
836842
if (!writable->written)
837843
{
838844
/* Sanitize name */
@@ -993,6 +999,12 @@ void ADIOS2IOHandlerImpl::extendDataset(
993999
VERIFY_ALWAYS(
9941000
access::write(m_handler->m_backendAccess),
9951001
"[ADIOS2] Cannot extend datasets in read-only mode.");
1002+
if (Dataset::undefinedExtent(parameters.extent))
1003+
{
1004+
throw error::OperationUnsupportedInBackend(
1005+
"ADIOS2", "No support for Datasets with undefined extent.");
1006+
}
1007+
9961008
setAndGetFilePosition(writable);
9971009
auto file = refreshFileFromParent(writable, /* preferParentFile = */ false);
9981010
std::string name = nameOfVariable(writable);

‎src/IO/HDF5/HDF5IOHandler.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -834,6 +834,11 @@ void HDF5IOHandlerImpl::createDataset(
834834
error::throwOperationUnsupportedInBackend(
835835
"HDF5", "Joined Arrays currently only supported in ADIOS2");
836836
}
837+
else if (Dataset::undefinedExtent(parameters.extent))
838+
{
839+
throw error::OperationUnsupportedInBackend(
840+
"HDF5", "No support for Datasets with undefined extent.");
841+
}
837842

838843
if (!writable->written)
839844
{
@@ -1114,6 +1119,11 @@ void HDF5IOHandlerImpl::extendDataset(
11141119
error::throwOperationUnsupportedInBackend(
11151120
"HDF5", "Joined Arrays currently only supported in ADIOS2");
11161121
}
1122+
else if (Dataset::undefinedExtent(parameters.extent))
1123+
{
1124+
throw error::OperationUnsupportedInBackend(
1125+
"HDF5", "No support for Datasets with undefined extent.");
1126+
}
11171127

11181128
File file =
11191129
requireFile("extendDataset", writable, /* checkParent = */ true);

‎src/RecordComponent.cpp‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ or alternatively the JSON option {"verify_homogeneous_extents": false}.
7777
Attributable const &callsite, RecordComponent &rc)
7878
{
7979
auto extent = rc.getExtent();
80-
if (extent.empty())
80+
if (Dataset::undefinedExtent(extent))
8181
{
8282
without_extent.emplace_back(rc);
8383
}
@@ -226,9 +226,8 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
226226
rc.m_hasBeenExtended = true;
227227
}
228228

229-
// @todo check this while flushing
230-
// if (d.extent.empty())
231-
// throw std::runtime_error("Dataset extent must be at least 1D.");
229+
if (d.extent.empty())
230+
throw std::runtime_error("Dataset extent must be at least 1D.");
232231
if (d.empty())
233232
{
234233
if (d.extent.empty())
@@ -291,7 +290,7 @@ Extent RecordComponent::getExtent() const
291290
}
292291
else
293292
{
294-
return {};
293+
return {Dataset::UNDEFINED_EXTENT};
295294
}
296295
}
297296

@@ -422,7 +421,7 @@ void RecordComponent::flush(
422421
}
423422
auto constant_component_write_shape = [&]() {
424423
auto extent = getExtent();
425-
return !extent.empty() &&
424+
return !Dataset::undefinedExtent(extent) &&
426425
std::none_of(extent.begin(), extent.end(), [](auto val) {
427426
return val == Dataset::JOINED_DIMENSION;
428427
});
@@ -561,7 +560,7 @@ void RecordComponent::readBase(bool require_unit_si)
561560
if (!containsAttribute("shape"))
562561
{
563562
setWritten(false, Attributable::EnqueueAsynchronously::No);
564-
resetDataset(Dataset(dtype, {}));
563+
resetDataset(Dataset(dtype, {Dataset::UNDEFINED_EXTENT}));
565564
setWritten(true, Attributable::EnqueueAsynchronously::No);
566565

567566
return;

‎test/Files_SerialIO/components_without_extent.cpp‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ void particle_offset_without_extent()
2727
position_comp.storeChunk(std::move(data), {0}, {5});
2828

2929
auto offset_comp = e["positionOffset"][comp_id];
30-
offset_comp.resetDataset({openPMD::Datatype::INT, {}});
30+
offset_comp.resetDataset(
31+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
3132
offset_comp.makeConstant(0);
3233
}
3334
write.close();
@@ -57,11 +58,13 @@ void particles_without_any_extent()
5758
for (auto comp_id : {"x", "y", "z"})
5859
{
5960
auto position_comp = e["position"][comp_id];
60-
position_comp.resetDataset({openPMD::Datatype::INT, {}});
61+
position_comp.resetDataset(
62+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
6163
position_comp.makeConstant(0);
6264

6365
auto offset_comp = e["positionOffset"][comp_id];
64-
offset_comp.resetDataset({openPMD::Datatype::INT, {}});
66+
offset_comp.resetDataset(
67+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
6568
offset_comp.makeConstant(0);
6669
}
6770
write.close();
@@ -84,7 +87,9 @@ void particles_without_any_extent()
8487
{
8588
for (auto const &component : record.second)
8689
{
87-
REQUIRE(component.second.getExtent() == openPMD::Extent{});
90+
REQUIRE(
91+
component.second.getExtent() ==
92+
openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT});
8893
}
8994
}
9095
}
@@ -152,7 +157,8 @@ void meshes_with_incomplete_extent()
152157
for (auto comp_id : {"y", "z"})
153158
{
154159
auto comp = E[comp_id];
155-
comp.resetDataset({openPMD::Datatype::INT, {}});
160+
comp.resetDataset(
161+
{openPMD::Datatype::INT, {openPMD::Dataset::UNDEFINED_EXTENT}});
156162
comp.makeConstant(0);
157163
}
158164
write.close();
@@ -220,7 +226,9 @@ void meshes_without_any_extent()
220226
for (auto comp_id : {"x", "y", "z"})
221227
{
222228
auto comp = E[comp_id];
223-
comp.resetDataset({openPMD::Datatype::FLOAT, {}});
229+
comp.resetDataset(
230+
{openPMD::Datatype::FLOAT,
231+
{openPMD::Dataset::UNDEFINED_EXTENT}});
224232
comp.makeConstant<float>(0);
225233
}
226234
write.close();
@@ -241,7 +249,9 @@ void meshes_without_any_extent()
241249
auto E = read.snapshots()[0].meshes["E"];
242250
for (auto const &component : E)
243251
{
244-
REQUIRE(component.second.getExtent() == openPMD::Extent{});
252+
REQUIRE(
253+
component.second.getExtent() ==
254+
openPMD::Extent{openPMD::Dataset::UNDEFINED_EXTENT});
245255
}
246256
}
247257
}

0 commit comments

Comments
 (0)