Skip to content

Commit c72422b

Browse files
BrianMichellBrianMichell
authored
Merge pull request #160 from TGSAI/MetadataGetter
Fix missing units field and add convenience getter
2 parents a61699e + 49f334b commit c72422b

6 files changed

Lines changed: 243 additions & 24 deletions

File tree

‎mdio/dataset_test.cc‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1204,6 +1204,46 @@ TEST(Dataset, create) {
12041204
<< "Dataset successfully overwrote an existing dataset!";
12051205
}
12061206

1207+
TEST(Dataset, getVariableUnits) {
1208+
const std::string path = "zarrs/acceptance";
1209+
std::filesystem::remove_all("zarrs/acceptance");
1210+
auto json_vars = GetToyExample();
1211+
1212+
auto datasetRes =
1213+
mdio::Dataset::from_json(json_vars, path, mdio::constants::kCreateClean);
1214+
ASSERT_TRUE(datasetRes.status().ok()) << datasetRes.status();
1215+
auto dataset = datasetRes.value();
1216+
1217+
auto imageRes = dataset.variables.at("velocity");
1218+
ASSERT_TRUE(imageRes.ok()) << imageRes.status();
1219+
auto image = imageRes.value();
1220+
1221+
auto unitsRes = image.get_units();
1222+
ASSERT_TRUE(unitsRes.status().ok()) << unitsRes.status();
1223+
auto units = unitsRes.value();
1224+
EXPECT_EQ(units.get<std::string>(), mdio::units::kMetersPerSecond);
1225+
}
1226+
1227+
TEST(Dataset, getVariableUnitsError) {
1228+
const std::string path = "zarrs/acceptance";
1229+
std::filesystem::remove_all("zarrs/acceptance");
1230+
auto json_vars = GetToyExample();
1231+
1232+
auto datasetRes =
1233+
mdio::Dataset::from_json(json_vars, path, mdio::constants::kCreateClean);
1234+
ASSERT_TRUE(datasetRes.status().ok()) << datasetRes.status();
1235+
auto dataset = datasetRes.value();
1236+
1237+
auto imageRes = dataset.variables.at("image");
1238+
ASSERT_TRUE(imageRes.ok()) << imageRes.status();
1239+
auto image = imageRes.value();
1240+
1241+
auto unitsRes = image.get_units();
1242+
ASSERT_FALSE(unitsRes.status().ok()) << unitsRes.status();
1243+
EXPECT_EQ(unitsRes.status().message(),
1244+
"This Variable does not contain units");
1245+
}
1246+
12071247
TEST(Dataset, commitMetadata) {
12081248
const std::string path = "zarrs/acceptance";
12091249
std::filesystem::remove_all("zarrs/acceptance");

‎mdio/impl.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,48 @@ using byte_t = tensorstore::dtypes::byte_t;
7474
using bool_t = tensorstore::dtypes::bool_t;
7575
} // namespace dtypes
7676

77+
namespace units {
78+
// Angle units
79+
constexpr std::string_view kDegrees = "deg";
80+
constexpr std::string_view kRadians = "rad";
81+
82+
// Density units
83+
constexpr std::string_view kGramsPerCubicCentimeter = "g/cm**3";
84+
constexpr std::string_view kKilogramsPerCubicMeter = "kg/m**3";
85+
constexpr std::string_view kPoundsPerGallon = "lb/gal";
86+
87+
// Frequency units
88+
constexpr std::string_view kHertz = "Hz";
89+
90+
// Length units
91+
constexpr std::string_view kMillimeters = "mm";
92+
constexpr std::string_view kCentimeters = "cm";
93+
constexpr std::string_view kMeters = "m";
94+
constexpr std::string_view kKilometers = "km";
95+
constexpr std::string_view kInches = "in";
96+
constexpr std::string_view kFeet = "ft";
97+
constexpr std::string_view kYards = "yd";
98+
constexpr std::string_view kMiles = "mi";
99+
100+
// Speed units
101+
constexpr std::string_view kMetersPerSecond = "m/s";
102+
constexpr std::string_view kFeetPerSecond = "ft/s";
103+
104+
// Time units
105+
constexpr std::string_view kNanoseconds = "ns";
106+
constexpr std::string_view kMicroseconds = "µs";
107+
constexpr std::string_view kMilliseconds = "ms";
108+
constexpr std::string_view kSeconds = "s";
109+
constexpr std::string_view kMinutes = "min";
110+
constexpr std::string_view kHours = "h";
111+
constexpr std::string_view kDays = "d";
112+
113+
// Voltage units
114+
constexpr std::string_view kMicrovolts = "µV";
115+
constexpr std::string_view kMillivolts = "mV";
116+
constexpr std::string_view kVolts = "V";
117+
} // namespace units
118+
77119
// Special constants bleeds
78120
constexpr DimensionIndex dynamic_rank = tensorstore::dynamic_rank;
79121
constexpr ArrayOriginKind zero_origin = tensorstore::zero_origin;

‎mdio/stats.h‎

Lines changed: 96 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -352,7 +352,7 @@ class UserAttributes {
352352
* @endcode
353353
*/
354354
UserAttributes(const UserAttributes& other)
355-
: stats(other.stats), attrs(other.attrs) {}
355+
: stats(other.stats), units(other.units), attrs(other.attrs) {}
356356

357357
/**
358358
* @brief Constructs a UserAttributes object from a JSON representation of a
@@ -410,35 +410,60 @@ class UserAttributes {
410410
// Because the user can supply JSON here, there's a chance that the JSON is
411411
// malformed.
412412
try {
413-
if (j.contains("statsV1")) {
414-
auto statsJson = j["statsV1"];
413+
if (j.contains("statsV1") || j.contains("unitsV1")) {
415414
std::vector<internal::SummaryStats> statsCollection;
416-
if (statsJson.is_array()) {
417-
for (auto& s : statsJson) {
418-
auto statsRes = internal::SummaryStats::FromJson<T>(s);
415+
if (j.contains("statsV1")) {
416+
auto statsJson = j["statsV1"];
417+
if (statsJson.is_array()) {
418+
for (auto& s : statsJson) {
419+
auto statsRes = internal::SummaryStats::FromJson<T>(s);
420+
if (!statsRes.status().ok()) {
421+
return statsRes.status();
422+
}
423+
statsCollection.emplace_back(statsRes.value());
424+
}
425+
} else {
426+
auto statsRes = internal::SummaryStats::FromJson<T>(statsJson);
419427
if (!statsRes.status().ok()) {
420428
return statsRes.status();
421429
}
422430
statsCollection.emplace_back(statsRes.value());
423431
}
424-
} else {
425-
auto statsRes = internal::SummaryStats::FromJson<T>(statsJson);
426-
if (!statsRes.status().ok()) {
427-
return statsRes.status();
432+
}
433+
std::vector<std::string> unitsCollection;
434+
if (j.contains("unitsV1")) {
435+
auto unitsJson = j["unitsV1"];
436+
if (unitsJson.is_array()) {
437+
for (auto& s : unitsJson) {
438+
if (s.is_object()) {
439+
// If the element is an object, iterate its key-value pairs.
440+
for (auto& kv : s.items()) {
441+
unitsCollection.push_back(kv.value().get<std::string>());
442+
}
443+
} else {
444+
unitsCollection.push_back(s.get<std::string>());
445+
}
446+
}
447+
} else if (unitsJson.is_object()) {
448+
// If unitsV1 itself is an object, iterate its key-value pairs.
449+
for (auto& kv : unitsJson.items()) {
450+
unitsCollection.push_back(kv.value().get<std::string>());
451+
}
452+
} else {
453+
unitsCollection.push_back(unitsJson.get<std::string>());
428454
}
429-
statsCollection.emplace_back(statsRes.value());
430455
}
431456
auto attrs =
432-
UserAttributes(statsCollection, j.contains("attributes")
433-
? j["attributes"]
434-
: nlohmann::json::object());
435-
return attrs;
457+
UserAttributes(statsCollection, unitsCollection,
458+
j.contains("attributes") ? j["attributes"]
459+
: nlohmann::json::object());
460+
return mdio::Result<UserAttributes>(attrs);
436461
} else if (j.contains("attributes")) {
437462
auto attrs = UserAttributes(j["attributes"]);
438-
return attrs;
463+
return mdio::Result<UserAttributes>(attrs);
439464
}
440465
auto attrs = UserAttributes(nlohmann::json::object());
441-
return attrs;
466+
return mdio::Result<UserAttributes>(attrs);
442467
} catch (const nlohmann::json::exception& e) {
443468
return absl::InvalidArgumentError(
444469
"There appeared to be some malformed JSON" + std::string(e.what()));
@@ -454,6 +479,12 @@ class UserAttributes {
454479
*/
455480
const nlohmann::json getStatsV1() const { return statsBindable(); }
456481

482+
/**
483+
* @brief Extracts just the unitsV1 JSON
484+
* @return The unitsV1 JSON representation of the data
485+
*/
486+
const nlohmann::json getUnitsV1() const { return unitsBindable(); }
487+
457488
/**
458489
* @brief Extracts just the attributes JSON
459490
* @return The attributes JSON representation of the data
@@ -469,6 +500,9 @@ class UserAttributes {
469500
if (stats.size() >= 1) {
470501
j["statsV1"] = statsBindable();
471502
}
503+
if (units.size() >= 1) {
504+
j["unitsV1"] = unitsBindable();
505+
}
472506
auto attrs = attrsBindable();
473507
if (attrs.empty()) {
474508
return j;
@@ -485,7 +519,7 @@ class UserAttributes {
485519
* static member function `FromJson(nlohmann::json)`
486520
*/
487521
explicit UserAttributes(const nlohmann::json& attrs)
488-
: attrs(attrs), stats({}) {}
522+
: attrs(attrs), stats({}), units({}) {}
489523

490524
/**
491525
* @brief A case where there are statsV1 objects but no attributes
@@ -494,10 +528,33 @@ class UserAttributes {
494528
* @note This constructor is intended for internal use only. Please use the
495529
* static member function `FromJson(nlohmann::json)`
496530
*/
531+
UserAttributes(const std::vector<internal::SummaryStats>& stats,
532+
const nlohmann::json attrs)
533+
: stats(stats), units({}), attrs(attrs) {}
534+
535+
/**
536+
* @brief A case where there are unitsV1 objects but no attributes
537+
* @param units A collection of SummaryStats objects
538+
* @param attrs User specified attributes
539+
* @note This constructor is intended for internal use only. Please use the
540+
* static member function `FromJson(nlohmann::json)`
541+
*/
542+
UserAttributes(const std::vector<std::string>& units,
543+
const nlohmann::json attrs)
544+
: units(units), attrs(attrs) {}
497545

546+
/**
547+
* @brief A case where there are both statsV1 and unitsV1 objects
548+
* @param stats A collection of SummaryStats objects
549+
* @param units A collection of SummaryStats objects
550+
* @param attrs User specified attributes
551+
* @note This constructor is intended for internal use only. Please use the
552+
* static member function `FromJson(nlohmann::json)`
553+
*/
498554
UserAttributes(const std::vector<internal::SummaryStats>& stats,
555+
const std::vector<std::string>& units,
499556
const nlohmann::json attrs)
500-
: stats(stats), attrs(attrs) {}
557+
: stats(stats), units(units), attrs(attrs) {}
501558

502559
/**
503560
* @brief Binds the existing statsV1 data to a JSON object
@@ -516,6 +573,23 @@ class UserAttributes {
516573
return statsRet;
517574
}
518575

576+
/**
577+
* @brief Binds the existing unitsV1 data to a JSON object
578+
* @return A bindable unitsV1 JSON object
579+
*/
580+
const nlohmann::json unitsBindable() const {
581+
if (units.empty()) {
582+
return nlohmann::json::object();
583+
} else if (units.size() == 1) {
584+
return units[0];
585+
}
586+
nlohmann::json unitsRet = nlohmann::json::array();
587+
for (const auto& unit : units) {
588+
unitsRet.push_back(unit);
589+
}
590+
return unitsRet;
591+
}
592+
519593
/**
520594
* @brief Binds the existing attributes data to a JSON object
521595
* @return A bindable attributes JSON object
@@ -547,13 +621,14 @@ class UserAttributes {
547621
return true; // Assumption #1
548622
}
549623
}
550-
return false; // If we get here then we have only integers
624+
return false;
551625
}
552626
}
553-
return true; // We don't care, a histogram doesn't exist in this Variable
627+
return true; // Default to float if no histogram is provided
554628
}
555629

556630
std::vector<internal::SummaryStats> stats;
631+
std::vector<std::string> units;
557632
const nlohmann::json attrs;
558633
};
559634

‎mdio/stats_test.cc‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -454,4 +454,43 @@ TEST(UserAttributes, locationAndReassignment) {
454454
// auto newAttrs = std::move(attr)
455455
}
456456

457+
TEST(Units, unitsFromJsonObject) {
458+
// Test when unitsV1 is provided as an object.
459+
nlohmann::json json_input = {{"unitsV1", {{"length", "m"}}}};
460+
auto uaRes = mdio::UserAttributes::FromJson(json_input);
461+
ASSERT_TRUE(uaRes.status().ok()) << uaRes.status();
462+
mdio::UserAttributes ua = uaRes.value();
463+
// When an object is provided, the FromJson implementation pushes back the
464+
// unit value, so with one element it will return directly (not wrapped in an
465+
// array)
466+
nlohmann::json ua_json = ua.ToJson();
467+
EXPECT_TRUE(ua_json.contains("unitsV1"));
468+
EXPECT_EQ(ua_json["unitsV1"], "m");
469+
}
470+
471+
TEST(Units, unitsFromJsonArrayOfObjects) {
472+
// Test when unitsV1 is provided as an array of objects.
473+
nlohmann::json json_input = {
474+
{"unitsV1", {{{"length", "m"}}, {{"time", "s"}}}}};
475+
auto uaRes = mdio::UserAttributes::FromJson(json_input);
476+
ASSERT_TRUE(uaRes.status().ok()) << uaRes.status();
477+
mdio::UserAttributes ua = uaRes.value();
478+
nlohmann::json ua_json = ua.ToJson();
479+
EXPECT_TRUE(ua_json.contains("unitsV1"));
480+
// With more than one unit, the units-bindable returns an array.
481+
nlohmann::json expected = {"m", "s"};
482+
EXPECT_EQ(ua_json["unitsV1"], expected);
483+
}
484+
485+
TEST(Units, unitsFromJsonString) {
486+
// Test when unitsV1 is provided as a plain string.
487+
nlohmann::json json_input = {{"unitsV1", "rad"}};
488+
auto uaRes = mdio::UserAttributes::FromJson(json_input);
489+
ASSERT_TRUE(uaRes.status().ok()) << uaRes.status();
490+
mdio::UserAttributes ua = uaRes.value();
491+
nlohmann::json ua_json = ua.ToJson();
492+
EXPECT_TRUE(ua_json.contains("unitsV1"));
493+
EXPECT_EQ(ua_json["unitsV1"], "rad");
494+
}
495+
457496
} // namespace

‎mdio/variable.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1378,6 +1378,18 @@ class Variable {
13781378
return (*attributes)->ToJson();
13791379
}
13801380

1381+
Result<nlohmann::json> get_units() const {
1382+
auto attrs = GetAttributes();
1383+
1384+
// Return units if they exist and are non-null.
1385+
if (attrs.contains("unitsV1") && !attrs["unitsV1"].is_null()) {
1386+
return attrs["unitsV1"];
1387+
}
1388+
1389+
// Return an error if the units do not exist.
1390+
return absl::InvalidArgumentError("This Variable does not contain units");
1391+
}
1392+
13811393
/**
13821394
* @brief Gets the entire metadata of the Variable.
13831395
* Returned object is expected to have a parent key of "metadata".

‎mdio/variable_test.cc‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,11 +51,12 @@ ::nlohmann::json json_good = ::nlohmann::json::object({
5151
{"job status", "win"}, // can be anything
5252
{"project code", "fail"}
5353
}
54-
}}
54+
},
55+
{"unitsV1", {"m", "ft"}} // moved to be directly in metadata
56+
}
5557
},
5658
{"long_name", "foooooo ....."}, // required
5759
{"dimension_names", {"x", "y"} }, // required
58-
{"dimension_units", {"m", "ft"} }, // optional (if coord).
5960
}},
6061
{"metadata",
6162
{
@@ -78,7 +79,7 @@ ::nlohmann::json json_bad_1 = {
7879
}
7980
},
8081
{"attributes",
81-
{{"dimension_units", {"m", "ft"} }, // optional (if coord).
82+
{{"unitsV1", {"m", "ft"} }, // optional (if coord).
8283
{"metadata",
8384
{{"attributes", // optional misc attributes.
8485
{
@@ -733,6 +734,16 @@ TEST(Variable, userAttributes) {
733734
<< "An update to the UserAttributes was not detected";
734735
}
735736

737+
TEST(Variable, getUnitsPresent) {
738+
auto var1Res =
739+
mdio::Variable<>::Open(json_good, mdio::constants::kCreateClean);
740+
ASSERT_TRUE(var1Res.status().ok()) << var1Res.status();
741+
auto var1 = var1Res.value();
742+
743+
auto unitsRes = var1.get_units();
744+
ASSERT_TRUE(unitsRes.status().ok()) << unitsRes.status();
745+
}
746+
736747
// If a slice is "out of bounds" it should automatically get resized to the
737748
// proper bounds.
738749
TEST(Variable, outOfBoundsSlice) {

0 commit comments

Comments
 (0)