diff --git a/mdio/dataset_test.cc b/mdio/dataset_test.cc index d30548d..2ddbc41 100644 --- a/mdio/dataset_test.cc +++ b/mdio/dataset_test.cc @@ -1204,6 +1204,46 @@ TEST(Dataset, create) { << "Dataset successfully overwrote an existing dataset!"; } +TEST(Dataset, getVariableUnits) { + const std::string path = "zarrs/acceptance"; + std::filesystem::remove_all("zarrs/acceptance"); + auto json_vars = GetToyExample(); + + auto datasetRes = + mdio::Dataset::from_json(json_vars, path, mdio::constants::kCreateClean); + ASSERT_TRUE(datasetRes.status().ok()) << datasetRes.status(); + auto dataset = datasetRes.value(); + + auto imageRes = dataset.variables.at("velocity"); + ASSERT_TRUE(imageRes.ok()) << imageRes.status(); + auto image = imageRes.value(); + + auto unitsRes = image.get_units(); + ASSERT_TRUE(unitsRes.status().ok()) << unitsRes.status(); + auto units = unitsRes.value(); + EXPECT_EQ(units.get(), mdio::units::kMetersPerSecond); +} + +TEST(Dataset, getVariableUnitsError) { + const std::string path = "zarrs/acceptance"; + std::filesystem::remove_all("zarrs/acceptance"); + auto json_vars = GetToyExample(); + + auto datasetRes = + mdio::Dataset::from_json(json_vars, path, mdio::constants::kCreateClean); + ASSERT_TRUE(datasetRes.status().ok()) << datasetRes.status(); + auto dataset = datasetRes.value(); + + auto imageRes = dataset.variables.at("image"); + ASSERT_TRUE(imageRes.ok()) << imageRes.status(); + auto image = imageRes.value(); + + auto unitsRes = image.get_units(); + ASSERT_FALSE(unitsRes.status().ok()) << unitsRes.status(); + EXPECT_EQ(unitsRes.status().message(), + "This Variable does not contain units"); +} + TEST(Dataset, commitMetadata) { const std::string path = "zarrs/acceptance"; std::filesystem::remove_all("zarrs/acceptance"); diff --git a/mdio/impl.h b/mdio/impl.h index 5607f11..c98e4ea 100644 --- a/mdio/impl.h +++ b/mdio/impl.h @@ -74,6 +74,48 @@ using byte_t = tensorstore::dtypes::byte_t; using bool_t = tensorstore::dtypes::bool_t; } // namespace dtypes +namespace units { +// Angle units +constexpr std::string_view kDegrees = "deg"; +constexpr std::string_view kRadians = "rad"; + +// Density units +constexpr std::string_view kGramsPerCubicCentimeter = "g/cm**3"; +constexpr std::string_view kKilogramsPerCubicMeter = "kg/m**3"; +constexpr std::string_view kPoundsPerGallon = "lb/gal"; + +// Frequency units +constexpr std::string_view kHertz = "Hz"; + +// Length units +constexpr std::string_view kMillimeters = "mm"; +constexpr std::string_view kCentimeters = "cm"; +constexpr std::string_view kMeters = "m"; +constexpr std::string_view kKilometers = "km"; +constexpr std::string_view kInches = "in"; +constexpr std::string_view kFeet = "ft"; +constexpr std::string_view kYards = "yd"; +constexpr std::string_view kMiles = "mi"; + +// Speed units +constexpr std::string_view kMetersPerSecond = "m/s"; +constexpr std::string_view kFeetPerSecond = "ft/s"; + +// Time units +constexpr std::string_view kNanoseconds = "ns"; +constexpr std::string_view kMicroseconds = "µs"; +constexpr std::string_view kMilliseconds = "ms"; +constexpr std::string_view kSeconds = "s"; +constexpr std::string_view kMinutes = "min"; +constexpr std::string_view kHours = "h"; +constexpr std::string_view kDays = "d"; + +// Voltage units +constexpr std::string_view kMicrovolts = "µV"; +constexpr std::string_view kMillivolts = "mV"; +constexpr std::string_view kVolts = "V"; +} // namespace units + // Special constants bleeds constexpr DimensionIndex dynamic_rank = tensorstore::dynamic_rank; constexpr ArrayOriginKind zero_origin = tensorstore::zero_origin; diff --git a/mdio/stats.h b/mdio/stats.h index c1b1179..d2d27a5 100644 --- a/mdio/stats.h +++ b/mdio/stats.h @@ -352,7 +352,7 @@ class UserAttributes { * @endcode */ UserAttributes(const UserAttributes& other) - : stats(other.stats), attrs(other.attrs) {} + : stats(other.stats), units(other.units), attrs(other.attrs) {} /** * @brief Constructs a UserAttributes object from a JSON representation of a @@ -410,35 +410,60 @@ class UserAttributes { // Because the user can supply JSON here, there's a chance that the JSON is // malformed. try { - if (j.contains("statsV1")) { - auto statsJson = j["statsV1"]; + if (j.contains("statsV1") || j.contains("unitsV1")) { std::vector statsCollection; - if (statsJson.is_array()) { - for (auto& s : statsJson) { - auto statsRes = internal::SummaryStats::FromJson(s); + if (j.contains("statsV1")) { + auto statsJson = j["statsV1"]; + if (statsJson.is_array()) { + for (auto& s : statsJson) { + auto statsRes = internal::SummaryStats::FromJson(s); + if (!statsRes.status().ok()) { + return statsRes.status(); + } + statsCollection.emplace_back(statsRes.value()); + } + } else { + auto statsRes = internal::SummaryStats::FromJson(statsJson); if (!statsRes.status().ok()) { return statsRes.status(); } statsCollection.emplace_back(statsRes.value()); } - } else { - auto statsRes = internal::SummaryStats::FromJson(statsJson); - if (!statsRes.status().ok()) { - return statsRes.status(); + } + std::vector unitsCollection; + if (j.contains("unitsV1")) { + auto unitsJson = j["unitsV1"]; + if (unitsJson.is_array()) { + for (auto& s : unitsJson) { + if (s.is_object()) { + // If the element is an object, iterate its key-value pairs. + for (auto& kv : s.items()) { + unitsCollection.push_back(kv.value().get()); + } + } else { + unitsCollection.push_back(s.get()); + } + } + } else if (unitsJson.is_object()) { + // If unitsV1 itself is an object, iterate its key-value pairs. + for (auto& kv : unitsJson.items()) { + unitsCollection.push_back(kv.value().get()); + } + } else { + unitsCollection.push_back(unitsJson.get()); } - statsCollection.emplace_back(statsRes.value()); } auto attrs = - UserAttributes(statsCollection, j.contains("attributes") - ? j["attributes"] - : nlohmann::json::object()); - return attrs; + UserAttributes(statsCollection, unitsCollection, + j.contains("attributes") ? j["attributes"] + : nlohmann::json::object()); + return mdio::Result(attrs); } else if (j.contains("attributes")) { auto attrs = UserAttributes(j["attributes"]); - return attrs; + return mdio::Result(attrs); } auto attrs = UserAttributes(nlohmann::json::object()); - return attrs; + return mdio::Result(attrs); } catch (const nlohmann::json::exception& e) { return absl::InvalidArgumentError( "There appeared to be some malformed JSON" + std::string(e.what())); @@ -454,6 +479,12 @@ class UserAttributes { */ const nlohmann::json getStatsV1() const { return statsBindable(); } + /** + * @brief Extracts just the unitsV1 JSON + * @return The unitsV1 JSON representation of the data + */ + const nlohmann::json getUnitsV1() const { return unitsBindable(); } + /** * @brief Extracts just the attributes JSON * @return The attributes JSON representation of the data @@ -469,6 +500,9 @@ class UserAttributes { if (stats.size() >= 1) { j["statsV1"] = statsBindable(); } + if (units.size() >= 1) { + j["unitsV1"] = unitsBindable(); + } auto attrs = attrsBindable(); if (attrs.empty()) { return j; @@ -485,7 +519,7 @@ class UserAttributes { * static member function `FromJson(nlohmann::json)` */ explicit UserAttributes(const nlohmann::json& attrs) - : attrs(attrs), stats({}) {} + : attrs(attrs), stats({}), units({}) {} /** * @brief A case where there are statsV1 objects but no attributes @@ -494,10 +528,33 @@ class UserAttributes { * @note This constructor is intended for internal use only. Please use the * static member function `FromJson(nlohmann::json)` */ + UserAttributes(const std::vector& stats, + const nlohmann::json attrs) + : stats(stats), units({}), attrs(attrs) {} + + /** + * @brief A case where there are unitsV1 objects but no attributes + * @param units A collection of SummaryStats objects + * @param attrs User specified attributes + * @note This constructor is intended for internal use only. Please use the + * static member function `FromJson(nlohmann::json)` + */ + UserAttributes(const std::vector& units, + const nlohmann::json attrs) + : units(units), attrs(attrs) {} + /** + * @brief A case where there are both statsV1 and unitsV1 objects + * @param stats A collection of SummaryStats objects + * @param units A collection of SummaryStats objects + * @param attrs User specified attributes + * @note This constructor is intended for internal use only. Please use the + * static member function `FromJson(nlohmann::json)` + */ UserAttributes(const std::vector& stats, + const std::vector& units, const nlohmann::json attrs) - : stats(stats), attrs(attrs) {} + : stats(stats), units(units), attrs(attrs) {} /** * @brief Binds the existing statsV1 data to a JSON object @@ -516,6 +573,23 @@ class UserAttributes { return statsRet; } + /** + * @brief Binds the existing unitsV1 data to a JSON object + * @return A bindable unitsV1 JSON object + */ + const nlohmann::json unitsBindable() const { + if (units.empty()) { + return nlohmann::json::object(); + } else if (units.size() == 1) { + return units[0]; + } + nlohmann::json unitsRet = nlohmann::json::array(); + for (const auto& unit : units) { + unitsRet.push_back(unit); + } + return unitsRet; + } + /** * @brief Binds the existing attributes data to a JSON object * @return A bindable attributes JSON object @@ -547,13 +621,14 @@ class UserAttributes { return true; // Assumption #1 } } - return false; // If we get here then we have only integers + return false; } } - return true; // We don't care, a histogram doesn't exist in this Variable + return true; // Default to float if no histogram is provided } std::vector stats; + std::vector units; const nlohmann::json attrs; }; diff --git a/mdio/stats_test.cc b/mdio/stats_test.cc index 334798d..468c874 100644 --- a/mdio/stats_test.cc +++ b/mdio/stats_test.cc @@ -454,4 +454,43 @@ TEST(UserAttributes, locationAndReassignment) { // auto newAttrs = std::move(attr) } +TEST(Units, unitsFromJsonObject) { + // Test when unitsV1 is provided as an object. + nlohmann::json json_input = {{"unitsV1", {{"length", "m"}}}}; + auto uaRes = mdio::UserAttributes::FromJson(json_input); + ASSERT_TRUE(uaRes.status().ok()) << uaRes.status(); + mdio::UserAttributes ua = uaRes.value(); + // When an object is provided, the FromJson implementation pushes back the + // unit value, so with one element it will return directly (not wrapped in an + // array) + nlohmann::json ua_json = ua.ToJson(); + EXPECT_TRUE(ua_json.contains("unitsV1")); + EXPECT_EQ(ua_json["unitsV1"], "m"); +} + +TEST(Units, unitsFromJsonArrayOfObjects) { + // Test when unitsV1 is provided as an array of objects. + nlohmann::json json_input = { + {"unitsV1", {{{"length", "m"}}, {{"time", "s"}}}}}; + auto uaRes = mdio::UserAttributes::FromJson(json_input); + ASSERT_TRUE(uaRes.status().ok()) << uaRes.status(); + mdio::UserAttributes ua = uaRes.value(); + nlohmann::json ua_json = ua.ToJson(); + EXPECT_TRUE(ua_json.contains("unitsV1")); + // With more than one unit, the units-bindable returns an array. + nlohmann::json expected = {"m", "s"}; + EXPECT_EQ(ua_json["unitsV1"], expected); +} + +TEST(Units, unitsFromJsonString) { + // Test when unitsV1 is provided as a plain string. + nlohmann::json json_input = {{"unitsV1", "rad"}}; + auto uaRes = mdio::UserAttributes::FromJson(json_input); + ASSERT_TRUE(uaRes.status().ok()) << uaRes.status(); + mdio::UserAttributes ua = uaRes.value(); + nlohmann::json ua_json = ua.ToJson(); + EXPECT_TRUE(ua_json.contains("unitsV1")); + EXPECT_EQ(ua_json["unitsV1"], "rad"); +} + } // namespace diff --git a/mdio/variable.h b/mdio/variable.h index 14cad32..3a28790 100644 --- a/mdio/variable.h +++ b/mdio/variable.h @@ -1378,6 +1378,18 @@ class Variable { return (*attributes)->ToJson(); } + Result get_units() const { + auto attrs = GetAttributes(); + + // Return units if they exist and are non-null. + if (attrs.contains("unitsV1") && !attrs["unitsV1"].is_null()) { + return attrs["unitsV1"]; + } + + // Return an error if the units do not exist. + return absl::InvalidArgumentError("This Variable does not contain units"); + } + /** * @brief Gets the entire metadata of the Variable. * Returned object is expected to have a parent key of "metadata". diff --git a/mdio/variable_test.cc b/mdio/variable_test.cc index df4ccc6..79cfeb8 100644 --- a/mdio/variable_test.cc +++ b/mdio/variable_test.cc @@ -51,11 +51,12 @@ ::nlohmann::json json_good = ::nlohmann::json::object({ {"job status", "win"}, // can be anything {"project code", "fail"} } - }} + }, + {"unitsV1", {"m", "ft"}} // moved to be directly in metadata + } }, {"long_name", "foooooo ....."}, // required {"dimension_names", {"x", "y"} }, // required - {"dimension_units", {"m", "ft"} }, // optional (if coord). }}, {"metadata", { @@ -78,7 +79,7 @@ ::nlohmann::json json_bad_1 = { } }, {"attributes", - {{"dimension_units", {"m", "ft"} }, // optional (if coord). + {{"unitsV1", {"m", "ft"} }, // optional (if coord). {"metadata", {{"attributes", // optional misc attributes. { @@ -733,6 +734,16 @@ TEST(Variable, userAttributes) { << "An update to the UserAttributes was not detected"; } +TEST(Variable, getUnitsPresent) { + auto var1Res = + mdio::Variable<>::Open(json_good, mdio::constants::kCreateClean); + ASSERT_TRUE(var1Res.status().ok()) << var1Res.status(); + auto var1 = var1Res.value(); + + auto unitsRes = var1.get_units(); + ASSERT_TRUE(unitsRes.status().ok()) << unitsRes.status(); +} + // If a slice is "out of bounds" it should automatically get resized to the // proper bounds. TEST(Variable, outOfBoundsSlice) {