From 10ff92c6f470ec66335e58966291dce4e9bbeeb6 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Wed, 17 Aug 2022 10:50:29 -0300 Subject: [PATCH 01/10] Add CSV data parsing (#402) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michel Hidalgo Co-authored-by: Carlos Agüero --- CMakeLists.txt | 3 +- io/BUILD.bazel | 76 ++++++++ io/include/CMakeLists.txt | 1 + io/include/gz/CMakeLists.txt | 1 + io/include/gz/common/CMakeLists.txt | 1 + io/include/gz/common/CSVStreams.hh | 177 +++++++++++++++++++ io/include/gz/common/DataFrame.hh | 210 ++++++++++++++++++++++ io/include/gz/common/Io.hh | 85 +++++++++ io/src/CMakeLists.txt | 15 ++ io/src/CSVStreams.cc | 262 ++++++++++++++++++++++++++++ io/src/CSVStreams_TEST.cc | 192 ++++++++++++++++++++ io/src/DataFrame_TEST.cc | 90 ++++++++++ src/CMakeLists.txt | 5 + 13 files changed, 1116 insertions(+), 2 deletions(-) create mode 100644 io/BUILD.bazel create mode 100644 io/include/CMakeLists.txt create mode 100644 io/include/gz/CMakeLists.txt create mode 100644 io/include/gz/common/CMakeLists.txt create mode 100644 io/include/gz/common/CSVStreams.hh create mode 100644 io/include/gz/common/DataFrame.hh create mode 100644 io/include/gz/common/Io.hh create mode 100644 io/src/CMakeLists.txt create mode 100644 io/src/CSVStreams.cc create mode 100644 io/src/CSVStreams_TEST.cc create mode 100644 io/src/DataFrame_TEST.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 019f921e4..f16df76aa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -127,7 +127,6 @@ gz_find_package(AVUTIL REQUIRED_BY av PRETTY libavutil) # Find assimp gz_find_package(GzAssimp REQUIRED_BY graphics PRETTY assimp) - message(STATUS "-------------------------------------------\n") @@ -138,7 +137,7 @@ configure_file("${PROJECT_SOURCE_DIR}/cppcheck.suppress.in" ${PROJECT_BINARY_DIR}/cppcheck.suppress) gz_configure_build(QUIT_IF_BUILD_ERRORS - COMPONENTS av events geospatial graphics profiler testing) + COMPONENTS av events geospatial graphics io profiler testing) #============================================================================ # Create package information diff --git a/io/BUILD.bazel b/io/BUILD.bazel new file mode 100644 index 000000000..89a00dbed --- /dev/null +++ b/io/BUILD.bazel @@ -0,0 +1,76 @@ +load( + "//gz_bazel:build_defs.bzl", + "GZ_ROOT", + "GZ_VISIBILITY", + "generate_include_header", + "gz_export_header", +) + +package( + default_visibility = GZ_VISIBILITY, + features = [ + "-parse_headers", + "-layering_check", + ], +) + +public_headers_no_gen = glob([ + "include/gz/common/*.hh", +]) + +sources = glob( + ["src/*.cc"], + exclude = ["src/*_TEST.cc"], +) + +test_sources = glob(["src/*_TEST.cc"]) + +gz_export_header( + name = "include/gz/common/io/Export.hh", + export_base = "GZ_COMMON_IO", + lib_name = "gz-common-io", + visibility = ["//visibility:private"], +) + +generate_include_header( + name = "iohh_genrule", + out = "include/gz/common/io.hh", + hdrs = public_headers_no_gen + [ + "include/gz/common/io/Export.hh", + ], +) + +public_headers = public_headers_no_gen + [ + "include/gz/common/io/Export.hh", + "include/gz/common/io.hh", +] + +cc_library( + name = "io", + srcs = sources, + hdrs = public_headers, + includes = ["include"], + deps = [ + GZ_ROOT + "gz_common", + GZ_ROOT + "gz_math", + ], +) + +cc_binary( + name = "libgz-common5-io.so", + includes = ["include"], + linkopts = ["-Wl,-soname,libgz-common5-io.so"], + linkshared = True, + deps = [":events"], +) + +[cc_test( + name = src.replace("/", "_").replace(".cc", "").replace("src_", ""), + srcs = [src], + deps = [ + ":io", + GZ_ROOT + "gz_common/test:test_utils", + "@gtest", + "@gtest//:gtest_main", + ], +) for src in test_sources] diff --git a/io/include/CMakeLists.txt b/io/include/CMakeLists.txt new file mode 100644 index 000000000..a35a0475e --- /dev/null +++ b/io/include/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(gz) diff --git a/io/include/gz/CMakeLists.txt b/io/include/gz/CMakeLists.txt new file mode 100644 index 000000000..e4717b2d6 --- /dev/null +++ b/io/include/gz/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(common) diff --git a/io/include/gz/common/CMakeLists.txt b/io/include/gz/common/CMakeLists.txt new file mode 100644 index 000000000..82eb256f2 --- /dev/null +++ b/io/include/gz/common/CMakeLists.txt @@ -0,0 +1 @@ +gz_install_all_headers(COMPONENT io) diff --git a/io/include/gz/common/CSVStreams.hh b/io/include/gz/common/CSVStreams.hh new file mode 100644 index 000000000..58d3ecb48 --- /dev/null +++ b/io/include/gz/common/CSVStreams.hh @@ -0,0 +1,177 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#ifndef GZ_COMMON_CSVSTREAMS_HH_ +#define GZ_COMMON_CSVSTREAMS_HH_ + +#include +#include +#include + +#include + +#include + +namespace gz +{ + namespace common + { + /// \brief A CSV specification. + struct GZ_COMMON_IO_VISIBLE CSVDialect + { + /// Field delimiter character. + char delimiter; + + /// Row termination character. + char terminator; + + /// Field quoting character. + char quote; + + /// CSV dialect as expected by Unix tools. + static const CSVDialect Unix; + }; + + /// \brief Check CSV dialects for equality. + /// \param[in] _lhs Left-hand side CSV dialect. + /// \param[in] _rhs Right-hand side CSV dialect. + /// \return true if CSV dialects are equal, false otherwise. + bool GZ_COMMON_IO_VISIBLE operator==(const CSVDialect &_lhs, + const CSVDialect &_rhs); + + /// \brief A token in CSV data. + /// + /// Lexical specifications are typically dictated by a CSV dialect. + struct CSVToken + { + /// Token type. + enum { + TEXT = 0, ///< A pure text token (e.g. a letter). + QUOTE, ///< A field quoting token (e.g. a double quote). + DELIMITER, ///< A field delimiter token (e.g. a comma). + TERMINATOR ///< A row termination token (e.g. a newline). + } type; + + /// Token character. + char character; + }; + + /// \brief Extract a single token from an input stream of CSV data. + /// + /// If tokenization fails, the CSV data stream ``failbit`` will be set. + /// + /// \param[in] _stream A stream of CSV data to tokenize. + /// \param[out] _token Output CSV token to extract into. + /// \param[in] _dialect CSV data dialect. Defaults to the Unix dialect. + /// \return same CSV data stream. + GZ_COMMON_IO_VISIBLE std::istream &ExtractCSVToken( + std::istream &_stream, CSVToken &_token, + const CSVDialect &_dialect = CSVDialect::Unix); + + /// \brief Parse a single row from an input stream of CSV data. + /// + /// If parsing fails, the CSV data stream ``failbit`` will be set. + /// + /// \param[in] _stream CSV data stream to parse. + /// \param[out] _row Output CSV row to parse into. + /// \param[in] _dialect CSV data dialect. Defaults to the Unix dialect. + /// \returns same CSV data stream. + GZ_COMMON_IO_VISIBLE std::istream &ParseCSVRow( + std::istream &_stream, std::vector &_row, + const CSVDialect &_dialect = CSVDialect::Unix); + + /// \brief A single-pass row iterator on an input stream of CSV data. + /// + /// Similar to std::istream_iterator, this iterator parses a stream of + /// CSV data, one row at a time. \see `ParseCSVRow`. + class GZ_COMMON_IO_VISIBLE CSVIStreamIterator + { + public: using iterator_category = std::input_iterator_tag; + public: using value_type = std::vector; + public: using difference_type = std::ptrdiff_t; + public: using pointer = const value_type*; + public: using reference = const value_type&; + + /// \brief Construct an end-of-stream iterator. + public: CSVIStreamIterator(); + + /// \brief Construct an iterator over `_stream`. + /// + /// The first row will be read from the underlying stream to + /// initialize the iterator. If there are parsing errors while + /// reading, the underlying stream ``failbit`` will be set. + /// + /// \param[in] _stream A stream of CSV data to iterate. + /// \param[in] _dialect CSV data dialect. Defaults to the Unix dialect. + public: explicit CSVIStreamIterator( + std::istream &_stream, + const CSVDialect &_dialect = CSVDialect::Unix); + + /// \brief Read the next row from the underlying stream. + /// + /// If the read fails, the iterator becomes an end-of-stream iterator. + /// If there are parsing errors while reading, the underlying stream + /// ``failbit`` will be set. If the iterator already is an end-of-stream + /// iterator, behavior is undefined. + /// + /// \return A reference to the iterator once modified. + public: CSVIStreamIterator &operator++(); + + /// \brief Read the next row from the underlying stream. + /// + /// If the read fails, the iterator becomes an end-of-stream iterator. + /// If there are parsing errors while reading, the underlying stream + /// ``failbit`` will be set. If the iterator already is an end-of-stream + /// iterator, behavior is undefined. + /// + /// \return A copy of the iterator before modification. Note that, + /// while an iterator copy retains its state, the underlying stream + /// may still be advanced. + public: CSVIStreamIterator operator++(int); + + /// \brief Check for iterator equality. + /// \param[in] _other Iterator to compare with. + /// \return true if both iterators are end-of-stream iterators + /// or if both iterator wrap the same stream and use the same dialect, + /// false otherwise. + public: bool operator==(const CSVIStreamIterator &_other) const; + + /// \brief Check for iterator inequality. + /// \param[in] _other Iterator to compare with. + /// \return true if both iterators are not equal, false otherwise. + public: bool operator!=(const CSVIStreamIterator &_other) const; + + /// \brief Access current row. + /// + /// Behavior is undefined if the iterator is an end-of-stream iterator. + /// + /// \return reference to the current row. + public: reference operator*() const; + + /// \brief Access current row. + /// + /// Behavior is undefined if the iterator is an end-of-stream iterator. + /// + /// \return pointer to the current row. + public: pointer operator->() const; + + /// \brief Pointer to private data. + private: GZ_UTILS_IMPL_PTR(dataPtr) + }; + } +} + +#endif diff --git a/io/include/gz/common/DataFrame.hh b/io/include/gz/common/DataFrame.hh new file mode 100644 index 000000000..b5a124dc6 --- /dev/null +++ b/io/include/gz/common/DataFrame.hh @@ -0,0 +1,210 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#ifndef GZ_COMMON_DATAFRAME_HH_ +#define GZ_COMMON_DATAFRAME_HH_ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include + +namespace gz +{ + namespace common + { + /// \brief An abstract data frame. + /// + /// \tparam K Column key type + /// \tparam V Column value type + template + class DataFrame + { + /// \brief Check if column is present. + /// \param[in] _key Key to column to look up. + /// \return whether the given column is present + /// in the data frame. + public: bool Has(const K &_key) const + { + return this->storage.count(_key) > 0; + } + + /// \brief Fetch mutable reference to column. + /// \param[in] _key Key to column to look up. + /// \return Mutable reference to column in the + /// data frame. + public: V &operator[](const K &_key) + { + return this->storage[_key]; + } + + /// \brief Fetch immutable reference to column + /// \param[in] _key Key to column to look up. + /// \return Immutable reference to column in the + /// data frame. + public: const V &operator[](const K &_key) const + { + return this->storage.at(_key); + } + + /// \brief Data frame storage + private: std::unordered_map storage; + }; + + /// \brief Traits for IO of data frames comprised of time varying volumetric grids. + /// + /// \tparam K Data frame key type. + /// \tparam T Time coordinate type. + /// \tparam V Grid value type. + /// \tparam P Spatial dimensions type. + template + struct IO>> + { + /// \brief Read data frame from CSV data stream. + /// + /// \param[in] _begin Beginning-of-stream iterator to CSV data stream. + /// \param[in] _end End-of-stream iterator to CSV data stream. + /// \param[in] _timeColumnName CSV data column name to use as time + /// dimension. + /// \param[in] _spatialColumnNames CSV data columns' names to use + /// as spatial (x, y, z) dimensions, in that order. + /// \throws std::invalid_argument if the CSV data stream is empty, or + /// if the CSV data stream has no header, or if the given columns + /// cannot be found in the CSV data stream header. + /// \return data frame read. + static DataFrame> + ReadFrom(CSVIStreamIterator _begin, + CSVIStreamIterator _end, + const std::string &_timeColumnName, + const std::array &_spatialColumnNames) + { + if (_begin == _end) + { + throw std::invalid_argument("CSV data stream is empty"); + } + const std::vector &header = *_begin; + if (header.empty()) + { + throw std::invalid_argument("CSV data stream has no header"); + } + + auto it = std::find(header.begin(), header.end(), _timeColumnName); + if (it == header.end()) + { + std::stringstream sstream; + sstream << "CSV data stream has no '" + << _timeColumnName << "' column"; + throw std::invalid_argument(sstream.str()); + } + const size_t timeIndex = it - header.begin(); + + std::array spatialColumnIndices; + for (size_t i = 0; i < _spatialColumnNames.size(); ++i) + { + it = std::find(header.begin(), header.end(), _spatialColumnNames[i]); + if (it == header.end()) + { + std::stringstream sstream; + sstream << "CSV data stream has no '" + << _spatialColumnNames[i] << "' column"; + throw std::invalid_argument(sstream.str()); + } + spatialColumnIndices[i] = it - header.begin(); + } + + return ReadFrom(_begin, _end, timeIndex, spatialColumnIndices); + } + + /// \brief Read data frame from CSV data stream. + /// + /// \param[in] _begin Beginning-of-stream iterator to CSV data stream. + /// \param[in] _end End-of-stream iterator to CSV data stream. + /// \param[in] _timeColumnIndex CSV data column index to use as + /// time dimension. + /// \param[in] _spatialColumnIndices CSV data columns indices + /// to use as spatial (x, y, z) dimensions, in that order. + /// \throws std::invalid_argument if the CSV data stream is empty, or + /// if the CSV data stream has no header, or if the given columns + /// cannot be found in the CSV data stream header. + /// \return data frame read. + static DataFrame> + ReadFrom(CSVIStreamIterator _begin, + CSVIStreamIterator _end, + const size_t &_timeColumnIndex = 0, + const std::array &_spatialColumnIndices = {1, 2, 3}) + { + if (_begin == _end) + { + throw std::invalid_argument("CSV data stream is empty"); + } + std::vector dataColumnIndices(_begin->size()); + std::iota(dataColumnIndices.begin(), dataColumnIndices.end(), 0); + auto last = dataColumnIndices.end(); + for (size_t index : {_timeColumnIndex, _spatialColumnIndices[0], + _spatialColumnIndices[1], _spatialColumnIndices[2]}) + { + auto it = std::find(dataColumnIndices.begin(), last, index); + if (it == last) + { + std::stringstream sstream; + sstream << "Column index " << index << " is" + << "out of range for CSV data stream"; + throw std::invalid_argument(sstream.str()); + } + *it = *(--last); + } + dataColumnIndices.erase(last, dataColumnIndices.end()); + + using FactoryT = + math::InMemoryTimeVaryingVolumetricGridFactory; + std::vector factories(dataColumnIndices.size()); + for (auto it = _begin; it != _end; ++it) + { + const T time = IO::ReadFrom(it->at(_timeColumnIndex)); + const math::Vector3

position{ + IO

::ReadFrom(it->at(_spatialColumnIndices[0])), + IO

::ReadFrom(it->at(_spatialColumnIndices[1])), + IO

::ReadFrom(it->at(_spatialColumnIndices[2]))}; + + for (size_t i = 0; i < dataColumnIndices.size(); ++i) + { + const V value = IO::ReadFrom(it->at(dataColumnIndices[i])); + factories[i].AddPoint(time, position, value); + } + } + + DataFrame> df; + for (size_t i = 0; i < dataColumnIndices.size(); ++i) + { + const std::string key = !_begin->empty() ? + _begin->at(dataColumnIndices[i]) : + "var" + std::to_string(dataColumnIndices[i]); + df[IO::ReadFrom(key)] = factories[i].Build(); + } + return df; + } + }; + } +} +#endif diff --git a/io/include/gz/common/Io.hh b/io/include/gz/common/Io.hh new file mode 100644 index 000000000..434b10baf --- /dev/null +++ b/io/include/gz/common/Io.hh @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#ifndef GZ_COMMON_IO_HH_ +#define GZ_COMMON_IO_HH_ + +#include +#include + +namespace gz +{ + namespace common + { + /// \brief Traits for type-specific object I/O. + /// + /// To be fully specialized as needed. + template + struct IO + { + /// \brief Read object from stream. + /// + /// This default implementation relies on stream operator overloads. + /// + /// \param[in] _istream Stream to read object from. + /// \return object instance. + static T ReadFrom(std::istream &_istream) + { + T value; + _istream >> value; + return value; + } + + /// \brief Read object from string. + /// + /// This default implementation relies on stream operator overloads. + /// + /// \param[in] _string String to read object from. + /// \return object instance. + static T ReadFrom(const std::string &_string) + { + std::istringstream stream{_string}; + return ReadFrom(stream); + } + }; + + /// \brief Traits for string I/O. + template<> + struct IO + { + /// \brief Read object from stream. + /// + /// This default implementation relies on stream operator overloads. + /// + /// \param[in] _istream Stream to read object from. + /// \return object instance. + static std::string ReadFrom(std::istream &_istream) + { + std::string value; + _istream >> value; + return value; + } + + /// \brief Read string from string (copy as-is). + static std::string ReadFrom(std::string _string) + { + return _string; + } + }; + } +} + +#endif diff --git a/io/src/CMakeLists.txt b/io/src/CMakeLists.txt new file mode 100644 index 000000000..783bfd265 --- /dev/null +++ b/io/src/CMakeLists.txt @@ -0,0 +1,15 @@ +gz_get_libsources_and_unittests(sources gtest_sources) + +gz_add_component(io SOURCES ${sources} GET_TARGET_NAME io_target) + +target_link_libraries(${io_target} + PUBLIC + gz-math${GZ_MATH_VER}::gz-math${GZ_MATH_VER}) + +gz_build_tests( + TYPE UNIT + SOURCES ${gtest_sources} + LIB_DEPS + ${io_target} + gz-common${GZ_COMMON_VER}-testing +) diff --git a/io/src/CSVStreams.cc b/io/src/CSVStreams.cc new file mode 100644 index 000000000..a286e14ca --- /dev/null +++ b/io/src/CSVStreams.cc @@ -0,0 +1,262 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +#include "gz/common/CSVStreams.hh" + +#include + +namespace gz +{ +namespace common +{ + +///////////////////////////////////////////////// +const CSVDialect CSVDialect::Unix = {',', '\n', '"'}; + +///////////////////////////////////////////////// +bool operator==(const CSVDialect &_lhs, const CSVDialect &_rhs) +{ + return (_lhs.delimiter == _rhs.delimiter && + _lhs.terminator == _rhs.terminator && + _lhs.quote == _rhs.quote); +} + +///////////////////////////////////////////////// +std::istream &ExtractCSVToken( + std::istream &_stream, CSVToken &_token, + const CSVDialect &_dialect) +{ + char character; + if (_stream.peek(), !_stream.fail() && _stream.eof()) + { + _token = {CSVToken::TERMINATOR, EOF}; + } + else if (_stream.get(character)) + { + if (character == _dialect.terminator) + { + _token = {CSVToken::TERMINATOR, character}; + } + else if (character == _dialect.delimiter) + { + _token = {CSVToken::DELIMITER, character}; + } + else if (character == _dialect.quote) + { + if (_stream.peek() == _dialect.quote) + { + _token = {CSVToken::TEXT, character}; + _stream.ignore(); + } + else + { + _token = {CSVToken::QUOTE, character}; + } + } + else + { + _token = {CSVToken::TEXT, character}; + } + } + return _stream; +} + +///////////////////////////////////////////////// +std::istream &ParseCSVRow( + std::istream &_stream, + std::vector &_row, + const CSVDialect &_dialect) +{ + std::stringstream text; + enum { + FIELD_START = 0, + ESCAPED_FIELD, + NONESCAPED_FIELD, + FIELD_END, + RECORD_END + } state = FIELD_START; + + _row.clear(); + + CSVToken token; + while (state != RECORD_END && ExtractCSVToken(_stream, token, _dialect)) + { + switch (state) + { + case FIELD_START: + if (token.type == CSVToken::QUOTE) + { + state = ESCAPED_FIELD; + break; + } + state = NONESCAPED_FIELD; + [[fallthrough]]; + case NONESCAPED_FIELD: + if (token.type == CSVToken::TEXT) + { + text << token.character; + break; + } + state = FIELD_END; + [[fallthrough]]; + case FIELD_END: + switch (token.type) + { + case CSVToken::DELIMITER: + _row.push_back(text.str()); + state = FIELD_START; + break; + case CSVToken::TERMINATOR: + if (token.character != EOF || !_row.empty() || text.tellp() > 0) + { + _row.push_back(text.str()); + state = RECORD_END; + break; + } + [[fallthrough]]; + default: + _stream.setstate(std::istream::failbit); + break; + } + text.str(""), text.clear(); + break; + case ESCAPED_FIELD: + if (token.type == CSVToken::QUOTE) + { + state = FIELD_END; + break; + } + if (token.type != CSVToken::TERMINATOR || token.character != EOF) + { + text << token.character; + break; + } + [[fallthrough]]; + default: + _stream.setstate(std::istream::failbit); + break; + } + } + return _stream; +} + +/// \brief Private data for the CSVIStreamIterator class +class CSVIStreamIterator::Implementation +{ + /// \brief Default constructor for end iterator. + public: Implementation() = default; + + /// \brief Constructor for begin iterator. + public: Implementation(std::istream &_stream, const CSVDialect &_dialect) + : stream(&_stream), dialect(_dialect) + { + } + + /// \brief Copy constructor. + public: Implementation(const Implementation &_other) + : stream(_other.stream), dialect(_other.dialect), row(_other.row) + { + } + + /// \brief Advance iterator to next row if possible. + public: void Next() + { + if (this->stream) + { + try + { + if (!ParseCSVRow(*this->stream, this->row, this->dialect)) + { + this->stream = nullptr; + } + } + catch (...) + { + this->stream = nullptr; + throw; + } + } + } + + /// \brief CSV data stream to iterate, if any. + public: std::istream *stream{nullptr}; + + /// \brief CSV dialect for data parsing. + public: CSVDialect dialect{}; + + /// \brief Current CSV data row. + public: std::vector row; +}; + +///////////////////////////////////////////////// +CSVIStreamIterator::CSVIStreamIterator() + : dataPtr(gz::utils::MakeImpl()) +{ +} + +///////////////////////////////////////////////// +CSVIStreamIterator::CSVIStreamIterator(std::istream &_stream, + const CSVDialect &_dialect) + : dataPtr(gz::utils::MakeImpl(_stream, _dialect)) +{ + this->dataPtr->Next(); +} + +///////////////////////////////////////////////// +bool CSVIStreamIterator::operator==(const CSVIStreamIterator &_other) const +{ + return this->dataPtr->stream == _other.dataPtr->stream && ( + this->dataPtr->stream == nullptr || + this->dataPtr->dialect == _other.dataPtr->dialect); +} + +///////////////////////////////////////////////// +bool CSVIStreamIterator::operator!=(const CSVIStreamIterator &_other) const +{ + return !(*this == _other); +} + +///////////////////////////////////////////////// +CSVIStreamIterator &CSVIStreamIterator::operator++() +{ + this->dataPtr->Next(); + return *this; +} + +///////////////////////////////////////////////// +// NOLINTNEXTLINE(readability/casting) +CSVIStreamIterator CSVIStreamIterator::operator++(int) +{ + CSVIStreamIterator it(*this); + this->dataPtr->Next(); + return it; +} + +///////////////////////////////////////////////// +const std::vector &CSVIStreamIterator::operator*() const +{ + return this->dataPtr->row; +} + +///////////////////////////////////////////////// +const std::vector *CSVIStreamIterator::operator->() const +{ + return &this->dataPtr->row; +} + +} +} diff --git a/io/src/CSVStreams_TEST.cc b/io/src/CSVStreams_TEST.cc new file mode 100644 index 000000000..ec6533291 --- /dev/null +++ b/io/src/CSVStreams_TEST.cc @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ +#include +#include +#include +#include + +#include "gz/common/CSVStreams.hh" + +using namespace gz; +using namespace common; + +///////////////////////////////////////////////// +TEST(CSVStreams, CanExtractCSVTokens) +{ + std::stringstream sstream; + sstream << "\"a,\n\"\""; + + CSVToken token; + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::QUOTE); + EXPECT_EQ(token.character, '"'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TEXT); + EXPECT_EQ(token.character, 'a'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::DELIMITER); + EXPECT_EQ(token.character, ','); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TERMINATOR); + EXPECT_EQ(token.character, '\n'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TEXT); + EXPECT_EQ(token.character, '"'); + + EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); + EXPECT_EQ(token.type, CSVToken::TERMINATOR); + EXPECT_EQ(token.character, EOF); +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanParseCSVRows) +{ + { + std::stringstream sstream; + sstream << ","; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"", ""}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << "foo"; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"foo"}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << "foo" << std::endl; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"foo"}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << ",foo"; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"", "foo"}; + EXPECT_EQ(row, expectedRow); + } + + { + std::stringstream sstream; + sstream << ",\"foo,bar\nbaz\","; + std::vector row; + EXPECT_TRUE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + const std::vector expectedRow{"", "foo,bar\nbaz", ""}; + EXPECT_EQ(row, expectedRow); + } +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanHandleInvalidCSVRows) +{ + { + std::stringstream sstream; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } + + { + std::stringstream sstream; + sstream << "\""; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } + + { + std::stringstream sstream; + sstream << "\"foo\"?"; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } + + { + std::stringstream sstream; + sstream << "foo\"bar\""; + std::vector row; + EXPECT_FALSE(ParseCSVRow(sstream, row, CSVDialect::Unix)); + } +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanIterateValidCSV) +{ + { + std::stringstream sstream; + EXPECT_EQ(CSVIStreamIterator(sstream), + CSVIStreamIterator()); + } + + { + std::stringstream ss; + ss << std::endl; + const std::vector> expectedRows{{""}}; + const auto rows = std::vector>( + CSVIStreamIterator(ss), CSVIStreamIterator()); + EXPECT_EQ(expectedRows, rows); + EXPECT_TRUE(ss.eof()); + } + + { + std::stringstream ss; + ss << "foo,bar" << std::endl + << "bar," << std::endl + << ",foo" << std::endl + << "," << std::endl + << "baz,baz"; + const std::vector> expectedRows{ + {"foo", "bar"}, {"bar", ""}, {"", "foo"}, {"", ""}, {"baz", "baz"}}; + const auto rows = std::vector>( + CSVIStreamIterator(ss), CSVIStreamIterator()); + EXPECT_EQ(expectedRows, rows); + } +} + +///////////////////////////////////////////////// +TEST(CSVStreams, CanIterateInvalidCSVSafely) +{ + { + std::stringstream sstream; + sstream << "\"" << std::endl; + auto it = CSVIStreamIterator(sstream, CSVDialect::Unix); + EXPECT_EQ(it, CSVIStreamIterator()); + } + + { + std::stringstream sstream; + sstream.exceptions(std::stringstream::failbit); + sstream << "foo" << std::endl + << "\"bar" << std::endl; + auto it = CSVIStreamIterator(sstream, CSVDialect::Unix); + EXPECT_THROW({ ++it; }, std::stringstream::failure); + EXPECT_EQ(it, CSVIStreamIterator()); + } +} diff --git a/io/src/DataFrame_TEST.cc b/io/src/DataFrame_TEST.cc new file mode 100644 index 000000000..9100ae6db --- /dev/null +++ b/io/src/DataFrame_TEST.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2022 Open Source Robotics Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +#include +#include +#include +#include + +#include + +#include "gz/common/CSVStreams.hh" +#include "gz/common/DataFrame.hh" +#include "gz/common/Filesystem.hh" + +using namespace gz; + +///////////////////////////////////////////////// +TEST(DataFrameTests, SimpleCSV) +{ + std::stringstream ss; + ss << "t,x,y,z,temperature" << std::endl + << "0,0,0,0,25.2" << std::endl + << "0,10,0,0,25.2" << std::endl + << "0,0,10,0,25.2" << std::endl + << "0,10,10,0,25.2" << std::endl + << "1,0,0,0,24.9" << std::endl + << "1,10,0,0,24.9" << std::endl + << "1,0,10,0,25.1" << std::endl + << "1,10,10,0,25.1" << std::endl; + + using DataT = + math::InMemoryTimeVaryingVolumetricGrid; + using DataFrameT = common::DataFrame; + const auto df = common::IO::ReadFrom( + common::CSVIStreamIterator(ss), + common::CSVIStreamIterator()); + + ASSERT_TRUE(df.Has("temperature")); + const DataT &temperatureData = df["temperature"]; + auto temperatureSession = temperatureData.StepTo( + temperatureData.CreateSession(), 0.5); + ASSERT_TRUE(temperatureSession.has_value()); + const math::Vector3d position{5., 5., 0.}; + auto temperature = temperatureData.LookUp( + temperatureSession.value(), position); + ASSERT_TRUE(temperature.has_value()); + EXPECT_DOUBLE_EQ(25.1, temperature.value()); +} + +///////////////////////////////////////////////// +TEST(DataFrameTests, ComplexCSV) +{ + std::stringstream ss; + ss << "timestamp,temperature,pressure,humidity,lat,lon,altitude" << std::endl + << "1658923062,13.1,101490,91,36.80029505,-121.788972517,0.8" << std::endl + << "1658923062,13,101485,88,36.80129505,-121.788972517,0.8" << std::endl + << "1658923062,13.1,101485,89,36.80029505,-121.789972517,0.8" << std::endl + << "1658923062,13.5,101490,92,36.80129505,-121.789972517,0.8" << std::endl; + + using DataT = + math::InMemoryTimeVaryingVolumetricGrid; + using DataFrameT = common::DataFrame; + const auto df = common::IO::ReadFrom( + common::CSVIStreamIterator(ss), common::CSVIStreamIterator(), + "timestamp", {"lat", "lon", "altitude"}); + EXPECT_TRUE(df.Has("temperature")); + EXPECT_TRUE(df.Has("humidity")); + ASSERT_TRUE(df.Has("pressure")); + + const DataT &pressureData = df["pressure"]; + auto pressureSession = pressureData.CreateSession(); + const math::Vector3d position{36.80079505, -121.789472517, 0.8}; + auto pressure = pressureData.LookUp(pressureSession, position); + ASSERT_TRUE(pressure.has_value()); + EXPECT_DOUBLE_EQ(101487.5, pressure.value()); +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d14341b9d..bcb1ff4c4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -49,6 +49,11 @@ gz_build_tests( # Used to make internal source file headers visible to the unit tests ${CMAKE_CURRENT_SOURCE_DIR}) +if(TARGET UNIT_DataFrame_TEST) + target_include_directories(UNIT_DataFrame_TEST PRIVATE + ${gz-math${GZ_MATH_VER}_INCLUDE_DIRS}) +endif() + if(TARGET UNIT_MovingWindowFilter_TEST) target_include_directories(UNIT_MovingWindowFilter_TEST PRIVATE ${gz-math${GZ_MATH_VER}_INCLUDE_DIRS}) From 2d2f73c7535349c5eddf16f5a8af0a148d4e86d2 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Tue, 6 Sep 2022 10:23:30 -0300 Subject: [PATCH 02/10] Skip CSV header when reading DataFrame (#435) Signed-off-by: Michel Hidalgo --- io/include/gz/common/DataFrame.hh | 2 +- io/src/DataFrame_TEST.cc | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/io/include/gz/common/DataFrame.hh b/io/include/gz/common/DataFrame.hh index b5a124dc6..bc340015e 100644 --- a/io/include/gz/common/DataFrame.hh +++ b/io/include/gz/common/DataFrame.hh @@ -179,7 +179,7 @@ namespace gz using FactoryT = math::InMemoryTimeVaryingVolumetricGridFactory; std::vector factories(dataColumnIndices.size()); - for (auto it = _begin; it != _end; ++it) + for (auto it = std::next(_begin); it != _end; ++it) { const T time = IO::ReadFrom(it->at(_timeColumnIndex)); const math::Vector3

position{ diff --git a/io/src/DataFrame_TEST.cc b/io/src/DataFrame_TEST.cc index 9100ae6db..5418d6879 100644 --- a/io/src/DataFrame_TEST.cc +++ b/io/src/DataFrame_TEST.cc @@ -52,12 +52,19 @@ TEST(DataFrameTests, SimpleCSV) ASSERT_TRUE(df.Has("temperature")); const DataT &temperatureData = df["temperature"]; auto temperatureSession = temperatureData.StepTo( - temperatureData.CreateSession(), 0.5); + temperatureData.CreateSession(), 0.); ASSERT_TRUE(temperatureSession.has_value()); const math::Vector3d position{5., 5., 0.}; auto temperature = temperatureData.LookUp( temperatureSession.value(), position); ASSERT_TRUE(temperature.has_value()); + EXPECT_DOUBLE_EQ(25.2, temperature.value()); + temperatureSession = temperatureData.StepTo( + temperatureData.CreateSession(), 0.5); + ASSERT_TRUE(temperatureSession.has_value()); + temperature = temperatureData.LookUp( + temperatureSession.value(), position); + ASSERT_TRUE(temperature.has_value()); EXPECT_DOUBLE_EQ(25.1, temperature.value()); } From 3bdeca4e96e7b36c4bd0f61b26e7bc326f68ac93 Mon Sep 17 00:00:00 2001 From: Arjo Chakravarty Date: Thu, 6 Oct 2022 23:53:31 +0800 Subject: [PATCH 03/10] Adds an API to retrieve keys. (#446) This is useful for when we perform visualization. Signed-off-by: Arjo Chakravarty Co-authored-by: Michael Carroll --- io/include/gz/common/DataFrame.hh | 12 ++++++++++++ io/src/DataFrame_TEST.cc | 6 +++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/io/include/gz/common/DataFrame.hh b/io/include/gz/common/DataFrame.hh index bc340015e..fda99d060 100644 --- a/io/include/gz/common/DataFrame.hh +++ b/io/include/gz/common/DataFrame.hh @@ -68,6 +68,18 @@ namespace gz return this->storage.at(_key); } + /// \brief Retrieve all keys + /// \return A vector with keys + public: const std::vector Keys() const + { + std::vector keyList; + for (auto &[k, _]: this->storage) + { + keyList.push_back(k); + } + return keyList; + } + /// \brief Data frame storage private: std::unordered_map storage; }; diff --git a/io/src/DataFrame_TEST.cc b/io/src/DataFrame_TEST.cc index 5418d6879..fd2e8a215 100644 --- a/io/src/DataFrame_TEST.cc +++ b/io/src/DataFrame_TEST.cc @@ -66,6 +66,9 @@ TEST(DataFrameTests, SimpleCSV) temperatureSession.value(), position); ASSERT_TRUE(temperature.has_value()); EXPECT_DOUBLE_EQ(25.1, temperature.value()); + auto keys = df.Keys(); + ASSERT_EQ(keys.size(), 1); + ASSERT_EQ(keys[0], "temperature"); } ///////////////////////////////////////////////// @@ -87,7 +90,8 @@ TEST(DataFrameTests, ComplexCSV) EXPECT_TRUE(df.Has("temperature")); EXPECT_TRUE(df.Has("humidity")); ASSERT_TRUE(df.Has("pressure")); - + auto keys = df.Keys(); + ASSERT_EQ(keys.size(), 3); const DataT &pressureData = df["pressure"]; auto pressureSession = pressureData.CreateSession(); const math::Vector3d position{36.80079505, -121.789472517, 0.8}; From 5c74a6326463bfc206311d0657190848b362434f Mon Sep 17 00:00:00 2001 From: Nate Koenig Date: Tue, 18 Oct 2022 08:27:32 -0700 Subject: [PATCH 04/10] 5.2.0 release (#461) Signed-off-by: Nate Koenig Signed-off-by: Nate Koenig Co-authored-by: Nate Koenig --- CMakeLists.txt | 2 +- Changelog.md | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f16df76aa..3a7a34f1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.10.2 FATAL_ERROR) #============================================================================ # Initialize the project #============================================================================ -project(gz-common5 VERSION 5.1.0) +project(gz-common5 VERSION 5.2.0) set(GZ_COMMON_VER ${PROJECT_VERSION_MAJOR}) #============================================================================ diff --git a/Changelog.md b/Changelog.md index ac32be692..f1c537185 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,16 @@ ## Gazebo Common 5.x +## Gazebo Common 5.2.0 (2022-10-18) + +1. Add CSV data parsing + * [Pull request #402](https://github.com/gazebosim/gz-common/pull/402) + +1. Skip CSV header when reading DataFrame. + * [Pull request #435](https://github.com/gazebosim/gz-common/pull/435) + +1. Adds an API to retrieve keys. + * [Pull request #446](https://github.com/gazebosim/gz-common/pull/446) + ## Gazebo Common 5.1.0 (2022-10-13) 1. 4 ➡️ 5 From b3313c27f3703cd714b472ca8c44cbcd7ab9b6c5 Mon Sep 17 00:00:00 2001 From: Nate Koenig Date: Tue, 18 Oct 2022 10:43:27 -0700 Subject: [PATCH 05/10] Fix arm builds (#462) * Fix arm builds Signed-off-by: Nate Koenig * Adding in pre-release 5.2.1 Signed-off-by: Nate Koenig Signed-off-by: Nate Koenig Co-authored-by: Nate Koenig --- CMakeLists.txt | 4 ++-- io/src/CSVStreams.cc | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a7a34f1c..b41e559ba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.10.2 FATAL_ERROR) #============================================================================ # Initialize the project #============================================================================ -project(gz-common5 VERSION 5.2.0) +project(gz-common5 VERSION 5.2.1) set(GZ_COMMON_VER ${PROJECT_VERSION_MAJOR}) #============================================================================ @@ -18,7 +18,7 @@ set(GZ_CMAKE_VER ${gz-cmake3_VERSION_MAJOR}) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -gz_configure_project(VERSION_SUFFIX) +gz_configure_project(VERSION_SUFFIX pre1) #============================================================================ # Set project-specific options diff --git a/io/src/CSVStreams.cc b/io/src/CSVStreams.cc index a286e14ca..f561edba3 100644 --- a/io/src/CSVStreams.cc +++ b/io/src/CSVStreams.cc @@ -43,7 +43,7 @@ std::istream &ExtractCSVToken( char character; if (_stream.peek(), !_stream.fail() && _stream.eof()) { - _token = {CSVToken::TERMINATOR, EOF}; + _token = {CSVToken::TERMINATOR, static_cast(EOF)}; } else if (_stream.get(character)) { @@ -121,7 +121,8 @@ std::istream &ParseCSVRow( state = FIELD_START; break; case CSVToken::TERMINATOR: - if (token.character != EOF || !_row.empty() || text.tellp() > 0) + if (token.character != static_cast(EOF) || !_row.empty() || + text.tellp() > 0) { _row.push_back(text.str()); state = RECORD_END; @@ -140,7 +141,8 @@ std::istream &ParseCSVRow( state = FIELD_END; break; } - if (token.type != CSVToken::TERMINATOR || token.character != EOF) + if (token.type != CSVToken::TERMINATOR || + token.character != static_cast(EOF)) { text << token.character; break; From 1a26b9ae9e7d0d709be47ef8e82dc8c5c8b22f26 Mon Sep 17 00:00:00 2001 From: Nate Koenig Date: Tue, 18 Oct 2022 21:02:03 -0700 Subject: [PATCH 06/10] Fix arm test (#463) * Fix arm test Signed-off-by: Nate Koenig Co-authored-by: Nate Koenig --- CMakeLists.txt | 2 +- io/src/CSVStreams_TEST.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b41e559ba..4040a9d22 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ set(GZ_CMAKE_VER ${gz-cmake3_VERSION_MAJOR}) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -gz_configure_project(VERSION_SUFFIX pre1) +gz_configure_project(VERSION_SUFFIX pre2) #============================================================================ # Set project-specific options diff --git a/io/src/CSVStreams_TEST.cc b/io/src/CSVStreams_TEST.cc index ec6533291..4846bbf09 100644 --- a/io/src/CSVStreams_TEST.cc +++ b/io/src/CSVStreams_TEST.cc @@ -53,7 +53,7 @@ TEST(CSVStreams, CanExtractCSVTokens) EXPECT_TRUE(ExtractCSVToken(sstream, token, CSVDialect::Unix)); EXPECT_EQ(token.type, CSVToken::TERMINATOR); - EXPECT_EQ(token.character, EOF); + EXPECT_EQ(token.character, static_cast(EOF)); } ///////////////////////////////////////////////// From 56b82ad501b64a0c51b06a79fd68aeec2766f86e Mon Sep 17 00:00:00 2001 From: Nate Koenig Date: Wed, 19 Oct 2022 15:51:51 -0700 Subject: [PATCH 07/10] 5.2.1 release (#464) Signed-off-by: Nate Koenig Signed-off-by: Nate Koenig Co-authored-by: Nate Koenig --- CMakeLists.txt | 2 +- Changelog.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4040a9d22..8bc4a70c3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,7 +18,7 @@ set(GZ_CMAKE_VER ${gz-cmake3_VERSION_MAJOR}) set(CMAKE_CXX_STANDARD 17) set(CMAKE_CXX_STANDARD_REQUIRED ON) -gz_configure_project(VERSION_SUFFIX pre2) +gz_configure_project(VERSION_SUFFIX) #============================================================================ # Set project-specific options diff --git a/Changelog.md b/Changelog.md index f1c537185..566383691 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,11 @@ ## Gazebo Common 5.x +## Gazebo Common 5.2.1 (2022-10-19) + +1. Fix arm builds and tests + * [Pull request #462](https://github.com/gazebosim/gz-common/pull/462) + * [Pull request #463](https://github.com/gazebosim/gz-common/pull/463) + ## Gazebo Common 5.2.0 (2022-10-18) 1. Add CSV data parsing From 9521e1bb348133bf98e7bf23fba79115c1c9cb55 Mon Sep 17 00:00:00 2001 From: Michel Hidalgo Date: Wed, 26 Oct 2022 17:24:58 -0300 Subject: [PATCH 08/10] Backport: Avoid Io.hh header name clash (#472) Backports #471 to Garden. Signed-off-by: Michel Hidalgo --- io/include/gz/common/DataFrame.hh | 2 +- io/include/gz/common/{Io.hh => IOBase.hh} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename io/include/gz/common/{Io.hh => IOBase.hh} (100%) diff --git a/io/include/gz/common/DataFrame.hh b/io/include/gz/common/DataFrame.hh index fda99d060..70b7b85fa 100644 --- a/io/include/gz/common/DataFrame.hh +++ b/io/include/gz/common/DataFrame.hh @@ -26,7 +26,7 @@ #include #include -#include +#include #include diff --git a/io/include/gz/common/Io.hh b/io/include/gz/common/IOBase.hh similarity index 100% rename from io/include/gz/common/Io.hh rename to io/include/gz/common/IOBase.hh From 431a2adf2fed46de03257eeb7326a1f06a2cc792 Mon Sep 17 00:00:00 2001 From: Michael Carroll Date: Wed, 26 Oct 2022 14:14:58 -0700 Subject: [PATCH 09/10] =?UTF-8?q?=F0=9F=8E=88=20=205.2.2=20(#473)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Michael Carroll --- CMakeLists.txt | 2 +- Changelog.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8bc4a70c3..d797adaba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -3,7 +3,7 @@ cmake_minimum_required(VERSION 3.10.2 FATAL_ERROR) #============================================================================ # Initialize the project #============================================================================ -project(gz-common5 VERSION 5.2.1) +project(gz-common5 VERSION 5.2.2) set(GZ_COMMON_VER ${PROJECT_VERSION_MAJOR}) #============================================================================ diff --git a/Changelog.md b/Changelog.md index 566383691..90c183b11 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,5 +1,10 @@ ## Gazebo Common 5.x +## Gazebo Common 5.2.2 (2022-10-26) + +1. [Backport] Avoid Io.hh header name clash (#471) + * [Pull request #472](https://github.com/gazebosim/gz-common/pull/472) + ## Gazebo Common 5.2.1 (2022-10-19) 1. Fix arm builds and tests From 4a61ca7a11925152fccc78d5a502208ae0b1ba57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Onur=20Berk=20T=C3=B6re?= Date: Thu, 10 Nov 2022 08:22:11 +0300 Subject: [PATCH 10/10] Fix silly mistakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds paranthesis Signed-off-by: Onur Berk Töre --- graphics/src/AssimpLoader_TEST.cc | 17 ++ graphics/src/ColladaLoader_TEST.cc | 17 ++ test/data/box_with_no_animation_name.dae | 194 +++++++++++++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 test/data/box_with_no_animation_name.dae diff --git a/graphics/src/AssimpLoader_TEST.cc b/graphics/src/AssimpLoader_TEST.cc index 87e6b8210..3a00f4954 100644 --- a/graphics/src/AssimpLoader_TEST.cc +++ b/graphics/src/AssimpLoader_TEST.cc @@ -738,3 +738,20 @@ TEST_F(AssimpLoader, CheckNonRootDisplacement) auto xDisplacement = skelAnim->XDisplacement(); ASSERT_TRUE(xDisplacement); } + +///////////////////////////////////////////////// +// Load animation without a name +TEST_F(AssimpLoader, NoAnimName) +{ + common::AssimpLoader loader; + + std::string meshFilename = + common::testing::TestFile("data", "box_with_no_animation_name.dae"); + + common::Mesh *mesh = loader.Load(meshFilename); + common::SkeletonPtr skeleton = mesh->MeshSkeleton(); + ASSERT_EQ(1u, skeleton->AnimationCount()); + common::SkeletonAnimation *anim = skeleton->Animation(0); + auto animName = anim->Name(); + EXPECT_EQ(animName, "animation1"); +} diff --git a/graphics/src/ColladaLoader_TEST.cc b/graphics/src/ColladaLoader_TEST.cc index 63e0fe75d..23b94fdc0 100644 --- a/graphics/src/ColladaLoader_TEST.cc +++ b/graphics/src/ColladaLoader_TEST.cc @@ -489,3 +489,20 @@ TEST_F(ColladaLoader, LoadCylinderAnimatedFrom3dsMax) EXPECT_EQ(1u, anim->NodeCount()); EXPECT_TRUE(anim->HasNode("Bone02")); } + +///////////////////////////////////////////////// +// Load animation without a name +TEST_F(ColladaLoader, NoAnimName) +{ + common::ColladaLoader loader; + + std::string meshFilename = + common::testing::TestFile("data", "box_with_no_animation_name.dae"); + + common::Mesh *mesh = loader.Load(meshFilename); + common::SkeletonPtr skeleton = mesh->MeshSkeleton(); + ASSERT_EQ(1u, skeleton->AnimationCount()); + common::SkeletonAnimation *anim = skeleton->Animation(0); + auto animName = anim->Name(); + EXPECT_EQ(animName, "animation1"); +} \ No newline at end of file diff --git a/test/data/box_with_no_animation_name.dae b/test/data/box_with_no_animation_name.dae new file mode 100644 index 000000000..4a3a42273 --- /dev/null +++ b/test/data/box_with_no_animation_name.dae @@ -0,0 +1,194 @@ + + + + + Blender User + Blender 2.80.40 commit date:2019-01-07, commit time:23:37, hash:91a155833e59 + + 2019-01-08T17:44:11 + 2019-01-08T17:44:11 + + Z_UP + + + + + + + + 0.8 0.8 0.8 1 + + + 0 0.5 0 1 + + + + + + + + + + + + + + + + + 1 1 1 1 1 -1 1 -1 1 1 -1 -1 -1 1 1 -1 1 -1 -1 -1 1 -1 -1 -1 + + + + + + + + + + 0 0 1 0 -1 0 -1 0 0 0 0 -1 1 0 0 0 1 0 + + + + + + + + + + 0.625 0 0.375 0.25 0.375 0 0.625 0.25 0.375 0.5 0.375 0.25 0.625 0.5 0.375 0.75 0.375 0.5 0.625 0.75 0.375 1 0.375 0.75 0.375 0.5 0.125 0.75 0.125 0.5 0.875 0.5 0.625 0.75 0.625 0.5 0.625 0 0.625 0.25 0.375 0.25 0.625 0.25 0.625 0.5 0.375 0.5 0.625 0.5 0.625 0.75 0.375 0.75 0.625 0.75 0.625 1 0.375 1 0.375 0.5 0.375 0.75 0.125 0.75 0.875 0.5 0.875 0.75 0.625 0.75 + + + + + + + + + + + + + + + 3 3 3 3 3 3 3 3 3 3 3 3 +

4 0 0 2 0 1 0 0 2 2 1 3 7 1 4 3 1 5 6 2 6 5 2 7 7 2 8 1 3 9 7 3 10 5 3 11 0 4 12 3 4 13 1 4 14 4 5 15 1 5 16 5 5 17 4 0 18 6 0 19 2 0 20 2 1 21 6 1 22 7 1 23 6 2 24 4 2 25 5 2 26 1 3 27 3 3 28 7 3 29 0 4 30 2 4 31 3 4 32 4 5 33 0 5 34 1 5 35

+ + + + + + + + 1 0 0 -1 0 1 0 1 0 0 1 1 0 0 0 1 + + Bone + + + + + + + + 0.7886752 0.2113248 0.5773504 -0.5773504 -0.5773503 0.5773503 0.5773503 1.154701 -0.2113249 -0.7886752 0.5773503 -0.5773502 0 0 0 1 + + + + + + + + 1 1 1 1 1 1 1 1 + + + + + + + + + + + + + + 1 1 1 1 1 1 1 1 + 0 0 0 1 0 2 0 3 0 4 0 5 0 6 0 7 + + + + + + + + 0.04166662 0.08333331 0.125 0.1666666 0.2083333 0.25 0.2916666 0.3333333 0.375 0.4166666 0.4583333 0.5 0.5416667 0.5833333 0.625 0.6666667 0.7083333 0.75 0.7916667 0.8333333 0.875 0.9166667 0.9583333 1 1.041667 1.083333 1.125 1.166667 1.208333 1.25 1.291667 1.333333 1.375 1.416667 1.458333 1.5 1.541667 1.583333 1.625 1.666667 + + + + + + + + 1 0 0 1 0 1 0 -1 0 0 1 0 0 0 0 1 0.9999878 3.10816e-5 0.004935208 1 0 0.9999802 -0.006297799 -1 -0.004935306 0.006297722 0.999968 0 0 0 0 1 0.999819 4.61727e-4 0.01901668 1 0 0.9997054 -0.02427293 -1 -0.01902229 0.02426853 0.9995245 0 0 0 0 1 0.9991519 0.002163141 0.04111904 1 0 0.9986191 -0.05253414 -1 -0.04117589 0.05248959 0.9977722 0 0 0 0 1 0.9975264 0.006301912 0.07000974 1 0 0.9959731 -0.08965231 -1 -0.0702928 0.08943056 0.9935095 0 0 0 0 1 0.9944467 0.01411698 0.1042901 1 0 0.9909625 -0.1341392 -1 -0.1052413 0.1333943 0.9854594 0 0 0 0 1 0.9894527 0.02671701 0.1423712 1 0 0.9828442 -0.184438 -1 -0.1448563 0.1824927 0.9724778 0 0 0 0 1 0.9821799 0.04490547 0.1825 1 0 0.9710366 -0.2389307 -1 -0.1879434 0.234673 0.9537326 0 0 0 0 1 0.9724072 0.06904543 0.2228386 1 0 0.9551992 -0.2959637 -1 -0.2332902 0.2877972 0.9288425 0 0 0 0 1 0.9600915 0.09897761 0.261587 1 0 0.9352878 -0.3538882 -1 -0.2796861 0.339765 0.8979618 0 0 0 0 1 0.9453882 0.1340003 0.2971281 1 0 0.9115852 -0.4111113 -1 -0.3259466 0.3886598 0.8618018 0 0 0 0 1 0.9286572 0.1729132 0.328172 1 0 0.8847058 -0.4661497 -1 -0.3709391 0.4328933 0.8215885 0 0 0 0 1 0.9104556 0.2141147 0.3538722 1 0 0.8555763 -0.5176768 -1 -0.4136069 0.4713217 0.7789642 0 0 0 0 1 0.8915175 0.2557371 0.3738919 1 0 0.8253933 -0.5645581 -1 -0.4529863 0.5033134 0.7358525 0 0 0 0 1 0.8727233 0.2957927 0.388408 1 0 0.7955672 -0.6058654 -1 -0.4882152 0.5287529 0.6943099 0 0 0 0 1 0.8550603 0.332307 0.3980502 1 0 0.7676533 -0.6408653 -1 -0.5185286 0.5479785 0.6563899 0 0 0 0 1 0.8395769 0.3634188 0.4037789 1 0 0.7432778 -0.6689829 -1 -0.5432408 0.5616626 0.6240388 0 0 0 0 1 0.8273312 0.3874339 0.4067161 1 0 0.7240622 -0.6897347 -1 -0.5617144 0.5706391 0.5990393 0 0 0 0 1 0.8193359 0.4028329 0.4079393 1 0 0.7115462 -0.7026393 -1 -0.5733138 0.5756976 0.5829953 0 0 0 0 1 0.8164964 0.4082482 0.4082486 1 7.75722e-8 0.707107 -0.7071065 -1 -0.5773504 0.57735 0.5773503 0 0 0 0 1 0.8190646 0.4033515 0.4079717 1 7.78161e-8 0.7111219 -0.7030687 -1 -0.5737014 0.5758587 0.5824547 0 0 0 0 1 0.8263245 0.3893851 0.4068995 1 7.85059e-8 0.7224849 -0.6913868 -1 -0.5631944 0.5713098 0.5970069 0 0 0 0 1 0.8375081 0.3675125 0.4043696 1 7.95684e-8 0.7400277 -0.6725764 -1 -0.5464249 0.5632883 0.6197791 0 0 0 0 1 0.8517552 0.3390183 0.3994742 1 8.0922e-8 0.7624427 -0.6470557 -1 -0.5239399 0.5511332 0.6494145 0 0 0 0 1 0.8681612 0.3053284 0.3912425 1 8.24806e-8 0.7883466 -0.6152314 -1 -0.4962822 0.5341201 0.6844119 0 0 0 0 1 0.8858209 0.2680094 0.3788038 1 8.41584e-8 0.8163394 -0.5775725 -1 -0.4640273 0.5116258 0.7231305 0 0 0 0 1 0.9038687 0.2287352 0.3615268 1 8.58731e-8 0.8450637 -0.5346656 -1 -0.42781 0.4832675 0.7638266 0 0 0 0 1 0.9215156 0.1892192 0.339124 1 8.75496e-8 0.8732626 -0.4872499 -1 -0.3883413 0.4490085 0.8047251 0 0 0 0 1 0.9380813 0.1511175 0.3117163 1 8.91235e-8 0.899834 -0.4362323 -1 -0.3464153 0.4092214 0.8441175 0 0 0 0 1 0.9530206 0.1159168 0.2798482 1 9.05428e-8 0.9238796 -0.3826832 -1 -0.3029055 0.3647051 0.8804763 0 0 0 0 1 0.965943 0.08482374 0.2444564 1 9.17705e-8 0.9447417 -0.3278156 -1 -0.2587547 0.3166512 0.9125667 0 0 0 0 1 0.9766233 0.05867312 0.2067956 1 9.27852e-8 0.9620277 -0.2729518 -1 -0.2149581 0.2665711 0.9395387 0 0 0 0 1 0.9850019 0.03787052 0.1683363 1 9.35812e-8 0.975616 -0.2194843 -1 -0.1725436 0.2161924 0.9609836 0 0 0 0 1 0.991176 0.02237916 0.1306496 1 9.41678e-8 0.9856446 -0.1688333 -1 -0.1325524 0.1673435 0.9769473 0 0 0 0 1 0.9953793 0.01175384 0.09529842 1 9.45671e-8 0.9924796 -0.1224106 -1 -0.09602053 0.121845 0.9878936 0 0 0 0 1 0.997952 0.005218936 0.06375288 1 9.48115e-8 0.996666 -0.08159051 -1 -0.06396614 0.08142342 0.9946249 0 0 0 0 1 0.9993011 0.001782816 0.03733916 1 9.49397e-8 0.998862 -0.04769476 -1 -0.0373817 0.04766143 0.9981638 0 0 0 0 1 0.9998515 3.78837e-4 0.01722835 1 9.4992e-8 0.9997582 -0.02198936 -1 -0.01723252 0.0219861 0.9996098 0 0 0 0 1 0.99999 2.53135e-5 0.004462156 1 9.50052e-8 0.9999838 -0.00569412 -1 -0.004462227 0.005694063 0.9999738 0 0 0 0 1 1 0 0 2 0 1 0 -1 0 0 1 0 0 0 0 1 + + + + + + + + LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR LINEAR + + + + + + + + + + + + + + + + + + 1 0 0 1 0 1 0 -1 0 0 1 0 0 0 0 1 + + 0.7886751 -0.5773503 -0.211325 0 0.2113248 0.5773503 -0.7886751 0 0.5773503 0.5773503 0.5773502 0 0 0 0 1 + + + 0 + -0.5235989 + -2 + 2 + 2 + + + + + 0 0 0 + 0 0 1 0 + 0 1 0 0 + 1 0 0 0 + 1 1 1 + + #Armature_Bone + + + + + + + + + + + + + + + +