Changed the C file interface to C++ generic stream buffer interface.

Note that this is a major change.

CMakeLists.txt

@@ -87,7 +87,7 @@ if((CMAKE_BUILD_TYPE MATCHES "^[Rr]elease") AND MDF_BUILD_DOC)
     set(DOXYGEN_RECURSIVE NO)
     set(DOXYGEN_REPEAT_BRIEF NO)
     set(DOXYGEN_PROJECT_NAME "MDF Lib")
-    set(DOXYGEN_PROJECT_NUMBER "2.2")
+    set(DOXYGEN_PROJECT_NUMBER "2.3")
     set(DOXYGEN_HTML_EXTRA_STYLESHEET doxygen/utillib.css)
     set(DOXYGEN_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/docs/manual)
     doxygen_add_docs(

include/mdf/ichannel.h

@@ -480,7 +480,7 @@ class IChannel : public IBlock  {
   static bool GetVirtualSample(uint64_t sample, V& value) {
     // No need for array index here. Array is weird usage for virtual channels
     // as the channel value = sample.
-    value = static_cast<V>(sample);
+    value = static_cast<V>( static_cast<size_t>(sample) );
     return true;
   }
 

include/mdf/idatagroup.h

@@ -82,7 +82,7 @@ class IDataGroup : public IBlock {
   /** \brief Detaches all observers from the measurement. */
   void DetachAllSampleObservers() const;
   /** \brief Notifies the observer that a new sample record have been read.*/
-  bool NotifySampleObservers(size_t sample, uint64_t record_id,
+  bool NotifySampleObservers(uint64_t sample, uint64_t record_id,
                              const std::vector<uint8_t>& record) const;
 
   /** \brief Clear all temporary sample and data buffers.

include/mdf/itimestamp.h

@@ -79,8 +79,7 @@ class LocalTimestamp : public ITimestamp {
   [[nodiscard]] uint64_t GetUtcTimeNs() const override;
 
  private:
-  uint64_t local_timestamp_;  ///< The local timestamp in nanoseconds, with
-                              ///< timezone and DST offset.
+  uint64_t local_timestamp_;  ///< The local timestamp in nanoseconds, with timezone and DST offset.
   int16_t timezone_offset_min_ = 0;  ///< The timezone offset in minutes.
   int16_t dst_offset_min_ = 0;  ///< The daylight saving time offset in minutes.
 };

include/mdf/mdffactory.h

@@ -51,12 +51,12 @@ class MdfFactory {
  public:
   /** \brief Creates an MDF writer object. */
   static std::unique_ptr<MdfWriter> CreateMdfWriter(MdfWriterType type);
-  /** \brief Create an MDF reader object.*/
+  /** \brief Create an MDF file object.*/
   static std::unique_ptr<MdfFile> CreateMdfFile(MdfFileType type);
 
   /** \brief Creates an MDF writer object. */
   static MdfWriter* CreateMdfWriterEx(MdfWriterType type);
-  /** \brief Create an MDF reader object.*/
+  /** \brief Create an MDF file object.*/
   static MdfFile* CreateMdfFileEx(MdfFileType type);
 
   /** \brief Sets the log function. */

include/mdf/mdffile.h

@@ -143,7 +143,7 @@ class MdfFile {
    * read in any other information as measurement information.
    * @param file Pointer to an opened file.
    */
-  virtual void ReadHeader(std::FILE* file) = 0;
+  virtual void ReadHeader(std::streambuf& buffer) = 0;
 
   /** \brief Reads the measurement information about the file.
    *
@@ -154,7 +154,7 @@ class MdfFile {
    *
    * @param file Pointer to an opened file.
    */
-  virtual void ReadMeasurementInfo(std::FILE* file) = 0;
+  virtual void ReadMeasurementInfo(std::streambuf& buffer) = 0;
 
   /** \brief Reads in all expect raw data from the file.
    *
@@ -167,7 +167,7 @@ class MdfFile {
    * @param file Pointer to an opened file.
    */
   virtual void
-  ReadEverythingButData(std::FILE* file) = 0;
+  ReadEverythingButData(std::streambuf& buffer) = 0;
 
   /** \brief Saves all blocks onto the file.
    *
@@ -178,7 +178,7 @@ class MdfFile {
    * @param file Pointer to an open file.
    * @return True on success.
    */
-  virtual bool Write(std::FILE* file) = 0;
+  virtual bool Write(std::streambuf& buffer) = 0;
 
   /** \brief Display name of the file.
    *
@@ -211,7 +211,7 @@ class MdfFile {
   void FileName(const std::string& filename);
 
   /** \brief Sets the finalize state for the file. */
-  virtual void IsFinalized(bool finalized, std::FILE* file,
+  virtual void IsFinalized(bool finalized, std::streambuf& buffer,
                            uint16_t standard_flags, uint16_t custom_flags) = 0;
   /** \brief Returns true if the file is finalized. */
   [[nodiscard]] virtual bool IsFinalized(uint16_t& standard_flags,

include/mdf/mdfreader.h

@@ -9,6 +9,7 @@
 #include <string>
 #include <functional>
 #include <vector>
+#include <fstream>
 
 #include "mdf/ichannelobserver.h"
 #include "mdf/mdffile.h"
@@ -67,9 +68,18 @@ void CreateChannelObserverForDataGroup(const IDataGroup& data_group,
  */
 class MdfReader {
  public:
-  explicit MdfReader(
-      const std::string& filename);  ///< Constructor that opens the file and
-                                     ///< read ID and HD block.
+  /** \brief Constructor for readers using an external file.
+   *
+   * This constructor is used when reading from an external file. The other
+   * way of reading from a C++ stream buffer. The latter is more complicated
+   * but more generic as it support environment without a file block
+   * storage.
+   *
+   * @param filename Full file path to the input file.
+   */
+  explicit MdfReader(std::string filename);
+
+  explicit MdfReader(std::shared_ptr<std::streambuf>& buffer);
   virtual ~MdfReader();  ///< Destructor that close any open file and destructs.
 
   MdfReader() = delete;
@@ -118,14 +128,21 @@ class MdfReader {
   [[nodiscard]] std::string ShortName()
       const;  ///< Returns the file name without paths.
 
-  bool Open();   ///< Opens the file stream for reading.
+  /** \brief Opens the internal stream buffer.
+   *
+   * Normally the stream buffer is an ordinary file but the use may also
+   * attach a generic C++ stream
+   * @return True if the stream was opened.
+   */
+  [[nodiscard]] bool Open();
+  [[nodiscard]] bool IsOpen() const;
   void Close();  ///< Closes the file stream.
 
   bool ReadHeader();             ///< Reads the ID and the HD block.
   bool ReadMeasurementInfo();    ///< Reads everything but not CG and raw data.
   bool ReadEverythingButData();  ///< Reads all blocks but not raw data.
 
-  /** \brief Export the attachment data to a detination file. */
+  /** \brief Export the attachment data to a destination file. */
   bool ExportAttachmentData(const IAttachment& attachment,
                             const std::string& dest_file);
 
@@ -184,7 +201,7 @@ class MdfReader {
    * bytes typical a video stream. These files tends to be huge so the
    * application runs out of memory.
    *
-   * This function reads in VLSD stored data in smaller batcher. The
+   * This function reads in VLSD stored data in smaller batches. The
    * application first reads in all offsets to the raw data. Using these offsets
    * the application can read in typically one sample (offset) at a time. This
    * tactic saves primary memory.
@@ -203,11 +220,23 @@ class MdfReader {
 
 
  private:
-  std::FILE* file_ = nullptr;          ///< Pointer to the file stream.
+  /** \brief Internal pointer to the active stream buffer.
+   *
+   * Shared pointer to the C++ stream buffer. This buffer is either
+   * created by this class (file streams) or outside the
+   */
+  std::shared_ptr<std::streambuf> file_;
+
   std::string filename_;               ///< The file name with full path.
   std::unique_ptr<MdfFile> instance_;  ///< Pointer to the MDF file object.
   int64_t index_ = 0;  ///< Unique (database) file index that can be used to
                        ///< identify a file instead of its path.
+
+  /** \brief Reads in the
+   *
+   */
+  void VerifyMdfFile();
+
 };
 
 }  // namespace mdf

include/mdf/mdfwriter.h

@@ -309,8 +309,8 @@ class MdfWriter {
 
   virtual void CreateMdfFile() = 0; ///< Creates an MDF file
   virtual bool PrepareForWriting() = 0; ///< Prepare for writing.
-  virtual void SetDataPosition(std::FILE* file); ///< Set the data position.
-  virtual bool WriteSignalData(std::FILE* file); ///< Write an SD block.
+  virtual void SetDataPosition(std::streambuf& file); ///< Set the data position.
+  virtual bool WriteSignalData(std::streambuf& file); ///< Write an SD block.
 
   void StopWorkThread(); ///< Stops the worker thread
   void WorkThread(); ///< Worker thread function
@@ -323,7 +323,7 @@ class MdfWriter {
   /** \brief Increment the sample counter. */
   void IncrementNofSamples(uint64_t record_id) const;
   /** \brief Set the last file position. */
-  virtual void SetLastPosition(std::FILE* file) = 0;
+  virtual void SetLastPosition(std::streambuf& buffer) = 0;
 
 
  private:

include/mdf/zlibutil.h

@@ -23,7 +23,7 @@ using ByteArray = std::vector<uint8_t>;  ///< Defines a dynamic byte array
  * @param out Pointer to the output file stream.
  * @return True on success.
  */
-bool Deflate(std::FILE* in, std::FILE* out);  ///< Compress file to file.
+bool Deflate(std::streambuf& in, std::streambuf& out);  ///< Compress file to file.
 
 /**
  * Compress a byte array directly to another array.
@@ -55,13 +55,13 @@ bool Deflate(const ByteArray& buf_in,
 bool Deflate(const std::string& filename,
              ByteArray& buf_out);  ///< Compress array to array.
 
-bool Inflate(std::FILE* in, std::FILE* out);  ///< Decompress file to file.
-bool Inflate(std::FILE* in, std::FILE* out,
+bool Inflate(std::streambuf& in, std::streambuf& out);  ///< Decompress file to file.
+bool Inflate(std::streambuf& in, std::streambuf& out,
              uint64_t nof_bytes);  ///< Decompress part of file to file
 bool Inflate(const ByteArray& in,
              ByteArray& out);  ///< Decompress array to array.
 bool Inflate(const ByteArray& in,
-             std::FILE* out);  ///< Decompress array to file.
+             std::streambuf& out);  ///< Decompress array to file.
 
 void Transpose(ByteArray& data,
                size_t record_size);  ///< Transpose of an array.

mdflib/src/at4block.cpp

@@ -6,6 +6,7 @@
 
 #include <cerrno>
 #include <sstream>
+#include <fstream>
 
 #include "mdf/cryptoutil.h"
 #include "mdf/mdfhelper.h"
@@ -46,15 +47,11 @@ std::string MakeFlagString(uint16_t flag) {
   return s.str();
 }
 
-bool CopyBytes(std::FILE* source, std::FILE* dest, uint64_t nof_bytes) {
+bool CopyBytes(std::streambuf& source, std::streambuf& dest, uint64_t nof_bytes) {
   uint8_t temp = 0;
   for (uint64_t ii = 0; ii < nof_bytes; ++ii) {
-    if (fread(&temp, 1, 1, source) != 1) {
-      return false;
-    }
-    if (fwrite(&temp, 1, 1, dest) != 1) {
-      return false;
-    }
+    temp = source.sbumpc();
+    dest.sputc(static_cast<char>(temp));
   }
   return true;
 }
@@ -144,27 +141,27 @@ void At4Block::GetBlockProperty(BlockPropertyList& dest) const {
   }
 }
 
-size_t At4Block::Read(std::FILE* file) {
-  size_t bytes = ReadHeader4(file);
-  bytes += ReadNumber(file, flags_);
-  bytes += ReadNumber(file, creator_index_);
+uint64_t At4Block::Read(std::streambuf& buffer) {
+  uint64_t bytes = ReadHeader4(buffer);
+  bytes += ReadNumber(buffer, flags_);
+  bytes += ReadNumber(buffer, creator_index_);
   std::vector<uint8_t> reserved;
-  bytes += ReadByte(file, reserved, 4);
-  bytes += ReadByte(file, md5_, 16);
-  bytes += ReadNumber(file, original_size_);
-  bytes += ReadNumber(file, nof_bytes_);
+  bytes += ReadByte(buffer, reserved, 4);
+  bytes += ReadByte(buffer, md5_, 16);
+  bytes += ReadNumber(buffer, original_size_);
+  bytes += ReadNumber(buffer, nof_bytes_);
   // Do not read in the data BLOB at this point but store the file position for
   // that data, so it is fast to get the data later
-  data_position_ = GetFilePosition(file);
+  data_position_ = GetFilePosition(buffer);
 
-  filename_ = ReadTx4(file, kIndexFilename);
-  file_type_ = ReadTx4(file, kIndexType);
-  ReadMdComment(file, kIndexMd);
+  filename_ = ReadTx4(buffer, kIndexFilename);
+  file_type_ = ReadTx4(buffer, kIndexType);
+  ReadMdComment(buffer, kIndexMd);
 
   return bytes;
 }
 
-size_t At4Block::Write(std::FILE* file) {
+uint64_t At4Block::Write(std::streambuf& buffer) {
   const bool update = FilePosition() > 0;
   if (update) {
     return block_size_;
@@ -190,8 +187,8 @@ size_t At4Block::Write(std::FILE* file) {
         return 0;
       }
     } else if (IsEmbedded()) {
-      const auto buffer = FileToBuffer(filename_, data_buffer);
-      if (!buffer) {
+      const auto read = FileToBuffer(filename_, data_buffer);
+      if (!read) {
         MDF_ERROR() << "File to buffer failure. File: " << filename;
         return 0;
       }
@@ -209,44 +206,47 @@ size_t At4Block::Write(std::FILE* file) {
   block_length_ = 24 + (4 * 8) + 2 + 2 + 4 + 16 + 8 + 8 + nof_bytes_;
   link_list_.resize(4, 0);
 
-  WriteTx4(file, kIndexFilename, filename_);
-  WriteTx4(file, kIndexType, file_type_);
-  WriteMdComment(file, kIndexMd);
+  WriteTx4(buffer, kIndexFilename, filename_);
+  WriteTx4(buffer, kIndexType, file_type_);
+  WriteMdComment(buffer, kIndexMd);
 
-  auto bytes = MdfBlock::Write(file);
-  bytes += WriteNumber(file, flags_);
-  bytes += WriteNumber(file, creator_index_);
-  bytes += WriteBytes(file, 4);
+  uint64_t bytes = MdfBlock::Write(buffer);
+  bytes += WriteNumber(buffer, flags_);
+  bytes += WriteNumber(buffer, creator_index_);
+  bytes += WriteBytes(buffer, 4);
   if (md5_.size() == 16) {
-    bytes += WriteByte(file, md5_);
+    bytes += WriteByte(buffer, md5_);
   } else {
-    bytes += WriteBytes(file, 16);
+    bytes += WriteBytes(buffer, 16);
   }
-  bytes += WriteNumber(file, original_size_);
-  bytes += WriteNumber(file, nof_bytes_);
+  bytes += WriteNumber(buffer, original_size_);
+  bytes += WriteNumber(buffer, nof_bytes_);
   data_position_ = FilePosition();
   if (nof_bytes_ > 0) {
-    bytes += WriteByte(file, data_buffer);
+    bytes += WriteByte(buffer, data_buffer);
   }
-  UpdateBlockSize(file, bytes);
+  UpdateBlockSize(buffer, bytes);
 
   return bytes;
 }
 
-void At4Block::ReadData(std::FILE* file, const std::string& dest_file) const {
-  if (file == nullptr || data_position_ <= 0) {
+void At4Block::ReadData(std::streambuf& buffer,
+                        const std::string& dest_file) const {
+  if (data_position_ <= 0) {
     throw std::invalid_argument("File is not opened or data position not read");
   }
-  SetFilePosition(file, data_position_);
+  SetFilePosition(buffer, data_position_);
   if (IsEmbedded()) {
-    FILE* dest = nullptr;
-    Platform::fileopen(&dest, dest_file.c_str(), "wb");
-    if (dest == nullptr) {
+    std::filebuf dest;
+    dest.open( dest_file,
+            std::ios_base::out | std::ios_base::trunc | std::ios_base::binary);
+    if (!dest.is_open()) {
       throw std::ios_base::failure("Failed to open the destination file");
     }
-    const bool error = IsCompressed() ? !Inflate(file, dest, nof_bytes_)
-                                      : !CopyBytes(file, dest, nof_bytes_);
-    if (const int close = fclose(dest); error || close != 0) {
+    const bool error = IsCompressed() ? !Inflate(buffer, dest, nof_bytes_)
+                                      : !CopyBytes(buffer, dest, nof_bytes_);
+    dest.close();
+    if ( error ) {
       throw std::ios_base::failure("Failed to copy correct number of bytes");
     }
   } else {

mdflib/src/at4block.h

@@ -51,11 +51,11 @@ class At4Block : public MdfBlock, public IAttachment {
   [[nodiscard]] bool IsCompressed() const override;
 
   void GetBlockProperty(BlockPropertyList& dest) const override;
-  size_t Read(std::FILE* file) override;
+  uint64_t Read(std::streambuf& buffer) override;
 
-  void ReadData(std::FILE* file, const std::string& dest_file) const;
+  void ReadData(std::streambuf& buffer, const std::string& dest_file) const;
 
-  size_t Write(std::FILE* file) override;
+  uint64_t Write(std::streambuf& buffer) override;
   [[nodiscard]] std::optional<std::string> Md5() const override;
 
  private: