From 8e3d0b07ec0db6863a2185683beff7d011d63fd3 Mon Sep 17 00:00:00 2001 From: Jose Date: Thu, 17 Oct 2024 10:10:47 +0200 Subject: [PATCH] Bring DataStorm into Ice 3.8 (#2902) --- cpp/include/DataStorm/Config.h | 26 + cpp/include/DataStorm/DataStorm.h | 2075 +++++++++++++++++ cpp/include/DataStorm/InternalI.h | 276 +++ cpp/include/DataStorm/InternalT.h | 576 +++++ cpp/include/DataStorm/Node.h | 210 ++ cpp/include/DataStorm/Types.h | 320 +++ cpp/msbuild/ice.sln | 27 + cpp/msbuild/ice.test.sln | 166 ++ cpp/src/DataStorm/CallbackExecutor.cpp | 78 + cpp/src/DataStorm/CallbackExecutor.h | 40 + cpp/src/DataStorm/ConnectionManager.cpp | 97 + cpp/src/DataStorm/ConnectionManager.h | 41 + cpp/src/DataStorm/Contract.ice | 247 ++ cpp/src/DataStorm/DataElementI.cpp | 1411 +++++++++++ cpp/src/DataStorm/DataElementI.h | 429 ++++ cpp/src/DataStorm/DataStorm.rc | 33 + cpp/src/DataStorm/ForwarderManager.cpp | 88 + cpp/src/DataStorm/ForwarderManager.h | 49 + cpp/src/DataStorm/Instance.cpp | 193 ++ cpp/src/DataStorm/Instance.h | 140 ++ cpp/src/DataStorm/LookupI.cpp | 74 + cpp/src/DataStorm/LookupI.h | 46 + cpp/src/DataStorm/Makefile.mk | 16 + cpp/src/DataStorm/Node.cpp | 93 + cpp/src/DataStorm/NodeI.cpp | 607 +++++ cpp/src/DataStorm/NodeI.h | 115 + cpp/src/DataStorm/NodeSessionI.cpp | 241 ++ cpp/src/DataStorm/NodeSessionI.h | 50 + cpp/src/DataStorm/NodeSessionManager.cpp | 451 ++++ cpp/src/DataStorm/NodeSessionManager.h | 92 + cpp/src/DataStorm/SessionI.cpp | 1389 +++++++++++ cpp/src/DataStorm/SessionI.h | 384 +++ cpp/src/DataStorm/TimerTaskI.h | 27 + cpp/src/DataStorm/TopicFactoryI.cpp | 320 +++ cpp/src/DataStorm/TopicFactoryI.h | 85 + cpp/src/DataStorm/TopicI.cpp | 1099 +++++++++ cpp/src/DataStorm/TopicI.h | 225 ++ cpp/src/DataStorm/TraceUtil.cpp | 24 + cpp/src/DataStorm/TraceUtil.h | 255 ++ .../msbuild/datastorm/datastorm.vcxproj | 270 +++ .../datastorm/datastorm.vcxproj.filters | 230 ++ .../msbuild/datastorm/packages.config | 4 + cpp/src/dsnode/Makefile.mk | 10 + cpp/src/dsnode/Node.cpp | 75 + cpp/src/dsnode/msbuild/dsnode.vcxproj | 105 + cpp/src/dsnode/msbuild/dsnode.vcxproj.filters | 16 + cpp/test/DataStorm/api/DuplicateSymbols.cpp | 18 + cpp/test/DataStorm/api/Makefile.mk | 10 + cpp/test/DataStorm/api/Test.ice | 17 + cpp/test/DataStorm/api/Writer.cpp | 321 +++ .../api/msbuild/writer/packages.config | 5 + .../api/msbuild/writer/writer.vcxproj | 163 ++ .../api/msbuild/writer/writer.vcxproj.filters | 90 + cpp/test/DataStorm/api/test.py | 17 + cpp/test/DataStorm/callbacks/Makefile.mk | 11 + cpp/test/DataStorm/callbacks/Reader.cpp | 360 +++ cpp/test/DataStorm/callbacks/Writer.cpp | 244 ++ .../callbacks/msbuild/reader/packages.config | 4 + .../callbacks/msbuild/reader/reader.vcxproj | 102 + .../msbuild/reader/reader.vcxproj.filters | 19 + .../callbacks/msbuild/writer/packages.config | 4 + .../callbacks/msbuild/writer/writer.vcxproj | 102 + .../msbuild/writer/writer.vcxproj.filters | 19 + cpp/test/DataStorm/callbacks/test.py | 17 + cpp/test/DataStorm/config/Makefile.mk | 11 + cpp/test/DataStorm/config/Reader.cpp | 317 +++ cpp/test/DataStorm/config/Writer.cpp | 332 +++ .../config/msbuild/reader/packages.config | 4 + .../config/msbuild/reader/reader.vcxproj | 102 + .../msbuild/reader/reader.vcxproj.filters | 19 + .../config/msbuild/writer/packages.config | 4 + .../config/msbuild/writer/writer.vcxproj | 102 + .../msbuild/writer/writer.vcxproj.filters | 19 + cpp/test/DataStorm/config/test.py | 17 + cpp/test/DataStorm/events/Makefile.mk | 11 + cpp/test/DataStorm/events/Reader.cpp | 395 ++++ cpp/test/DataStorm/events/Test.ice | 24 + cpp/test/DataStorm/events/Writer.cpp | 394 ++++ .../events/msbuild/reader/packages.config | 5 + .../events/msbuild/reader/reader.vcxproj | 163 ++ .../msbuild/reader/reader.vcxproj.filters | 88 + .../events/msbuild/writer/packages.config | 5 + .../events/msbuild/writer/writer.vcxproj | 163 ++ .../msbuild/writer/writer.vcxproj.filters | 88 + cpp/test/DataStorm/events/test.py | 19 + cpp/test/DataStorm/partial/Makefile.mk | 11 + cpp/test/DataStorm/partial/Reader.cpp | 69 + cpp/test/DataStorm/partial/Test.ice | 14 + cpp/test/DataStorm/partial/Writer.cpp | 42 + .../partial/msbuild/reader/packages.config | 5 + .../partial/msbuild/reader/reader.vcxproj | 163 ++ .../msbuild/reader/reader.vcxproj.filters | 88 + .../partial/msbuild/writer/packages.config | 5 + .../partial/msbuild/writer/writer.vcxproj | 163 ++ .../msbuild/writer/writer.vcxproj.filters | 88 + cpp/test/DataStorm/partial/test.py | 17 + cpp/test/DataStorm/reliability/Makefile.mk | 11 + cpp/test/DataStorm/reliability/Reader.cpp | 64 + cpp/test/DataStorm/reliability/Writer.cpp | 54 + .../msbuild/reader/packages.config | 4 + .../reliability/msbuild/reader/reader.vcxproj | 102 + .../msbuild/reader/reader.vcxproj.filters | 19 + .../msbuild/writer/packages.config | 4 + .../reliability/msbuild/writer/writer.vcxproj | 102 + .../msbuild/writer/writer.vcxproj.filters | 19 + cpp/test/DataStorm/reliability/test.py | 81 + cpp/test/DataStorm/types/Makefile.mk | 11 + cpp/test/DataStorm/types/Reader.cpp | 119 + cpp/test/DataStorm/types/Test.ice | 24 + cpp/test/DataStorm/types/Writer.cpp | 135 ++ .../types/msbuild/reader/packages.config | 5 + .../types/msbuild/reader/reader.vcxproj | 163 ++ .../msbuild/reader/reader.vcxproj.filters | 88 + .../types/msbuild/writer/packages.config | 5 + .../types/msbuild/writer/writer.vcxproj | 163 ++ .../msbuild/writer/writer.vcxproj.filters | 90 + cpp/test/DataStorm/types/test.py | 17 + scripts/DataStormUtil.py | 98 + scripts/Util.py | 2 + slice/DataStorm/Sample.ice | 38 + 120 files changed, 18713 insertions(+) create mode 100644 cpp/include/DataStorm/Config.h create mode 100644 cpp/include/DataStorm/DataStorm.h create mode 100644 cpp/include/DataStorm/InternalI.h create mode 100644 cpp/include/DataStorm/InternalT.h create mode 100644 cpp/include/DataStorm/Node.h create mode 100644 cpp/include/DataStorm/Types.h create mode 100644 cpp/src/DataStorm/CallbackExecutor.cpp create mode 100644 cpp/src/DataStorm/CallbackExecutor.h create mode 100644 cpp/src/DataStorm/ConnectionManager.cpp create mode 100644 cpp/src/DataStorm/ConnectionManager.h create mode 100644 cpp/src/DataStorm/Contract.ice create mode 100644 cpp/src/DataStorm/DataElementI.cpp create mode 100644 cpp/src/DataStorm/DataElementI.h create mode 100644 cpp/src/DataStorm/DataStorm.rc create mode 100644 cpp/src/DataStorm/ForwarderManager.cpp create mode 100644 cpp/src/DataStorm/ForwarderManager.h create mode 100644 cpp/src/DataStorm/Instance.cpp create mode 100644 cpp/src/DataStorm/Instance.h create mode 100644 cpp/src/DataStorm/LookupI.cpp create mode 100644 cpp/src/DataStorm/LookupI.h create mode 100644 cpp/src/DataStorm/Makefile.mk create mode 100644 cpp/src/DataStorm/Node.cpp create mode 100644 cpp/src/DataStorm/NodeI.cpp create mode 100644 cpp/src/DataStorm/NodeI.h create mode 100644 cpp/src/DataStorm/NodeSessionI.cpp create mode 100644 cpp/src/DataStorm/NodeSessionI.h create mode 100644 cpp/src/DataStorm/NodeSessionManager.cpp create mode 100644 cpp/src/DataStorm/NodeSessionManager.h create mode 100644 cpp/src/DataStorm/SessionI.cpp create mode 100644 cpp/src/DataStorm/SessionI.h create mode 100644 cpp/src/DataStorm/TimerTaskI.h create mode 100644 cpp/src/DataStorm/TopicFactoryI.cpp create mode 100644 cpp/src/DataStorm/TopicFactoryI.h create mode 100644 cpp/src/DataStorm/TopicI.cpp create mode 100644 cpp/src/DataStorm/TopicI.h create mode 100644 cpp/src/DataStorm/TraceUtil.cpp create mode 100644 cpp/src/DataStorm/TraceUtil.h create mode 100644 cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj create mode 100644 cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj.filters create mode 100644 cpp/src/DataStorm/msbuild/datastorm/packages.config create mode 100644 cpp/src/dsnode/Makefile.mk create mode 100644 cpp/src/dsnode/Node.cpp create mode 100644 cpp/src/dsnode/msbuild/dsnode.vcxproj create mode 100644 cpp/src/dsnode/msbuild/dsnode.vcxproj.filters create mode 100644 cpp/test/DataStorm/api/DuplicateSymbols.cpp create mode 100644 cpp/test/DataStorm/api/Makefile.mk create mode 100644 cpp/test/DataStorm/api/Test.ice create mode 100644 cpp/test/DataStorm/api/Writer.cpp create mode 100644 cpp/test/DataStorm/api/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/api/test.py create mode 100644 cpp/test/DataStorm/callbacks/Makefile.mk create mode 100644 cpp/test/DataStorm/callbacks/Reader.cpp create mode 100644 cpp/test/DataStorm/callbacks/Writer.cpp create mode 100644 cpp/test/DataStorm/callbacks/msbuild/reader/packages.config create mode 100644 cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj create mode 100644 cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj.filters create mode 100644 cpp/test/DataStorm/callbacks/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/callbacks/test.py create mode 100644 cpp/test/DataStorm/config/Makefile.mk create mode 100644 cpp/test/DataStorm/config/Reader.cpp create mode 100644 cpp/test/DataStorm/config/Writer.cpp create mode 100644 cpp/test/DataStorm/config/msbuild/reader/packages.config create mode 100644 cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj create mode 100644 cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj.filters create mode 100644 cpp/test/DataStorm/config/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/config/test.py create mode 100644 cpp/test/DataStorm/events/Makefile.mk create mode 100644 cpp/test/DataStorm/events/Reader.cpp create mode 100644 cpp/test/DataStorm/events/Test.ice create mode 100644 cpp/test/DataStorm/events/Writer.cpp create mode 100644 cpp/test/DataStorm/events/msbuild/reader/packages.config create mode 100644 cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj create mode 100644 cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj.filters create mode 100644 cpp/test/DataStorm/events/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/events/test.py create mode 100644 cpp/test/DataStorm/partial/Makefile.mk create mode 100644 cpp/test/DataStorm/partial/Reader.cpp create mode 100644 cpp/test/DataStorm/partial/Test.ice create mode 100644 cpp/test/DataStorm/partial/Writer.cpp create mode 100644 cpp/test/DataStorm/partial/msbuild/reader/packages.config create mode 100644 cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj create mode 100644 cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj.filters create mode 100644 cpp/test/DataStorm/partial/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/partial/test.py create mode 100644 cpp/test/DataStorm/reliability/Makefile.mk create mode 100644 cpp/test/DataStorm/reliability/Reader.cpp create mode 100644 cpp/test/DataStorm/reliability/Writer.cpp create mode 100644 cpp/test/DataStorm/reliability/msbuild/reader/packages.config create mode 100644 cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj create mode 100644 cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj.filters create mode 100644 cpp/test/DataStorm/reliability/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/reliability/test.py create mode 100644 cpp/test/DataStorm/types/Makefile.mk create mode 100644 cpp/test/DataStorm/types/Reader.cpp create mode 100644 cpp/test/DataStorm/types/Test.ice create mode 100644 cpp/test/DataStorm/types/Writer.cpp create mode 100644 cpp/test/DataStorm/types/msbuild/reader/packages.config create mode 100644 cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj create mode 100644 cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj.filters create mode 100644 cpp/test/DataStorm/types/msbuild/writer/packages.config create mode 100644 cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj create mode 100644 cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj.filters create mode 100644 cpp/test/DataStorm/types/test.py create mode 100644 scripts/DataStormUtil.py create mode 100644 slice/DataStorm/Sample.ice diff --git a/cpp/include/DataStorm/Config.h b/cpp/include/DataStorm/Config.h new file mode 100644 index 00000000000..98942944a2f --- /dev/null +++ b/cpp/include/DataStorm/Config.h @@ -0,0 +1,26 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_CONFIG_H +#define DATASTORM_CONFIG_H + +#include "Ice/Config.h" + +#if !defined(ICE_BUILDING_DATASTORM) && defined(DATASTORM_API_EXPORTS) +# define ICE_BUILDING_DATASTORM +#endif + +#if defined(_MSC_VER) && !defined(ICE_BUILDING_DATASTORM) +# pragma comment(lib, ICE_LIBNAME("DataStorm")) // Automatically link with DataStorm[D].lib +#endif + +#ifndef DATASTORM_API +# if defined(DATASTORM_API_EXPORTS) +# define DATASTORM_API ICE_DECLSPEC_EXPORT +# else +# define DATASTORM_API ICE_DECLSPEC_IMPORT +# endif +#endif + +#endif diff --git a/cpp/include/DataStorm/DataStorm.h b/cpp/include/DataStorm/DataStorm.h new file mode 100644 index 00000000000..e409baf47ff --- /dev/null +++ b/cpp/include/DataStorm/DataStorm.h @@ -0,0 +1,2075 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_DATASTORM_H +#define DATASTORM_DATASTORM_H + +#include "DataStorm/Config.h" +#include "DataStorm/InternalI.h" +#include "DataStorm/InternalT.h" +#include "DataStorm/Node.h" +#include "DataStorm/Sample.h" +#include "DataStorm/Types.h" + +#include + +namespace DataStorm +{ + + /** + * A sample provides information about a data element update. + * + * The Sample template provides access to the key, value as well as additional information such as the event, + * timestamp, update tag. Samples are generated and published by writers and received by readers. + * + * @headerfile DataStorm/DataStorm.h + */ + template class Sample + { + public: + /** + * The type of the sample key. + */ + using KeyType = Key; + + /** + * The type of the sample value. + */ + using ValueType = Value; + + /** + * The type of the update tag. The update tag type defaults to string if it's not explicitly specified + * with the Sample template parameters. + */ + using UpdateTagType = UpdateTag; + + /** + * The event associated with the sample. + * + * @return The sample event. + */ + SampleEvent getEvent() const noexcept; + + /** + * The key of the sample. + * + * @return The sample key. + */ + const Key& getKey() const noexcept; + + /** + * The value of the sample. + * + * Depending on the sample event, the sample value might not always be available. It's the case if the + * sample event is Remove where this method will return a default value. + * + * @return The sample value. + */ + const Value& getValue() const noexcept; + + /** + * The update tag for the partial update. + * + * This method should only be called if the sample event is PartialUpdate. + * + * @return The update tag. + */ + UpdateTag getUpdateTag() const noexcept; + + /** + * The timestamp of the sample. + * + * The timestamp is generated by the writer and corresponds to the time of sending. + * + * @return The timestamp. + */ + std::chrono::time_point getTimeStamp() const noexcept; + + /** + * The origin of the sample. + * + * The origin of the sample identifies uniquely on the node the writer that created the sample. It's the + * name of the writer if a name was explicitly provided on creation of the writer. Otherwise, if no name + * was provided, an unique identifier is generated by DataStorm. + * + * @return The origin of the sample. + */ + std::string getOrigin() const noexcept; + + /** + * Get the session identifier of the session that received this sample. + * + * This session identifier can be used to retrieve the Ice connection with the node. + * + * @return The session identifier. + */ + std::string getSession() const noexcept; + + /** @private */ + Sample(const std::shared_ptr&) noexcept; + + private: + std::shared_ptr> _impl; + }; + + /** + * Convert the given sample type to a string and add it to the stream. + * + * @param os The output stream + * @param sampleType The sample type to add to the stream + * @return The output stream + */ + inline std::ostream& operator<<(std::ostream& os, SampleEvent sampleType) + { + switch (sampleType) + { + case SampleEvent::Add: + os << "Add"; + break; + case SampleEvent::Update: + os << "Update"; + break; + case SampleEvent::Remove: + os << "Remove"; + break; + case SampleEvent::PartialUpdate: + os << "PartialUpdate"; + break; + default: + os << static_cast(sampleType); + break; + } + return os; + } + + /** + * Convert the given sample type vector to a string and add it to the stream. + * + * @param os The output stream + * @param types The sample type vector to add to the stream + * @return The output stream + */ + inline std::ostream& operator<<(std::ostream& os, const std::vector& types) + { + os << "["; + for (auto p = types.begin(); p != types.end(); ++p) + { + if (p != types.begin()) + { + os << ','; + } + os << *p; + } + os << "]"; + return os; + } + + /** + * Convert the given sample to a string and add it to the stream. The implementation outputs the sample value. + * + * @param os The output stream + * @param sample The sample to add to the stream + * @return The output stream + */ + template + std::ostream& operator<<(std::ostream& os, const Sample& sample) + { + os << sample.getValue(); + return os; + } + + /** + * The Reader class is used to retrieve samples for a data element. + * + * @headerfile DataStorm/DataStorm.h + */ + template class Reader + { + public: + /** + * The key type. + */ + using KeyType = Key; + + /** + * The value type. + */ + using ValueType = Value; + + /** + * Transfers the given reader to this reader. + * + * @param reader The reader. + **/ + Reader(Reader&& reader) noexcept; + + /** + * Destruct the reader. The destruction of the reader disconnects the reader from the writers. + */ + ~Reader(); + + /** + * Move assignment operator + * + * @param reader The reader. + **/ + Reader& operator=(Reader&& reader) noexcept; + + /** + * Indicates whether or not writers are online. + * + * @return True if writers are connected, false otherwise. + */ + bool hasWriters() const noexcept; + + /** + * Wait for given number of writers to be online. The node shutdown will cause this method to raise + * NodeShutdownException. + * + * @param count The number of writers to wait. + */ + void waitForWriters(unsigned int count = 1) const; + + /** + * Wait for readers to be offline. The node shutdown will cause this method to raise + * NodeShutdownException. + */ + void waitForNoWriters() const; + + /** + * Get the connected writers. + * + * @return The names of the connected writers. + */ + std::vector getConnectedWriters() const noexcept; + + /** + * Get the keys for which writers are connected to this reader. + * + * @return The keys for which we have writers connected. + **/ + std::vector getConnectedKeys() const noexcept; + + /** + * Returns all the unread samples. + * + * @return The unread samples. + */ + std::vector> getAllUnread() noexcept; + + /** + * Wait for given number of unread samples to be available. The node shutdown will cause this method to + * raise NodeShutdownException. + */ + void waitForUnread(unsigned int count = 1) const; + + /** + * Returns wether or not unread samples are available. + * + * @return True if there unread samples are queued, false otherwise. + */ + bool hasUnread() const noexcept; + + /** + * Returns the next unread sample. The node shutdown will cause this method to raise + * NodeShutdownException. + * + * @return The unread sample. + */ + Sample getNextUnread(); + + /** + * Calls the given functions to provide the initial set of connected keys and when a key is added or + * removed from the set of connected keys. If callback functions are already set, they will be replaced. + * + * The connected keys represent the set of keys for which writers are connected to this reader. + * + * The init callback is always called after this method returns to provide the initial set of connected + * keys. The update callback is called when new keys are added or removed from the set of connected keys. + * + * @param init The function to call with the initial set of connected keys. + * @param update The function to call when a key is added or removed from the set. + **/ + void onConnectedKeys( + std::function)> init, + std::function update) noexcept; + + /** + * Calls the given functions to provide the initial set of connected writers and when a new writer + * connects or disconnects. If callback functions are already set, they will be replaced. + * + * The init callback is always called after this method returns to provide the initial set of connected + * writers. The update callback is called when new writers connect or disconnect. + * + * @param init The function to call with the initial set of connected writers. + * @param update The function to call when a new writer connects or disconnects. + **/ + void onConnectedWriters( + std::function)> init, + std::function update) noexcept; + + /** + * Calls the given function to provide the initial set of unread samples and when new samples are queued. + * + * If a function is already set, it will be replaced. + * + * The init callback is always called after this method returns to provide the initial set of unread + * samples. The queue callback is called when a new sample is received. + * + * @param init The function to call with the initial set of unread samples. + * @param queue The function to call when a new sample is received. + **/ + void onSamples( + std::function>)> init, + std::function)> queue) noexcept; + + protected: + /** @private */ + Reader(const std::shared_ptr& impl) noexcept : _impl(impl) {} + + /** @private */ + std::shared_ptr _impl; + }; + + /** + * The writer class is used to write samples for a data element. + * + * @headerfile DataStorm/DataStorm.h + */ + template class Writer + { + public: + /** + * The key type. + */ + using KeyType = Key; + + /** + * The value type. + */ + using ValueType = Value; + + /** + * Transfers the given writer to this writer. + * + * @param writer The writer. + **/ + Writer(Writer&& writer) noexcept; + + /** + * Move assignment operator. + * + * @param writer The writer. + **/ + Writer& operator=(Writer&& writer) noexcept; + + /** + * Destruct the writer. The destruction of the writer disconnects the writer from the readers. + */ + ~Writer(); + + /** + * Indicates whether or not readers are online. + * + * @return True if readers are connected, false otherwise. + */ + bool hasReaders() const noexcept; + + /** + * Wait for given number of readers to be online. The node shutdown will cause this method to raise + * NodeShutdownException. + * + * @param count The number of readers to wait. + */ + void waitForReaders(unsigned int count = 1) const; + + /** + * Wait for readers to be offline. The node shutdown this method to raise NodeShutdownException. + */ + void waitForNoReaders() const; + + /** + * Get the connected readers. + * + * @return The names of the connected readers. + */ + std::vector getConnectedReaders() const noexcept; + + /** + * Get the keys for which readers are connected to this writer. + * + * @return The keys for which we have writers connected. + **/ + std::vector getConnectedKeys() const noexcept; + + /** + * Get the last written sample. If there's no sample, the std::logic_error exception is raised. + * + * @return The last written sample. + **/ + Sample getLast(); + + /** + * Get all the written sample kept in the writer history. + * + * @return The sample history. + **/ + std::vector> getAll() noexcept; + + /** + * Calls the given functions to provide the initial set of connected keys and when a key is added or + * removed from the set of connected keys. If callback functions are already set, they will be replaced. + * + * The connected keys represent the set of keys for which writers are connected to this reader. + * + * The init callback is always called after this method returns to provide the initial set of connected + * keys. The update callback is called when new keys are added or removed from the set of connected keys. + * + * @param init The function to call with the initial set of connected keys. + * @param update The function to call when a key is added or removed from the set. + **/ + void onConnectedKeys( + std::function)> init, + std::function update) noexcept; + + /** + * Calls the given functions to provide the initial set of connected readers and when a new reader + * connects or disconnects. If callback functions are already set, they will be replaced. + * + * The init callback is always called after this method returns to provide the initial set of connected + * readers. The update callback is called when new readers connect or disconnect. + * + * @param init The function to call with the initial set of connected readers. + * @param update The function to call when a new reader connects or disconnects. + **/ + void onConnectedReaders( + std::function)> init, + std::function update) noexcept; + + protected: + /** @private */ + Writer(const std::shared_ptr& impl) noexcept : _impl(impl) {} + + /** @private */ + std::shared_ptr _impl; + }; + + /** + * The Topic class. + * + * This class allows constructing reader and writer objects. It's also used to setup filter and updater + * functions. + * + * @headerfile DataStorm/DataStorm.h + */ + template class Topic + { + public: + /** + * The topic's key type. + */ + using KeyType = Key; + + /** + * The topic's value type. + */ + using ValueType = Value; + + /** + * The topic's update tag type (defaults to std::string if not specified). + */ + using UpdateTagType = UpdateTag; + + /** + * The topic's writer type. + */ + using WriterType = Writer; + + /** + * The topic's reader type. + */ + using ReaderType = Reader; + + /** + * The topic's sample type. + */ + using SampleType = Sample; + + /** + * Construct a new Topic for the topic with the given name. + * + * @param node The node. + * @param name The name of the topic. + */ + Topic(const Node& node, const std::string& name) noexcept; + + /** + * Construct a new Topic by taking ownership of the given topic. + * + * @param topic The topic to transfer ownership from. + */ + Topic(Topic&& topic) noexcept + : _name(std::move(topic._name)), + _topicFactory(std::move(topic._topicFactory)), + _keyFactory(std::move(topic._keyFactory)), + _tagFactory(std::move(topic._tagFactory)), + _keyFilterFactories(std::move(topic._keyFilterFactories)), + _sampleFilterFactories(std::move(topic._sampleFilterFactories)), + _reader(std::move(topic._reader)), + _writer(std::move(topic._writer)), + _updaters(std::move(topic._updaters)) + { + } + + /** + * Destruct the Topic. This disconnects the topic from peers. + */ + ~Topic(); + + /** + * Move assignment operator. + * + * @param topic The topic. + **/ + Topic& operator=(Topic&& topic) noexcept; + + /** + * Indicates whether or not data writers are online. + * + * @return True if data writers are connected, false otherwise. + */ + bool hasWriters() const noexcept; + + /** + * Wait for given number of data writers to be online. The node shutdown will cause this method to raise + * NodeShutdownException. + * + * @param count The number of date writers to wait. + */ + void waitForWriters(unsigned int count = 1) const; + + /** + * Wait for data writers to be offline. The node shutdown will cause this method to raise + * NodeShutdownException. + */ + void waitForNoWriters() const; + + /** + * Set the default configuration used to construct readers. + * + * @param config The default writer configuration. + */ + void setWriterDefaultConfig(const WriterConfig& config) noexcept; + + /** + * Indicates whether or not data readers are online. + * + * @return True if data readers are connected, false otherwise. + */ + bool hasReaders() const noexcept; + + /** + * Wait for given number of data readers to be online. The node shutdown will cause this method to raise + * NodeShutdownException. + * + * @param count The number of data readers to wait. + */ + void waitForReaders(unsigned int count = 1) const; + + /** + * Wait for data readers to be offline. The node shutdown will cause this method to raise + * NodeShutdownException. + */ + void waitForNoReaders() const; + + /** + * Set the default configuration used to construct readers. + * + * @param config The default reader configuration. + */ + void setReaderDefaultConfig(const ReaderConfig& config) noexcept; + + /** + * Set an updater function for the given update tag. The function is called when a partial update is + * received or sent to compute the new value. The function is provided the latest value and the partial + * update. It should return the new value. + * + * @param tag The update tag. + * @param updater The updater function. + */ + template + void setUpdater(const UpdateTag& tag, std::function updater) noexcept; + + /** + * Set a key filter factory. The given factory function must return a filter function that returns true if + * the key matches the filter criteria, false otherwise. + * + * @param name The name of the key filter. + * @param factory The filter factory function. + */ + template + void setKeyFilter( + const std::string& name, + std::function(const Criteria&)> factory) noexcept; + + /** + * Set a sample filter factory. The given factory function must return a filter function that returns true + * if the sample matches the filter criteria, false otherwise. + * + * @param name The name of the sample filter. + * @param factory The filter factory function. + */ + template + void setSampleFilter( + const std::string& name, + std::function(const Criteria&)> factory) noexcept; + + private: + std::shared_ptr getReader() const noexcept; + std::shared_ptr getWriter() const noexcept; + Ice::CommunicatorPtr getCommunicator() const noexcept; + + template friend class SingleKeyWriter; + template friend class MultiKeyWriter; + template friend class SingleKeyReader; + template friend class MultiKeyReader; + template friend class FilteredKeyReader; + + const std::string _name; + const std::shared_ptr _topicFactory; + const std::shared_ptr> _keyFactory; + const std::shared_ptr> _tagFactory; + + const std::shared_ptr>> _keyFilterFactories; + const std::shared_ptr>> + _sampleFilterFactories; + + mutable std::mutex _mutex; + mutable std::shared_ptr _reader; + mutable std::shared_ptr _writer; + mutable std::map, DataStormI::Topic::Updater> _updaters; + }; + + /** + * Filter structure to specify the filter name and criteria value. + * + * @headerfile DataStorm/DataStorm.h + */ + template struct Filter + { + /** + * Construct a filter structure with the given name and criteria. + * + * @param name The filter name + * @param criteria The criteria + */ + template + Filter(const std::string& name, TT&& criteria) noexcept : name(name), + criteria(std::forward(criteria)) + { + } + + /** The filter name. */ + std::string name; + + /** The filter criteria value. */ + T criteria; + }; + + /** + * The key reader to read the data element associated with a given key. + * + * @headerfile DataStorm/DataStorm.h + */ + template + class SingleKeyReader : public Reader + { + public: + /** + * Construct a new reader for the given key. The construction of the reader connects the reader to writers + * with a matching key. + * + * @param topic The topic. + * @param key The key of the data element to read. + * @param name The optional reader name. + * @param config The reader configuration. + */ + SingleKeyReader( + const Topic& topic, + const Key& key, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept; + + /** + * Construct a new reader for the given key and sample filter criteria. The construction of the reader + * connects the reader to writers with a matching key. The writer will only send samples matching the + * given sample filter criteria to the reader. + * + * @param topic The topic. + * @param key The key of the data element to read. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The reader configuration. + */ + template + SingleKeyReader( + const Topic& topic, + const Key& key, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept; + + /** + * Transfers the given reader to this reader. + * + * @param reader The reader. + **/ + SingleKeyReader(SingleKeyReader&& reader) noexcept; + + /** + * Move assignment operator. + * + * @param reader The reader. + **/ + SingleKeyReader& operator=(SingleKeyReader&& reader) noexcept; + }; + + /** + * The key reader to read the data element associated with a given set of keys. + * + * @headerfile DataStorm/DataStorm.h + */ + template + class MultiKeyReader : public Reader + { + public: + /** + * Construct a new reader for the given keys. The construction of the reader connects the reader to + * writers with matching keys. If an empty vector of keys is provided, the reader will connect to all the + * available writers. + * + * @param topic The topic. + * @param keys The keys of the data elements to read. + * @param name The optional reader name. + * @param config The reader configuration. + */ + MultiKeyReader( + const Topic& topic, + const std::vector& keys, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept; + + /** + * Construct a new reader for the given keys and sample filter criteria. The construction of the reader + * connects the reader to writers with matching keys. If an empty vector of keys is provided, the reader + * will connect to all the available writers. The writer will only send samples matching the given sample + * filter criteria to the reader. + * + * @param topic The topic. + * @param keys The keys of the data elements to read. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The reader configuration. + */ + template + MultiKeyReader( + const Topic& topic, + const std::vector& keys, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept; + + /** + * Transfers the given reader to this reader. + * + * @param reader The reader. + **/ + MultiKeyReader(MultiKeyReader&& reader) noexcept; + + /** + * Move assignment operator. + * + * @param reader The reader. + **/ + MultiKeyReader& operator=(MultiKeyReader&& reader) noexcept; + }; + + /** + * Creates a key reader for the given topic and key. This helper method deduces the topic Key, Value and + * UpdateTag types from the topic argument. + * + * @param topic The topic. + * @param key The key. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + SingleKeyReader makeSingleKeyReader( + const Topic& topic, + const typename Topic::KeyType& key, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept + { + return SingleKeyReader(topic, key, name, config); + } + + /** + * Creates a key reader for the given topic, key and sample filter. This helper method deduces the topic Key + * and Value types from the topic argument. + * + * @param topic The topic. + * @param key The key. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + SingleKeyReader makeSingleKeyReader( + const Topic& topic, + const typename Topic::KeyType& key, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept + { + return SingleKeyReader(topic, key, sampleFilter, name, config); + } + + /** + * Creates a multi-key reader for the given topic. This helper method deduces the topic Key, Value and + * UpdateTag types from the topic argument. + * + * The reader will only receive samples for the given set of keys. + * + * @param topic The topic. + * @param keys The keys. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + MultiKeyReader makeMultiKeyReader( + const Topic& topic, + const std::vector::KeyType>& keys, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept + { + return MultiKeyReader(topic, keys, name, config); + } + + /** + * Creates a multi-key reader for the given topic, keys and sample filter. This helper method deduces the + * topic Key and Value types from the topic argument. + * + * The reader will only receive samples for the given set of keys. + * + * @param topic The topic. + * @param keys The keys. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + MultiKeyReader makeMultiKeyReader( + const Topic& topic, + const std::vector::KeyType>& keys, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept + { + return MultiKeyReader(topic, keys, sampleFilter, name, config); + } + + /** + * Creates an any-key reader for the given topic. This helper method deduces the topic Key, Value and + * UpdateTag types from the topic argument. + * + * The reader will receive samples for any keys from the topic. + * + * @param topic The topic. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + MultiKeyReader makeAnyKeyReader( + const Topic& topic, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept + { + return MultiKeyReader(topic, {}, name, config); + } + + /** + * Creates an any-key reader for the given topic and sample filter. This helper method deduces the topic Key + * and Value types from the topic argument. + * + * The reader will receive samples for the keys from the topic. + * + * @param topic The topic. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + MultiKeyReader makeAnyKeyReader( + const Topic& topic, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) noexcept + { + return MultiKeyReader(topic, {}, sampleFilter, name, config); + } + + /** + * The filtered reader to read data elements whose key match a given filter. + * + * @headerfile DataStorm/DataStorm.h + */ + template + class FilteredKeyReader : public Reader + { + public: + /** + * Construct a new reader for the given key filter. The construction of the reader connects the reader to + * writers whose key matches the key filter criteria. + * + * If the key filter is not registered with the topic or the filter invalid, std::invalid_argument is + * raised. + * + * @param topic The topic. + * @param keyFilter The key filter. + * @param name The optional reader name. + * @param config The reader configuration. + */ + template + FilteredKeyReader( + const Topic& topic, + const Filter& keyFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()); + + /** + * Construct a new reader for the given key filter and sample filter criteria. The construction of the + * reader connects the reader to writers whose key matches the key filter criteria. + * + * If the key filter is not registered with the topic or the filter invalid, std::invalid_argument is + * raised. + * + * @param topic The topic. + * @param keyFilter The key filter. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The reader configuration. + */ + template + FilteredKeyReader( + const Topic& topic, + const Filter& keyFilter, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()); + + /** + * Transfers the given reader to this reader. + * + * @param reader The reader. + **/ + FilteredKeyReader(FilteredKeyReader&& reader) noexcept; + + /** + * Move assignment operator. + * + * @param reader The reader. + **/ + FilteredKeyReader& operator=(FilteredKeyReader&& reader) noexcept; + }; + + /** + * Creates a new filtered reader for the given topic and key filter. This helper method deduces the topic Key, + * Value and UpdateTag types from the topic argument. + * + * @param topic The topic. + * @param filter The key filter. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + FilteredKeyReader makeFilteredKeyReader( + const Topic& topic, + const Filter& filter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) + { + return FilteredKeyReader(topic, filter, name, config); + } + + /** + * Creates a new filter reader for the given topic, key filter and sample filter. This helper method deduces + * the topic Key, Value and UpdateTag types from the topic argument. + * + * @param topic The topic. + * @param keyFilter The key filter. + * @param sampleFilter The sample filter. + * @param name The optional reader name. + * @param config The optional reader configuration. + */ + template + FilteredKeyReader makeFilteredKeyReader( + const Topic& topic, + const Filter& keyFilter, + const Filter& sampleFilter, + const std::string& name = std::string(), + const ReaderConfig& config = ReaderConfig()) + { + return FilteredKeyReader(topic, keyFilter, sampleFilter, name, config); + } + + /** + * The key writer to write the data element associated with a given key. + * + * @headerfile DataStorm/DataStorm.h + */ + template + class SingleKeyWriter : public Writer + { + public: + /** + * Construct a new writer for the given key. The construction of the writer connects the writer to readers + * with a matching key. + * + * @param topic The topic. + * @param key The key of the data element to write. + * @param name The optional writer name. + * @param config The writer configuration. + */ + SingleKeyWriter( + const Topic& topic, + const Key& key, + const std::string& name = std::string(), + const WriterConfig& config = WriterConfig()) noexcept; + + /** + * Move constructor. + * + * @param writer The writer. + **/ + SingleKeyWriter(SingleKeyWriter&& writer) noexcept; + + /** + * Move assignment operator. + * + * @param writer The writer. + **/ + SingleKeyWriter& operator=(SingleKeyWriter&& writer) noexcept; + + /** + * Add the data element. This generates an {@link Add} sample with the + * given value. + * + * @param value The data element value. + */ + void add(const Value& value) noexcept; + + /** + * Update the data element. This generates an {@link Update} sample with the + * given value. + * + * @param value The data element value. + */ + void update(const Value& value) noexcept; + + /** + * Get a partial update generator function for the given partial update tag. When called, the returned + * function generates a {@link PartialUpdate} sample with the given partial update value. + * + * The UpdateValue template parameter must match the UpdateValue type used to register the updater with + * the {@link Topic::setUpdater} method. + * + * @param tag The partial update tag. + */ + template + std::function partialUpdate(const UpdateTag& tag) noexcept; + + /** + * Remove the data element. This generates a {@link Remove} sample. + */ + void remove() noexcept; + + private: + const std::shared_ptr> _tagFactory; + }; + + /** + * The key writer to write data elements associated with a given set of keys. + * + * @headerfile DataStorm/DataStorm.h + */ + template + class MultiKeyWriter : public Writer + { + public: + /** + * Construct a new writer for the given keys. The construction of the writer connects the writer to + * readers with matching keys. If an empty vector of keys is provided, the writer will connect to all the + * available readers. + * + * @param topic The topic. + * @param keys The keys. + * @param name The optional writer name. + * @param config The writer configuration. + */ + MultiKeyWriter( + const Topic& topic, + const std::vector& keys, + const std::string& name = std::string(), + const WriterConfig& config = WriterConfig()) noexcept; + + /** + * Transfers the given writer to this writer. + * + * @param writer The writer. + **/ + MultiKeyWriter(MultiKeyWriter&& writer) noexcept; + + /** + * Move assignment operator. + * + * @param writer The writer. + **/ + MultiKeyWriter& operator=(MultiKeyWriter&& writer) noexcept; + + /** + * Add the data element. This generates an {@link Add} sample with the given value. + * + * @param key The key + * @param value The data element value. + */ + void add(const Key& key, const Value& value) noexcept; + + /** + * Update the data element. This generates an {@link Update} sample with the given value. + * + * @param key The key + * @param value The data element value. + */ + void update(const Key& key, const Value& value) noexcept; + + /** + * Get a partial update generator function for the given partial update tag. When called, the returned + * function generates a {@link PartialUpdate} sample with the given partial update value. + * + * The UpdateValue template parameter must match the UpdateValue type used to register the updater with + * the {@link Topic::setUpdater} method. + * + * @param tag The partial update tag. + */ + template + std::function partialUpdate(const UpdateTag& tag) noexcept; + + /** + * Remove the data element. This generates a {@link Remove} sample. + + * @param key The key + */ + void remove(const Key& key) noexcept; + + private: + const std::shared_ptr> _keyFactory; + const std::shared_ptr> _tagFactory; + }; + + /** + * Creates a key writer for the given topic and key. This helper method deduces the topic Key, Value and + * UpdateTag types from the topic argument. + * + * @param topic The topic. + * @param key The key. + * @param name The optional writer name. + * @param config The optional writer configuration. + */ + template + SingleKeyWriter makeSingleKeyWriter( + const Topic& topic, + const typename Topic::KeyType& key, + const std::string& name = std::string(), + const WriterConfig& config = WriterConfig()) noexcept + { + return SingleKeyWriter(topic, key, name, config); + } + + /** + * Creates a multi-key writer for the given topic and keys. This helper method deduces the topic Key, Value + * and UpdateTag types from the topic argument. + * + * @param topic The topic. + * @param keys The keys. + * @param name The optional writer name. + * @param config The optional writer configuration. + */ + template + MultiKeyWriter makeMultiKeyWriter( + const Topic& topic, + const std::vector::KeyType>& keys, + const std::string& name = std::string(), + const WriterConfig& config = WriterConfig()) noexcept + { + return MultiKeyWriter(topic, keys, name, config); + } + + /** + * Creates an any-key writer for the given topic. This helper method deduces the topic Key, Value and + * UpdateTag types from the topic argument. + * + * @param topic The topic. + * @param name The optional writer name. + * @param config The optional writer configuration. + */ + template + MultiKeyWriter makeAnyKeyWriter( + const Topic& topic, + const std::string& name = std::string(), + const WriterConfig& config = WriterConfig()) noexcept + { + return MultiKeyWriter(topic, {}, name, config); + } + +} + +// +// Public template based API implementation +// + +namespace DataStorm +{ + + // + // Sample template implementation + // + template + SampleEvent Sample::getEvent() const noexcept + { + return _impl->event; + } + + template + const Key& Sample::getKey() const noexcept + { + return _impl->getKey(); + } + + template + const Value& Sample::getValue() const noexcept + { + return _impl->getValue(); + } + + template + UpdateTag Sample::getUpdateTag() const noexcept + { + return _impl->getTag(); + } + + template + std::chrono::time_point Sample::getTimeStamp() const noexcept + { + return _impl->timestamp; + } + + template + std::string Sample::getOrigin() const noexcept + { + return _impl->origin; + } + + template + std::string Sample::getSession() const noexcept + { + return _impl->session; + } + + template + Sample::Sample(const std::shared_ptr& impl) noexcept + : _impl(std::static_pointer_cast>(impl)) + { + } + + // + // Reader template implementation + // + template + Reader::Reader(Reader&& reader) noexcept + : _impl(std::move(reader._impl)) + { + } + + template Reader::~Reader() + { + if (_impl) + { + _impl->destroy(); + } + } + + template + Reader& Reader::operator=(Reader&& reader) noexcept + { + if (_impl) + { + _impl->destroy(); + } + _impl = std::move(reader._impl); + return *this; + } + + template + bool Reader::hasWriters() const noexcept + { + return _impl->hasWriters(); + } + + template + void Reader::waitForWriters(unsigned int count) const + { + _impl->waitForWriters(static_cast(count)); + } + + template + void Reader::waitForNoWriters() const + { + _impl->waitForWriters(-1); + } + + template + std::vector Reader::getConnectedWriters() const noexcept + { + return _impl->getConnectedElements(); + } + + template + std::vector Reader::getConnectedKeys() const noexcept + { + std::vector keys; + auto connectedKeys = _impl->getConnectedKeys(); + keys.reserve(connectedKeys.size()); + for (const auto& k : connectedKeys) + { + keys.push_back(std::static_pointer_cast>(k)->get()); + } + return keys; + } + + template + std::vector> Reader::getAllUnread() noexcept + { + auto unread = _impl->getAllUnread(); + std::vector> samples; + samples.reserve(unread.size()); + for (auto sample : unread) + { + samples.emplace_back(sample); + } + return samples; + } + + template + void Reader::waitForUnread(unsigned int count) const + { + _impl->waitForUnread(count); + } + + template + bool Reader::hasUnread() const noexcept + { + return _impl->hasUnread(); + } + + template + Sample Reader::getNextUnread() + { + return Sample(_impl->getNextUnread()); + } + + template + void Reader::onConnectedKeys( + std::function)> init, + std::function update) noexcept + { + _impl->onConnectedKeys(init ? [init](std::vector> connectedKeys) + { + std::vector keys; + keys.reserve(connectedKeys.size()); + for(const auto& k : connectedKeys) + { + keys.push_back(std::static_pointer_cast>(k)->get()); + } + init(std::move(keys)); + } : std::function>)>(), + update ? [update](CallbackReason action, std::shared_ptr key) + { + update(action, std::static_pointer_cast>(key)->get()); + } : std::function)>()); + } + + template + void Reader::onConnectedWriters( + std::function)> init, + std::function update) noexcept + { + _impl->onConnectedElements(init, update); + } + + template + void Reader::onSamples( + std::function>)> init, + std::function)> update) noexcept + { + auto communicator = _impl->getCommunicator(); + _impl->onSamples(init ? [communicator, init](const std::vector>& samplesI) + { + std::vector> samples; + samples.reserve(samplesI.size()); + for(const auto& s : samplesI) + { + samples.emplace_back(s); + } + init(std::move(samples)); + } : std::function>&)>(), + update ? [communicator, update](const std::shared_ptr& sampleI) + { + update(sampleI); + } : std::function&)>()); + } + + template + SingleKeyReader::SingleKeyReader( + const Topic& topic, + const Key& key, + const std::string& name, + const ReaderConfig& config) noexcept + : Reader(topic.getReader()->create({topic._keyFactory->create(key)}, name, config)) + { + } + + template + template + SingleKeyReader::SingleKeyReader( + const Topic& topic, + const Key& key, + const Filter& sampleFilter, + const std::string& name, + const ReaderConfig& config) noexcept + : Reader(topic.getReader()->create( + {topic._keyFactory->create(key)}, + name, + config, + sampleFilter.name, + DataStormI::EncoderT::encode(topic.getCommunicator(), sampleFilter.criteria))) + { + } + + template + SingleKeyReader::SingleKeyReader(SingleKeyReader&& reader) noexcept + : Reader(std::move(reader)) + { + } + + template + SingleKeyReader& + SingleKeyReader::operator=(SingleKeyReader&& reader) noexcept + { + Reader::operator=(std::move(reader)); + return *this; + } + + template + MultiKeyReader::MultiKeyReader( + const Topic& topic, + const std::vector& keys, + const std::string& name, + const ReaderConfig& config) noexcept + : Reader(topic.getReader()->create(topic._keyFactory->create(keys), name, config)) + { + } + + template + template + MultiKeyReader::MultiKeyReader( + const Topic& topic, + const std::vector& keys, + const Filter& sampleFilter, + const std::string& name, + const ReaderConfig& config) noexcept + : Reader(topic.getReader()->create( + topic._keyFactory->create(keys), + name, + config, + sampleFilter.name, + Encoder::encode(topic.getCommunicator(), sampleFilter.criteria))) + { + } + + template + MultiKeyReader::MultiKeyReader(MultiKeyReader&& reader) noexcept + : Reader(std::move(reader)) + { + } + + template + MultiKeyReader& + MultiKeyReader::operator=(MultiKeyReader&& reader) noexcept + { + Reader::operator=(std::move(reader)); + return *this; + } + + template + template + FilteredKeyReader::FilteredKeyReader( + const Topic& topic, + const Filter& filter, + const std::string& name, + const ReaderConfig& config) + : Reader(topic.getReader()->createFiltered( + topic._keyFilterFactories->create(filter.name, filter.criteria), + name, + config)) + { + } + + template + template + FilteredKeyReader::FilteredKeyReader( + const Topic& topic, + const Filter& keyFilter, + const Filter& sampleFilter, + const std::string& name, + const ReaderConfig& config) + : Reader(topic.getReader()->createFiltered( + topic._keyFilterFactories->create(keyFilter.name, keyFilter.criteria), + name, + config, + sampleFilter.name, + Encoder::encode(topic.getCommunicator(), sampleFilter.criteria))) + { + } + + template + FilteredKeyReader::FilteredKeyReader( + FilteredKeyReader&& reader) noexcept + : Reader(std::move(reader)) + { + } + + template + FilteredKeyReader& + FilteredKeyReader::operator=(FilteredKeyReader&& reader) noexcept + { + Reader::operator=(std::move(reader)); + return *this; + } + + // + // Writer template implementation + // + template + Writer::Writer(Writer&& writer) noexcept : _impl(std::move(writer._impl)) + { + } + + template Writer::~Writer() + { + if (_impl) + { + _impl->destroy(); + } + } + + template + Writer& Writer::operator=(Writer&& writer) noexcept + { + if (_impl) + { + _impl->destroy(); + } + _impl = std::move(writer._impl); + return *this; + } + + template + bool Writer::hasReaders() const noexcept + { + return _impl->hasReaders(); + } + + template + void Writer::waitForReaders(unsigned int count) const + { + return _impl->waitForReaders(static_cast(count)); + } + + template + void Writer::waitForNoReaders() const + { + return _impl->waitForReaders(-1); + } + + template + std::vector Writer::getConnectedReaders() const noexcept + { + return _impl->getConnectedElements(); + } + + template + std::vector Writer::getConnectedKeys() const noexcept + { + std::vector keys; + auto connectedKeys = _impl->getConnectedKeys(); + keys.reserve(connectedKeys.size()); + for (const auto& k : connectedKeys) + { + keys.push_back(std::static_pointer_cast>(k)->get()); + } + return keys; + } + + template + Sample Writer::getLast() + { + auto sample = _impl->getLast(); + if (!sample) + { + throw std::logic_error("no sample"); + } + return Sample(sample); + } + + template + std::vector> Writer::getAll() noexcept + { + auto all = _impl->getAll(); + std::vector> samples; + samples.reserve(all.size()); + for (auto sample : all) + { + samples.emplace_back(sample); + } + return samples; + } + + template + void Writer::onConnectedKeys( + std::function)> init, + std::function update) noexcept + { + _impl->onConnectedKeys(init ? [init](std::vector> connectedKeys) + { + std::vector keys; + keys.reserve(connectedKeys.size()); + for(const auto& k : connectedKeys) + { + keys.push_back(std::static_pointer_cast>(k)->get()); + } + init(std::move(keys)); + } : std::function>)>(), + update ? [update](CallbackReason action, std::shared_ptr key) + { + update(action, std::static_pointer_cast>(key)->get()); + } : std::function)>()); + } + + template + void Writer::onConnectedReaders( + std::function)> init, + std::function update) noexcept + { + _impl->onConnectedElements(init, update); + } + + template + SingleKeyWriter::SingleKeyWriter( + const Topic& topic, + const Key& key, + const std::string& name, + const WriterConfig& config) noexcept + : Writer(topic.getWriter()->create({topic._keyFactory->create(key)}, name, config)), + _tagFactory(topic._tagFactory) + { + } + + template + SingleKeyWriter::SingleKeyWriter(SingleKeyWriter&& writer) noexcept + : Writer(std::move(writer)), + _tagFactory(std::move(writer._tagFactory)) + { + } + + template + SingleKeyWriter& + SingleKeyWriter::operator=(SingleKeyWriter&& writer) noexcept + { + Writer::operator=(std::move(writer)); + return *this; + } + + template + void SingleKeyWriter::add(const Value& value) noexcept + { + Writer::_impl->publish( + nullptr, + std::make_shared>(SampleEvent::Add, value)); + } + + template + void SingleKeyWriter::update(const Value& value) noexcept + { + Writer::_impl->publish( + nullptr, + std::make_shared>(SampleEvent::Update, value)); + } + + template + template + std::function + SingleKeyWriter::partialUpdate(const UpdateTag& tag) noexcept + { + auto impl = Writer::_impl; + auto updateTag = _tagFactory->create(tag); + return [impl, updateTag](const UpdateValue& value) + { + auto encoded = Encoder::encode(impl->getCommunicator(), value); + impl->publish(nullptr, std::make_shared>(encoded, updateTag)); + }; + } + + template + void SingleKeyWriter::remove() noexcept + { + Writer::_impl->publish( + nullptr, + std::make_shared>(SampleEvent::Remove)); + } + + template + MultiKeyWriter::MultiKeyWriter( + const Topic& topic, + const std::vector& keys, + const std::string& name, + const WriterConfig& config) noexcept + : Writer(topic.getWriter()->create(topic._keyFactory->create(keys), name, config)), + _keyFactory(topic._keyFactory), + _tagFactory(topic._tagFactory) + { + } + + template + MultiKeyWriter::MultiKeyWriter(MultiKeyWriter&& writer) noexcept + : Writer(std::move(writer)), + _keyFactory(std::move(writer._keyFactory)), + _tagFactory(std::move(writer._tagFactory)) + { + } + + template + MultiKeyWriter& + MultiKeyWriter::operator=(MultiKeyWriter&& writer) noexcept + { + Writer::operator=(std::move(writer)); + return *this; + } + + template + void MultiKeyWriter::add(const Key& key, const Value& value) noexcept + { + Writer::_impl->publish( + _keyFactory->create(key), + std::make_shared>(SampleEvent::Add, value)); + } + + template + void MultiKeyWriter::update(const Key& key, const Value& value) noexcept + { + Writer::_impl->publish( + _keyFactory->create(key), + std::make_shared>(SampleEvent::Update, value)); + } + + template + template + std::function + MultiKeyWriter::partialUpdate(const UpdateTag& tag) noexcept + { + auto impl = Writer::_impl; + auto updateTag = _tagFactory->create(tag); + auto keyFactory = _keyFactory; + return [impl, updateTag, keyFactory](const Key& key, const UpdateValue& value) + { + auto encoded = Encoder::encode(impl->getCommunicator(), value); + impl->publish( + keyFactory->create(key), + std::make_shared>(encoded, updateTag)); + }; + } + + template + void MultiKeyWriter::remove(const Key& key) noexcept + { + Writer::_impl->publish( + _keyFactory->create(key), + std::make_shared>(SampleEvent::Remove)); + } + + /** @private */ + template + std::function(const std::string&)> makeRegexFilter() noexcept + { + return [](const std::string& criteria) + { + std::regex expr(criteria); + return [expr](const Value& value) + { + std::ostringstream os; + os << value; + return std::regex_match(os.str(), expr); + }; + }; + } + + /** @private */ + template + std::function&)>(const std::vector&)> + makeSampleEventFilter(const Topic&) noexcept + { + return [](const std::vector& criteria) + { + return [criteria](const Sample& sample) + { return std::find(criteria.begin(), criteria.end(), sample.getEvent()) != criteria.end(); }; + }; + } + + /** @private */ + template struct RegexFilter + { + template static void add(F) {} + }; + + /** @private */ + template + struct RegexFilter::value>::type> + { + template static void add(F factory) + { + factory->set("_regex", makeRegexFilter()); // Only set the _regex filter if the value is streamable + } + }; + + // + // Topic template implementation + // + template + Topic::Topic(const Node& node, const std::string& name) noexcept + : _name(name), + _topicFactory(node._factory), + _keyFactory(DataStormI::KeyFactoryT::createFactory()), + _tagFactory(DataStormI::TagFactoryT::createFactory()), + _keyFilterFactories(DataStormI::FilterManagerT>::create()), + _sampleFilterFactories(DataStormI::FilterManagerT>::create()) + { + RegexFilter::add(_keyFilterFactories); + RegexFilter, Value>::add(_sampleFilterFactories); + _sampleFilterFactories->set("_event", makeSampleEventFilter(*this)); + } + + template Topic::~Topic() + { + std::lock_guard lock(_mutex); + if (_reader) + { + _reader->destroy(); + } + if (_writer) + { + _writer->destroy(); + } + } + + template + Topic& Topic::operator=(Topic&& topic) noexcept + { + std::lock_guard lock(_mutex); + if (_reader) + { + _reader->destroy(); + } + if (_writer) + { + _writer->destroy(); + } + _name = std::move(topic._name); + _topicFactory = std::move(topic._topicFactory); + _keyFactory = std::move(topic._keyFactory); + _tagFactory = std::move(topic._tagFactory); + _keyFilterFactories = std::move(topic._keyFilterFactories); + _sampleFilterFactories = std::move(topic._sampleFilterFactories); + _reader = std::move(topic._reader); + _writer = std::move(topic._writer); + _updaters = std::move(topic._updaters); + return *this; + } + + template + bool Topic::hasWriters() const noexcept + { + return getReader()->hasWriters(); + } + + template + void Topic::waitForWriters(unsigned int count) const + { + getReader()->waitForWriters(static_cast(count)); + } + + template + void Topic::waitForNoWriters() const + { + getReader()->waitForWriters(-1); + } + + template + void Topic::setReaderDefaultConfig(const ReaderConfig& config) noexcept + { + getReader()->setDefaultConfig(config); + } + + template + bool Topic::hasReaders() const noexcept + { + return getWriter()->hasReaders(); + } + + template + void Topic::waitForReaders(unsigned int count) const + { + getWriter()->waitForReaders(static_cast(count)); + } + + template + void Topic::waitForNoReaders() const + { + getWriter()->waitForReaders(-1); + } + + template + void Topic::setWriterDefaultConfig(const WriterConfig& config) noexcept + { + getWriter()->setDefaultConfig(config); + } + + template + template + void Topic::setUpdater( + const UpdateTag& tag, + std::function updater) noexcept + { + std::lock_guard lock(_mutex); + auto tagI = _tagFactory->create(std::move(tag)); + auto updaterImpl = updater ? [updater](const std::shared_ptr& previous, + const std::shared_ptr& next, + const Ice::CommunicatorPtr& communicator) + { + Value value; + if(previous) + { + value = Cloner::clone( + std::static_pointer_cast>(previous)->getValue()); + } + updater(value, Decoder::decode(communicator, next->getEncodedValue())); + std::static_pointer_cast>(next)->setValue(std::move(value)); + } : std::function&, + const std::shared_ptr&, + const Ice::CommunicatorPtr&)>(); + + if (_reader && !_writer) + { + _reader->setUpdater(tagI, updaterImpl); + } + else if (_writer && !_reader) + { + _writer->setUpdater(tagI, updaterImpl); + } + else if (_reader && _writer) + { + _reader->setUpdater(tagI, updaterImpl); + _writer->setUpdater(tagI, updaterImpl); + } + else + { + _updaters[tagI] = updaterImpl; + } + } + + template + template + void Topic::setKeyFilter( + const std::string& name, + std::function(const Criteria&)> factory) noexcept + { + std::lock_guard lock(_mutex); + _keyFilterFactories->set(name, factory); + } + + template + template + void Topic::setSampleFilter( + const std::string& name, + std::function&)>(const Criteria&)> factory) noexcept + { + std::lock_guard lock(_mutex); + _sampleFilterFactories->set(name, factory); + } + + template + std::shared_ptr Topic::getReader() const noexcept + { + std::lock_guard lock(_mutex); + if (!_reader) + { + auto sampleFactory = std::make_shared>(); + _reader = _topicFactory->createTopicReader( + _name, + _keyFactory, + _tagFactory, + std::move(sampleFactory), + _keyFilterFactories, + _sampleFilterFactories); + _reader->setUpdaters(_writer ? _writer->getUpdaters() : _updaters); + _updaters.clear(); + } + return _reader; + } + + template + std::shared_ptr Topic::getWriter() const noexcept + { + std::lock_guard lock(_mutex); + if (!_writer) + { + _writer = _topicFactory->createTopicWriter( + _name, + _keyFactory, + _tagFactory, + nullptr, + _keyFilterFactories, + _sampleFilterFactories); + _writer->setUpdaters(_reader ? _reader->getUpdaters() : _updaters); + _updaters.clear(); + } + return _writer; + } + + template + Ice::CommunicatorPtr Topic::getCommunicator() const noexcept + { + return _topicFactory->getCommunicator(); + } + +} +#endif diff --git a/cpp/include/DataStorm/InternalI.h b/cpp/include/DataStorm/InternalI.h new file mode 100644 index 00000000000..31c9ef6d504 --- /dev/null +++ b/cpp/include/DataStorm/InternalI.h @@ -0,0 +1,276 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_INTERNALI_H +#define DATASTORM_INTERNALI_H + +#include "DataStorm/Sample.h" +#include "Ice/Ice.h" +#include "Types.h" + +#include +#include +#include + +// +// Private abstract API used by the template based API and the internal DataStorm implementation. +// +namespace DataStormI +{ + + class Instance; + + class Filterable + { + public: + virtual ~Filterable() = default; + }; + + class Element + { + public: + virtual ~Element() = default; + virtual std::string toString() const = 0; + virtual Ice::ByteSeq encode(const Ice::CommunicatorPtr&) const = 0; + virtual std::int64_t getId() const = 0; + }; + + class Key : public Filterable, virtual public Element + { + }; + + class KeyFactory + { + public: + virtual ~KeyFactory() = default; + virtual std::shared_ptr get(std::int64_t) const = 0; + virtual std::shared_ptr decode(const Ice::CommunicatorPtr&, const Ice::ByteSeq&) = 0; + }; + + class Tag : virtual public Element + { + }; + + class TagFactory + { + public: + virtual ~TagFactory() = default; + virtual std::shared_ptr get(std::int64_t) const = 0; + virtual std::shared_ptr decode(const Ice::CommunicatorPtr&, const Ice::ByteSeq&) = 0; + }; + + class Sample : public Filterable + { + public: + Sample( + const std::string& session, + const std::string& origin, + std::int64_t id, + DataStorm::SampleEvent event, + const std::shared_ptr& key, + const std::shared_ptr& tag, + Ice::ByteSeq value, + std::int64_t timestamp) + : session(session), + origin(origin), + id(id), + event(event), + key(key), + tag(tag), + timestamp(std::chrono::microseconds(timestamp)), + _encodedValue(std::move(value)) + { + } + + Sample(DataStorm::SampleEvent event, const std::shared_ptr& tag = nullptr) : event(event), tag(tag) {} + + virtual bool hasValue() const = 0; + virtual void setValue(const std::shared_ptr&) = 0; + + virtual void decode(const Ice::CommunicatorPtr&) = 0; + virtual const Ice::ByteSeq& encode(const Ice::CommunicatorPtr&) = 0; + virtual Ice::ByteSeq encodeValue(const Ice::CommunicatorPtr&) = 0; + + const Ice::ByteSeq& getEncodedValue() const { return _encodedValue; } + + std::string session; + std::string origin; + std::int64_t id; + DataStorm::SampleEvent event; + std::shared_ptr key; + std::shared_ptr tag; + std::chrono::time_point timestamp; + + protected: + Ice::ByteSeq _encodedValue; + }; + + class SampleFactory + { + public: + virtual ~SampleFactory() = default; + + virtual std::shared_ptr create( + const std::string&, + const std::string&, + std::int64_t, + DataStorm::SampleEvent, + const std::shared_ptr&, + const std::shared_ptr&, + Ice::ByteSeq, + std::int64_t) = 0; + }; + + class Filter : virtual public Element + { + public: + virtual bool match(const std::shared_ptr&) const = 0; + virtual const std::string& getName() const = 0; + }; + + class FilterFactory + { + public: + virtual ~FilterFactory() = default; + + virtual std::shared_ptr get(std::int64_t) const = 0; + virtual std::shared_ptr decode(const Ice::CommunicatorPtr&, const Ice::ByteSeq&) = 0; + }; + + class FilterManager + { + public: + virtual ~FilterManager() = default; + + virtual std::shared_ptr get(const std::string&, std::int64_t) const = 0; + + virtual std::shared_ptr + decode(const Ice::CommunicatorPtr&, const std::string&, const Ice::ByteSeq&) = 0; + }; + + class DataElement + { + public: + virtual ~DataElement() = default; + + using Id = std::tuple; + + virtual std::vector getConnectedElements() const = 0; + virtual std::vector> getConnectedKeys() const = 0; + virtual void onConnectedKeys( + std::function>)>, + std::function)>) = 0; + virtual void onConnectedElements( + std::function)>, + std::function) = 0; + + virtual void destroy() = 0; + virtual Ice::CommunicatorPtr getCommunicator() const = 0; + }; + + class DataReader : virtual public DataElement + { + public: + virtual bool hasWriters() = 0; + virtual void waitForWriters(int) = 0; + virtual int getInstanceCount() const = 0; + + virtual std::vector> getAllUnread() = 0; + virtual void waitForUnread(unsigned int) const = 0; + virtual bool hasUnread() const = 0; + virtual std::shared_ptr getNextUnread() = 0; + + virtual void onSamples( + std::function>&)>, + std::function&)>) = 0; + }; + + class DataWriter : virtual public DataElement + { + public: + virtual bool hasReaders() const = 0; + virtual void waitForReaders(int) const = 0; + + virtual std::shared_ptr getLast() const = 0; + virtual std::vector> getAll() const = 0; + + virtual void publish(const std::shared_ptr&, const std::shared_ptr&) = 0; + }; + + class Topic + { + public: + virtual ~Topic() = default; + + using Updater = std::function< + void(const std::shared_ptr&, const std::shared_ptr&, const Ice::CommunicatorPtr&)>; + + virtual void setUpdater(const std::shared_ptr&, Updater) = 0; + + virtual void setUpdaters(std::map, Updater>) = 0; + virtual std::map, Updater> getUpdaters() const = 0; + + virtual std::string getName() const = 0; + virtual void destroy() = 0; + }; + + class TopicReader : virtual public Topic + { + public: + virtual std::shared_ptr createFiltered( + const std::shared_ptr&, + const std::string&, + DataStorm::ReaderConfig, + const std::string& = std::string(), + Ice::ByteSeq = {}) = 0; + + virtual std::shared_ptr create( + const std::vector>&, + const std::string&, + DataStorm::ReaderConfig, + const std::string& = std::string(), + Ice::ByteSeq = {}) = 0; + + virtual void setDefaultConfig(DataStorm::ReaderConfig) = 0; + virtual bool hasWriters() const = 0; + virtual void waitForWriters(int) const = 0; + }; + + class TopicWriter : virtual public Topic + { + public: + virtual std::shared_ptr + create(const std::vector>&, const std::string&, DataStorm::WriterConfig) = 0; + + virtual void setDefaultConfig(DataStorm::WriterConfig) = 0; + virtual bool hasReaders() const = 0; + virtual void waitForReaders(int) const = 0; + }; + + class TopicFactory + { + public: + virtual ~TopicFactory() = default; + + virtual std::shared_ptr createTopicReader( + const std::string&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&) = 0; + + virtual std::shared_ptr createTopicWriter( + const std::string&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&) = 0; + + virtual Ice::CommunicatorPtr getCommunicator() const = 0; + }; + +} +#endif diff --git a/cpp/include/DataStorm/InternalT.h b/cpp/include/DataStorm/InternalT.h new file mode 100644 index 00000000000..af228b1869d --- /dev/null +++ b/cpp/include/DataStorm/InternalT.h @@ -0,0 +1,576 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#pragma once + +#include "Config.h" +#include "Ice/Ice.h" +#include "InternalI.h" +#include "Types.h" + +namespace DataStorm +{ + + template class Sample; + +} + +namespace DataStormI +{ + + template class has_communicator_parameter + { + template + static auto testE(int) + -> decltype(TT::encode(std::declval(), std::declval()), std::true_type()); + + template static auto testE(...) -> std::false_type; + + template + static auto testD(int) + -> decltype(TT::decode(std::declval(), Ice::ByteSeq()), std::true_type()); + + template static auto testD(...) -> std::false_type; + + public: + static const bool value = + decltype(testE, T>(0))::value && decltype(testD, T>(0))::value; + }; + + template struct EncoderT + { + static Ice::ByteSeq encode(const Ice::CommunicatorPtr&, const T& value) + { + return DataStorm::Encoder::encode(value); + } + }; + + template struct DecoderT + { + static T decode(const Ice::CommunicatorPtr&, const Ice::ByteSeq& data) + { + return DataStorm::Decoder::decode(data); + } + }; + + template struct EncoderT::value>::type> + { + static Ice::ByteSeq encode(const Ice::CommunicatorPtr& communicator, const T& value) + { + return DataStorm::Encoder::encode(communicator, value); + } + }; + + template struct DecoderT::value>::type> + { + static T decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& data) + { + return DataStorm::Decoder::decode(communicator, data); + } + }; + + template class is_streamable + { + template + static auto test(int) -> decltype(std::declval() << std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static const bool value = decltype(test(0))::value; + }; + + template struct Stringifier + { + static std::string toString(const T& value) + { + std::ostringstream os; + os << typeid(value).name() << '(' << &value << ')'; + return os.str(); + } + }; + + template struct Stringifier::value>::type> + { + static std::string toString(const T& value) + { + std::ostringstream os; + os << value; + return os.str(); + } + }; + + template class AbstractElementT : virtual public Element + { + public: + template AbstractElementT(TT&& v, std::int64_t id) : _value(std::forward(v)), _id(id) {} + + virtual std::string toString() const override + { + std::ostringstream os; + os << _id << ':' << Stringifier::toString(_value); + return os.str(); + } + + virtual Ice::ByteSeq encode(const Ice::CommunicatorPtr& communicator) const override + { + return EncoderT::encode(communicator, _value); + } + + virtual std::int64_t getId() const override { return _id; } + + const T& get() const { return _value; } + + protected: + const T _value; + const std::int64_t _id; + }; + + template + class AbstractFactoryT : public std::enable_shared_from_this> + { + struct Deleter + { + void operator()(V* obj) + { + auto factory = _factory.lock(); + if (factory) + { + factory->remove(obj); + } + delete obj; + } + + std::weak_ptr> _factory; + + } _deleter; + + public: + AbstractFactoryT() : _nextId(1) {} + + void init() { _deleter = {std::enable_shared_from_this>::shared_from_this()}; } + + template + std::shared_ptr create(F&& value, Args&&... args) + { + std::lock_guard lock(_mutex); + return createImpl(std::forward(value), std::forward(args)...); + } + + std::vector> create(std::vector values) + { + std::lock_guard lock(_mutex); + std::vector> seq; + for (auto& v : values) + { + seq.push_back(createImpl(std::move(v))); + } + return seq; + } + + protected: + friend struct Deleter; + + std::shared_ptr getImpl(long long id) const + { + std::lock_guard lock(_mutex); + auto p = _elementsById.find(id); + if (p != _elementsById.end()) + { + auto k = p->second.lock(); + if (k) + { + return k; + } + } + return nullptr; + } + + template std::shared_ptr createImpl(F&& value, Args&&... args) + { + auto p = _elements.find(value); + if (p != _elements.end()) + { + auto k = p->second.lock(); + if (k) + { + return k; + } + + // + // The key is being removed concurrently by the deleter, remove it now + // to allow the insertion of a new key. The deleter won't remove the + // new key. + // + _elements.erase(p); + } + + auto k = + std::shared_ptr(new V(std::forward(value), std::forward(args)..., ++_nextId), _deleter); + _elements[k->get()] = k; + _elementsById[k->getId()] = k; + return k; + } + + void remove(V* v) + { + // Make sure to declare the variable outside the synchronization in case the element needs + // to be deleted if it's not the same. + std::shared_ptr e; + std::lock_guard lock(_mutex); + auto p = _elements.find(v->get()); + if (p != _elements.end()) + { + e = p->second.lock(); + if (e && e.get() == v) + { + _elements.erase(p); + } + } + _elementsById.erase(v->getId()); + } + + mutable std::mutex _mutex; + std::map> _elements; + std::map> _elementsById; + std::int64_t _nextId; + }; + + template class KeyT : public Key, public AbstractElementT + { + public: + virtual std::string toString() const override { return "k" + AbstractElementT::toString(); } + + using AbstractElementT::AbstractElementT; + using BaseClassType = Key; + }; + + template class KeyFactoryT : public KeyFactory, public AbstractFactoryT> + { + public: + using AbstractFactoryT>::AbstractFactoryT; + + virtual std::shared_ptr get(std::int64_t id) const override + { + return AbstractFactoryT>::getImpl(id); + } + + virtual std::shared_ptr decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& data) override + { + return AbstractFactoryT>::create(DecoderT::decode(communicator, data)); + } + + static std::shared_ptr> createFactory() + { + auto f = std::make_shared>(); + f->init(); + return f; + } + }; + + template class TagT : public Tag, public AbstractElementT + { + public: + virtual std::string toString() const override { return "t" + AbstractElementT::toString(); } + + using AbstractElementT::AbstractElementT; + using BaseClassType = Tag; + }; + + template class TagFactoryT : public TagFactory, public AbstractFactoryT> + { + public: + using AbstractFactoryT>::AbstractFactoryT; + + virtual std::shared_ptr get(std::int64_t id) const override + { + return AbstractFactoryT>::getImpl(id); + } + + virtual std::shared_ptr decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& data) override + { + return AbstractFactoryT>::create(DecoderT::decode(communicator, data)); + } + + static std::shared_ptr> createFactory() + { + auto f = std::make_shared>(); + f->init(); + return f; + } + }; + + template + class SampleT : public Sample, public std::enable_shared_from_this> + { + public: + SampleT( + const std::string& session, + const std::string& origin, + std::int64_t id, + DataStorm::SampleEvent event, + const std::shared_ptr& key, + const std::shared_ptr& tag, + Ice::ByteSeq value, + std::int64_t timestamp) + : Sample(session, origin, id, event, key, tag, value, timestamp), + _hasValue(false) + { + } + + SampleT(DataStorm::SampleEvent event) : Sample(event), _hasValue(false) {} + + SampleT(DataStorm::SampleEvent event, Value value) : Sample(event), _hasValue(true), _value(std::move(value)) {} + + SampleT(Ice::ByteSeq value, const std::shared_ptr& tag) + : Sample(DataStorm::SampleEvent::PartialUpdate, tag), + _hasValue(false) + { + _encodedValue = std::move(value); + } + + DataStorm::Sample get() + { + auto impl = std::enable_shared_from_this>::shared_from_this(); + return DataStorm::Sample(impl); + } + + const Key& getKey() + { + assert(key); + return std::static_pointer_cast>(key)->get(); + } + + const Value& getValue() const { return _value; } + + UpdateTag getTag() const { return tag ? std::static_pointer_cast>(tag)->get() : UpdateTag(); } + + void setValue(Value value) + { + _value = std::move(value); + _hasValue = true; + } + + virtual bool hasValue() const override { return _hasValue; } + + virtual void setValue(const std::shared_ptr& sample) override + { + if (sample) + { + _value = DataStorm::Cloner::clone( + std::static_pointer_cast>(sample)->getValue()); + } + else + { + _value = Value(); + } + _hasValue = true; + } + + virtual const Ice::ByteSeq& encode(const Ice::CommunicatorPtr& communicator) override + { + if (_encodedValue.empty()) + { + _encodedValue = encodeValue(communicator); + } + return _encodedValue; + } + + virtual Ice::ByteSeq encodeValue(const Ice::CommunicatorPtr& communicator) override + { + assert(_hasValue || event == DataStorm::SampleEvent::Remove); + return EncoderT::encode(communicator, _value); + } + + virtual void decode(const Ice::CommunicatorPtr& communicator) override + { + if (!_encodedValue.empty()) + { + _hasValue = true; + _value = DecoderT::decode(communicator, _encodedValue); + _encodedValue.clear(); + } + } + + private: + bool _hasValue; + Value _value; + }; + + template class SampleFactoryT : public SampleFactory + { + public: + virtual std::shared_ptr create( + const std::string& session, + const std::string& origin, + std::int64_t id, + DataStorm::SampleEvent type, + const std::shared_ptr& key, + const std::shared_ptr& tag, + Ice::ByteSeq value, + std::int64_t timestamp) + { + return std::make_shared< + SampleT>(session, origin, id, type, key, tag, std::move(value), timestamp); + } + }; + + template class FilterT : public Filter, public AbstractElementT + { + public: + template + FilterT(CC&& criteria, std::int64_t id) : AbstractElementT::AbstractElementT(std::forward(criteria), id) + { + } + + virtual std::string toString() const override { return "f" + AbstractElementT::toString(); } + + virtual bool match(const std::shared_ptr& value) const override + { + return _lambda(std::static_pointer_cast(value)->get()); + } + + virtual const std::string& getName() const override { return _name; } + + template void init(const std::string& name, FF&& lambda) + { + _name = name; + _lambda = std::forward(lambda); + } + + using BaseClassType = Filter; + + private: + std::string _name; + std::function().get())>::type&)> _lambda; + }; + + template + class FilterFactoryT : public FilterFactory, public AbstractFactoryT> + { + public: + FilterFactoryT() {} + + virtual std::shared_ptr get(std::int64_t id) const override + { + return AbstractFactoryT>::getImpl(id); + } + + virtual std::shared_ptr + decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& data) override + { + return AbstractFactoryT>::create(DecoderT::decode(communicator, data)); + } + + static std::shared_ptr> createFactory() + { + auto f = std::make_shared>(); + f->init(); + return f; + } + }; + + template class FilterManagerT : public FilterManager + { + using Value = typename std::remove_reference().get())>::type; + + struct Factory + { + virtual ~Factory() = default; + + virtual std::shared_ptr get(std::int64_t) const = 0; + + virtual std::shared_ptr decode(const Ice::CommunicatorPtr&, const Ice::ByteSeq&) = 0; + }; + + template struct FactoryT : Factory + { + FactoryT(const std::string& name, std::function(const Criteria&)> lambda) + : name(name), + lambda(std::move(lambda)) + { + } + + std::shared_ptr create(Criteria criteria) + { + auto filter = std::static_pointer_cast>(filterFactory.create(criteria)); + filter->init(name, lambda(filter->get())); + return filter; + } + + virtual std::shared_ptr get(std::int64_t id) const { return filterFactory.get(id); } + + virtual std::shared_ptr decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& data) + { + return create(DecoderT::decode(communicator, data)); + } + + const std::string name; + std::function(const Criteria&)> lambda; + FilterFactoryT filterFactory; + }; + + public: + FilterManagerT() {} + + template std::shared_ptr create(const std::string& name, const Criteria& criteria) + { + auto p = _factories.find(name); + if (p == _factories.end()) + { + throw std::invalid_argument("unknown filter `" + name + "'"); + } + + auto factory = dynamic_cast*>(p->second.get()); + if (!factory) + { + throw std::invalid_argument("filter `" + name + "' type doesn't match"); + } + + return factory->create(criteria); + } + + virtual std::shared_ptr + decode(const Ice::CommunicatorPtr& communicator, const std::string& name, const Ice::ByteSeq& data) override + { + auto p = _factories.find(name); + if (p == _factories.end()) + { + return nullptr; + } + + return p->second->decode(communicator, data); + } + + virtual std::shared_ptr get(const std::string& name, std::int64_t id) const override + { + auto p = _factories.find(name); + if (p == _factories.end()) + { + return nullptr; + } + + return p->second->get(id); + } + + template + void set(const std::string& name, std::function(const Criteria&)> lambda) + { + if (lambda) + { + _factories[name] = std::unique_ptr(new FactoryT(name, std::move(lambda))); + } + else + { + _factories.erase(name); + } + } + + static std::shared_ptr> create() { return std::make_shared>(); } + + private: + std::map> _factories; + }; + +} diff --git a/cpp/include/DataStorm/Node.h b/cpp/include/DataStorm/Node.h new file mode 100644 index 00000000000..e5f2d467709 --- /dev/null +++ b/cpp/include/DataStorm/Node.h @@ -0,0 +1,210 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_NODE_H +#define DATASTORM_NODE_H + +#include "Config.h" +#include "InternalI.h" + +namespace DataStorm +{ + + template class Topic; + + /** + * The exception NodeShutdownException. It's raised by methods which might block waiting for a condition to + * occur and after the node has been shutdown. It informs the application that the condition won't occur + * because the DataStorm node is being shutdown and will disconnect from other nodes. + * + * @headerfile DataStorm/DataStorm.h + */ + class DATASTORM_API NodeShutdownException : public std::exception + { + public: + virtual const char* what() const noexcept; + }; + + /** + * The Node class allows creating topic readers and writers. + * + * A Node is the main DataStorm object which allows creating topic readers or writers. + * + * @headerfile DataStorm/DataStorm.h + */ + class DATASTORM_API Node + { + public: + /** + * Construct a DataStorm node. + * + * A node is the main DataStorm object. It is required to construct topics. The node uses the given Ice + * communicator. + * + * @param communicator The Ice communicator used by the topic factory for its configuration and + * communications. + */ + Node(Ice::CommunicatorPtr communicator); + + /** + * Construct a DataStorm node. + * + * A node is the main DataStorm object. It is required to construct topics. This constructor parses the + * command line arguments into Ice properties and initialize a new Node. The constructor initializes the + * Ice communicator using the given Ice arguments. If the communicator creation fails, an Ice exception is + * raised. + * + * @param argc The number of command line arguments in the argv array. + * @param argv The command line arguments. + * @param iceArgs Additional arguments which are passed to the Ice::initialize function in addition to the + * argc and argv arguments. + */ + template Node(int& argc, const char* argv[], T&&... iceArgs) : _ownsCommunicator(true) + { + init(argc, argv, std::forward(iceArgs)...); + } + + /** + * Construct a DataStorm node. + * + * A node is the main DataStorm object. It is required to construct topics. This constructor parses the + * command line arguments into Ice properties and initialize a new Node. The constructor initializes the + * Ice communicator using the given Ice arguments. If the communicator creation fails, an Ice exception is + * raised. + * + * @param argc The number of command line arguments in the argv array. + * @param argv The command line arguments. + * @param iceArgs Additional arguments which are passed to the Ice::initialize function in addition to the + * argc and argv arguments. + */ + template Node(int& argc, char* argv[], T&&... iceArgs) : _ownsCommunicator(true) + { + init(argc, argv, std::forward(iceArgs)...); + } + +#ifdef _WIN32 + /** + * Construct a DataStorm node. + * + * A node is the main DataStorm object. It is required to construct topics. This constructor parses the + * command line arguments into Ice properties and initialize a new Node. The constructor initializes the + * Ice communicator using the given Ice arguments. If the communicator creation fails, an Ice exception is + * raised. + * + * @param argc The number of command line arguments in the argv array. + * @param argv The command line arguments. + * @param iceArgs Additional arguments which are passed to the Ice::initialize function in addition to the + * argc and argv arguments. + */ + template Node(int& argc, const wchar_t* argv[], T&&... iceArgs) : _ownsCommunicator(true) + { + init(argc, argv, std::forward(iceArgs)...); + } + + /** + * Construct a DataStorm node. + * + * A node is the main DataStorm object. It is required to construct topics. This constructor parses the + * command line arguments into Ice properties and initialize a new Node. The constructor initializes the + * Ice communicator using the given Ice arguments. If the communicator creation fails, an Ice exception is + * raised. + * + * @param argc The number of command line arguments in the argv array. + * @param argv The command line arguments. + * @param iceArgs Additional arguments which are passed to the Ice::initialize function in addition to the + * argc and argv arguments. + */ + template Node(int& argc, wchar_t* argv[], T&&... iceArgs) : _ownsCommunicator(true) + { + init(argc, argv, std::forward(iceArgs)...); + } +#endif + + /** + * Construct a DataStorm node. + * + * A node is the main DataStorm object. It is required to construct topics. The constructor initializes + * the Ice communicator using the given arguments. If the communicator creation fails, an Ice exception is + * raised. + * + * @param iceArgs Arguments which are passed to the Ice::initialize function. + */ + template Node(T&&... iceArgs) : _ownsCommunicator(true) + { + init(Ice::initialize(std::forward(iceArgs)...)); + } + + /** + * Construct a new Node by taking ownership of the given node. + * + * @param node The node to transfer ownership from. + */ + Node(Node&& node) noexcept; + + /** + * Node destructor. The node destruction releases associated resources. If the node created the Ice + * communicator, the communicator is destroyed. + */ + ~Node(); + + /** + * Shutdown the node. The shutdown interrupts calls which are waiting for events, writers or readers. + **/ + void shutdown() noexcept; + + /** + * Return whether or not the node shutdown has been initiated. + * + * @return True if the node is shutdown, false otherwise. + */ + bool isShutdown() const noexcept; + + /** + * Wait for shutdown to be called. + */ + void waitForShutdown() const noexcept; + + /** + * Move assignment operator. + * + * @param node The node. + **/ + Node& operator=(Node&& node) noexcept; + + /** + * Returns the Ice communicator associated with the node. + */ + Ice::CommunicatorPtr getCommunicator() const noexcept; + + /** + * Returns the Ice connection associated with a session given a session identifier. Session identifiers are + * provided with the sample origin data member as the first tuple element. + * + * @param ident The session identifier. + * @return The connection associated with the given session + * @see DataStorm::Sample::ElementId DataStorm::Sample::getSession + */ + Ice::ConnectionPtr getSessionConnection(const std::string& ident) const noexcept; + + private: + template void init(int& argc, V argv, T&&... iceArgs) + { + auto communicator = Ice::initialize(argc, argv, std::forward(iceArgs)...); + auto args = Ice::argsToStringSeq(argc, argv); + args = communicator->getProperties()->parseCommandLineOptions("DataStorm", args); + Ice::stringSeqToArgs(args, argc, argv); + init(communicator); + } + + void init(const Ice::CommunicatorPtr&); + + std::shared_ptr _instance; + std::shared_ptr _factory; + bool _ownsCommunicator; + + template friend class Topic; + }; + +} +#endif diff --git a/cpp/include/DataStorm/Types.h b/cpp/include/DataStorm/Types.h new file mode 100644 index 00000000000..460402e76b6 --- /dev/null +++ b/cpp/include/DataStorm/Types.h @@ -0,0 +1,320 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_TYPES_H +#define DATASTORM_TYPES_H + +#include "Config.h" +#include "Ice/Ice.h" + +namespace DataStorm +{ + + /** + * The discard policy specifies how samples are discarded by readers upon receive. + */ + enum struct DiscardPolicy + { + /** Samples are never discarded. */ + None, + + /** + * Samples are discared based on the sample timestamp. If the received sample timestamp is older than the + * last received sample, the sample is discarded. This ensures that readers will eventually always end up + * with the same view of the data if multiple writers are sending samples. + **/ + SendTime, + + /** + * Samples are discarded based on the writer priority. Only samples from the highest priority connected + * writers are kept, others are discarded. + */ + Priority + }; + + /** + * The clear history policy specifies when the history is cleared. The history can be cleared based on the + * event of the received sample. + **/ + enum struct ClearHistoryPolicy + { + /** Clear the sample history when a Add sample is received. */ + OnAdd, + + /** Clear the sample history when a Remove sample is received. */ + OnRemove, + + /** Clear the sample history when a new sample is received. */ + OnAll, + + /** Clear the sample history when a new sample which is not a partial update is received. */ + OnAllExceptPartialUpdate, + + /** Never clear the sample history. */ + Never + }; + + /** + * The configuration base class holds configuration options common to readers and writers. + * + * @headerfile DataStorm/DataStorm.h + */ + class Config + { + public: + /** + * Construct a Config object. + * + * The constructor accepts optional parameters for each of the Config data members. + * + * @param sampleCount The optional sample count. + * @param sampleLifetime The optional sample lifetime. + * @param clearHistory The optional clear history policy. + */ + Config( + std::optional sampleCount = std::nullopt, + std::optional sampleLifetime = std::nullopt, + std::optional clearHistory = std::nullopt) noexcept + : sampleCount(std::move(sampleCount)), + sampleLifetime(std::move(sampleLifetime)), + clearHistory(std::move(clearHistory)) + { + } + + /** + * The sampleCount configuration specifies how many samples are kept by the reader or writer in its sample + * history. By default, the sample count is unlimited. + */ + std::optional sampleCount; + + /** + * The sampleLifetime configuration specifies samples to keep in the writer or reader history based on + * their age. Samples with a timestamp older than the sampleLifetime value (in milliseconds) are discarded + * from the history. By default, the samples are kept for an unlimited amount of time. + */ + std::optional sampleLifetime; + + /** + * The clear history policy specifies when samples are removed from the sample history. By default, + * samples are removed when a new sample is is received which effectively disables the sample history. + */ + std::optional clearHistory; + }; + + /** + * The ReaderConfig class specifies configuration options specific to readers. + * + * It extends the Config class and therefore inherits its configuration options. + * + * @headerfile DataStorm/DataStorm.h + */ + class ReaderConfig : public Config + { + public: + /** + * Construct a ReaderConfig object. + * + * The constructor accepts optional parameters for each of the ReaderConfig data members. + * + * @param sampleCount The optional sample count. + * @param sampleLifetime The optional sample lifetime. + * @param clearHistory The optional clear history policy. + * @param discardPolicy The discard policy. + */ + ReaderConfig( + std::optional sampleCount = std::nullopt, + std::optional sampleLifetime = std::nullopt, + std::optional clearHistory = std::nullopt, + std::optional discardPolicy = std::nullopt) noexcept + : Config(std::move(sampleCount), std::move(sampleLifetime), std::move(clearHistory)), + discardPolicy(std::move(discardPolicy)) + { + } + + /** + * Specifies if and how samples are discarded after being received by a reader. + */ + std::optional discardPolicy; + }; + + /** + * The WriterConfig class specifies configuration options specific to writers. + * + * It extends the Config class and therefore inherits its configuration + * options. + * + * @headerfile DataStorm/DataStorm.h + */ + class WriterConfig : public Config + { + public: + /** + * Construct a WriterConfig object. + * + * The constructor accepts optional parameters for each of the WriterConfig data members. + * + * @param sampleCount The optional sample count. + * @param sampleLifetime The optional sample lifetime. + * @param clearHistory The optional clear history policy. + * @param priority The writer priority. + */ + WriterConfig( + std::optional sampleCount = std::nullopt, + std::optional sampleLifetime = std::nullopt, + std::optional clearHistory = std::nullopt, + std::optional priority = std::nullopt) noexcept + : Config(std::move(sampleCount), std::move(sampleLifetime), std::move(clearHistory)), + priority(std::move(priority)) + { + } + + /** + * Specifies the writer priority. The priority is used by readers using the priority discard policy. + */ + std::optional priority; + }; + + /** + * The callback action enumerator specifies the reason why a callback is called. + */ + enum struct CallbackReason + { + /** The callback is called because of connection. */ + Connect, + + /** The callback is called because of a disconnection. */ + Disconnect + }; + + /** + * The Encoder template provides a method to encode decode user types. + * + * The encoder template can be specialized to provide encoding for types that don't support being encoded with + * Ice. By default, the Ice encoding is used if no Encoder template specialization is provided for the type. + * + * @headerfile DataStorm/DataStorm.h + */ + template struct Encoder + { + /** + * Encode the given value. This method encodes the given value and returns the resulting byte sequence. + * The communicator parameter is provided to allow the implementation to eventually use the Ice encoding. + * + * @see decode + * + * @param communicator The communicator associated with the node + * @param value The value to encode + * @return The resulting byte sequence + */ + static Ice::ByteSeq encode(const Ice::CommunicatorPtr& communicator, const T& value) noexcept; + }; + + /** + * The Decoder template provides a method to decode user types. + * + * The decoder template can be specialized to provide decoding for types that don't support being decoded with + * Ice. By default, the Ice decoding is used if no Decoder template specialization is provided for the type. + * + * @headerfile DataStorm/DataStorm.h + */ + template struct Decoder + { + /** + * Decode a value. This method decodes the given byte sequence and returns the resulting value. The + * communicator parameter is provided to allow the implementation to eventually use the Ice encoding. + * + * @see encode + * + * @param communicator The communicator associated with the node + * @param value The byte sequence to decode + * @return The resulting value + */ + static T decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& value) noexcept; + }; + + /** + * The Cloner template provides a method to clone user types. + * + * The cloner template can be specialized to provide cloning for types that require special cloning. By + * defaut, the template uses plain C++ copy. + * + * @headerfile DataStorm/DataStorm.h + */ + template struct Cloner + { + /** + * Clone the given value. This helper is used when processing partial update to clone the previous value + * and compute the new value with the partial update. The default implementation performs a plain C++ copy + * with the copy constructor. + * + * @param value The value to encode + * @return The cloned value + */ + static T clone(const T& value) noexcept { return value; } + }; + + /** + * Encoder template specialization to encode Ice::Value instances. + **/ + template struct Encoder::value>::type> + { + static Ice::ByteSeq encode(const Ice::CommunicatorPtr& communicator, const T& value) noexcept + { + return Encoder>::encode(communicator, std::make_shared(value)); + } + }; + + /** + * Decoder template specialization to decode Ice::Value instances. + **/ + template struct Decoder::value>::type> + { + static T decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& data) noexcept + { + return *Decoder>::decode(communicator, data); + } + }; + + /** + * Cloner template specialization to clone shared Ice values using ice_clone. + */ + template + struct Cloner, typename std::enable_if::value>::type> + { + static std::shared_ptr clone(const std::shared_ptr& value) noexcept { return value->ice_clone(); } + }; + + /** + * Encoder template implementation + */ + template + Ice::ByteSeq Encoder::encode(const Ice::CommunicatorPtr& communicator, const T& value) noexcept + { + Ice::ByteSeq v; + Ice::OutputStream stream(communicator); + stream.write(value); + stream.finished(v); + return v; + } + + /** + * Decoder template implementation + */ + template + T Decoder::decode(const Ice::CommunicatorPtr& communicator, const Ice::ByteSeq& value) noexcept + { + T v; + if (value.empty()) + { + v = T(); + } + else + { + Ice::InputStream(communicator, value).read(v); + } + return v; + } + +} +#endif diff --git a/cpp/msbuild/ice.sln b/cpp/msbuild/ice.sln index 67d610c45a6..e8b6cc8e833 100644 --- a/cpp/msbuild/ice.sln +++ b/cpp/msbuild/ice.sln @@ -203,6 +203,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "glacier2cryptpermissionsver {2940A3C2-A9BA-44AA-AF65-00479C783407} = {2940A3C2-A9BA-44AA-AF65-00479C783407} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "datastorm", "..\src\DataStorm\msbuild\datastorm\datastorm.vcxproj", "{BD2AC148-422F-49E0-9185-ACCBA7AECF26}" + ProjectSection(ProjectDependencies) = postProject + {3AB9772C-6113-4F1C-90FB-5368E7486C11} = {3AB9772C-6113-4F1C-90FB-5368E7486C11} + {763F88BB-37FD-4BCC-9D13-A7103596EA06} = {763F88BB-37FD-4BCC-9D13-A7103596EA06} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dsnode", "..\src\dsnode\msbuild\dsnode.vcxproj", "{85F74421-2B78-448B-91F0-496526EB365F}" + ProjectSection(ProjectDependencies) = postProject + {BD2AC148-422F-49E0-9185-ACCBA7AECF26} = {BD2AC148-422F-49E0-9185-ACCBA7AECF26} + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -475,6 +486,22 @@ Global {BF2077E4-D837-486B-9356-24FA5F659808}.Release|Win32.Build.0 = Release|Win32 {BF2077E4-D837-486B-9356-24FA5F659808}.Release|x64.ActiveCfg = Release|x64 {BF2077E4-D837-486B-9356-24FA5F659808}.Release|x64.Build.0 = Release|x64 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Debug|Win32.ActiveCfg = Debug|Win32 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Debug|Win32.Build.0 = Debug|Win32 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Debug|x64.ActiveCfg = Debug|x64 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Debug|x64.Build.0 = Debug|x64 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Release|Win32.ActiveCfg = Release|Win32 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Release|Win32.Build.0 = Release|Win32 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Release|x64.ActiveCfg = Release|x64 + {BD2AC148-422F-49E0-9185-ACCBA7AECF26}.Release|x64.Build.0 = Release|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|Win32.ActiveCfg = Debug|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|Win32.Build.0 = Debug|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|x64.ActiveCfg = Debug|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|x64.Build.0 = Debug|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|Win32.ActiveCfg = Release|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|Win32.Build.0 = Release|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|x64.ActiveCfg = Release|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/cpp/msbuild/ice.test.sln b/cpp/msbuild/ice.test.sln index a69827b3b4b..0d1146f54b2 100644 --- a/cpp/msbuild/ice.test.sln +++ b/cpp/msbuild/ice.test.sln @@ -1143,6 +1143,48 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "..\test\Ice\maxCo {C7223CC8-0AAA-470B-ACB3-12B9DE75525C} = {C7223CC8-0AAA-470B-ACB3-12B9DE75525C} EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "DataStorm", "DataStorm", "{8050D625-A26F-4EA0-A74D-6095A1F13B92}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "api", "api", "{834A92DD-5964-4F16-8A9B-77F764620F55}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "callbacks", "callbacks", "{4564A657-073F-470E-A4AD-A5D947DFFEE3}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{C5A4D258-2623-42C0-95AB-8CF48ACECE56}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "events", "events", "{3A4E17C2-28FB-46AF-BE15-E1D8DBC232C9}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "partial", "partial", "{7D50B677-D295-4CAF-B555-9C87FAF4E8D6}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "reliability", "reliability", "{E435395E-2CFB-4933-941D-69EE60805093}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "types", "types", "{2618669B-54B1-4013-B6EC-C1389391E3AA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\api\msbuild\writer\writer.vcxproj", "{85F74421-2B78-448B-91F0-496526EB365F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "reader", "..\test\DataStorm\callbacks\msbuild\reader\reader.vcxproj", "{783EE047-9119-4412-960A-600729D1A0F1}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\callbacks\msbuild\writer\writer.vcxproj", "{8ADD3C14-F6A9-4683-94BE-EB87C60078F1}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "reader", "..\test\DataStorm\config\msbuild\reader\reader.vcxproj", "{735292B7-70CB-43B2-B7BB-D123B7497CC7}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\config\msbuild\writer\writer.vcxproj", "{1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "reader", "..\test\DataStorm\events\msbuild\reader\reader.vcxproj", "{5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\events\msbuild\writer\writer.vcxproj", "{A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "reader", "..\test\DataStorm\partial\msbuild\reader\reader.vcxproj", "{3EAAF1DE-8673-4D16-AA36-F522C5C21F79}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\partial\msbuild\writer\writer.vcxproj", "{71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "reader", "..\test\DataStorm\reliability\msbuild\reader\reader.vcxproj", "{18B9E886-A355-4CCC-8E27-84900C516DA8}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\reliability\msbuild\writer\writer.vcxproj", "{71A33CBC-71F6-434B-BB90-620710269310}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "reader", "..\test\DataStorm\types\msbuild\reader\reader.vcxproj", "{924B7545-4B49-41E6-9FAC-30F7BAB7A518}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "writer", "..\test\DataStorm\types\msbuild\writer\writer.vcxproj", "{6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Win32 = Debug|Win32 @@ -2663,6 +2705,110 @@ Global {2750A8E9-34AB-4A5E-ABC4-78CE41328A12}.Release|Win32.Build.0 = Release|Win32 {2750A8E9-34AB-4A5E-ABC4-78CE41328A12}.Release|x64.ActiveCfg = Release|x64 {2750A8E9-34AB-4A5E-ABC4-78CE41328A12}.Release|x64.Build.0 = Release|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|Win32.ActiveCfg = Debug|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|Win32.Build.0 = Debug|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|x64.ActiveCfg = Debug|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Debug|x64.Build.0 = Debug|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|Win32.ActiveCfg = Release|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|Win32.Build.0 = Release|Win32 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|x64.ActiveCfg = Release|x64 + {85F74421-2B78-448B-91F0-496526EB365F}.Release|x64.Build.0 = Release|x64 + {783EE047-9119-4412-960A-600729D1A0F1}.Debug|Win32.ActiveCfg = Debug|Win32 + {783EE047-9119-4412-960A-600729D1A0F1}.Debug|Win32.Build.0 = Debug|Win32 + {783EE047-9119-4412-960A-600729D1A0F1}.Debug|x64.ActiveCfg = Debug|x64 + {783EE047-9119-4412-960A-600729D1A0F1}.Debug|x64.Build.0 = Debug|x64 + {783EE047-9119-4412-960A-600729D1A0F1}.Release|Win32.ActiveCfg = Release|Win32 + {783EE047-9119-4412-960A-600729D1A0F1}.Release|Win32.Build.0 = Release|Win32 + {783EE047-9119-4412-960A-600729D1A0F1}.Release|x64.ActiveCfg = Release|x64 + {783EE047-9119-4412-960A-600729D1A0F1}.Release|x64.Build.0 = Release|x64 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Debug|Win32.ActiveCfg = Debug|Win32 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Debug|Win32.Build.0 = Debug|Win32 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Debug|x64.ActiveCfg = Debug|x64 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Debug|x64.Build.0 = Debug|x64 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Release|Win32.ActiveCfg = Release|Win32 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Release|Win32.Build.0 = Release|Win32 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Release|x64.ActiveCfg = Release|x64 + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1}.Release|x64.Build.0 = Release|x64 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Debug|Win32.ActiveCfg = Debug|Win32 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Debug|Win32.Build.0 = Debug|Win32 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Debug|x64.ActiveCfg = Debug|x64 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Debug|x64.Build.0 = Debug|x64 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Release|Win32.ActiveCfg = Release|Win32 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Release|Win32.Build.0 = Release|Win32 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Release|x64.ActiveCfg = Release|x64 + {735292B7-70CB-43B2-B7BB-D123B7497CC7}.Release|x64.Build.0 = Release|x64 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Debug|Win32.ActiveCfg = Debug|Win32 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Debug|Win32.Build.0 = Debug|Win32 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Debug|x64.ActiveCfg = Debug|x64 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Debug|x64.Build.0 = Debug|x64 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Release|Win32.ActiveCfg = Release|Win32 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Release|Win32.Build.0 = Release|Win32 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Release|x64.ActiveCfg = Release|x64 + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55}.Release|x64.Build.0 = Release|x64 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Debug|Win32.ActiveCfg = Debug|Win32 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Debug|Win32.Build.0 = Debug|Win32 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Debug|x64.ActiveCfg = Debug|x64 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Debug|x64.Build.0 = Debug|x64 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Release|Win32.ActiveCfg = Release|Win32 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Release|Win32.Build.0 = Release|Win32 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Release|x64.ActiveCfg = Release|x64 + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00}.Release|x64.Build.0 = Release|x64 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Debug|Win32.ActiveCfg = Debug|Win32 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Debug|Win32.Build.0 = Debug|Win32 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Debug|x64.ActiveCfg = Debug|x64 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Debug|x64.Build.0 = Debug|x64 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Release|Win32.ActiveCfg = Release|Win32 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Release|Win32.Build.0 = Release|Win32 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Release|x64.ActiveCfg = Release|x64 + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB}.Release|x64.Build.0 = Release|x64 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Debug|Win32.ActiveCfg = Debug|Win32 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Debug|Win32.Build.0 = Debug|Win32 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Debug|x64.ActiveCfg = Debug|x64 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Debug|x64.Build.0 = Debug|x64 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Release|Win32.ActiveCfg = Release|Win32 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Release|Win32.Build.0 = Release|Win32 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Release|x64.ActiveCfg = Release|x64 + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79}.Release|x64.Build.0 = Release|x64 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Debug|Win32.ActiveCfg = Debug|Win32 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Debug|Win32.Build.0 = Debug|Win32 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Debug|x64.ActiveCfg = Debug|x64 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Debug|x64.Build.0 = Debug|x64 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Release|Win32.ActiveCfg = Release|Win32 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Release|Win32.Build.0 = Release|Win32 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Release|x64.ActiveCfg = Release|x64 + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C}.Release|x64.Build.0 = Release|x64 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Debug|Win32.ActiveCfg = Debug|Win32 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Debug|Win32.Build.0 = Debug|Win32 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Debug|x64.ActiveCfg = Debug|x64 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Debug|x64.Build.0 = Debug|x64 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Release|Win32.ActiveCfg = Release|Win32 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Release|Win32.Build.0 = Release|Win32 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Release|x64.ActiveCfg = Release|x64 + {18B9E886-A355-4CCC-8E27-84900C516DA8}.Release|x64.Build.0 = Release|x64 + {71A33CBC-71F6-434B-BB90-620710269310}.Debug|Win32.ActiveCfg = Debug|Win32 + {71A33CBC-71F6-434B-BB90-620710269310}.Debug|Win32.Build.0 = Debug|Win32 + {71A33CBC-71F6-434B-BB90-620710269310}.Debug|x64.ActiveCfg = Debug|x64 + {71A33CBC-71F6-434B-BB90-620710269310}.Debug|x64.Build.0 = Debug|x64 + {71A33CBC-71F6-434B-BB90-620710269310}.Release|Win32.ActiveCfg = Release|Win32 + {71A33CBC-71F6-434B-BB90-620710269310}.Release|Win32.Build.0 = Release|Win32 + {71A33CBC-71F6-434B-BB90-620710269310}.Release|x64.ActiveCfg = Release|x64 + {71A33CBC-71F6-434B-BB90-620710269310}.Release|x64.Build.0 = Release|x64 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Debug|Win32.ActiveCfg = Debug|Win32 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Debug|Win32.Build.0 = Debug|Win32 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Debug|x64.ActiveCfg = Debug|x64 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Debug|x64.Build.0 = Debug|x64 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Release|Win32.ActiveCfg = Release|Win32 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Release|Win32.Build.0 = Release|Win32 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Release|x64.ActiveCfg = Release|x64 + {924B7545-4B49-41E6-9FAC-30F7BAB7A518}.Release|x64.Build.0 = Release|x64 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Debug|Win32.ActiveCfg = Debug|Win32 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Debug|Win32.Build.0 = Debug|Win32 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Debug|x64.ActiveCfg = Debug|x64 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Debug|x64.Build.0 = Debug|x64 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Release|Win32.ActiveCfg = Release|Win32 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Release|Win32.Build.0 = Release|Win32 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Release|x64.ActiveCfg = Release|x64 + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2944,6 +3090,26 @@ Global {286A7273-091A-4C8A-84ED-8CBB52D790A6} = {2CAF9731-CB18-498C-A3EF-24F3D8A334AC} {9A7E939A-DE60-431B-ABD2-5A44EE2CABD6} = {286A7273-091A-4C8A-84ED-8CBB52D790A6} {2750A8E9-34AB-4A5E-ABC4-78CE41328A12} = {286A7273-091A-4C8A-84ED-8CBB52D790A6} + {834A92DD-5964-4F16-8A9B-77F764620F55} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {4564A657-073F-470E-A4AD-A5D947DFFEE3} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {C5A4D258-2623-42C0-95AB-8CF48ACECE56} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {3A4E17C2-28FB-46AF-BE15-E1D8DBC232C9} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {7D50B677-D295-4CAF-B555-9C87FAF4E8D6} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {E435395E-2CFB-4933-941D-69EE60805093} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {2618669B-54B1-4013-B6EC-C1389391E3AA} = {8050D625-A26F-4EA0-A74D-6095A1F13B92} + {85F74421-2B78-448B-91F0-496526EB365F} = {834A92DD-5964-4F16-8A9B-77F764620F55} + {783EE047-9119-4412-960A-600729D1A0F1} = {4564A657-073F-470E-A4AD-A5D947DFFEE3} + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1} = {4564A657-073F-470E-A4AD-A5D947DFFEE3} + {735292B7-70CB-43B2-B7BB-D123B7497CC7} = {C5A4D258-2623-42C0-95AB-8CF48ACECE56} + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55} = {C5A4D258-2623-42C0-95AB-8CF48ACECE56} + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00} = {3A4E17C2-28FB-46AF-BE15-E1D8DBC232C9} + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB} = {3A4E17C2-28FB-46AF-BE15-E1D8DBC232C9} + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79} = {7D50B677-D295-4CAF-B555-9C87FAF4E8D6} + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C} = {7D50B677-D295-4CAF-B555-9C87FAF4E8D6} + {18B9E886-A355-4CCC-8E27-84900C516DA8} = {E435395E-2CFB-4933-941D-69EE60805093} + {71A33CBC-71F6-434B-BB90-620710269310} = {E435395E-2CFB-4933-941D-69EE60805093} + {924B7545-4B49-41E6-9FAC-30F7BAB7A518} = {2618669B-54B1-4013-B6EC-C1389391E3AA} + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8} = {2618669B-54B1-4013-B6EC-C1389391E3AA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E6FDB68A-64BA-4577-ABCD-40A01257F8AB} diff --git a/cpp/src/DataStorm/CallbackExecutor.cpp b/cpp/src/DataStorm/CallbackExecutor.cpp new file mode 100644 index 00000000000..a12e0b3eb5b --- /dev/null +++ b/cpp/src/DataStorm/CallbackExecutor.cpp @@ -0,0 +1,78 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "CallbackExecutor.h" + +using namespace std; +using namespace DataStormI; + +CallbackExecutor::CallbackExecutor() : _flush(false), _destroyed(false) +{ + _thread = thread( + [this] + { + std::vector, std::function>> queue; + while (true) + { + unique_lock lock(_mutex); + if (_destroyed) + { + break; + } + _cond.wait(lock, [this] { return _flush || _destroyed; }); + if (_flush) + { + _flush = false; + _queue.swap(queue); + } + + lock.unlock(); + for (const auto& p : queue) + { + try + { + p.second(); + } + catch (...) + { + std::terminate(); + } + } + queue.clear(); + } + }); +} + +void +CallbackExecutor::queue(const std::shared_ptr& element, std::function cb, bool flush) +{ + unique_lock lock(_mutex); + _queue.emplace_back(element, cb); + if (flush) + { + _flush = true; + _cond.notify_one(); + } +} + +void +CallbackExecutor::flush() +{ + unique_lock lock(_mutex); + if (!_queue.empty()) + { + _flush = true; + _cond.notify_one(); + } +} + +void +CallbackExecutor::destroy() +{ + unique_lock lock(_mutex); + _destroyed = true; + _cond.notify_one(); + lock.unlock(); + _thread.join(); +} diff --git a/cpp/src/DataStorm/CallbackExecutor.h b/cpp/src/DataStorm/CallbackExecutor.h new file mode 100644 index 00000000000..0c3b725f2f3 --- /dev/null +++ b/cpp/src/DataStorm/CallbackExecutor.h @@ -0,0 +1,40 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_CALLBACK_EXECUTOR_H +#define DATASTORM_CALLBACK_EXECUTOR_H + +#include +#include +#include +#include +#include +#include + +namespace DataStormI +{ + + class DataElementI; + + class CallbackExecutor + { + public: + CallbackExecutor(); + + void queue(const std::shared_ptr&, std::function, bool = false); + void flush(); + void destroy(); + + private: + std::mutex _mutex; + std::thread _thread; + std::condition_variable _cond; + bool _flush; + bool _destroyed; + std::vector, std::function>> _queue; + }; + +} + +#endif diff --git a/cpp/src/DataStorm/ConnectionManager.cpp b/cpp/src/DataStorm/ConnectionManager.cpp new file mode 100644 index 00000000000..02171cfed33 --- /dev/null +++ b/cpp/src/DataStorm/ConnectionManager.cpp @@ -0,0 +1,97 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "ConnectionManager.h" +#include "CallbackExecutor.h" + +#include + +using namespace std; +using namespace DataStormI; + +ConnectionManager::ConnectionManager(const shared_ptr& executor) : _executor(executor) {} + +void +ConnectionManager::add( + const Ice::ConnectionPtr& connection, + shared_ptr object, + function callback) +{ + lock_guard lock(_mutex); + auto& objects = _connections[connection]; + if (objects.empty()) + { + connection->setCloseCallback([self = shared_from_this()](const Ice::ConnectionPtr& con) { self->remove(con); }); + } + objects.emplace(std::move(object), std::move(callback)); +} + +void +ConnectionManager::remove(const shared_ptr& object, const Ice::ConnectionPtr& connection) +{ + lock_guard lock(_mutex); + auto p = _connections.find(connection); + if (p == _connections.end()) + { + return; + } + auto& objects = p->second; + objects.erase(object); + if (objects.empty()) + { + connection->setCloseCallback(nullptr); + _connections.erase(p); + } +} + +void +ConnectionManager::remove(const Ice::ConnectionPtr& connection) +{ + map, Callback> objects; + { + lock_guard lock(_mutex); + auto p = _connections.find(connection); + if (p == _connections.end()) + { + return; + } + objects.swap(p->second); + connection->setCloseCallback(nullptr); + _connections.erase(p); + } + exception_ptr ex; + try + { + connection->getInfo(); + } + catch (const std::exception&) + { + ex = current_exception(); + } + for (const auto& object : objects) + { + try + { + object.second(connection, ex); + } + catch (const std::exception& ex) + { + cerr << ex.what() << endl; + assert(false); + throw; + } + } + _executor->flush(); +} + +void +ConnectionManager::destroy() +{ + lock_guard lock(_mutex); + for (const auto& connection : _connections) + { + connection.first->setCloseCallback(nullptr); + } + _connections.clear(); +} diff --git a/cpp/src/DataStorm/ConnectionManager.h b/cpp/src/DataStorm/ConnectionManager.h new file mode 100644 index 00000000000..25b04bcc257 --- /dev/null +++ b/cpp/src/DataStorm/ConnectionManager.h @@ -0,0 +1,41 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_CONNECTION_MANAGER_H +#define DATASTORM_CONNECTION_MANAGER_H + +#include "DataStorm/Config.h" +#include "Ice/Ice.h" + +namespace DataStormI +{ + + class CallbackExecutor; + + class ConnectionManager : public std::enable_shared_from_this + { + public: + ConnectionManager(const std::shared_ptr&); + + void + add(const Ice::ConnectionPtr&, + std::shared_ptr, + std::function); + + void remove(const std::shared_ptr&, const Ice::ConnectionPtr&); + void remove(const Ice::ConnectionPtr&); + + void destroy(); + + private: + using Callback = std::function; + + std::mutex _mutex; + std::map, Callback>> _connections; + std::shared_ptr _executor; + }; + +} + +#endif diff --git a/cpp/src/DataStorm/Contract.ice b/cpp/src/DataStorm/Contract.ice new file mode 100644 index 00000000000..a8673a83d97 --- /dev/null +++ b/cpp/src/DataStorm/Contract.ice @@ -0,0 +1,247 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +#pragma once + +#include +#include + +[["cpp:include:deque"]] + +module DataStormContract +{ + +enum ClearHistoryPolicy +{ + OnAdd, + OnRemove, + OnAll, + OnAllExceptPartialUpdate, + Never +}; + +/** A sequence of bytes use to hold the encoded key or value */ +sequence ByteSeq; + +/** A sequence of long */ +sequence LongSeq; + +/** A sequence of strings */ +sequence StringSeq; + +/** A dictionary of */ +dictionary LongLongDict; + +struct DataSample +{ + /** The sample id. */ + long id; + + /** The key id. */ + long keyId; + + /** The key value if the key ID <= 0. */ + ByteSeq keyValue; + + /** The timestamp of the sample (write time). */ + long timestamp; + + /** The update tag if the sample event is PartialUpdate. */ + long tag; + + /** The sample event. */ + DataStorm::SampleEvent event; + + /** The value of the sample. */ + ByteSeq value; +} +["cpp:type:std::deque"] sequence DataSampleSeq; + +struct DataSamples +{ + /** The id of the writer or reader. */ + long id; + + /** The samples. */ + DataSampleSeq samples; +} +sequence DataSamplesSeq; + +struct ElementInfo +{ + /** The key or filter id. */ + long id; + + /** The filter name. */ + string name; + + /** The key or filter value. */ + ByteSeq value; +} +sequence ElementInfoSeq; + +struct TopicInfo +{ + /** The topic name. */ + string name; + + /** The id of topic writers or readers. */ + LongSeq ids; +} +sequence TopicInfoSeq; + +struct TopicSpec +{ + /** The id of the topic. */ + long id; + + /* The name of the topic. */ + string name; + + /** The topic keys or filters. */ + ElementInfoSeq elements; + + /** The topic update tags. */ + ElementInfoSeq tags; +}; + +struct FilterInfo +{ + string name; + ByteSeq criteria; +}; + +class ElementConfig(1) +{ + optional(1) string facet; + optional(2) FilterInfo sampleFilter; + optional(3) string name; + optional(4) int priority; + + optional(10) int sampleCount; + optional(11) int sampleLifetime; + optional(12) ClearHistoryPolicy clearHistory; +}; + +struct ElementData +{ + /** The id of the writer or reader */ + long id; + + /** The config of the writer or reader */ + ElementConfig config; + + /** The lastIds received by the reader. */ + LongLongDict lastIds; +} +sequence ElementDataSeq; + +struct ElementSpec +{ + /** The readers and writers associated with the key or filter. */ + ElementDataSeq elements; + + /** The id of the key or filter */ + long id; + + /** The name of the filter. */ + string name; + + /** The value of the key or filter. */ + ByteSeq value; + + /** The id of the key or filter from the peer. */ + long peerId; + + /** The name of the filter from the peer. */ + string peerName; +} +sequence ElementSpecSeq; + +struct ElementDataAck +{ + /** The id of the writer or filter. */ + long id; + + /** The config of the writer or reader */ + ElementConfig config; + + /** The lastIds received by the reader. */ + LongLongDict lastIds; + + /** The samples of the writer or reader */ + DataSampleSeq samples; + + /** The id of the writer or reader on the peer. */ + long peerId; +} +sequence ElementDataAckSeq; + +struct ElementSpecAck +{ + /** The readers or writers associated with the key or filter. */ + ElementDataAckSeq elements; + + /** The id of the key or filter. */ + long id; + + /** The name of the filter. */ + string name; + + /** The key or filter value. */ + ByteSeq value; + + /** The id of the key or filter on the peer. */ + long peerId; + + /** The name of the filter on the peer. */ + string peerName; +} +sequence ElementSpecAckSeq; + +interface Session +{ + void announceTopics(TopicInfoSeq topics, bool initialize); + void attachTopic(TopicSpec topic); + void detachTopic(long topic); + + void attachTags(long topic, ElementInfoSeq tags, bool initialize); + void detachTags(long topic, LongSeq tags); + + void announceElements(long topic, ElementInfoSeq keys); + void attachElements(long topic, ElementSpecSeq elements, bool initialize); + void attachElementsAck(long topic, ElementSpecAckSeq elements); + void detachElements(long topic, LongSeq keys); + + void initSamples(long topic, DataSamplesSeq samples); + + void disconnected(); +} + +interface PublisherSession extends Session +{ +} + +interface SubscriberSession extends Session +{ + void s(long topicId, long elementId, DataSample sample); +} + +interface Node +{ + void initiateCreateSession(Node* publisher); + void createSession(Node* subscriber, SubscriberSession* session, bool fromRelay); + void confirmCreateSession(Node* publisher, PublisherSession* session); +} + +interface Lookup +{ + idempotent void announceTopicReader(string topic, Node* node); + idempotent void announceTopicWriter(string topic, Node* node); + + idempotent void announceTopics(StringSeq readers, StringSeq writers, Node* node); + + Node* createSession(Node* node); +} + +} diff --git a/cpp/src/DataStorm/DataElementI.cpp b/cpp/src/DataStorm/DataElementI.cpp new file mode 100644 index 00000000000..d4527b2b87d --- /dev/null +++ b/cpp/src/DataStorm/DataElementI.cpp @@ -0,0 +1,1411 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataElementI.h" +#include "CallbackExecutor.h" +#include "Ice/Ice.h" +#include "Instance.h" +#include "NodeI.h" +#include "TopicI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; +using namespace DataStormContract; + +namespace +{ + + DataSample toSample(const shared_ptr& sample, const Ice::CommunicatorPtr& communicator, bool marshalKey) + { + return { + sample->id, + marshalKey ? 0 : sample->key->getId(), + marshalKey ? sample->key->encode(communicator) : Ice::ByteSeq{}, + chrono::time_point_cast(sample->timestamp).time_since_epoch().count(), + sample->tag ? sample->tag->getId() : 0, + sample->event, + sample->encode(communicator)}; + } + + void cleanOldSamples( + deque>& samples, + const chrono::time_point& now, + int lifetime) + { + chrono::time_point staleTime = now - chrono::milliseconds(lifetime); + auto p = stable_partition( + samples.begin(), + samples.end(), + [&](const shared_ptr& s) { return s->timestamp < staleTime; }); + if (p != samples.begin()) + { + samples.erase(samples.begin(), p); + } + } + +} + +DataElementI::DataElementI(TopicI* parent, const string& name, int64_t id, const DataStorm::Config& config) + : _traceLevels(parent->getInstance()->getTraceLevels()), + _name(name), + _id(id), + _config(make_shared()), + _executor(parent->getInstance()->getCallbackExecutor()), + _listenerCount(0), + _parent(parent->shared_from_this()), + _waiters(0), + _notified(0), + _destroyed(false) +{ + _config->sampleCount = config.sampleCount; + _config->sampleLifetime = config.sampleLifetime; + if (!name.empty()) + { + _config->name = name; + } + if (config.clearHistory) + { + _config->clearHistory = static_cast(*config.clearHistory); + } +} + +void +DataElementI::init() +{ + auto forwarder = [self = shared_from_this()](Ice::ByteSeq e, const Ice::Current& c) { self->forward(e, c); }; + _forwarder = + Ice::uncheckedCast(_parent->getInstance()->getCollocatedForwarder()->add(std::move(forwarder))); +} + +DataElementI::~DataElementI() +{ + assert(_destroyed); + assert(_listeners.empty()); + assert(_listenerCount == 0); +} + +void +DataElementI::destroy() +{ + { + unique_lock lock(_parent->_mutex); + assert(!_destroyed); + _destroyed = true; + destroyImpl(); // Must be called first. + } + disconnect(); + _parent->getInstance()->getCollocatedForwarder()->remove(_forwarder->ice_getIdentity()); +} + +void +DataElementI::attach( + int64_t topicId, + int64_t id, + const shared_ptr& key, + const shared_ptr& filter, + const shared_ptr& session, + optional prx, + const ElementData& data, + const chrono::time_point& now, + ElementDataAckSeq& acks) +{ + shared_ptr sampleFilter; + if (data.config->sampleFilter) + { + auto info = *data.config->sampleFilter; + sampleFilter = _parent->getSampleFilterFactories()->decode(getCommunicator(), info.name, info.criteria); + } + string facet = data.config->facet ? *data.config->facet : string(); + int priority = data.config->priority ? *data.config->priority : 0; + string name; + if (data.config->name) + { + name = *data.config->name; + } + else + { + ostringstream os; + os << session->getId() << '-' << topicId << '-' << data.id; + name = os.str(); + } + if ((id > 0 && attachKey(topicId, data.id, key, sampleFilter, session, prx, facet, id, name, priority)) || + (id < 0 && attachFilter(topicId, data.id, key, sampleFilter, session, prx, facet, id, filter, name, priority))) + { + auto q = data.lastIds.find(_id); + long long lastId = q != data.lastIds.end() ? q->second : 0; + LongLongDict lastIds = key ? session->getLastIds(topicId, id, shared_from_this()) : LongLongDict(); + DataSamples samples = getSamples(key, sampleFilter, data.config, lastId, now); + acks.push_back({_id, _config, lastIds, samples.samples, data.id}); + } +} + +std::function +DataElementI::attach( + int64_t topicId, + int64_t id, + const shared_ptr& key, + const shared_ptr& filter, + const shared_ptr& session, + optional prx, + const ElementDataAck& data, + const chrono::time_point& now, + DataSamplesSeq& samples) +{ + shared_ptr sampleFilter; + if (data.config->sampleFilter) + { + auto info = *data.config->sampleFilter; + sampleFilter = _parent->getSampleFilterFactories()->decode(getCommunicator(), info.name, info.criteria); + } + string facet = data.config->facet ? *data.config->facet : string(); + int priority = data.config->priority ? *data.config->priority : 0; + string name; + if (data.config->name) + { + name = *data.config->name; + } + else + { + ostringstream os; + os << session->getId() << '-' << topicId << '-' << data.id; + name = os.str(); + } + if ((id > 0 && attachKey(topicId, data.id, key, sampleFilter, session, prx, facet, id, name, priority)) || + (id < 0 && attachFilter(topicId, data.id, key, sampleFilter, session, prx, facet, id, filter, name, priority))) + { + auto q = data.lastIds.find(_id); + long long lastId = q != data.lastIds.end() ? q->second : 0; + samples.push_back(getSamples(key, sampleFilter, data.config, lastId, now)); + } + + auto samplesI = + session->subscriberInitialized(topicId, id > 0 ? data.id : -data.id, data.samples, key, shared_from_this()); + if (!samplesI.empty()) + { + return [=, this]() { initSamples(samplesI, topicId, data.id, priority, now, id < 0); }; + } + return nullptr; +} + +bool +DataElementI::attachKey( + int64_t topicId, + int64_t elementId, + const shared_ptr& key, + const shared_ptr& sampleFilter, + const shared_ptr& session, + optional prx, + const string& facet, + int64_t keyId, + const string& name, + int priority) +{ + // No locking necessary, called by the session with the mutex locked + auto p = _listeners.find({session, facet}); + if (p == _listeners.end()) + { + p = _listeners.emplace(ListenerKey{session, facet}, Listener(std::move(prx), facet)).first; + } + + bool added = false; + auto subscriber = p->second.addOrGet(topicId, elementId, keyId, nullptr, sampleFilter, name, priority, added); + if (_onConnectedElements && added) + { + _executor->queue( + shared_from_this(), + [=, this] { _onConnectedElements(DataStorm::CallbackReason::Connect, name); }); + } + if (addConnectedKey(key, subscriber)) + { + if (key) + { + subscriber->keys.insert(key); + } + if (_traceLevels->data > 1) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": attach e" << elementId << ":" << name; + if (!facet.empty()) + { + out << ":" << facet; + } + out << ":[" << key << "]@" << topicId; + } + + ++_listenerCount; + _parent->incListenerCount(session); + session->subscribeToKey(topicId, elementId, shared_from_this(), facet, key, keyId, name, priority); + notifyListenerWaiters(session->getTopicLock()); + return true; + } + return false; +} + +void +DataElementI::detachKey( + int64_t topicId, + int64_t elementId, + const shared_ptr& key, + const shared_ptr& session, + const string& facet, + bool unsubscribe) +{ + // No locking necessary, called by the session with the mutex locked + auto p = _listeners.find({session, facet}); + if (p == _listeners.end()) + { + return; + } + + auto subscriber = p->second.get(topicId, elementId); + if (removeConnectedKey(key, subscriber)) + { + if (key) + { + subscriber->keys.erase(key); + } + if (subscriber->keys.empty()) + { + if (_onConnectedElements) + { + _executor->queue( + shared_from_this(), + [=, this] { _onConnectedElements(DataStorm::CallbackReason::Disconnect, subscriber->name); }); + } + if (p->second.remove(topicId, elementId)) + { + _listeners.erase(p); + } + } + + if (_traceLevels->data > 1) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": detach e" << elementId << ":" << subscriber->name; + if (!facet.empty()) + { + out << ":" << facet; + } + out << ":[" << key << "]@" << topicId; + } + --_listenerCount; + _parent->decListenerCount(session); + if (unsubscribe) + { + session->unsubscribeFromKey(topicId, elementId, shared_from_this(), subscriber->id); + } + notifyListenerWaiters(session->getTopicLock()); + } +} + +bool +DataElementI::attachFilter( + int64_t topicId, + int64_t elementId, + const shared_ptr& key, + const shared_ptr& sampleFilter, + const shared_ptr& session, + optional prx, + const string& facet, + int64_t filterId, + const shared_ptr& filter, + const string& name, + int priority) +{ + // No locking necessary, called by the session with the mutex locked + auto p = _listeners.find({session, facet}); + if (p == _listeners.end()) + { + p = _listeners.emplace(ListenerKey{session, facet}, Listener(prx, facet)).first; + } + + bool added = false; + auto subscriber = p->second.addOrGet(topicId, -elementId, filterId, filter, sampleFilter, name, priority, added); + if (_onConnectedElements && added) + { + _executor->queue( + shared_from_this(), + [=, this] { _onConnectedElements(DataStorm::CallbackReason::Connect, name); }); + } + if (addConnectedKey(key, subscriber)) + { + if (key) + { + subscriber->keys.insert(key); + } + if (_traceLevels->data > 1) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": attach e" << elementId << ":" << name; + if (!facet.empty()) + { + out << ":" << facet; + } + out << ":[" << filter << "]@" << topicId; + } + + ++_listenerCount; + _parent->incListenerCount(session); + session->subscribeToFilter(topicId, elementId, shared_from_this(), facet, key, name, priority); + notifyListenerWaiters(session->getTopicLock()); + return true; + } + return false; +} + +void +DataElementI::detachFilter( + int64_t topicId, + int64_t elementId, + const shared_ptr& key, + const shared_ptr& session, + const string& facet, + bool unsubscribe) +{ + // No locking necessary, called by the session with the mutex locked + auto p = _listeners.find({session, facet}); + if (p == _listeners.end()) + { + return; + } + + auto subscriber = p->second.get(topicId, -elementId); + if (removeConnectedKey(key, subscriber)) + { + if (key) + { + subscriber->keys.erase(key); + } + if (subscriber->keys.empty()) + { + if (_onConnectedElements) + { + _executor->queue( + shared_from_this(), + [=, this] { _onConnectedElements(DataStorm::CallbackReason::Disconnect, subscriber->name); }); + } + if (p->second.remove(topicId, -elementId)) + { + _listeners.erase(p); + } + } + + if (_traceLevels->data > 1) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": detach e" << elementId << ":" << subscriber->name; + if (!facet.empty()) + { + out << ":" << facet; + } + out << ":[" << subscriber->filter << "]@" << topicId; + } + + --_listenerCount; + _parent->decListenerCount(session); + if (unsubscribe) + { + session->unsubscribeFromFilter(topicId, elementId, shared_from_this(), subscriber->id); + } + notifyListenerWaiters(session->getTopicLock()); + } +} + +vector> +DataElementI::getConnectedKeys() const +{ + unique_lock lock(_parent->_mutex); + vector> keys; + for (const auto& key : _connectedKeys) + { + keys.push_back(key.first); + } + return keys; +} + +vector +DataElementI::getConnectedElements() const +{ + unique_lock lock(_parent->_mutex); + vector elements; + elements.reserve(_listeners.size()); + for (const auto& listener : _listeners) + { + for (const auto& subscriber : listener.second.subscribers) + { + elements.push_back(subscriber.second->name); + } + } + return elements; +} + +void +DataElementI::onConnectedKeys( + function>)> init, + function)> update) +{ + unique_lock lock(_parent->_mutex); + _onConnectedKeys = std::move(update); + if (init) + { + vector> keys; + for (const auto& key : _connectedKeys) + { + keys.push_back(key.first); + } + _executor->queue(shared_from_this(), [init, keys] { init(keys); }, true); + } +} + +void +DataElementI::onConnectedElements( + function)> init, + function update) +{ + unique_lock lock(_parent->_mutex); + _onConnectedElements = std::move(update); + if (init) + { + vector elements; + for (const auto& listener : _listeners) + { + for (const auto& subscriber : listener.second.subscribers) + { + elements.push_back(subscriber.second->name); + } + } + _executor->queue(shared_from_this(), [init, elements] { init(elements); }, true); + } +} + +void +DataElementI::initSamples( + const vector>&, + int64_t, + int64_t, + int, + const chrono::time_point&, + bool) +{ +} + +DataSamples +DataElementI::getSamples( + const shared_ptr&, + const shared_ptr&, + const shared_ptr&, + int64_t, + const chrono::time_point&) +{ + return {}; +} + +void +DataElementI::queue( + const shared_ptr&, + int, + const shared_ptr&, + const string&, + const chrono::time_point&, + bool) +{ + assert(false); +} + +shared_ptr +DataElementI::getConfig() const +{ + return _config; +} + +void +DataElementI::waitForListeners(int count) const +{ + unique_lock lock(_parent->_mutex); + ++_waiters; + while (true) + { + _parent->getInstance()->checkShutdown(); + if (count < 0 && _listenerCount == 0) + { + --_waiters; + return; + } + else if (count >= 0 && _listenerCount >= static_cast(count)) + { + --_waiters; + return; + } + _parent->_cond.wait(lock); + ++_notified; + } +} + +bool +DataElementI::hasListeners() const +{ + unique_lock lock(_parent->_mutex); + return _listenerCount > 0; +} + +Ice::CommunicatorPtr +DataElementI::getCommunicator() const +{ + return _parent->getInstance()->getCommunicator(); +} + +bool +DataElementI::addConnectedKey(const shared_ptr& key, const shared_ptr& subscriber) +{ + auto& subscribers = _connectedKeys[key]; + if (find(subscribers.begin(), subscribers.end(), subscriber) == subscribers.end()) + { + if (key && subscribers.empty() && _onConnectedKeys) + { + _executor->queue( + shared_from_this(), + [=, this] { _onConnectedKeys(DataStorm::CallbackReason::Connect, key); }); + } + subscribers.push_back(subscriber); + return true; + } + return false; +} + +bool +DataElementI::removeConnectedKey(const shared_ptr& key, const shared_ptr& subscriber) +{ + auto& subscribers = _connectedKeys[key]; + auto p = find(subscribers.begin(), subscribers.end(), subscriber); + if (p != subscribers.end()) + { + subscribers.erase(p); + if (subscribers.empty()) + { + if (key && _onConnectedKeys) + { + _executor->queue( + shared_from_this(), + [=, this] { _onConnectedKeys(DataStorm::CallbackReason::Disconnect, key); }); + } + _connectedKeys.erase(key); + } + return true; + } + return false; +} + +void +DataElementI::notifyListenerWaiters(unique_lock& lock) const +{ + if (_waiters > 0) + { + _notified = 0; + _parent->_cond.notify_all(); + _parent->_cond.wait(lock, [&]() { return _notified < _waiters; }); // Wait until all the waiters are notified. + } +} + +void +DataElementI::disconnect() +{ + map listeners; + { + unique_lock lock(_parent->_mutex); + listeners.swap(_listeners); + _parent->decListenerCount(_listenerCount); + _listenerCount = 0; + notifyListenerWaiters(lock); + } + for (const auto& listener : listeners) + { + for (const auto& ks : listener.second.subscribers) + { + const auto& k = ks.first; + if (k.second < 0) + { + listener.first.session->disconnectFromFilter(k.first, k.second, shared_from_this(), ks.second->id); + } + else + { + listener.first.session->disconnectFromKey(k.first, k.second, shared_from_this(), ks.second->id); + } + } + } +} + +void +DataElementI::forward(const Ice::ByteSeq& inEncaps, const Ice::Current& current) const +{ + for (const auto& listener : _listeners) + { + // If there's at least one subscriber interested in the update + if (!_sample || listener.second.matchOne(_sample, false)) + { + // TODO do we need to check the result? + auto _ = listener.second.proxy->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } + } +} + +DataReaderI::DataReaderI( + TopicReaderI* topic, + const string& name, + int64_t id, + const string& sampleFilterName, + Ice::ByteSeq sampleFilterCriteria, + const DataStorm::ReaderConfig& config) + : DataElementI(topic, name, id, config), + _parent(topic), + _discardPolicy(config.discardPolicy ? *config.discardPolicy : DataStorm::DiscardPolicy::None) +{ + if (!sampleFilterName.empty()) + { + _config->sampleFilter = FilterInfo{sampleFilterName, std::move(sampleFilterCriteria)}; + } +} + +int +DataReaderI::getInstanceCount() const +{ + lock_guard lock(_parent->_mutex); + return _instanceCount; +} + +vector> +DataReaderI::getAllUnread() +{ + lock_guard lock(_parent->_mutex); + vector> unread(_samples.begin(), _samples.end()); + _samples.clear(); + return unread; +} + +void +DataReaderI::waitForUnread(unsigned int count) const +{ + unique_lock lock(_parent->_mutex); + _parent->_cond.wait( + lock, + [&]() + { + _parent->getInstance()->checkShutdown(); + return _samples.size() >= count; + }); +} + +bool +DataReaderI::hasUnread() const +{ + unique_lock lock(_parent->_mutex); + return !_samples.empty(); +} + +shared_ptr +DataReaderI::getNextUnread() +{ + unique_lock lock(_parent->_mutex); + _parent->_cond.wait( + lock, + [&]() + { + _parent->getInstance()->checkShutdown(); + return !_samples.empty(); + }); + shared_ptr sample = _samples.front(); + _samples.pop_front(); + return sample; +} + +void +DataReaderI::initSamples( + const vector>& samples, + int64_t topic, + int64_t element, + int priority, + const chrono::time_point& now, + bool checkKey) +{ + if (_traceLevels->data > 1) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": initialized " << samples.size() << " samples from `" << element << '@' << topic << "'"; + } + + vector> valid; + shared_ptr previous = _last; + for (const auto& sample : samples) + { + if (checkKey && !matchKey(sample->key)) + { + continue; + } + else if (_discardPolicy == DataStorm::DiscardPolicy::SendTime && sample->timestamp <= _lastSendTime) + { + continue; + } + else if ( + _discardPolicy == DataStorm::DiscardPolicy::Priority && + priority < _connectedKeys[sample->key].back()->priority) + { + continue; + } + assert(sample->key); + valid.push_back(sample); + + if (!sample->hasValue()) + { + if (sample->event == DataStorm::SampleEvent::PartialUpdate) + { + _parent->getUpdater(sample->tag)(previous, sample, _parent->getInstance()->getCommunicator()); + } + else + { + sample->decode(_parent->getInstance()->getCommunicator()); + } + } + previous = sample; + } + + if (_traceLevels->data > 2 && valid.size() < samples.size()) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": discarded " << samples.size() - valid.size() << " samples from `" << element << '@' << topic + << "'"; + } + + if (_onSamples) + { + _executor->queue( + shared_from_this(), + [this, valid] + { + for (const auto& s : valid) + { + _onSamples(s); + } + }); + } + + if (valid.empty()) + { + return; + } + _lastSendTime = valid.back()->timestamp; + + if (_config->sampleLifetime && *_config->sampleLifetime > 0) + { + cleanOldSamples(_samples, now, *_config->sampleLifetime); + } + + if (_config->sampleCount) + { + if (*_config->sampleCount > 0) + { + size_t count = _samples.size(); + size_t maxCount = static_cast(*_config->sampleCount); + if (count + valid.size() > maxCount) + { + count = count + valid.size() - maxCount; + while (!_samples.empty() && count-- > 0) + { + _samples.pop_front(); + } + assert(_samples.size() + valid.size() == maxCount); + } + } + else if (*_config->sampleCount == 0) + { + return; // Don't keep history + } + } + + if (_config->clearHistory && *_config->clearHistory == ClearHistoryPolicy::OnAll) + { + _samples.clear(); + _samples.push_back(valid.back()); + } + else + { + for (const auto& s : valid) + { + if (_config->clearHistory && + ((s->event == DataStorm::SampleEvent::Add && *_config->clearHistory == ClearHistoryPolicy::OnAdd) || + (s->event == DataStorm::SampleEvent::Remove && + *_config->clearHistory == ClearHistoryPolicy::OnRemove) || + (s->event != DataStorm::SampleEvent::PartialUpdate && + *_config->clearHistory == ClearHistoryPolicy::OnAllExceptPartialUpdate))) + { + _samples.clear(); + } + _samples.push_back(s); + } + } + assert(!_samples.empty()); + _last = _samples.back(); + _parent->_cond.notify_all(); +} + +void +DataReaderI::queue( + const shared_ptr& sample, + int priority, + const shared_ptr&, + const string& facet, + const chrono::time_point& now, + bool checkKey) +{ + if (_config->facet && *_config->facet != facet) + { + if (_traceLevels->data > 2) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": skipped sample " << sample->id << " (facet doesn't match)"; + } + return; + } + else if (checkKey && !matchKey(sample->key)) + { + if (_traceLevels->data > 2) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": skipped sample " << sample->id << " (key doesn't match)"; + } + return; + } + + if (_traceLevels->data > 2) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": queued sample " << sample->id << " listeners=" << _listenerCount; + } + + if ((_discardPolicy == DataStorm::DiscardPolicy::SendTime && sample->timestamp <= _lastSendTime) || + (_discardPolicy == DataStorm::DiscardPolicy::Priority && + priority < _connectedKeys[sample->key].back()->priority)) + { + if (_traceLevels->data > 2) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": discarded sample" << sample->id; + } + return; + } + + if (!sample->hasValue()) + { + if (sample->event == DataStorm::SampleEvent::PartialUpdate) + { + _parent->getUpdater(sample->tag)(_last, sample, _parent->getInstance()->getCommunicator()); + } + else + { + sample->decode(_parent->getInstance()->getCommunicator()); + } + } + _lastSendTime = sample->timestamp; + + if (_onSamples) + { + _executor->queue(shared_from_this(), [this, sample] { _onSamples(sample); }); + } + + if (_config->sampleLifetime && *_config->sampleLifetime > 0) + { + cleanOldSamples(_samples, now, *_config->sampleLifetime); + } + + if (_config->sampleCount) + { + if (*_config->sampleCount > 0) + { + size_t count = _samples.size(); + size_t maxCount = static_cast(*_config->sampleCount); + if (count + 1 > maxCount) + { + if (!_samples.empty()) + { + _samples.pop_front(); + } + assert(_samples.size() + 1 == maxCount); + } + } + else if (*_config->sampleCount == 0) + { + return; // Don't keep history + } + } + + if (_config->clearHistory && + (*_config->clearHistory == ClearHistoryPolicy::OnAll || + (sample->event == DataStorm::SampleEvent::Add && *_config->clearHistory == ClearHistoryPolicy::OnAdd) || + (sample->event == DataStorm::SampleEvent::Remove && *_config->clearHistory == ClearHistoryPolicy::OnRemove) || + (sample->event != DataStorm::SampleEvent::PartialUpdate && + *_config->clearHistory == ClearHistoryPolicy::OnAllExceptPartialUpdate))) + { + _samples.clear(); + } + _samples.push_back(sample); + _last = sample; + _parent->_cond.notify_all(); +} + +void +DataReaderI::onSamples( + function>&)> init, + function&)> update) +{ + unique_lock lock(_parent->_mutex); + _onSamples = std::move(update); + if (init && !_samples.empty()) + { + vector> samples(_samples.begin(), _samples.end()); + _executor->queue(shared_from_this(), [init, samples] { init(samples); }, true); + } +} + +bool +DataReaderI::addConnectedKey(const shared_ptr& key, const shared_ptr& subscriber) +{ + if (DataElementI::addConnectedKey(key, subscriber)) + { + if (_discardPolicy == DataStorm::DiscardPolicy::Priority) + { + auto& subscribers = _connectedKeys[key]; + sort( + subscribers.begin(), + subscribers.end(), + [](const shared_ptr& lhs, const shared_ptr& rhs) + { return lhs->priority < rhs->priority; }); + } + return true; + } + else + { + return false; + } +} + +DataWriterI::DataWriterI(TopicWriterI* topic, const string& name, int64_t id, const DataStorm::WriterConfig& config) + : DataElementI(topic, name, id, config), + _parent(topic) +{ + _config->priority = config.priority; +} + +void +DataWriterI::init() +{ + DataElementI::init(); + _subscribers = Ice::uncheckedCast(_forwarder); +} + +void +DataWriterI::publish(const shared_ptr& key, const shared_ptr& sample) +{ + lock_guard lock(_parent->_mutex); + if (sample->event == DataStorm::SampleEvent::PartialUpdate) + { + assert(!sample->hasValue()); + _parent->getUpdater(sample->tag)(_last, sample, _parent->getInstance()->getCommunicator()); + } + + sample->id = ++_parent->_nextSampleId; + sample->timestamp = chrono::system_clock::now(); + + if (_traceLevels->data > 2) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": publishing sample " << sample->id << " listeners=" << _listenerCount; + } + send(key, sample); + + if (_config->sampleLifetime && *_config->sampleLifetime > 0) + { + cleanOldSamples(_samples, sample->timestamp, *_config->sampleLifetime); + } + + if (_config->sampleCount) + { + if (*_config->sampleCount > 0) + { + if (_samples.size() + 1 > static_cast(*_config->sampleCount)) + { + _samples.pop_front(); + } + } + else if (*_config->sampleCount == 0) + { + return; // Don't keep history + } + } + + if (_config->clearHistory && + (*_config->clearHistory == ClearHistoryPolicy::OnAll || + (sample->event == DataStorm::SampleEvent::Add && *_config->clearHistory == ClearHistoryPolicy::OnAdd) || + (sample->event == DataStorm::SampleEvent::Remove && *_config->clearHistory == ClearHistoryPolicy::OnRemove) || + (sample->event != DataStorm::SampleEvent::PartialUpdate && + *_config->clearHistory == ClearHistoryPolicy::OnAllExceptPartialUpdate))) + { + _samples.clear(); + } + assert(sample->key); + _samples.push_back(sample); + _last = sample; +} + +KeyDataReaderI::KeyDataReaderI( + TopicReaderI* topic, + const string& name, + int64_t id, + const vector>& keys, + const string& sampleFilterName, + const Ice::ByteSeq sampleFilterCriteria, + const DataStorm::ReaderConfig& config) + : DataReaderI(topic, name, id, sampleFilterName, sampleFilterCriteria, config), + _keys(keys) +{ + if (_traceLevels->data > 0) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": created key reader"; + } + + // + // If sample filtering is enabled, ensure the updates are received using a session + // facet specific to this reader. + // + if (_config->sampleFilter) + { + ostringstream os; + os << "fa" << _id; + _config->facet = os.str(); + } +} + +void +KeyDataReaderI::destroyImpl() +{ + if (_traceLevels->data > 0) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": destroyed key reader"; + } + try + { + _forwarder->detachElements(_parent->getId(), {_keys.empty() ? -_id : _id}); + } + catch (const std::exception&) + { + _parent->forwarderException(); + } + _parent->remove(shared_from_this(), _keys); +} + +void +KeyDataReaderI::waitForWriters(int count) +{ + waitForListeners(count); +} + +bool +KeyDataReaderI::hasWriters() +{ + return hasListeners(); +} + +string +KeyDataReaderI::toString() const +{ + ostringstream os; + os << 'e' << _id << ":"; + if (_config->name) + { + os << *_config->name << ":"; + } + os << "["; + for (auto q = _keys.begin(); q != _keys.end(); ++q) + { + if (q != _keys.begin()) + { + os << ","; + } + os << (*q)->toString(); + } + os << "]@" << _parent; + return os.str(); +} + +bool +KeyDataReaderI::matchKey(const shared_ptr& key) const +{ + return _keys.empty() || find(_keys.begin(), _keys.end(), key) != _keys.end(); +} + +KeyDataWriterI::KeyDataWriterI( + TopicWriterI* topic, + const string& name, + int64_t id, + const vector>& keys, + const DataStorm::WriterConfig& config) + : DataWriterI(topic, name, id, config), + _keys(keys) +{ + if (_traceLevels->data > 0) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": created key writer"; + } +} + +void +KeyDataWriterI::destroyImpl() +{ + if (_traceLevels->data > 0) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": destroyed key writer"; + } + try + { + _forwarder->detachElements(_parent->getId(), {_keys.empty() ? -_id : _id}); + } + catch (const std::exception&) + { + _parent->forwarderException(); + } + _parent->remove(shared_from_this(), _keys); +} + +void +KeyDataWriterI::waitForReaders(int count) const +{ + waitForListeners(count); +} + +bool +KeyDataWriterI::hasReaders() const +{ + return hasListeners(); +} + +shared_ptr +KeyDataWriterI::getLast() const +{ + unique_lock lock(_parent->_mutex); + return _samples.empty() ? nullptr : _samples.back(); +} + +vector> +KeyDataWriterI::getAll() const +{ + unique_lock lock(_parent->_mutex); + vector> all(_samples.begin(), _samples.end()); + return all; +} + +string +KeyDataWriterI::toString() const +{ + ostringstream os; + os << 'e' << _id << ":"; + if (_config->name) + { + os << *_config->name << ":"; + } + os << "["; + for (auto q = _keys.begin(); q != _keys.end(); ++q) + { + if (q != _keys.begin()) + { + os << ","; + } + os << (*q)->toString(); + } + os << "]@" << _parent; + return os.str(); +} + +DataSamples +KeyDataWriterI::getSamples( + const shared_ptr& key, + const shared_ptr& sampleFilter, + const shared_ptr& config, + int64_t lastId, + const chrono::time_point& now) +{ + DataSamples samples; + samples.id = _keys.empty() ? -_id : _id; + + if (config->sampleCount && *config->sampleCount == 0) + { + return samples; + } + + if (_config->sampleLifetime && *_config->sampleLifetime > 0) + { + cleanOldSamples(_samples, now, *_config->sampleLifetime); + } + + chrono::time_point staleTime = chrono::time_point::min(); + if (config->sampleLifetime && *config->sampleLifetime > 0) + { + staleTime = now - chrono::milliseconds(*config->sampleLifetime); + } + + shared_ptr first; + for (auto p = _samples.rbegin(); p != _samples.rend(); ++p) + { + if ((*p)->timestamp < staleTime) + { + break; + } + if ((*p)->id <= lastId) + { + break; + } + + if ((!key || key == (*p)->key) && (!sampleFilter || sampleFilter->match(*p))) + { + first = *p; + samples.samples.push_front(toSample(*p, getCommunicator(), _keys.empty())); + if (config->sampleCount && *config->sampleCount > 0 && + static_cast(*config->sampleCount) == samples.samples.size()) + { + break; + } + + if (config->clearHistory && + (*config->clearHistory == ClearHistoryPolicy::OnAll || + ((*p)->event == DataStorm::SampleEvent::Add && *config->clearHistory == ClearHistoryPolicy::OnAdd) || + ((*p)->event == DataStorm::SampleEvent::Remove && + *config->clearHistory == ClearHistoryPolicy::OnRemove) || + ((*p)->event != DataStorm::SampleEvent::PartialUpdate && + *config->clearHistory == ClearHistoryPolicy::OnAllExceptPartialUpdate))) + { + break; + } + } + } + if (!samples.samples.empty()) + { + // If the first sample is a partial update, transform it to a full Update + if (first->event == DataStorm::SampleEvent::PartialUpdate) + { + samples.samples[0] = { + first->id, + samples.samples[0].keyId, + samples.samples[0].keyValue, + chrono::time_point_cast(first->timestamp).time_since_epoch().count(), + 0, + DataStorm::SampleEvent::Update, + first->encodeValue(getCommunicator())}; + } + } + return samples; +} + +void +KeyDataWriterI::send(const shared_ptr& key, const shared_ptr& sample) const +{ + assert(key || _keys.size() == 1); + _sample = sample; + _sample->key = key ? key : _keys[0]; + _subscribers->s(_parent->getId(), _keys.empty() ? -_id : _id, toSample(sample, getCommunicator(), _keys.empty())); + _sample = nullptr; +} + +void +KeyDataWriterI::forward(const Ice::ByteSeq& inEncaps, const Ice::Current& current) const +{ + for (const auto& listener : _listeners) + { + // If there's at least one subscriber interested in the update (check the key if any writer) + if (!_sample || listener.second.matchOne(_sample, _keys.empty())) + { + // TODO do we need to check the result? + auto _ = listener.second.proxy->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } + } +} + +FilteredDataReaderI::FilteredDataReaderI( + TopicReaderI* topic, + const string& name, + int64_t id, + const shared_ptr& filter, + const string& sampleFilterName, + Ice::ByteSeq sampleFilterCriteria, + const DataStorm::ReaderConfig& config) + : DataReaderI(topic, name, id, sampleFilterName, sampleFilterCriteria, config), + _filter(filter) +{ + if (_traceLevels->data > 0) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": created filtered reader"; + } + + // + // If sample filtering is enabled, ensure the updates are received using a session + // facet specific to this reader. + // + if (_config->sampleFilter) + { + ostringstream os; + os << "fa" << _id; + _config->facet = os.str(); + } +} + +void +FilteredDataReaderI::destroyImpl() +{ + if (_traceLevels->data > 0) + { + Trace out(_traceLevels, _traceLevels->dataCat); + out << this << ": destroyed filter reader"; + } + try + { + _forwarder->detachElements(_parent->getId(), {-_id}); + } + catch (const std::exception&) + { + _parent->forwarderException(); + } + _parent->removeFiltered(shared_from_this(), _filter); +} + +void +FilteredDataReaderI::waitForWriters(int count) +{ + waitForListeners(count); +} + +bool +FilteredDataReaderI::hasWriters() +{ + return hasListeners(); +} + +string +FilteredDataReaderI::toString() const +{ + ostringstream os; + os << 'e' << _id << ':'; + if (_config->name) + { + os << *_config->name << ":"; + } + os << "[" << _filter->toString() << "]@" << _parent; + return os.str(); +} + +bool +FilteredDataReaderI::matchKey(const shared_ptr& key) const +{ + return _filter->match(key); +} diff --git a/cpp/src/DataStorm/DataElementI.h b/cpp/src/DataStorm/DataElementI.h new file mode 100644 index 00000000000..10e2e64bc13 --- /dev/null +++ b/cpp/src/DataStorm/DataElementI.h @@ -0,0 +1,429 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_DATA_ELEMENTI_H +#define DATASTORM_DATA_ELEMENTI_H + +#include "DataStorm/Contract.h" +#include "DataStorm/InternalI.h" +#include "ForwarderManager.h" + +namespace DataStormI +{ + + class SessionI; + class TopicI; + class TopicReaderI; + class TopicWriterI; + class CallbackExecutor; + class TraceLevels; + + class DataElementI : virtual public DataElement, public std::enable_shared_from_this + { + protected: + struct Subscriber + { + Subscriber( + std::int64_t id, + const std::shared_ptr& filter, + const std::shared_ptr& sampleFilter, + const std::string& name, + int priority) + : id(id), + filter(filter), + sampleFilter(sampleFilter), + name(name), + priority(priority) + { + } + + std::int64_t topicId; + std::int64_t elementId; + std::int64_t id; + std::set> keys; + std::shared_ptr filter; + std::shared_ptr sampleFilter; + std::string name; + int priority; + }; + + private: + struct ListenerKey + { + std::shared_ptr session; + std::string facet; + + bool operator<(const ListenerKey& other) const + { + if (session < other.session) + { + return true; + } + else if (other.session < session) + { + return false; + } + return facet < other.facet; + } + }; + + struct Listener + { + Listener(std::optional proxy, const std::string& facet) + : proxy(facet.empty() ? proxy : proxy->ice_facet(facet)) + { + } + + bool matchOne(const std::shared_ptr& sample, bool matchKey) const + { + for (const auto& s : subscribers) + { + if ((!matchKey || s.second->keys.empty() || + s.second->keys.find(sample->key) != s.second->keys.end()) && + (!s.second->filter || s.second->filter->match(sample->key)) && + (!s.second->sampleFilter || s.second->sampleFilter->match(sample))) + { + return true; + } + } + return false; + } + + std::shared_ptr addOrGet( + std::int64_t topicId, + std::int64_t elementId, + std::int64_t id, + const std::shared_ptr& filter, + const std::shared_ptr& sampleFilter, + const std::string& name, + int priority, + bool& added) + { + auto k = std::make_pair(topicId, elementId); + auto p = subscribers.find(k); + if (p == subscribers.end()) + { + added = true; + p = subscribers.emplace(k, std::make_shared(id, filter, sampleFilter, name, priority)) + .first; + } + return p->second; + } + + std::shared_ptr get(std::int64_t topicId, std::int64_t elementId) + { + return subscribers.find(std::make_pair(topicId, elementId))->second; + } + + bool remove(std::int64_t topicId, std::int64_t elementId) + { + subscribers.erase(std::make_pair(topicId, elementId)); + return subscribers.empty(); + } + + std::optional proxy; + std::map, std::shared_ptr> subscribers; + }; + + public: + DataElementI(TopicI*, const std::string&, std::int64_t, const DataStorm::Config&); + virtual ~DataElementI(); + + void init(); + + virtual void destroy() override; + + void attach( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + std::optional, + const DataStormContract::ElementData&, + const std::chrono::time_point&, + DataStormContract::ElementDataAckSeq&); + + std::function attach( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + std::optional, + const DataStormContract::ElementDataAck&, + const std::chrono::time_point&, + DataStormContract::DataSamplesSeq&); + + bool attachKey( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + std::optional, + const std::string&, + std::int64_t, + const std::string&, + int); + + void detachKey( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::shared_ptr&, + const std::string&, + bool); + + bool attachFilter( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + std::optional, + const std::string&, + std::int64_t, + const std::shared_ptr&, + const std::string&, + int); + + void detachFilter( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::shared_ptr&, + const std::string&, + bool); + + virtual std::vector> getConnectedKeys() const override; + virtual std::vector getConnectedElements() const override; + virtual void onConnectedKeys( + std::function>)>, + std::function)>) override; + virtual void onConnectedElements( + std::function)>, + std::function) override; + + virtual void initSamples( + const std::vector>&, + std::int64_t, + std::int64_t, + int, + const std::chrono::time_point&, + bool); + virtual DataStormContract::DataSamples getSamples( + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + std::int64_t, + const std::chrono::time_point&); + + virtual void queue( + const std::shared_ptr&, + int, + const std::shared_ptr&, + const std::string&, + const std::chrono::time_point&, + bool); + + virtual std::string toString() const = 0; + virtual Ice::CommunicatorPtr getCommunicator() const override; + + std::int64_t getId() const { return _id; } + + std::shared_ptr getConfig() const; + + void waitForListeners(int count) const; + bool hasListeners() const; + + TopicI* getTopic() const { return _parent.get(); } + + protected: + virtual bool addConnectedKey(const std::shared_ptr&, const std::shared_ptr&); + virtual bool removeConnectedKey(const std::shared_ptr&, const std::shared_ptr&); + + void notifyListenerWaiters(std::unique_lock&) const; + void disconnect(); + virtual void destroyImpl() = 0; + + const std::shared_ptr _traceLevels; + const std::string _name; + const std::int64_t _id; + const std::shared_ptr _config; + const std::shared_ptr _executor; + + size_t _listenerCount; + mutable std::shared_ptr _sample; + std::optional _forwarder; + std::map, std::vector>> _connectedKeys; + std::map _listeners; + + private: + virtual void forward(const Ice::ByteSeq&, const Ice::Current&) const; + + const std::shared_ptr _parent; + mutable size_t _waiters; + mutable size_t _notified; + bool _destroyed; + + std::function)> _onConnectedKeys; + std::function _onConnectedElements; + }; + + class DataReaderI : public DataElementI, public DataReader + { + public: + DataReaderI( + TopicReaderI*, + const std::string&, + std::int64_t, + const std::string&, + Ice::ByteSeq, + const DataStorm::ReaderConfig&); + + virtual int getInstanceCount() const override; + + virtual std::vector> getAllUnread() override; + virtual void waitForUnread(unsigned int) const override; + virtual bool hasUnread() const override; + virtual std::shared_ptr getNextUnread() override; + + virtual void initSamples( + const std::vector>&, + std::int64_t, + std::int64_t, + int, + const std::chrono::time_point&, + bool) override; + virtual void queue( + const std::shared_ptr&, + int, + const std::shared_ptr&, + const std::string&, + const std::chrono::time_point&, + bool) override; + + virtual void onSamples( + std::function>&)>, + std::function&)>) override; + + protected: + virtual bool matchKey(const std::shared_ptr&) const = 0; + virtual bool addConnectedKey(const std::shared_ptr&, const std::shared_ptr&) override; + + TopicReaderI* _parent; + + std::deque> _samples; + std::shared_ptr _last; + int _instanceCount; + DataStorm::DiscardPolicy _discardPolicy; + std::chrono::time_point _lastSendTime; + std::function&)> _onSamples; + }; + + class DataWriterI : public DataElementI, public DataWriter + { + public: + DataWriterI(TopicWriterI*, const std::string&, std::int64_t, const DataStorm::WriterConfig&); + void init(); + + virtual void publish(const std::shared_ptr&, const std::shared_ptr&) override; + + protected: + virtual void send(const std::shared_ptr&, const std::shared_ptr&) const = 0; + + TopicWriterI* _parent; + // TODO it should be probably a non-nullable proxy + std::optional _subscribers; + std::deque> _samples; + std::shared_ptr _last; + }; + + class KeyDataReaderI : public DataReaderI + { + public: + KeyDataReaderI( + TopicReaderI*, + const std::string&, + std::int64_t, + const std::vector>&, + const std::string&, + Ice::ByteSeq, + const DataStorm::ReaderConfig&); + + virtual void destroyImpl() override; + + virtual void waitForWriters(int) override; + virtual bool hasWriters() override; + + virtual std::string toString() const override; + + private: + virtual bool matchKey(const std::shared_ptr&) const override; + + const std::vector> _keys; + }; + + class KeyDataWriterI : public DataWriterI + { + public: + KeyDataWriterI( + TopicWriterI*, + const std::string&, + std::int64_t, + const std::vector>&, + const DataStorm::WriterConfig&); + + virtual void destroyImpl() override; + + virtual void waitForReaders(int) const override; + virtual bool hasReaders() const override; + + virtual std::shared_ptr getLast() const override; + virtual std::vector> getAll() const override; + + virtual std::string toString() const override; + virtual DataStormContract::DataSamples getSamples( + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + std::int64_t, + const std::chrono::time_point&) override; + + private: + virtual void send(const std::shared_ptr&, const std::shared_ptr&) const override; + virtual void forward(const Ice::ByteSeq&, const Ice::Current&) const override; + + const std::vector> _keys; + }; + + class FilteredDataReaderI : public DataReaderI + { + public: + FilteredDataReaderI( + TopicReaderI*, + const std::string&, + std::int64_t, + const std::shared_ptr&, + const std::string&, + Ice::ByteSeq, + const DataStorm::ReaderConfig&); + + virtual void destroyImpl() override; + + virtual void waitForWriters(int) override; + virtual bool hasWriters() override; + + virtual std::string toString() const override; + + private: + virtual bool matchKey(const std::shared_ptr&) const override; + + const std::shared_ptr _filter; + }; + +} + +#endif diff --git a/cpp/src/DataStorm/DataStorm.rc b/cpp/src/DataStorm/DataStorm.rc new file mode 100644 index 00000000000..bfcbac26703 --- /dev/null +++ b/cpp/src/DataStorm/DataStorm.rc @@ -0,0 +1,33 @@ +#include "../Ice/ResourceConfig.h" + +#define ICE_INTERNALNAME ICE_LIBNAME("datastorm") "\0" +#define ICE_ORIGINALFILENAME ICE_LIBNAME("datastorm") ".dll\0" + +VS_VERSION_INFO VERSIONINFO +FILEVERSION ICE_VERSION +PRODUCTVERSION ICE_VERSION +FILEFLAGSMASK VS_FFI_FILEFLAGSMASK +FILEOS VOS_NT_WINDOWS32 +FILETYPE VFT_DLL +FILESUBTYPE VFT2_UNKNOWN +FILEFLAGS VER_DEBUG +BEGIN + BLOCK "StringFileInfo" + BEGIN + BLOCK "040904e4" + BEGIN + VALUE "CompanyName", ICE_COMPANY_NAME + VALUE "FileDescription", "DataStorm C++ Library\0" + VALUE "FileVersion", ICE_STRING_VERSION + VALUE "InternalName", ICE_INTERNALNAME + VALUE "LegalCopyright", ICE_COPYRIGHT + VALUE "OriginalFilename", ICE_ORIGINALFILENAME + VALUE "ProductName", ICE_PRODUCT_NAME + VALUE "ProductVersion", ICE_STRING_VERSION + END + END + BLOCK "VarFileInfo" + BEGIN + VALUE "Translation", 0x409, 1252 + END +END diff --git a/cpp/src/DataStorm/ForwarderManager.cpp b/cpp/src/DataStorm/ForwarderManager.cpp new file mode 100644 index 00000000000..f3b6298dc15 --- /dev/null +++ b/cpp/src/DataStorm/ForwarderManager.cpp @@ -0,0 +1,88 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "ForwarderManager.h" + +using namespace std; +using namespace DataStormI; + +ForwarderManager::ForwarderManager(const Ice::ObjectAdapterPtr& adapter, const string& category) + : _adapter(adapter), + _category(category), + _nextId(0) +{ +} + +optional +ForwarderManager::add(function forwarder) +{ + lock_guard lock(_mutex); + ostringstream os; + os << _nextId++; + _forwarders.emplace(os.str(), std::move(forwarder)); + try + { + return _adapter->createProxy({os.str(), _category}); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + return nullopt; + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + return nullopt; + } +} + +optional +ForwarderManager::add(function forwarder) +{ + return add( + [forwarder = std::move(forwarder)](auto inEncaps, auto response, auto exception, auto current) + { + try + { + forwarder(inEncaps, current); + response(true, Ice::ByteSeq()); + } + catch (...) + { + exception(current_exception()); + } + }); +} + +void +ForwarderManager::remove(const Ice::Identity& id) +{ + lock_guard lock(_mutex); + _forwarders.erase(id.name); +} + +void +ForwarderManager::destroy() +{ + lock_guard lock(_mutex); + _forwarders.clear(); +} + +void +ForwarderManager::ice_invokeAsync( + Ice::ByteSeq inEncaps, + function response, + function exception, + const Ice::Current& current) +{ + std::function forwarder; + { + lock_guard lock(_mutex); + auto p = _forwarders.find(current.id.name); + if (p == _forwarders.end()) + { + throw Ice::ObjectNotExistException(__FILE__, __LINE__, current.id, current.facet, current.operation); + } + forwarder = p->second; + } + forwarder(std::move(inEncaps), std::move(response), std::move(exception), current); +} diff --git a/cpp/src/DataStorm/ForwarderManager.h b/cpp/src/DataStorm/ForwarderManager.h new file mode 100644 index 00000000000..6faab81cb44 --- /dev/null +++ b/cpp/src/DataStorm/ForwarderManager.h @@ -0,0 +1,49 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_FORWARDER_MANAGER_H +#define DATASTORM_FORWARDER_MANAGER_H + +#include "DataStorm/Config.h" +#include "Ice/Ice.h" + +#include +#include + +namespace DataStormI +{ + + class Instance; + + class ForwarderManager : public Ice::BlobjectAsync + { + public: + using Response = std::function; + using Exception = std::function; + + ForwarderManager(const Ice::ObjectAdapterPtr&, const std::string&); + + std::optional add(std::function); + std::optional add(std::function); + void remove(const Ice::Identity&); + + void destroy(); + + private: + virtual void ice_invokeAsync( + Ice::ByteSeq, + std::function, + std::function, + const Ice::Current&); + + const Ice::ObjectAdapterPtr _adapter; + const std::string _category; + + std::mutex _mutex; + std::map> _forwarders; + unsigned int _nextId; + }; + +} +#endif diff --git a/cpp/src/DataStorm/Instance.cpp b/cpp/src/DataStorm/Instance.cpp new file mode 100644 index 00000000000..15014bb9f81 --- /dev/null +++ b/cpp/src/DataStorm/Instance.cpp @@ -0,0 +1,193 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "Instance.h" +#include "CallbackExecutor.h" +#include "ConnectionManager.h" +#include "DataStorm/Node.h" +#include "LookupI.h" +#include "NodeI.h" +#include "NodeSessionManager.h" +#include "TopicFactoryI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; + +Instance::Instance(const Ice::CommunicatorPtr& communicator) : _communicator(communicator), _shutdown(false) +{ + Ice::PropertiesPtr properties = _communicator->getProperties(); + + if (properties->getPropertyAsIntWithDefault("DataStorm.Node.Server.Enabled", 1) > 0) + { + properties->setProperty("DataStorm.Node.Adapters.Server.ThreadPool.SizeMax", "1"); + properties->setProperty("DataStorm.Node.Adapters.Server.Endpoints", "tcp"); + + const string pfx = "DataStorm.Node.Server"; + auto props = properties->getPropertiesForPrefix(pfx); + for (const auto& p : props) + { + if (p.first != "DataStorm.Node.Server.Enabled" && + p.first != "DataStorm.Node.Server.ForwardDiscoveryToMulticast") + { + properties->setProperty("DataStorm.Node.Adapters.Server" + p.first.substr(pfx.length()), p.second); + } + } + + try + { + _adapter = _communicator->createObjectAdapter("DataStorm.Node.Adapters.Server"); + } + catch (const Ice::LocalException& ex) + { + ostringstream os; + os << "failed to listen on server endpoints `"; + os << properties->getProperty("DataStorm.Node.Adapters.Server.Endpoints") << "':\n"; + os << ex.what(); + throw invalid_argument(os.str()); + } + } + else + { + _adapter = _communicator->createObjectAdapter(""); + } + + if (properties->getPropertyAsIntWithDefault("DataStorm.Node.Multicast.Enabled", 1) > 0) + { + properties->setProperty("DataStorm.Node.Adapters.Multicast.Endpoints", "udp -h 239.255.0.1 -p 10000"); + properties->setProperty("DataStorm.Node.Adapters.Multicast.ProxyOptions", "-d"); + properties->setProperty("DataStorm.Node.Adapters.Multicast.ThreadPool.SizeMax", "1"); + + const string pfx = "DataStorm.Node.Multicast"; + auto props = properties->getPropertiesForPrefix(pfx); + for (const auto& p : props) + { + if (p.first != "DataStorm.Node.Multicast.Enabled") + { + properties->setProperty("DataStorm.Node.Adapters.Multicast" + p.first.substr(pfx.length()), p.second); + } + } + + try + { + _multicastAdapter = _communicator->createObjectAdapter("DataStorm.Node.Adapters.Multicast"); + } + catch (const Ice::LocalException& ex) + { + ostringstream os; + os << "failed to listen on multicast endpoints `"; + os << properties->getProperty("DataStorm.Node.Adapters.Server.Endpoints") << "':\n"; + os << ex.what(); + throw invalid_argument(os.str()); + } + } + + _retryDelay = chrono::milliseconds(properties->getPropertyAsIntWithDefault("DataStorm.Node.RetryDelay", 500)); + _retryMultiplier = properties->getPropertyAsIntWithDefault("DataStorm.Node.RetryMultiplier", 2); + _retryCount = properties->getPropertyAsIntWithDefault("DataStorm.Node.RetryCount", 6); + + // + // Create a collocated object adapter with a random name to prevent user configuration + // of the adapter. + // + auto collocated = Ice::generateUUID(); + properties->setProperty(collocated + ".AdapterId", collocated); + _collocatedAdapter = _communicator->createObjectAdapter(collocated); + + _collocatedForwarder = make_shared(_collocatedAdapter, "forwarders"); + _collocatedAdapter->addDefaultServant(_collocatedForwarder, "forwarders"); + + _executor = make_shared(); + _connectionManager = make_shared(_executor); + _timer = make_shared(); + _traceLevels = make_shared(_communicator); +} + +void +Instance::init() +{ + auto self = shared_from_this(); + + _topicFactory = make_shared(self); + + _node = make_shared(self); + _node->init(); + + _nodeSessionManager = make_shared(self, _node); + _nodeSessionManager->init(); + + auto lookupI = make_shared(_nodeSessionManager, _topicFactory, _node->getProxy()); + _adapter->add(lookupI, {"Lookup", "DataStorm"}); + if (_multicastAdapter) + { + auto lookup = _multicastAdapter->add(lookupI, {"Lookup", "DataStorm"}); + _lookup = lookup->ice_collocationOptimized(false); + } + + _adapter->activate(); + _collocatedAdapter->activate(); + if (_multicastAdapter) + { + _multicastAdapter->activate(); + } +} + +void +Instance::shutdown() +{ + unique_lock lock(_mutex); + _shutdown = true; + _cond.notify_all(); + _topicFactory->shutdown(); +} + +bool +Instance::isShutdown() const +{ + unique_lock lock(_mutex); + return _shutdown; +} + +void +Instance::checkShutdown() const +{ + unique_lock lock(_mutex); + if (_shutdown) + { + throw DataStorm::NodeShutdownException(); + } +} + +void +Instance::waitForShutdown() const +{ + unique_lock lock(_mutex); + _cond.wait(lock, [&]() { return _shutdown; }); // Wait until shutdown is called +} + +void +Instance::destroy(bool ownsCommunicator) +{ + if (ownsCommunicator) + { + _communicator->destroy(); + } + else + { + _adapter->destroy(); + _collocatedAdapter->destroy(); + if (_multicastAdapter) + { + _multicastAdapter->destroy(); + } + } + _node->destroy(ownsCommunicator); + + _executor->destroy(); + _connectionManager->destroy(); + _collocatedForwarder->destroy(); + // Destroy the session manager before the timer to avoid scheduling new tasks after the timer has been destroyed. + _nodeSessionManager->destroy(); + _timer->destroy(); +} diff --git a/cpp/src/DataStorm/Instance.h b/cpp/src/DataStorm/Instance.h new file mode 100644 index 00000000000..1a99e47b637 --- /dev/null +++ b/cpp/src/DataStorm/Instance.h @@ -0,0 +1,140 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_INSTANCE_H +#define DATASTORM_INSTANCE_H + +#include "DataStorm/Config.h" +#include "DataStorm/Contract.h" +#include "Ice/Ice.h" + +#include +#include + +namespace DataStorm +{ + + class TopicFactory; + +} + +namespace DataStormI +{ + + class TopicFactoryI; + class ConnectionManager; + class NodeSessionManager; + class TraceLevels; + class ForwarderManager; + class NodeI; + class CallbackExecutor; + + class Instance : public std::enable_shared_from_this + { + public: + Instance(const Ice::CommunicatorPtr&); + + void init(); + + std::shared_ptr getConnectionManager() const + { + assert(_connectionManager); + return _connectionManager; + } + + std::shared_ptr getNodeSessionManager() const + { + assert(_nodeSessionManager); + return _nodeSessionManager; + } + + Ice::CommunicatorPtr getCommunicator() const + { + assert(_communicator); + return _communicator; + } + + Ice::ObjectAdapterPtr getObjectAdapter() const + { + assert(_adapter); + return _adapter; + } + + std::shared_ptr getCollocatedForwarder() const + { + assert(_collocatedForwarder); + return _collocatedForwarder; + } + + std::optional getLookup() const { return _lookup; } + + std::shared_ptr getTopicFactory() const + { + assert(_topicFactory); + return _topicFactory; + } + + std::shared_ptr getTraceLevels() const + { + assert(_traceLevels); + return _traceLevels; + } + + std::shared_ptr getNode() const + { + assert(_node); + return _node; + } + + std::shared_ptr getCallbackExecutor() const + { + assert(_executor); + return _executor; + } + + Ice::TimerPtr getTimer() const + { + assert(_timer); + return _timer; + } + + std::chrono::milliseconds getRetryDelay(int count) const + { + return _retryDelay * static_cast(std::pow(_retryMultiplier, std::min(count, _retryCount))); + } + + int getRetryCount() const { return _retryCount; } + + void shutdown(); + bool isShutdown() const; + void checkShutdown() const; + void waitForShutdown() const; + + void destroy(bool); + + private: + std::shared_ptr _topicFactory; + std::shared_ptr _connectionManager; + std::shared_ptr _nodeSessionManager; + std::shared_ptr _collocatedForwarder; + std::shared_ptr _node; + Ice::CommunicatorPtr _communicator; + Ice::ObjectAdapterPtr _adapter; + Ice::ObjectAdapterPtr _collocatedAdapter; + Ice::ObjectAdapterPtr _multicastAdapter; + std::optional _lookup; + std::shared_ptr _traceLevels; + std::shared_ptr _executor; + Ice::TimerPtr _timer; + std::chrono::milliseconds _retryDelay; + int _retryMultiplier; + int _retryCount; + + mutable std::mutex _mutex; + mutable std::condition_variable _cond; + bool _shutdown; + }; + +} +#endif diff --git a/cpp/src/DataStorm/LookupI.cpp b/cpp/src/DataStorm/LookupI.cpp new file mode 100644 index 00000000000..0ce7c3fee47 --- /dev/null +++ b/cpp/src/DataStorm/LookupI.cpp @@ -0,0 +1,74 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "LookupI.h" +#include "Instance.h" +#include "NodeI.h" +#include "NodeSessionManager.h" +#include "TopicFactoryI.h" + +using namespace std; +using namespace DataStormContract; +using namespace DataStormI; + +LookupI::LookupI( + shared_ptr nodeSessionManager, + shared_ptr topicFactory, + optional nodePrx) + : _nodeSessionManager(std::move(nodeSessionManager)), + _topicFactory(std::move(topicFactory)), + _nodePrx(std::move(nodePrx)) +{ +} + +void +LookupI::announceTopicReader(string name, optional proxy, const Ice::Current& current) +{ + if (proxy) + { + _nodeSessionManager->announceTopicReader(name, proxy, current.con); + _topicFactory->createSubscriberSession(name, proxy, current.con); + } +} + +void +LookupI::announceTopicWriter(string name, optional proxy, const Ice::Current& current) +{ + if (proxy) + { + _nodeSessionManager->announceTopicWriter(name, proxy, current.con); + _topicFactory->createPublisherSession(name, proxy, current.con); + } +} + +void +LookupI::announceTopics(StringSeq readers, StringSeq writers, optional proxy, const Ice::Current& current) +{ + if (proxy) + { + _nodeSessionManager->announceTopics(readers, writers, proxy, current.con); + for (const auto& name : readers) + { + _topicFactory->createSubscriberSession(name, proxy, current.con); + } + for (const auto& name : writers) + { + _topicFactory->createPublisherSession(name, proxy, current.con); + } + } +} + +optional +LookupI::createSession(optional node, const Ice::Current& current) +{ + if (node) + { + _nodeSessionManager->createOrGet(std::move(node), current.con, true); + return _nodePrx; + } + else + { + return nullopt; + } +} diff --git a/cpp/src/DataStorm/LookupI.h b/cpp/src/DataStorm/LookupI.h new file mode 100644 index 00000000000..730adf35134 --- /dev/null +++ b/cpp/src/DataStorm/LookupI.h @@ -0,0 +1,46 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_LOOKUPI_H +#define DATASTORM_LOOKUPI_H + +#include "DataStorm/Contract.h" + +namespace DataStormI +{ + + class NodeSessionManager; + class TopicFactoryI; + + class LookupI : public DataStormContract::Lookup + { + public: + LookupI( + std::shared_ptr, + std::shared_ptr, + std::optional); + + virtual void + announceTopicReader(std::string, std::optional, const Ice::Current&) override; + + virtual void + announceTopicWriter(std::string, std::optional, const Ice::Current&) override; + + virtual void announceTopics( + DataStormContract::StringSeq, + DataStormContract::StringSeq, + std::optional, + const Ice::Current&) override; + + virtual std::optional + createSession(std::optional, const Ice::Current&) override; + + private: + std::shared_ptr _nodeSessionManager; + std::shared_ptr _topicFactory; + std::optional _nodePrx; + }; + +} +#endif diff --git a/cpp/src/DataStorm/Makefile.mk b/cpp/src/DataStorm/Makefile.mk new file mode 100644 index 00000000000..befac610d83 --- /dev/null +++ b/cpp/src/DataStorm/Makefile.mk @@ -0,0 +1,16 @@ +# ********************************************************************** +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# +# ********************************************************************** + +$(project)_libraries = DataStorm + +$(project)_generated_includedir := $(project)/generated/DataStorm + +DataStorm_sliceflags := --include-dir DataStorm +DataStorm_targetdir := $(libdir) +DataStorm_cppflags := -DDATASTORM_API_EXPORTS +DataStorm_dependencies := Ice + +projects += $(project) diff --git a/cpp/src/DataStorm/Node.cpp b/cpp/src/DataStorm/Node.cpp new file mode 100644 index 00000000000..38f9ae2d260 --- /dev/null +++ b/cpp/src/DataStorm/Node.cpp @@ -0,0 +1,93 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/Node.h" +#include "Ice/Ice.h" +#include "Instance.h" +#include "NodeI.h" +#include "TopicFactoryI.h" + +using namespace std; +using namespace DataStorm; + +const char* +NodeShutdownException::what() const noexcept +{ + return "::DataStorm::NodeShutdownException"; +} + +Node::Node(Ice::CommunicatorPtr communicator) : _ownsCommunicator(false) { init(communicator); } + +void +Node::init(const Ice::CommunicatorPtr& communicator) +{ + try + { + _instance = make_shared(communicator); + _instance->init(); + } + catch (...) + { + if (_ownsCommunicator) + { + communicator->destroy(); + } + throw; + } + _factory = _instance->getTopicFactory(); +} + +Node::Node(Node&& node) noexcept +{ + _instance = std::move(node._instance); + _factory = std::move(node._factory); + _ownsCommunicator = node._ownsCommunicator; +} + +Node::~Node() +{ + if (_instance) + { + _instance->destroy(_ownsCommunicator); + } +} + +void +Node::shutdown() noexcept +{ + _instance->shutdown(); +} + +bool +Node::isShutdown() const noexcept +{ + return _instance->isShutdown(); +} + +void +Node::waitForShutdown() const noexcept +{ + _instance->waitForShutdown(); +} + +Node& +Node::operator=(Node&& node) noexcept +{ + _instance = std::move(node._instance); + _factory = std::move(node._factory); + _ownsCommunicator = node._ownsCommunicator; + return *this; +} + +Ice::CommunicatorPtr +Node::getCommunicator() const noexcept +{ + return _instance ? _instance->getCommunicator() : nullptr; +} + +Ice::ConnectionPtr +Node::getSessionConnection(const string& ident) const noexcept +{ + return _instance ? _instance->getNode()->getSessionConnection(ident) : nullptr; +} diff --git a/cpp/src/DataStorm/NodeI.cpp b/cpp/src/DataStorm/NodeI.cpp new file mode 100644 index 00000000000..ead2defe6c6 --- /dev/null +++ b/cpp/src/DataStorm/NodeI.cpp @@ -0,0 +1,607 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "NodeI.h" +#include "CallbackExecutor.h" +#include "Instance.h" +#include "NodeSessionI.h" +#include "NodeSessionManager.h" +#include "SessionI.h" +#include "TopicFactoryI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; +using namespace DataStormContract; + +namespace +{ + + // TODO convert to a middleware + class DispatchInterceptorI : public Ice::Object + { + public: + DispatchInterceptorI(shared_ptr node, shared_ptr executor) + : _node(std::move(node)), + _executor(std::move(executor)) + { + } + + void dispatch(Ice::IncomingRequest& request, std::function sendResponse) final + { + auto session = _node->getSession(request.current().id); + if (!session) + { + throw Ice::ObjectNotExistException(__FILE__, __LINE__); + } + session->dispatch(request, std::move(sendResponse)); + _executor->flush(); + } + + private: + shared_ptr _node; + shared_ptr _executor; + }; + +} + +NodeI::NodeI(const shared_ptr& instance) + : _instance(instance), + _nextSubscriberSessionId(0), + _nextPublisherSessionId(0) +{ +} + +NodeI::~NodeI() +{ + assert(_subscribers.empty()); + assert(_publishers.empty()); + assert(_subscriberSessions.empty()); + assert(_publisherSessions.empty()); +} + +void +NodeI::init() +{ + auto self = shared_from_this(); + auto instance = getInstance(); + auto forwarder = [self = shared_from_this()](Ice::ByteSeq e, const Ice::Current& c) { self->forward(e, c); }; + _subscriberForwarder = Ice::uncheckedCast(instance->getCollocatedForwarder()->add(forwarder)); + _publisherForwarder = Ice::uncheckedCast(instance->getCollocatedForwarder()->add(forwarder)); + try + { + auto adapter = instance->getObjectAdapter(); + _proxy = adapter->addWithUUID(self); + + auto interceptor = make_shared(self, instance->getCallbackExecutor()); + adapter->addDefaultServant(interceptor, "s"); + adapter->addDefaultServant(interceptor, "p"); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } +} + +void +NodeI::destroy(bool ownsCommunicator) +{ + unique_lock lock(_mutex); + if (!ownsCommunicator) + { + // + // Notifies peer sessions of the disconnection. + // + for (const auto& p : _subscribers) + { + auto s = p.second->getSession(); + if (s) + { + try + { + // TODO check the return value? + auto _ = s->disconnectedAsync(); + } + catch (const Ice::LocalException&) + { + } + } + } + + for (const auto& p : _publishers) + { + auto s = p.second->getSession(); + if (s) + { + try + { + // TODO check the return value? + auto _ = s->disconnectedAsync(); + } + catch (const Ice::LocalException&) + { + } + } + } + } + _subscribers.clear(); + _publishers.clear(); + _subscriberSessions.clear(); + _publisherSessions.clear(); +} + +void +NodeI::initiateCreateSession(optional publisher, const Ice::Current& current) +{ + if (publisher == nullopt) + { + // TODO throw if the publisher is null? + return; + } + + // + // Create a session with the given publisher. + // + createPublisherSession(std::move(publisher), current.con, nullptr); +} + +void +NodeI::createSession( + optional subscriber, + optional subscriberSession, + bool fromRelay, + const Ice::Current& current) +{ + if (subscriber == nullopt || subscriberSession == nullopt) + { + return; + } + + shared_ptr session; + try + { + auto s = subscriber; + if (fromRelay) + { + // + // If the call is from a relay, we check if we already have a connection to this node + // and eventually re-use it. Otherwise, we'll try to establish a connection to the node + // if it has endpoints. If it doesn't, we'll re-use the current connection to send the + // confirmation. + // + s = getNodeWithExistingConnection(subscriber, current.con); + } + else if (current.con) + { + s = subscriber->ice_fixed(current.con); + } + + unique_lock lock(_mutex); + session = createPublisherSessionServant(subscriber); + if (!session || session->checkSession()) + { + return; // Shutting down or already connected + } + + auto self = shared_from_this(); + s->ice_getConnectionAsync( + [=, this](auto connection) mutable + { + if (session->checkSession()) + { + return; + } + + if (connection && !connection->getAdapter()) + { + connection->setAdapter(getInstance()->getObjectAdapter()); + } + + if (connection) + { + subscriberSession = subscriberSession->ice_fixed(connection); + } + + try + { + // Must be called before connected + s->confirmCreateSessionAsync( + _proxy, + session->getProxy(), + nullptr, + [=](auto ex) { self->removePublisherSession(subscriber, session, ex); }); + assert(!s->ice_getCachedConnection() || s->ice_getCachedConnection() == connection); + session->connected( + subscriberSession, + connection, + getInstance()->getTopicFactory()->getTopicWriters()); + } + catch (const Ice::LocalException&) + { + removePublisherSession(subscriber, session, current_exception()); + } + }, + [=](auto ex) { self->removePublisherSession(subscriber, session, ex); }); + } + catch (const Ice::LocalException&) + { + removePublisherSession(subscriber, session, current_exception()); + } +} + +void +NodeI::confirmCreateSession( + optional publisher, + optional publisherSession, + const Ice::Current& current) +{ + if (publisher == nullopt || publisherSession == nullopt) + { + return; + } + + unique_lock lock(_mutex); + auto p = _subscribers.find(publisher->ice_getIdentity()); + if (p == _subscribers.end()) + { + return; + } + + auto session = p->second; + if (session->checkSession()) + { + return; + } + + if (current.con && publisherSession->ice_getEndpoints().empty() && publisherSession->ice_getAdapterId().empty()) + { + publisherSession = publisherSession->ice_fixed(current.con); + } + + session->connected(publisherSession, current.con, getInstance()->getTopicFactory()->getTopicReaders()); +} + +void +NodeI::createSubscriberSession( + optional subscriber, + const Ice::ConnectionPtr& connection, + const shared_ptr& session) +{ + try + { + subscriber = getNodeWithExistingConnection(subscriber, connection); + + auto self = shared_from_this(); + subscriber->ice_getConnectionAsync( + [=, this](auto connection) + { + if (connection && !connection->getAdapter()) + { + connection->setAdapter(getInstance()->getObjectAdapter()); + } + subscriber->initiateCreateSessionAsync( + _proxy, + nullptr, + [=](auto ex) { self->removePublisherSession(subscriber, session, ex); }); + }, + [=](auto ex) { self->removePublisherSession(subscriber, session, ex); }); + } + catch (const Ice::LocalException&) + { + removePublisherSession(subscriber, session, current_exception()); + } +} + +void +NodeI::createPublisherSession( + optional publisher, + const Ice::ConnectionPtr& con, + shared_ptr session) +{ + try + { + auto p = getNodeWithExistingConnection(publisher, con); + + unique_lock lock(_mutex); + if (!session) + { + session = createSubscriberSessionServant(publisher); + if (!session) + { + return; // Shutting down. + } + } + + auto self = shared_from_this(); + p->ice_getConnectionAsync( + [=, this](auto connection) + { + if (session->checkSession()) + { + return; + } + + if (connection && !connection->getAdapter()) + { + connection->setAdapter(getInstance()->getObjectAdapter()); + } + + try + { + p->createSessionAsync( + _proxy, + session->getProxy(), + false, + nullptr, + [=](auto ex) { self->removeSubscriberSession(publisher, session, ex); }); + } + catch (const Ice::LocalException&) + { + removeSubscriberSession(publisher, session, current_exception()); + } + }, + [=](auto ex) { self->removeSubscriberSession(publisher, session, ex); }); + } + catch (const Ice::LocalException&) + { + removeSubscriberSession(publisher, session, current_exception()); + } +} + +void +NodeI::removeSubscriberSession( + optional node, + const shared_ptr& session, + const exception_ptr& ex) +{ + if (session && !session->retry(node, ex)) + { + unique_lock lock(_mutex); + auto sessionNode = session->getNode(); + if (!session->checkSession() && node && sessionNode && + node->ice_getIdentity() == sessionNode->ice_getIdentity()) + { + auto p = _subscribers.find(sessionNode->ice_getIdentity()); + if (p != _subscribers.end() && p->second == session) + { + _subscribers.erase(p); + _subscriberSessions.erase(session->getProxy()->ice_getIdentity()); + session->destroyImpl(ex); + } + } + } +} + +void +NodeI::removePublisherSession( + optional node, + const shared_ptr& session, + const exception_ptr& ex) +{ + if (session && !session->retry(node, ex)) + { + unique_lock lock(_mutex); + auto sessionNode = session->getNode(); + if (!session->checkSession() && node && sessionNode && + node->ice_getIdentity() == sessionNode->ice_getIdentity()) + { + auto p = _publishers.find(sessionNode->ice_getIdentity()); + if (p != _publishers.end() && p->second == session) + { + _publishers.erase(p); + _publisherSessions.erase(session->getProxy()->ice_getIdentity()); + session->destroyImpl(ex); + } + } + } +} + +Ice::ConnectionPtr +NodeI::getSessionConnection(const string& id) const +{ + auto session = getSession(Ice::stringToIdentity(id)); + if (session) + { + return session->getConnection(); + } + else + { + return nullptr; + } +} + +shared_ptr +NodeI::getSession(const Ice::Identity& ident) const +{ + unique_lock lock(_mutex); + if (ident.category == "s") + { + auto p = _subscriberSessions.find(ident); + if (p != _subscriberSessions.end()) + { + return p->second; + } + } + else if (ident.category == "p") + { + auto p = _publisherSessions.find(ident); + if (p != _publisherSessions.end()) + { + return p->second; + } + } + return nullptr; +} + +shared_ptr +NodeI::createSubscriberSessionServant(optional node) +{ + auto p = _subscribers.find(node->ice_getIdentity()); + if (p != _subscribers.end()) + { + p->second->setNode(node); + return p->second; + } + else + { + try + { + auto session = make_shared(shared_from_this(), node); + ostringstream os; + os << ++_nextSubscriberSessionId; + auto prx = getInstance()->getObjectAdapter()->createProxy({os.str(), "s"})->ice_oneway(); + session->init(prx); + _subscribers.emplace(node->ice_getIdentity(), session); + _subscriberSessions.emplace(session->getProxy()->ice_getIdentity(), session); + return session; + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + return nullptr; + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + return nullptr; + } + } +} + +shared_ptr +NodeI::createPublisherSessionServant(optional node) +{ + auto p = _publishers.find(node->ice_getIdentity()); + if (p != _publishers.end()) + { + p->second->setNode(node); + return p->second; + } + else + { + try + { + auto session = make_shared(shared_from_this(), node); + ostringstream os; + os << ++_nextPublisherSessionId; + auto prx = getInstance()->getObjectAdapter()->createProxy({os.str(), "p"})->ice_oneway(); + session->init(prx); + _publishers.emplace(node->ice_getIdentity(), session); + _publisherSessions.emplace(session->getProxy()->ice_getIdentity(), session); + return session; + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + return nullptr; + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + return nullptr; + } + } +} + +void +NodeI::forward(const Ice::ByteSeq& inEncaps, const Ice::Current& current) const +{ + lock_guard lock(_mutex); + if (current.id == _subscriberForwarder->ice_getIdentity()) + { + for (const auto& s : _subscribers) + { + optional session = s.second->getSession(); + if (session) + { + // TODO check the return value? + auto _ = session->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } + } + } + else + { + assert(current.id == _publisherForwarder->ice_getIdentity()); + for (const auto& s : _publishers) + { + optional session = s.second->getSession(); + if (session) + { + // TODO check the return value? + auto _ = session->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } + } + } +} + +optional +NodeI::getNodeWithExistingConnection(optional node, const Ice::ConnectionPtr& con) +{ + Ice::ConnectionPtr connection; + + // + // If the node has a session with this node, use a bi-dir proxy associated with + // node session's connection. + // + auto instance = _instance.lock(); + if (instance) + { + auto nodeSession = instance->getNodeSessionManager()->getSession(node->ice_getIdentity()); + if (nodeSession) + { + connection = nodeSession->getConnection(); + } + } + + // + // Otherwise, check if the node already has a session established and use the connection + // from the session. + // + { + lock_guard lock(_mutex); + auto p = _subscribers.find(node->ice_getIdentity()); + if (p != _subscribers.end()) + { + connection = p->second->getConnection(); + } + + auto q = _publishers.find(node->ice_getIdentity()); + if (q != _publishers.end()) + { + connection = q->second->getConnection(); + } + } + + // + // Make sure the connection is still valid. + // + if (connection) + { + try + { + connection->throwException(); + } + catch (...) + { + connection = nullptr; + } + } + + if (!connection && node->ice_getEndpoints().empty() && node->ice_getAdapterId().empty()) + { + connection = con; + } + + if (connection) + { + if (!connection->getAdapter()) + { + connection->setAdapter(instance->getObjectAdapter()); + } + return node->ice_fixed(connection); + } + + // + // Ensure that the returned proxy doesn't have a cached connection. + // + return node->ice_connectionCached(false)->ice_connectionCached(true); +} diff --git a/cpp/src/DataStorm/NodeI.h b/cpp/src/DataStorm/NodeI.h new file mode 100644 index 00000000000..755052bc0ba --- /dev/null +++ b/cpp/src/DataStorm/NodeI.h @@ -0,0 +1,115 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_NODEI_H +#define DATASTORM_NODEI_H + +#include "DataStorm/Contract.h" +#include "DataStorm/InternalI.h" +#include "ForwarderManager.h" + +#include "Ice/Ice.h" + +#include + +namespace DataStormI +{ + + class TraceLevels; + class Instance; + class SessionI; + class PublisherSessionI; + class SubscriberSessionI; + + class NodeI final : virtual public DataStormContract::Node, public std::enable_shared_from_this + { + public: + NodeI(const std::shared_ptr&); + virtual ~NodeI(); + + void init(); + void destroy(bool); + + void initiateCreateSession(std::optional, const Ice::Current&) final; + + void createSession( + std::optional, + std::optional, + bool, + const Ice::Current&) final; + + void confirmCreateSession( + std::optional, + std::optional, + const Ice::Current&) final; + + void createSubscriberSession( + std::optional, + const Ice::ConnectionPtr&, + const std::shared_ptr&); + + void createPublisherSession( + std::optional, + const Ice::ConnectionPtr&, + std::shared_ptr); + + void removeSubscriberSession( + std::optional, + const std::shared_ptr&, + const std::exception_ptr&); + + void removePublisherSession( + std::optional, + const std::shared_ptr&, + const std::exception_ptr&); + + Ice::ConnectionPtr getSessionConnection(const std::string&) const; + + std::shared_ptr getSession(const Ice::Identity&) const; + + std::optional + getNodeWithExistingConnection(std::optional, const Ice::ConnectionPtr&); + + std::optional getProxy() const { return _proxy; } + + std::shared_ptr getInstance() const + { + auto instance = _instance.lock(); + assert(instance); + return instance; + } + + std::optional getPublisherForwarder() const + { + return _publisherForwarder; + } + + std::optional getSubscriberForwarder() const + { + return _subscriberForwarder; + } + + private: + std::shared_ptr createSubscriberSessionServant(std::optional); + + std::shared_ptr createPublisherSessionServant(std::optional); + + void forward(const Ice::ByteSeq&, const Ice::Current&) const; + + mutable std::mutex _mutex; + mutable std::condition_variable _cond; + std::weak_ptr _instance; + std::optional _proxy; + std::optional _subscriberForwarder; + std::optional _publisherForwarder; + std::map> _subscribers; + std::map> _publishers; + std::map> _subscriberSessions; + std::map> _publisherSessions; + std::int64_t _nextSubscriberSessionId; + std::int64_t _nextPublisherSessionId; + }; + +} +#endif diff --git a/cpp/src/DataStorm/NodeSessionI.cpp b/cpp/src/DataStorm/NodeSessionI.cpp new file mode 100644 index 00000000000..a2c9a104696 --- /dev/null +++ b/cpp/src/DataStorm/NodeSessionI.cpp @@ -0,0 +1,241 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "NodeSessionI.h" +#include "ConnectionManager.h" +#include "Instance.h" +#include "NodeSessionManager.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; +using namespace DataStormContract; + +namespace +{ + + class NodeForwarderI : public Node, public enable_shared_from_this + { + public: + NodeForwarderI( + shared_ptr nodeSessionManager, + shared_ptr session, + optional node) + : _nodeSessionManager(std::move(nodeSessionManager)), + _session(std::move(session)), + _node(std::move(node)) + { + } + + virtual void initiateCreateSession(optional publisher, const Ice::Current& current) override + { + if (publisher == nullopt) + { + return; + } + + auto session = _session.lock(); + if (!session) + { + return; + } + + try + { + optional session; + updateNodeAndSessionProxy(publisher, session, current); + // TODO check the return value? + auto _ = _node->initiateCreateSessionAsync(publisher); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + } + + virtual void createSession( + optional subscriber, + optional subscriberSession, + bool /* fromRelay */, + const Ice::Current& current) override + { + if (subscriber == nullopt || subscriberSession == nullopt) + { + return; + } + + auto session = _session.lock(); + if (!session) + { + return; + } + + try + { + updateNodeAndSessionProxy(subscriber, subscriberSession, current); + session->addSession(subscriberSession); + // TODO check the return value? + auto _ = _node->createSessionAsync(subscriber, subscriberSession, true); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + } + + virtual void confirmCreateSession( + optional publisher, + optional publisherSession, + const Ice::Current& current) override + { + if (publisher == nullopt || publisherSession == nullopt) + { + return; + } + + auto session = _session.lock(); + if (!session) + { + return; + } + try + { + updateNodeAndSessionProxy(publisher, publisherSession, current); + session->addSession(publisherSession); + // TODO check the return value? + auto _ = _node->confirmCreateSessionAsync(publisher, publisherSession); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + } + + private: + template + void updateNodeAndSessionProxy(optional& node, optional& session, const Ice::Current& current) + { + assert(node != nullopt); + if (node->ice_getEndpoints().empty() && node->ice_getAdapterId().empty()) + { + auto peerSession = _nodeSessionManager->createOrGet(node, current.con, false); + assert(peerSession); + node = peerSession->getPublicNode(); + if (session) + { + session = peerSession->getSessionForwarder(session); + } + } + } + + const shared_ptr _nodeSessionManager; + const weak_ptr _session; + const optional _node; + }; + +} + +NodeSessionI::NodeSessionI( + shared_ptr instance, + optional node, + Ice::ConnectionPtr connection, + bool forwardAnnouncements) + : _instance(std::move(instance)), + _traceLevels(_instance->getTraceLevels()), + _node(std::move(node)), + _connection(std::move(connection)) +{ + if (forwardAnnouncements) + { + _lookup = _connection->createProxy({"Lookup", "DataStorm"}); + } +} + +void +NodeSessionI::init() +{ + if (_node->ice_getEndpoints().empty() && _node->ice_getAdapterId().empty()) + { + auto bidirNode = _node->ice_fixed(_connection); + auto fwd = make_shared(_instance->getNodeSessionManager(), shared_from_this(), bidirNode); + _publicNode = _instance->getObjectAdapter()->add(fwd, _node->ice_getIdentity()); + } + else + { + _publicNode = _node; + } + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << "created node session (peer = `" << _publicNode << "'):\n" << _connection->toString(); + } +} + +void +NodeSessionI::destroy() +{ + lock_guard lock(_mutex); + _destroyed = true; + + try + { + if (_publicNode != _node) + { + _instance->getObjectAdapter()->remove(_publicNode->ice_getIdentity()); + } + + for (const auto& session : _sessions) + { + // TODO check the return value? + auto _ = session.second->disconnectedAsync(); + } + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << "destroyed node session (peer = `" << _publicNode << "')"; + } +} + +void +NodeSessionI::addSession(optional session) +{ + lock_guard lock(_mutex); + _sessions[session->ice_getIdentity()] = std::move(session); +} + +optional +NodeSessionI::forwarder(optional session) const +{ + auto id = session->ice_getIdentity(); + auto proxy = _instance->getObjectAdapter()->createProxy( + {id.name + '-' + _node->ice_getIdentity().name, id.category + 'f'}); + return proxy->ice_oneway(); +} diff --git a/cpp/src/DataStorm/NodeSessionI.h b/cpp/src/DataStorm/NodeSessionI.h new file mode 100644 index 00000000000..71508d740c0 --- /dev/null +++ b/cpp/src/DataStorm/NodeSessionI.h @@ -0,0 +1,50 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_NODE_SESSIONI_H +#define DATASTORM_NODE_SESSIONI_H + +#include "DataStorm/Contract.h" +#include "Ice/Ice.h" + +namespace DataStormI +{ + + class Instance; + class TraceLevels; + + class NodeSessionI : public std::enable_shared_from_this + { + public: + NodeSessionI(std::shared_ptr, std::optional, Ice::ConnectionPtr, bool); + + void init(); + void destroy(); + void addSession(std::optional); + + std::optional getPublicNode() const { return _publicNode; } + std::optional getLookup() const { return _lookup; } + const Ice::ConnectionPtr& getConnection() const { return _connection; } + template std::optional getSessionForwarder(std::optional session) const + { + return Ice::uncheckedCast(forwarder(session)); + } + + private: + std::optional forwarder(std::optional) const; + + const std::shared_ptr _instance; + const std::shared_ptr _traceLevels; + std::optional _node; + const Ice::ConnectionPtr _connection; + + std::mutex _mutex; + bool _destroyed; + std::optional _publicNode; + std::optional _lookup; + std::map> _sessions; + }; + +} +#endif diff --git a/cpp/src/DataStorm/NodeSessionManager.cpp b/cpp/src/DataStorm/NodeSessionManager.cpp new file mode 100644 index 00000000000..065d00223c8 --- /dev/null +++ b/cpp/src/DataStorm/NodeSessionManager.cpp @@ -0,0 +1,451 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "NodeSessionManager.h" +#include "ConnectionManager.h" +#include "ForwarderManager.h" +#include "Instance.h" +#include "NodeI.h" +#include "NodeSessionI.h" +#include "TimerTaskI.h" +#include "TopicFactoryI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; +using namespace DataStormContract; + +namespace +{ + + class SessionForwarderI : public Ice::Blobject + { + public: + SessionForwarderI(shared_ptr nodeSessionManager) + : _nodeSessionManager(std::move(nodeSessionManager)) + { + } + + virtual bool ice_invoke(Ice::ByteSeq inEncaps, Ice::ByteSeq&, const Ice::Current& curr) + { + auto pos = curr.id.name.find('-'); + if (pos != string::npos && pos < curr.id.name.length()) + { + auto s = _nodeSessionManager->getSession(curr.id.name.substr(pos + 1)); + if (s) + { + auto id = Ice::Identity{curr.id.name.substr(0, pos), curr.id.category.substr(0, 1)}; + // TODO check the return value? + auto _ = s->getConnection()->createProxy(id)->ice_invokeAsync( + curr.operation, + curr.mode, + inEncaps, + curr.ctx); + return true; + } + } + throw Ice::ObjectNotExistException(__FILE__, __LINE__, curr.id, curr.facet, curr.operation); + } + + private: + const shared_ptr _nodeSessionManager; + }; + +} + +NodeSessionManager::NodeSessionManager(const shared_ptr& instance, const shared_ptr& node) + : _instance(instance), + _traceLevels(instance->getTraceLevels()), + _nodePrx(node->getProxy()), + _forwardToMulticast( + instance->getCommunicator()->getProperties()->getPropertyAsInt( + "DataStorm.Node.Server.ForwardDiscoveryToMulticast") > 0), + _retryCount(0) +{ +} + +void +NodeSessionManager::init() +{ + auto instance = getInstance(); + + auto forwarder = [self = shared_from_this()](Ice::ByteSeq e, const Ice::Current& c) { self->forward(e, c); }; + _forwarder = Ice::uncheckedCast(instance->getCollocatedForwarder()->add(std::move(forwarder))); + + try + { + auto sessionForwader = make_shared(shared_from_this()); + instance->getObjectAdapter()->addDefaultServant(sessionForwader, "sf"); + instance->getObjectAdapter()->addDefaultServant(sessionForwader, "pf"); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + + auto communicator = instance->getCommunicator(); + auto connectTo = communicator->getProperties()->getProperty("DataStorm.Node.ConnectTo"); + if (!connectTo.empty()) + { + connect(LookupPrx{communicator, "DataStorm/Lookup:" + connectTo}, _nodePrx); + } +} + +void +NodeSessionManager::destroy() +{ + unique_lock lock(_mutex); + _instance.reset(); +} + +shared_ptr +NodeSessionManager::createOrGet(optional node, const Ice::ConnectionPtr& connection, bool forwardAnnouncements) +{ + // TODO node should be non-optional + unique_lock lock(_mutex); + + auto p = _sessions.find(node->ice_getIdentity()); + if (p != _sessions.end()) + { + if (p->second->getConnection() != connection) + { + p->second->destroy(); + _sessions.erase(p); + } + else + { + return p->second; + } + } + + auto instance = getInstance(); + + if (!connection->getAdapter()) + { + connection->setAdapter(instance->getObjectAdapter()); + } + + auto session = make_shared(instance, node, connection, forwardAnnouncements); + session->init(); + _sessions.emplace(node->ice_getIdentity(), session); + + // TODO we should review this code, to avoid using the proxy shared_ptr as a map key. + // Specially the connection manager doesn't use this proxy for lookup. + instance->getConnectionManager()->add( + connection, + make_shared(*node), + [self = shared_from_this(), node](const Ice::ConnectionPtr&, std::exception_ptr) + { self->destroySession(std::move(node)); }); + + return session; +} + +void +NodeSessionManager::announceTopicReader( + const string& topic, + optional node, + const Ice::ConnectionPtr& connection) const +{ + unique_lock lock(_mutex); + if (connection && node->ice_getIdentity() == _nodePrx->ice_getIdentity()) + { + return; // Ignore requests from self + } + + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + if (connection) + { + out << "topic reader `" << topic << "' announced (peer = `" << node << "')"; + } + else + { + out << "announcing topic reader `" << topic << "' (peer = `" << node << "')"; + } + } + + auto p = _sessions.find(node->ice_getIdentity()); + auto nodePrx = p != _sessions.end() ? p->second->getPublicNode() : node; + + _exclude = connection; + _forwarder->announceTopicReader(topic, nodePrx); + + lock.unlock(); + + if (!connection || (_forwardToMulticast && connection->type() != "udp")) + { + auto instance = _instance.lock(); + if (instance && instance->getLookup()) + { + // TODO check the return value? + auto _ = instance->getLookup()->announceTopicReaderAsync(topic, nodePrx); + } + } +} + +void +NodeSessionManager::announceTopicWriter( + const string& topic, + optional node, + const Ice::ConnectionPtr& connection) const +{ + unique_lock lock(_mutex); + if (connection && node->ice_getIdentity() == _nodePrx->ice_getIdentity()) + { + return; // Ignore requests from self + } + + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + if (connection) + { + out << "topic writer `" << topic << "' announced (peer = `" << node << "')"; + } + else + { + out << "announcing topic writer `" << topic << "' (peer = `" << node << "')"; + } + } + + _exclude = connection; + auto p = _sessions.find(node->ice_getIdentity()); + auto nodePrx = p != _sessions.end() ? p->second->getPublicNode() : node; + _forwarder->announceTopicWriter(topic, nodePrx); + + lock.unlock(); + + if (!connection || (_forwardToMulticast && connection->type() != "udp")) + { + auto instance = _instance.lock(); + if (instance && instance->getLookup()) + { + // TODO check the return value? + auto _ = instance->getLookup()->announceTopicWriterAsync(topic, nodePrx); + } + } +} + +void +NodeSessionManager::announceTopics( + const StringSeq& readers, + const StringSeq& writers, + optional node, + const Ice::ConnectionPtr& connection) const +{ + unique_lock lock(_mutex); + if (connection && node->ice_getIdentity() == _nodePrx->ice_getIdentity()) + { + return; // Ignore requests from self + } + + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + if (connection) + { + if (!readers.empty()) + { + out << "topic reader(s) `" << readers << "' announced (peer = `" << node << "')"; + } + if (!writers.empty()) + { + out << "topic writer(s) `" << writers << "' announced (peer = `" << node << "')"; + } + } + else + { + if (!readers.empty()) + { + out << "announcing topic reader(s) `" << readers << "' (peer = `" << node << "')"; + } + if (!writers.empty()) + { + out << "announcing topic writer(s) `" << writers << "' (peer = `" << node << "')"; + } + } + } + + _exclude = connection; + auto p = _sessions.find(node->ice_getIdentity()); + auto nodePrx = p != _sessions.end() ? p->second->getPublicNode() : node; + _forwarder->announceTopics(readers, writers, nodePrx); + + lock.unlock(); + + if (!connection || (_forwardToMulticast && connection->type() != "udp")) + { + auto instance = _instance.lock(); + if (instance && instance->getLookup()) + { + // TODO check the return value? + auto _ = instance->getLookup()->announceTopicsAsync(readers, writers, nodePrx); + } + } +} + +shared_ptr +NodeSessionManager::getSession(const Ice::Identity& node) const +{ + unique_lock lock(_mutex); + auto p = _sessions.find(node); + if (p != _sessions.end()) + { + return p->second; + } + return nullptr; +} + +void +NodeSessionManager::forward(const Ice::ByteSeq& inEncaps, const Ice::Current& current) const +{ + for (const auto& session : _sessions) + { + if (session.second->getConnection() != _exclude) + { + auto l = session.second->getLookup(); + if (l) + { + // TODO check the return value? + auto _ = l->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } + } + } + for (const auto& lookup : _connectedTo) + { + if (lookup.second.second->ice_getCachedConnection() != _exclude) + { + // TODO check the return value? + auto _ = lookup.second.second->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } + } +} + +void +NodeSessionManager::connect(optional lookup, optional proxy) +{ + try + { + lookup->createSessionAsync( + proxy, + [=, self = shared_from_this()](auto node) { self->connected(node, lookup); }, + [=, self = shared_from_this()](std::exception_ptr) { self->disconnected(nullopt, lookup); }); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + disconnected(nullopt, lookup); + } + catch (const Ice::CommunicatorDestroyedException&) + { + disconnected(nullopt, lookup); + } +} + +void +NodeSessionManager::connected(optional node, optional lookup) +{ + // TODO lookup and node should be non-optional + unique_lock lock(_mutex); + auto instance = _instance.lock(); + if (!instance) + { + return; + } + + auto p = _sessions.find(node->ice_getIdentity()); + auto connection = p != _sessions.end() ? p->second->getConnection() : lookup->ice_getCachedConnection(); + if (!connection->getAdapter()) + { + connection->setAdapter(instance->getObjectAdapter()); + } + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << "established node session (peer = `" << node << "'):\n" << connection->toString(); + } + + instance->getConnectionManager()->add( + connection, + make_shared(*lookup), + [=, self = shared_from_this()](const Ice::ConnectionPtr&, std::exception_ptr) + { self->disconnected(node, lookup); }); + auto l = p != _sessions.end() ? lookup->ice_fixed(connection) : lookup; + _connectedTo.emplace(node->ice_getIdentity(), make_pair(node, l)); + + auto readerNames = instance->getTopicFactory()->getTopicReaderNames(); + auto writerNames = instance->getTopicFactory()->getTopicWriterNames(); + if (!readerNames.empty() || !writerNames.empty()) + { + try + { + // TODO check the return value? + auto _ = l->announceTopicsAsync(readerNames, writerNames, _nodePrx); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + } +} + +void +NodeSessionManager::disconnected(optional node, optional lookup) +{ + unique_lock lock(_mutex); + auto instance = _instance.lock(); + if (!instance) + { + return; + } + + if (node) + { + _retryCount = 0; + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << "disconnected node session (peer = `" << node << "')"; + } + _connectedTo.erase(node->ice_getIdentity()); + lock.unlock(); + connect(lookup, _nodePrx); + } + else + { + instance->getTimer()->schedule( + make_shared( + [=, this, self = shared_from_this()] + { + auto instance = _instance.lock(); + if (instance) + { + self->connect(lookup, _nodePrx); + } + }), + instance->getRetryDelay(_retryCount++)); + } +} + +void +NodeSessionManager::destroySession(optional node) +{ + unique_lock lock(_mutex); + auto p = _sessions.find(node->ice_getIdentity()); + if (p == _sessions.end()) + { + return; + } + + p->second->destroy(); + _sessions.erase(p); +} diff --git a/cpp/src/DataStorm/NodeSessionManager.h b/cpp/src/DataStorm/NodeSessionManager.h new file mode 100644 index 00000000000..061bf7aaf33 --- /dev/null +++ b/cpp/src/DataStorm/NodeSessionManager.h @@ -0,0 +1,92 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_NODE_SESSION_MANAGER_H +#define DATASTORM_NODE_SESSION_MANAGER_H + +#include "DataStorm/Config.h" +#include "DataStorm/Contract.h" +#include "Ice/Ice.h" + +namespace DataStormI +{ + + class NodeSessionI; + class CallbackExecutor; + class Instance; + class TraceLevels; + class NodeI; + + class NodeSessionManager : public std::enable_shared_from_this + { + public: + NodeSessionManager(const std::shared_ptr&, const std::shared_ptr&); + + void init(); + void destroy(); + + std::shared_ptr + createOrGet(std::optional, const Ice::ConnectionPtr&, bool); + + void announceTopicReader( + const std::string&, + std::optional, + const Ice::ConnectionPtr& = nullptr) const; + + void announceTopicWriter( + const std::string&, + std::optional, + const Ice::ConnectionPtr& = nullptr) const; + + void announceTopics( + const DataStormContract::StringSeq&, + const DataStormContract::StringSeq&, + std::optional, + const Ice::ConnectionPtr& = nullptr) const; + + std::shared_ptr getSession(const Ice::Identity&) const; + std::shared_ptr getSession(const std::string& name) const + { + return getSession(Ice::Identity{name, ""}); + } + + void forward(const Ice::ByteSeq&, const Ice::Current&) const; + + private: + void connect(std::optional, std::optional); + + void connected(std::optional, std::optional); + + void disconnected(std::optional, std::optional); + + void destroySession(std::optional); + + std::shared_ptr getInstance() const + { + auto instance = _instance.lock(); + assert(instance); + return instance; + } + + std::weak_ptr _instance; + const std::shared_ptr _traceLevels; + const std::optional _nodePrx; + const bool _forwardToMulticast; + + mutable std::mutex _mutex; + + int _retryCount; + + std::map> _sessions; + std::map< + Ice::Identity, + std::pair, std::optional>> + _connectedTo; + + mutable Ice::ConnectionPtr _exclude; + std::optional _forwarder; + }; + +} +#endif diff --git a/cpp/src/DataStorm/SessionI.cpp b/cpp/src/DataStorm/SessionI.cpp new file mode 100644 index 00000000000..0c1fc1be8d5 --- /dev/null +++ b/cpp/src/DataStorm/SessionI.cpp @@ -0,0 +1,1389 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "SessionI.h" +#include "CallbackExecutor.h" +#include "ConnectionManager.h" +#include "Instance.h" +#include "NodeI.h" +#include "TimerTaskI.h" +#include "TopicFactoryI.h" +#include "TopicI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; +using namespace DataStormContract; + +namespace +{ + + class DispatchInterceptorI : public Ice::Object + { + public: + DispatchInterceptorI(Ice::ObjectPtr servant, shared_ptr executor) + : _servant(std::move(servant)), + _executor(std::move(executor)) + { + } + + void dispatch(Ice::IncomingRequest& request, std::function sendResponse) final + { + _servant->dispatch(request, sendResponse); + _executor->flush(); + } + + private: + Ice::ObjectPtr _servant; + shared_ptr _executor; + }; + +} + +SessionI::SessionI(const std::shared_ptr& parent, optional node) + : _instance(parent->getInstance()), + _traceLevels(_instance->getTraceLevels()), + _parent(parent), + _node(node), + _destroyed(false), + _sessionInstanceId(0), + _retryCount(0) +{ +} + +void +SessionI::init(optional prx) +{ + assert(_node); + _proxy = prx; + _id = Ice::identityToString(prx->ice_getIdentity()); + + // + // Even though the node register a default servant for sessions, we still need to + // register the session servant explicitly here to ensure collocation works. The + // default servant from the node is used for facet calls. + // + _instance->getObjectAdapter()->add( + make_shared(shared_from_this(), _instance->getCallbackExecutor()), + _proxy->ice_getIdentity()); + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": created session (peer = `" << _node << "')"; + } +} + +void +SessionI::announceTopics(TopicInfoSeq topics, bool, const Ice::Current&) +{ + // + // Retain topics outside the synchronization. This is necessary to ensure the topic destructor + // doesn't get called within the synchronization. The topic destructor can callback on the + // session to disconnect. + // + vector> retained; + { + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": announcing topics `" << topics << "'"; + } + + for (const auto& info : topics) + { + runWithTopics( + info.name, + retained, + [&](const shared_ptr& topic) + { + for (auto id : info.ids) + { + topic->attach(id, shared_from_this(), _session); + } + // TODO check the return value? + auto _ = _session->attachTopicAsync(topic->getTopicSpec()); + }); + } + + // Reap un-visited topics + auto p = _topics.begin(); + while (p != _topics.end()) + { + if (p->second.reap(_sessionInstanceId)) + { + _topics.erase(p++); + } + else + { + ++p; + } + } + } +} + +void +SessionI::attachTopic(TopicSpec spec, const Ice::Current&) +{ + // + // Retain topics outside the synchronization. This is necessary to ensure the topic destructor + // doesn't get called within the synchronization. The topic destructor can callback on the + // session to disconnect. + // + vector> retained; + { + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + runWithTopics( + spec.name, + retained, + [&](const shared_ptr& topic) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": attaching topic `" << spec << "' to `" << topic << "'"; + } + + topic->attach(spec.id, shared_from_this(), _session); + + if (!spec.tags.empty()) + { + auto& subscriber = _topics.at(spec.id).getSubscriber(topic.get()); + for (const auto& tag : spec.tags) + { + subscriber.tags[tag.id] = + topic->getTagFactory()->decode(_instance->getCommunicator(), tag.value); + } + } + + auto tags = topic->getTags(); + if (!tags.empty()) + { + // TODO check the return value? + auto _ = _session->attachTagsAsync(topic->getId(), tags, true); + } + + auto specs = topic->getElementSpecs(spec.id, spec.elements, shared_from_this()); + if (!specs.empty()) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": matched elements `" << spec << "' on `" << topic << "'"; + } + // TODO check the return value? + auto _ = _session->attachElementsAsync(topic->getId(), specs, true); + } + }); + } +} + +void +SessionI::detachTopic(int64_t id, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + runWithTopics( + id, + [&](TopicI* topic, TopicSubscriber&) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": detaching topic `" << id << "' from `" << topic << "'"; + } + topic->detach(id, shared_from_this()); + }); + + // Erase the topic + _topics.erase(id); +} + +void +SessionI::attachTags(int64_t topicId, ElementInfoSeq tags, bool initialize, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + runWithTopics( + topicId, + [&](TopicI* topic, TopicSubscriber& subscriber, TopicSubscribers&) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": attaching tags `[" << tags << "]@" << topicId << "' on topic `" << topic << "'"; + } + + if (initialize) + { + subscriber.tags.clear(); + } + for (const auto& tag : tags) + { + subscriber.tags[tag.id] = topic->getTagFactory()->decode(_instance->getCommunicator(), tag.value); + } + }); +} + +void +SessionI::detachTags(int64_t topicId, LongSeq tags, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + runWithTopics( + topicId, + [&](TopicI* topic, TopicSubscriber& subscriber, TopicSubscribers&) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": detaching tags `[" << tags << "]@" << topicId << "' on topic `" << topic << "'"; + } + + for (auto tag : tags) + { + subscriber.tags.erase(tag); + } + }); +} + +void +SessionI::announceElements(int64_t topicId, ElementInfoSeq elements, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + runWithTopics( + topicId, + [&](TopicI* topic, TopicSubscriber&, TopicSubscribers&) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": announcing elements `[" << elements << "]@" << topicId << "' on topic `" << topic + << "'"; + } + + auto specs = topic->getElementSpecs(topicId, elements, shared_from_this()); + if (!specs.empty()) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": announcing elements matched `[" << specs << "]@" << topicId << "' on topic `" + << topic << "'"; + } + // TODO check the return value? + auto _ = _session->attachElementsAsync(topic->getId(), specs, false); + } + }); +} + +void +SessionI::attachElements(int64_t id, ElementSpecSeq elements, bool initialize, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + auto now = chrono::system_clock::now(); + runWithTopics( + id, + [&](TopicI* topic, TopicSubscriber& subscriber, TopicSubscribers&) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": attaching elements `[" << elements << "]@" << id << "' on topic `" << topic << "'"; + if (initialize) + { + out << " (initializing)"; + } + } + + auto specAck = topic->attachElements(id, elements, shared_from_this(), _session, now); + + if (initialize) + { + // Reap unused keys and filters from the topic subscriber + subscriber.reap(_sessionInstanceId); + + // TODO: reap keys / filters + } + + if (!specAck.empty()) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": attaching elements matched `[" << specAck << "]@" << id << "' on topic `" << topic + << "'"; + } + // TODO check the return value? + auto _ = _session->attachElementsAckAsync(topic->getId(), specAck); + } + }); +} + +void +SessionI::attachElementsAck(int64_t id, ElementSpecAckSeq elements, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + auto now = chrono::system_clock::now(); + runWithTopics( + id, + [&](TopicI* topic, TopicSubscriber&, TopicSubscribers&) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": attaching elements ack `[" << elements << "]@" << id << "' on topic `" << topic << "'"; + } + + LongSeq removedIds; + auto samples = topic->attachElementsAck(id, elements, shared_from_this(), _session, now, removedIds); + if (!samples.empty()) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": initializing elements `[" << samples << "]@" << id << "' on topic `" << topic + << "'"; + } + // TODO check the return value? + auto _ = _session->initSamplesAsync(topic->getId(), samples); + } + if (!removedIds.empty()) + { + // TODO check the return value? + auto _ = _session->detachElementsAsync(topic->getId(), removedIds); + } + }); +} + +void +SessionI::detachElements(int64_t id, LongSeq elements, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + runWithTopics( + id, + [&](TopicI* topic, TopicSubscriber& subscriber) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": detaching elements `[" << elements << "]@" << id << "' on topic `" << topic << "'"; + } + + for (auto e : elements) + { + auto k = subscriber.remove(e); + for (auto& s : k.getSubscribers()) + { + for (auto key : s.second.keys) + { + if (e > 0) + { + s.first->detachKey(id, e, key, shared_from_this(), s.second.facet, true); + } + else + { + s.first->detachFilter(id, -e, key, shared_from_this(), s.second.facet, true); + } + } + } + } + }); +} + +void +SessionI::initSamples(int64_t topicId, DataSamplesSeq samplesSeq, const Ice::Current&) +{ + lock_guard lock(_mutex); + if (!_session) + { + return; + } + + auto now = chrono::system_clock::now(); + auto communicator = _instance->getCommunicator(); + for (const auto& samples : samplesSeq) + { + runWithTopics( + topicId, + [&](TopicI* topic, TopicSubscriber& subscriber) + { + auto k = subscriber.get(samples.id); + if (k) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": initializing samples from `" << samples.id << "'" + << " on ["; + for (auto q = k->getSubscribers().begin(); q != k->getSubscribers().end(); ++q) + { + if (q != k->getSubscribers().begin()) + { + out << ", "; + } + out << q->first; + if (!q->second.facet.empty()) + { + out << ":" << q->second.facet; + } + } + out << "]"; + } + + vector> samplesI; + samplesI.reserve(samples.samples.size()); + auto sampleFactory = topic->getSampleFactory(); + for (auto& s : samples.samples) + { + shared_ptr key; + if (s.keyValue.empty()) + { + key = subscriber.keys[s.keyId].first; + } + else + { + key = topic->getKeyFactory()->decode(_instance->getCommunicator(), s.keyValue); + } + assert(key); + + samplesI.push_back(sampleFactory->create( + _id, + k->name, + s.id, + s.event, + key, + subscriber.tags[s.tag], + s.value, + s.timestamp)); + } + for (auto& ks : k->getSubscribers()) + { + if (!ks.second.initialized) + { + ks.second.initialized = true; + if (!samplesI.empty()) + { + ks.second.lastId = samplesI.back()->id; + ks.first->initSamples(samplesI, topicId, samples.id, k->priority, now, samples.id < 0); + } + } + } + } + }); + } +} + +void +SessionI::disconnected(const Ice::Current& current) +{ + if (disconnected(current.con, nullptr)) + { + if (!retry(getNode(), nullptr)) + { + remove(); + } + } +} + +void +SessionI::connected(optional session, const Ice::ConnectionPtr& connection, const TopicInfoSeq& topics) +{ + lock_guard lock(_mutex); + if (_destroyed || _session) + { + assert(_connectedCallbacks.empty()); + return; + } + + _session = session; + _connection = connection; + if (connection) + { + auto self = shared_from_this(); + _instance->getConnectionManager()->add( + connection, + self, + [self](auto connection, auto ex) + { + if (self->disconnected(connection, ex)) + { + if (!self->retry(self->getNode(), nullptr)) + { + self->remove(); + } + } + }); + } + + if (_retryTask) + { + _instance->getTimer()->cancel(_retryTask); + _retryTask = nullptr; + } + + ++_sessionInstanceId; + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": session `" << _session->ice_getIdentity() << "' connected"; + if (_connection) + { + out << "\n" << _connection->toString(); + } + } + + if (!topics.empty()) + { + try + { + // TODO check the return value? + auto _ = _session->announceTopicsAsync(topics, true); + } + catch (const Ice::LocalException&) + { + // Ignore + } + } + + for (auto c : _connectedCallbacks) + { + c(_proxy); + } + _connectedCallbacks.clear(); +} + +bool +SessionI::disconnected(const Ice::ConnectionPtr& connection, exception_ptr ex) +{ + lock_guard lock(_mutex); + if (_destroyed || (connection && _connection != connection) || !_session) + { + return false; + } + + if (_traceLevels->session > 0) + { + try + { + if (ex) + { + rethrow_exception(ex); + } + else + { + throw Ice::CloseConnectionException(__FILE__, __LINE__); + } + } + catch (const std::exception& e) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": session `" << _session->ice_getIdentity() << "' disconnected:\n"; + out << (_connection ? _connection->toString() : "") << "\n"; + out << e.what(); + } + } + + for (auto& t : _topics) + { + runWithTopics(t.first, [&](TopicI* topic, TopicSubscriber&) { topic->detach(t.first, shared_from_this()); }); + } + + _session = nullopt; + _connection = nullptr; + _retryCount = 0; + return true; +} + +bool +SessionI::retry(optional node, exception_ptr exception) +{ + lock_guard lock(_mutex); + + if (exception) + { + // + // Don't retry if we are shutting down. + // + try + { + rethrow_exception(exception); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + return false; + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + return false; + } + catch (const Ice::CommunicatorDestroyedException&) + { + return false; + } + catch (const std::exception&) + { + } + } + + if (node->ice_getEndpoints().empty() && node->ice_getAdapterId().empty()) + { + if (_retryTask) + { + _instance->getTimer()->cancel(_retryTask); + _retryTask = nullptr; + } + _retryCount = 0; + + // + // If we can't retry connecting to the node because we don't have its endpoints, + // we just wait for the duration of the last retry delay for the peer to reconnect. + // If it doesn't reconnect, we'll destroy this session after the timeout. + // + auto delay = _instance->getRetryDelay(_instance->getRetryCount()) * 2; + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": can't retry connecting to `" << node->ice_toString() << "`, waiting " << delay.count() + << " (ms) for peer to reconnect"; + } + + _retryTask = make_shared([self = shared_from_this()] { self->remove(); }); + _instance->getTimer()->schedule(_retryTask, delay); + } + else + { + // + // If we can retry the connection attempt, we schedule a timer to retry. Always + // retry immediately on the first attempt. + // + auto delay = _retryCount == 0 ? 0ms : _instance->getRetryDelay(_retryCount - 1); + ++_retryCount; + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + if (_retryCount <= _instance->getRetryCount()) + { + out << _id << ": retrying connecting to `" << node->ice_toString() << "` in " << delay.count(); + out << " (ms), retry " << _retryCount << '/' << _instance->getRetryCount(); + } + else + { + out << _id << ": connection to `" << node->ice_toString() + << "` failed and the retry limit has been reached"; + } + if (exception) + { + try + { + rethrow_exception(exception); + } + catch (const Ice::LocalException& ex) + { + out << '\n' << ex.what(); + } + } + } + + if (_retryCount > _instance->getRetryCount()) + { + return false; + } + + _retryTask = make_shared([node, self = shared_from_this()] { self->reconnect(node); }); + _instance->getTimer()->schedule(_retryTask, delay); + } + return true; +} + +void +SessionI::destroyImpl(const exception_ptr& ex) +{ + lock_guard lock(_mutex); + assert(!_destroyed); + _destroyed = true; + + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": destroyed session"; + if (ex) + { + try + { + rethrow_exception(ex); + } + catch (const Ice::Exception& e) + { + out << "\n:" << e.what() << "\n" << e.ice_stackTrace(); + } + catch (const exception& e) + { + out << "\n:" << e.what(); + } + catch (...) + { + out << "\n: unexpected exception"; + } + } + } + + if (_session) + { + if (_connection) + { + _instance->getConnectionManager()->remove(shared_from_this(), _connection); + } + + _session = nullopt; + _connection = nullptr; + + for (const auto& t : _topics) + { + runWithTopics( + t.first, + [&](TopicI* topic, TopicSubscriber&) { topic->detach(t.first, shared_from_this()); }); + } + _topics.clear(); + } + + for (auto c : _connectedCallbacks) + { + c(nullopt); + } + _connectedCallbacks.clear(); + + try + { + _instance->getObjectAdapter()->remove(_proxy->ice_getIdentity()); + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } +} + +Ice::ConnectionPtr +SessionI::getConnection() const +{ + lock_guard lock(_mutex); + return _connection; +} + +bool +SessionI::checkSession() +{ + while (true) + { + unique_lock lock(_mutex); + if (_session) + { + if (_connection) + { + // + // Make sure the connection is still established. It's possible that the connection got closed + // and we're not notified yet by the connection manager. Check session explicitly check for the + // connection to make sure that if we get a session creation request from a peer (which might + // detect the connection closure before), it doesn't get ignored. + // + try + { + _connection->throwException(); + } + catch (const Ice::LocalException&) + { + auto connection = _connection; + lock.unlock(); + if (!disconnected(connection, current_exception())) + { + continue; + } + return false; + } + } + return true; + } + else + { + return false; + } + } +} + +optional +SessionI::getSession() const +{ + lock_guard lock(_mutex); + return _session; +} + +optional +SessionI::getNode() const +{ + lock_guard lock(_mutex); + return _node; +} + +void +SessionI::setNode(optional node) +{ + lock_guard lock(_mutex); + _node = node; +} + +void +SessionI::subscribe(int64_t id, TopicI* topic) +{ + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": subscribed topic `" << id << "' to topic `" << topic << "'"; + } + _topics[id].addSubscriber(topic, _sessionInstanceId); +} + +void +SessionI::unsubscribe(int64_t id, TopicI* topic) +{ + assert(_topics.find(id) != _topics.end()); + auto& subscriber = _topics.at(id).getSubscriber(topic); + for (auto& k : subscriber.getAll()) + { + for (auto& e : k.second.getSubscribers()) + { + for (auto key : e.second.keys) + { + if (k.first > 0) + { + e.first->detachKey(id, k.first, key, shared_from_this(), e.second.facet, false); + } + else + { + e.first->detachFilter(id, -k.first, key, shared_from_this(), e.second.facet, false); + } + } + } + } + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": unsubscribed topic `" << id << "' from topic `" << topic << "'"; + } +} + +void +SessionI::disconnect(int64_t id, TopicI* topic) +{ + lock_guard lock(_mutex); // Called by TopicI::destroy + if (!_session) + { + return; + } + + if (_topics.find(id) == _topics.end()) + { + return; // Peer topic detached first. + } + + runWithTopic(id, topic, [&](TopicSubscriber&) { unsubscribe(id, topic); }); + + auto& subscriber = _topics.at(id); + subscriber.removeSubscriber(topic); + if (subscriber.getSubscribers().empty()) + { + _topics.erase(id); + } +} + +void +SessionI::subscribeToKey( + int64_t topicId, + int64_t elementId, + const std::shared_ptr& element, + const string& facet, + const shared_ptr& key, + int64_t keyId, + const string& name, + int priority) +{ + assert(_topics.find(topicId) != _topics.end()); + auto& subscriber = _topics.at(topicId).getSubscriber(element->getTopic()); + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": subscribed `e" << elementId << ":[k" << keyId << "]@" << topicId << "' to `" << element << "'"; + if (!facet.empty()) + { + out << " (facet=" << facet << ')'; + } + } + + subscriber.add(elementId, name, priority)->addSubscriber(element, key, facet, _sessionInstanceId); + + auto& p = subscriber.keys[keyId]; + if (!p.first) + { + p.first = key; + } + ++p.second[elementId]; +} + +void +SessionI::unsubscribeFromKey( + int64_t topicId, + int64_t elementId, + const std::shared_ptr& element, + int64_t keyId) +{ + assert(_topics.find(topicId) != _topics.end()); + auto& subscriber = _topics.at(topicId).getSubscriber(element->getTopic()); + auto k = subscriber.get(elementId); + if (k) + { + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": unsubscribed `e" << elementId << "[k" << keyId << "]@" << topicId << "' from `" << element + << "'"; + } + k->removeSubscriber(element); + if (k->getSubscribers().empty()) + { + subscriber.remove(elementId); + } + } + + auto& p = subscriber.keys[keyId]; + if (--p.second[elementId] == 0) + { + p.second.erase(elementId); + if (p.second.empty()) + { + subscriber.keys.erase(keyId); + } + } +} + +void +SessionI::disconnectFromKey( + int64_t topicId, + int64_t elementId, + const std::shared_ptr& element, + int64_t keyId) +{ + lock_guard lock(_mutex); // Called by DataElementI::destroy + if (!_session) + { + return; + } + + runWithTopic( + topicId, + element->getTopic(), + [&](TopicSubscriber&) { unsubscribeFromKey(topicId, elementId, element, keyId); }); +} + +void +SessionI::subscribeToFilter( + int64_t topicId, + int64_t elementId, + const std::shared_ptr& element, + const string& facet, + const shared_ptr& key, + const string& name, + int priority) +{ + assert(_topics.find(topicId) != _topics.end()); + auto& subscriber = _topics.at(topicId).getSubscriber(element->getTopic()); + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": subscribed `e" << elementId << '@' << topicId << "' to `" << element << "'"; + if (!facet.empty()) + { + out << " (facet=" << facet << ')'; + } + } + subscriber.add(-elementId, name, priority)->addSubscriber(element, key, facet, _sessionInstanceId); +} + +void +SessionI::unsubscribeFromFilter( + int64_t topicId, + int64_t elementId, + const std::shared_ptr& element, + int64_t) +{ + assert(_topics.find(topicId) != _topics.end()); + auto& subscriber = _topics.at(topicId).getSubscriber(element->getTopic()); + auto f = subscriber.get(-elementId); + if (f) + { + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": unsubscribed `e" << elementId << '@' << topicId << "' from `" << element << "'"; + } + f->removeSubscriber(element); + if (f->getSubscribers().empty()) + { + subscriber.remove(-elementId); + } + } +} + +void +SessionI::disconnectFromFilter( + int64_t topicId, + int64_t elementId, + const std::shared_ptr& element, + int64_t filterId) +{ + lock_guard lock(_mutex); // Called by DataElementI::destroy + if (!_session) + { + return; + } + + runWithTopic( + topicId, + element->getTopic(), + [&](TopicSubscriber&) { unsubscribeFromFilter(topicId, elementId, element, filterId); }); +} + +LongLongDict +SessionI::getLastIds(int64_t topicId, int64_t keyId, const std::shared_ptr& element) +{ + LongLongDict lastIds; + auto p = _topics.find(topicId); + if (p != _topics.end()) + { + auto& subscriber = p->second.getSubscriber(element->getTopic()); + for (auto q : subscriber.keys[keyId].second) + { + lastIds.emplace(q.first, subscriber.get(q.first)->getSubscriber(element)->lastId); + } + } + return lastIds; +} + +vector> +SessionI::subscriberInitialized( + int64_t topicId, + int64_t elementId, + const DataSampleSeq& samples, + const shared_ptr& key, + const std::shared_ptr& element) +{ + assert(_topics.find(topicId) != _topics.end()); + auto& subscriber = _topics.at(topicId).getSubscriber(element->getTopic()); + auto e = subscriber.get(elementId); + auto s = e->getSubscriber(element); + assert(s); + + if (_traceLevels->session > 1) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": initialized `" << element << "' from `e" << elementId << '@' << topicId << "'"; + } + s->initialized = true; + s->lastId = samples.empty() ? 0 : samples.back().id; + + vector> samplesI; + samplesI.reserve(samples.size()); + auto sampleFactory = element->getTopic()->getSampleFactory(); + auto keyFactory = element->getTopic()->getKeyFactory(); + for (auto& s : samples) + { + assert((!key && !s.keyValue.empty()) || key == subscriber.keys[s.keyId].first); + + samplesI.push_back(sampleFactory->create( + _id, + e->name, + s.id, + s.event, + key ? key : keyFactory->decode(_instance->getCommunicator(), s.keyValue), + subscriber.tags[s.tag], + s.value, + s.timestamp)); + assert(samplesI.back()->key); + } + return samplesI; +} + +void +SessionI::runWithTopics( + const std::string& name, + vector>& retained, + function&)> fn) +{ + auto topics = getTopics(name); + for (auto topic : topics) + { + retained.push_back(topic); + unique_lock l(topic->getMutex()); + if (topic->isDestroyed()) + { + continue; + } + _topicLock = &l; + fn(topic); + _topicLock = nullptr; + l.unlock(); + } +} + +void +SessionI::runWithTopics(int64_t id, function fn) +{ + auto t = _topics.find(id); + if (t != _topics.end()) + { + for (auto& s : t->second.getSubscribers()) + { + unique_lock l(s.first->getMutex()); + if (s.first->isDestroyed()) + { + continue; + } + _topicLock = &l; + fn(s.first, s.second); + _topicLock = nullptr; + l.unlock(); + } + } +} + +void +SessionI::runWithTopics(int64_t id, function fn) +{ + auto t = _topics.find(id); + if (t != _topics.end()) + { + for (auto& s : t->second.getSubscribers()) + { + if (s.first->isDestroyed()) + { + continue; + } + unique_lock l(s.first->getMutex()); + _topicLock = &l; + fn(s.first, s.second, t->second); + _topicLock = nullptr; + l.unlock(); + } + } +} + +void +SessionI::runWithTopic(int64_t id, TopicI* topic, function fn) +{ + auto t = _topics.find(id); + if (t != _topics.end()) + { + auto p = t->second.getSubscribers().find(topic); + if (p != t->second.getSubscribers().end()) + { + unique_lock l(topic->getMutex()); + if (topic->isDestroyed()) + { + return; + } + _topicLock = &l; + fn(p->second); + _topicLock = nullptr; + l.unlock(); + } + } +} + +SubscriberSessionI::SubscriberSessionI(const std::shared_ptr& parent, optional node) + : SessionI(parent, node) +{ +} + +vector> +SubscriberSessionI::getTopics(const string& name) const +{ + return _instance->getTopicFactory()->getTopicReaders(name); +} + +void +SubscriberSessionI::s(int64_t topicId, int64_t elementId, DataSample s, const Ice::Current& current) +{ + lock_guard lock(_mutex); + if (!_session || current.con != _connection) + { + if (current.con != _connection) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": discarding sample `" << s.id << "' from `e" << elementId << '@' << topicId << "'\n"; + if (_connection) + { + out << current.con->toString() << "\n" << _connection->toString(); + } + else + { + out << ""; + } + } + return; + } + auto now = chrono::system_clock::now(); + runWithTopics( + topicId, + [&](TopicI* topic, TopicSubscriber& subscriber, TopicSubscribers&) + { + auto e = subscriber.get(elementId); + if (e && !e->getSubscribers().empty()) + { + if (_traceLevels->session > 2) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": queuing sample `" << s.id << "[k" << s.keyId << "]' from `e" << elementId << '@' + << topicId << "'"; + if (!current.facet.empty()) + { + out << " facet=" << current.facet; + } + out << " to ["; + for (auto q = e->getSubscribers().begin(); q != e->getSubscribers().end(); ++q) + { + if (q != e->getSubscribers().begin()) + { + out << ", "; + } + out << q->first; + if (!q->second.facet.empty()) + { + out << ":" << q->second.facet; + } + } + out << "]"; + } + + shared_ptr key; + if (s.keyValue.empty()) + { + key = subscriber.keys[s.keyId].first; + } + else + { + key = topic->getKeyFactory()->decode(_instance->getCommunicator(), s.keyValue); + } + assert(key); + + auto impl = topic->getSampleFactory()->create( + _id, + e->name, + + s.id, + s.event, + key, + subscriber.tags[s.tag], + s.value, + s.timestamp); + for (auto& es : e->getSubscribers()) + { + if (es.second.initialized && (s.keyId <= 0 || es.second.keys.find(key) != es.second.keys.end())) + { + es.second.lastId = s.id; + es.first->queue(impl, e->priority, shared_from_this(), current.facet, now, !s.keyValue.empty()); + } + } + } + }); +} + +void +SubscriberSessionI::reconnect(optional node) +{ + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": trying to reconnect session with `" << node->ice_toString() << "'"; + } + _parent->createPublisherSession(node, nullptr, dynamic_pointer_cast(shared_from_this())); +} + +void +SubscriberSessionI::remove() +{ + _parent->removeSubscriberSession(getNode(), dynamic_pointer_cast(shared_from_this()), nullptr); +} + +PublisherSessionI::PublisherSessionI(const std::shared_ptr& parent, optional node) + : SessionI(parent, node) +{ +} + +vector> +PublisherSessionI::getTopics(const string& name) const +{ + return _instance->getTopicFactory()->getTopicWriters(name); +} + +void +PublisherSessionI::reconnect(optional node) +{ + if (_traceLevels->session > 0) + { + Trace out(_traceLevels, _traceLevels->sessionCat); + out << _id << ": trying to reconnect session with `" << node->ice_toString() << "'"; + } + _parent->createSubscriberSession(node, nullptr, dynamic_pointer_cast(shared_from_this())); +} + +void +PublisherSessionI::remove() +{ + _parent->removePublisherSession(getNode(), dynamic_pointer_cast(shared_from_this()), nullptr); +} diff --git a/cpp/src/DataStorm/SessionI.h b/cpp/src/DataStorm/SessionI.h new file mode 100644 index 00000000000..66283e73cfd --- /dev/null +++ b/cpp/src/DataStorm/SessionI.h @@ -0,0 +1,384 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_SESSIONI_H +#define DATASTORM_SESSIONI_H + +#include "DataStorm/Contract.h" +#include "Ice/Ice.h" +#include "NodeI.h" + +#if defined(_MSC_VER) +# pragma warning(push) +# pragma warning(disable : 4456) // ... : declaration of 'identifier' hides previous local declaration +# pragma warning(disable : 4458) // ... : declaration of 'identifier' hides class member +#elif defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wshadow" +#endif + +namespace DataStormI +{ + + class TopicI; + class TopicReaderI; + class TopicWriterI; + class DataElementI; + class Instance; + class TraceLevels; + + class SessionI : virtual public DataStormContract::Session, public std::enable_shared_from_this + { + protected: + struct ElementSubscriber + { + ElementSubscriber(const std::string& facet, const std::shared_ptr& key, int sessionInstanceId) + : facet(facet), + initialized(false), + lastId(0), + sessionInstanceId(sessionInstanceId) + { + keys.insert(key); + } + + const std::string facet; + bool initialized; + std::int64_t lastId; + std::set> keys; + int sessionInstanceId; + }; + + class ElementSubscribers + { + public: + ElementSubscribers(const std::string& name, int priority) + : name(name), + priority(priority), + _sessionInstanceId(0) + { + } + + void addSubscriber( + const std::shared_ptr& element, + const std::shared_ptr& key, + const std::string& facet, + int sessionInstanceId) + { + _sessionInstanceId = sessionInstanceId; + auto p = _subscribers.find(element); + if (p != _subscribers.end()) + { + p->second.keys.insert(key); + p->second.sessionInstanceId = sessionInstanceId; + p->second.initialized = false; + } + else + { + _subscribers.emplace(element, ElementSubscriber(facet, key, sessionInstanceId)); + } + } + + void removeSubscriber(const std::shared_ptr& element) { _subscribers.erase(element); } + + std::map, ElementSubscriber>& getSubscribers() { return _subscribers; } + + ElementSubscriber* getSubscriber(const std::shared_ptr& element) + { + auto p = _subscribers.find(element); + if (p != _subscribers.end()) + { + return &p->second; + } + return 0; + } + + bool reap(int sessionInstanceId) + { + if (_sessionInstanceId != sessionInstanceId) + { + return true; + } + + auto p = _subscribers.begin(); + while (p != _subscribers.end()) + { + if (p->second.sessionInstanceId != sessionInstanceId) + { + _subscribers.erase(p++); + } + else + { + ++p; + } + } + return _subscribers.empty(); + } + + std::string name; + int priority; + + private: + std::map, ElementSubscriber> _subscribers; + int _sessionInstanceId; + }; + + class TopicSubscriber + { + public: + TopicSubscriber(int sessionInstanceId) : sessionInstanceId(sessionInstanceId) {} + + ElementSubscribers* add(std::int64_t id, const std::string& name, int priority) + { + auto p = _elements.find(id); + if (p == _elements.end()) + { + p = _elements.emplace(id, ElementSubscribers(name, priority)).first; + } + return &p->second; + } + + ElementSubscribers* get(std::int64_t id) + { + auto p = _elements.find(id); + if (p == _elements.end()) + { + return 0; + } + return &p->second; + } + + ElementSubscribers remove(std::int64_t id) + { + auto p = _elements.find(id); + if (p != _elements.end()) + { + ElementSubscribers tmp(std::move(p->second)); + _elements.erase(p); + return tmp; + } + return ElementSubscribers("", 0); + } + + std::map& getAll() { return _elements; } + + void reap(int sessionInstanceId) + { + auto p = _elements.begin(); + while (p != _elements.end()) + { + if (p->second.reap(sessionInstanceId)) + { + _elements.erase(p++); + } + else + { + ++p; + } + } + } + + std::map, std::map>> keys; + std::map> tags; + int sessionInstanceId; + + private: + std::map _elements; + }; + + class TopicSubscribers + { + public: + TopicSubscribers() {} + + void addSubscriber(TopicI* topic, int sessionInstanceId) + { + _sessionInstanceId = sessionInstanceId; + auto p = _subscribers.find(topic); + if (p != _subscribers.end()) + { + p->second.sessionInstanceId = sessionInstanceId; + } + else + { + _subscribers.emplace(topic, TopicSubscriber(sessionInstanceId)); + } + } + + TopicSubscriber& getSubscriber(TopicI* topic) + { + assert(_subscribers.find(topic) != _subscribers.end()); + return _subscribers.at(topic); + } + + void removeSubscriber(TopicI* topic) { _subscribers.erase(topic); } + + std::map& getSubscribers() { return _subscribers; } + + bool reap(int sessionInstanceId) + { + if (sessionInstanceId != _sessionInstanceId) + { + return true; + } + + auto p = _subscribers.begin(); + while (p != _subscribers.end()) + { + if (p->second.sessionInstanceId != sessionInstanceId) + { + _subscribers.erase(p++); + } + else + { + ++p; + } + } + return _subscribers.empty(); + } + + private: + std::map _subscribers; + int _sessionInstanceId; + }; + + public: + SessionI(const std::shared_ptr&, std::optional); + void init(std::optional); + + virtual void announceTopics(DataStormContract::TopicInfoSeq, bool, const Ice::Current&) override; + virtual void attachTopic(DataStormContract::TopicSpec, const Ice::Current&) override; + virtual void detachTopic(std::int64_t, const Ice::Current&) override; + + virtual void attachTags(std::int64_t, DataStormContract::ElementInfoSeq, bool, const Ice::Current&) override; + virtual void detachTags(std::int64_t, DataStormContract::LongSeq, const Ice::Current&) override; + + virtual void announceElements(std::int64_t, DataStormContract::ElementInfoSeq, const Ice::Current&) override; + virtual void + attachElements(std::int64_t, DataStormContract::ElementSpecSeq, bool, const Ice::Current&) override; + virtual void + attachElementsAck(std::int64_t, DataStormContract::ElementSpecAckSeq, const Ice::Current&) override; + virtual void detachElements(std::int64_t, DataStormContract::LongSeq, const Ice::Current&) override; + + virtual void initSamples(std::int64_t, DataStormContract::DataSamplesSeq, const Ice::Current&) override; + + virtual void disconnected(const Ice::Current&) override; + + void connected( + std::optional, + const Ice::ConnectionPtr&, + const DataStormContract::TopicInfoSeq&); + bool disconnected(const Ice::ConnectionPtr&, std::exception_ptr); + bool retry(std::optional, std::exception_ptr); + void destroyImpl(const std::exception_ptr&); + + const std::string& getId() const { return _id; } + + Ice::ConnectionPtr getConnection() const; + std::optional getSession() const; + bool checkSession(); + + template std::optional getProxy() const + { + return Ice::uncheckedCast(_proxy); + } + + std::optional getNode() const; + void setNode(std::optional); + + std::unique_lock& getTopicLock() { return *_topicLock; } + + void subscribe(std::int64_t, TopicI*); + void unsubscribe(std::int64_t, TopicI*); + void disconnect(std::int64_t, TopicI*); + + void subscribeToKey( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::string&, + const std::shared_ptr&, + std::int64_t, + const std::string&, + int); + void unsubscribeFromKey(std::int64_t, std::int64_t, const std::shared_ptr&, std::int64_t); + void disconnectFromKey(std::int64_t, std::int64_t, const std::shared_ptr&, std::int64_t); + + void subscribeToFilter( + std::int64_t, + std::int64_t, + const std::shared_ptr&, + const std::string&, + const std::shared_ptr&, + const std::string&, + int); + void unsubscribeFromFilter(std::int64_t, std::int64_t, const std::shared_ptr&, std::int64_t); + void disconnectFromFilter(std::int64_t, std::int64_t, const std::shared_ptr&, std::int64_t); + + DataStormContract::LongLongDict getLastIds(std::int64_t, std::int64_t, const std::shared_ptr&); + std::vector> subscriberInitialized( + std::int64_t, + std::int64_t, + const DataStormContract::DataSampleSeq&, + const std::shared_ptr&, + const std::shared_ptr&); + + protected: + void runWithTopics( + const std::string&, + std::vector>&, + std::function&)>); + void runWithTopics(std::int64_t, std::function); + void runWithTopics(std::int64_t, std::function); + void runWithTopic(std::int64_t, TopicI*, std::function); + + virtual std::vector> getTopics(const std::string&) const = 0; + virtual void reconnect(std::optional) = 0; + virtual void remove() = 0; + + const std::shared_ptr _instance; + std::shared_ptr _traceLevels; + mutable std::mutex _mutex; + std::shared_ptr _parent; + std::string _id; + std::optional _proxy; + std::optional _node; + bool _destroyed; + int _sessionInstanceId; + int _retryCount; + Ice::TimerTaskPtr _retryTask; + + std::map _topics; + std::unique_lock* _topicLock; + + std::optional _session; + Ice::ConnectionPtr _connection; + std::vector)>> _connectedCallbacks; + }; + + class SubscriberSessionI : public SessionI, public DataStormContract::SubscriberSession + { + public: + SubscriberSessionI(const std::shared_ptr&, std::optional); + + virtual void s(std::int64_t, std::int64_t, DataStormContract::DataSample, const Ice::Current&) override; + + private: + virtual std::vector> getTopics(const std::string&) const override; + virtual void reconnect(std::optional) override; + virtual void remove() override; + }; + + class PublisherSessionI : public SessionI, public DataStormContract::PublisherSession + { + public: + PublisherSessionI(const std::shared_ptr&, std::optional); + + private: + virtual std::vector> getTopics(const std::string&) const override; + virtual void reconnect(std::optional) override; + virtual void remove() override; + }; + +} +#endif diff --git a/cpp/src/DataStorm/TimerTaskI.h b/cpp/src/DataStorm/TimerTaskI.h new file mode 100644 index 00000000000..f4397f33fa4 --- /dev/null +++ b/cpp/src/DataStorm/TimerTaskI.h @@ -0,0 +1,27 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORMI_TIMER_TASKI_H +#define DATASTORMI_TIMER_TASKI_H + +#include "Ice/Ice.h" + +#include + +namespace DataStormI +{ + // TODO temporary until we fix https://github.com/zeroc-ice/ice/issues/2877 + class TimerTaskI : public Ice::TimerTask + { + public: + TimerTaskI(std::function task) : _task(std::move(task)) {} + + void runTimerTask() override { _task(); } + + private: + std::function _task; + }; +} + +#endif diff --git a/cpp/src/DataStorm/TopicFactoryI.cpp b/cpp/src/DataStorm/TopicFactoryI.cpp new file mode 100644 index 00000000000..a9faad73c64 --- /dev/null +++ b/cpp/src/DataStorm/TopicFactoryI.cpp @@ -0,0 +1,320 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "TopicFactoryI.h" +#include "Instance.h" +#include "NodeI.h" +#include "NodeSessionManager.h" +#include "TopicI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; + +TopicFactoryI::TopicFactoryI(const shared_ptr& instance) + : _instance(instance), + _traceLevels(instance->getTraceLevels()), + _nextReaderId(0), + _nextWriterId(0) +{ +} + +shared_ptr +TopicFactoryI::createTopicReader( + const string& name, + const shared_ptr& keyFactory, + const shared_ptr& tagFactory, + const shared_ptr& sampleFactory, + const shared_ptr& keyFilterFactories, + const shared_ptr& sampleFilterFactories) +{ + shared_ptr reader; + bool hasWriters; + { + lock_guard lock(_mutex); + reader = make_shared( + shared_from_this(), + keyFactory, + tagFactory, + sampleFactory, + keyFilterFactories, + sampleFilterFactories, + name, + _nextReaderId++); + reader->init(); + _readers[name].push_back(reader); + if (_traceLevels->topic > 0) + { + Trace out(_traceLevels, _traceLevels->topicCat); + out << name << ": created topic reader"; + } + + hasWriters = _writers.find(name) != _writers.end(); + } + + try + { + auto instance = getInstance(); + auto node = instance->getNode(); + auto nodePrx = node->getProxy(); + if (hasWriters) + { + node->createSubscriberSession(nodePrx, nullptr, nullptr); + } + node->getSubscriberForwarder()->announceTopics({{name, {reader->getId()}}}, false); + instance->getNodeSessionManager()->announceTopicReader(name, nodePrx); + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const std::exception&) + { + assert(false); + } + return reader; +} + +shared_ptr +TopicFactoryI::createTopicWriter( + const string& name, + const shared_ptr& keyFactory, + const shared_ptr& tagFactory, + const shared_ptr& sampleFactory, + const shared_ptr& keyFilterFactories, + const shared_ptr& sampleFilterFactories) +{ + shared_ptr writer; + bool hasReaders; + { + lock_guard lock(_mutex); + writer = make_shared( + shared_from_this(), + keyFactory, + tagFactory, + sampleFactory, + keyFilterFactories, + sampleFilterFactories, + name, + _nextWriterId++); + writer->init(); + _writers[name].push_back(writer); + if (_traceLevels->topic > 0) + { + Trace out(_traceLevels, _traceLevels->topicCat); + out << name << ": created topic writer"; + } + + hasReaders = _readers.find(name) != _readers.end(); + } + + try + { + auto instance = getInstance(); + auto node = instance->getNode(); + auto nodePrx = node->getProxy(); + if (hasReaders) + { + node->createPublisherSession(nodePrx, nullptr, nullptr); + } + node->getPublisherForwarder()->announceTopics({{name, {writer->getId()}}}, false); + instance->getNodeSessionManager()->announceTopicWriter(name, nodePrx); + } + catch (const Ice::CommunicatorDestroyedException&) + { + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + } + catch (const std::exception&) + { + assert(false); + } + + return writer; +} + +void +TopicFactoryI::removeTopicReader(const string& name, const shared_ptr& reader) +{ + lock_guard lock(_mutex); + if (_traceLevels->topic > 0) + { + Trace out(_traceLevels, _traceLevels->topicCat); + out << name << ": destroyed topic reader"; + } + auto& readers = _readers[name]; + readers.erase(find(readers.begin(), readers.end(), reader)); + if (readers.empty()) + { + _readers.erase(name); + } +} + +void +TopicFactoryI::removeTopicWriter(const string& name, const shared_ptr& writer) +{ + lock_guard lock(_mutex); + if (_traceLevels->topic > 0) + { + Trace out(_traceLevels, _traceLevels->topicCat); + out << name << ": destroyed topic writer"; + } + auto& writers = _writers[name]; + writers.erase(find(writers.begin(), writers.end(), writer)); + if (writers.empty()) + { + _writers.erase(name); + } +} + +vector> +TopicFactoryI::getTopicReaders(const string& name) const +{ + lock_guard lock(_mutex); + auto p = _readers.find(name); + if (p == _readers.end()) + { + return vector>(); + } + return p->second; +} + +vector> +TopicFactoryI::getTopicWriters(const string& name) const +{ + lock_guard lock(_mutex); + auto p = _writers.find(name); + if (p == _writers.end()) + { + return vector>(); + } + return p->second; +} + +void +TopicFactoryI::createPublisherSession( + const string& topic, + optional publisher, + const Ice::ConnectionPtr& connection) +{ + auto readers = getTopicReaders(topic); + if (!readers.empty()) + { + getInstance()->getNode()->createPublisherSession(publisher, connection, nullptr); + } +} + +void +TopicFactoryI::createSubscriberSession( + const string& topic, + optional subscriber, + const Ice::ConnectionPtr& connection) +{ + auto writers = getTopicWriters(topic); + if (!writers.empty()) + { + getInstance()->getNode()->createSubscriberSession(subscriber, connection, nullptr); + } +} + +DataStormContract::TopicInfoSeq +TopicFactoryI::getTopicReaders() const +{ + lock_guard lock(_mutex); + DataStormContract::TopicInfoSeq readers; + readers.reserve(_readers.size()); + for (const auto& p : _readers) + { + DataStormContract::TopicInfo info; + info.name = p.first; + info.ids.reserve(p.second.size()); + for (const auto& q : p.second) + { + info.ids.push_back(q->getId()); + } + readers.push_back(std::move(info)); + } + return readers; +} + +DataStormContract::TopicInfoSeq +TopicFactoryI::getTopicWriters() const +{ + lock_guard lock(_mutex); + DataStormContract::TopicInfoSeq writers; + writers.reserve(_writers.size()); + for (const auto& p : _writers) + { + DataStormContract::TopicInfo info; + info.name = p.first; + info.ids.reserve(p.second.size()); + for (const auto& q : p.second) + { + info.ids.push_back(q->getId()); + } + writers.push_back(std::move(info)); + } + return writers; +} + +DataStormContract::StringSeq +TopicFactoryI::getTopicReaderNames() const +{ + lock_guard lock(_mutex); + DataStormContract::StringSeq readers; + readers.reserve(_readers.size()); + for (const auto& p : _readers) + { + readers.push_back(p.first); + } + return readers; +} + +DataStormContract::StringSeq +TopicFactoryI::getTopicWriterNames() const +{ + lock_guard lock(_mutex); + DataStormContract::StringSeq writers; + writers.reserve(_writers.size()); + for (const auto& p : _writers) + { + writers.push_back(p.first); + } + return writers; +} + +void +TopicFactoryI::shutdown() const +{ + lock_guard lock(_mutex); + for (const auto& p : _writers) + { + for (const auto& w : p.second) + { + w->shutdown(); + } + } + for (const auto& p : _readers) + { + for (const auto& r : p.second) + { + r->shutdown(); + } + } +} + +Ice::CommunicatorPtr +TopicFactoryI::getCommunicator() const +{ + return getInstance()->getCommunicator(); +} diff --git a/cpp/src/DataStorm/TopicFactoryI.h b/cpp/src/DataStorm/TopicFactoryI.h new file mode 100644 index 00000000000..ca8547d8312 --- /dev/null +++ b/cpp/src/DataStorm/TopicFactoryI.h @@ -0,0 +1,85 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_TOPIC_FACTORYI_H +#define DATASTORM_TOPIC_FACTORYI_H + +#include "DataStorm/Contract.h" +#include "DataStorm/InternalI.h" + +namespace DataStormI +{ + + class Instance; + class TraceLevels; + + class TopicI; + class TopicReaderI; + class TopicWriterI; + + class TopicFactoryI : public TopicFactory, public std::enable_shared_from_this + { + public: + TopicFactoryI(const std::shared_ptr&); + + virtual std::shared_ptr createTopicReader( + const std::string&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&) override; + + virtual std::shared_ptr createTopicWriter( + const std::string&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&) override; + + virtual Ice::CommunicatorPtr getCommunicator() const override; + + void removeTopicReader(const std::string&, const std::shared_ptr&); + void removeTopicWriter(const std::string&, const std::shared_ptr&); + + std::vector> getTopicReaders(const std::string&) const; + std::vector> getTopicWriters(const std::string&) const; + + void createSubscriberSession( + const std::string&, + std::optional, + const Ice::ConnectionPtr&); + void createPublisherSession( + const std::string&, + std::optional, + const Ice::ConnectionPtr&); + + std::shared_ptr getInstance() const + { + auto instance = _instance.lock(); + assert(instance); + return instance; + } + + DataStormContract::TopicInfoSeq getTopicReaders() const; + DataStormContract::TopicInfoSeq getTopicWriters() const; + + DataStormContract::StringSeq getTopicReaderNames() const; + DataStormContract::StringSeq getTopicWriterNames() const; + + void shutdown() const; + + private: + mutable std::mutex _mutex; + std::weak_ptr _instance; + std::shared_ptr _traceLevels; + std::map>> _readers; + std::map>> _writers; + std::int64_t _nextReaderId; + std::int64_t _nextWriterId; + }; + +} +#endif diff --git a/cpp/src/DataStorm/TopicI.cpp b/cpp/src/DataStorm/TopicI.cpp new file mode 100644 index 00000000000..33af53aab9f --- /dev/null +++ b/cpp/src/DataStorm/TopicI.cpp @@ -0,0 +1,1099 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "TopicI.h" +#include "NodeI.h" +#include "SessionI.h" +#include "TopicFactoryI.h" +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; +using namespace DataStormContract; + +namespace +{ + + template vector toSeq(const map& map) + { + vector seq; + seq.reserve(map.size()); + for (const auto& p : map) + { + seq.push_back(p.second); + } + return seq; + } + + int toInt(const string& v, int value = 0) + { + istringstream is(v); + is >> value; + return value; + } + + static Topic::Updater noOpUpdater = + [](const shared_ptr& previous, const shared_ptr& next, const Ice::CommunicatorPtr&) + { next->setValue(previous); }; + + // The always match filter always matches the value, it's used by the any key reader/writer. + class AlwaysMatchFilter : public Filter + { + public: + virtual string toString() const { return "f1:alwaysmatch"; } + + virtual const string& getName() const + { + static string alwaysmatch("alwaysmatch"); + return alwaysmatch; + } + + virtual Ice::ByteSeq encode(const Ice::CommunicatorPtr&) const { return Ice::ByteSeq{}; } + + virtual int64_t getId() const + { + return 1; // 1 is reserved for the match all filter. + } + + virtual bool match(const shared_ptr&) const { return true; } + }; + const auto alwaysMatchFilter = make_shared(); + +} + +TopicI::TopicI( + const weak_ptr& factory, + const shared_ptr& keyFactory, + const shared_ptr& tagFactory, + const shared_ptr& sampleFactory, + const shared_ptr& keyFilterFactories, + const shared_ptr& sampleFilterFactories, + const string& name, + int64_t id) + : _factory(factory), + _keyFactory(keyFactory), + _tagFactory(tagFactory), + _sampleFactory(std::move(sampleFactory)), + _keyFilterFactories(keyFilterFactories), + _sampleFilterFactories(sampleFilterFactories), + _name(name), + _instance(factory.lock()->getInstance()), + _traceLevels(_instance->getTraceLevels()), + _id(id), + _destroyed(false), + _listenerCount(0), + _waiters(0), + _notified(0), + _nextId(0), + _nextFilteredId(0), + _nextSampleId(0) +{ +} + +TopicI::~TopicI() { assert(_destroyed); } + +void +TopicI::init() +{ + auto forwarder = [self = shared_from_this()](Ice::ByteSeq e, const Ice::Current& c) { self->forward(e, c); }; + _forwarder = Ice::uncheckedCast(_instance->getCollocatedForwarder()->add(forwarder)); +} + +string +TopicI::getName() const +{ + return _name; +} + +void +TopicI::destroy() +{ + std::map, std::set>> keyElements; + std::map, std::set>> filteredElements; + { + lock_guard lock(_mutex); + assert(!_destroyed); + _destroyed = true; + try + { + _forwarder->detachTopic(_id); // Must be called before disconnect() + } + catch (const std::exception&) + { + forwarderException(); + } + _keyElements.swap(keyElements); + _filteredElements.swap(filteredElements); + _instance->getCollocatedForwarder()->remove(_forwarder->ice_getIdentity()); + } + disconnect(); +} + +void +TopicI::shutdown() +{ + lock_guard lock(_mutex); + _cond.notify_all(); +} + +TopicSpec +TopicI::getTopicSpec() const +{ + TopicSpec spec; + spec.id = _id; + spec.name = _name; + spec.elements.reserve(_keyElements.size() + _filteredElements.size()); + for (auto k : _keyElements) + { + spec.elements.push_back({k.first->getId(), "", k.first->encode(_instance->getCommunicator())}); + } + for (auto f : _filteredElements) + { + spec.elements.push_back({-f.first->getId(), f.first->getName(), f.first->encode(_instance->getCommunicator())}); + } + spec.tags = getTags(); + return spec; +} + +ElementInfoSeq +TopicI::getTags() const +{ + ElementInfoSeq tags; + tags.reserve(_updaters.size()); + for (auto u : _updaters) + { + tags.push_back({u.first->getId(), "", u.first->encode(_instance->getCommunicator())}); + } + return tags; +} + +ElementSpecSeq +TopicI::getElementSpecs(int64_t topicId, const ElementInfoSeq& infos, const shared_ptr& session) +{ + ElementSpecSeq specs; + for (const auto& info : infos) + { + if (info.id > 0) // Key + { + auto key = _keyFactory->decode(_instance->getCommunicator(), info.value); + auto p = _keyElements.find(key); + if (p != _keyElements.end()) + { + ElementDataSeq elements; + for (auto k : p->second) + { + elements.push_back({k->getId(), k->getConfig(), session->getLastIds(topicId, info.id, k)}); + } + specs.push_back({std::move(elements), key->getId(), "", {}, info.id, ""}); + } + for (auto e : _filteredElements) + { + if (e.first->match(key)) + { + ElementDataSeq elements; + for (auto f : e.second) + { + elements.push_back({f->getId(), f->getConfig(), session->getLastIds(topicId, info.id, f)}); + } + specs.push_back( + {std::move(elements), + -e.first->getId(), + e.first->getName(), + e.first->encode(_instance->getCommunicator()), + info.id, + ""}); + } + } + } + else + { + shared_ptr filter; + if (info.value.empty()) + { + filter = alwaysMatchFilter; + } + else + { + filter = _keyFilterFactories->decode(_instance->getCommunicator(), info.name, info.value); + } + + for (auto e : _keyElements) + { + if (filter->match(e.first)) + { + ElementDataSeq elements; + for (auto k : e.second) + { + elements.push_back({k->getId(), k->getConfig(), session->getLastIds(topicId, info.id, k)}); + } + specs.push_back( + {std::move(elements), + e.first->getId(), + "", + e.first->encode(_instance->getCommunicator()), + info.id, + info.name}); + } + } + + if (filter == alwaysMatchFilter) + { + for (auto e : _filteredElements) + { + ElementDataSeq elements; + for (auto f : e.second) + { + elements.push_back({f->getId(), f->getConfig(), session->getLastIds(topicId, info.id, f)}); + } + specs.push_back( + {std::move(elements), + -e.first->getId(), + e.first->getName(), + e.first->encode(_instance->getCommunicator()), + info.id, + info.name}); + } + } + else + { + auto p = _filteredElements.find(alwaysMatchFilter); + if (p != _filteredElements.end()) + { + ElementDataSeq elements; + for (auto f : p->second) + { + elements.push_back({f->getId(), f->getConfig(), session->getLastIds(topicId, info.id, f)}); + } + specs.push_back( + {std::move(elements), + -alwaysMatchFilter->getId(), + alwaysMatchFilter->getName(), + alwaysMatchFilter->encode(_instance->getCommunicator()), + info.id, + info.name}); + } + } + } + } + return specs; +} + +void +TopicI::attach(int64_t id, const shared_ptr& session, optional prx) +{ + auto p = _listeners.find({session}); + if (p == _listeners.end()) + { + p = _listeners.emplace(ListenerKey{session}, Listener(prx)).first; + } + + if (p->second.topics.insert(id).second) + { + session->subscribe(id, this); + } +} + +void +TopicI::detach(int64_t id, const shared_ptr& session) +{ + auto p = _listeners.find({session}); + if (p != _listeners.end() && p->second.topics.erase(id)) + { + session->unsubscribe(id, this); + if (p->second.topics.empty()) + { + _listeners.erase(p); + } + } +} + +ElementSpecAckSeq +TopicI::attachElements( + int64_t topicId, + const ElementSpecSeq& elements, + const shared_ptr& session, + optional prx, + const chrono::time_point& now) +{ + ElementSpecAckSeq specs; + for (const auto& spec : elements) + { + if (spec.peerId > 0) // Key + { + auto key = _keyFactory->get(spec.peerId); + auto p = _keyElements.find(key); + if (p != _keyElements.end()) + { + shared_ptr filter; + if (spec.id < 0) // Filter + { + if (spec.value.empty()) + { + filter = alwaysMatchFilter; + } + else + { + filter = _keyFilterFactories->decode(_instance->getCommunicator(), spec.name, spec.value); + } + } + for (auto e : p->second) + { + ElementDataAckSeq acks; + for (const auto& data : spec.elements) + { + if (spec.id > 0) // Key + { + e->attach(topicId, spec.id, key, nullptr, session, prx, data, now, acks); + } + else if (filter->match(key)) + { + e->attach(topicId, spec.id, key, filter, session, prx, data, now, acks); + } + } + if (!acks.empty()) + { + specs.push_back( + {std::move(acks), + key->getId(), + "", + spec.id < 0 ? key->encode(_instance->getCommunicator()) : ByteSeq(), + spec.id, + spec.name}); + } + } + } + } + else + { + shared_ptr filter; + if (spec.peerId == -1) + { + filter = alwaysMatchFilter; + } + else + { + filter = _keyFilterFactories->get(spec.peerName, -spec.peerId); + } + + auto p = _filteredElements.find(filter); + if (p != _filteredElements.end()) + { + shared_ptr key; + if (spec.id > 0) // Key + { + key = _keyFactory->decode(_instance->getCommunicator(), spec.value); + } + + for (auto e : p->second) + { + ElementDataAckSeq acks; + for (const auto& data : spec.elements) + { + if (spec.id < 0) // Filter + { + e->attach(topicId, spec.id, nullptr, filter, session, prx, data, now, acks); + } + else if (filter->match(key)) + { + e->attach(topicId, spec.id, key, filter, session, prx, data, now, acks); + } + } + if (!acks.empty()) + { + specs.push_back( + {std::move(acks), + -filter->getId(), + filter->getName(), + spec.id > 0 ? filter->encode(_instance->getCommunicator()) : ByteSeq(), + spec.id, + spec.name}); + } + } + } + } + } + return specs; +} + +DataSamplesSeq +TopicI::attachElementsAck( + int64_t topicId, + const ElementSpecAckSeq& elements, + const shared_ptr& session, + const optional prx, + const chrono::time_point& now, + LongSeq& removedIds) +{ + DataSamplesSeq samples; + vector> initCallbacks; + for (const auto& spec : elements) + { + if (spec.peerId > 0) // Key + { + auto key = _keyFactory->get(spec.peerId); + auto p = _keyElements.find(key); + if (p != _keyElements.end()) + { + shared_ptr filter; + if (spec.id < 0) + { + if (spec.value.empty()) + { + filter = alwaysMatchFilter; + } + else + { + filter = _keyFilterFactories->decode(_instance->getCommunicator(), spec.name, spec.value); + } + } + + vector> samplesI; + for (const auto& data : spec.elements) + { + bool found = false; + for (auto e : p->second) + { + if (data.peerId == e->getId()) + { + function initCb; + if (spec.id > 0) // Key + { + initCb = e->attach(topicId, spec.id, key, nullptr, session, prx, data, now, samples); + } + else if (filter->match(key)) // Filter + { + initCb = e->attach(topicId, spec.id, key, filter, session, prx, data, now, samples); + } + if (initCb) + { + initCallbacks.push_back(initCb); + } + found = true; + break; + } + } + if (!found) + { + removedIds.push_back(data.peerId); + } + } + } + else + { + for (const auto& data : spec.elements) + { + removedIds.push_back(data.peerId); + } + } + } + else // Filter + { + shared_ptr filter; + if (spec.peerId == -1) + { + filter = alwaysMatchFilter; + } + else + { + filter = _keyFilterFactories->get(spec.peerName, -spec.peerId); + } + + auto p = _filteredElements.find(filter); + if (p != _filteredElements.end()) + { + shared_ptr key; + if (spec.id > 0) // Key + { + key = _keyFactory->decode(_instance->getCommunicator(), spec.value); + } + + for (const auto& data : spec.elements) + { + bool found = false; + for (auto e : p->second) + { + if (data.peerId == e->getId()) + { + function initCb; + if (spec.id < 0) // Filter + { + initCb = e->attach(topicId, spec.id, nullptr, filter, session, prx, data, now, samples); + } + else if (filter->match(key)) + { + initCb = e->attach(topicId, spec.id, key, nullptr, session, prx, data, now, samples); + } + if (initCb) + { + initCallbacks.push_back(initCb); + } + found = true; + break; + } + } + if (!found) + { + removedIds.push_back(-data.peerId); + } + } + } + else + { + for (const auto& data : spec.elements) + { + removedIds.push_back(-data.peerId); + } + } + } + } + + // + // Initialize samples on data elements once all the elements have been attached. This is + // important for the priority configuration in case 2 writers with different priorities are + // attached from the same session. + // + for (auto initCb : initCallbacks) + { + initCb(); + } + return samples; +} + +void +TopicI::setUpdater(const shared_ptr& tag, Updater updater) +{ + unique_lock lock(_mutex); + if (updater) + { + _updaters[tag] = updater; + try + { + _forwarder->attachTags(_id, {{tag->getId(), "", tag->encode(_instance->getCommunicator())}}, false); + } + catch (const std::exception&) + { + forwarderException(); + } + } + else + { + _updaters.erase(tag); + try + { + _forwarder->detachTags(_id, {tag->getId()}); + } + catch (const std::exception&) + { + forwarderException(); + } + } +} + +const Topic::Updater& +TopicI::getUpdater(const shared_ptr& tag) const +{ + // Called with mutex locked + auto p = _updaters.find(tag); + if (p != _updaters.end()) + { + return p->second; + } + return noOpUpdater; +} + +void +TopicI::setUpdaters(map, Updater> updaters) +{ + unique_lock lock(_mutex); + _updaters = std::move(updaters); +} + +map, Topic::Updater> +TopicI::getUpdaters() const +{ + unique_lock lock(_mutex); + return _updaters; +} + +void +TopicI::incListenerCount(const shared_ptr& session) +{ + ++_listenerCount; + notifyListenerWaiters(session->getTopicLock()); +} + +void +TopicI::decListenerCount(const shared_ptr& session) +{ + --_listenerCount; + notifyListenerWaiters(session->getTopicLock()); +} + +void +TopicI::decListenerCount(size_t listenerCount) +{ + _listenerCount -= listenerCount; +} + +void +TopicI::removeFiltered(const shared_ptr& element, const shared_ptr& filter) +{ + auto p = _filteredElements.find(filter); + if (p != _filteredElements.end()) + { + p->second.erase(element); + if (p->second.empty()) + { + _filteredElements.erase(p); + } + } +} + +void +TopicI::remove(const shared_ptr& element, const vector>& keys) +{ + if (keys.empty()) + { + removeFiltered(element, alwaysMatchFilter); + return; + } + + for (auto key : keys) + { + auto p = _keyElements.find(key); + if (p != _keyElements.end()) + { + p->second.erase(element); + if (p->second.empty()) + { + _keyElements.erase(p); + } + } + } +} + +void +TopicI::waitForListeners(int count) const +{ + unique_lock lock(_mutex); + ++_waiters; + while (true) + { + _instance->checkShutdown(); + if (count < 0 && _listenerCount == 0) + { + --_waiters; + return; + } + else if (count >= 0 && _listenerCount >= static_cast(count)) + { + --_waiters; + return; + } + _cond.wait(lock); + ++_notified; + } +} + +bool +TopicI::hasListeners() const +{ + unique_lock lock(_mutex); + return _listenerCount > 0; +} + +void +TopicI::notifyListenerWaiters(unique_lock& lock) const +{ + if (_waiters > 0) + { + _notified = 0; + _cond.notify_all(); + _cond.wait(lock, [&]() { return _notified < _waiters; }); // Wait until all the waiters are notified. + } +} + +void +TopicI::disconnect() +{ + map listeners; + { + unique_lock lock(_mutex); + listeners.swap(_listeners); + } + for (auto s : listeners) + { + for (auto t : s.second.topics) + { + s.first.session->disconnect(t, this); + } + } + +#ifndef NDEBUG + { + unique_lock lock(_mutex); + assert(_listenerCount == 0); + } +#endif +} + +void +TopicI::forward(const Ice::ByteSeq& inEncaps, const Ice::Current& current) const +{ + // Forwarder proxy must be called with the mutex locked! + for (const auto& listener : _listeners) + { + // TODO check return value + auto _ = listener.second.proxy->ice_invokeAsync(current.operation, current.mode, inEncaps, current.ctx); + } +} + +void +TopicI::forwarderException() const +{ + try + { + rethrow_exception(current_exception()); + } + catch (const Ice::CommunicatorDestroyedException&) + { + // Ignore + } + catch (const Ice::ObjectAdapterDeactivatedException&) + { + // Ignore + } + catch (const Ice::ObjectAdapterDestroyedException&) + { + // Ignore + } +} + +void +TopicI::add(const shared_ptr& element, const vector>& keys) +{ + if (keys.empty()) + { + addFiltered(element, alwaysMatchFilter); + return; + } + + ElementInfoSeq infos; + for (const auto& key : keys) + { + auto p = _keyElements.find(key); + if (p == _keyElements.end()) + { + p = _keyElements.emplace(key, set>()).first; + } + assert(element); + infos.push_back({key->getId(), "", key->encode(_instance->getCommunicator())}); + p->second.insert(element); + } + if (!infos.empty()) + { + try + { + _forwarder->announceElements(_id, infos); + } + catch (const std::exception&) + { + forwarderException(); + } + } +} + +void +TopicI::addFiltered(const shared_ptr& element, const shared_ptr& filter) +{ + auto p = _filteredElements.find(filter); + if (p == _filteredElements.end()) + { + p = _filteredElements.emplace(filter, set>()).first; + } + assert(element); + p->second.insert(element); + try + { + _forwarder->announceElements( + _id, + {{-filter->getId(), filter->getName(), filter->encode(_instance->getCommunicator())}}); + } + catch (const std::exception&) + { + forwarderException(); + } +} + +void +TopicI::parseConfigImpl(const Ice::PropertyDict& properties, const string& prefix, DataStorm::Config& config) const +{ + // Override defaults with properties + auto p = properties.find(prefix + ".SampleLifetime"); + if (p != properties.end()) + { + config.sampleLifetime = toInt(p->second); + } + p = properties.find(prefix + ".SampleCount"); + if (p != properties.end()) + { + config.sampleCount = toInt(p->second); + } + p = properties.find(prefix + ".ClearHistory"); + if (p != properties.end()) + { + if (p->second == "OnAdd") + { + config.clearHistory = DataStorm::ClearHistoryPolicy::OnAdd; + } + else if (p->second == "OnRemove") + { + config.clearHistory = DataStorm::ClearHistoryPolicy::OnRemove; + } + else if (p->second == "OnAll") + { + config.clearHistory = DataStorm::ClearHistoryPolicy::OnAll; + } + else if (p->second == "OnAllExceptPartialUpdate") + { + config.clearHistory = DataStorm::ClearHistoryPolicy::OnAllExceptPartialUpdate; + } + else if (p->second == "Never") + { + config.clearHistory = DataStorm::ClearHistoryPolicy::Never; + } + } +} + +TopicReaderI::TopicReaderI( + const shared_ptr& factory, + const shared_ptr& keyFactory, + const shared_ptr& tagFactory, + const shared_ptr& sampleFactory, + const shared_ptr& keyFilterFactories, + const shared_ptr& sampleFilterFactories, + const string& name, + int64_t id) + : TopicI(factory, keyFactory, tagFactory, sampleFactory, keyFilterFactories, sampleFilterFactories, name, id) +{ + _defaultConfig = {-1, 0, DataStorm::ClearHistoryPolicy::OnAll, DataStorm::DiscardPolicy::None}; + _defaultConfig = mergeConfigs(parseConfig("DataStorm.Topic")); +} + +shared_ptr +TopicReaderI::createFiltered( + const shared_ptr& filter, + const string& name, + DataStorm::ReaderConfig config, + const string& sampleFilterName, + Ice::ByteSeq sampleFilterCriteria) +{ + lock_guard lock(_mutex); + auto element = make_shared( + this, + name, + ++_nextFilteredId, + filter, + sampleFilterName, + std::move(sampleFilterCriteria), + mergeConfigs(std::move(config))); + element->init(); + addFiltered(element, filter); + return element; +} + +shared_ptr +TopicReaderI::create( + const vector>& keys, + const string& name, + DataStorm::ReaderConfig config, + const string& sampleFilterName, + Ice::ByteSeq sampleFilterCriteria) +{ + lock_guard lock(_mutex); + auto element = make_shared( + this, + name, + ++_nextId, + keys, + sampleFilterName, + std::move(sampleFilterCriteria), + mergeConfigs(std::move(config))); + element->init(); + add(element, keys); + return element; +} + +void +TopicReaderI::setDefaultConfig(DataStorm::ReaderConfig config) +{ + lock_guard lock(_mutex); + _defaultConfig = mergeConfigs(std::move(config)); +} + +void +TopicReaderI::waitForWriters(int count) const +{ + waitForListeners(count); +} + +bool +TopicReaderI::hasWriters() const +{ + return hasListeners(); +} + +void +TopicReaderI::destroy() +{ + TopicI::destroy(); + + auto factory = _factory.lock(); + if (factory) + { + factory->removeTopicReader(_name, shared_from_this()); + } +} + +DataStorm::ReaderConfig +TopicReaderI::parseConfig(const string& prefix) const +{ + DataStorm::ReaderConfig config; + auto properties = _instance->getCommunicator()->getProperties()->getPropertiesForPrefix(prefix); + parseConfigImpl(properties, prefix, config); + auto p = properties.find(prefix + ".DiscardPolicy"); + if (p != properties.end()) + { + if (p->second == "None") + { + config.discardPolicy = DataStorm::DiscardPolicy::None; + } + else if (p->second == "SendTime") + { + config.discardPolicy = DataStorm::DiscardPolicy::SendTime; + } + else if (p->second == "SendTime") + { + config.discardPolicy = DataStorm::DiscardPolicy::Priority; + } + } + return config; +} + +DataStorm::ReaderConfig +TopicReaderI::mergeConfigs(DataStorm::ReaderConfig config) const +{ + if (!config.sampleCount && _defaultConfig.sampleCount) + { + config.sampleCount = _defaultConfig.sampleCount; + } + if (!config.sampleLifetime && _defaultConfig.sampleLifetime) + { + config.sampleLifetime = _defaultConfig.sampleLifetime; + } + if (!config.clearHistory && _defaultConfig.clearHistory) + { + config.clearHistory = _defaultConfig.clearHistory; + } + if (!config.discardPolicy && _defaultConfig.discardPolicy) + { + config.discardPolicy = _defaultConfig.discardPolicy; + } + return config; +} + +TopicWriterI::TopicWriterI( + const shared_ptr& factory, + const shared_ptr& keyFactory, + const shared_ptr& tagFactory, + const shared_ptr& sampleFactory, + const shared_ptr& keyFilterFactories, + const shared_ptr& sampleFilterFactories, + const string& name, + int64_t id) + : TopicI(factory, keyFactory, tagFactory, sampleFactory, keyFilterFactories, sampleFilterFactories, name, id) +{ + _defaultConfig = {-1, 0, DataStorm::ClearHistoryPolicy::OnAll}; + _defaultConfig = mergeConfigs(parseConfig("DataStorm.Topic")); +} + +shared_ptr +TopicWriterI::create(const vector>& keys, const string& name, DataStorm::WriterConfig config) +{ + lock_guard lock(_mutex); + auto element = make_shared(this, name, ++_nextId, keys, mergeConfigs(std::move(config))); + element->init(); + add(element, keys); + return element; +} + +void +TopicWriterI::setDefaultConfig(DataStorm::WriterConfig config) +{ + lock_guard lock(_mutex); + _defaultConfig = mergeConfigs(std::move(config)); +} + +void +TopicWriterI::waitForReaders(int count) const +{ + waitForListeners(count); +} + +bool +TopicWriterI::hasReaders() const +{ + return hasListeners(); +} + +void +TopicWriterI::destroy() +{ + TopicI::destroy(); + + auto factory = _factory.lock(); + if (factory) + { + factory->removeTopicWriter(_name, shared_from_this()); + } +} + +DataStorm::WriterConfig +TopicWriterI::parseConfig(const string& prefix) const +{ + DataStorm::WriterConfig config; + auto properties = _instance->getCommunicator()->getProperties()->getPropertiesForPrefix(prefix); + parseConfigImpl(properties, prefix, config); + auto p = properties.find(prefix + ".Priority"); + if (p != properties.end()) + { + istringstream is(p->second); + int priority; + is >> priority; + config.priority = priority; + } + return config; +} + +DataStorm::WriterConfig +TopicWriterI::mergeConfigs(DataStorm::WriterConfig config) const +{ + if (!config.sampleCount && _defaultConfig.sampleCount) + { + config.sampleCount = _defaultConfig.sampleCount; + } + if (!config.sampleLifetime && _defaultConfig.sampleLifetime) + { + config.sampleLifetime = _defaultConfig.sampleLifetime; + } + if (!config.clearHistory && _defaultConfig.clearHistory) + { + config.clearHistory = _defaultConfig.clearHistory; + } + if (!config.priority && _defaultConfig.priority) + { + config.priority = _defaultConfig.priority; + } + return config; +} diff --git a/cpp/src/DataStorm/TopicI.h b/cpp/src/DataStorm/TopicI.h new file mode 100644 index 00000000000..75493837304 --- /dev/null +++ b/cpp/src/DataStorm/TopicI.h @@ -0,0 +1,225 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_TOPICI_H +#define DATASTORM_TOPICI_H + +#include "DataElementI.h" +#include "DataStorm/InternalI.h" +#include "DataStorm/Types.h" +#include "ForwarderManager.h" +#include "Instance.h" + +namespace DataStormI +{ + + class SessionI; + class TopicFactoryI; + + class TopicI : virtual public Topic, public std::enable_shared_from_this + { + struct ListenerKey + { + std::shared_ptr session; + + bool operator<(const ListenerKey& other) const { return session < other.session; } + }; + + struct Listener + { + Listener(std::optional proxy) : proxy(std::move(proxy)) {} + + std::set topics; + std::optional proxy; + }; + + public: + TopicI( + const std::weak_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::string&, + std::int64_t); + + virtual ~TopicI(); + + void init(); + + virtual std::string getName() const override; + virtual void destroy() override; + + void shutdown(); + + const std::shared_ptr& getInstance() const { return _instance; } + + DataStormContract::TopicSpec getTopicSpec() const; + DataStormContract::ElementInfoSeq getTags() const; + DataStormContract::ElementSpecSeq + getElementSpecs(std::int64_t, const DataStormContract::ElementInfoSeq&, const std::shared_ptr&); + + void attach(std::int64_t, const std::shared_ptr&, std::optional); + void detach(std::int64_t, const std::shared_ptr&); + + DataStormContract::ElementSpecAckSeq attachElements( + std::int64_t, + const DataStormContract::ElementSpecSeq&, + const std::shared_ptr&, + std::optional, + const std::chrono::time_point&); + + DataStormContract::DataSamplesSeq attachElementsAck( + std::int64_t, + const DataStormContract::ElementSpecAckSeq&, + const std::shared_ptr&, + std::optional, + const std::chrono::time_point&, + DataStormContract::LongSeq&); + + virtual void setUpdater(const std::shared_ptr&, Updater) override; + const Updater& getUpdater(const std::shared_ptr&) const; + + virtual void setUpdaters(std::map, Updater>) override; + virtual std::map, Updater> getUpdaters() const override; + + bool isDestroyed() const { return _destroyed; } + + std::int64_t getId() const { return _id; } + + std::mutex& getMutex() { return _mutex; } + + const std::shared_ptr& getKeyFactory() const { return _keyFactory; } + + const std::shared_ptr& getTagFactory() const { return _tagFactory; } + + const std::shared_ptr& getSampleFactory() const { return _sampleFactory; } + + const std::shared_ptr& getSampleFilterFactories() const { return _sampleFilterFactories; } + + void incListenerCount(const std::shared_ptr&); + void decListenerCount(const std::shared_ptr&); + void decListenerCount(size_t); + + void removeFiltered(const std::shared_ptr&, const std::shared_ptr&); + void remove(const std::shared_ptr&, const std::vector>&); + + protected: + void waitForListeners(int count) const; + bool hasListeners() const; + void notifyListenerWaiters(std::unique_lock&) const; + + void disconnect(); + + void forward(const Ice::ByteSeq&, const Ice::Current&) const; + void forwarderException() const; + + void add(const std::shared_ptr&, const std::vector>&); + void addFiltered(const std::shared_ptr&, const std::shared_ptr&); + + void parseConfigImpl(const Ice::PropertyDict&, const std::string&, DataStorm::Config&) const; + + friend class DataElementI; + friend class DataReaderI; + friend class FilteredDataReaderI; + friend class DataWriterI; + friend class KeyDataWriterI; + friend class KeyDataReaderI; + + const std::weak_ptr _factory; + const std::shared_ptr _keyFactory; + const std::shared_ptr _tagFactory; + const std::shared_ptr _sampleFactory; + const std::shared_ptr _keyFilterFactories; + const std::shared_ptr _sampleFilterFactories; + const std::string _name; + const std::shared_ptr _instance; + const std::shared_ptr _traceLevels; + const std::int64_t _id; + std::optional _forwarder; + + mutable std::mutex _mutex; + mutable std::condition_variable _cond; + bool _destroyed; + std::map, std::set>> _keyElements; + std::map, std::set>> _filteredElements; + std::map _listeners; + std::map, Updater> _updaters; + size_t _listenerCount; + mutable size_t _waiters; + mutable size_t _notified; + std::int64_t _nextId; + std::int64_t _nextFilteredId; + std::int64_t _nextSampleId; + }; + + class TopicReaderI : public TopicReader, public TopicI + { + public: + TopicReaderI( + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::string&, + std::int64_t); + + virtual std::shared_ptr createFiltered( + const std::shared_ptr&, + const std::string&, + DataStorm::ReaderConfig, + const std::string&, + Ice::ByteSeq) override; + virtual std::shared_ptr create( + const std::vector>&, + const std::string&, + DataStorm::ReaderConfig, + const std::string&, + Ice::ByteSeq) override; + + virtual void setDefaultConfig(DataStorm::ReaderConfig) override; + virtual void waitForWriters(int) const override; + virtual bool hasWriters() const override; + virtual void destroy() override; + + private: + DataStorm::ReaderConfig parseConfig(const std::string&) const; + DataStorm::ReaderConfig mergeConfigs(DataStorm::ReaderConfig) const; + + DataStorm::ReaderConfig _defaultConfig; + }; + + class TopicWriterI : public TopicWriter, public TopicI + { + public: + TopicWriterI( + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::shared_ptr&, + const std::string&, + std::int64_t); + + virtual std::shared_ptr + create(const std::vector>&, const std::string&, DataStorm::WriterConfig) override; + + virtual void setDefaultConfig(DataStorm::WriterConfig) override; + virtual void waitForReaders(int) const override; + virtual bool hasReaders() const override; + virtual void destroy() override; + + private: + DataStorm::WriterConfig parseConfig(const std::string&) const; + DataStorm::WriterConfig mergeConfigs(DataStorm::WriterConfig) const; + + DataStorm::WriterConfig _defaultConfig; + }; + +} +#endif diff --git a/cpp/src/DataStorm/TraceUtil.cpp b/cpp/src/DataStorm/TraceUtil.cpp new file mode 100644 index 00000000000..de724ec4d33 --- /dev/null +++ b/cpp/src/DataStorm/TraceUtil.cpp @@ -0,0 +1,24 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "TraceUtil.h" + +using namespace std; +using namespace DataStormI; + +TraceLevels::TraceLevels(Ice::CommunicatorPtr communicator) + : topic(0), + topicCat("Topic"), + data(0), + dataCat("Data"), + session(0), + sessionCat("Session"), + logger(communicator->getLogger()) +{ + auto properties = communicator->getProperties(); + const string keyBase = "DataStorm.Trace."; + const_cast(topic) = properties->getPropertyAsInt(keyBase + topicCat); + const_cast(data) = properties->getPropertyAsInt(keyBase + dataCat); + const_cast(session) = properties->getPropertyAsInt(keyBase + sessionCat); +} diff --git a/cpp/src/DataStorm/TraceUtil.h b/cpp/src/DataStorm/TraceUtil.h new file mode 100644 index 00000000000..014209c4443 --- /dev/null +++ b/cpp/src/DataStorm/TraceUtil.h @@ -0,0 +1,255 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#ifndef DATASTORM_TRACE_UTIL_H +#define DATASTORM_TRACE_UTIL_H + +#include "DataElementI.h" +#include "DataStorm/InternalI.h" +#include "Ice/Ice.h" +#include "SessionI.h" +#include "TopicI.h" + +namespace std +{ + + template inline std::ostream& operator<<(std::ostream& os, const std::vector& p) + { + if (!p.empty()) + { + for (auto q = p.begin(); q != p.end(); ++q) + { + if (q != p.begin()) + { + os << ", "; + } + os << *q; + } + } + return os; + } + + template inline std::ostream& operator<<(std::ostream& os, const std::map& p) + { + if (!p.empty()) + { + for (auto q = p.begin(); q != p.end(); ++q) + { + if (q != p.begin()) + { + os << ", "; + } + os << q->first << "=" << q->second; + } + } + return os; + } + +} + +namespace Ice +{ + + inline std::ostream& operator<<(std::ostream& os, const Ice::Identity& id) + { + return os << (id.category.empty() ? "" : id.category + "/") << id.name; + } + +} + +namespace DataStormContract +{ + + inline std::string valueIdToString(std::int64_t valueId) + { + std::ostringstream os; + if (valueId < 0) + { + os << "f" << -valueId; + } + else + { + os << "k" << valueId; + } + return os.str(); + } + + inline std::ostream& operator<<(std::ostream& os, const ElementInfo& info) + { + os << valueIdToString(info.id); + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const ElementData& data) + { + os << 'e' << data.id; + if (data.config && data.config->facet) + { + os << ':' << *data.config->facet; + } + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const ElementSpec& spec) + { + for (auto q = spec.elements.begin(); q != spec.elements.end(); ++q) + { + if (q != spec.elements.begin()) + { + os << ','; + } + os << *q << ':' << valueIdToString(spec.id) << ":pv" << valueIdToString(spec.peerId); + } + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const ElementDataAck& data) + { + os << 'e' << data.id << ":pe" << data.peerId; + if (data.config && data.config->facet) + { + os << ':' << *data.config->facet; + } + os << ":sz" << data.samples.size(); + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const ElementSpecAck& spec) + { + for (auto q = spec.elements.begin(); q != spec.elements.end(); ++q) + { + if (q != spec.elements.begin()) + { + os << ','; + } + os << *q << ':' << valueIdToString(spec.id) << ":pv" << valueIdToString(spec.peerId); + } + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const TopicInfo& info) + { + os << "[" << info.ids << "]:" << info.name; + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const TopicSpec& info) + { + os << info.id << ':' << info.name << ":[" << info.elements << "]"; + return os; + } + + inline std::ostream& operator<<(std::ostream& os, const DataSamples& samples) + { + os << 'e' << samples.id << ":sz" << samples.samples.size(); + return os; + } + +} + +namespace DataStormI +{ + + template::value>::type* = nullptr> + inline std::ostream& operator<<(std::ostream& os, const std::shared_ptr& p) + { + os << (p ? p->toString() : ""); + return os; + } + + template< + typename T, + typename ::std::enable_if<::std::is_base_of::value>::type* = nullptr> + inline std::ostream& operator<<(std::ostream& os, T* element) + { + os << (element ? element->toString() : ""); + return os; + } + + template< + typename T, + typename ::std::enable_if<::std::is_base_of::value>::type* = nullptr> + inline std::ostream& operator<<(std::ostream& os, const std::shared_ptr& element) + { + os << element.get(); + return os; + } + + template::value>::type* = nullptr> + inline std::ostream& operator<<(std::ostream& os, T* session) + { + if (session) + { + Ice::Identity id = session->getNode()->ice_getIdentity(); + os << id.name; + if (!id.category.empty()) + { + os << '/' << id.category; + } + } + else + { + os << ""; + } + return os; + } + + template< + typename T, + typename ::std::enable_if< + ::std::is_base_of::type>::value>::type* = nullptr> + inline std::ostream& operator<<(std::ostream& os, T topic) + { + if (topic) + { + os << topic->getId() << ":" << topic->getName(); + } + else + { + os << ""; + } + return os; + } + + template::value>::type* = nullptr> + inline std::ostream& operator<<(std::ostream& os, const std::shared_ptr& topic) + { + os << topic.get(); + return os; + } + + class TraceLevels + { + public: + TraceLevels(Ice::CommunicatorPtr); + + const int topic; + const char* topicCat; + + const int data; + const char* dataCat; + + const int session; + const char* sessionCat; + + const Ice::LoggerPtr logger; + }; + + class Trace : public Ice::Trace + { + public: + Trace(std::shared_ptr traceLevels, const std::string& category) + : Ice::Trace(traceLevels->logger, category) + { + } + }; + + class Warning : public Ice::Warning + { + public: + Warning(std::shared_ptr traceLevels) : Ice::Warning(traceLevels->logger) {} + }; + +} +#endif diff --git a/cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj b/cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj new file mode 100644 index 00000000000..e973d8b8ad0 --- /dev/null +++ b/cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj @@ -0,0 +1,270 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {BD2AC148-422F-49E0-9185-ACCBA7AECF26} + + + + DynamicLibrary + true + $(DefaultPlatformToolset) + + + DynamicLibrary + false + $(DefaultPlatformToolset) + + + DynamicLibrary + true + $(DefaultPlatformToolset) + + + DynamicLibrary + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + DATASTORM_API_EXPORTS;%(PreprocessorDefinitions) + $(Platform)\$(Configuration);%(AdditionalIncludeDirectories) + + + %(AdditionalDependencies) + + + + + DATASTORM_API_EXPORTS;%(PreprocessorDefinitions) + $(Platform)\$(Configuration);%(AdditionalIncludeDirectories) + + + %(AdditionalDependencies) + + + + + DATASTORM_API_EXPORTS;%(PreprocessorDefinitions) + $(Platform)\$(Configuration);%(AdditionalIncludeDirectories) + + + %(AdditionalDependencies) + + + + + DATASTORM_API_EXPORTS;%(PreprocessorDefinitions) + $(Platform)\$(Configuration);%(AdditionalIncludeDirectories) + + + %(AdditionalDependencies) + + + + + $(Platform)\$(Configuration) + $(IceSrcRootDir)\include\generated\$(Platform)\$(Configuration)\DataStorm + DataStorm + + + + + + + + + + + + + + + + + + + + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + + + + + + + + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + true + true + true + ..\..\..\..\..\slice\DataStorm\Sample.ice + + + + + + + + + + + + + + + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\Contract.ice + + + true + true + true + ..\..\Contract.ice + + + + + + $(Platform)\$(Configuration)\DataStorm + + + + + Designer + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj.filters b/cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj.filters new file mode 100644 index 00000000000..ac542688d75 --- /dev/null +++ b/cpp/src/DataStorm/msbuild/datastorm/datastorm.vcxproj.filters @@ -0,0 +1,230 @@ + + + + + {da1f64c2-3af6-4ad6-9be4-651fcf51e176} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {E6261236-F264-49AD-97FE-C4A2103E10C4} + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + {6CF148F1-D3A5-45BD-A9B8-6E1E209A8E92} + ice + + + {6dc90333-23f5-4ed6-b6d0-48df24ec0545} + h + + + {e7499dd2-5f0e-49bc-b32a-8e8b9a321763} + + + {e12d6501-6091-4eb6-971d-e803fc2ea3bc} + + + {e384c485-17c6-4b18-9663-999bcea9bb47} + + + {62d1e5e7-deb6-47f0-a6bc-681bd24288fb} + + + {85607007-5e31-479f-a1e4-4dd811576f3e} + + + {3aaeea7e-9693-410c-b276-169476ff8817} + + + {aa92b0d4-1476-4c82-bc8d-2a53fab43cd8} + + + {eaaf90ac-2b23-4311-96ea-110f4dfb1aeb} + + + {1f715f2e-ed49-497e-9476-5df658443a74} + + + {1a41548b-3a42-4cca-8612-a6f88dc89f63} + + + {ae30d32a-f00b-4e29-bd04-d8a87c0c3d2d} + + + {2918b451-9c4f-4b55-9915-1ced485a503b} + + + + + Resource Files + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + Source Files\x64\Release + + + Source Files + + + Source Files + + + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + Header Files + + + Header Files + + + Header Files + + + Header Files + + + + + + + + Slice Files + + + Slice Files + + + \ No newline at end of file diff --git a/cpp/src/DataStorm/msbuild/datastorm/packages.config b/cpp/src/DataStorm/msbuild/datastorm/packages.config new file mode 100644 index 00000000000..2bece8161b2 --- /dev/null +++ b/cpp/src/DataStorm/msbuild/datastorm/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/src/dsnode/Makefile.mk b/cpp/src/dsnode/Makefile.mk new file mode 100644 index 00000000000..fcaf15fdbc0 --- /dev/null +++ b/cpp/src/dsnode/Makefile.mk @@ -0,0 +1,10 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(project)_programs = dsnode + +$(project)_dependencies := DataStorm Ice +$(project)_targetdir := $(bindir) + +projects += $(project) diff --git a/cpp/src/dsnode/Node.cpp b/cpp/src/dsnode/Node.cpp new file mode 100644 index 00000000000..4eb815d2d57 --- /dev/null +++ b/cpp/src/dsnode/Node.cpp @@ -0,0 +1,75 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" + +#include + +using namespace std; + +static void +usage(const string& n) +{ + cerr << "Usage: " << n << " [options]\n"; + cerr << "Options:\n" + "-h, --help Show this message.\n" + "-v, --version Display the DataStorm version.\n"; +} + +int +main(int argc, char* argv[]) +{ + try + { + // + // Parse arguments. + // + for (int i = 0; i < argc; ++i) + { + string arg = argv[i]; + if (arg == "-v" || arg == "--version") + { + cout << ICE_STRING_VERSION << endl; + return 0; + } + else if (arg == "-h" || arg == "--help") + { + usage(argv[0]); + return 0; + } + } + + // + // CtrlCHandler must be created before the node is created or any other threads are started. + // + Ice::CtrlCHandler ctrlCHandler; + + // + // Instantiates node. + // + DataStorm::Node node(argc, argv); + + if (argc > 1) + { + cerr << "unrecognized arguments" << endl; + usage(argv[0]); + return 1; + } + // + // Shutdown the node on Ctrl-C. + // + ctrlCHandler.setCallback([&node](int) { node.shutdown(); }); + + // + // Exit once the user hits Ctrl-C to shutdown the node. + // + node.waitForShutdown(); + } + catch (const std::exception& ex) + { + cerr << ex.what() << endl; + return 1; + } + return 0; +} diff --git a/cpp/src/dsnode/msbuild/dsnode.vcxproj b/cpp/src/dsnode/msbuild/dsnode.vcxproj new file mode 100644 index 00000000000..3ac48804714 --- /dev/null +++ b/cpp/src/dsnode/msbuild/dsnode.vcxproj @@ -0,0 +1,105 @@ + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {85F74421-2B78-448B-91F0-496526EB365F} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + dsnode + + + dsnode + + + dsnode + + + dsnode + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + diff --git a/cpp/src/dsnode/msbuild/dsnode.vcxproj.filters b/cpp/src/dsnode/msbuild/dsnode.vcxproj.filters new file mode 100644 index 00000000000..e6a9cc3bf9a --- /dev/null +++ b/cpp/src/dsnode/msbuild/dsnode.vcxproj.filters @@ -0,0 +1,16 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/api/DuplicateSymbols.cpp b/cpp/test/DataStorm/api/DuplicateSymbols.cpp new file mode 100644 index 00000000000..77201acc3ad --- /dev/null +++ b/cpp/test/DataStorm/api/DuplicateSymbols.cpp @@ -0,0 +1,18 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +#include + +using namespace DataStorm; +using namespace std; + +void +testDuplicateSymbols() +{ + // Make sure these don't cause duplicate symbols errors on link + ostringstream os; + os << SampleEvent::Add; + os << vector(); + Sample* sample = nullptr; + os << *sample; +} diff --git a/cpp/test/DataStorm/api/Makefile.mk b/cpp/test/DataStorm/api/Makefile.mk new file mode 100644 index 00000000000..8f004f9246c --- /dev/null +++ b/cpp/test/DataStorm/api/Makefile.mk @@ -0,0 +1,10 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_writer_sources = Writer.cpp DuplicateSymbols.cpp Test.ice + +tests += $(test) diff --git a/cpp/test/DataStorm/api/Test.ice b/cpp/test/DataStorm/api/Test.ice new file mode 100644 index 00000000000..2c8be6b3b5d --- /dev/null +++ b/cpp/test/DataStorm/api/Test.ice @@ -0,0 +1,17 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +module Test +{ + +struct StructKey +{ + int value; +}; + +class ClassKey +{ + int value; +}; + +}; diff --git a/cpp/test/DataStorm/api/Writer.cpp b/cpp/test/DataStorm/api/Writer.cpp new file mode 100644 index 00000000000..9c146288677 --- /dev/null +++ b/cpp/test/DataStorm/api/Writer.cpp @@ -0,0 +1,321 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" + +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; +using namespace Test; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + cout << "testing node..." << flush; + { + Node n; + Node nm(std::move(n)); + auto nm2 = std::move(nm); + nm2.getCommunicator(); + nm2.getSessionConnection("s"); + + Node n2(Ice::initialize()); + n2.getCommunicator()->destroy(); + + auto c = Ice::initialize(); + { + Node n22(c); + } + { + const Ice::CommunicatorPtr& c2 = c; + Node n23(c2); + } + { + const Ice::CommunicatorPtr c3 = c; + Node n24(c3); + } + { + Ice::CommunicatorPtr& c4 = c; + Node n25(c4); + } + c->destroy(); + + Node n3(Ice::InitializationData{}); + + try + { + Node n4(argc, argv, "config.file"); + } + catch (const Ice::FileException&) + { + } + + Node n5(argc, argv, Ice::InitializationData{}); + + { + Node n6; + test(!n6.isShutdown()); + n6.shutdown(); + test(n6.isShutdown()); + n6.waitForShutdown(); + + auto testException = [](function fn) + { + try + { + fn(); + test(false); + } + catch (NodeShutdownException&) + { + } + catch (...) + { + test(false); + } + }; + + Topic t1(n6, "t1"); + testException([&t1]() { t1.waitForWriters(); }); + testException([&t1]() { t1.waitForNoWriters(); }); + testException([&t1]() { t1.waitForReaders(); }); + testException([&t1]() { t1.waitForNoReaders(); }); + + auto writer = makeSingleKeyWriter(t1, 0); + testException([&writer] { writer.waitForReaders(); }); + testException([&writer] { writer.waitForNoReaders(); }); + + auto reader = makeSingleKeyReader(t1, 0); + testException([&reader] { reader.waitForWriters(); }); + testException([&reader] { reader.waitForNoWriters(); }); + testException([&reader] { reader.waitForUnread(); }); + testException([&reader] { reader.getNextUnread(); }); + } + } + cout << "ok" << endl; + + cout << "testing topic... " << flush; + { + Topic t1(node, "t1"); + Topic t2(node, "t2"); + Topic t3(node, "t3"); + + // TODO fix class cloner + // Topic t4(node, "t4"); + + Topic::KeyType k1 = 5; + Topic::ValueType v1("string"); + Topic::UpdateTagType tag1("string"); + + Topic::WriterType* writer = nullptr; + if (writer != nullptr) + { + test(writer->getConnectedKeys()[0] == k1); // Use variables to prevent unused variable warnings + } + Topic::ReaderType* reader = nullptr; + if (reader != nullptr) + { + reader->getConnectedKeys(); + } + + auto tc1 = std::move(t1); + + test(!tc1.hasWriters()); + tc1.waitForWriters(0); + tc1.waitForNoWriters(); + + test(!t2.hasReaders()); + t2.waitForReaders(0); + t2.waitForNoReaders(); + + tc1.setWriterDefaultConfig(WriterConfig()); + t2.setReaderDefaultConfig(ReaderConfig()); + + tc1.setUpdater("test", [](string&, string) {}); + } + cout << "ok" << endl; + + cout << "testing writer... " << flush; + { + Topic topic(node, "topic"); + + auto testWriter = [](Topic::WriterType& writer) + { + writer.hasReaders(); + writer.waitForReaders(0); + writer.waitForNoReaders(); + writer.getConnectedReaders(); + writer.getConnectedKeys(); + test(writer.getAll().empty()); + try + { + writer.getLast(); + } + catch (const std::logic_error&) + { + } + writer.getAll(); + writer.onConnectedKeys([](vector) {}, [](CallbackReason, string) {}); + }; + + auto skw = makeSingleKeyWriter(topic, "key"); + skw = makeSingleKeyWriter(topic, "key", "", WriterConfig()); + SingleKeyWriter skw1(topic, "key"); + + auto skwm = std::move(skw); + testWriter(skwm); + skwm.add("test"); + skwm.update(string("test")); + skwm.partialUpdate("updatetag")(10); + skwm.remove(); + + auto skws = make_shared>(topic, "key"); + skws = make_shared>(topic, "key", "", WriterConfig()); + + auto mkw = makeMultiKeyWriter(topic, {"key"}); + mkw = makeMultiKeyWriter(topic, {"key"}, "", WriterConfig()); + MultiKeyWriter mkw1(topic, {"key"}); + + auto mkwm = std::move(mkw); + testWriter(mkwm); + mkwm.add("key", "test"); + mkwm.update("key", string("test")); + mkwm.partialUpdate("updatetag")("key", 10); + mkwm.remove("key"); + + auto mkws = make_shared>(topic, vector{"key"}); + mkws = make_shared>(topic, vector{"key"}, "", WriterConfig()); + + auto akw = makeAnyKeyWriter(topic); + akw = makeAnyKeyWriter(topic, "", WriterConfig()); + MultiKeyWriter akw1(topic, {}); + + auto akwm = std::move(akw); + testWriter(akwm); + + auto akws = make_shared>(topic, vector{}); + akws = make_shared>(topic, vector{}, "", WriterConfig()); + } + cout << "ok" << endl; + + cout << "testing reader... " << flush; + { + Topic topic(node, "topic"); + + auto testReader = [](Topic::ReaderType& reader) + { + reader.hasWriters(); + reader.waitForWriters(0); + reader.waitForNoWriters(); + reader.getConnectedWriters(); + reader.getConnectedKeys(); + reader.getAllUnread(); + reader.waitForUnread(0); + reader.hasUnread(); + if (false) + { + reader.getNextUnread(); + } + reader.onConnectedKeys([](vector) {}, [](CallbackReason, string) {}); + reader.onSamples([](vector>) {}, [](Sample) {}); + }; + + auto skr = makeSingleKeyReader(topic, "key"); + skr = makeSingleKeyReader(topic, "key", "", ReaderConfig()); + testReader(skr); + auto skrsf = makeSingleKeyReader(topic, "key", Filter("_regex", ".*")); + skrsf = makeSingleKeyReader(topic, "key", Filter("_regex", ".*"), "", ReaderConfig()); + + auto mkr = makeMultiKeyReader(topic, {"key"}); + mkr = makeMultiKeyReader(topic, {"key"}, "", ReaderConfig()); + testReader(mkr); + auto mkrsf = makeMultiKeyReader(topic, {"key"}, Filter("_regex", ".*")); + mkrsf = makeMultiKeyReader(topic, {"key"}, Filter("_regex", ".*"), "", ReaderConfig()); + + auto akr = makeAnyKeyReader(topic); + akr = makeAnyKeyReader(topic, "", ReaderConfig()); + testReader(akr); + auto akrsf = makeAnyKeyReader(topic, Filter("_regex", ".*")); + akrsf = makeAnyKeyReader(topic, Filter("_regex", ".*"), "", ReaderConfig()); + + auto fr = makeFilteredKeyReader(topic, Filter(string("_regex"), string(".*"))); + fr = makeFilteredKeyReader(topic, Filter("_regex", ".*"), "", ReaderConfig()); + testReader(fr); + auto frsf = makeFilteredKeyReader(topic, Filter("_regex", ".*"), Filter("_regex", ".*")); + frsf = makeFilteredKeyReader( + topic, + Filter("_regex", ".*"), + Filter("_regex", ".*"), + "", + ReaderConfig()); + + auto skrs = make_shared>(topic, "key"); + skrs = make_shared>(topic, "key", "", ReaderConfig()); + + auto mkrs = make_shared>(topic, vector{"key"}); + mkrs = make_shared>(topic, vector{"key"}, "", ReaderConfig()); + + auto akrs = make_shared>(topic, vector{}); + akrs = make_shared>(topic, vector{}, "", ReaderConfig()); + + auto frs = make_shared>(topic, Filter("_regex", ".*")); + frs = make_shared>(topic, Filter("_regex", ".*"), "", ReaderConfig()); + + try + { + makeFilteredKeyReader(topic, Filter("unknown", "")); + test(false); + } + catch (const std::invalid_argument&) + { + } + + try + { + makeFilteredKeyReader(topic, Filter("_regex", "(")); + test(false); + } + catch (const std::invalid_argument&) + { + } + catch (const std::regex_error&) + { + } + } + cout << "ok" << endl; + + cout << "testing sample... " << flush; + { + Topic topic(node, "topic"); + auto skw = makeSingleKeyWriter(topic, "key"); + skw.add("test"); + test(skw.getLast().getKey() == "key"); + test(skw.getLast().getValue() == "test"); + test(skw.getLast().getEvent() == SampleEvent::Add); + skw.update("test2"); + test(skw.getLast().getKey() == "key"); + test(skw.getLast().getValue() == "test2"); + test(skw.getLast().getEvent() == SampleEvent::Update); + skw.remove(); + test(skw.getLast().getKey() == "key"); + test(skw.getLast().getValue() == ""); + test(skw.getLast().getEvent() == SampleEvent::Remove); + skw.partialUpdate("partialupdate")("update"); + test(skw.getLast().getKey() == "key"); + test(skw.getLast().getValue() == ""); + test(skw.getLast().getUpdateTag() == "partialupdate"); + test(skw.getLast().getEvent() == SampleEvent::PartialUpdate); + + ostringstream os; + os << skw.getLast(); + os << skw.getLast().getEvent(); + } + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/api/msbuild/writer/packages.config b/cpp/test/DataStorm/api/msbuild/writer/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/api/msbuild/writer/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..15b01824c80 --- /dev/null +++ b/cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {85F74421-2B78-448B-91F0-496526EB365F} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..7a2ec951b94 --- /dev/null +++ b/cpp/test/DataStorm/api/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,90 @@ + + + + + {5e53da0e-935d-4a13-9d99-57bbceb72232} + ice + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {0aed96b7-db8e-4e25-90f0-f2c094a30d2e} + + + {6a38eb93-4e2d-49ee-aae2-0c72446fb0b6} + + + {2664ac65-4c5f-4b19-984c-60ed1107e713} + + + {542193f4-8c4a-4b3b-9144-27928a2adbf1} + + + {b79f9f38-28cb-426b-86e9-4130d503af48} + + + {8364a897-8ac4-4ebd-8e80-f2be4886e5d6} + + + {6b0957c2-59a0-4960-883b-f251e3720af8} + + + {439ab8a9-0a28-4045-bad6-e16d0e5423ca} + + + {53d8216c-7e21-48cd-a9fd-67fda6c26f6d} + + + {3529a985-a84b-4e31-9afc-8fa942f6c3c9} + + + {a57a87d7-a8db-45fb-b001-cfd5a9820ce1} + + + {806d8945-97cd-451f-80da-c744d3956c60} + + + + + Source Files + + + Source Files\x64\Debug + + + Source Files\x64\Release + + + Source Files\Win32\Debug + + + Source Files\Win32\Release + + + + + + + + Slice Files + + + + + Header Files\x64\Debug + + + Header Files\x64\Release + + + Header Files\Win32\Debug + + + Header Files\Win32\Release + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/api/test.py b/cpp/test/DataStorm/api/test.py new file mode 100644 index 00000000000..3e549ca7b50 --- /dev/null +++ b/cpp/test/DataStorm/api/test.py @@ -0,0 +1,17 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Writer +from Util import ClientTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2, +} + +TestSuite( + __file__, + [ ClientTestCase(client = Writer(), traceProps=traceProps) ], + runOnMainThread=True) diff --git a/cpp/test/DataStorm/callbacks/Makefile.mk b/cpp/test/DataStorm/callbacks/Makefile.mk new file mode 100644 index 00000000000..e1761617d4d --- /dev/null +++ b/cpp/test/DataStorm/callbacks/Makefile.mk @@ -0,0 +1,11 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = reader writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_reader_sources = Reader.cpp +$(test)_writer_sources = Writer.cpp + +tests += $(test) diff --git a/cpp/test/DataStorm/callbacks/Reader.cpp b/cpp/test/DataStorm/callbacks/Reader.cpp new file mode 100644 index 00000000000..4db846f3793 --- /dev/null +++ b/cpp/test/DataStorm/callbacks/Reader.cpp @@ -0,0 +1,360 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + ReaderConfig config; + config.sampleCount = -1; // Unlimited sample count + config.clearHistory = ClearHistoryPolicy::Never; + + Topic controller(node, "controller"); + auto writers = makeSingleKeyReader(controller, "writers"); + auto readers = makeSingleKeyWriter(controller, "readers"); + readers.waitForReaders(); + + Topic topic(node, "string"); + + // onSamples + { + { + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(); + promise p; + reader.onSamples( + [&p](const vector>& samples) + { + test(samples.size() == 1); + p.set_value(); + }, + [](const Sample&) { test(false); }); + p.get_future().wait(); + readers.update(1); + } + { + auto reader = makeSingleKeyReader(topic, "elem2", "", config); + reader.waitForWriters(); + promise p; + reader.onSamples( + [](const vector>&) { test(false); }, + [&p](const Sample&) { p.set_value(); }); + readers.update(2); + p.get_future().wait(); + readers.update(3); + } + { + auto reader = makeSingleKeyReader(topic, "elem3", "", config); + reader.waitForUnread(3); + promise p; + reader.onSamples( + [&p](const vector>& samples) + { + test(samples.size() == 3); + p.set_value(); + }, + [](const Sample&) { test(false); }); + p.get_future().wait(); + readers.update(4); + } + } + // onConnectedKeys + { + { + test(writers.getNextUnread().getValue() == 1); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + test(writers.getNextUnread().getValue() == 2); + } + { + auto reader = makeSingleKeyReader(topic, "elem2", "", config); + test(writers.getNextUnread().getValue() == 3); + } + { + auto reader = makeSingleKeyReader(topic, "elem3", "", config); + promise p1, p2, p3; + reader.onConnectedKeys( + [&p1](vector keys) { p1.set_value(keys.empty()); }, + [&p2, &p3](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + p2.set_value(key == "elem3"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(key == "elem3"); + } + }); + test(p1.get_future().get()); + readers.update(1); + test(p2.get_future().get()); + readers.update(2); + test(p3.get_future().get()); + } + { + auto reader = makeSingleKeyReader(topic, "elem4", "", config); + reader.waitForWriters(); + promise p1, p2; + reader.onConnectedKeys( + [&p1](vector keys) { p1.set_value(!keys.empty() && keys[0] == "elem4"); }, + [&p2](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + test(false); + } + else if (action == CallbackReason::Disconnect) + { + p2.set_value(key == "elem4"); + } + }); + test(p1.get_future().get()); + readers.update(3); + test(p2.get_future().get()); + } + readers.update(4); + { + test(writers.getNextUnread().getValue() == 1); + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem1"), "", config); + test(writers.getNextUnread().getValue() == 2); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem2"), "", config); + test(writers.getNextUnread().getValue() == 3); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem3"), "", config); + promise p1, p2, p3; + reader.onConnectedKeys( + [&p1](vector keys) { p1.set_value(keys.empty()); }, + [&p2, &p3](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + p2.set_value(key == "elem3"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(key == "elem3"); + } + }); + test(p1.get_future().get()); + readers.update(1); + test(p2.get_future().get()); + readers.update(2); + test(p3.get_future().get()); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem4"), "", config); + reader.waitForWriters(); + promise p1, p2; + reader.onConnectedKeys( + [&p1](vector keys) { p1.set_value(!keys.empty() && keys[0] == "elem4"); }, + [&p2](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + test(false); + } + else if (action == CallbackReason::Disconnect) + { + p2.set_value(key == "elem4"); + } + }); + test(p1.get_future().get()); + readers.update(3); + test(p2.get_future().get()); + } + + readers.update(4); + + { + test(writers.getNextUnread().getValue() == 1); + auto reader = makeSingleKeyReader(topic, "anyelem1", "", config); + test(writers.getNextUnread().getValue() == 2); + } + + test(writers.getNextUnread().getValue() == 3); + + { + auto reader = makeAnyKeyReader(topic, "", config); + promise p1, p2, p3; + reader.onConnectedKeys( + [&p1](vector keys) { p1.set_value(keys.empty()); }, + [&p2, &p3](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + p2.set_value(key == "anyelem3"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(key == "anyelem3"); + } + }); + test(p1.get_future().get()); + readers.update(1); + test(p2.get_future().get()); + readers.update(2); + test(p3.get_future().get()); + } + + readers.update(3); + } + // onConnected + { + { + test(writers.getNextUnread().getValue() == 1); + auto reader = makeSingleKeyReader(topic, "elem1", "reader1", config); + test(writers.getNextUnread().getValue() == 2); + } + { + auto reader = makeSingleKeyReader(topic, "elem2", "reader2", config); + test(writers.getNextUnread().getValue() == 3); + } + { + auto reader = makeSingleKeyReader(topic, "elem3", "", config); + promise p1, p2, p3; + reader.onConnectedWriters( + [&p1](vector writers) { p1.set_value(writers.empty()); }, + [&p2, &p3](CallbackReason action, string writer) + { + if (action == CallbackReason::Connect) + { + p2.set_value(writer == "writer1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(writer == "writer1"); + } + }); + test(p1.get_future().get()); + test(reader.getConnectedWriters().empty()); + readers.update(1); + test(p2.get_future().get()); + test(reader.getConnectedWriters() == vector{"writer1"}); + readers.update(2); + test(p3.get_future().get()); + } + { + auto reader = makeSingleKeyReader(topic, "elem4", "", config); + reader.waitForWriters(); + promise p1, p2; + reader.onConnectedWriters( + [&p1](vector writers) { p1.set_value(!writers.empty() && writers[0] == "writer2"); }, + [&p2](CallbackReason action, string writer) + { + if (action == CallbackReason::Connect) + { + test(false); + } + else if (action == CallbackReason::Disconnect) + { + p2.set_value(writer == "writer2"); + } + }); + test(p1.get_future().get()); + readers.update(3); + test(p2.get_future().get()); + } + + readers.update(4); + + { + test(writers.getNextUnread().getValue() == 1); + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem1"), "reader1", config); + test(writers.getNextUnread().getValue() == 2); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem2"), "reader2", config); + test(writers.getNextUnread().getValue() == 3); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem3"), "", config); + promise p1, p2, p3; + reader.onConnectedWriters( + [&p1](vector writers) { p1.set_value(writers.empty()); }, + [&p2, &p3](CallbackReason action, string writer) + { + if (action == CallbackReason::Connect) + { + p2.set_value(writer == "writer1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(writer == "writer1"); + } + }); + test(p1.get_future().get()); + readers.update(1); + test(p2.get_future().get()); + readers.update(2); + test(p3.get_future().get()); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem4"), "", config); + reader.waitForWriters(); + promise p1, p2; + reader.onConnectedWriters( + [&p1](vector writers) { p1.set_value(!writers.empty() && writers[0] == "writer2"); }, + [&p2](CallbackReason action, string writer) + { + if (action == CallbackReason::Connect) + { + test(false); + } + else if (action == CallbackReason::Disconnect) + { + p2.set_value(writer == "writer2"); + } + }); + test(p1.get_future().get()); + readers.update(3); + test(p2.get_future().get()); + } + + readers.update(4); + + { + test(writers.getNextUnread().getValue() == 1); + auto reader = makeSingleKeyReader(topic, "anyelem1", "reader1", config); + test(writers.getNextUnread().getValue() == 2); + } + + test(writers.getNextUnread().getValue() == 3); + + { + auto reader = makeAnyKeyReader(topic, "", config); + promise p1, p2, p3; + reader.onConnectedWriters( + [&p1](vector writers) { p1.set_value(writers.empty()); }, + [&p2, &p3](CallbackReason action, string writer) + { + if (action == CallbackReason::Connect) + { + p2.set_value(writer == "writer1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(writer == "writer1"); + } + }); + test(p1.get_future().get()); + readers.update(1); + test(p2.get_future().get()); + readers.update(2); + test(p3.get_future().get()); + } + + readers.update(3); + } + return 0; +} diff --git a/cpp/test/DataStorm/callbacks/Writer.cpp b/cpp/test/DataStorm/callbacks/Writer.cpp new file mode 100644 index 00000000000..94e49725b9f --- /dev/null +++ b/cpp/test/DataStorm/callbacks/Writer.cpp @@ -0,0 +1,244 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + WriterConfig config; + config.sampleCount = -1; // Unlimited sample count + config.clearHistory = ClearHistoryPolicy::Never; + + Topic controller(node, "controller"); + auto writers = makeSingleKeyWriter(controller, "writers"); + auto readers = makeSingleKeyReader(controller, "readers"); + writers.waitForReaders(); + + Topic topic(node, "string"); + + cout << "testing onSamples... " << flush; + { + { + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + test(readers.getNextUnread().getValue() == 1); + } + { + auto writer = makeSingleKeyWriter(topic, "elem2", "", config); + test(readers.getNextUnread().getValue() == 2); + writer.add("value1"); + test(readers.getNextUnread().getValue() == 3); + } + { + auto writer = makeSingleKeyWriter(topic, "elem3", "", config); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + test(readers.getNextUnread().getValue() == 4); + } + } + cout << "ok" << endl; + + cout << "testing onConnectedKeys... " << flush; + { + for (int i = 0; i < 2; ++i) + { + { + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + promise p1, p2, p3; + writer.onConnectedKeys( + [&p1](vector keys) { p1.set_value(keys.empty()); }, + [&p2, &p3](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + p2.set_value(key == "elem1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(key == "elem1"); + } + }); + test(p1.get_future().get()); + writers.update(1); + test(p2.get_future().get()); + writers.update(2); + test(p3.get_future().get()); + } + { + auto writer = makeSingleKeyWriter(topic, "elem2", "", config); + writer.waitForReaders(); + promise p1, p2; + writer.onConnectedKeys( + [&p1](vector keys) { p1.set_value(!keys.empty() && keys[0] == "elem2"); }, + [&p2](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + test(false); + } + else if (action == CallbackReason::Disconnect) + { + p2.set_value(key == "elem2"); + } + }); + test(p1.get_future().get()); + writers.update(3); + test(p2.get_future().get()); + } + { + test(readers.getNextUnread().getValue() == 1); + auto writer = makeSingleKeyWriter(topic, "elem3", "", config); + test(readers.getNextUnread().getValue() == 2); + } + { + auto writer = makeSingleKeyWriter(topic, "elem4", "", config); + test(readers.getNextUnread().getValue() == 3); + } + + test(readers.getNextUnread().getValue() == 4); + } + + { + auto writer = makeAnyKeyWriter(topic, "", config); + promise p1, p2, p3; + writer.onConnectedKeys( + [&p1](vector keys) { p1.set_value(keys.empty()); }, + [&p2, &p3](CallbackReason action, string key) + { + if (action == CallbackReason::Connect) + { + p2.set_value(key == "anyelem1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(key == "anyelem1"); + } + }); + test(p1.get_future().get()); + writers.update(1); + test(p2.get_future().get()); + writers.update(2); + test(p3.get_future().get()); + } + + writers.update(3); + + { + test(readers.getNextUnread().getValue() == 1); + auto writer = makeSingleKeyWriter(topic, "anyelem3", "", config); + test(readers.getNextUnread().getValue() == 2); + } + + test(readers.getNextUnread().getValue() == 3); + } + cout << "ok" << endl; + + cout << "testing onConnected... " << flush; + { + for (int i = 0; i < 2; ++i) + { + { + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + promise p1, p2, p3; + writer.onConnectedReaders( + [&p1](vector readers) { p1.set_value(readers.empty()); }, + [&p2, &p3](CallbackReason action, string reader) + { + if (action == CallbackReason::Connect) + { + p2.set_value(reader == "reader1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(reader == "reader1"); + } + }); + test(p1.get_future().get()); + test(writer.getConnectedReaders().empty()); + writers.update(1); + test(p2.get_future().get()); + test(writer.getConnectedReaders() == vector{"reader1"}); + writers.update(2); + test(p3.get_future().get()); + } + { + auto writer = makeSingleKeyWriter(topic, "elem2", "", config); + writer.waitForReaders(); + promise p1, p2; + writer.onConnectedReaders( + [&p1](vector readers) { p1.set_value(!readers.empty() && readers[0] == "reader2"); }, + [&p2](CallbackReason action, string reader) + { + if (action == CallbackReason::Connect) + { + test(false); + } + else if (action == CallbackReason::Disconnect) + { + p2.set_value(reader == "reader2"); + } + }); + test(p1.get_future().get()); + writers.update(3); + test(p2.get_future().get()); + } + { + test(readers.getNextUnread().getValue() == 1); + auto writer = makeSingleKeyWriter(topic, "elem3", "writer1", config); + test(readers.getNextUnread().getValue() == 2); + } + { + auto writer = makeSingleKeyWriter(topic, "elem4", "writer2", config); + test(readers.getNextUnread().getValue() == 3); + } + + test(readers.getNextUnread().getValue() == 4); + } + + { + auto writer = makeAnyKeyWriter(topic, "", config); + promise p1, p2, p3; + writer.onConnectedReaders( + [&p1](vector readers) { p1.set_value(readers.empty()); }, + [&p2, &p3](CallbackReason action, string reader) + { + if (action == CallbackReason::Connect) + { + p2.set_value(reader == "reader1"); + } + else if (action == CallbackReason::Disconnect) + { + p3.set_value(reader == "reader1"); + } + }); + test(p1.get_future().get()); + writers.update(1); + test(p2.get_future().get()); + test(writer.getConnectedReaders() == vector{"reader1"}); + writers.update(2); + test(p3.get_future().get()); + } + + writers.update(3); + + { + test(readers.getNextUnread().getValue() == 1); + auto writer = makeSingleKeyWriter(topic, "anyelem3", "writer1", config); + test(readers.getNextUnread().getValue() == 2); + } + + test(readers.getNextUnread().getValue() == 3); + } + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/callbacks/msbuild/reader/packages.config b/cpp/test/DataStorm/callbacks/msbuild/reader/packages.config new file mode 100644 index 00000000000..9a5f7b2c74a --- /dev/null +++ b/cpp/test/DataStorm/callbacks/msbuild/reader/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj b/cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj new file mode 100644 index 00000000000..2281cf47cd3 --- /dev/null +++ b/cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj @@ -0,0 +1,102 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {783EE047-9119-4412-960A-600729D1A0F1} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj.filters b/cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj.filters new file mode 100644 index 00000000000..9eff3ef7ab8 --- /dev/null +++ b/cpp/test/DataStorm/callbacks/msbuild/reader/reader.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/callbacks/msbuild/writer/packages.config b/cpp/test/DataStorm/callbacks/msbuild/writer/packages.config new file mode 100644 index 00000000000..9a5f7b2c74a --- /dev/null +++ b/cpp/test/DataStorm/callbacks/msbuild/writer/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..f9d91232844 --- /dev/null +++ b/cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj @@ -0,0 +1,102 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {8ADD3C14-F6A9-4683-94BE-EB87C60078F1} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..bc887ef6719 --- /dev/null +++ b/cpp/test/DataStorm/callbacks/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/callbacks/test.py b/cpp/test/DataStorm/callbacks/test.py new file mode 100644 index 00000000000..3caada15cec --- /dev/null +++ b/cpp/test/DataStorm/callbacks/test.py @@ -0,0 +1,17 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Reader, Writer +from Util import ClientServerTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2, +} + +TestSuite( + __file__, + [ ClientServerTestCase(client = Writer(), server = Reader(), traceProps=traceProps) ], + runOnMainThread=True) diff --git a/cpp/test/DataStorm/config/Makefile.mk b/cpp/test/DataStorm/config/Makefile.mk new file mode 100644 index 00000000000..e1761617d4d --- /dev/null +++ b/cpp/test/DataStorm/config/Makefile.mk @@ -0,0 +1,11 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = reader writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_reader_sources = Reader.cpp +$(test)_writer_sources = Writer.cpp + +tests += $(test) diff --git a/cpp/test/DataStorm/config/Reader.cpp b/cpp/test/DataStorm/config/Reader.cpp new file mode 100644 index 00000000000..f59e27bcb54 --- /dev/null +++ b/cpp/test/DataStorm/config/Reader.cpp @@ -0,0 +1,317 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + Topic topic(node, "stringtopic"); + Topic controller(node, "controller"); + + auto readers = makeSingleKeyWriter(controller, "readers"); + auto writers = makeSingleKeyReader(controller, "writers", "", {-1, 0, ClearHistoryPolicy::Never}); + + writers.waitForWriters(); + + // Writer/reader name + { + auto reader = makeSingleKeyReader(topic, "key1", "readername1"); + test(reader.getNextUnread().getOrigin() == "writername1"); + readers.update(true); // Reader is done + } + + // Writer sample count + { + auto read = [&topic, &writers, &readers](int count) + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + + readers.update(false); + + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + test(count < 6 || reader.getNextUnread().getValue() == "value1"); + test(count < 5 || reader.getNextUnread().getValue() == "value2"); + test(count < 4 || reader.getNextUnread().getEvent() == SampleEvent::Remove); + test(count < 3 || reader.getNextUnread().getValue() == "value3"); + test(count < 2 || reader.getNextUnread().getValue() == "value4"); + test(count < 1 || reader.getNextUnread().getEvent() == SampleEvent::Remove); + readers.update(true); // Reader is done + }; + + read(6); // Writer keeps all the samples + read(4); // Writer keeps 4 samples + read(3); // Writer keeps last instance samples + } + + // Reader sample count + { + auto read = [&topic, &writers, &readers](int count, ReaderConfig config) + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + test(count < 6 || reader.getNextUnread().getValue() == "value1"); + test(count < 5 || reader.getNextUnread().getValue() == "value2"); + test(count < 4 || reader.getNextUnread().getEvent() == SampleEvent::Remove); + test(count < 3 || reader.getNextUnread().getValue() == "value3"); + test(count < 2 || reader.getNextUnread().getValue() == "value4"); + test(count < 1 || reader.getNextUnread().getEvent() == SampleEvent::Remove); + readers.update(true); // Reader is done + }; + + // Keep all the samples in the history. + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + read(6, config); + } + + // Keep 4 samples in the history + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + config.sampleCount = 4; + read(4, config); + } + + // Keep last instance samples in the history + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::OnAdd; + read(3, config); + } + } + + // Writer sample lifetime + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + + // Writer keeps 3ms worth of samples + readers.update(false); + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + test(reader.getNextUnread().getValue() == "value3"); + test(reader.getNextUnread().getValue() == "value4"); + test(reader.getNextUnread().getEvent() == SampleEvent::Remove); + readers.update(true); // Reader is done + } + + // Reader sample lifetime + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + + ReaderConfig config; + config.sampleLifetime = 390; + config.clearHistory = ClearHistoryPolicy::Never; + + auto now = chrono::system_clock::now(); + + // Reader wants 390ms worth of samples + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(3); + auto samples = reader.getAllUnread(); + test(samples[0].getValue() == "value3"); + test(samples[1].getValue() == "value4"); + test(samples[2].getEvent() == SampleEvent::Remove); + readers.update(true); // Reader is done + + for (const auto& s : samples) + { + test(s.getTimeStamp() >= (now - chrono::milliseconds(150))); + } + } + + // Writer clearHistory + { + topic.setUpdater("concat", [](string& value, string update) { value += update; }); + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(22); + readers.update(true); // Reader is done + } + + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(2); + test(reader.getNextUnread().getValue() == "value3"); + test(reader.getNextUnread().getValue() == "value4"); + readers.update(true); // Reader is done + } + + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(3); + test(reader.getNextUnread().getEvent() == DataStorm::SampleEvent::Remove); + test(reader.getNextUnread().getValue() == "value3"); + test(reader.getNextUnread().getValue() == "value4"); + readers.update(true); // Reader is done + } + + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(1); + test(reader.getNextUnread().getValue() == "value4"); + readers.update(true); // Reader is done + } + + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + readers.update(false); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(4); + test(reader.getNextUnread().getValue() == "value"); + auto s = reader.getNextUnread(); + test(s.getEvent() == SampleEvent::PartialUpdate); + test(s.getUpdateTag() == "concat"); + test(s.getValue() == "value1"); + test(reader.getNextUnread().getValue() == "value12"); + test(reader.getNextUnread().getValue() == "value123"); + readers.update(true); // Reader is done + } + } + + // Reader clearHistory + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + readers.update(false); + + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(9); + } + + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::OnAdd; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(5); + } + + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::OnRemove; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(6); + } + + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::OnAll; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(1); + } + + { + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::OnAllExceptPartialUpdate; + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + reader.waitForUnread(4); + } + + readers.update(true); // Reader is done + } + + // Reader priority discard policy + { + readers.update(false); + + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + config.discardPolicy = DiscardPolicy::Priority; + auto reader = makeSingleKeyReader(topic, "elemdp1", "", config); + test(reader.getNextUnread().getValue() == "value1"); + test(reader.getNextUnread().getValue() == "value2"); + test(reader.getNextUnread().getValue() == "value21"); + test(reader.getNextUnread().getEvent() == DataStorm::SampleEvent::Remove); + + readers.update(true); // Reader is done + } + { + readers.update(false); + + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + config.discardPolicy = DiscardPolicy::Priority; + auto reader = makeSingleKeyReader(topic, "elemdp2", "", config); + test(reader.getNextUnread().getValue() == "value1"); + test(reader.getNextUnread().getValue() == "value2"); + test(reader.getNextUnread().getValue() == "value21"); + test(reader.getNextUnread().getEvent() == DataStorm::SampleEvent::Remove); + + readers.update(true); // Reader is done + } + + // Reader sending discard policy + { + while (!writers.getNextUnread().getValue()) + ; // Wait for writer to write the samples before reading + + readers.update(false); + while (true) + { + auto lastUnread = chrono::time_point::min(); + Topic topic1(node, "sendTimeTopic"); + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + config.discardPolicy = DiscardPolicy::SendTime; + auto reader = makeSingleKeyReader(topic1, "elem", "", config); + int writerCount = 10; + int sampleCount = 0; + while (true) + { + auto sample = reader.getNextUnread(); + test(sample.getTimeStamp() > lastUnread); + lastUnread = sample.getTimeStamp(); + ++sampleCount; + if (sample.getValue() == (writerCount - 1)) + { + break; + } + } + if (sampleCount < writerCount) + { + test(!reader.hasUnread()); + break; + } + } + readers.update(true); // Reader is done + } + + return 0; +} diff --git a/cpp/test/DataStorm/config/Writer.cpp b/cpp/test/DataStorm/config/Writer.cpp new file mode 100644 index 00000000000..04dda35ae3a --- /dev/null +++ b/cpp/test/DataStorm/config/Writer.cpp @@ -0,0 +1,332 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "TestHelper.h" + +#include + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + Topic topic(node, "stringtopic"); + Topic controller(node, "controller"); + + auto writers = makeSingleKeyWriter(controller, "writers"); + auto readers = makeSingleKeyReader(controller, "readers", "", {-1, 0, ClearHistoryPolicy::Never}); + + { + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + topic.setWriterDefaultConfig(config); + } + + cout << "testing writer/reader name... " << flush; + { + auto writer = makeSingleKeyWriter(topic, "key1", "writername1"); + writer.update("update"); + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + cout << "ok" << endl; + + cout << "testing writer sampleCount... " << flush; + { + auto write = [&topic, &writers, &readers](WriterConfig config) + { + writers.update(false); // Not ready + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + writer.add("value3"); + writer.update("value4"); + writer.remove(); + writers.update(true); // Ready + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + }; + + // Keep all the samples in the history. + { + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + write(config); + } + + // Keep 4 samples in the history + { + WriterConfig config; + config.sampleCount = 4; + write(config); + } + + // Keep last instance samples in the history + { + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::OnAdd; + write(config); + } + } + cout << "ok" << endl; + + cout << "testing reader sampleCount... " << flush; + { + auto write = [&topic, &writers, &readers]() + { + writers.update(false); // Not ready + auto writer = makeSingleKeyWriter(topic, "elem1"); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + writer.add("value3"); + writer.update("value4"); + writer.remove(); + writers.update(true); // Ready + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + }; + + write(); // Reader keeps all the samples in the history. + write(); // Reader keeps 4 samples in the history. + write(); // Reader keeps last instance samples in the history. + } + cout << "ok" << endl; + + cout << "testing writer sampleLifetime... " << flush; + { + writers.update(false); // Not ready + + // Keep 3ms worth of samples in the history + WriterConfig config; + config.sampleLifetime = 20; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + this_thread::sleep_for(chrono::milliseconds(20)); + writers.update(true); // Ready + writer.waitForReaders(); + writer.add("value3"); + writer.update("value4"); + writer.remove(); + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + cout << "ok" << endl; + + cout << "testing reader sampleLifetime... " << flush; + { + writers.update(false); // Not ready + auto writer = makeSingleKeyWriter(topic, "elem1"); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + this_thread::sleep_for(chrono::milliseconds(400)); + writer.add("value3"); + writer.update("value4"); + writer.remove(); + writers.update(true); // Ready + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + cout << "ok" << endl; + + cout << "testing writer clearHistory... " << flush; + { + topic.setUpdater("concat", [](string& value, string update) { value += update; }); + + { + writers.update(false); // Not ready + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + for (int i = 0; i < 20; ++i) + { + writer.update("value2"); + } + writer.remove(); + test(writer.getAll().size() == 22); + writers.update(true); // Ready + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + + { + writers.update(false); // Not ready + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::OnAdd; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + writer.add("value3"); + writer.update("value4"); + test(writer.getAll().size() == 2); + writers.update(true); // Ready + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + + { + writers.update(false); // Not ready + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::OnRemove; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + writer.add("value3"); + writer.update("value4"); + test(writer.getAll().size() == 3); + writers.update(true); // Ready + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + + { + writers.update(false); // Not ready + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::OnAll; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.remove(); + writer.add("value3"); + writer.update("value4"); + test(writer.getAll().size() == 1); + writers.update(true); // Ready + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + + { + writers.update(false); // Not ready + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::OnAllExceptPartialUpdate; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.partialUpdate("concat")("1"); + writer.remove(); + writer.add("value3"); + writer.update("value"); + writer.partialUpdate("concat")("1"); + writer.partialUpdate("concat")("2"); + writer.partialUpdate("concat")("3"); + test(writer.getAll().size() == 4); + test(writer.getAll()[1].getValue() == "value1"); + writers.update(true); // Ready + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + } + cout << "ok" << endl; + + cout << "testing reader clearHistory... " << flush; + { + writers.update(false); // Not ready + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.add("value1"); + writer.update("value2"); + writer.partialUpdate("concat")("1"); + writer.remove(); + writer.add("value3"); + writer.update("value"); + writer.partialUpdate("concat")("1"); + writer.partialUpdate("concat")("2"); + writer.partialUpdate("concat")("3"); + writers.update(true); // Ready + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + cout << "ok" << endl; + + cout << "testing priority discard policy... " << flush; + { + WriterConfig config; + config.priority = 10; + auto writer1 = makeSingleKeyWriter(topic, "elemdp1", "", config); + config.priority = 1; + auto writer2 = makeSingleKeyWriter(topic, "elemdp1", "", config); + + writer1.add("value1"); + writer1.update("value2"); + writer1.partialUpdate("concat")("1"); + writer1.remove(); + + writer2.add("novalue1"); + writer2.update("novalue2"); + writer2.partialUpdate("concat")("no1"); + writer2.remove(); + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + { + WriterConfig config; + config.priority = 1; + auto writer1 = makeSingleKeyWriter(topic, "elemdp2", "", config); + config.priority = 10; + auto writer2 = makeSingleKeyWriter(topic, "elemdp2", "", config); + + writer2.waitForReaders(); + + writer1.add("novalue1"); + writer1.update("novalue2"); + writer1.partialUpdate("concat")("no1"); + writer1.remove(); + + writer2.add("value1"); + writer2.update("value2"); + writer2.partialUpdate("concat")("1"); + writer2.remove(); + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + cout << "ok" << endl; + + cout << "testing send time discard policy... " << flush; + { + writers.update(false); // Not ready + vector, SingleKeyWriter>> w; + size_t writerCount = 10; + for (size_t i = 0; i < writerCount; ++i) + { + Ice::InitializationData initData; + initData.properties = node.getCommunicator()->getProperties()->clone(); + Ice::CommunicatorHolder holder(initData); + Node node1(holder.communicator()); + Topic topic1(node1, "sendTimeTopic"); + auto singleKeyWriter = makeSingleKeyWriter(topic1, "elem"); + w.emplace_back(std::move(holder), std::move(node1), std::move(topic1), std::move(singleKeyWriter)); + } + for (size_t i = 0; i < writerCount; ++i) + { + this_thread::sleep_for(chrono::microseconds(200)); + get<3>(w[i]).update(static_cast(i)); + } + writers.update(true); // Ready + + while (!readers.getNextUnread().getValue()) + ; // Wait for reader to be done + } + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/config/msbuild/reader/packages.config b/cpp/test/DataStorm/config/msbuild/reader/packages.config new file mode 100644 index 00000000000..9a5f7b2c74a --- /dev/null +++ b/cpp/test/DataStorm/config/msbuild/reader/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj b/cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj new file mode 100644 index 00000000000..4279ae9109e --- /dev/null +++ b/cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj @@ -0,0 +1,102 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {735292B7-70CB-43B2-B7BB-D123B7497CC7} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj.filters b/cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj.filters new file mode 100644 index 00000000000..9eff3ef7ab8 --- /dev/null +++ b/cpp/test/DataStorm/config/msbuild/reader/reader.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/config/msbuild/writer/packages.config b/cpp/test/DataStorm/config/msbuild/writer/packages.config new file mode 100644 index 00000000000..9a5f7b2c74a --- /dev/null +++ b/cpp/test/DataStorm/config/msbuild/writer/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..4bfc57888e7 --- /dev/null +++ b/cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj @@ -0,0 +1,102 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {1C5D0A06-731C-4FBB-8250-0B09E7D5BF55} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..bc887ef6719 --- /dev/null +++ b/cpp/test/DataStorm/config/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/config/test.py b/cpp/test/DataStorm/config/test.py new file mode 100644 index 00000000000..3caada15cec --- /dev/null +++ b/cpp/test/DataStorm/config/test.py @@ -0,0 +1,17 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Reader, Writer +from Util import ClientServerTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2, +} + +TestSuite( + __file__, + [ ClientServerTestCase(client = Writer(), server = Reader(), traceProps=traceProps) ], + runOnMainThread=True) diff --git a/cpp/test/DataStorm/events/Makefile.mk b/cpp/test/DataStorm/events/Makefile.mk new file mode 100644 index 00000000000..a275bcdbe7f --- /dev/null +++ b/cpp/test/DataStorm/events/Makefile.mk @@ -0,0 +1,11 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = reader writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_reader_sources = Reader.cpp Test.ice +$(test)_writer_sources = Writer.cpp Test.ice + +tests += $(test) diff --git a/cpp/test/DataStorm/events/Reader.cpp b/cpp/test/DataStorm/events/Reader.cpp new file mode 100644 index 00000000000..82f6cf1666d --- /dev/null +++ b/cpp/test/DataStorm/events/Reader.cpp @@ -0,0 +1,395 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + ReaderConfig config; + config.sampleCount = -1; // Unlimited sample count + config.clearHistory = ClearHistoryPolicy::Never; + + { + Topic topic(node, "string"); + { + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + + reader.waitForWriters(1); + test(reader.hasWriters()); + + auto testSample = [&reader](SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue() == value); + } + }; + + testSample(SampleEvent::Add, "elem1", "value1"); + testSample(SampleEvent::Update, "elem1", "value2"); + testSample(SampleEvent::Remove, "elem1"); + + test(reader.getAllUnread().empty()); + } + { + auto reader1 = makeSingleKeyReader(topic, "elem2", "", config); + auto reader2 = makeSingleKeyReader(topic, "elem2", "", config); + reader1.waitForWriters(1); + reader2.waitForWriters(1); + reader1.waitForUnread(); + reader2.waitForUnread(); + } + } + + { + Topic topic(node, "struct"); + auto reader = makeSingleKeyReader(topic, 10, "", config); + + reader.waitForWriters(1); + test(reader.hasWriters()); + + auto testSample = [&reader](SampleEvent event, Test::StructValue value = Test::StructValue()) + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == 10); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue() == value); + } + }; + + testSample(SampleEvent::Add, Test::StructValue({"firstName", "lastName", 10})); + testSample(SampleEvent::Update, Test::StructValue({"firstName", "lastName", 11})); + testSample(SampleEvent::Remove); + } + + { + Topic> topic(node, "baseclass"); + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + + reader.waitForWriters(1); + test(reader.hasWriters()); + + auto testSample = [&reader](SampleEvent event, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == "elem1"); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue()->b == value); + } + }; + + testSample(SampleEvent::Add, "value1"); + testSample(SampleEvent::Update, "value2"); + testSample(SampleEvent::Remove); + } + + { + Topic> topic(node, "baseclass2"); + + auto testSample = + [](typename decltype(topic)::ReaderType& reader, SampleEvent event, string key, string value = "") + { + reader.waitForWriters(1); + test(reader.hasWriters()); + + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue()->b == value); + } + }; + + { + auto reader = makeSingleKeyReader(topic, "elem1", "", config); + testSample(reader, SampleEvent::Add, "elem1", "value1"); + testSample(reader, SampleEvent::Update, "elem1", "value2"); + testSample(reader, SampleEvent::Remove, "elem1"); + } + { + auto reader = makeSingleKeyReader(topic, "elem2", "", config); + testSample(reader, SampleEvent::Update, "elem2", "value1"); + } + { + auto reader = makeSingleKeyReader(topic, "elem3", "", config); + testSample(reader, SampleEvent::Remove, "elem3"); + } + { + auto reader = makeSingleKeyReader(topic, "elem4", "", config); + testSample(reader, SampleEvent::Add, "elem4", "value1"); + } + } + + { + Topic topic(node, "multikey1"); + + auto reader = makeMultiKeyReader(topic, {"elem1", "elem2"}, "", config); + reader.waitForWriters(2); + reader.waitForUnread(6); + test(reader.getAllUnread().size() == 6); + } + + { + Topic topic(node, "anykey1"); + + auto reader = makeAnyKeyReader(topic, "", config); + reader.waitForWriters(2); + reader.waitForUnread(6); + test(reader.getAllUnread().size() == 6); + } + + { + Topic topic(node, "multikey2"); + + auto reader1 = makeSingleKeyReader(topic, "elem1", "", config); + auto reader2 = makeSingleKeyReader(topic, "elem2", "", config); + + reader1.waitForWriters(1); + reader1.waitForUnread(3); + test(reader1.getAllUnread().size() == 3); + + reader2.waitForWriters(1); + reader2.waitForUnread(3); + test(reader2.getAllUnread().size() == 3); + } + + { + Topic topic(node, "anykey2"); + + auto reader1 = makeSingleKeyReader(topic, "elem1", "", config); + auto reader2 = makeSingleKeyReader(topic, "elem2", "", config); + + reader1.waitForWriters(1); + reader1.waitForUnread(3); + test(reader1.getAllUnread().size() == 3); + + reader2.waitForWriters(1); + reader2.waitForUnread(3); + test(reader2.getAllUnread().size() == 3); + } + + { + Topic topic(node, "multikey3"); + + auto reader = makeMultiKeyReader(topic, {"elem1", "elem2"}, "", config); + reader.waitForWriters(2); + reader.waitForUnread(6); + test(reader.getAllUnread().size() == 6); + } + + { + Topic topic(node, "anykey3"); + + auto reader = makeAnyKeyReader(topic, "", config); + reader.waitForWriters(1); + reader.waitForUnread(6); + test(reader.getAllUnread().size() == 6); + } + + { + Topic> topic(node, "filtered1"); + topic.setKeyFilter( + "startswith", + [](const string& prefix) + { + return [prefix](const string& key) + { return key.size() >= prefix.size() && key.compare(0, prefix.size(), prefix) == 0; }; + }); + + { + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem[0-4]"), "", config); + + reader.waitForWriters(1); + test(reader.hasWriters()); + + auto testSample = [&reader](SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue()->b == value); + } + }; + + testSample(SampleEvent::Add, "elem1", "value1"); + testSample(SampleEvent::Update, "elem1", "value2"); + testSample(SampleEvent::Remove, "elem1"); + + testSample(SampleEvent::Update, "elem2", "value1"); + testSample(SampleEvent::Remove, "elem3"); + testSample(SampleEvent::Add, "elem4", "value1"); + } + { + auto reader = makeFilteredKeyReader(topic, Filter("startswith", "val"), "", config); + reader.waitForWriters(1); + test(reader.hasWriters()); + reader.getNextUnread(); + } + } + + { + Topic> topic(node, "filtered2"); + + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem[0-4]"), "", config); + + reader.waitForWriters(1); + test(reader.hasWriters()); + + auto testSample = [&reader](SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue()->b == value); + } + }; + + testSample(SampleEvent::Add, "elem1", "value1"); + testSample(SampleEvent::Update, "elem1", "value2"); + testSample(SampleEvent::Remove, "elem1"); + + testSample(SampleEvent::Update, "elem2", "value1"); + testSample(SampleEvent::Remove, "elem3"); + testSample(SampleEvent::Add, "elem4", "value1"); + } + + { + Topic> topic(node, "filtered3"); + + auto reader = makeFilteredKeyReader(topic, Filter("_regex", "elem[0-4]"), "", config); + + reader.waitForWriters(1); + test(reader.hasWriters()); + + auto testSample = [&reader](SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue()->b == value); + } + }; + + testSample(SampleEvent::Add, "elem1", "value1"); + testSample(SampleEvent::Update, "elem1", "value2"); + testSample(SampleEvent::Remove, "elem1"); + + testSample(SampleEvent::Update, "elem2", "value1"); + testSample(SampleEvent::Remove, "elem3"); + testSample(SampleEvent::Add, "elem4", "value1"); + } + + { + Topic topic(node, "filtered reader key/value filter"); + + { + auto testSample = + [](typename decltype(topic)::ReaderType& reader, SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue() == value); + } + }; + + auto reader11 = makeFilteredKeyReader( + topic, + Filter("_regex", "elem[1]"), + Filter("_event", SampleEventSeq{SampleEvent::Add}), + "", + config); + auto reader12 = makeFilteredKeyReader( + topic, + Filter("_regex", "elem[1]"), + Filter("_event", SampleEventSeq{SampleEvent::Update}), + "", + config); + auto reader13 = makeFilteredKeyReader( + topic, + Filter("_regex", "elem[1]"), + Filter("_event", SampleEventSeq{SampleEvent::Remove}), + "", + config); + testSample(reader11, SampleEvent::Add, "elem1", "value1"); + testSample(reader12, SampleEvent::Update, "elem1", "value2"); + testSample(reader13, SampleEvent::Remove, "elem1"); + } + { + auto testSample = + [](typename decltype(topic)::ReaderType& reader, SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue() == value); + } + }; + + auto reader2 = makeFilteredKeyReader( + topic, + Filter("_regex", "elem[2]"), + Filter("_regex", "value[2-4]"), + "", + config); + testSample(reader2, SampleEvent::Update, "elem2", "value2"); + testSample(reader2, SampleEvent::Update, "elem2", "value3"); + testSample(reader2, SampleEvent::Update, "elem2", "value4"); + } + { + auto testSample = + [](typename decltype(topic)::ReaderType& reader, SampleEvent event, string key, string value = "") + { + reader.waitForUnread(1); + auto sample = reader.getNextUnread(); + test(sample.getKey() == key); + test(sample.getEvent() == event); + if (event != SampleEvent::Remove) + { + test(sample.getValue() == value); + } + }; + + auto reader2 = makeSingleKeyReader(topic, "elem3", Filter("startswith", "val"), "", config); + testSample(reader2, SampleEvent::Update, "elem3", "value"); + } + } + + return 0; +} diff --git a/cpp/test/DataStorm/events/Test.ice b/cpp/test/DataStorm/events/Test.ice new file mode 100644 index 00000000000..162d4abfd1a --- /dev/null +++ b/cpp/test/DataStorm/events/Test.ice @@ -0,0 +1,24 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +module Test +{ + +struct StructValue +{ + string firstName; + string lastName; + int age; +}; + +class Base +{ + string b; +}; + +class Extended extends Base +{ + int e; +}; + +}; diff --git a/cpp/test/DataStorm/events/Writer.cpp b/cpp/test/DataStorm/events/Writer.cpp new file mode 100644 index 00000000000..e5a1f9b17cd --- /dev/null +++ b/cpp/test/DataStorm/events/Writer.cpp @@ -0,0 +1,394 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + WriterConfig config; + config.sampleCount = -1; // Unlimited sample count + config.clearHistory = ClearHistoryPolicy::Never; + + cout << "testing single key reader/writer... " << flush; + { + { + Topic topic(node, "string"); + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + + writer.waitForReaders(1); + test(writer.hasReaders()); + + writer.add("value1"); + writer.update("value2"); + writer.remove(); + + writer.waitForNoReaders(); + + auto writer2 = makeSingleKeyWriter(topic, "elem2", "", config); + writer2.waitForReaders(2); + writer2.add("value"); + writer2.waitForNoReaders(); + } + { + Topic topic(node, "struct"); + auto writer = makeSingleKeyWriter(topic, 10, "", config); + + writer.waitForReaders(1); + test(writer.hasReaders()); + + writer.add({"firstName", "lastName", 10}); + writer.update({"firstName", "lastName", 11}); + writer.remove(); + + writer.waitForNoReaders(); + } + { + Topic> topic(node, "baseclass"); + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + + writer.waitForReaders(1); + test(writer.hasReaders()); + + writer.add(make_shared("value1")); + writer.update(make_shared("value2")); + writer.remove(); + + writer.waitForNoReaders(); + } + { + Topic> topic(node, "baseclass2"); + { + auto writer = makeSingleKeyWriter(topic, "elem1", "", config); + writer.waitForReaders(1); + test(writer.hasReaders()); + + writer.add(make_shared("value1")); + writer.update(make_shared("value2")); + writer.remove(); + writer.waitForNoReaders(); + } + { + auto writer = makeSingleKeyWriter(topic, "elem2", "", config); + writer.waitForReaders(1); + writer.update(make_shared("value1")); + writer.waitForNoReaders(); + } + { + auto writer = makeSingleKeyWriter(topic, "elem3", "", config); + writer.waitForReaders(1); + writer.remove(); + writer.waitForNoReaders(); + } + { + auto writer = makeSingleKeyWriter(topic, "elem4", "", config); + writer.waitForReaders(1); + writer.add(make_shared("value1")); + writer.waitForNoReaders(); + } + } + } + cout << "ok" << endl; + + cout << "testing multi-key reader/writer... " << flush; + { + { + Topic topic(node, "multikey1"); + auto writer1 = makeSingleKeyWriter(topic, "elem1", "", config); + auto writer2 = makeSingleKeyWriter(topic, "elem2", "", config); + + writer1.waitForReaders(1); + writer2.waitForReaders(1); + + writer1.add("value1"); + writer1.update("value2"); + writer1.remove(); + + writer2.add("value1"); + writer2.update("value2"); + writer2.remove(); + + writer1.waitForNoReaders(); + writer2.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing any-key reader/writer... " << flush; + { + { + Topic topic(node, "anykey1"); + auto writer1 = makeSingleKeyWriter(topic, "elem1", "", config); + auto writer2 = makeSingleKeyWriter(topic, "elem2", "", config); + + writer1.waitForReaders(1); + writer2.waitForReaders(1); + + writer1.add("value1"); + writer1.update("value2"); + writer1.remove(); + + writer2.add("value1"); + writer2.update("value2"); + writer2.remove(); + + writer1.waitForNoReaders(); + writer2.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing reader/multi-key writer... " << flush; + { + { + Topic topic(node, "multikey2"); + auto writer = makeMultiKeyWriter(topic, {"elem1", "elem2"}, "", config); + + writer.waitForReaders(2); + + writer.add("elem1", "value1"); + writer.update("elem1", "value2"); + writer.remove("elem1"); + + writer.add("elem2", "value1"); + writer.update("elem2", "value2"); + writer.remove("elem2"); + + writer.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing reader/any-key writer... " << flush; + { + { + Topic topic(node, "anykey2"); + auto writer = makeAnyKeyWriter(topic, "", config); + + writer.waitForReaders(2); + + writer.add("elem1", "value1"); + writer.update("elem1", "value2"); + writer.remove("elem1"); + + writer.add("elem2", "value1"); + writer.update("elem2", "value2"); + writer.remove("elem2"); + + writer.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing multi-key reader/multi-key writer... " << flush; + { + { + Topic topic(node, "multikey3"); + auto writer = makeMultiKeyWriter(topic, {"elem1", "elem2"}, "", config); + + writer.waitForReaders(1); + + writer.add("elem1", "value1"); + writer.update("elem1", "value2"); + writer.remove("elem1"); + + writer.add("elem2", "value1"); + writer.update("elem2", "value2"); + writer.remove("elem2"); + + writer.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing any-key reader/any-key writer... " << flush; + { + { + Topic topic(node, "anykey3"); + auto writer = makeAnyKeyWriter(topic, "", config); + + writer.waitForReaders(1); + + writer.add("elem1", "value1"); + writer.update("elem1", "value2"); + writer.remove("elem1"); + + writer.add("elem2", "value1"); + writer.update("elem2", "value2"); + writer.remove("elem2"); + + writer.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing filtered reader/writer... " << flush; + { + Topic> topic(node, "filtered1"); + + topic.setKeyFilter( + "startswith", + [](const string& prefix) + { + return [prefix](const string& key) + { return key.size() >= prefix.size() && key.compare(0, prefix.size(), prefix) == 0; }; + }); + + { + auto writer5 = makeSingleKeyWriter(topic, "elem5", "", config); + writer5.add(make_shared("value1")); + writer5.update(make_shared("value2")); + writer5.remove(); + + auto writer1 = makeSingleKeyWriter(topic, "elem1", "", config); + writer1.waitForReaders(1); + test(writer1.hasReaders()); + writer1.add(make_shared("value1")); + writer1.update(make_shared("value2")); + writer1.remove(); + + auto writer2 = makeSingleKeyWriter(topic, "elem2", "", config); + writer2.waitForReaders(1); + writer2.update(make_shared("value1")); + + auto writer3 = makeSingleKeyWriter(topic, "elem3", "", config); + writer3.waitForReaders(1); + writer3.remove(); + + auto writer4 = makeSingleKeyWriter(topic, "elem4", "", config); + writer4.waitForReaders(1); + writer4.add(make_shared("value1")); + writer4.waitForNoReaders(); + } + { + auto writer1 = makeSingleKeyWriter(topic, "nonvalue", "", config); + auto writer2 = makeSingleKeyWriter(topic, "value", "", config); + test(!writer1.hasReaders()); + writer2.waitForReaders(1); + test(!writer1.hasReaders()); + writer2.remove(); + writer2.waitForNoReaders(); + } + } + cout << "ok" << endl; + + cout << "testing filtered reader/multi-key writer... " << flush; + { + Topic> topic(node, "filtered2"); + + auto writer = makeMultiKeyWriter(topic, {"elem1", "elem2", "elem3", "elem4", "elem5"}, "", config); + writer.waitForReaders(1); + + writer.add("elem5", make_shared("value1")); + writer.update("elem5", make_shared("value2")); + writer.remove("elem5"); + + test(writer.hasReaders()); + writer.add("elem1", make_shared("value1")); + writer.update("elem1", make_shared("value2")); + writer.remove("elem1"); + + writer.update("elem2", make_shared("value1")); + + writer.remove("elem3"); + + writer.add("elem4", make_shared("value1")); + writer.waitForNoReaders(); + } + cout << "ok" << endl; + + cout << "testing filtered reader/any-key writer... " << flush; + { + Topic> topic(node, "filtered3"); + + auto writer = makeAnyKeyWriter(topic, "", config); + writer.waitForReaders(1); + + writer.add("elem5", make_shared("value1")); + writer.update("elem5", make_shared("value2")); + writer.remove("elem5"); + + test(writer.hasReaders()); + writer.add("elem1", make_shared("value1")); + writer.update("elem1", make_shared("value2")); + writer.remove("elem1"); + + writer.update("elem2", make_shared("value1")); + + writer.remove("elem3"); + + writer.add("elem4", make_shared("value1")); + writer.waitForNoReaders(); + } + cout << "ok" << endl; + + cout << "testing filtered sample reader... " << flush; + { + Topic topic(node, "filtered reader key/value filter"); + topic.setSampleFilter( + "startswith", + [](const string& prefix) + { + return [prefix](const Sample& sample) + { + auto value = sample.getValue(); + return value.size() >= prefix.size() && value.compare(0, prefix.size(), prefix) == 0; + }; + }); + + auto writer1 = makeSingleKeyWriter(topic, "elem1", "", config); + writer1.waitForReaders(3); + test(writer1.hasReaders()); + writer1.add("value1"); + writer1.update("value2"); + writer1.remove(); + + auto writer2 = makeSingleKeyWriter(topic, "elem2", "", config); + writer2.waitForReaders(1); + writer2.update("value1"); + writer2.update("value2"); + writer2.update("value3"); + writer2.update("value4"); + writer2.update("value5"); + + auto writer3 = makeSingleKeyWriter(topic, "elem3", "", config); + writer3.waitForReaders(1); + writer3.update("nonvalue"); + writer3.update("value"); + + writer1.waitForNoReaders(); + writer2.waitForNoReaders(); + } + cout << "ok" << endl; + + cout << "testing topic collocated key reader and writer... " << flush; + { + Topic topic(node, "collocated"); + { + auto writer = makeSingleKeyWriter(topic, "test"); + writer.add("add"); + + auto reader = makeSingleKeyReader(topic, "test"); + test(reader.getNextUnread().getValue() == "add"); + } + { + auto reader = makeSingleKeyReader(topic, "test"); + + auto writer = makeSingleKeyWriter(topic, "test"); + writer.update("update"); + + test(reader.getNextUnread().getValue() == "update"); + } + } + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/events/msbuild/reader/packages.config b/cpp/test/DataStorm/events/msbuild/reader/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/events/msbuild/reader/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj b/cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj new file mode 100644 index 00000000000..09c79fe95da --- /dev/null +++ b/cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {5EE1FFB1-A9CF-4737-A1FA-575B51F8AE00} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj.filters b/cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj.filters new file mode 100644 index 00000000000..fa784cd87e6 --- /dev/null +++ b/cpp/test/DataStorm/events/msbuild/reader/reader.vcxproj.filters @@ -0,0 +1,88 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {fcef0a0c-3c3e-43f2-8584-8850460418b6} + ice + + + {6d35b729-7499-4927-8b1d-344747798f3a} + + + {a97fe3cb-c593-4e50-bbbe-415511aed4dd} + + + {0c3eb08a-c284-489a-8e6f-c7501179ee69} + + + {a0fecb4c-510f-4425-9f5d-e80b53d6c905} + + + {f0a4408b-c54f-4b07-8092-d2394aea6c07} + + + {f9da6281-b997-452f-80b1-47d9b981c1b8} + + + {fe4db0c6-e9d2-4eb0-93b2-daa9fcd4e42a} + + + {0281fd82-ed95-42a3-a941-d84d5daf06c0} + + + {da1bc916-c3e8-423f-b68d-9446a40a33b4} + + + {a5c7ef45-3e6a-4729-b215-cfb91a156ec2} + + + {4cba710e-1737-4813-a306-3e3d46610032} + + + {2f850c71-5340-46fe-ae9a-a0ab87d74d65} + + + + + + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + + + + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/events/msbuild/writer/packages.config b/cpp/test/DataStorm/events/msbuild/writer/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/events/msbuild/writer/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..d532ffc4bb1 --- /dev/null +++ b/cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {A368BD9C-2CBC-4302-9A7F-C96BA11D62BB} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..ce58f7e3e38 --- /dev/null +++ b/cpp/test/DataStorm/events/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,88 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {595c4e14-d980-4550-bde7-13a7d7d28da1} + ice + + + {94cc4fb1-b503-4d53-aed0-92ffb05e2751} + + + {150f8f3b-6806-43f8-87e5-f3f4e02166ea} + + + {17bf4a06-3c0f-462b-bfa3-75845698d90b} + + + {3c711416-11a7-4d1c-b147-562628767972} + + + {05c6ebcc-9a77-4505-b505-446bead24086} + + + {ce343315-9386-415e-bd3f-aad0ac619039} + + + {e283e6ed-a7d5-4535-8fd8-cc323e4f56ca} + + + {f4a18664-766b-448d-a927-afa520583c91} + + + {90839bd1-a2a9-405b-87cb-8fdef9f13d16} + + + {dedf3f23-3983-4832-82ec-ff80ec96cd78} + + + {d2a17450-c1af-497f-9a4e-78dab58a0f0e} + + + {57141d1c-4e07-46d9-a545-5ca850c763f3} + + + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + + + + + + + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/events/test.py b/cpp/test/DataStorm/events/test.py new file mode 100644 index 00000000000..39aaca84ea3 --- /dev/null +++ b/cpp/test/DataStorm/events/test.py @@ -0,0 +1,19 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Reader, Writer +from Util import ClientServerTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2, + "Ice.Trace.Protocol" : 2, + "Ice.Trace.Network" : 2 +} + +TestSuite( + __file__, + [ ClientServerTestCase(client = Writer(), server = Reader(), traceProps=traceProps) ], + runOnMainThread=True) diff --git a/cpp/test/DataStorm/partial/Makefile.mk b/cpp/test/DataStorm/partial/Makefile.mk new file mode 100644 index 00000000000..a275bcdbe7f --- /dev/null +++ b/cpp/test/DataStorm/partial/Makefile.mk @@ -0,0 +1,11 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = reader writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_reader_sources = Reader.cpp Test.ice +$(test)_writer_sources = Writer.cpp Test.ice + +tests += $(test) diff --git a/cpp/test/DataStorm/partial/Reader.cpp b/cpp/test/DataStorm/partial/Reader.cpp new file mode 100644 index 00000000000..bb6dcbfbb36 --- /dev/null +++ b/cpp/test/DataStorm/partial/Reader.cpp @@ -0,0 +1,69 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; +using namespace Test; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + Topic topic(node, "topic"); + + ReaderConfig config; + config.sampleCount = -1; + config.clearHistory = ClearHistoryPolicy::Never; + topic.setReaderDefaultConfig(config); + + topic.setUpdater("price", [](Stock& stock, float price) { stock.price = price; }); + + { + auto reader = makeSingleKeyReader(topic, "AAPL"); + auto sample = reader.getNextUnread(); + test(sample.getEvent() == SampleEvent::Add); + test(sample.getValue().price == 12.0f); + + sample = reader.getNextUnread(); + test(sample.getEvent() == SampleEvent::PartialUpdate); + test(sample.getUpdateTag() == "price"); + test(sample.getValue().price == 15.0f); + + sample = reader.getNextUnread(); + test(sample.getEvent() == SampleEvent::PartialUpdate); + test(sample.getUpdateTag() == "price"); + test(sample.getValue().price == 18.0f); + + // Late joining reader should still receive update events instead of partial updates + auto reader2 = makeSingleKeyReader(topic, "AAPL"); + sample = reader2.getNextUnread(); + test(sample.getEvent() == SampleEvent::Add); + test(sample.getValue().price == 12.0f); + + sample = reader2.getNextUnread(); + test(sample.getEvent() == SampleEvent::PartialUpdate); + test(sample.getValue().price == 15.0f); + + sample = reader2.getNextUnread(); + test(sample.getEvent() == SampleEvent::PartialUpdate); + test(sample.getValue().price == 18.0f); + + // Late joining reader with limited sample count should receive one Update event and a PartialUpdate event + auto reader3 = makeSingleKeyReader(topic, "AAPL", "", ReaderConfig(2)); + sample = reader3.getNextUnread(); + test(sample.getEvent() == SampleEvent::Update); + test(sample.getValue().price == 15.0f); + + sample = reader3.getNextUnread(); + test(sample.getEvent() == SampleEvent::PartialUpdate); + test(sample.getValue().price == 18.0f); + } + + return 0; +} diff --git a/cpp/test/DataStorm/partial/Test.ice b/cpp/test/DataStorm/partial/Test.ice new file mode 100644 index 00000000000..7ff05617d3e --- /dev/null +++ b/cpp/test/DataStorm/partial/Test.ice @@ -0,0 +1,14 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +module Test +{ + +struct Stock +{ + float price; + float lastBid; + float laskAsk; +}; + +}; diff --git a/cpp/test/DataStorm/partial/Writer.cpp b/cpp/test/DataStorm/partial/Writer.cpp new file mode 100644 index 00000000000..4f5f0267527 --- /dev/null +++ b/cpp/test/DataStorm/partial/Writer.cpp @@ -0,0 +1,42 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +#if defined(_MSC_VER) +# pragma warning(4 : 4244) +#endif + +#include "DataStorm/DataStorm.h" +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; +using namespace Test; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + Topic topic(node, "topic"); + + WriterConfig config; + config.sampleCount = -1; + config.clearHistory = ClearHistoryPolicy::Never; + topic.setWriterDefaultConfig(config); + + topic.setUpdater("price", [](Stock& stock, float price) { stock.price = price; }); + + cout << "testing partial update... " << flush; + { + auto writer = makeSingleKeyWriter(topic, "AAPL"); + writer.waitForReaders(); + writer.add(Stock{12.0f, 13.0f, 14.0f}); + writer.partialUpdate("price")(15.0f); + writer.partialUpdate("price")(18); + writer.waitForNoReaders(); + } + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/partial/msbuild/reader/packages.config b/cpp/test/DataStorm/partial/msbuild/reader/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/partial/msbuild/reader/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj b/cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj new file mode 100644 index 00000000000..f58636d4119 --- /dev/null +++ b/cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {3EAAF1DE-8673-4D16-AA36-F522C5C21F79} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj.filters b/cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj.filters new file mode 100644 index 00000000000..fa784cd87e6 --- /dev/null +++ b/cpp/test/DataStorm/partial/msbuild/reader/reader.vcxproj.filters @@ -0,0 +1,88 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {fcef0a0c-3c3e-43f2-8584-8850460418b6} + ice + + + {6d35b729-7499-4927-8b1d-344747798f3a} + + + {a97fe3cb-c593-4e50-bbbe-415511aed4dd} + + + {0c3eb08a-c284-489a-8e6f-c7501179ee69} + + + {a0fecb4c-510f-4425-9f5d-e80b53d6c905} + + + {f0a4408b-c54f-4b07-8092-d2394aea6c07} + + + {f9da6281-b997-452f-80b1-47d9b981c1b8} + + + {fe4db0c6-e9d2-4eb0-93b2-daa9fcd4e42a} + + + {0281fd82-ed95-42a3-a941-d84d5daf06c0} + + + {da1bc916-c3e8-423f-b68d-9446a40a33b4} + + + {a5c7ef45-3e6a-4729-b215-cfb91a156ec2} + + + {4cba710e-1737-4813-a306-3e3d46610032} + + + {2f850c71-5340-46fe-ae9a-a0ab87d74d65} + + + + + + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + + + + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/partial/msbuild/writer/packages.config b/cpp/test/DataStorm/partial/msbuild/writer/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/partial/msbuild/writer/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..e237415ba9a --- /dev/null +++ b/cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {71506AF2-B7C4-48D7-AE6E-4ADA8B72A87C} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..ce58f7e3e38 --- /dev/null +++ b/cpp/test/DataStorm/partial/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,88 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {595c4e14-d980-4550-bde7-13a7d7d28da1} + ice + + + {94cc4fb1-b503-4d53-aed0-92ffb05e2751} + + + {150f8f3b-6806-43f8-87e5-f3f4e02166ea} + + + {17bf4a06-3c0f-462b-bfa3-75845698d90b} + + + {3c711416-11a7-4d1c-b147-562628767972} + + + {05c6ebcc-9a77-4505-b505-446bead24086} + + + {ce343315-9386-415e-bd3f-aad0ac619039} + + + {e283e6ed-a7d5-4535-8fd8-cc323e4f56ca} + + + {f4a18664-766b-448d-a927-afa520583c91} + + + {90839bd1-a2a9-405b-87cb-8fdef9f13d16} + + + {dedf3f23-3983-4832-82ec-ff80ec96cd78} + + + {d2a17450-c1af-497f-9a4e-78dab58a0f0e} + + + {57141d1c-4e07-46d9-a545-5ca850c763f3} + + + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + + + + + + + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/partial/test.py b/cpp/test/DataStorm/partial/test.py new file mode 100644 index 00000000000..11c49ec03ba --- /dev/null +++ b/cpp/test/DataStorm/partial/test.py @@ -0,0 +1,17 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Reader, Writer +from Util import ClientServerTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2 +} + +TestSuite( + __file__, + [ ClientServerTestCase(client = Writer(), server = Reader(), traceProps=traceProps) ], + runOnMainThread=True) diff --git a/cpp/test/DataStorm/reliability/Makefile.mk b/cpp/test/DataStorm/reliability/Makefile.mk new file mode 100644 index 00000000000..e1761617d4d --- /dev/null +++ b/cpp/test/DataStorm/reliability/Makefile.mk @@ -0,0 +1,11 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = reader writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_reader_sources = Reader.cpp +$(test)_writer_sources = Writer.cpp + +tests += $(test) diff --git a/cpp/test/DataStorm/reliability/Reader.cpp b/cpp/test/DataStorm/reliability/Reader.cpp new file mode 100644 index 00000000000..86133abcda0 --- /dev/null +++ b/cpp/test/DataStorm/reliability/Reader.cpp @@ -0,0 +1,64 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "TestHelper.h" + +#include + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + ReaderConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + + { + Topic topic(node, "string"); + auto barrier = makeSingleKeyWriter(topic, "barrier"); + barrier.waitForReaders(); + barrier.update(""); + auto reader = makeSingleKeyReader(topic, "element", "", config); + test(reader.getNextUnread().getValue() == "add"); + test(reader.getNextUnread().getValue() == "update1"); + test(reader.getNextUnread().getValue() == "update2"); + barrier.update(""); + test(reader.getNextUnread().getValue() == "update3"); + barrier.update(""); + barrier.waitForNoReaders(); + } + + { + Topic topic(node, "int"); + auto reader = makeSingleKeyReader(topic, "element", "", config); + for (int i = 0; i < 1000; ++i) + { + auto sample = reader.getNextUnread(); + if (sample.getValue() != i) + { + cerr << "unexpected sample: " << sample.getValue() << " expected:" << i << endl; + test(false); + } + if ((i % 50) == 0) + { + auto connection = node.getSessionConnection(sample.getSession()); + while (!connection) + { + this_thread::sleep_for(chrono::milliseconds(200)); + connection = node.getSessionConnection(sample.getSession()); + } + connection->close().get(); + } + } + auto writer = makeSingleKeyWriter(topic, "barrier"); + writer.waitForReaders(); + writer.update(0); + writer.waitForNoReaders(); + } + return 0; +} diff --git a/cpp/test/DataStorm/reliability/Writer.cpp b/cpp/test/DataStorm/reliability/Writer.cpp new file mode 100644 index 00000000000..f4124c1a212 --- /dev/null +++ b/cpp/test/DataStorm/reliability/Writer.cpp @@ -0,0 +1,54 @@ +// ********************************************************************** +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +// ********************************************************************** + +#include "DataStorm/DataStorm.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + WriterConfig config; + config.clearHistory = ClearHistoryPolicy::Never; + + cout << "testing writer connection closure... " << flush; + { + Topic topic(node, "string"); + auto writer = makeSingleKeyWriter(topic, "element", "", config); + writer.add("add"); + writer.update("update1"); + auto barrier = makeSingleKeyReader(topic, "barrier"); + auto sample = barrier.getNextUnread(); + writer.waitForReaders(); + auto connection = node.getSessionConnection(sample.getSession()); + test(connection); + connection->close().get(); + writer.update("update2"); + barrier.getNextUnread(); + writer.update("update3"); + barrier.getNextUnread(); + } + cout << "ok" << endl; + + cout << "testing reader connection closure... " << flush; + { + Topic topic(node, "int"); + auto writer = makeSingleKeyWriter(topic, "element", "", config); + writer.waitForReaders(); + for (int i = 0; i < 1000; ++i) + { + writer.update(i); + } + makeSingleKeyReader(topic, "barrier").getNextUnread(); + } + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/reliability/msbuild/reader/packages.config b/cpp/test/DataStorm/reliability/msbuild/reader/packages.config new file mode 100644 index 00000000000..9a5f7b2c74a --- /dev/null +++ b/cpp/test/DataStorm/reliability/msbuild/reader/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj b/cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj new file mode 100644 index 00000000000..0d530ff4695 --- /dev/null +++ b/cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj @@ -0,0 +1,102 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {18B9E886-A355-4CCC-8E27-84900C516DA8} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj.filters b/cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj.filters new file mode 100644 index 00000000000..9eff3ef7ab8 --- /dev/null +++ b/cpp/test/DataStorm/reliability/msbuild/reader/reader.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/reliability/msbuild/writer/packages.config b/cpp/test/DataStorm/reliability/msbuild/writer/packages.config new file mode 100644 index 00000000000..9a5f7b2c74a --- /dev/null +++ b/cpp/test/DataStorm/reliability/msbuild/writer/packages.config @@ -0,0 +1,4 @@ + + + + diff --git a/cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..f743ec72d15 --- /dev/null +++ b/cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj @@ -0,0 +1,102 @@ + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + {71A33CBC-71F6-434B-BB90-620710269310} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + ..\..;%(AdditionalIncludeDirectories) + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + diff --git a/cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..bc887ef6719 --- /dev/null +++ b/cpp/test/DataStorm/reliability/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,19 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + + + Source Files + + + + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/reliability/test.py b/cpp/test/DataStorm/reliability/test.py new file mode 100644 index 00000000000..af217346cf8 --- /dev/null +++ b/cpp/test/DataStorm/reliability/test.py @@ -0,0 +1,81 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Reader, Writer, Node, NodeTestCase +from Util import ClientServerTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2, + "Ice.Trace.Protocol": 1, +# "Ice.Trace.Network": 3 +} + +clientProps = { + "DataStorm.Node.Multicast.Enabled": 0, + "DataStorm.Node.Server.Enabled": 0, + "DataStorm.Node.ConnectTo": "tcp -p {port1}" +} + +client2Props = { + "DataStorm.Node.Multicast.Enabled": 0, + "DataStorm.Node.Server.Enabled": 0, + "DataStorm.Node.ConnectTo": "tcp -p {port2}" +} + +serverProps = { + "DataStorm.Node.Multicast.Enabled": 0, + "DataStorm.Node.Server.Enabled": 1, + "DataStorm.Node.Server.Endpoints": "tcp -p {port1}", + "DataStorm.Node.ConnectTo": "" +} + +server2Props = { + "DataStorm.Node.Multicast.Enabled": 0, + "DataStorm.Node.Server.Enabled": 1, + "DataStorm.Node.Server.Endpoints": "tcp -p {port2}", + "DataStorm.Node.ConnectTo": "tcp -p {port1}" +} + +serverAnyProps = { + "DataStorm.Node.Multicast.Enabled": 0, + "DataStorm.Node.Server.Endpoints": "tcp", + "DataStorm.Node.ConnectTo": "tcp -p {port1}", +} + +props = [ + ("reader as client with writer as server", clientProps, serverProps, None, None, False), + ("reader as server with writer as client", serverProps, clientProps, None, None, False), + ("reader as client with writer as server", clientProps, serverProps, None, None, True), + ("reader as server with writer as client", serverProps, clientProps, None, None, True), + ("reader/writer as client with node", clientProps, clientProps, serverProps, None, False), + ("reader as client and writer as server with node", clientProps, serverAnyProps, serverProps, None, False), + ("reader as server and writer as client with node", serverAnyProps, clientProps, serverProps, None, False), + ("reader/writer as client with node", clientProps, clientProps, serverProps, None, True), + ("reader as client and writer as server with node", clientProps, serverAnyProps, serverProps, None, True), + ("reader as server and writer as client with node", serverAnyProps, clientProps, serverProps, None, True), + ("reader/writer as client with 2 nodes", clientProps, client2Props, serverProps, server2Props, False), + ("reader as client and writer as server with 2 nodes", clientProps, serverAnyProps, serverProps, server2Props, False), + ("reader as server and writer as client with 2 nodes", serverAnyProps, clientProps, serverProps, server2Props, False), + ("reader/writer as client with 2 nodes", clientProps, client2Props, serverProps, server2Props, True), + ("reader as client and writer as server with 2 nodes", clientProps, serverAnyProps, serverProps, server2Props, True), + ("reader as server and writer as client with 2 nodes", serverAnyProps, clientProps, serverProps, server2Props, True), +] + +testcases = [] +for (name, readerProps, writerProps, nodeProps, node2Props, reversedStart) in props: + if reversedStart: + name += " (reversed start order)" + c = Writer(props=writerProps) if not reversedStart else Reader(props=readerProps) + s = Reader(props=readerProps) if not reversedStart else Writer(props=writerProps) + if node2Props: + nodes = [Node(desc="node1", props=nodeProps), Node(desc="node2", props=node2Props)] + testcases.append(NodeTestCase(name=name, client=c, server=s, nodes=nodes, traceProps=traceProps)) + elif nodeProps: + testcases.append(NodeTestCase(name=name, client=c, server=s, nodeProps=nodeProps, traceProps=traceProps)) + else: + testcases.append(ClientServerTestCase(name=name, client=c, server=s, traceProps=traceProps)) + +TestSuite(__file__, testcases) diff --git a/cpp/test/DataStorm/types/Makefile.mk b/cpp/test/DataStorm/types/Makefile.mk new file mode 100644 index 00000000000..a275bcdbe7f --- /dev/null +++ b/cpp/test/DataStorm/types/Makefile.mk @@ -0,0 +1,11 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +$(test)_programs = reader writer +$(test)_dependencies = DataStorm Ice TestCommon + +$(test)_reader_sources = Reader.cpp Test.ice +$(test)_writer_sources = Writer.cpp Test.ice + +tests += $(test) diff --git a/cpp/test/DataStorm/types/Reader.cpp b/cpp/test/DataStorm/types/Reader.cpp new file mode 100644 index 00000000000..95ec567677a --- /dev/null +++ b/cpp/test/DataStorm/types/Reader.cpp @@ -0,0 +1,119 @@ +// ********************************************************************** +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +// ********************************************************************** + +#if defined(_WIN32) +# pragma warning(disable : 4503) // decorated name length exceeded, name was truncated +#endif + +#include "DataStorm/DataStorm.h" +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; +using namespace Test; + +namespace +{ + + enum class color : unsigned char + { + blue, + red, + }; + + template bool compare(T v1, T v2) { return v1 == v2; } + + template bool compare(shared_ptr v1, shared_ptr v2) { return *v1 == *v2; } + + template void testReader(T topic, A add, U update) + { + topic.setReaderDefaultConfig(ReaderConfig(-1, std::nullopt, ClearHistoryPolicy::Never)); + map readers; + for (auto p : add) + { + readers.emplace(p.first, makeSingleKeyReader(topic, p.first)); + auto s = readers.at(p.first).getNextUnread(); + test(s.getEvent() == SampleEvent::Add && compare(s.getValue(), p.second)); + } + for (auto p : update) + { + auto s = readers.at(p.first).getNextUnread(); + test(s.getEvent() == SampleEvent::Update && compare(s.getValue(), p.second)); + } + for (auto p : add) + { + auto s = readers.at(p.first).getNextUnread(); + test(s.getEvent() == SampleEvent::Remove); + } + }; + +} // namespace + +namespace DataStorm +{ + + template<> struct Decoder + { + static color decode(const Ice::CommunicatorPtr&, const vector& data) + { + return static_cast(data[0]); + } + }; + + template<> struct Encoder + { + static vector encode(const Ice::CommunicatorPtr&, const color& value) + { + return {static_cast(value)}; + } + }; + +} + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + testReader( + Topic(node, "stringstring"), + map{{"k1", "v1"}, {"k2", "v2"}}, + map{{"k1", "u1"}, {"k2", "u2"}}); + testReader( + Topic(node, "intstring"), + map{{1, "v1"}, {2, "v2"}}, + map{{1, "u1"}, {2, "u2"}}); + testReader( + Topic(node, "intdouble"), + map{{1, 2.0}, {2, 8.7}}, + map{{1, 4.0}, {2, 7.8}}); + testReader( + Topic(node, "stringstruct"), + map{{"k1", {"firstName", "lastName", 10}}, {"k2", {"fn", "ln", 12}}}, + map{{"k1", {"firstName", "lastName", 18}}, {"k2", {"fn", "ln", 15}}}); + testReader( + Topic(node, "structstring"), + map{{{"firstName", "lastName", 10}, "v2"}, {{"fn", "ln", 12}, "v3"}}, + map{{{"firstName", "lastName", 10}, "v4"}, {{"fn", "ln", 12}, "v5"}}); + // TODO enable class testing + /*testReader(Topic(node, "stringclassbyvalue"), + map { { "k1", Extended("v1", 8) }, + { "k2", Extended("v2", 8) } }, + map { { "k1", Extended("v1", 10) }, + { "k2", Extended("v2", 10) } }); + testReader(Topic>(node, "stringclassbyref"), + map> { { "k1", make_shared("v1") }, + { "k2", make_shared("v2") } + }, map> { { "k1", make_shared("v1", 10) }, + { "k2", make_shared("v2", + 10) } });*/ + testReader( + Topic(node, "enumstring"), + map{{color::blue, "v1"}, {color::red, "v2"}}, + map{{color::blue, "u1"}, {color::red, "u2"}}); + return 0; +} diff --git a/cpp/test/DataStorm/types/Test.ice b/cpp/test/DataStorm/types/Test.ice new file mode 100644 index 00000000000..162d4abfd1a --- /dev/null +++ b/cpp/test/DataStorm/types/Test.ice @@ -0,0 +1,24 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +module Test +{ + +struct StructValue +{ + string firstName; + string lastName; + int age; +}; + +class Base +{ + string b; +}; + +class Extended extends Base +{ + int e; +}; + +}; diff --git a/cpp/test/DataStorm/types/Writer.cpp b/cpp/test/DataStorm/types/Writer.cpp new file mode 100644 index 00000000000..5a5ffff8d33 --- /dev/null +++ b/cpp/test/DataStorm/types/Writer.cpp @@ -0,0 +1,135 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// + +#include "DataStorm/DataStorm.h" +#include "Test.h" +#include "TestHelper.h" + +using namespace DataStorm; +using namespace std; +using namespace Test; + +namespace +{ + + enum class color : unsigned char + { + blue, + red, + }; + + template void testWriter(T topic, A add, U update) + { + topic.setWriterDefaultConfig(WriterConfig(-1, std::nullopt, ClearHistoryPolicy::Never)); + using WriterType = decltype(makeSingleKeyWriter(topic, typename decltype(add)::key_type())); + map writers; + for (auto p : add) + { + writers.emplace(p.first, makeSingleKeyWriter(topic, p.first)); + writers.at(p.first).waitForReaders(); + writers.at(p.first).add(p.second); + } + for (auto p : update) + { + writers.at(p.first).update(p.second); + } + for (auto p : add) + { + writers.at(p.first).remove(); + } + for (auto p : add) + { + writers.at(p.first).waitForNoReaders(); + } + }; + +} + +namespace DataStorm +{ + + template<> struct Decoder + { + static color decode(const Ice::CommunicatorPtr&, const vector& data) + { + return static_cast(data[0]); + } + }; + + template<> struct Encoder + { + static vector encode(const Ice::CommunicatorPtr&, const color& value) + { + return {static_cast(value)}; + } + }; + +} + +int +main(int argc, char* argv[]) +{ + Node node(argc, argv); + + cout << "testing string/string... " << flush; + testWriter( + Topic(node, "stringstring"), + map{{"k1", "v1"}, {"k2", "v2"}}, + map{{"k1", "u1"}, {"k2", "u2"}}); + cout << "ok" << endl; + + cout << "testing int/string... " << flush; + testWriter( + Topic(node, "intstring"), + map{{1, "v1"}, {2, "v2"}}, + map{{1, "u1"}, {2, "u2"}}); + cout << "ok" << endl; + + cout << "testing int/double... " << flush; + testWriter( + Topic(node, "intdouble"), + map{{1, 2.0}, {2, 8.7}}, + map{{1, 4.0}, {2, 7.8}}); + cout << "ok" << endl; + + cout << "testing string/struct... " << flush; + testWriter( + Topic(node, "stringstruct"), + map{{"k1", {"firstName", "lastName", 10}}, {"k2", {"fn", "ln", 12}}}, + map{{"k1", {"firstName", "lastName", 18}}, {"k2", {"fn", "ln", 15}}}); + cout << "ok" << endl; + + cout << "testing struct/string... " << flush; + testWriter( + Topic(node, "structstring"), + map{{{"firstName", "lastName", 10}, "v2"}, {{"fn", "ln", 12}, "v3"}}, + map{{{"firstName", "lastName", 10}, "v4"}, {{"fn", "ln", 12}, "v5"}}); + cout << "ok" << endl; + + // TODO enable class tests + /*cout << "testing string/class by value... " << flush; + testWriter(Topic(node, "stringclassbyvalue"), + map { { string("k1"), Extended("v1", 8) }, + { string("k2"), Extended("v2", 8) } }, + map { { "k1", Extended("v1", 10) }, + { "k2", Extended("v2", 10) } }); + cout << "ok" << endl; + + cout << "testing string/class by ref... " << flush; + testWriter(Topic>(node, "stringclassbyref"), + map> { { "k1", make_shared("v1") }, + { "k2", make_shared("v2") } + }, map> { { "k1", make_shared("v1", 10) }, + { "k2", make_shared("v2", + 10) } }); cout << "ok" << endl;*/ + + cout << "testing enum/string... " << flush; + testWriter( + Topic(node, "enumstring"), + map{{color::blue, "v1"}, {color::red, "v2"}}, + map{{color::blue, "u1"}, {color::red, "u2"}}); + cout << "ok" << endl; + + return 0; +} diff --git a/cpp/test/DataStorm/types/msbuild/reader/packages.config b/cpp/test/DataStorm/types/msbuild/reader/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/types/msbuild/reader/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj b/cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj new file mode 100644 index 00000000000..35164ca2eb6 --- /dev/null +++ b/cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {924B7545-4B49-41E6-9FAC-30F7BAB7A518} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj.filters b/cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj.filters new file mode 100644 index 00000000000..fa784cd87e6 --- /dev/null +++ b/cpp/test/DataStorm/types/msbuild/reader/reader.vcxproj.filters @@ -0,0 +1,88 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {fcef0a0c-3c3e-43f2-8584-8850460418b6} + ice + + + {6d35b729-7499-4927-8b1d-344747798f3a} + + + {a97fe3cb-c593-4e50-bbbe-415511aed4dd} + + + {0c3eb08a-c284-489a-8e6f-c7501179ee69} + + + {a0fecb4c-510f-4425-9f5d-e80b53d6c905} + + + {f0a4408b-c54f-4b07-8092-d2394aea6c07} + + + {f9da6281-b997-452f-80b1-47d9b981c1b8} + + + {fe4db0c6-e9d2-4eb0-93b2-daa9fcd4e42a} + + + {0281fd82-ed95-42a3-a941-d84d5daf06c0} + + + {da1bc916-c3e8-423f-b68d-9446a40a33b4} + + + {a5c7ef45-3e6a-4729-b215-cfb91a156ec2} + + + {4cba710e-1737-4813-a306-3e3d46610032} + + + {2f850c71-5340-46fe-ae9a-a0ab87d74d65} + + + + + + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + + + + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/types/msbuild/writer/packages.config b/cpp/test/DataStorm/types/msbuild/writer/packages.config new file mode 100644 index 00000000000..d67b054a90d --- /dev/null +++ b/cpp/test/DataStorm/types/msbuild/writer/packages.config @@ -0,0 +1,5 @@ + + + + + diff --git a/cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj b/cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj new file mode 100644 index 00000000000..be01c0fb374 --- /dev/null +++ b/cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj @@ -0,0 +1,163 @@ + + + + + + + Debug + Win32 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + + + + + + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + true + true + true + ..\..\Test.ice + + + + {6CFEBEB6-88C6-4A41-847E-E02CFDCD12C8} + + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + Application + true + $(DefaultPlatformToolset) + + + Application + false + $(DefaultPlatformToolset) + + + + + + + + + + + + + + + + + + + + + + + + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + generated;..\..;%(AdditionalIncludeDirectories) + + + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + + diff --git a/cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj.filters b/cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj.filters new file mode 100644 index 00000000000..9c9f23f1651 --- /dev/null +++ b/cpp/test/DataStorm/types/msbuild/writer/writer.vcxproj.filters @@ -0,0 +1,90 @@ + + + + + {2efb87e2-44aa-4907-b445-4ded9dc175c7} + + + {fa2de026-c14d-4caf-904b-245988a34bec} + + + {595c4e14-d980-4550-bde7-13a7d7d28da1} + ice + + + {94cc4fb1-b503-4d53-aed0-92ffb05e2751} + + + {150f8f3b-6806-43f8-87e5-f3f4e02166ea} + + + {17bf4a06-3c0f-462b-bfa3-75845698d90b} + + + {3c711416-11a7-4d1c-b147-562628767972} + + + {05c6ebcc-9a77-4505-b505-446bead24086} + + + {ce343315-9386-415e-bd3f-aad0ac619039} + + + {e283e6ed-a7d5-4535-8fd8-cc323e4f56ca} + + + {f4a18664-766b-448d-a927-afa520583c91} + + + {90839bd1-a2a9-405b-87cb-8fdef9f13d16} + + + {dedf3f23-3983-4832-82ec-ff80ec96cd78} + + + {d2a17450-c1af-497f-9a4e-78dab58a0f0e} + + + {57141d1c-4e07-46d9-a545-5ca850c763f3} + + + + + Source Files + + + Source Files\Win32\Debug + + + Source Files\x64\Debug + + + Source Files\Win32\Release + + + Source Files\x64\Release + + + + + + + + Header Files\Win32\Debug + + + Header Files\x64\Debug + + + Header Files\Win32\Release + + + Header Files\x64\Release + + + + + Slice Files + + + \ No newline at end of file diff --git a/cpp/test/DataStorm/types/test.py b/cpp/test/DataStorm/types/test.py new file mode 100644 index 00000000000..11c49ec03ba --- /dev/null +++ b/cpp/test/DataStorm/types/test.py @@ -0,0 +1,17 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from DataStormUtil import Reader, Writer +from Util import ClientServerTestCase, TestSuite + +traceProps = { + "DataStorm.Trace.Topic" : 1, + "DataStorm.Trace.Session" : 3, + "DataStorm.Trace.Data" : 2 +} + +TestSuite( + __file__, + [ ClientServerTestCase(client = Writer(), server = Reader(), traceProps=traceProps) ], + runOnMainThread=True) diff --git a/scripts/DataStormUtil.py b/scripts/DataStormUtil.py new file mode 100644 index 00000000000..ec034a4f8f8 --- /dev/null +++ b/scripts/DataStormUtil.py @@ -0,0 +1,98 @@ +# +# Copyright (c) ZeroC, Inc. All rights reserved. +# + +from Util import ( + Client, + ClientServerTestCase, + Mapping, + ProcessFromBinDir, + Server, + Process, +) + +import re + +# Regex pattern to match placeholders like {port1}, {port2}, ..., {portXX} +port_pattern = re.compile(r'{port(\d+)}') + +class DataStormProcess(Process): + + def getEffectiveProps(self, current, props): + props = Process.getEffectiveProps(self, current, props) + for key, value in props.items(): + if key.startswith("DataStorm.Node.") and type(value) is str: + props[key] = port_pattern.sub( + lambda match: str(current.driver.getTestPort(10 + int(match.group(1)))), value) + + return props + +class Writer(Client, DataStormProcess): + processType = "writer" + + def __init__(self, instanceName=None, instance=None, *args, **kargs): + Client.__init__(self, *args, **kargs) + + def getEffectiveProps(self, current, props): + props = DataStormProcess.getEffectiveProps(self, current, props) + if not any(key.startswith("DataStorm.Node.") for key in props): + # Default properties for tests that don't specify any DataStorm.Node.* properties + props.update( + { + "DataStorm.Node.Server.Enabled": 0, + "DataStorm.Node.ConnectTo": f"tcp -p {current.driver.getTestPort(10)}" + }) + return props + +class Reader(Server, DataStormProcess): + processType = "reader" + + def __init__(self, instanceName=None, instance=None, *args, **kargs): + Server.__init__(self, *args, **kargs) + + def getEffectiveProps(self, current, props): + props = DataStormProcess.getEffectiveProps(self, current, props) + if not any(key.startswith("DataStorm.Node.") for key in props): + # Default properties for tests that don't specify any DataStorm.Node.* properties + props.update( + { + "DataStorm.Node.Server.Endpoints": f"tcp -p {current.driver.getTestPort(10)}" + }) + return props + +class Node(ProcessFromBinDir, Server, DataStormProcess): + def __init__(self, desc=None, *args, **kargs): + Server.__init__(self, "dsnode", mapping=Mapping.getByName("cpp"), desc=desc or "DataStorm node", *args, **kargs) + + def shutdown(self, current): + if self in current.processes: + current.processes[self].terminate() + + def getProps(self, current): + props = Server.getProps(self, current) + props['Ice.ProgramName'] = self.desc + return props + + def getEffectiveProps(self, current, props): + return DataStormProcess.getEffectiveProps(self, current, props) + +class NodeTestCase(ClientServerTestCase): + + def __init__(self, nodes=None, nodeProps=None, *args, **kargs): + ClientServerTestCase.__init__(self, *args, **kargs) + if nodes: + self.nodes = nodes + elif nodeProps: + self.nodes = [Node(props=nodeProps)] + else: + self.nodes = None + + def init(self, mapping, testsuite): + ClientServerTestCase.init(self, mapping, testsuite) + if self.nodes: + self.servers = self.nodes + self.servers + + def teardownClientSide(self, current, success): + if self.nodes: + for n in self.nodes: + n.shutdown(current) diff --git a/scripts/Util.py b/scripts/Util.py index 03d60dd4909..8c18b052db3 100644 --- a/scripts/Util.py +++ b/scripts/Util.py @@ -3683,6 +3683,8 @@ def _getDefaultSource(self, processType): "collocated": "Collocated.cpp", "subscriber": "Subscriber.cpp", "publisher": "Publisher.cpp", + "reader": "Reader.cpp", + "writer": "Writer.cpp" }[processType] def _getDefaultExe(self, processType): diff --git a/slice/DataStorm/Sample.ice b/slice/DataStorm/Sample.ice new file mode 100644 index 00000000000..62b7d59d39b --- /dev/null +++ b/slice/DataStorm/Sample.ice @@ -0,0 +1,38 @@ +// +// Copyright (c) ZeroC, Inc. All rights reserved. +// +#pragma once + +module DataStorm +{ + +/** + * The sample event. + * + * The sample event matches the operation used by the DataWriter to update + * the data element. It also provides information on what to expect from + * the sample. A sample with the Add or Update event always provide a value + * while a sample with the Remove type doesn't. + * + */ +enum SampleEvent +{ + /** The element has been added. */ + Add, + + /** The element has been updated. */ + Update, + + /** The element has been partially updated. */ + PartialUpdate, + + /** The element has been removed. */ + Remove, +} + +/** + * A sequence of sample type enumeration. + */ +sequence SampleEventSeq; + +}