From 934d871da0332bdc6cacee86fae458478c606d4a Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 10:01:11 -0800 Subject: [PATCH 01/19] PSync: move face,scheduler,keychain to children; move sendApplicationNack to PartialProducer Change-Id: Ia3c56b732c1e774570abba20f834089e2774e9e0 --- PSync/full-producer.cpp | 5 ++++- PSync/full-producer.hpp | 7 +++++++ PSync/partial-producer.cpp | 21 ++++++++++++++++++++- PSync/partial-producer.hpp | 16 ++++++++++++++++ PSync/producer-base.cpp | 20 -------------------- PSync/producer-base.hpp | 18 ------------------ tests/test-partial-producer.cpp | 15 +++++++++++++++ tests/test-producer-base.cpp | 19 ++----------------- 8 files changed, 64 insertions(+), 57 deletions(-) diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index a7a0b24..62d995d 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -38,7 +38,10 @@ FullProducer::FullProducer(const size_t expectedNumEntries, const UpdateCallback& onUpdateCallBack, ndn::time::milliseconds syncInterestLifetime, ndn::time::milliseconds syncReplyFreshness) - : ProducerBase(expectedNumEntries, face, syncPrefix, userPrefix, syncReplyFreshness) + : ProducerBase(expectedNumEntries, syncPrefix, userPrefix, syncReplyFreshness) + , m_face(face) + , m_scheduler(m_face.getIoService()) + , m_segmentPublisher(m_face, m_keyChain) , m_syncInterestLifetime(syncInterestLifetime) , m_onUpdate(onUpdateCallBack) { diff --git a/PSync/full-producer.hpp b/PSync/full-producer.hpp index f51ea7d..2c2015b 100644 --- a/PSync/full-producer.hpp +++ b/PSync/full-producer.hpp @@ -184,6 +184,13 @@ class FullProducer : public ProducerBase isFutureHash(const ndn::Name& prefix, const std::set& negative); private: + ndn::Face& m_face; + ndn::KeyChain m_keyChain; + ndn::Scheduler m_scheduler; + +PUBLIC_WITH_TESTS_ELSE_PROTECTED: + SegmentPublisher m_segmentPublisher; + std::map m_pendingEntries; ndn::time::milliseconds m_syncInterestLifetime; UpdateCallback m_onUpdate; diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index e75bc38..fd4368f 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -35,8 +35,11 @@ PartialProducer::PartialProducer(size_t expectedNumEntries, const ndn::Name& userPrefix, ndn::time::milliseconds syncReplyFreshness, ndn::time::milliseconds helloReplyFreshness) - : ProducerBase(expectedNumEntries, face, syncPrefix, + : ProducerBase(expectedNumEntries, syncPrefix, userPrefix, syncReplyFreshness, helloReplyFreshness) + , m_face(face) + , m_scheduler(m_face.getIoService()) + , m_segmentPublisher(m_face, m_keyChain) { m_registeredPrefix = m_face.registerPrefix(m_syncPrefix, [this] (const ndn::Name& syncPrefix) { @@ -249,4 +252,20 @@ PartialProducer::satisfyPendingSyncInterests(const ndn::Name& prefix) { } } +void +PartialProducer::sendApplicationNack(const ndn::Name& name) +{ + NDN_LOG_DEBUG("Sending application nack"); + ndn::Name dataName(name); + m_iblt.appendToName(dataName); + + dataName.appendSegment(0); + ndn::Data data(dataName); + data.setFreshnessPeriod(m_syncReplyFreshness); + data.setContentType(ndn::tlv::ContentType_Nack); + data.setFinalBlock(dataName[-1]); + m_keyChain.sign(data); + m_face.put(data); +} + } // namespace psync diff --git a/PSync/partial-producer.hpp b/PSync/partial-producer.hpp index 782078b..daa6faa 100644 --- a/PSync/partial-producer.hpp +++ b/PSync/partial-producer.hpp @@ -117,7 +117,23 @@ class PartialProducer : public ProducerBase void onSyncInterest(const ndn::Name& prefix, const ndn::Interest& interest); + /** + * @brief Sends a data packet with content type nack + * + * Producer sends a nack to consumer if consumer has very old IBF + * whose differences with latest IBF can't be decoded successfully + * + * @param name send application nack with this name + */ + void + sendApplicationNack(const ndn::Name& name); + PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: + ndn::Face& m_face; + ndn::KeyChain m_keyChain; + ndn::Scheduler m_scheduler; + + SegmentPublisher m_segmentPublisher; std::map m_pendingEntries; ndn::ScopedRegisteredPrefixHandle m_registeredPrefix; }; diff --git a/PSync/producer-base.cpp b/PSync/producer-base.cpp index d00fac0..5cffe7a 100644 --- a/PSync/producer-base.cpp +++ b/PSync/producer-base.cpp @@ -31,7 +31,6 @@ namespace psync { NDN_LOG_INIT(psync.ProducerBase); ProducerBase::ProducerBase(size_t expectedNumEntries, - ndn::Face& face, const ndn::Name& syncPrefix, const ndn::Name& userPrefix, ndn::time::milliseconds syncReplyFreshness, @@ -39,13 +38,10 @@ ProducerBase::ProducerBase(size_t expectedNumEntries, : m_iblt(expectedNumEntries) , m_expectedNumEntries(expectedNumEntries) , m_threshold(expectedNumEntries/2) - , m_face(face) - , m_scheduler(m_face.getIoService()) , m_syncPrefix(syncPrefix) , m_userPrefix(userPrefix) , m_syncReplyFreshness(syncReplyFreshness) , m_helloReplyFreshness(helloReplyFreshness) - , m_segmentPublisher(m_face, m_keyChain) , m_rng(ndn::random::getRandomNumberEngine()) { addUserNode(userPrefix); @@ -124,22 +120,6 @@ ProducerBase::updateSeqNo(const ndn::Name& prefix, uint64_t seq) m_iblt.insert(newHash); } -void -ProducerBase::sendApplicationNack(const ndn::Name& name) -{ - NDN_LOG_DEBUG("Sending application nack"); - ndn::Name dataName(name); - m_iblt.appendToName(dataName); - - dataName.appendSegment(0); - ndn::Data data(dataName); - data.setFreshnessPeriod(m_syncReplyFreshness); - data.setContentType(ndn::tlv::ContentType_Nack); - data.setFinalBlock(dataName[-1]); - m_keyChain.sign(data); - m_face.put(data); -} - void ProducerBase::onRegisterFailed(const ndn::Name& prefix, const std::string& msg) const { diff --git a/PSync/producer-base.hpp b/PSync/producer-base.hpp index 6463ea2..0570986 100644 --- a/PSync/producer-base.hpp +++ b/PSync/producer-base.hpp @@ -68,7 +68,6 @@ class ProducerBase * @param helloReplyFreshness freshness of hello data */ ProducerBase(size_t expectedNumEntries, - ndn::Face& face, const ndn::Name& syncPrefix, const ndn::Name& userPrefix, ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS, @@ -134,17 +133,6 @@ class ProducerBase return m_prefixes.find(prefix) != m_prefixes.end(); } - /** - * @brief Sends a data packet with content type nack - * - * Producer sends a nack to consumer if consumer has very old IBF - * whose differences with latest IBF can't be decoded successfully - * - * @param name send application nack with this name - */ - void - sendApplicationNack(const ndn::Name& name); - /** * @brief Logs a message if setting an interest filter fails * @@ -169,18 +157,12 @@ class ProducerBase // Value is prefix (and not prefix/seqNo) std::map m_hash2prefix; - ndn::Face& m_face; - ndn::KeyChain m_keyChain; - ndn::Scheduler m_scheduler; - ndn::Name m_syncPrefix; ndn::Name m_userPrefix; ndn::time::milliseconds m_syncReplyFreshness; ndn::time::milliseconds m_helloReplyFreshness; - SegmentPublisher m_segmentPublisher; - ndn::random::RandomNumberEngine& m_rng; }; diff --git a/tests/test-partial-producer.cpp b/tests/test-partial-producer.cpp index aa5df37..5c7e5ad 100644 --- a/tests/test-partial-producer.cpp +++ b/tests/test-partial-producer.cpp @@ -142,6 +142,21 @@ BOOST_AUTO_TEST_CASE(OnSyncInterest) BOOST_REQUIRE_NO_THROW(producer.onSyncInterest(syncInterestName, Interest(syncInterestName))); } +BOOST_AUTO_TEST_CASE(ApplicationNack) +{ + util::DummyClientFace face; + PartialProducer producer(40, face, Name("/psync"), Name("/testUser")); + + BOOST_CHECK_EQUAL(face.sentData.size(), 0); + producer.m_syncReplyFreshness = time::milliseconds(1000); + producer.sendApplicationNack(Name("test")); + face.processEvents(time::milliseconds(10)); + BOOST_CHECK_EQUAL(face.sentData.size(), 1); + + Data data = *face.sentData.begin(); + BOOST_CHECK_EQUAL(data.getContentType(), ndn::tlv::ContentType_Nack); +} + BOOST_AUTO_TEST_SUITE_END() } // namespace psync \ No newline at end of file diff --git a/tests/test-producer-base.cpp b/tests/test-producer-base.cpp index e775ad8..dc6e785 100644 --- a/tests/test-producer-base.cpp +++ b/tests/test-producer-base.cpp @@ -35,14 +35,14 @@ BOOST_AUTO_TEST_SUITE(TestProducerBase) BOOST_AUTO_TEST_CASE(Ctor) { util::DummyClientFace face; - BOOST_REQUIRE_NO_THROW(ProducerBase(40, face, Name("/psync"), Name("/testUser"))); + BOOST_REQUIRE_NO_THROW(ProducerBase(40, Name("/psync"), Name("/testUser"))); } BOOST_AUTO_TEST_CASE(Basic) { util::DummyClientFace face; Name userNode("/testUser"); - ProducerBase producerBase(40, face, Name("/psync"), userNode); + ProducerBase producerBase(40, Name("/psync"), userNode); // Hash table size should be 40 + 40/2 = 60 (which is perfectly divisible by N_HASH = 3) BOOST_CHECK_EQUAL(producerBase.m_iblt.getHashTable().size(), 60); BOOST_CHECK_EQUAL(producerBase.getSeqNo(userNode).value(), 0); @@ -65,21 +65,6 @@ BOOST_AUTO_TEST_CASE(Basic) producerBase.m_prefix2hash.end()); } -BOOST_AUTO_TEST_CASE(ApplicationNack) -{ - util::DummyClientFace face; - ProducerBase producerBase(40, face, Name("/psync"), Name("/testUser")); - - BOOST_CHECK_EQUAL(face.sentData.size(), 0); - producerBase.m_syncReplyFreshness = time::milliseconds(1000); - producerBase.sendApplicationNack(Name("test")); - face.processEvents(time::milliseconds(10)); - BOOST_CHECK_EQUAL(face.sentData.size(), 1); - - Data data = *face.sentData.begin(); - BOOST_CHECK_EQUAL(data.getContentType(), ndn::tlv::ContentType_Nack); -} - BOOST_AUTO_TEST_SUITE_END() } // namespace psync From 68ed347c3917311e38629bf235117779a0eb841a Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 11:08:52 -0800 Subject: [PATCH 02/19] PSync: prefix2hash->name2hash, hash2prefix->hash2name; ProducerBase: new function insertToIBF called by updateSeqNo Change-Id: I9c85017f6497bb1cf36f72ba2fa2d9f1ec3d188b --- PSync/full-producer.cpp | 6 ++++-- PSync/partial-producer.cpp | 3 ++- PSync/producer-base.cpp | 28 +++++++++++++++++----------- PSync/producer-base.hpp | 11 +++++++---- tests/test-producer-base.cpp | 13 +++++++------ 5 files changed, 37 insertions(+), 24 deletions(-) diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index 62d995d..32240f5 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -194,7 +194,8 @@ FullProducer::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& i State state; for (const auto& hash : positive) { - const ndn::Name& prefix = m_hash2prefix[hash]; + ndn::Name prefix = m_hash2name[hash]; + prefix = prefix.getPrefix(prefix.size() - 1); // Don't sync up sequence number zero if (m_prefixes[prefix] != 0 && !isFutureHash(prefix.toUri(), negative)) { state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); @@ -309,7 +310,8 @@ FullProducer::satisfyPendingInterests() State state; for (const auto& hash : positive) { - ndn::Name prefix = m_hash2prefix[hash]; + ndn::Name prefix = m_hash2name[hash]; + prefix = prefix.getPrefix(prefix.size() - 1); if (m_prefixes[prefix] != 0) { state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index fd4368f..8e2f5db 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -174,7 +174,8 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in NDN_LOG_TRACE("Size of positive set " << positive.size()); NDN_LOG_TRACE("Size of negative set " << negative.size()); for (const auto& hash : positive) { - ndn::Name prefix = m_hash2prefix[hash]; + ndn::Name prefix = m_hash2name[hash]; + prefix = prefix.getPrefix(prefix.size() - 1); if (bf.contains(prefix.toUri())) { // generate data state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); diff --git a/PSync/producer-base.cpp b/PSync/producer-base.cpp index 5cffe7a..312113a 100644 --- a/PSync/producer-base.cpp +++ b/PSync/producer-base.cpp @@ -68,11 +68,11 @@ ProducerBase::removeUserNode(const ndn::Name& prefix) m_prefixes.erase(it); ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seqNo); - auto hashIt = m_prefix2hash.find(prefixWithSeq); - if (hashIt != m_prefix2hash.end()) { + auto hashIt = m_name2hash.find(prefixWithSeq); + if (hashIt != m_name2hash.end()) { uint32_t hash = hashIt->second; - m_prefix2hash.erase(hashIt); - m_hash2prefix.erase(hash); + m_name2hash.erase(hashIt); + m_hash2name.erase(hash); m_iblt.erase(hash); } } @@ -102,11 +102,11 @@ ProducerBase::updateSeqNo(const ndn::Name& prefix, uint64_t seq) // Because we don't insert zeroth prefix in IBF so no need to delete that if (oldSeq != 0) { ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(oldSeq); - auto hashIt = m_prefix2hash.find(prefixWithSeq); - if (hashIt != m_prefix2hash.end()) { + auto hashIt = m_name2hash.find(prefixWithSeq); + if (hashIt != m_name2hash.end()) { uint32_t hash = hashIt->second; - m_prefix2hash.erase(hashIt); - m_hash2prefix.erase(hash); + m_name2hash.erase(hashIt); + m_hash2name.erase(hash); m_iblt.erase(hash); } } @@ -114,9 +114,15 @@ ProducerBase::updateSeqNo(const ndn::Name& prefix, uint64_t seq) // Insert the new seq no it->second = seq; ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seq); - uint32_t newHash = murmurHash3(N_HASHCHECK, prefixWithSeq.toUri()); - m_prefix2hash[prefixWithSeq] = newHash; - m_hash2prefix[newHash] = prefix; + insertToIBF(prefixWithSeq); +} + +void +ProducerBase::insertToIBF(const ndn::Name& prefix) +{ + uint32_t newHash = murmurHash3(N_HASHCHECK, prefix.toUri()); + m_name2hash[prefix] = newHash; + m_hash2name[newHash] = prefix; m_iblt.insert(newHash); } diff --git a/PSync/producer-base.hpp b/PSync/producer-base.hpp index 0570986..8f4c529 100644 --- a/PSync/producer-base.hpp +++ b/PSync/producer-base.hpp @@ -112,6 +112,9 @@ class ProducerBase void removeUserNode(const ndn::Name& prefix); + void + insertToIBF(const ndn::Name& prefix); + PSYNC_PUBLIC_WITH_TESTS_ELSE_PROTECTED: /** * @brief Update m_prefixes and IBF with the given prefix and seq @@ -152,10 +155,10 @@ class ProducerBase // prefix and sequence number std::map m_prefixes; // Just for looking up hash faster (instead of calculating it again) - // Only used in updateSeqNo, prefix/seqNo is the key - std::map m_prefix2hash; - // Value is prefix (and not prefix/seqNo) - std::map m_hash2prefix; + // Only used in updateSeqNo, name (arbitrary or /prefix/seq) is the key + std::map m_name2hash; + // Value is name + std::map m_hash2name; ndn::Name m_syncPrefix; ndn::Name m_userPrefix; diff --git a/tests/test-producer-base.cpp b/tests/test-producer-base.cpp index dc6e785..5b32a06 100644 --- a/tests/test-producer-base.cpp +++ b/tests/test-producer-base.cpp @@ -51,18 +51,19 @@ BOOST_AUTO_TEST_CASE(Basic) BOOST_CHECK(producerBase.getSeqNo(userNode.toUri()).value() == 1); std::string prefixWithSeq = Name(userNode).appendNumber(1).toUri(); - uint32_t hash = producerBase.m_prefix2hash[prefixWithSeq]; - BOOST_CHECK_EQUAL(producerBase.m_hash2prefix[hash], userNode.toUri()); + uint32_t hash = producerBase.m_name2hash[prefixWithSeq]; + ndn::Name prefix = producerBase.m_hash2name[hash]; + BOOST_CHECK_EQUAL(prefix.getPrefix(prefix.size() - 1), userNode.toUri()); producerBase.removeUserNode(userNode); BOOST_CHECK(producerBase.getSeqNo(userNode.toUri()) == ndn::nullopt); - BOOST_CHECK(producerBase.m_prefix2hash.find(prefixWithSeq) == producerBase.m_prefix2hash.end()); - BOOST_CHECK(producerBase.m_hash2prefix.find(hash) == producerBase.m_hash2prefix.end()); + BOOST_CHECK(producerBase.m_name2hash.find(prefixWithSeq) == producerBase.m_name2hash.end()); + BOOST_CHECK(producerBase.m_hash2name.find(hash) == producerBase.m_hash2name.end()); Name nonExistentUserNode("/notAUser"); producerBase.updateSeqNo(nonExistentUserNode, 1); - BOOST_CHECK(producerBase.m_prefix2hash.find(Name(nonExistentUserNode).appendNumber(1).toUri()) == - producerBase.m_prefix2hash.end()); + BOOST_CHECK(producerBase.m_name2hash.find(Name(nonExistentUserNode).appendNumber(1).toUri()) == + producerBase.m_name2hash.end()); } BOOST_AUTO_TEST_SUITE_END() From e6b2f01efa270c205085c015551d1e77e0e43bf6 Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 11:25:42 -0800 Subject: [PATCH 03/19] PSync: minor modifications for better readability Change-Id: I29e3b3fba8e6eaee67a324bb66467d4d3bab5ecf --- PSync/full-producer.cpp | 13 +++++++------ PSync/partial-producer.cpp | 7 ++++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index 32240f5..c2e4e66 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -194,11 +194,12 @@ FullProducer::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& i State state; for (const auto& hash : positive) { - ndn::Name prefix = m_hash2name[hash]; - prefix = prefix.getPrefix(prefix.size() - 1); + ndn::Name name = m_hash2name[hash]; + ndn::Name prefix = name.getPrefix(-1); + // Don't sync up sequence number zero if (m_prefixes[prefix] != 0 && !isFutureHash(prefix.toUri(), negative)) { - state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); + state.addContent(name); } } @@ -310,11 +311,11 @@ FullProducer::satisfyPendingInterests() State state; for (const auto& hash : positive) { - ndn::Name prefix = m_hash2name[hash]; - prefix = prefix.getPrefix(prefix.size() - 1); + ndn::Name name = m_hash2name[hash]; + ndn::Name prefix = name.getPrefix(-1); if (m_prefixes[prefix] != 0) { - state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); + state.addContent(name); } } diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index 8e2f5db..2614de3 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -174,11 +174,12 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in NDN_LOG_TRACE("Size of positive set " << positive.size()); NDN_LOG_TRACE("Size of negative set " << negative.size()); for (const auto& hash : positive) { - ndn::Name prefix = m_hash2name[hash]; - prefix = prefix.getPrefix(prefix.size() - 1); + ndn::Name name = m_hash2name[hash]; + ndn::Name prefix = name.getPrefix(-1); + if (bf.contains(prefix.toUri())) { // generate data - state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); + state.addContent(name); NDN_LOG_DEBUG("Content: " << prefix << " " << std::to_string(m_prefixes[prefix])); } } From d56f9315a3ec98deb169f67906d9eb0ec3ee4cfb Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 12:32:20 -0800 Subject: [PATCH 04/19] small readability improvements Change-Id: Ie9b9470cf3608ad1b207ed438cb20d78f2b3863f --- PSync/consumer.cpp | 4 ++-- PSync/full-producer.cpp | 4 ++-- PSync/partial-producer.cpp | 17 +++++++++-------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/PSync/consumer.cpp b/PSync/consumer.cpp index 24483a7..94e3653 100644 --- a/PSync/consumer.cpp +++ b/PSync/consumer.cpp @@ -134,7 +134,7 @@ Consumer::onHelloData(const ndn::ConstBufferPtr& bufferPtr) for (const auto& content : state.getContent()) { const ndn::Name& prefix = content.getPrefix(-1); - uint64_t seq = content.get(content.size()-1).toNumber(); + uint64_t seq = content.get(-1).toNumber(); // If consumer is subscribed then prefix must already be present in // m_prefixes (see addSubscription). So [] operator is safe to use. if (isSubscribed(prefix) && seq > m_prefixes[prefix]) { @@ -225,7 +225,7 @@ Consumer::onSyncData(const ndn::ConstBufferPtr& bufferPtr) for (const auto& content : state.getContent()) { NDN_LOG_DEBUG(content); const ndn::Name& prefix = content.getPrefix(-1); - uint64_t seq = content.get(content.size() - 1).toNumber(); + uint64_t seq = content.get(-1).toNumber(); if (m_prefixes.find(prefix) == m_prefixes.end() || seq > m_prefixes[prefix]) { // If this is just the next seq number then we had already informed the consumer about // the previous sequence number and hence seq low and seq high should be equal to current seq diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index c2e4e66..01c5064 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -148,7 +148,7 @@ FullProducer::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& i return; } - ndn::name::Component ibltName = interestName.get(interestName.size()-1); + ndn::name::Component ibltName = interestName.get(-1); NDN_LOG_DEBUG("Full Sync Interest Received, nonce: " << interest.getNonce() << ", hash: " << std::hash{}(interestName)); @@ -264,7 +264,7 @@ FullProducer::onSyncData(const ndn::Interest& interest, const ndn::ConstBufferPt for (const auto& content : state.getContent()) { const ndn::Name& prefix = content.getPrefix(-1); - uint64_t seq = content.get(content.size() - 1).toNumber(); + uint64_t seq = content.get(-1).toNumber(); if (m_prefixes.find(prefix) == m_prefixes.end() || m_prefixes[prefix] < seq) { updates.push_back(MissingDataInfo{prefix, m_prefixes[prefix] + 1, seq}); diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index 2614de3..06eef7c 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -70,14 +70,15 @@ PartialProducer::publishName(const ndn::Name& prefix, ndn::optional se void PartialProducer::onHelloInterest(const ndn::Name& prefix, const ndn::Interest& interest) { - if (m_segmentPublisher.replyFromStore(interest.getName())) { + ndn::Name interestName = interest.getName(); + if (m_segmentPublisher.replyFromStore(interestName)) { return; } // Last component or fourth last component (in case of interest with version and segment) // needs to be hello - if (interest.getName().get(interest.getName().size()-1).toUri() != "hello" && - interest.getName().get(interest.getName().size()-4).toUri() != "hello") { + if (interestName.get(-1).toUri() != "hello" && + interestName.get(-4).toUri() != "hello") { return; } @@ -93,7 +94,7 @@ PartialProducer::onHelloInterest(const ndn::Name& prefix, const ndn::Interest& i ndn::Name helloDataName = prefix; m_iblt.appendToName(helloDataName); - m_segmentPublisher.publish(interest.getName(), helloDataName, + m_segmentPublisher.publish(interestName, helloDataName, state.wireEncode(), m_helloReplyFreshness); } @@ -126,11 +127,11 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in unsigned int projectedCount; double falsePositiveProb; try { - projectedCount = interestName.get(interestName.size()-4).toNumber(); - falsePositiveProb = interestName.get(interestName.size()-3).toNumber()/1000.; - bfName = interestName.get(interestName.size()-2); + projectedCount = interestName.get(-4).toNumber(); + falsePositiveProb = interestName.get(-3).toNumber()/1000.; + bfName = interestName.get(-2); - ibltName = interestName.get(interestName.size()-1); + ibltName = interestName.get(-1); } catch (const std::exception& e) { NDN_LOG_ERROR("Cannot extract bloom filter and IBF from sync interest: " << e.what()); From 5a54a5ca808f05b30fd459442f086631829774ca Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 12:46:43 -0800 Subject: [PATCH 05/19] ProducerBase: remove unused m_userPrefix refs: #4844 Change-Id: I4c74c97ef7d14ea67c4430563340aafd43e00c3c --- PSync/producer-base.cpp | 1 - PSync/producer-base.hpp | 1 - 2 files changed, 2 deletions(-) diff --git a/PSync/producer-base.cpp b/PSync/producer-base.cpp index 312113a..ad1754f 100644 --- a/PSync/producer-base.cpp +++ b/PSync/producer-base.cpp @@ -39,7 +39,6 @@ ProducerBase::ProducerBase(size_t expectedNumEntries, , m_expectedNumEntries(expectedNumEntries) , m_threshold(expectedNumEntries/2) , m_syncPrefix(syncPrefix) - , m_userPrefix(userPrefix) , m_syncReplyFreshness(syncReplyFreshness) , m_helloReplyFreshness(helloReplyFreshness) , m_rng(ndn::random::getRandomNumberEngine()) diff --git a/PSync/producer-base.hpp b/PSync/producer-base.hpp index 8f4c529..c8ba922 100644 --- a/PSync/producer-base.hpp +++ b/PSync/producer-base.hpp @@ -161,7 +161,6 @@ class ProducerBase std::map m_hash2name; ndn::Name m_syncPrefix; - ndn::Name m_userPrefix; ndn::time::milliseconds m_syncReplyFreshness; ndn::time::milliseconds m_helloReplyFreshness; From 6864873e230f5f8dbefb6e23c90ec8ac14856d4d Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 13:07:55 -0800 Subject: [PATCH 06/19] ProducerBase: add and use removeFromIBF Change-Id: I72aaf3119dc697509388c12224ad4d7902a71c2e --- PSync/producer-base.cpp | 30 ++++++++++++++---------------- PSync/producer-base.hpp | 3 +++ 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/PSync/producer-base.cpp b/PSync/producer-base.cpp index ad1754f..f7d4360 100644 --- a/PSync/producer-base.cpp +++ b/PSync/producer-base.cpp @@ -66,14 +66,7 @@ ProducerBase::removeUserNode(const ndn::Name& prefix) uint64_t seqNo = it->second; m_prefixes.erase(it); - ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seqNo); - auto hashIt = m_name2hash.find(prefixWithSeq); - if (hashIt != m_name2hash.end()) { - uint32_t hash = hashIt->second; - m_name2hash.erase(hashIt); - m_hash2name.erase(hash); - m_iblt.erase(hash); - } + removeFromIBF(ndn::Name(prefix).appendNumber(seqNo)); } } @@ -100,14 +93,7 @@ ProducerBase::updateSeqNo(const ndn::Name& prefix, uint64_t seq) // Delete the last sequence prefix from the iblt // Because we don't insert zeroth prefix in IBF so no need to delete that if (oldSeq != 0) { - ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(oldSeq); - auto hashIt = m_name2hash.find(prefixWithSeq); - if (hashIt != m_name2hash.end()) { - uint32_t hash = hashIt->second; - m_name2hash.erase(hashIt); - m_hash2name.erase(hash); - m_iblt.erase(hash); - } + removeFromIBF(ndn::Name(prefix).appendNumber(oldSeq)); } // Insert the new seq no @@ -125,6 +111,18 @@ ProducerBase::insertToIBF(const ndn::Name& prefix) m_iblt.insert(newHash); } +void +ProducerBase::removeFromIBF(const ndn::Name& prefix) +{ + auto hashIt = m_name2hash.find(prefix); + if (hashIt != m_name2hash.end()) { + uint32_t hash = hashIt->second; + m_name2hash.erase(hashIt); + m_hash2name.erase(hash); + m_iblt.erase(hash); + } +} + void ProducerBase::onRegisterFailed(const ndn::Name& prefix, const std::string& msg) const { diff --git a/PSync/producer-base.hpp b/PSync/producer-base.hpp index c8ba922..94bea6f 100644 --- a/PSync/producer-base.hpp +++ b/PSync/producer-base.hpp @@ -115,6 +115,9 @@ class ProducerBase void insertToIBF(const ndn::Name& prefix); + void + removeFromIBF(const ndn::Name& prefix); + PSYNC_PUBLIC_WITH_TESTS_ELSE_PROTECTED: /** * @brief Update m_prefixes and IBF with the given prefix and seq From 692af7381c82bd9b01bea0919ba05388988cd4ab Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Sat, 9 Mar 2019 14:02:29 -0800 Subject: [PATCH 07/19] Added UserPrefixes. --- PSync/detail/user-prefixes.cpp | 46 ++++++++++++++++ PSync/detail/user-prefixes.hpp | 98 ++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 PSync/detail/user-prefixes.cpp create mode 100644 PSync/detail/user-prefixes.hpp diff --git a/PSync/detail/user-prefixes.cpp b/PSync/detail/user-prefixes.cpp new file mode 100644 index 0000000..8d7744f --- /dev/null +++ b/PSync/detail/user-prefixes.cpp @@ -0,0 +1,46 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#include "PSync/detail/user-prefixes.hpp" + +namespace psync { + +bool +UserPrefixes::addUserNode(const ndn::Name& prefix) +{ + if (!isUserNode(prefix)) { + m_prefixes[prefix] = 0; + return true; + } + else { + return false; + } +} + +void +UserPrefixes::removeUserNode(const ndn::Name& prefix) +{ + auto it = m_prefixes.find(prefix); + if (it != m_prefixes.end()) { + uint64_t seqNo = it->second; + m_prefixes.erase(it); + } +} + +} // namespace psync diff --git a/PSync/detail/user-prefixes.hpp b/PSync/detail/user-prefixes.hpp new file mode 100644 index 0000000..af3a2ff --- /dev/null +++ b/PSync/detail/user-prefixes.hpp @@ -0,0 +1,98 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#ifndef PSYNC_USER_PREFIXES_HPP +#define PSYNC_USER_PREFIXES_HPP + +#include +#include + +namespace psync { + +/** + * @brief UserPrefixes holds the m_prefixes map from prefix to sequence number, + * used by PartialProducer and FullProducer. + * + * Contains code common to both + */ +class UserPrefixes +{ +public: + /** + * @brief Check if the prefix is in m_prefixes. + * + * @param prefix The prefix to check. + * @return True if the prefix is in m_prefixes. + */ + bool + isUserNode(const ndn::Name& prefix) const + { + return m_prefixes.find(prefix) != m_prefixes.end(); + } + + /** + * @brief Returns the current sequence number of the given prefix + * + * @param prefix prefix to get the sequence number of + */ + ndn::optional + getSeqNo(const ndn::Name& prefix) const + { + auto it = m_prefixes.find(prefix); + if (it == m_prefixes.end()) { + return ndn::nullopt; + } + return it->second; + } + + /** + * @brief Adds a user node for synchronization + * + * Initializes m_prefixes[prefix] to zero + * Does not add zero-th sequence number to IBF + * because if a large number of user nodes are added + * then decoding of the difference between own IBF and + * other IBF will not be possible + * + * @param prefix the user node to be added + * @return True if the prefix was added, false if the prefix was already in + * m_prefixes. + */ + bool + addUserNode(const ndn::Name& prefix); + + /** + * @brief Remove the user node from synchronization. If the prefix is not in + * m_prefixes, then do nothing. + * + * The caller should first check isUserNode(prefix) and erase the prefix from + * the IBF and other maps if needed. + * + * @param prefix the user node to be removed + */ + void + removeUserNode(const ndn::Name& prefix); + + // prefix and sequence number + std::map m_prefixes; +}; + +} // namespace psync + +#endif // PSYNC_USER_PREFIXES_HPP From ad00e3f0056f7d601c5816083985472ecc5bd654 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Sat, 9 Mar 2019 14:30:17 -0800 Subject: [PATCH 08/19] UserPrefixes: Added updateSeqNo. --- PSync/detail/user-prefixes.cpp | 29 +++++++++++++++++++++++++++++ PSync/detail/user-prefixes.hpp | 27 +++++++++++++++++++++------ 2 files changed, 50 insertions(+), 6 deletions(-) diff --git a/PSync/detail/user-prefixes.cpp b/PSync/detail/user-prefixes.cpp index 8d7744f..e510253 100644 --- a/PSync/detail/user-prefixes.cpp +++ b/PSync/detail/user-prefixes.cpp @@ -17,10 +17,13 @@ * PSync, e.g., in COPYING.md file. If not, see . **/ +#include #include "PSync/detail/user-prefixes.hpp" namespace psync { +NDN_LOG_INIT(psync.UserPrefixes); + bool UserPrefixes::addUserNode(const ndn::Name& prefix) { @@ -43,4 +46,30 @@ UserPrefixes::removeUserNode(const ndn::Name& prefix) } } +bool +UserPrefixes::updateSeqNo + (const ndn::Name& prefix, uint64_t seqNo, uint64_t& oldSeqNo) +{ + oldSeqNo = 0; + NDN_LOG_DEBUG("UpdateSeq: " << prefix << " " << seqNo); + + auto it = m_prefixes.find(prefix); + if (it != m_prefixes.end()) { + oldSeqNo = it->second; + } + else { + NDN_LOG_WARN("Prefix not found in m_prefixes"); + return false; + } + + if (oldSeqNo >= seqNo) { + NDN_LOG_WARN("Update has lower/equal seq no for prefix, doing nothing!"); + return false; + } + + // Insert the new sequence number + it->second = seqNo; + return true; +} + } // namespace psync diff --git a/PSync/detail/user-prefixes.hpp b/PSync/detail/user-prefixes.hpp index af3a2ff..23e13ad 100644 --- a/PSync/detail/user-prefixes.hpp +++ b/PSync/detail/user-prefixes.hpp @@ -65,13 +65,9 @@ class UserPrefixes * @brief Adds a user node for synchronization * * Initializes m_prefixes[prefix] to zero - * Does not add zero-th sequence number to IBF - * because if a large number of user nodes are added - * then decoding of the difference between own IBF and - * other IBF will not be possible * * @param prefix the user node to be added - * @return True if the prefix was added, false if the prefix was already in + * @return true if the prefix was added, false if the prefix was already in * m_prefixes. */ bool @@ -82,13 +78,32 @@ class UserPrefixes * m_prefixes, then do nothing. * * The caller should first check isUserNode(prefix) and erase the prefix from - * the IBF and other maps if needed. + * the IBLT and other maps if needed. * * @param prefix the user node to be removed */ void removeUserNode(const ndn::Name& prefix); + /** + * @brief Update m_prefixes with the given prefix and sequence number. This + * does not update the IBLT. This logs a message for the update. + * + * Whoever calls this needs to make sure that isUserNode(prefix) is true. + * + * @param prefix the prefix of the update + * @param seqNo the sequence number of the update + * @param oldSeqNo This sets oldSeqNo to the old sequence number for the + * prefix. If this method returns true and oldSeqNo is not zero, the caller + * can remove the old prefix from the IBLT. + * @return True if the sequence number was updated, false if the prefix was + * not in m_prefixes, or if the seqNo is less than or equal to the old + * sequence number. If this returns false, the caller should not update the + * IBLT. + */ + bool + updateSeqNo(const ndn::Name& prefix, uint64_t seqNo, uint64_t& oldSeqNo); + // prefix and sequence number std::map m_prefixes; }; From d91524ebe5bb282dbd3439bd7087a403d2c54c21 Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 17:17:50 -0800 Subject: [PATCH 09/19] putting everything together Change-Id: If9d06c73f9d6e01f140295fc482a82af1e416c08 --- PSync/full-producer-arbitrary.cpp | 349 ++++++++++++++++++++++++++++++ PSync/full-producer-arbitrary.hpp | 216 ++++++++++++++++++ PSync/full-producer.cpp | 334 +++++----------------------- PSync/full-producer.hpp | 152 +++++-------- PSync/partial-producer.cpp | 36 ++- PSync/partial-producer.hpp | 55 +++++ PSync/producer-base.cpp | 74 +------ PSync/producer-base.hpp | 62 +----- tests/test-full-producer.cpp | 3 +- tests/test-full-sync.cpp | 10 +- tests/test-partial-sync.cpp | 12 +- tests/test-producer-base.cpp | 8 +- 12 files changed, 777 insertions(+), 534 deletions(-) create mode 100644 PSync/full-producer-arbitrary.cpp create mode 100644 PSync/full-producer-arbitrary.hpp diff --git a/PSync/full-producer-arbitrary.cpp b/PSync/full-producer-arbitrary.cpp new file mode 100644 index 0000000..4d66ab7 --- /dev/null +++ b/PSync/full-producer-arbitrary.cpp @@ -0,0 +1,349 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2014-2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#include "PSync/full-producer-arbitrary.hpp" + +#include +#include +#include + +#include +#include +#include + +namespace psync { + +NDN_LOG_INIT(psync.FullProducerArbitrary); + +FullProducerArbitrary::FullProducerArbitrary(const size_t expectedNumEntries, + ndn::Face& face, + const ndn::Name& syncPrefix, + const ndn::Name& userPrefix, + const ArbitraryUpdateCallback& onArbitraryUpdateCallback, + ndn::time::milliseconds syncInterestLifetime, + ndn::time::milliseconds syncReplyFreshness, + const ShouldAddToSyncDataCallback& onShouldAddToSyncDataCallback, + const SyncDataCallback onSyncDataCallBack) + : ProducerBase(expectedNumEntries, syncPrefix, syncReplyFreshness) + , m_face(face) + , m_scheduler(m_face.getIoService()) + , m_segmentPublisher(m_face, m_keyChain) + , m_syncInterestLifetime(syncInterestLifetime) + , m_onArbitraryUpdateCallback(onArbitraryUpdateCallback) + , m_onShouldAddToSyncDataCallback(onShouldAddToSyncDataCallback) + , m_onSyncDataCallback(onSyncDataCallBack) +{ + int jitter = m_syncInterestLifetime.count() * .20; + m_jitter = std::uniform_int_distribution<>(-jitter, jitter); + + m_registeredPrefix = m_face.setInterestFilter( + ndn::InterestFilter(m_syncPrefix).allowLoopback(false), + std::bind(&FullProducerArbitrary::onSyncInterest, this, _1, _2), + std::bind(&FullProducerArbitrary::onRegisterFailed, this, _1, _2)); + + // Should we do this after setInterestFilter success call back + // (Currently following ChronoSync's way) + sendSyncInterest(); +} + +FullProducerArbitrary::~FullProducerArbitrary() +{ + if (m_fetcher) { + m_fetcher->stop(); + } +} + +void +FullProducerArbitrary::publishName(const ndn::Name& name) +{ + NDN_LOG_INFO("Publish: " << name); + + insertToIBF(name); + + satisfyPendingInterests(); +} + +void +FullProducerArbitrary::sendSyncInterest() +{ + // If we send two sync interest one after the other + // since there is no new data in the network yet, + // when data is available it may satisfy both of them + if (m_fetcher) { + m_fetcher->stop(); + } + + // Sync Interest format for full sync: // + ndn::Name syncInterestName = m_syncPrefix; + + // Append our latest IBF + m_iblt.appendToName(syncInterestName); + + m_outstandingInterestName = syncInterestName; + + m_scheduledSyncInterestId = + m_scheduler.scheduleEvent(m_syncInterestLifetime / 2 + ndn::time::milliseconds(m_jitter(m_rng)), + [this] { sendSyncInterest(); }); + + ndn::Interest syncInterest(syncInterestName); + + ndn::util::SegmentFetcher::Options options; + options.interestLifetime = m_syncInterestLifetime; + options.maxTimeout = m_syncInterestLifetime; + + m_fetcher = ndn::util::SegmentFetcher::start(m_face, + syncInterest, + ndn::security::v2::getAcceptAllValidator(), + options); + + m_fetcher->onComplete.connect([this, syncInterest] (ndn::ConstBufferPtr bufferPtr) { + onSyncData(syncInterest, bufferPtr); + }); + + m_fetcher->onError.connect([] (uint32_t errorCode, const std::string& msg) { + NDN_LOG_ERROR("Cannot fetch sync data, error: " << + errorCode << " message: " << msg); + }); + + NDN_LOG_DEBUG("sendFullSyncInterest, nonce: " << syncInterest.getNonce() << + ", hash: " << std::hash{}(syncInterestName)); +} + +void +FullProducerArbitrary::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& interest) +{ + if (m_segmentPublisher.replyFromStore(interest.getName())) { + return; + } + + ndn::Name nameWithoutSyncPrefix = interest.getName().getSubName(prefixName.size()); + ndn::Name interestName; + + if (nameWithoutSyncPrefix.size() == 1) { + // Get //IBF from //IBF + interestName = interest.getName(); + } + else if (nameWithoutSyncPrefix.size() == 3) { + // Get //IBF from //IBF// + interestName = interest.getName().getPrefix(-2); + } + else { + return; + } + + ndn::name::Component ibltName = interestName.get(-1); + + NDN_LOG_DEBUG("Full Sync Interest Received, nonce: " << interest.getNonce() << + ", hash: " << std::hash{}(interestName)); + + IBLT iblt(m_expectedNumEntries); + try { + iblt.initialize(ibltName); + } + catch (const std::exception& e) { + NDN_LOG_WARN(e.what()); + return; + } + + IBLT diff = m_iblt - iblt; + + std::set positive; + std::set negative; + + if (!diff.listEntries(positive, negative)) { + NDN_LOG_TRACE("Cannot decode differences, positive: " << positive.size() + << " negative: " << negative.size() << " m_threshold: " + << m_threshold); + + // Send all data if greater then threshold, else send positive below as usual + // Or send if we can't get neither positive nor negative differences + if (positive.size() + negative.size() >= m_threshold || + (positive.size() == 0 && negative.size() == 0)) { + State state; + for (const auto& it : m_name2hash) { + state.addContent(it.first); + } + + if (!state.getContent().empty()) { + m_segmentPublisher.publish(interest.getName(), interest.getName(), + state.wireEncode(), m_syncReplyFreshness); + } + + return; + } + } + + State state; + for (const auto& hash : positive) { + ndn::Name name = m_hash2name[hash]; + ndn::Name prefix = name.getPrefix(-1); + + // Don't sync up sequence number zero + if (m_name2hash.find(name) != m_name2hash.end() && + m_onShouldAddToSyncDataCallback(prefix.toUri(), negative)) { + state.addContent(name); + } + } + + if (!state.getContent().empty()) { + NDN_LOG_DEBUG("Sending sync content: " << state); + sendSyncData(interestName, state.wireEncode()); + return; + } + else { + NDN_LOG_WARN("State is empty"); + } + + auto& entry = m_pendingEntries.emplace(interestName, PendingEntryInfoFull{iblt, {}}).first->second; + entry.expirationEvent = m_scheduler.scheduleEvent(interest.getInterestLifetime(), + [this, interest] { + NDN_LOG_TRACE("Erase Pending Interest " << interest.getNonce()); + m_pendingEntries.erase(interest.getName()); + }); +} + +void +FullProducerArbitrary::sendSyncData(const ndn::Name& name, const ndn::Block& block) +{ + NDN_LOG_DEBUG("Checking if data will satisfy our own pending interest"); + + ndn::Name nameWithIblt; + m_iblt.appendToName(nameWithIblt); + + // Append hash of our IBF so that data name maybe different for each node answering + ndn::Name dataName(ndn::Name(name).appendNumber(std::hash{}(nameWithIblt))); + + // checking if our own interest got satisfied + if (m_outstandingInterestName == name) { + NDN_LOG_DEBUG("Satisfied our own pending interest"); + // remove outstanding interest + if (m_fetcher) { + NDN_LOG_DEBUG("Removing our pending interest from face (stop fetcher)"); + m_fetcher->stop(); + m_outstandingInterestName = ndn::Name(""); + } + + NDN_LOG_DEBUG("Sending Sync Data"); + + // Send data after removing pending sync interest on face + m_segmentPublisher.publish(name, dataName, block, m_syncReplyFreshness); + + NDN_LOG_TRACE("Renewing sync interest"); + sendSyncInterest(); + } + else { + NDN_LOG_DEBUG("Sending Sync Data"); + m_segmentPublisher.publish(name, dataName, block, m_syncReplyFreshness); + } +} + +void +FullProducerArbitrary::onSyncData(const ndn::Interest& interest, const ndn::ConstBufferPtr& bufferPtr) +{ + deletePendingInterests(interest.getName()); + + State state(ndn::Block(std::move(bufferPtr))); + std::vector updates; + + NDN_LOG_DEBUG("Sync Data Received: " << state); + + for (const auto& name : state.getContent()) { + if (m_name2hash.find(name) == m_name2hash.end()) { + updates.push_back(name); + if (m_onSyncDataCallback) { + m_onSyncDataCallback(name); + } + else { + insertToIBF(name); + } + // We should not call satisfyPendingSyncInterests here because we just + // got data and deleted pending interest by calling deletePendingFullSyncInterests + // But we might have interests not matching to this interest that might not have deleted + // from pending sync interest + } + } + + // We just got the data, so send a new sync interest + if (!updates.empty()) { + m_onArbitraryUpdateCallback(updates); + NDN_LOG_TRACE("Renewing sync interest"); + sendSyncInterest(); + } + else { + NDN_LOG_TRACE("No new update, interest nonce: " << interest.getNonce() << + " , hash: " << std::hash{}(interest.getName())); + } +} + +void +FullProducerArbitrary::satisfyPendingInterests() +{ + NDN_LOG_DEBUG("Satisfying full sync interest: " << m_pendingEntries.size()); + + for (auto it = m_pendingEntries.begin(); it != m_pendingEntries.end();) { + const PendingEntryInfoFull& entry = it->second; + IBLT diff = m_iblt - entry.iblt; + std::set positive; + std::set negative; + + if (!diff.listEntries(positive, negative)) { + NDN_LOG_TRACE("Decode failed for pending interest"); + if (positive.size() + negative.size() >= m_threshold || + (positive.size() == 0 && negative.size() == 0)) { + NDN_LOG_TRACE("pos + neg > threshold or no diff can be found, erase pending interest"); + m_pendingEntries.erase(it++); + continue; + } + } + + State state; + for (const auto& hash : positive) { + ndn::Name name = m_hash2name[hash]; + ndn::Name prefix = name.getPrefix(-1); + + if (m_name2hash.find(name) != m_name2hash.end()) { + state.addContent(name); + } + } + + if (!state.getContent().empty()) { + NDN_LOG_DEBUG("Satisfying sync content: " << state); + sendSyncData(it->first, state.wireEncode()); + m_pendingEntries.erase(it++); + } + else { + ++it; + } + } +} + +void +FullProducerArbitrary::deletePendingInterests(const ndn::Name& interestName) { + for (auto it = m_pendingEntries.begin(); it != m_pendingEntries.end();) { + if (it->first == interestName) { + NDN_LOG_TRACE("Delete pending interest: " << interestName); + m_pendingEntries.erase(it++); + } + else { + ++it; + } + } +} + +} // namespace psync diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp new file mode 100644 index 0000000..80ba1ac --- /dev/null +++ b/PSync/full-producer-arbitrary.hpp @@ -0,0 +1,216 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2014-2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#ifndef PSYNC_FULL_PRODUCER_ARBITRARY_HPP +#define PSYNC_FULL_PRODUCER_ARBITRARY_HPP + +#include "PSync/producer-base.hpp" +#include "PSync/detail/state.hpp" + +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace psync { + +// Name has to be different than PendingEntryInfo +// used in partial-producer otherwise get strange segmentation-faults +// when partial producer is destructed +struct PendingEntryInfoFull +{ + IBLT iblt; + ndn::util::scheduler::ScopedEventId expirationEvent; +}; + +typedef std::function&)> ArbitraryUpdateCallback; +typedef std::function SyncDataCallback; +typedef std::function&)> ShouldAddToSyncDataCallback; + +const ndn::time::milliseconds SYNC_INTEREST_LIFTIME = 1_s; + +/** + * @brief Full sync logic to synchronize with other nodes + * where all nodes wants to get all names prefixes synced. + * + * Application should call publishName whenever it wants to + * let consumers know that new data is available for the userPrefix. + * Multiple userPrefixes can be added by using addUserNode. + * Currently, fetching and publishing of data needs to be handled by the application. + */ +class FullProducerArbitrary : public ProducerBase +{ +public: + /** + * @brief constructor + * + * Registers syncPrefix in NFD and sends a sync interest + * + * @param expectedNumEntries expected entries in IBF + * @param face application's face + * @param syncPrefix The prefix of the sync group + * @param userPrefix The prefix of the first user in the group + * @param onUpdateCallBack The call back to be called when there is new data + * @param syncInterestLifetime lifetime of the sync interest + * @param syncReplyFreshness freshness of sync data + */ + FullProducerArbitrary(size_t expectedNumEntries, + ndn::Face& face, + const ndn::Name& syncPrefix, + const ndn::Name& userPrefix, + const ArbitraryUpdateCallback& onArbitraryUpdateCallback, + ndn::time::milliseconds syncInterestLifetime = SYNC_INTEREST_LIFTIME, + ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS, + const ShouldAddToSyncDataCallback& onShouldAddToSyncDataCallback = ShouldAddToSyncDataCallback(), + const SyncDataCallback onSyncDataCallBack = SyncDataCallback()); + + ~FullProducerArbitrary(); + + void + removeName(const ndn::Name& name) { + removeFromIBF(name); + } + + void + addName(const ndn::Name& name) { + insertToIBF(name); + } + + /** + * @brief Publish name to let others know + * + * addUserNode needs to be called before this to add the prefix + * if not already added via the constructor. + * If seq is null then the seq of prefix is incremented by 1 else + * the supplied sequence is set in the IBF. + * + * @param prefix the prefix to be updated + * @param seq the sequence number of the prefix + */ + void + publishName(const ndn::Name& name); + +private: + /** + * @brief Send sync interest for full synchronization + * + * Forms the interest name: // + * Cancels any pending sync interest we sent earlier on the face + * Sends the sync interest + */ + void + sendSyncInterest(); + +PUBLIC_WITH_TESTS_ELSE_PRIVATE: + /** + * @brief Process sync interest from other parties + * + * Get differences b/w our IBF and IBF in the sync interest. + * If we cannot get the differences successfully then send an application nack. + * + * If we have some things in our IBF that the other side does not have, reply with the content or + * If no. of different items is greater than threshold or equals zero then send a nack. + * Otherwise add the sync interest into a map with interest name as key and PendingEntryInfoFull + * as value. + * + * @param prefixName prefix for sync group which we registered + * @param interest the interest we got + */ + void + onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& interest); + +private: + /** + * @brief Send sync data + * + * Check if the data will satisfy our own pending interest, + * remove it first if it does, and then renew the sync interest + * Otherwise just send the data + * + * @param name name to be set as data name + * @param block the content of the data + */ + void + sendSyncData(const ndn::Name& name, const ndn::Block& block); + + /** + * @brief Process sync data + * + * Call deletePendingInterests to delete any pending sync interest with + * interest name would have been satisfied once NFD got the data. + * + * For each prefix/seq in data content check that we don't already have the + * prefix/seq and updateSeq(prefix, seq) + * + * Notify the application about the updates + * sendSyncInterest because the last one was satisfied by the incoming data + * + * @param interest interest for which we got the data + * @param bufferPtr sync data content + */ + void + onSyncData(const ndn::Interest& interest, const ndn::ConstBufferPtr& bufferPtr); + +public: + /** + * @brief Satisfy pending sync interests + * + * For pending sync interests SI, if IBF of SI has any difference from our own IBF: + * send data back. + * If we can't decode differences from the stored IBF, then delete it. + */ + void + satisfyPendingInterests(); + +PUBLIC_WITH_TESTS_ELSE_PRIVATE: + /** + * @brief Delete pending sync interests that match given name + */ + void + deletePendingInterests(const ndn::Name& interestName); + +private: + ndn::Face& m_face; + ndn::KeyChain m_keyChain; + ndn::Scheduler m_scheduler; + +PUBLIC_WITH_TESTS_ELSE_PROTECTED: + SegmentPublisher m_segmentPublisher; + + std::map m_pendingEntries; + ndn::time::milliseconds m_syncInterestLifetime; + ArbitraryUpdateCallback m_onArbitraryUpdateCallback; + ShouldAddToSyncDataCallback m_onShouldAddToSyncDataCallback; + SyncDataCallback m_onSyncDataCallback; + ndn::util::scheduler::ScopedEventId m_scheduledSyncInterestId; + std::uniform_int_distribution<> m_jitter; + ndn::Name m_outstandingInterestName; + ndn::ScopedRegisteredPrefixHandle m_registeredPrefix; + std::shared_ptr m_fetcher; +}; + +} // namespace psync + +#endif // PSYNC_FULL_PRODUCER_ARBITRARY_HPP diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index 01c5064..0a59dfc 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -38,324 +38,88 @@ FullProducer::FullProducer(const size_t expectedNumEntries, const UpdateCallback& onUpdateCallBack, ndn::time::milliseconds syncInterestLifetime, ndn::time::milliseconds syncReplyFreshness) - : ProducerBase(expectedNumEntries, syncPrefix, userPrefix, syncReplyFreshness) - , m_face(face) - , m_scheduler(m_face.getIoService()) - , m_segmentPublisher(m_face, m_keyChain) - , m_syncInterestLifetime(syncInterestLifetime) - , m_onUpdate(onUpdateCallBack) + : m_producerArbitrary(expectedNumEntries, face, syncPrefix, userPrefix, + [this] (const std::vector& names) { + arbitraryUpdateCallBack(names); + }, + syncInterestLifetime, syncReplyFreshness, + [this] (const ndn::Name& prefix, const std::set& negative) { + return isNotFutureHash(prefix, negative); + }, + [this] (const ndn::Name& name) { + ndn::Name prefix = name.getPrefix(-1); + uint64_t seq = name.get(-1).toNumber(); + + if (m_prefixes.m_prefixes.find(prefix) == m_prefixes.m_prefixes.end() || + m_prefixes.m_prefixes[prefix] < seq) { + updateSeqNo(prefix, seq); + } + }) + , m_onUpdateCallback(onUpdateCallBack) { - int jitter = m_syncInterestLifetime.count() * .20; - m_jitter = std::uniform_int_distribution<>(-jitter, jitter); - - m_registeredPrefix = m_face.setInterestFilter( - ndn::InterestFilter(m_syncPrefix).allowLoopback(false), - std::bind(&FullProducer::onSyncInterest, this, _1, _2), - std::bind(&FullProducer::onRegisterFailed, this, _1, _2)); - - // Should we do this after setInterestFilter success call back - // (Currently following ChronoSync's way) - sendSyncInterest(); -} - -FullProducer::~FullProducer() -{ - if (m_fetcher) { - m_fetcher->stop(); - } + addUserNode(userPrefix); } void FullProducer::publishName(const ndn::Name& prefix, ndn::optional seq) { - if (m_prefixes.find(prefix) == m_prefixes.end()) { - NDN_LOG_WARN("Prefix not added: " << prefix); + if (!m_prefixes.isUserNode(prefix)) { return; } - uint64_t newSeq = seq.value_or(m_prefixes[prefix] + 1); + uint64_t newSeq = seq.value_or(m_prefixes.m_prefixes[prefix] + 1); - NDN_LOG_INFO("Publish: "<< prefix << "/" << newSeq); + NDN_LOG_INFO("Publish: " << prefix << "/" << newSeq); updateSeqNo(prefix, newSeq); - satisfyPendingInterests(); + m_producerArbitrary.satisfyPendingInterests(); } void -FullProducer::sendSyncInterest() +FullProducer::updateSeqNo(const ndn::Name& prefix, uint64_t seq) { - // If we send two sync interest one after the other - // since there is no new data in the network yet, - // when data is available it may satisfy both of them - if (m_fetcher) { - m_fetcher->stop(); - } - - // Sync Interest format for full sync: // - ndn::Name syncInterestName = m_syncPrefix; - - // Append our latest IBF - m_iblt.appendToName(syncInterestName); - - m_outstandingInterestName = syncInterestName; - - m_scheduledSyncInterestId = - m_scheduler.schedule(m_syncInterestLifetime / 2 + ndn::time::milliseconds(m_jitter(m_rng)), - [this] { sendSyncInterest(); }); - - ndn::Interest syncInterest(syncInterestName); - - using ndn::util::SegmentFetcher; - SegmentFetcher::Options options; - options.interestLifetime = m_syncInterestLifetime; - options.maxTimeout = m_syncInterestLifetime; - - m_fetcher = SegmentFetcher::start(m_face, syncInterest, - ndn::security::v2::getAcceptAllValidator(), options); - - m_fetcher->onComplete.connect([this, syncInterest] (const ndn::ConstBufferPtr& bufferPtr) { - onSyncData(syncInterest, bufferPtr); - }); - - m_fetcher->onError.connect([] (uint32_t errorCode, const std::string& msg) { - NDN_LOG_ERROR("Cannot fetch sync data, error: " << errorCode << " message: " << msg); - }); - - NDN_LOG_DEBUG("sendFullSyncInterest, nonce: " << syncInterest.getNonce() << - ", hash: " << std::hash{}(syncInterestName)); -} - -void -FullProducer::onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& interest) -{ - if (m_segmentPublisher.replyFromStore(interest.getName())) { - return; - } - - ndn::Name nameWithoutSyncPrefix = interest.getName().getSubName(prefixName.size()); - ndn::Name interestName; - - if (nameWithoutSyncPrefix.size() == 1) { - // Get //IBF from //IBF - interestName = interest.getName(); - } - else if (nameWithoutSyncPrefix.size() == 3) { - // Get //IBF from //IBF// - interestName = interest.getName().getPrefix(-2); - } - else { - return; - } - - ndn::name::Component ibltName = interestName.get(-1); - - NDN_LOG_DEBUG("Full Sync Interest Received, nonce: " << interest.getNonce() << - ", hash: " << std::hash{}(interestName)); - - IBLT iblt(m_expectedNumEntries); - try { - iblt.initialize(ibltName); - } - catch (const std::exception& e) { - NDN_LOG_WARN(e.what()); - return; - } - - IBLT diff = m_iblt - iblt; - - std::set positive; - std::set negative; - - if (!diff.listEntries(positive, negative)) { - NDN_LOG_TRACE("Cannot decode differences, positive: " << positive.size() - << " negative: " << negative.size() << " m_threshold: " - << m_threshold); - - // Send all data if greater then threshold, else send positive below as usual - // Or send if we can't get neither positive nor negative differences - if (positive.size() + negative.size() >= m_threshold || - (positive.size() == 0 && negative.size() == 0)) { - State state; - for (const auto& content : m_prefixes) { - if (content.second != 0) { - state.addContent(ndn::Name(content.first).appendNumber(content.second)); - } - } - - if (!state.getContent().empty()) { - m_segmentPublisher.publish(interest.getName(), interest.getName(), - state.wireEncode(), m_syncReplyFreshness); - } - - return; - } - } - - State state; - for (const auto& hash : positive) { - ndn::Name name = m_hash2name[hash]; - ndn::Name prefix = name.getPrefix(-1); - - // Don't sync up sequence number zero - if (m_prefixes[prefix] != 0 && !isFutureHash(prefix.toUri(), negative)) { - state.addContent(name); - } - } - - if (!state.getContent().empty()) { - NDN_LOG_DEBUG("Sending sync content: " << state); - sendSyncData(interestName, state.wireEncode()); - return; - } - - auto& entry = m_pendingEntries.emplace(interestName, PendingEntryInfoFull{iblt, {}}).first->second; - entry.expirationEvent = m_scheduler.schedule(interest.getInterestLifetime(), - [this, interest] { - NDN_LOG_TRACE("Erase Pending Interest " << interest.getNonce()); - m_pendingEntries.erase(interest.getName()); - }); -} - -void -FullProducer::sendSyncData(const ndn::Name& name, const ndn::Block& block) -{ - NDN_LOG_DEBUG("Checking if data will satisfy our own pending interest"); - - ndn::Name nameWithIblt; - m_iblt.appendToName(nameWithIblt); - - // Append hash of our IBF so that data name maybe different for each node answering - ndn::Name dataName(ndn::Name(name).appendNumber(std::hash{}(nameWithIblt))); - - // checking if our own interest got satisfied - if (m_outstandingInterestName == name) { - NDN_LOG_DEBUG("Satisfied our own pending interest"); - // remove outstanding interest - if (m_fetcher) { - NDN_LOG_DEBUG("Removing our pending interest from face (stop fetcher)"); - m_fetcher->stop(); - m_outstandingInterestName = ndn::Name(""); - } - - NDN_LOG_DEBUG("Sending Sync Data"); - - // Send data after removing pending sync interest on face - m_segmentPublisher.publish(name, dataName, block, m_syncReplyFreshness); - - NDN_LOG_TRACE("Renewing sync interest"); - sendSyncInterest(); - } - else { - NDN_LOG_DEBUG("Sending Sync Data"); - m_segmentPublisher.publish(name, dataName, block, m_syncReplyFreshness); - } -} - -void -FullProducer::onSyncData(const ndn::Interest& interest, const ndn::ConstBufferPtr& bufferPtr) -{ - deletePendingInterests(interest.getName()); - - State state{ndn::Block{bufferPtr}}; - std::vector updates; - - NDN_LOG_DEBUG("Sync Data Received: " << state); - - for (const auto& content : state.getContent()) { - const ndn::Name& prefix = content.getPrefix(-1); - uint64_t seq = content.get(-1).toNumber(); - - if (m_prefixes.find(prefix) == m_prefixes.end() || m_prefixes[prefix] < seq) { - updates.push_back(MissingDataInfo{prefix, m_prefixes[prefix] + 1, seq}); - updateSeqNo(prefix, seq); - // We should not call satisfyPendingSyncInterests here because we just - // got data and deleted pending interest by calling deletePendingFullSyncInterests - // But we might have interests not matching to this interest that might not have deleted - // from pending sync interest - } - } - - // We just got the data, so send a new sync interest - if (!updates.empty()) { - m_onUpdate(updates); - NDN_LOG_TRACE("Renewing sync interest"); - sendSyncInterest(); - } - else { - NDN_LOG_TRACE("No new update, interest nonce: " << interest.getNonce() << - " , hash: " << std::hash{}(interest.getName())); - } -} - -void -FullProducer::satisfyPendingInterests() -{ - NDN_LOG_DEBUG("Satisfying full sync interest: " << m_pendingEntries.size()); - - for (auto it = m_pendingEntries.begin(); it != m_pendingEntries.end();) { - const PendingEntryInfoFull& entry = it->second; - IBLT diff = m_iblt - entry.iblt; - std::set positive; - std::set negative; - - if (!diff.listEntries(positive, negative)) { - NDN_LOG_TRACE("Decode failed for pending interest"); - if (positive.size() + negative.size() >= m_threshold || - (positive.size() == 0 && negative.size() == 0)) { - NDN_LOG_TRACE("pos + neg > threshold or no diff can be found, erase pending interest"); - it = m_pendingEntries.erase(it); - continue; - } - } - - State state; - for (const auto& hash : positive) { - ndn::Name name = m_hash2name[hash]; - ndn::Name prefix = name.getPrefix(-1); - - if (m_prefixes[prefix] != 0) { - state.addContent(name); - } - } - - if (!state.getContent().empty()) { - NDN_LOG_DEBUG("Satisfying sync content: " << state); - sendSyncData(it->first, state.wireEncode()); - it = m_pendingEntries.erase(it); - } - else { - ++it; - } - } + uint64_t oldSeq; + if (!m_prefixes.updateSeqNo(prefix, seq, oldSeq)) + return; // Delete the last sequence prefix from the iblt + // Because we don't insert zeroth prefix in IBF so no need to delete that + if (oldSeq != 0) { + m_producerArbitrary.removeName(ndn::Name(prefix).appendNumber(oldSeq)); + } // Insert the new seq no + ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seq); + m_producerArbitrary.addName(prefixWithSeq); } bool -FullProducer::isFutureHash(const ndn::Name& prefix, const std::set& negative) +FullProducer::isNotFutureHash(const ndn::Name& prefix, const std::set& negative) { uint32_t nextHash = murmurHash3(N_HASHCHECK, - ndn::Name(prefix).appendNumber(m_prefixes[prefix] + 1).toUri()); + ndn::Name(prefix).appendNumber(m_prefixes.m_prefixes[prefix] + 1).toUri()); for (const auto& nHash : negative) { if (nHash == nextHash) { - return true; + return false; break; } } - return false; + return true; } void -FullProducer::deletePendingInterests(const ndn::Name& interestName) +FullProducer::arbitraryUpdateCallBack(const std::vector& names) { - for (auto it = m_pendingEntries.begin(); it != m_pendingEntries.end();) { - if (it->first == interestName) { - NDN_LOG_TRACE("Delete pending interest: " << interestName); - it = m_pendingEntries.erase(it); - } - else { - ++it; + std::vector updates; + + for (const auto& name : names) { + ndn::Name prefix = name.getPrefix(-1); + uint64_t seq = name.get(-1).toNumber(); + + if (m_prefixes.m_prefixes.find(prefix) == m_prefixes.m_prefixes.end() || + m_prefixes.m_prefixes[prefix] < seq) { + updates.push_back(MissingDataInfo{prefix, m_prefixes.m_prefixes[prefix] + 1, seq}); } } + + m_onUpdateCallback(updates); } } // namespace psync diff --git a/PSync/full-producer.hpp b/PSync/full-producer.hpp index 2c2015b..7f1a924 100644 --- a/PSync/full-producer.hpp +++ b/PSync/full-producer.hpp @@ -21,7 +21,9 @@ #define PSYNC_FULL_PRODUCER_HPP #include "PSync/producer-base.hpp" +#include "PSync/full-producer-arbitrary.hpp" #include "PSync/detail/state.hpp" +#include "PSync/detail/user-prefixes.hpp" #include #include @@ -35,19 +37,8 @@ namespace psync { -// Name has to be different than PendingEntryInfo -// used in partial-producer otherwise get strange segmentation-faults -// when partial producer is destructed -struct PendingEntryInfoFull -{ - IBLT iblt; - ndn::scheduler::ScopedEventId expirationEvent; -}; - typedef std::function&)> UpdateCallback; -const ndn::time::milliseconds SYNC_INTEREST_LIFTIME = 1_s; - /** * @brief Full sync logic to synchronize with other nodes * where all nodes wants to get all names prefixes synced. @@ -57,7 +48,7 @@ const ndn::time::milliseconds SYNC_INTEREST_LIFTIME = 1_s; * Multiple userPrefixes can be added by using addUserNode. * Currently, fetching and publishing of data needs to be handled by the application. */ -class FullProducer : public ProducerBase +class FullProducer { public: /** @@ -77,103 +68,71 @@ class FullProducer : public ProducerBase ndn::Face& face, const ndn::Name& syncPrefix, const ndn::Name& userPrefix, - const UpdateCallback& onUpdateCallBack, + const UpdateCallback& onUpdateCallback, ndn::time::milliseconds syncInterestLifetime = SYNC_INTEREST_LIFTIME, ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS); - ~FullProducer(); - /** - * @brief Publish name to let others know + * @brief Returns the current sequence number of the given prefix * - * addUserNode needs to be called before this to add the prefix - * if not already added via the constructor. - * If seq is null then the seq of prefix is incremented by 1 else - * the supplied sequence is set in the IBF. - * - * @param prefix the prefix to be updated - * @param seq the sequence number of the prefix + * @param prefix prefix to get the sequence number of */ - void - publishName(const ndn::Name& prefix, ndn::optional seq = ndn::nullopt); - -private: - /** - * @brief Send sync interest for full synchronization - * - * Forms the interest name: // - * Cancels any pending sync interest we sent earlier on the face - * Sends the sync interest - */ - void - sendSyncInterest(); + ndn::optional + getSeqNo(const ndn::Name& prefix) const + { + return m_prefixes.getSeqNo(prefix); + } PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: /** - * @brief Process sync interest from other parties - * - * Get differences b/w our IBF and IBF in the sync interest. - * If we cannot get the differences successfully then send an application nack. + * @brief Adds a user node for synchronization * - * If we have some things in our IBF that the other side does not have, reply with the content or - * If no. of different items is greater than threshold or equals zero then send a nack. - * Otherwise add the sync interest into a map with interest name as key and PendingEntryInfoFull - * as value. + * Initializes m_prefixes[prefix] to zero + * Does not add zero-th sequence number to IBF + * because if a large number of user nodes are added + * then decoding of the difference between own IBF and + * other IBF will not be possible * - * @param prefixName prefix for sync group which we registered - * @param interest the interest we got + * @param prefix the user node to be added */ - void - onSyncInterest(const ndn::Name& prefixName, const ndn::Interest& interest); + bool + addUserNode(const ndn::Name& prefix) + { + return m_prefixes.addUserNode(prefix); + } -private: /** - * @brief Send sync data + * @brief Remove the user node from synchronization * - * Check if the data will satisfy our own pending interest, - * remove it first if it does, and then renew the sync interest - * Otherwise just send the data + * Erases prefix from IBF and other maps * - * @param name name to be set as data name - * @param block the content of the data + * @param prefix the user node to be removed */ void - sendSyncData(const ndn::Name& name, const ndn::Block& block); + removeUserNode(const ndn::Name& prefix) + { + if (m_prefixes.isUserNode(prefix)) { + uint64_t seqNo = m_prefixes.m_prefixes[prefix]; + m_prefixes.removeUserNode(prefix); + m_producerArbitrary.removeName(ndn::Name(prefix).appendNumber(seqNo)); + } + } /** - * @brief Process sync data - * - * Call deletePendingInterests to delete any pending sync interest with - * interest name would have been satisfied once NFD got the data. - * - * For each prefix/seq in data content check that we don't already have the - * prefix/seq and updateSeq(prefix, seq) - * - * Notify the application about the updates - * sendSyncInterest because the last one was satisfied by the incoming data + * @brief Publish name to let others know * - * @param interest interest for which we got the data - * @param bufferPtr sync data content - */ - void - onSyncData(const ndn::Interest& interest, const ndn::ConstBufferPtr& bufferPtr); - - /** - * @brief Satisfy pending sync interests + * addUserNode needs to be called before this to add the prefix + * if not already added via the constructor. + * If seq is null then the seq of prefix is incremented by 1 else + * the supplied sequence is set in the IBF. * - * For pending sync interests SI, if IBF of SI has any difference from our own IBF: - * send data back. - * If we can't decode differences from the stored IBF, then delete it. - */ - void - satisfyPendingInterests(); - - /** - * @brief Delete pending sync interests that match given name + * @param prefix the prefix to be updated + * @param seq the sequence number of the prefix */ void - deletePendingInterests(const ndn::Name& interestName); + publishName(const ndn::Name& prefix, ndn::optional seq = ndn::nullopt); +private: /** * @brief Check if hash(prefix + 1) is in negative * @@ -181,24 +140,21 @@ class FullProducer : public ProducerBase * gets to us before the data */ bool - isFutureHash(const ndn::Name& prefix, const std::set& negative); + isNotFutureHash(const ndn::Name& prefix, const std::set& negative); + +PUBLIC_WITH_TESTS_ELSE_PROTECTED: + void + updateSeqNo(const ndn::Name& prefix, uint64_t seq); private: - ndn::Face& m_face; - ndn::KeyChain m_keyChain; - ndn::Scheduler m_scheduler; + void + arbitraryUpdateCallBack(const std::vector& names); PUBLIC_WITH_TESTS_ELSE_PROTECTED: - SegmentPublisher m_segmentPublisher; - - std::map m_pendingEntries; - ndn::time::milliseconds m_syncInterestLifetime; - UpdateCallback m_onUpdate; - ndn::scheduler::ScopedEventId m_scheduledSyncInterestId; - std::uniform_int_distribution<> m_jitter; - ndn::Name m_outstandingInterestName; - ndn::ScopedRegisteredPrefixHandle m_registeredPrefix; - std::shared_ptr m_fetcher; + FullProducerArbitrary m_producerArbitrary; + UpdateCallback m_onUpdateCallback; + + UserPrefixes m_prefixes; }; } // namespace psync diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index 06eef7c..eb7c2cc 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -35,12 +35,14 @@ PartialProducer::PartialProducer(size_t expectedNumEntries, const ndn::Name& userPrefix, ndn::time::milliseconds syncReplyFreshness, ndn::time::milliseconds helloReplyFreshness) - : ProducerBase(expectedNumEntries, syncPrefix, - userPrefix, syncReplyFreshness, helloReplyFreshness) + : ProducerBase(expectedNumEntries, syncPrefix, syncReplyFreshness) , m_face(face) , m_scheduler(m_face.getIoService()) , m_segmentPublisher(m_face, m_keyChain) + , m_helloReplyFreshness(helloReplyFreshness) { + addUserNode(userPrefix); + m_registeredPrefix = m_face.registerPrefix(m_syncPrefix, [this] (const ndn::Name& syncPrefix) { m_face.setInterestFilter(ndn::Name(m_syncPrefix).append("hello"), @@ -54,11 +56,11 @@ PartialProducer::PartialProducer(size_t expectedNumEntries, void PartialProducer::publishName(const ndn::Name& prefix, ndn::optional seq) { - if (m_prefixes.find(prefix) == m_prefixes.end()) { + if (!m_prefixes.isUserNode(prefix)) { return; } - uint64_t newSeq = seq.value_or(m_prefixes[prefix] + 1); + uint64_t newSeq = seq.value_or(m_prefixes.m_prefixes[prefix] + 1); NDN_LOG_INFO("Publish: " << prefix << "/" << newSeq); @@ -67,6 +69,20 @@ PartialProducer::publishName(const ndn::Name& prefix, ndn::optional se satisfyPendingSyncInterests(prefix); } +void +PartialProducer::updateSeqNo(const ndn::Name& prefix, uint64_t seq) +{ + uint64_t oldSeq; + if (!m_prefixes.updateSeqNo(prefix, seq, oldSeq)) + return; // Delete the last sequence prefix from the iblt + // Because we don't insert zeroth prefix in IBF so no need to delete that + if (oldSeq != 0) { + removeFromIBF(ndn::Name(prefix).appendNumber(oldSeq)); + } // Insert the new seq no + ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seq); + insertToIBF(prefixWithSeq); +} + void PartialProducer::onHelloInterest(const ndn::Name& prefix, const ndn::Interest& interest) { @@ -86,7 +102,7 @@ PartialProducer::onHelloInterest(const ndn::Name& prefix, const ndn::Interest& i State state; - for (const auto& prefix : m_prefixes) { + for (const auto& prefix : m_prefixes.m_prefixes) { state.addContent(ndn::Name(prefix.first).appendNumber(prefix.second)); } NDN_LOG_DEBUG("sending content p: " << state); @@ -158,7 +174,7 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in std::set positive; std::set negative; - NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.size()); + NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.m_prefixes.size()); bool peel = diff.listEntries(positive, negative); @@ -181,7 +197,7 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in if (bf.contains(prefix.toUri())) { // generate data state.addContent(name); - NDN_LOG_DEBUG("Content: " << prefix << " " << std::to_string(m_prefixes[prefix])); + NDN_LOG_DEBUG("Content: " << prefix << " " << std::to_string(m_prefixes.m_prefixes[prefix])); } } @@ -221,7 +237,7 @@ PartialProducer::satisfyPendingSyncInterests(const ndn::Name& prefix) { NDN_LOG_TRACE("Result of listEntries on the difference: " << peel); - NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.size()); + NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.m_prefixes.size()); NDN_LOG_TRACE("m_threshold: " << m_threshold << " Total: " << positive.size() + negative.size()); if (!peel) { @@ -233,8 +249,8 @@ PartialProducer::satisfyPendingSyncInterests(const ndn::Name& prefix) { State state; if (entry.bf.contains(prefix.toUri()) || positive.size() + negative.size() >= m_threshold) { if (entry.bf.contains(prefix.toUri())) { - state.addContent(ndn::Name(prefix).appendNumber(m_prefixes[prefix])); - NDN_LOG_DEBUG("sending sync content " << prefix << " " << std::to_string(m_prefixes[prefix])); + state.addContent(ndn::Name(prefix).appendNumber(m_prefixes.m_prefixes[prefix])); + NDN_LOG_DEBUG("sending sync content " << prefix << " " << std::to_string(m_prefixes.m_prefixes[prefix])); } else { NDN_LOG_DEBUG("Sending with empty content to send latest IBF to consumer"); diff --git a/PSync/partial-producer.hpp b/PSync/partial-producer.hpp index daa6faa..4076495 100644 --- a/PSync/partial-producer.hpp +++ b/PSync/partial-producer.hpp @@ -21,6 +21,7 @@ #define PSYNC_PARTIAL_PRODUCER_HPP #include "PSync/detail/bloom-filter.hpp" +#include "PSync/detail/user-prefixes.hpp" #include "PSync/producer-base.hpp" #include @@ -32,6 +33,9 @@ namespace psync { +using namespace ndn::time_literals; +const ndn::time::milliseconds HELLO_REPLY_FRESHNESS = 1_s; + struct PendingEntryInfo { BloomFilter bf; @@ -70,6 +74,51 @@ class PartialProducer : public ProducerBase ndn::time::milliseconds helloReplyFreshness = HELLO_REPLY_FRESHNESS, ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS); + /** + * @brief Returns the current sequence number of the given prefix + * + * @param prefix prefix to get the sequence number of + */ + ndn::optional + getSeqNo(const ndn::Name& prefix) const + { + return m_prefixes.getSeqNo(prefix); + } + + /** + * @brief Adds a user node for synchronization + * + * Initializes m_prefixes[prefix] to zero + * Does not add zero-th sequence number to IBF + * because if a large number of user nodes are added + * then decoding of the difference between own IBF and + * other IBF will not be possible + * + * @param prefix the user node to be added + */ + bool + addUserNode(const ndn::Name& prefix) + { + return m_prefixes.addUserNode(prefix); + } + + /** + * @brief Remove the user node from synchronization + * + * Erases prefix from IBF and other maps + * + * @param prefix the user node to be removed + */ + void + removeUserNode(const ndn::Name& prefix) + { + if (m_prefixes.isUserNode(prefix)) { + uint64_t seqNo = m_prefixes.m_prefixes[prefix]; + m_prefixes.removeUserNode(prefix); + removeFromIBF(ndn::Name(prefix).appendNumber(seqNo)); + } + } + /** * @brief Publish name to let subscribed consumers know * @@ -94,6 +143,9 @@ class PartialProducer : public ProducerBase satisfyPendingSyncInterests(const ndn::Name& prefix); PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: + void + updateSeqNo(const ndn::Name& prefix, uint64_t seq); + /** * @brief Receive hello interest from consumer and respond with hello data * @@ -136,6 +188,9 @@ class PartialProducer : public ProducerBase SegmentPublisher m_segmentPublisher; std::map m_pendingEntries; ndn::ScopedRegisteredPrefixHandle m_registeredPrefix; + + UserPrefixes m_prefixes; + ndn::time::milliseconds m_helloReplyFreshness; }; } // namespace psync diff --git a/PSync/producer-base.cpp b/PSync/producer-base.cpp index f7d4360..7f2c78c 100644 --- a/PSync/producer-base.cpp +++ b/PSync/producer-base.cpp @@ -32,89 +32,29 @@ NDN_LOG_INIT(psync.ProducerBase); ProducerBase::ProducerBase(size_t expectedNumEntries, const ndn::Name& syncPrefix, - const ndn::Name& userPrefix, - ndn::time::milliseconds syncReplyFreshness, - ndn::time::milliseconds helloReplyFreshness) + ndn::time::milliseconds syncReplyFreshness) : m_iblt(expectedNumEntries) , m_expectedNumEntries(expectedNumEntries) , m_threshold(expectedNumEntries/2) , m_syncPrefix(syncPrefix) , m_syncReplyFreshness(syncReplyFreshness) - , m_helloReplyFreshness(helloReplyFreshness) , m_rng(ndn::random::getRandomNumberEngine()) { - addUserNode(userPrefix); -} - -bool -ProducerBase::addUserNode(const ndn::Name& prefix) -{ - if (m_prefixes.find(prefix) == m_prefixes.end()) { - m_prefixes[prefix] = 0; - return true; - } - else { - return false; - } -} - -void -ProducerBase::removeUserNode(const ndn::Name& prefix) -{ - auto it = m_prefixes.find(prefix); - if (it != m_prefixes.end()) { - uint64_t seqNo = it->second; - m_prefixes.erase(it); - - removeFromIBF(ndn::Name(prefix).appendNumber(seqNo)); - } -} - -void -ProducerBase::updateSeqNo(const ndn::Name& prefix, uint64_t seq) -{ - NDN_LOG_DEBUG("UpdateSeq: " << prefix << " " << seq); - - uint64_t oldSeq; - auto it = m_prefixes.find(prefix); - if (it != m_prefixes.end()) { - oldSeq = it->second; - } - else { - NDN_LOG_WARN("Prefix not found in m_prefixes"); - return; - } - - if (oldSeq >= seq) { - NDN_LOG_WARN("Update has lower/equal seq no for prefix, doing nothing!"); - return; - } - - // Delete the last sequence prefix from the iblt - // Because we don't insert zeroth prefix in IBF so no need to delete that - if (oldSeq != 0) { - removeFromIBF(ndn::Name(prefix).appendNumber(oldSeq)); - } - - // Insert the new seq no - it->second = seq; - ndn::Name prefixWithSeq = ndn::Name(prefix).appendNumber(seq); - insertToIBF(prefixWithSeq); } void -ProducerBase::insertToIBF(const ndn::Name& prefix) +ProducerBase::insertToIBF(const ndn::Name& name) { - uint32_t newHash = murmurHash3(N_HASHCHECK, prefix.toUri()); - m_name2hash[prefix] = newHash; - m_hash2name[newHash] = prefix; + uint32_t newHash = murmurHash3(N_HASHCHECK, name.toUri()); + m_name2hash[name] = newHash; + m_hash2name[newHash] = name; m_iblt.insert(newHash); } void -ProducerBase::removeFromIBF(const ndn::Name& prefix) +ProducerBase::removeFromIBF(const ndn::Name& name) { - auto hashIt = m_name2hash.find(prefix); + auto hashIt = m_name2hash.find(name); if (hashIt != m_name2hash.end()) { uint32_t hash = hashIt->second; m_name2hash.erase(hashIt); diff --git a/PSync/producer-base.hpp b/PSync/producer-base.hpp index 94bea6f..39fa236 100644 --- a/PSync/producer-base.hpp +++ b/PSync/producer-base.hpp @@ -41,7 +41,6 @@ namespace psync { using namespace ndn::time_literals; const ndn::time::milliseconds SYNC_REPLY_FRESHNESS = 1_s; -const ndn::time::milliseconds HELLO_REPLY_FRESHNESS = 1_s; /** * @brief Base class for PartialProducer and FullProducer @@ -69,54 +68,7 @@ class ProducerBase */ ProducerBase(size_t expectedNumEntries, const ndn::Name& syncPrefix, - const ndn::Name& userPrefix, - ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS, - ndn::time::milliseconds helloReplyFreshness = HELLO_REPLY_FRESHNESS); -public: - /** - * @brief Returns the current sequence number of the given prefix - * - * @param prefix prefix to get the sequence number of - */ - ndn::optional - getSeqNo(const ndn::Name& prefix) const - { - auto it = m_prefixes.find(prefix); - if (it == m_prefixes.end()) { - return ndn::nullopt; - } - return it->second; - } - - /** - * @brief Adds a user node for synchronization - * - * Initializes m_prefixes[prefix] to zero - * Does not add zero-th sequence number to IBF - * because if a large number of user nodes are added - * then decoding of the difference between own IBF and - * other IBF will not be possible - * - * @param prefix the user node to be added - */ - bool - addUserNode(const ndn::Name& prefix); - - /** - * @brief Remove the user node from synchronization - * - * Erases prefix from IBF and other maps - * - * @param prefix the user node to be removed - */ - void - removeUserNode(const ndn::Name& prefix); - - void - insertToIBF(const ndn::Name& prefix); - - void - removeFromIBF(const ndn::Name& prefix); + ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS); PSYNC_PUBLIC_WITH_TESTS_ELSE_PROTECTED: /** @@ -131,13 +83,10 @@ class ProducerBase * @param seq sequence number of the update */ void - updateSeqNo(const ndn::Name& prefix, uint64_t seq); + insertToIBF(const ndn::Name& name); - bool - isUserNode(const ndn::Name& prefix) const - { - return m_prefixes.find(prefix) != m_prefixes.end(); - } + void + removeFromIBF(const ndn::Name& name); /** * @brief Logs a message if setting an interest filter fails @@ -155,8 +104,6 @@ class ProducerBase // than it and whether we need to update the other side. uint32_t m_threshold; - // prefix and sequence number - std::map m_prefixes; // Just for looking up hash faster (instead of calculating it again) // Only used in updateSeqNo, name (arbitrary or /prefix/seq) is the key std::map m_name2hash; @@ -166,7 +113,6 @@ class ProducerBase ndn::Name m_syncPrefix; ndn::time::milliseconds m_syncReplyFreshness; - ndn::time::milliseconds m_helloReplyFreshness; ndn::random::RandomNumberEngine& m_rng; }; diff --git a/tests/test-full-producer.cpp b/tests/test-full-producer.cpp index 8190446..e74744e 100644 --- a/tests/test-full-producer.cpp +++ b/tests/test-full-producer.cpp @@ -47,7 +47,8 @@ BOOST_AUTO_TEST_CASE(OnInterest) Name syncInterestName(syncPrefix); syncInterestName.append("malicious-IBF"); - BOOST_REQUIRE_NO_THROW(node.onSyncInterest(syncPrefix, Interest(syncInterestName))); + BOOST_REQUIRE_NO_THROW(node.m_producerArbitrary.onSyncInterest(syncPrefix, + Interest(syncInterestName))); } BOOST_AUTO_TEST_SUITE_END() diff --git a/tests/test-full-sync.cpp b/tests/test-full-sync.cpp index af7c238..f3b66a9 100644 --- a/tests/test-full-sync.cpp +++ b/tests/test-full-sync.cpp @@ -410,14 +410,14 @@ BOOST_AUTO_TEST_CASE(DelayedSecondSegment) IBLT iblt(40); iblt.appendToName(syncInterestName); - nodes[0]->onSyncInterest(syncPrefix, Interest(syncInterestName)); + nodes[0]->m_producerArbitrary.onSyncInterest(syncPrefix, Interest(syncInterestName)); advanceClocks(ndn::time::milliseconds(10)); - BOOST_CHECK_EQUAL(nodes[0]->m_segmentPublisher.m_ims.size(), 2); + BOOST_CHECK_EQUAL(nodes[0]->m_producerArbitrary.m_segmentPublisher.m_ims.size(), 2); // Expire contents from segmentPublisher advanceClocks(ndn::time::milliseconds(10), 100); - BOOST_CHECK_EQUAL(nodes[0]->m_segmentPublisher.m_ims.size(), 0); + BOOST_CHECK_EQUAL(nodes[0]->m_producerArbitrary.m_segmentPublisher.m_ims.size(), 0); // Get data name from face and increase segment number to form next interest Name dataName = faces[0]->sentData.front().getName(); @@ -425,11 +425,11 @@ BOOST_AUTO_TEST_CASE(DelayedSecondSegment) interestName.appendSegment(1); faces[0]->sentData.clear(); - nodes[0]->onSyncInterest(syncPrefix, Interest(interestName)); + nodes[0]->m_producerArbitrary.onSyncInterest(syncPrefix, Interest(interestName)); advanceClocks(ndn::time::milliseconds(10)); // Should have repopulated SegmentPublisher - BOOST_CHECK_EQUAL(nodes[0]->m_segmentPublisher.m_ims.size(), 2); + BOOST_CHECK_EQUAL(nodes[0]->m_producerArbitrary.m_segmentPublisher.m_ims.size(), 2); // Should have received the second data segment this time BOOST_CHECK_EQUAL(faces[0]->sentData.front().getName()[-1].toSegment(), 1); } diff --git a/tests/test-partial-sync.cpp b/tests/test-partial-sync.cpp index 8938483..abd28a6 100644 --- a/tests/test-partial-sync.cpp +++ b/tests/test-partial-sync.cpp @@ -83,7 +83,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture for (const auto& update : updates) { BOOST_CHECK(consumers[id]->isSubscribed(update.prefix)); BOOST_CHECK_EQUAL(oldSeqMap.at(update.prefix) + 1, update.lowSeq); - BOOST_CHECK_EQUAL(producer->m_prefixes.at(update.prefix), update.highSeq); + BOOST_CHECK_EQUAL(producer->m_prefixes.m_prefixes.at(update.prefix), update.highSeq); BOOST_CHECK_EQUAL(consumers[id]->getSeqNo(update.prefix).value(), update.highSeq); } }, 40, 0.001); @@ -102,7 +102,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture bool checkSubList(const vector& availableSubs) { - for (const auto& prefix : producer->m_prefixes ) { + for (const auto& prefix : producer->m_prefixes.m_prefixes ) { if (std::find(availableSubs.begin(), availableSubs.end(), prefix.first) == availableSubs.end()) { return false; } @@ -122,7 +122,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture void publishUpdateFor(const std::string& prefix) { - oldSeqMap = producer->m_prefixes; + oldSeqMap = producer->m_prefixes.m_prefixes; producer->publishName(prefix); advanceClocks(ndn::time::milliseconds(10)); } @@ -130,7 +130,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture void updateSeqFor(const std::string& prefix, uint64_t seq) { - oldSeqMap = producer->m_prefixes; + oldSeqMap = producer->m_prefixes.m_prefixes; producer->updateSeqNo(prefix, seq); } @@ -326,7 +326,7 @@ BOOST_AUTO_TEST_CASE(ApplicationNack) publishUpdateFor("testUser-2"); BOOST_CHECK_EQUAL(numSyncDataRcvd, 1); - oldSeqMap = producer->m_prefixes; + oldSeqMap = producer->m_prefixes.m_prefixes; for (int i = 0; i < 50; i++) { ndn::Name prefix("testUser-" + to_string(i)); producer->updateSeqNo(prefix, producer->getSeqNo(prefix).value() + 1); @@ -402,7 +402,7 @@ BOOST_AUTO_TEST_CASE(SegmentedSync) syncInterestName.appendVersion(); syncInterestName.appendSegment(1); - oldSeqMap = producer->m_prefixes; + oldSeqMap = producer->m_prefixes.m_prefixes; for (int i = 1; i < 10; i++) { producer->updateSeqNo(longNameToExceedDataSize.toUri() + "-" + to_string(i), 1); } diff --git a/tests/test-producer-base.cpp b/tests/test-producer-base.cpp index 5b32a06..334ebbe 100644 --- a/tests/test-producer-base.cpp +++ b/tests/test-producer-base.cpp @@ -35,14 +35,14 @@ BOOST_AUTO_TEST_SUITE(TestProducerBase) BOOST_AUTO_TEST_CASE(Ctor) { util::DummyClientFace face; - BOOST_REQUIRE_NO_THROW(ProducerBase(40, Name("/psync"), Name("/testUser"))); + BOOST_REQUIRE_NO_THROW(ProducerBase(40, Name("/psync"))); } -BOOST_AUTO_TEST_CASE(Basic) +/*BOOST_AUTO_TEST_CASE(Basic) { util::DummyClientFace face; Name userNode("/testUser"); - ProducerBase producerBase(40, Name("/psync"), userNode); + ProducerBase producerBase(40, Name("/psync")); // Hash table size should be 40 + 40/2 = 60 (which is perfectly divisible by N_HASH = 3) BOOST_CHECK_EQUAL(producerBase.m_iblt.getHashTable().size(), 60); BOOST_CHECK_EQUAL(producerBase.getSeqNo(userNode).value(), 0); @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(Basic) producerBase.updateSeqNo(nonExistentUserNode, 1); BOOST_CHECK(producerBase.m_name2hash.find(Name(nonExistentUserNode).appendNumber(1).toUri()) == producerBase.m_name2hash.end()); -} +}*/ BOOST_AUTO_TEST_SUITE_END() From 672052b6b2e9b92f6382822183fb986c7f5e86ad Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Sat, 9 Mar 2019 17:50:35 -0800 Subject: [PATCH 10/19] FullProducerArbitrary: Remove unused userPrefix. --- PSync/full-producer-arbitrary.cpp | 1 - PSync/full-producer-arbitrary.hpp | 2 -- PSync/full-producer.cpp | 2 +- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/PSync/full-producer-arbitrary.cpp b/PSync/full-producer-arbitrary.cpp index 4d66ab7..ea0706a 100644 --- a/PSync/full-producer-arbitrary.cpp +++ b/PSync/full-producer-arbitrary.cpp @@ -34,7 +34,6 @@ NDN_LOG_INIT(psync.FullProducerArbitrary); FullProducerArbitrary::FullProducerArbitrary(const size_t expectedNumEntries, ndn::Face& face, const ndn::Name& syncPrefix, - const ndn::Name& userPrefix, const ArbitraryUpdateCallback& onArbitraryUpdateCallback, ndn::time::milliseconds syncInterestLifetime, ndn::time::milliseconds syncReplyFreshness, diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp index 80ba1ac..47c9a86 100644 --- a/PSync/full-producer-arbitrary.hpp +++ b/PSync/full-producer-arbitrary.hpp @@ -71,7 +71,6 @@ class FullProducerArbitrary : public ProducerBase * @param expectedNumEntries expected entries in IBF * @param face application's face * @param syncPrefix The prefix of the sync group - * @param userPrefix The prefix of the first user in the group * @param onUpdateCallBack The call back to be called when there is new data * @param syncInterestLifetime lifetime of the sync interest * @param syncReplyFreshness freshness of sync data @@ -79,7 +78,6 @@ class FullProducerArbitrary : public ProducerBase FullProducerArbitrary(size_t expectedNumEntries, ndn::Face& face, const ndn::Name& syncPrefix, - const ndn::Name& userPrefix, const ArbitraryUpdateCallback& onArbitraryUpdateCallback, ndn::time::milliseconds syncInterestLifetime = SYNC_INTEREST_LIFTIME, ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS, diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index 0a59dfc..cae8ee1 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -38,7 +38,7 @@ FullProducer::FullProducer(const size_t expectedNumEntries, const UpdateCallback& onUpdateCallBack, ndn::time::milliseconds syncInterestLifetime, ndn::time::milliseconds syncReplyFreshness) - : m_producerArbitrary(expectedNumEntries, face, syncPrefix, userPrefix, + : m_producerArbitrary(expectedNumEntries, face, syncPrefix, [this] (const std::vector& names) { arbitraryUpdateCallBack(names); }, From d88d137ecf5378995956e72f9ada8d5b61d8cd75 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Sat, 9 Mar 2019 18:31:29 -0800 Subject: [PATCH 11/19] examples: Added full-sync-arbitrary.cpp. --- examples/full-sync-arbitrary.cpp | 125 +++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 examples/full-sync-arbitrary.cpp diff --git a/examples/full-sync-arbitrary.cpp b/examples/full-sync-arbitrary.cpp new file mode 100644 index 0000000..768e3c8 --- /dev/null +++ b/examples/full-sync-arbitrary.cpp @@ -0,0 +1,125 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#include + +#include +#include + +#include + +NDN_LOG_INIT(examples.FullSyncApp); + +using namespace ndn::time_literals; + +class Producer +{ +public: + /** + * @brief Initialize producer and schedule updates + * + * Set IBF size as 80 expecting 80 updates to IBF in a sync cycle + * Set syncInterestLifetime and syncReplyFreshness to 1.6 seconds + */ + Producer(const ndn::Name& syncPrefix, const std::string& userPrefix, + int numDataPrefixes, int maxNumPublish) + : m_scheduler(m_face.getIoService()) + , m_fullProducer(80, m_face, syncPrefix, + std::bind(&Producer::processSyncUpdate, this, _1), + 1600_ms, 1600_ms) + , m_maxNumPublish(maxNumPublish) + , m_rng(ndn::random::getRandomNumberEngine()) + , m_rangeUniformRandom(0, 6000) + { + // Add name prefixes and schedule updates for them in specified the interval. + for (int i = 0; i < numDataPrefixes; i++) { + ndn::Name dataPrefix(userPrefix + "-" + ndn::to_string(i)); + m_nPublished[dataPrefix] = 0; + + m_scheduler.scheduleEvent(ndn::time::milliseconds(m_rangeUniformRandom(m_rng)), + [this, dataPrefix] { + doUpdate(dataPrefix); + }); + } + } + + void + run() + { + m_face.processEvents(); + } + +private: + void + doUpdate(const ndn::Name& dataPrefix) + { + ndn::Name name = ndn::Name(dataPrefix).appendVersion(); + + NDN_LOG_INFO("Publish: " << name); + m_fullProducer.publishName(name); + + ++m_nPublished[dataPrefix]; + + if (m_nPublished[dataPrefix] < m_maxNumPublish) { + m_scheduler.scheduleEvent(ndn::time::milliseconds(m_rangeUniformRandom(m_rng)), + [this, dataPrefix] { + doUpdate(dataPrefix); + }); + } + } + + void + processSyncUpdate(const std::vector& updates) + { + for (const auto& name : updates) { + NDN_LOG_INFO("Update " << name); + } + } + +private: + ndn::Face m_face; + ndn::util::Scheduler m_scheduler; + + psync::FullProducerArbitrary m_fullProducer; + + uint64_t m_maxNumPublish; + std::map m_nPublished; + + ndn::random::RandomNumberEngine& m_rng; + std::uniform_int_distribution<> m_rangeUniformRandom; +}; + +int +main(int argc, char* argv[]) +{ + if (argc != 5) { + std::cout << "usage: " << argv[0] << " " + << " " + << std::endl; + return 1; + } + + try { + Producer producer(argv[1], argv[2], std::stoi(argv[3]), std::stoi(argv[4])); + producer.run(); + } + catch (const std::exception& e) { + NDN_LOG_ERROR(e.what()); + } +} From 69bc30a9aa5a527b1b619c9ea60a5aba33a9f21e Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 9 Mar 2019 18:43:38 -0800 Subject: [PATCH 12/19] Add CanAddName callback Fix sync Change-Id: I672ddcba40a589b69168d7b6f1acf6a60e27bb6a --- PSync/full-producer-arbitrary.cpp | 22 +++++++++++----------- PSync/full-producer-arbitrary.hpp | 6 +++--- PSync/full-producer.cpp | 15 ++++++++++----- tests/test-full-sync.cpp | 4 ++-- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/PSync/full-producer-arbitrary.cpp b/PSync/full-producer-arbitrary.cpp index ea0706a..2bb5ec7 100644 --- a/PSync/full-producer-arbitrary.cpp +++ b/PSync/full-producer-arbitrary.cpp @@ -38,7 +38,7 @@ FullProducerArbitrary::FullProducerArbitrary(const size_t expectedNumEntries, ndn::time::milliseconds syncInterestLifetime, ndn::time::milliseconds syncReplyFreshness, const ShouldAddToSyncDataCallback& onShouldAddToSyncDataCallback, - const SyncDataCallback onSyncDataCallBack) + const CanAddName& onCanAddName) : ProducerBase(expectedNumEntries, syncPrefix, syncReplyFreshness) , m_face(face) , m_scheduler(m_face.getIoService()) @@ -46,7 +46,7 @@ FullProducerArbitrary::FullProducerArbitrary(const size_t expectedNumEntries, , m_syncInterestLifetime(syncInterestLifetime) , m_onArbitraryUpdateCallback(onArbitraryUpdateCallback) , m_onShouldAddToSyncDataCallback(onShouldAddToSyncDataCallback) - , m_onSyncDataCallback(onSyncDataCallBack) + , m_onCanAddName(onCanAddName) { int jitter = m_syncInterestLifetime.count() * .20; m_jitter = std::uniform_int_distribution<>(-jitter, jitter); @@ -193,10 +193,11 @@ FullProducerArbitrary::onSyncInterest(const ndn::Name& prefixName, const ndn::In ndn::Name name = m_hash2name[hash]; ndn::Name prefix = name.getPrefix(-1); - // Don't sync up sequence number zero - if (m_name2hash.find(name) != m_name2hash.end() && - m_onShouldAddToSyncDataCallback(prefix.toUri(), negative)) { - state.addContent(name); + if (m_name2hash.find(name) != m_name2hash.end()) { + if (!m_onShouldAddToSyncDataCallback || + m_onShouldAddToSyncDataCallback(prefix.toUri(), negative)) { + state.addContent(name); + } } } @@ -264,11 +265,10 @@ FullProducerArbitrary::onSyncData(const ndn::Interest& interest, const ndn::Cons for (const auto& name : state.getContent()) { if (m_name2hash.find(name) == m_name2hash.end()) { - updates.push_back(name); - if (m_onSyncDataCallback) { - m_onSyncDataCallback(name); - } - else { + NDN_LOG_DEBUG("Checking whether to add"); + if (!m_onCanAddName || m_onCanAddName(name)) { + NDN_LOG_DEBUG("Adding..."); + updates.push_back(name); insertToIBF(name); } // We should not call satisfyPendingSyncInterests here because we just diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp index 47c9a86..431b119 100644 --- a/PSync/full-producer-arbitrary.hpp +++ b/PSync/full-producer-arbitrary.hpp @@ -45,7 +45,7 @@ struct PendingEntryInfoFull }; typedef std::function&)> ArbitraryUpdateCallback; -typedef std::function SyncDataCallback; +typedef std::function CanAddName; typedef std::function&)> ShouldAddToSyncDataCallback; @@ -82,7 +82,7 @@ class FullProducerArbitrary : public ProducerBase ndn::time::milliseconds syncInterestLifetime = SYNC_INTEREST_LIFTIME, ndn::time::milliseconds syncReplyFreshness = SYNC_REPLY_FRESHNESS, const ShouldAddToSyncDataCallback& onShouldAddToSyncDataCallback = ShouldAddToSyncDataCallback(), - const SyncDataCallback onSyncDataCallBack = SyncDataCallback()); + const CanAddName& onCanAddName = CanAddName()); ~FullProducerArbitrary(); @@ -201,7 +201,7 @@ class FullProducerArbitrary : public ProducerBase ndn::time::milliseconds m_syncInterestLifetime; ArbitraryUpdateCallback m_onArbitraryUpdateCallback; ShouldAddToSyncDataCallback m_onShouldAddToSyncDataCallback; - SyncDataCallback m_onSyncDataCallback; + CanAddName m_onCanAddName; ndn::util::scheduler::ScopedEventId m_scheduledSyncInterestId; std::uniform_int_distribution<> m_jitter; ndn::Name m_outstandingInterestName; diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index cae8ee1..22f852a 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -52,8 +52,13 @@ FullProducer::FullProducer(const size_t expectedNumEntries, if (m_prefixes.m_prefixes.find(prefix) == m_prefixes.m_prefixes.end() || m_prefixes.m_prefixes[prefix] < seq) { - updateSeqNo(prefix, seq); + uint64_t oldSeq = m_prefixes.m_prefixes[prefix]; + if (oldSeq != 0) { + m_producerArbitrary.removeName(ndn::Name(prefix).appendNumber(oldSeq)); + } + return true; } + return false; }) , m_onUpdateCallback(onUpdateCallBack) { @@ -113,10 +118,10 @@ FullProducer::arbitraryUpdateCallBack(const std::vector& names) ndn::Name prefix = name.getPrefix(-1); uint64_t seq = name.get(-1).toNumber(); - if (m_prefixes.m_prefixes.find(prefix) == m_prefixes.m_prefixes.end() || - m_prefixes.m_prefixes[prefix] < seq) { - updates.push_back(MissingDataInfo{prefix, m_prefixes.m_prefixes[prefix] + 1, seq}); - } + NDN_LOG_INFO("Updates: " << prefix << " " << seq); + + updates.push_back(MissingDataInfo{prefix, m_prefixes.m_prefixes[prefix] + 1, seq}); + m_prefixes.m_prefixes[prefix] = seq; } m_onUpdateCallback(updates); diff --git a/tests/test-full-sync.cpp b/tests/test-full-sync.cpp index f3b66a9..1cee4b5 100644 --- a/tests/test-full-sync.cpp +++ b/tests/test-full-sync.cpp @@ -70,7 +70,7 @@ BOOST_AUTO_TEST_CASE(TwoNodesSimple) BOOST_CHECK_EQUAL(nodes[0]->getSeqNo(userPrefixes[0]).value_or(-1), 1); BOOST_CHECK_EQUAL(nodes[1]->getSeqNo(userPrefixes[0]).value_or(-1), 1); - nodes[1]->publishName(userPrefixes[1]); + /*nodes[1]->publishName(userPrefixes[1]); advanceClocks(ndn::time::milliseconds(10), 100); BOOST_CHECK_EQUAL(nodes[0]->getSeqNo(userPrefixes[1]).value_or(-1), 1); BOOST_CHECK_EQUAL(nodes[1]->getSeqNo(userPrefixes[1]).value_or(-1), 1); @@ -78,7 +78,7 @@ BOOST_AUTO_TEST_CASE(TwoNodesSimple) nodes[1]->publishName(userPrefixes[1]); advanceClocks(ndn::time::milliseconds(10), 100); BOOST_CHECK_EQUAL(nodes[0]->getSeqNo(userPrefixes[1]).value_or(-1), 2); - BOOST_CHECK_EQUAL(nodes[1]->getSeqNo(userPrefixes[1]).value_or(-1), 2); + BOOST_CHECK_EQUAL(nodes[1]->getSeqNo(userPrefixes[1]).value_or(-1), 2);*/ } BOOST_AUTO_TEST_CASE(TwoNodesForceSeqNo) From 3c0d1c56dd4b206aea7909f50a48ab16c6c9b12f Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Mon, 11 Mar 2019 15:48:08 -0700 Subject: [PATCH 13/19] UserPrefixes (internal): In removeUserNode, removed unused local variable. --- PSync/detail/user-prefixes.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/PSync/detail/user-prefixes.cpp b/PSync/detail/user-prefixes.cpp index e510253..cd6b961 100644 --- a/PSync/detail/user-prefixes.cpp +++ b/PSync/detail/user-prefixes.cpp @@ -41,7 +41,6 @@ UserPrefixes::removeUserNode(const ndn::Name& prefix) { auto it = m_prefixes.find(prefix); if (it != m_prefixes.end()) { - uint64_t seqNo = it->second; m_prefixes.erase(it); } } From dd9a4ac8020fc496adac115a2bc7b9b3aa519cbe Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Tue, 12 Mar 2019 11:20:00 -0700 Subject: [PATCH 14/19] FullProducerArbitrary (minor): In satisfyPendingInterests, remove unused local variable. --- PSync/full-producer-arbitrary.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/PSync/full-producer-arbitrary.cpp b/PSync/full-producer-arbitrary.cpp index 2bb5ec7..4b897c4 100644 --- a/PSync/full-producer-arbitrary.cpp +++ b/PSync/full-producer-arbitrary.cpp @@ -314,7 +314,6 @@ FullProducerArbitrary::satisfyPendingInterests() State state; for (const auto& hash : positive) { ndn::Name name = m_hash2name[hash]; - ndn::Name prefix = name.getPrefix(-1); if (m_name2hash.find(name) != m_name2hash.end()) { state.addContent(name); From f5721867471c6a55375597e4eff3c51df39a9697 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Mon, 18 Mar 2019 11:41:05 -0700 Subject: [PATCH 15/19] FullProducerArbitrary: Fix the doc for publishName for an arbitrary name. --- PSync/full-producer-arbitrary.hpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp index 431b119..f784374 100644 --- a/PSync/full-producer-arbitrary.hpp +++ b/PSync/full-producer-arbitrary.hpp @@ -99,13 +99,7 @@ class FullProducerArbitrary : public ProducerBase /** * @brief Publish name to let others know * - * addUserNode needs to be called before this to add the prefix - * if not already added via the constructor. - * If seq is null then the seq of prefix is incremented by 1 else - * the supplied sequence is set in the IBF. - * - * @param prefix the prefix to be updated - * @param seq the sequence number of the prefix + * @param name the Name to be updated */ void publishName(const ndn::Name& name); From a9e143d0f5389cd8fe35051201030d2a911a4e72 Mon Sep 17 00:00:00 2001 From: Jeff Thompson Date: Mon, 18 Mar 2019 11:42:06 -0700 Subject: [PATCH 16/19] FullProducerArbitrary: In publishName, do nothing if the name is already published. --- PSync/full-producer-arbitrary.cpp | 5 +++++ PSync/full-producer-arbitrary.hpp | 1 + 2 files changed, 6 insertions(+) diff --git a/PSync/full-producer-arbitrary.cpp b/PSync/full-producer-arbitrary.cpp index 4b897c4..3466e31 100644 --- a/PSync/full-producer-arbitrary.cpp +++ b/PSync/full-producer-arbitrary.cpp @@ -71,6 +71,11 @@ FullProducerArbitrary::~FullProducerArbitrary() void FullProducerArbitrary::publishName(const ndn::Name& name) { + if (m_name2hash.find(name) != m_name2hash.end()) { + NDN_LOG_DEBUG("Already published, ignoring: " << name); + return; + } + NDN_LOG_INFO("Publish: " << name); insertToIBF(name); diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp index f784374..4843295 100644 --- a/PSync/full-producer-arbitrary.hpp +++ b/PSync/full-producer-arbitrary.hpp @@ -99,6 +99,7 @@ class FullProducerArbitrary : public ProducerBase /** * @brief Publish name to let others know * + * However, if the name has already been published, do nothing. * @param name the Name to be updated */ void From 8fa2a988cb87705dba1fcb0ffa1b74c275dbdf5a Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 13 Jul 2019 10:48:14 -0700 Subject: [PATCH 17/19] full-producer*: fix compilation Fix some code style issues Change-Id: I718241514bc05516ba18910e57fa6f49e0ad6602 --- PSync/detail/user-prefixes.cpp | 16 +- PSync/detail/user-prefixes.hpp | 24 +-- PSync/full-producer-arbitrary.cpp | 8 +- PSync/full-producer-arbitrary.hpp | 10 +- PSync/full-producer.cpp | 14 +- PSync/full-producer.hpp | 6 +- PSync/partial-producer.cpp | 14 +- PSync/partial-producer.hpp | 2 +- tests/bk-test-bloom-filter.cpp.disable | 100 ++++++++++ tests/bk-test-iblt.cpp.disable | 247 +++++++++++++++++++++++++ tests/test-partial-sync.cpp | 14 +- 11 files changed, 401 insertions(+), 54 deletions(-) create mode 100644 tests/bk-test-bloom-filter.cpp.disable create mode 100644 tests/bk-test-iblt.cpp.disable diff --git a/PSync/detail/user-prefixes.cpp b/PSync/detail/user-prefixes.cpp index cd6b961..db4855b 100644 --- a/PSync/detail/user-prefixes.cpp +++ b/PSync/detail/user-prefixes.cpp @@ -17,8 +17,8 @@ * PSync, e.g., in COPYING.md file. If not, see . **/ -#include #include "PSync/detail/user-prefixes.hpp" +#include namespace psync { @@ -28,7 +28,7 @@ bool UserPrefixes::addUserNode(const ndn::Name& prefix) { if (!isUserNode(prefix)) { - m_prefixes[prefix] = 0; + prefixes[prefix] = 0; return true; } else { @@ -39,9 +39,9 @@ UserPrefixes::addUserNode(const ndn::Name& prefix) void UserPrefixes::removeUserNode(const ndn::Name& prefix) { - auto it = m_prefixes.find(prefix); - if (it != m_prefixes.end()) { - m_prefixes.erase(it); + auto it = prefixes.find(prefix); + if (it != prefixes.end()) { + prefixes.erase(it); } } @@ -52,12 +52,12 @@ UserPrefixes::updateSeqNo oldSeqNo = 0; NDN_LOG_DEBUG("UpdateSeq: " << prefix << " " << seqNo); - auto it = m_prefixes.find(prefix); - if (it != m_prefixes.end()) { + auto it = prefixes.find(prefix); + if (it != prefixes.end()) { oldSeqNo = it->second; } else { - NDN_LOG_WARN("Prefix not found in m_prefixes"); + NDN_LOG_WARN("Prefix not found in prefixes"); return false; } diff --git a/PSync/detail/user-prefixes.hpp b/PSync/detail/user-prefixes.hpp index 23e13ad..9ab8596 100644 --- a/PSync/detail/user-prefixes.hpp +++ b/PSync/detail/user-prefixes.hpp @@ -26,7 +26,7 @@ namespace psync { /** - * @brief UserPrefixes holds the m_prefixes map from prefix to sequence number, + * @brief UserPrefixes holds the prefixes map from prefix to sequence number, * used by PartialProducer and FullProducer. * * Contains code common to both @@ -35,15 +35,15 @@ class UserPrefixes { public: /** - * @brief Check if the prefix is in m_prefixes. + * @brief Check if the prefix is in prefixes. * * @param prefix The prefix to check. - * @return True if the prefix is in m_prefixes. + * @return True if the prefix is in prefixes. */ bool isUserNode(const ndn::Name& prefix) const { - return m_prefixes.find(prefix) != m_prefixes.end(); + return prefixes.find(prefix) != prefixes.end(); } /** @@ -54,8 +54,8 @@ class UserPrefixes ndn::optional getSeqNo(const ndn::Name& prefix) const { - auto it = m_prefixes.find(prefix); - if (it == m_prefixes.end()) { + auto it = prefixes.find(prefix); + if (it == prefixes.end()) { return ndn::nullopt; } return it->second; @@ -64,18 +64,18 @@ class UserPrefixes /** * @brief Adds a user node for synchronization * - * Initializes m_prefixes[prefix] to zero + * Initializes prefixes[prefix] to zero * * @param prefix the user node to be added * @return true if the prefix was added, false if the prefix was already in - * m_prefixes. + * prefixes. */ bool addUserNode(const ndn::Name& prefix); /** * @brief Remove the user node from synchronization. If the prefix is not in - * m_prefixes, then do nothing. + * prefixes, then do nothing. * * The caller should first check isUserNode(prefix) and erase the prefix from * the IBLT and other maps if needed. @@ -86,7 +86,7 @@ class UserPrefixes removeUserNode(const ndn::Name& prefix); /** - * @brief Update m_prefixes with the given prefix and sequence number. This + * @brief Update prefixes with the given prefix and sequence number. This * does not update the IBLT. This logs a message for the update. * * Whoever calls this needs to make sure that isUserNode(prefix) is true. @@ -97,7 +97,7 @@ class UserPrefixes * prefix. If this method returns true and oldSeqNo is not zero, the caller * can remove the old prefix from the IBLT. * @return True if the sequence number was updated, false if the prefix was - * not in m_prefixes, or if the seqNo is less than or equal to the old + * not in prefixes, or if the seqNo is less than or equal to the old * sequence number. If this returns false, the caller should not update the * IBLT. */ @@ -105,7 +105,7 @@ class UserPrefixes updateSeqNo(const ndn::Name& prefix, uint64_t seqNo, uint64_t& oldSeqNo); // prefix and sequence number - std::map m_prefixes; + std::map prefixes; }; } // namespace psync diff --git a/PSync/full-producer-arbitrary.cpp b/PSync/full-producer-arbitrary.cpp index 3466e31..ca1d2d0 100644 --- a/PSync/full-producer-arbitrary.cpp +++ b/PSync/full-producer-arbitrary.cpp @@ -102,8 +102,8 @@ FullProducerArbitrary::sendSyncInterest() m_outstandingInterestName = syncInterestName; m_scheduledSyncInterestId = - m_scheduler.scheduleEvent(m_syncInterestLifetime / 2 + ndn::time::milliseconds(m_jitter(m_rng)), - [this] { sendSyncInterest(); }); + m_scheduler.schedule(m_syncInterestLifetime / 2 + ndn::time::milliseconds(m_jitter(m_rng)), + [this] { sendSyncInterest(); }); ndn::Interest syncInterest(syncInterestName); @@ -201,7 +201,7 @@ FullProducerArbitrary::onSyncInterest(const ndn::Name& prefixName, const ndn::In if (m_name2hash.find(name) != m_name2hash.end()) { if (!m_onShouldAddToSyncDataCallback || m_onShouldAddToSyncDataCallback(prefix.toUri(), negative)) { - state.addContent(name); + state.addContent(name); } } } @@ -216,7 +216,7 @@ FullProducerArbitrary::onSyncInterest(const ndn::Name& prefixName, const ndn::In } auto& entry = m_pendingEntries.emplace(interestName, PendingEntryInfoFull{iblt, {}}).first->second; - entry.expirationEvent = m_scheduler.scheduleEvent(interest.getInterestLifetime(), + entry.expirationEvent = m_scheduler.schedule(interest.getInterestLifetime(), [this, interest] { NDN_LOG_TRACE("Erase Pending Interest " << interest.getNonce()); m_pendingEntries.erase(interest.getName()); diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp index 4843295..0956dc4 100644 --- a/PSync/full-producer-arbitrary.hpp +++ b/PSync/full-producer-arbitrary.hpp @@ -41,7 +41,7 @@ namespace psync { struct PendingEntryInfoFull { IBLT iblt; - ndn::util::scheduler::ScopedEventId expirationEvent; + ndn::scheduler::ScopedEventId expirationEvent; }; typedef std::function&)> ArbitraryUpdateCallback; @@ -116,7 +116,7 @@ class FullProducerArbitrary : public ProducerBase void sendSyncInterest(); -PUBLIC_WITH_TESTS_ELSE_PRIVATE: +PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: /** * @brief Process sync interest from other parties * @@ -177,7 +177,7 @@ class FullProducerArbitrary : public ProducerBase void satisfyPendingInterests(); -PUBLIC_WITH_TESTS_ELSE_PRIVATE: +PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: /** * @brief Delete pending sync interests that match given name */ @@ -189,7 +189,7 @@ class FullProducerArbitrary : public ProducerBase ndn::KeyChain m_keyChain; ndn::Scheduler m_scheduler; -PUBLIC_WITH_TESTS_ELSE_PROTECTED: +PSYNC_PUBLIC_WITH_TESTS_ELSE_PROTECTED: SegmentPublisher m_segmentPublisher; std::map m_pendingEntries; @@ -197,7 +197,7 @@ class FullProducerArbitrary : public ProducerBase ArbitraryUpdateCallback m_onArbitraryUpdateCallback; ShouldAddToSyncDataCallback m_onShouldAddToSyncDataCallback; CanAddName m_onCanAddName; - ndn::util::scheduler::ScopedEventId m_scheduledSyncInterestId; + ndn::scheduler::ScopedEventId m_scheduledSyncInterestId; std::uniform_int_distribution<> m_jitter; ndn::Name m_outstandingInterestName; ndn::ScopedRegisteredPrefixHandle m_registeredPrefix; diff --git a/PSync/full-producer.cpp b/PSync/full-producer.cpp index 22f852a..6c5a626 100644 --- a/PSync/full-producer.cpp +++ b/PSync/full-producer.cpp @@ -50,9 +50,9 @@ FullProducer::FullProducer(const size_t expectedNumEntries, ndn::Name prefix = name.getPrefix(-1); uint64_t seq = name.get(-1).toNumber(); - if (m_prefixes.m_prefixes.find(prefix) == m_prefixes.m_prefixes.end() || - m_prefixes.m_prefixes[prefix] < seq) { - uint64_t oldSeq = m_prefixes.m_prefixes[prefix]; + if (m_prefixes.prefixes.find(prefix) == m_prefixes.prefixes.end() || + m_prefixes.prefixes[prefix] < seq) { + uint64_t oldSeq = m_prefixes.prefixes[prefix]; if (oldSeq != 0) { m_producerArbitrary.removeName(ndn::Name(prefix).appendNumber(oldSeq)); } @@ -72,7 +72,7 @@ FullProducer::publishName(const ndn::Name& prefix, ndn::optional seq) return; } - uint64_t newSeq = seq.value_or(m_prefixes.m_prefixes[prefix] + 1); + uint64_t newSeq = seq.value_or(m_prefixes.prefixes[prefix] + 1); NDN_LOG_INFO("Publish: " << prefix << "/" << newSeq); @@ -99,7 +99,7 @@ bool FullProducer::isNotFutureHash(const ndn::Name& prefix, const std::set& negative) { uint32_t nextHash = murmurHash3(N_HASHCHECK, - ndn::Name(prefix).appendNumber(m_prefixes.m_prefixes[prefix] + 1).toUri()); + ndn::Name(prefix).appendNumber(m_prefixes.prefixes[prefix] + 1).toUri()); for (const auto& nHash : negative) { if (nHash == nextHash) { return false; @@ -120,8 +120,8 @@ FullProducer::arbitraryUpdateCallBack(const std::vector& names) NDN_LOG_INFO("Updates: " << prefix << " " << seq); - updates.push_back(MissingDataInfo{prefix, m_prefixes.m_prefixes[prefix] + 1, seq}); - m_prefixes.m_prefixes[prefix] = seq; + updates.push_back(MissingDataInfo{prefix, m_prefixes.prefixes[prefix] + 1, seq}); + m_prefixes.prefixes[prefix] = seq; } m_onUpdateCallback(updates); diff --git a/PSync/full-producer.hpp b/PSync/full-producer.hpp index 7f1a924..6bc56b9 100644 --- a/PSync/full-producer.hpp +++ b/PSync/full-producer.hpp @@ -112,7 +112,7 @@ class FullProducer removeUserNode(const ndn::Name& prefix) { if (m_prefixes.isUserNode(prefix)) { - uint64_t seqNo = m_prefixes.m_prefixes[prefix]; + uint64_t seqNo = m_prefixes.prefixes[prefix]; m_prefixes.removeUserNode(prefix); m_producerArbitrary.removeName(ndn::Name(prefix).appendNumber(seqNo)); } @@ -142,7 +142,7 @@ class FullProducer bool isNotFutureHash(const ndn::Name& prefix, const std::set& negative); -PUBLIC_WITH_TESTS_ELSE_PROTECTED: +PSYNC_PUBLIC_WITH_TESTS_ELSE_PROTECTED: void updateSeqNo(const ndn::Name& prefix, uint64_t seq); @@ -150,7 +150,7 @@ class FullProducer void arbitraryUpdateCallBack(const std::vector& names); -PUBLIC_WITH_TESTS_ELSE_PROTECTED: +PSYNC_PUBLIC_WITH_TESTS_ELSE_PROTECTED: FullProducerArbitrary m_producerArbitrary; UpdateCallback m_onUpdateCallback; diff --git a/PSync/partial-producer.cpp b/PSync/partial-producer.cpp index eb7c2cc..5c544d0 100644 --- a/PSync/partial-producer.cpp +++ b/PSync/partial-producer.cpp @@ -60,7 +60,7 @@ PartialProducer::publishName(const ndn::Name& prefix, ndn::optional se return; } - uint64_t newSeq = seq.value_or(m_prefixes.m_prefixes[prefix] + 1); + uint64_t newSeq = seq.value_or(m_prefixes.prefixes[prefix] + 1); NDN_LOG_INFO("Publish: " << prefix << "/" << newSeq); @@ -102,7 +102,7 @@ PartialProducer::onHelloInterest(const ndn::Name& prefix, const ndn::Interest& i State state; - for (const auto& prefix : m_prefixes.m_prefixes) { + for (const auto& prefix : m_prefixes.prefixes) { state.addContent(ndn::Name(prefix.first).appendNumber(prefix.second)); } NDN_LOG_DEBUG("sending content p: " << state); @@ -174,7 +174,7 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in std::set positive; std::set negative; - NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.m_prefixes.size()); + NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.prefixes.size()); bool peel = diff.listEntries(positive, negative); @@ -197,7 +197,7 @@ PartialProducer::onSyncInterest(const ndn::Name& prefix, const ndn::Interest& in if (bf.contains(prefix.toUri())) { // generate data state.addContent(name); - NDN_LOG_DEBUG("Content: " << prefix << " " << std::to_string(m_prefixes.m_prefixes[prefix])); + NDN_LOG_DEBUG("Content: " << prefix << " " << std::to_string(m_prefixes.prefixes[prefix])); } } @@ -237,7 +237,7 @@ PartialProducer::satisfyPendingSyncInterests(const ndn::Name& prefix) { NDN_LOG_TRACE("Result of listEntries on the difference: " << peel); - NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.m_prefixes.size()); + NDN_LOG_TRACE("Number elements in IBF: " << m_prefixes.prefixes.size()); NDN_LOG_TRACE("m_threshold: " << m_threshold << " Total: " << positive.size() + negative.size()); if (!peel) { @@ -249,8 +249,8 @@ PartialProducer::satisfyPendingSyncInterests(const ndn::Name& prefix) { State state; if (entry.bf.contains(prefix.toUri()) || positive.size() + negative.size() >= m_threshold) { if (entry.bf.contains(prefix.toUri())) { - state.addContent(ndn::Name(prefix).appendNumber(m_prefixes.m_prefixes[prefix])); - NDN_LOG_DEBUG("sending sync content " << prefix << " " << std::to_string(m_prefixes.m_prefixes[prefix])); + state.addContent(ndn::Name(prefix).appendNumber(m_prefixes.prefixes[prefix])); + NDN_LOG_DEBUG("sending sync content " << prefix << " " << std::to_string(m_prefixes.prefixes[prefix])); } else { NDN_LOG_DEBUG("Sending with empty content to send latest IBF to consumer"); diff --git a/PSync/partial-producer.hpp b/PSync/partial-producer.hpp index 4076495..e7ff487 100644 --- a/PSync/partial-producer.hpp +++ b/PSync/partial-producer.hpp @@ -113,7 +113,7 @@ class PartialProducer : public ProducerBase removeUserNode(const ndn::Name& prefix) { if (m_prefixes.isUserNode(prefix)) { - uint64_t seqNo = m_prefixes.m_prefixes[prefix]; + uint64_t seqNo = m_prefixes.prefixes[prefix]; m_prefixes.removeUserNode(prefix); removeFromIBF(ndn::Name(prefix).appendNumber(seqNo)); } diff --git a/tests/bk-test-bloom-filter.cpp.disable b/tests/bk-test-bloom-filter.cpp.disable new file mode 100644 index 0000000..eec8a7c --- /dev/null +++ b/tests/bk-test-bloom-filter.cpp.disable @@ -0,0 +1,100 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2014-2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#include "PSync/detail/bloom-filter.hpp" + +#include +#include + +#include + +#include +#include +#include +#include + +namespace psync { + +using namespace ndn; +namespace bio = boost::iostreams; + +BOOST_AUTO_TEST_SUITE(TestBloomFilter) + +BOOST_AUTO_TEST_CASE(Basic) +{ + BloomFilter bf(100, 0.001); + + std::string insertName("/memphis"); + bf.insert(insertName); + BOOST_CHECK(bf.contains(insertName)); +} + +BOOST_AUTO_TEST_CASE(NameAppendAndExtract) +{ + Name bfName("/test"); + BloomFilter bf(100, 0.001); + bf.insert("/memphis"); + + bf.appendToName(bfName); + + BloomFilter bfFromName(100, 0.001, bfName.get(-1)); + + BOOST_CHECK_EQUAL(bfName.get(1).toNumber(), 100); + BOOST_CHECK_EQUAL(bfName.get(2).toNumber(), 1); + BOOST_CHECK_EQUAL(bf, bfFromName); + + BOOST_CHECK_THROW(BloomFilter inCompatibleBf(200, 0.001, bfName.get(-1)), std::runtime_error); +} + +BOOST_AUTO_TEST_CASE(SizeTest) +{ + BloomFilter bf(1000, 0.01); + Name bfName("/test"); + + for(int i = 0; i < 1000; ++i) { + bf.insert("/memphis" + std::to_string(i)); + } + bf.appendToName(bfName); + std::cout << bfName << std::endl; + + std::cout << "Size of name: " << bfName.toUri() << std::endl; + std::cout << "Size of name: " << bfName.toUri().size() << std::endl; + // std::cout << "Size of name: " << bfName.wireEncode().size() << std::endl; + + std::vector table; + for (const auto i : bf.table()) { + table.push_back(i); + } + bio::filtering_streambuf in; + in.push(bio::zlib_compressor()); + in.push(bio::array_source(table.data(), table.size())); + + std::stringstream sstream; + bio::copy(in, sstream); + + Name bfName2("/test"); + std::string compressedIBF = sstream.str(); + bfName2.append(compressedIBF.begin(), compressedIBF.end()); + std::cout << "Size of compressed name: " << bfName2 << std::endl; + std::cout << "Size of compressed name: " << bfName2.toUri().size() << std::endl; +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace psync diff --git a/tests/bk-test-iblt.cpp.disable b/tests/bk-test-iblt.cpp.disable new file mode 100644 index 0000000..c377978 --- /dev/null +++ b/tests/bk-test-iblt.cpp.disable @@ -0,0 +1,247 @@ +/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ +/* + * Copyright (c) 2014-2019, The University of Memphis + * + * This file is part of PSync. + * See AUTHORS.md for complete list of PSync authors and contributors. + * + * PSync is free software: you can redistribute it and/or modify it under the terms + * of the GNU Lesser General Public License as published by the Free Software Foundation, + * either version 3 of the License, or (at your option) any later version. + * + * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; + * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR + * PURPOSE. See the GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License along with + * PSync, e.g., in COPYING.md file. If not, see . + **/ + +#include "PSync/detail/iblt.hpp" +#include "PSync/detail/util.hpp" + +#include +#include +#include + +#include + +namespace psync { + +using namespace ndn; + +BOOST_AUTO_TEST_SUITE(TestIBLT) + +BOOST_AUTO_TEST_CASE(Equal) +{ + int size = 10; + + IBLT iblt1(size); + IBLT iblt2(size); + BOOST_CHECK_EQUAL(iblt1, iblt2); + + std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + uint32_t newHash = murmurHash3(11, prefix); + iblt1.insert(newHash); + iblt2.insert(newHash); + + BOOST_CHECK_EQUAL(iblt1, iblt2); + + Name ibfName1("sync"), ibfName2("sync"); + iblt1.appendToName(ibfName1); + iblt2.appendToName(ibfName2); + BOOST_CHECK_EQUAL(ibfName1, ibfName2); +} + +BOOST_AUTO_TEST_CASE(NameAppendAndExtract) +{ + int size = 10; + + IBLT iblt(size); + std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + uint32_t newHash = murmurHash3(11, prefix); + iblt.insert(newHash); + + Name ibltName("sync"); + iblt.appendToName(ibltName); + + IBLT rcvd(size); + rcvd.initialize(ibltName.get(-1)); + + BOOST_CHECK_EQUAL(iblt, rcvd); + + IBLT rcvdDiffSize(20); + BOOST_CHECK_THROW(rcvdDiffSize.initialize(ibltName.get(-1)), std::runtime_error); +} + +BOOST_AUTO_TEST_CASE(CopyInsertErase) +{ + int size = 10; + + IBLT iblt1(size); + + std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + uint32_t hash1 = murmurHash3(11, prefix); + iblt1.insert(hash1); + + IBLT iblt2(iblt1); + iblt2.erase(hash1); + prefix = Name("/test/memphis").appendNumber(2).toUri(); + uint32_t hash3 = murmurHash3(11, prefix); + iblt2.insert(hash3); + + iblt1.erase(hash1); + prefix = Name("/test/memphis").appendNumber(5).toUri(); + uint32_t hash5 = murmurHash3(11, prefix); + iblt1.insert(hash5); + + iblt2.erase(hash3); + iblt2.insert(hash5); + + BOOST_CHECK_EQUAL(iblt1, iblt2); +} + +BOOST_AUTO_TEST_CASE(HigherSeqTest) +{ + // The case where we can't recognize if the rcvd IBF has higher sequence number + // Relevant to full sync case + int size = 10; + + IBLT ownIBF(size); + IBLT rcvdIBF(size); + + std::string prefix = Name("/test/memphis").appendNumber(3).toUri(); + uint32_t hash1 = murmurHash3(11, prefix); + ownIBF.insert(hash1); + + std::string prefix2 = Name("/test/memphis").appendNumber(4).toUri(); + uint32_t hash2 = murmurHash3(11, prefix2); + rcvdIBF.insert(hash2); + + IBLT diff = ownIBF - rcvdIBF; + std::set positive; + std::set negative; + + BOOST_CHECK(diff.listEntries(positive, negative)); + BOOST_CHECK(*positive.begin() == hash1); + BOOST_CHECK(*negative.begin() == hash2); +} + +BOOST_AUTO_TEST_CASE(Difference) +{ + int size = 10; + + IBLT ownIBF(size); + + IBLT rcvdIBF = ownIBF; + + IBLT diff = ownIBF - rcvdIBF; + + std::set positive; // non-empty Positive means we have some elements that the others don't + std::set negative; + + BOOST_CHECK(diff.listEntries(positive, negative)); + BOOST_CHECK_EQUAL(positive.size(), 0); + BOOST_CHECK_EQUAL(negative.size(), 0); + + std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); + uint32_t newHash = murmurHash3(11, prefix); + ownIBF.insert(newHash); + + diff = ownIBF - rcvdIBF; + BOOST_CHECK(diff.listEntries(positive, negative)); + BOOST_CHECK_EQUAL(positive.size(), 1); + BOOST_CHECK_EQUAL(negative.size(), 0); + + prefix = Name("/test/csu").appendNumber(1).toUri(); + newHash = murmurHash3(11, prefix); + rcvdIBF.insert(newHash); + + diff = ownIBF - rcvdIBF; + BOOST_CHECK(diff.listEntries(positive, negative)); + BOOST_CHECK_EQUAL(positive.size(), 1); + BOOST_CHECK_EQUAL(negative.size(), 1); +} + +BOOST_AUTO_TEST_CASE(DifferenceBwOversizedIBFs) +{ + // Insert 50 elements into IBF of size 10 + // Check that we can still list the difference + // even though we can't list the IBFs itself + + int size = 10; + + IBLT ownIBF(size); + + for (int i = 0; i < 50; i++) { + std::string prefix = Name("/test/memphis" + std::to_string(i)).appendNumber(1).toUri(); + uint32_t newHash = murmurHash3(11, prefix); + ownIBF.insert(newHash); + } + + IBLT rcvdIBF = ownIBF; + + std::string prefix = Name("/test/ucla").appendNumber(1).toUri(); + uint32_t newHash = murmurHash3(11, prefix); + ownIBF.insert(newHash); + + IBLT diff = ownIBF - rcvdIBF; + + std::set positive; + std::set negative; + BOOST_CHECK(diff.listEntries(positive, negative)); + BOOST_CHECK_EQUAL(positive.size(), 1); + BOOST_CHECK_EQUAL(*positive.begin(), newHash); + BOOST_CHECK_EQUAL(negative.size(), 0); + + BOOST_CHECK(!ownIBF.listEntries(positive, negative)); + BOOST_CHECK(!rcvdIBF.listEntries(positive, negative)); +} + + +BOOST_AUTO_TEST_CASE(FindSameElements) +{ + int size = 10; + + IBLT ownIBF(size); + std::cout << "A: " << murmurHash3(11, std::string("A")) << std::endl; + std::cout << "B: " << murmurHash3(11, std::string("B")) << std::endl; + std::cout << "C: " << murmurHash3(11, std::string("C")) << std::endl; + ownIBF.insert(murmurHash3(11, std::string("A"))); + ownIBF.insert(murmurHash3(11, std::string("B"))); + ownIBF.insert(murmurHash3(11, std::string("C"))); + + IBLT rcvdIBF(size); + rcvdIBF.insert(murmurHash3(11, std::string("C"))); + rcvdIBF.insert(murmurHash3(11, std::string("D"))); + rcvdIBF.insert(murmurHash3(11, std::string("E"))); + std::cout << "D: " << murmurHash3(11, std::string("D")) << std::endl; + std::cout << "E: " << murmurHash3(11, std::string("E")) << std::endl; + + IBLT diff = rcvdIBF - (ownIBF - rcvdIBF); + + std::set positive; + std::set negative; + + diff.listEntries(positive, negative); + + for (const auto& p : positive) { + std::cout << " positive: " << p << std::endl; + } + + for (const auto& n : negative) { + std::cout << " negative: " << n << std::endl; + } + + // IBLT diff2 = ownIBF - ; +} + +BOOST_AUTO_TEST_CASE(IBFSize) +{ + int size = 10; + +} + +BOOST_AUTO_TEST_SUITE_END() + +} // namespace psync diff --git a/tests/test-partial-sync.cpp b/tests/test-partial-sync.cpp index abd28a6..76618ef 100644 --- a/tests/test-partial-sync.cpp +++ b/tests/test-partial-sync.cpp @@ -83,7 +83,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture for (const auto& update : updates) { BOOST_CHECK(consumers[id]->isSubscribed(update.prefix)); BOOST_CHECK_EQUAL(oldSeqMap.at(update.prefix) + 1, update.lowSeq); - BOOST_CHECK_EQUAL(producer->m_prefixes.m_prefixes.at(update.prefix), update.highSeq); + BOOST_CHECK_EQUAL(producer->m_prefixes.prefixes.at(update.prefix), update.highSeq); BOOST_CHECK_EQUAL(consumers[id]->getSeqNo(update.prefix).value(), update.highSeq); } }, 40, 0.001); @@ -102,7 +102,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture bool checkSubList(const vector& availableSubs) { - for (const auto& prefix : producer->m_prefixes.m_prefixes ) { + for (const auto& prefix : producer->m_prefixes.prefixes ) { if (std::find(availableSubs.begin(), availableSubs.end(), prefix.first) == availableSubs.end()) { return false; } @@ -122,7 +122,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture void publishUpdateFor(const std::string& prefix) { - oldSeqMap = producer->m_prefixes.m_prefixes; + oldSeqMap = producer->m_prefixes.prefixes; producer->publishName(prefix); advanceClocks(ndn::time::milliseconds(10)); } @@ -130,7 +130,7 @@ class PartialSyncFixture : public tests::UnitTestTimeFixture void updateSeqFor(const std::string& prefix, uint64_t seq) { - oldSeqMap = producer->m_prefixes.m_prefixes; + oldSeqMap = producer->m_prefixes.prefixes; producer->updateSeqNo(prefix, seq); } @@ -326,7 +326,7 @@ BOOST_AUTO_TEST_CASE(ApplicationNack) publishUpdateFor("testUser-2"); BOOST_CHECK_EQUAL(numSyncDataRcvd, 1); - oldSeqMap = producer->m_prefixes.m_prefixes; + oldSeqMap = producer->m_prefixes.prefixes; for (int i = 0; i < 50; i++) { ndn::Name prefix("testUser-" + to_string(i)); producer->updateSeqNo(prefix, producer->getSeqNo(prefix).value() + 1); @@ -402,7 +402,7 @@ BOOST_AUTO_TEST_CASE(SegmentedSync) syncInterestName.appendVersion(); syncInterestName.appendSegment(1); - oldSeqMap = producer->m_prefixes.m_prefixes; + oldSeqMap = producer->m_prefixes.prefixes; for (int i = 1; i < 10; i++) { producer->updateSeqNo(longNameToExceedDataSize.toUri() + "-" + to_string(i), 1); } @@ -430,4 +430,4 @@ BOOST_AUTO_TEST_CASE(SegmentedSync) BOOST_AUTO_TEST_SUITE_END() -} // namespace psync \ No newline at end of file +} // namespace psync From 5b5074ad80fc37823536fc127ad3446df36d5a7b Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 13 Jul 2019 11:18:47 -0700 Subject: [PATCH 18/19] PSync: fix doxygen Change-Id: Ibde090c3fc7cbc72b536cfe08660cd667f600309 --- AUTHORS.md | 1 + PSync/detail/user-prefixes.cpp | 2 +- PSync/detail/user-prefixes.hpp | 2 +- PSync/full-producer-arbitrary.hpp | 4 +++- PSync/full-producer.hpp | 2 +- PSync/producer-base.hpp | 6 +----- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/AUTHORS.md b/AUTHORS.md index c143365..027f705 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -12,6 +12,7 @@ The University of Memphis: * Junxiao Shi * Davide Pesavento +* Jeff Thompson ## Technical advisor(s): diff --git a/PSync/detail/user-prefixes.cpp b/PSync/detail/user-prefixes.cpp index db4855b..c6cb8d7 100644 --- a/PSync/detail/user-prefixes.cpp +++ b/PSync/detail/user-prefixes.cpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2019, The University of Memphis + * Copyright (c) 2014-2019, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. diff --git a/PSync/detail/user-prefixes.hpp b/PSync/detail/user-prefixes.hpp index 9ab8596..d760519 100644 --- a/PSync/detail/user-prefixes.hpp +++ b/PSync/detail/user-prefixes.hpp @@ -1,6 +1,6 @@ /* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ /* - * Copyright (c) 2019, The University of Memphis + * Copyright (c) 2014-2019, The University of Memphis * * This file is part of PSync. * See AUTHORS.md for complete list of PSync authors and contributors. diff --git a/PSync/full-producer-arbitrary.hpp b/PSync/full-producer-arbitrary.hpp index 0956dc4..91ebc3e 100644 --- a/PSync/full-producer-arbitrary.hpp +++ b/PSync/full-producer-arbitrary.hpp @@ -71,9 +71,11 @@ class FullProducerArbitrary : public ProducerBase * @param expectedNumEntries expected entries in IBF * @param face application's face * @param syncPrefix The prefix of the sync group - * @param onUpdateCallBack The call back to be called when there is new data + * @param onArbitraryUpdateCallback The call back to be called when there is new data * @param syncInterestLifetime lifetime of the sync interest * @param syncReplyFreshness freshness of sync data + * @param onShouldAddToSyncDataCallback whether to add sync data to content being sent (FullProducer future hash) + * @param onCanAddName whether to add name to IBF */ FullProducerArbitrary(size_t expectedNumEntries, ndn::Face& face, diff --git a/PSync/full-producer.hpp b/PSync/full-producer.hpp index 6bc56b9..6e6f267 100644 --- a/PSync/full-producer.hpp +++ b/PSync/full-producer.hpp @@ -60,7 +60,7 @@ class FullProducer * @param face application's face * @param syncPrefix The prefix of the sync group * @param userPrefix The prefix of the first user in the group - * @param onUpdateCallBack The call back to be called when there is new data + * @param onUpdateCallback The call back to be called when there is new data * @param syncInterestLifetime lifetime of the sync interest * @param syncReplyFreshness freshness of sync data */ diff --git a/PSync/producer-base.hpp b/PSync/producer-base.hpp index 39fa236..b0a2591 100644 --- a/PSync/producer-base.hpp +++ b/PSync/producer-base.hpp @@ -60,11 +60,8 @@ class ProducerBase * @brief constructor * * @param expectedNumEntries expected number entries in IBF - * @param face application's face * @param syncPrefix The prefix of the sync group - * @param userPrefix The prefix of the first user in the group * @param syncReplyFreshness freshness of sync data - * @param helloReplyFreshness freshness of hello data */ ProducerBase(size_t expectedNumEntries, const ndn::Name& syncPrefix, @@ -79,8 +76,7 @@ class ProducerBase * (unless seq is zero because we don't insert zero seq into IBF) * Then we update m_prefix, m_prefix2hash, m_hash2prefix, and IBF * - * @param prefix prefix of the update - * @param seq sequence number of the update + * @param name prefix of the update */ void insertToIBF(const ndn::Name& name); From e1ec19f353e8f8bb52b14a33850df07de9e9617a Mon Sep 17 00:00:00 2001 From: Ashlesh Gawande Date: Sat, 13 Jul 2019 15:06:41 -0700 Subject: [PATCH 19/19] full-producer: fix function visibility examples: fix compilation Change-Id: I0293d3a661353b49d8f006b1d48259dbf923c785 --- PSync/full-producer.hpp | 1 - examples/full-sync-arbitrary.cpp | 18 +- tests/bk-test-bloom-filter.cpp.disable | 100 ---------- tests/bk-test-iblt.cpp.disable | 247 ------------------------- 4 files changed, 9 insertions(+), 357 deletions(-) delete mode 100644 tests/bk-test-bloom-filter.cpp.disable delete mode 100644 tests/bk-test-iblt.cpp.disable diff --git a/PSync/full-producer.hpp b/PSync/full-producer.hpp index 6e6f267..6723c4e 100644 --- a/PSync/full-producer.hpp +++ b/PSync/full-producer.hpp @@ -83,7 +83,6 @@ class FullProducer return m_prefixes.getSeqNo(prefix); } -PSYNC_PUBLIC_WITH_TESTS_ELSE_PRIVATE: /** * @brief Adds a user node for synchronization * diff --git a/examples/full-sync-arbitrary.cpp b/examples/full-sync-arbitrary.cpp index 768e3c8..275a37d 100644 --- a/examples/full-sync-arbitrary.cpp +++ b/examples/full-sync-arbitrary.cpp @@ -52,10 +52,10 @@ class Producer ndn::Name dataPrefix(userPrefix + "-" + ndn::to_string(i)); m_nPublished[dataPrefix] = 0; - m_scheduler.scheduleEvent(ndn::time::milliseconds(m_rangeUniformRandom(m_rng)), - [this, dataPrefix] { - doUpdate(dataPrefix); - }); + m_scheduler.schedule(ndn::time::milliseconds(m_rangeUniformRandom(m_rng)), + [this, dataPrefix] { + doUpdate(dataPrefix); + }); } } @@ -77,10 +77,10 @@ class Producer ++m_nPublished[dataPrefix]; if (m_nPublished[dataPrefix] < m_maxNumPublish) { - m_scheduler.scheduleEvent(ndn::time::milliseconds(m_rangeUniformRandom(m_rng)), - [this, dataPrefix] { - doUpdate(dataPrefix); - }); + m_scheduler.schedule(ndn::time::milliseconds(m_rangeUniformRandom(m_rng)), + [this, dataPrefix] { + doUpdate(dataPrefix); + }); } } @@ -94,7 +94,7 @@ class Producer private: ndn::Face m_face; - ndn::util::Scheduler m_scheduler; + ndn::Scheduler m_scheduler; psync::FullProducerArbitrary m_fullProducer; diff --git a/tests/bk-test-bloom-filter.cpp.disable b/tests/bk-test-bloom-filter.cpp.disable deleted file mode 100644 index eec8a7c..0000000 --- a/tests/bk-test-bloom-filter.cpp.disable +++ /dev/null @@ -1,100 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -/* - * Copyright (c) 2014-2019, The University of Memphis - * - * This file is part of PSync. - * See AUTHORS.md for complete list of PSync authors and contributors. - * - * PSync is free software: you can redistribute it and/or modify it under the terms - * of the GNU Lesser General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License along with - * PSync, e.g., in COPYING.md file. If not, see . - **/ - -#include "PSync/detail/bloom-filter.hpp" - -#include -#include - -#include - -#include -#include -#include -#include - -namespace psync { - -using namespace ndn; -namespace bio = boost::iostreams; - -BOOST_AUTO_TEST_SUITE(TestBloomFilter) - -BOOST_AUTO_TEST_CASE(Basic) -{ - BloomFilter bf(100, 0.001); - - std::string insertName("/memphis"); - bf.insert(insertName); - BOOST_CHECK(bf.contains(insertName)); -} - -BOOST_AUTO_TEST_CASE(NameAppendAndExtract) -{ - Name bfName("/test"); - BloomFilter bf(100, 0.001); - bf.insert("/memphis"); - - bf.appendToName(bfName); - - BloomFilter bfFromName(100, 0.001, bfName.get(-1)); - - BOOST_CHECK_EQUAL(bfName.get(1).toNumber(), 100); - BOOST_CHECK_EQUAL(bfName.get(2).toNumber(), 1); - BOOST_CHECK_EQUAL(bf, bfFromName); - - BOOST_CHECK_THROW(BloomFilter inCompatibleBf(200, 0.001, bfName.get(-1)), std::runtime_error); -} - -BOOST_AUTO_TEST_CASE(SizeTest) -{ - BloomFilter bf(1000, 0.01); - Name bfName("/test"); - - for(int i = 0; i < 1000; ++i) { - bf.insert("/memphis" + std::to_string(i)); - } - bf.appendToName(bfName); - std::cout << bfName << std::endl; - - std::cout << "Size of name: " << bfName.toUri() << std::endl; - std::cout << "Size of name: " << bfName.toUri().size() << std::endl; - // std::cout << "Size of name: " << bfName.wireEncode().size() << std::endl; - - std::vector table; - for (const auto i : bf.table()) { - table.push_back(i); - } - bio::filtering_streambuf in; - in.push(bio::zlib_compressor()); - in.push(bio::array_source(table.data(), table.size())); - - std::stringstream sstream; - bio::copy(in, sstream); - - Name bfName2("/test"); - std::string compressedIBF = sstream.str(); - bfName2.append(compressedIBF.begin(), compressedIBF.end()); - std::cout << "Size of compressed name: " << bfName2 << std::endl; - std::cout << "Size of compressed name: " << bfName2.toUri().size() << std::endl; -} - -BOOST_AUTO_TEST_SUITE_END() - -} // namespace psync diff --git a/tests/bk-test-iblt.cpp.disable b/tests/bk-test-iblt.cpp.disable deleted file mode 100644 index c377978..0000000 --- a/tests/bk-test-iblt.cpp.disable +++ /dev/null @@ -1,247 +0,0 @@ -/* -*- Mode:C++; c-file-style:"gnu"; indent-tabs-mode:nil; -*- */ -/* - * Copyright (c) 2014-2019, The University of Memphis - * - * This file is part of PSync. - * See AUTHORS.md for complete list of PSync authors and contributors. - * - * PSync is free software: you can redistribute it and/or modify it under the terms - * of the GNU Lesser General Public License as published by the Free Software Foundation, - * either version 3 of the License, or (at your option) any later version. - * - * PSync is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; - * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - * PURPOSE. See the GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License along with - * PSync, e.g., in COPYING.md file. If not, see . - **/ - -#include "PSync/detail/iblt.hpp" -#include "PSync/detail/util.hpp" - -#include -#include -#include - -#include - -namespace psync { - -using namespace ndn; - -BOOST_AUTO_TEST_SUITE(TestIBLT) - -BOOST_AUTO_TEST_CASE(Equal) -{ - int size = 10; - - IBLT iblt1(size); - IBLT iblt2(size); - BOOST_CHECK_EQUAL(iblt1, iblt2); - - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); - uint32_t newHash = murmurHash3(11, prefix); - iblt1.insert(newHash); - iblt2.insert(newHash); - - BOOST_CHECK_EQUAL(iblt1, iblt2); - - Name ibfName1("sync"), ibfName2("sync"); - iblt1.appendToName(ibfName1); - iblt2.appendToName(ibfName2); - BOOST_CHECK_EQUAL(ibfName1, ibfName2); -} - -BOOST_AUTO_TEST_CASE(NameAppendAndExtract) -{ - int size = 10; - - IBLT iblt(size); - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); - uint32_t newHash = murmurHash3(11, prefix); - iblt.insert(newHash); - - Name ibltName("sync"); - iblt.appendToName(ibltName); - - IBLT rcvd(size); - rcvd.initialize(ibltName.get(-1)); - - BOOST_CHECK_EQUAL(iblt, rcvd); - - IBLT rcvdDiffSize(20); - BOOST_CHECK_THROW(rcvdDiffSize.initialize(ibltName.get(-1)), std::runtime_error); -} - -BOOST_AUTO_TEST_CASE(CopyInsertErase) -{ - int size = 10; - - IBLT iblt1(size); - - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); - uint32_t hash1 = murmurHash3(11, prefix); - iblt1.insert(hash1); - - IBLT iblt2(iblt1); - iblt2.erase(hash1); - prefix = Name("/test/memphis").appendNumber(2).toUri(); - uint32_t hash3 = murmurHash3(11, prefix); - iblt2.insert(hash3); - - iblt1.erase(hash1); - prefix = Name("/test/memphis").appendNumber(5).toUri(); - uint32_t hash5 = murmurHash3(11, prefix); - iblt1.insert(hash5); - - iblt2.erase(hash3); - iblt2.insert(hash5); - - BOOST_CHECK_EQUAL(iblt1, iblt2); -} - -BOOST_AUTO_TEST_CASE(HigherSeqTest) -{ - // The case where we can't recognize if the rcvd IBF has higher sequence number - // Relevant to full sync case - int size = 10; - - IBLT ownIBF(size); - IBLT rcvdIBF(size); - - std::string prefix = Name("/test/memphis").appendNumber(3).toUri(); - uint32_t hash1 = murmurHash3(11, prefix); - ownIBF.insert(hash1); - - std::string prefix2 = Name("/test/memphis").appendNumber(4).toUri(); - uint32_t hash2 = murmurHash3(11, prefix2); - rcvdIBF.insert(hash2); - - IBLT diff = ownIBF - rcvdIBF; - std::set positive; - std::set negative; - - BOOST_CHECK(diff.listEntries(positive, negative)); - BOOST_CHECK(*positive.begin() == hash1); - BOOST_CHECK(*negative.begin() == hash2); -} - -BOOST_AUTO_TEST_CASE(Difference) -{ - int size = 10; - - IBLT ownIBF(size); - - IBLT rcvdIBF = ownIBF; - - IBLT diff = ownIBF - rcvdIBF; - - std::set positive; // non-empty Positive means we have some elements that the others don't - std::set negative; - - BOOST_CHECK(diff.listEntries(positive, negative)); - BOOST_CHECK_EQUAL(positive.size(), 0); - BOOST_CHECK_EQUAL(negative.size(), 0); - - std::string prefix = Name("/test/memphis").appendNumber(1).toUri(); - uint32_t newHash = murmurHash3(11, prefix); - ownIBF.insert(newHash); - - diff = ownIBF - rcvdIBF; - BOOST_CHECK(diff.listEntries(positive, negative)); - BOOST_CHECK_EQUAL(positive.size(), 1); - BOOST_CHECK_EQUAL(negative.size(), 0); - - prefix = Name("/test/csu").appendNumber(1).toUri(); - newHash = murmurHash3(11, prefix); - rcvdIBF.insert(newHash); - - diff = ownIBF - rcvdIBF; - BOOST_CHECK(diff.listEntries(positive, negative)); - BOOST_CHECK_EQUAL(positive.size(), 1); - BOOST_CHECK_EQUAL(negative.size(), 1); -} - -BOOST_AUTO_TEST_CASE(DifferenceBwOversizedIBFs) -{ - // Insert 50 elements into IBF of size 10 - // Check that we can still list the difference - // even though we can't list the IBFs itself - - int size = 10; - - IBLT ownIBF(size); - - for (int i = 0; i < 50; i++) { - std::string prefix = Name("/test/memphis" + std::to_string(i)).appendNumber(1).toUri(); - uint32_t newHash = murmurHash3(11, prefix); - ownIBF.insert(newHash); - } - - IBLT rcvdIBF = ownIBF; - - std::string prefix = Name("/test/ucla").appendNumber(1).toUri(); - uint32_t newHash = murmurHash3(11, prefix); - ownIBF.insert(newHash); - - IBLT diff = ownIBF - rcvdIBF; - - std::set positive; - std::set negative; - BOOST_CHECK(diff.listEntries(positive, negative)); - BOOST_CHECK_EQUAL(positive.size(), 1); - BOOST_CHECK_EQUAL(*positive.begin(), newHash); - BOOST_CHECK_EQUAL(negative.size(), 0); - - BOOST_CHECK(!ownIBF.listEntries(positive, negative)); - BOOST_CHECK(!rcvdIBF.listEntries(positive, negative)); -} - - -BOOST_AUTO_TEST_CASE(FindSameElements) -{ - int size = 10; - - IBLT ownIBF(size); - std::cout << "A: " << murmurHash3(11, std::string("A")) << std::endl; - std::cout << "B: " << murmurHash3(11, std::string("B")) << std::endl; - std::cout << "C: " << murmurHash3(11, std::string("C")) << std::endl; - ownIBF.insert(murmurHash3(11, std::string("A"))); - ownIBF.insert(murmurHash3(11, std::string("B"))); - ownIBF.insert(murmurHash3(11, std::string("C"))); - - IBLT rcvdIBF(size); - rcvdIBF.insert(murmurHash3(11, std::string("C"))); - rcvdIBF.insert(murmurHash3(11, std::string("D"))); - rcvdIBF.insert(murmurHash3(11, std::string("E"))); - std::cout << "D: " << murmurHash3(11, std::string("D")) << std::endl; - std::cout << "E: " << murmurHash3(11, std::string("E")) << std::endl; - - IBLT diff = rcvdIBF - (ownIBF - rcvdIBF); - - std::set positive; - std::set negative; - - diff.listEntries(positive, negative); - - for (const auto& p : positive) { - std::cout << " positive: " << p << std::endl; - } - - for (const auto& n : negative) { - std::cout << " negative: " << n << std::endl; - } - - // IBLT diff2 = ownIBF - ; -} - -BOOST_AUTO_TEST_CASE(IBFSize) -{ - int size = 10; - -} - -BOOST_AUTO_TEST_SUITE_END() - -} // namespace psync