Merge pull request #1533 from iotaledger/develop

Merge v0.7.3 changes to master

.github/workflows/release.yml

@@ -20,6 +20,8 @@ jobs:
           go-version: 1.16.3
       - name: Copy config.default.json to config.json
         run: cp config.default.json config.json
+      - name: Download latest snapshot file
+        run: wget -O snapshot.bin https://dbfiles-goshimmer.s3.eu-central-1.amazonaws.com/snapshots/nectar/snapshot-latest.bin
       - name: Run GoReleaser
         run: goreleaser --rm-dist
         env:

.gitignore

@@ -8,7 +8,7 @@
 *.dat.bkp
 
 # Snapshots are stored in bin format
-# *.bin
+*.bin
 
 # cli-wallet config
 tools/cli-wallet/config.json
@@ -40,4 +40,5 @@ goshimmer
 book
 
 config.json
-.env
+.env
+.vscode

CHANGELOG.md

@@ -1,3 +1,20 @@
+# v0.7.3 - 2021-07-01
+* Add latest snapshot download from S3
+* Add weak tips on local dashboard
+* Improve integration test framework
+* Add FCoB metrics to remote logger
+* Remove tips broadcaster
+* Improve APIs
+* Fix Sync status
+* Fix gossip and requester issues
+* Fix Inclusion state inconsistencies
+* Fix Prometheus issues when autopeering is not enabled
+* Increase maxVerticesWithoutFutureMarker to 3000000
+* Update snapshot file with DevNet UTXO at 2021-06-30 21:16 UTC
+* Update JS dependencies
+* Update hive.go
+* **Breaking**: bumps network and database versions
+
 # v0.7.2 - 2021-06-17
 * Add local double spend filter in webAPI
 * Add SetBranchFinalized to FCoB

Dockerfile

@@ -33,6 +33,8 @@ RUN --mount=target=. \
     -o /go/bin/goshimmer; \
     ./check_static.sh
 
+RUN wget -O /tmp/snapshot.bin https://dbfiles-goshimmer.s3.eu-central-1.amazonaws.com/snapshots/nectar/snapshot-latest.bin
+
 ############################
 # Image
 ############################
@@ -44,7 +46,7 @@ EXPOSE 14666/tcp
 EXPOSE 14626/udp
 
 # Copy configuration
-COPY snapshot.bin /snapshot.bin
+COPY --from=build /tmp/snapshot.bin /snapshot.bin
 COPY config.default.json /config.json
 
 # Copy the Pre-built binary file from the previous stage

README.md

@@ -64,7 +64,7 @@ We always recommend running your node via [Docker](https://goshimmer.docs.iota.o
 
 Please follow this guide: https://github.com/facebook/rocksdb/blob/master/INSTALL.md to build above libs.
 
-Finally, when compiling GoShimmer, just run the build script:
+When compiling GoShimmer, just run the build script:
 
 ```bash
 ./scripts/build.sh
@@ -75,6 +75,12 @@ If you also want to link the libraries statically (only on Linux) run this inste
 ./scripts/build_goshimmer_rocksdb_builtin.sh
 ```
 
+Finally, download the latest snapshot and make sure to place it in the root folder of GoShimmer:
+
+```bash
+wget -O snapshot.bin https://dbfiles-goshimmer.s3.eu-central-1.amazonaws.com/snapshots/nectar/snapshot-latest.bin
+```
+
 ## Supporting the project
 
 If you want to contribute to the code, consider posting a [bug report](https://github.com/iotaledger/goshimmer/issues/new-issue), feature request or a [pull request](https://github.com/iotaledger/goshimmer/pulls/).

client/health.go

@@ -0,0 +1,14 @@
+package client
+
+import (
+	"net/http"
+)
+
+const (
+	routeHealth = "healthz"
+)
+
+// HealthCheck checks whether the node is running and healthy.
+func (api *GoShimmerAPI) HealthCheck() error {
+	return api.do(http.MethodGet, routeHealth, nil, nil)
+}

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/go-resty/resty/v2 v2.6.0
 	github.com/golang/protobuf v1.4.3
 	github.com/gorilla/websocket v1.4.2
-	github.com/iotaledger/hive.go v0.0.0-20210528180853-73ecfbb76bd7
+	github.com/iotaledger/hive.go v0.0.0-20210625103722-68b2cf52ef4e
 	github.com/labstack/echo v3.3.10+incompatible
 	github.com/labstack/gommon v0.3.0
 	github.com/linxGnu/grocksdb v1.6.35 // indirect

go.sum

@@ -407,8 +407,8 @@ github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:q
 github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
-github.com/iotaledger/hive.go v0.0.0-20210528180853-73ecfbb76bd7 h1:fidetnahXOAJdKzwBDeSaNe3/6y3C0r51KsTHVxJWy8=
-github.com/iotaledger/hive.go v0.0.0-20210528180853-73ecfbb76bd7/go.mod h1:NyBg/Ny7FFAdDs59zdwTVoysU2ZbJVQnRwyLIDFKJYA=
+github.com/iotaledger/hive.go v0.0.0-20210625103722-68b2cf52ef4e h1:jUHV5HGa/1o9qoPPr8ms3dMueSVUb1ZB2Z1XyJ79qfU=
+github.com/iotaledger/hive.go v0.0.0-20210625103722-68b2cf52ef4e/go.mod h1:NyBg/Ny7FFAdDs59zdwTVoysU2ZbJVQnRwyLIDFKJYA=
 github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=
 github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM=

packages/consensus/fcob/consensusmechanism.go

@@ -51,7 +51,7 @@ func NewConsensusMechanism() *ConsensusMechanism {
 // Init initializes the ConsensusMechanism by making the Tangle object available that is using it.
 func (f *ConsensusMechanism) Init(tangle *tangle.Tangle) {
 	f.tangle = tangle
-	f.Storage = NewStorage(tangle.Options.Store)
+	f.Storage = NewStorage(tangle.Options.Store, tangle.Options.CacheTimeProvider)
 }
 
 // Setup sets up the behavior of the ConsensusMechanism by making it attach to the relevant events in the Tangle.
@@ -313,12 +313,6 @@ func (f *ConsensusMechanism) onPayloadOpinionFormed(messageID tangle.MessageID,
 				if err != nil {
 					panic(err)
 				}
-				if !liked {
-					_, err := f.tangle.LedgerState.BranchDAG.SetBranchFinalized(f.tangle.LedgerState.BranchID(transactionID), true)
-					if err != nil {
-						panic(err)
-					}
-				}
 			}
 		})
 	})

packages/consensus/fcob/consensusmechanism_test.go

@@ -6,6 +6,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/iotaledger/goshimmer/packages/database"
+
 	"github.com/iotaledger/hive.go/crypto/ed25519"
 	"github.com/iotaledger/hive.go/events"
 	"github.com/iotaledger/hive.go/identity"
@@ -38,8 +40,10 @@ func TestOpinionFormer_Scenario2(t *testing.T) {
 	LocallyFinalizedThreshold = 2 * time.Second
 
 	consensusProvider := NewConsensusMechanism()
+	cacheTimeProvider := database.NewCacheTimeProvider(0)
 
-	testTangle := tangle.New(tangle.Consensus(consensusProvider), tangle.SchedulerConfig(schedulerParams))
+	testTangle := tangle.New(tangle.Consensus(consensusProvider), tangle.SchedulerConfig(schedulerParams),
+		tangle.CacheTimeProvider(cacheTimeProvider))
 	defer testTangle.Shutdown()
 	testTangle.Setup()
 
@@ -241,8 +245,10 @@ func TestOpinionFormer(t *testing.T) {
 	LocallyFinalizedThreshold = 2 * time.Second
 
 	consensusProvider := NewConsensusMechanism()
+	cacheTimeProvider := database.NewCacheTimeProvider(0)
 
-	testTangle := tangle.New(tangle.Consensus(consensusProvider), tangle.SchedulerConfig(schedulerParams))
+	testTangle := tangle.New(tangle.Consensus(consensusProvider), tangle.SchedulerConfig(schedulerParams),
+		tangle.CacheTimeProvider(cacheTimeProvider))
 	defer testTangle.Shutdown()
 
 	messageA := newTestDataMessage("A")

packages/consensus/fcob/storage.go

@@ -30,14 +30,14 @@ type Storage struct {
 }
 
 // NewStorage is the constructor for a Storage.
-func NewStorage(store kvstore.KVStore) (storage *Storage) {
+func NewStorage(store kvstore.KVStore, cacheProvider *database.CacheTimeProvider) (storage *Storage) {
 	osFactory := objectstorage.NewFactory(store, database.PrefixFCOB)
 
 	storage = &Storage{
 		store:                   store,
-		opinionStorage:          osFactory.New(PrefixOpinion, OpinionFromObjectStorage, objectstorage.CacheTime(cacheTime), objectstorage.LeakDetectionEnabled(false)),
-		timestampOpinionStorage: osFactory.New(PrefixTimestampOpinion, TimestampOpinionFromObjectStorage, objectstorage.CacheTime(cacheTime), objectstorage.LeakDetectionEnabled(false)),
-		messageMetadataStorage:  osFactory.New(PrefixMessageMetadata, MessageMetadataFromObjectStorage, objectstorage.CacheTime(cacheTime), objectstorage.LeakDetectionEnabled(false)),
+		opinionStorage:          osFactory.New(PrefixOpinion, OpinionFromObjectStorage, cacheProvider.CacheTime(cacheTime), objectstorage.LeakDetectionEnabled(false)),
+		timestampOpinionStorage: osFactory.New(PrefixTimestampOpinion, TimestampOpinionFromObjectStorage, cacheProvider.CacheTime(cacheTime), objectstorage.LeakDetectionEnabled(false)),
+		messageMetadataStorage:  osFactory.New(PrefixMessageMetadata, MessageMetadataFromObjectStorage, cacheProvider.CacheTime(cacheTime), objectstorage.LeakDetectionEnabled(false)),
 	}
 
 	genesis := NewMessageMetadata(tangle.EmptyMessageID)

packages/database/cache_time_provider.go

@@ -0,0 +1,27 @@
+package database
+
+import (
+	"time"
+
+	"github.com/iotaledger/hive.go/objectstorage"
+)
+
+// CacheTimeProvider should be always used to get the CacheTime option for storage
+// It wraps around objectstorage.CacheTime() function and may override the input duration
+type CacheTimeProvider struct {
+	forceCacheTime time.Duration
+}
+
+// NewCacheTimeProvider creates an instance that forces cache time to always be a certain value.
+// If the given value is negative, hard coded defaults will be used
+func NewCacheTimeProvider(forceCacheTime time.Duration) *CacheTimeProvider {
+	return &CacheTimeProvider{forceCacheTime: forceCacheTime}
+}
+
+// CacheTime returns a CacheTime option. Duration may be overridden if CacheTimeProvider parameter is a non-negative integer
+func (m *CacheTimeProvider) CacheTime(duration time.Duration) objectstorage.Option {
+	if m.forceCacheTime >= 0 {
+		duration = m.forceCacheTime
+	}
+	return objectstorage.CacheTime(duration)
+}

packages/gossip/manager.go

@@ -3,7 +3,6 @@ package gossip
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"net"
 	"runtime"
 	"sync"
@@ -45,18 +44,16 @@ type Manager struct {
 	events          Events
 	neighborsEvents map[NeighborsGroup]NeighborsEvents
 
-	wg sync.WaitGroup
-
 	server      *server.TCP
 	serverMutex sync.RWMutex
 
 	neighbors      map[identity.ID]*Neighbor
 	neighborsMutex sync.RWMutex
 
 	// messageWorkerPool defines a worker pool where all incoming messages are processed.
-	messageWorkerPool *workerpool.WorkerPool
+	messageWorkerPool *workerpool.NonBlockingQueuedWorkerPool
 
-	messageRequestWorkerPool *workerpool.WorkerPool
+	messageRequestWorkerPool *workerpool.NonBlockingQueuedWorkerPool
 }
 
 // NewManager creates a new Manager.
@@ -76,13 +73,13 @@ func NewManager(local *peer.Local, f LoadMessageFunc, log *logger.Logger) *Manag
 		server:    nil,
 	}
 
-	m.messageWorkerPool = workerpool.New(func(task workerpool.Task) {
+	m.messageWorkerPool = workerpool.NewNonBlockingQueuedWorkerPool(func(task workerpool.Task) {
 		m.processPacketMessage(task.Param(0).([]byte), task.Param(1).(*Neighbor))
 
 		task.Return(nil)
 	}, workerpool.WorkerCount(messageWorkerCount), workerpool.QueueSize(messageWorkerQueueSize))
 
-	m.messageRequestWorkerPool = workerpool.New(func(task workerpool.Task) {
+	m.messageRequestWorkerPool = workerpool.NewNonBlockingQueuedWorkerPool(func(task workerpool.Task) {
 		m.processMessageRequest(task.Param(0).([]byte), task.Param(1).(*Neighbor))
 
 		task.Return(nil)
@@ -97,9 +94,6 @@ func (m *Manager) Start(srv *server.TCP) {
 	defer m.serverMutex.Unlock()
 
 	m.server = srv
-
-	m.messageWorkerPool.Start()
-	m.messageRequestWorkerPool.Start()
 }
 
 // Stop stops the manager and closes all established connections.
@@ -190,30 +184,12 @@ func (m *Manager) AllNeighbors() []*Neighbor {
 	return result
 }
 
-func (m *Manager) getNeighborsByIDOrRandom(ids ...identity.ID) []*Neighbor {
-	if len(ids) > 0 {
-		return m.getNeighborsByID(ids)
-	}
-	const randomNeighborsNum = 3
-	return m.getRandomNeighbors(randomNeighborsNum)
-}
-
-func (m *Manager) getRandomNeighbors(amount int) []*Neighbor {
-	allNeighbors := m.AllNeighbors()
-	neighborsNum := len(allNeighbors)
-	if amount > neighborsNum {
-		amount = neighborsNum
-	}
-	randomIndexes := rand.Perm(neighborsNum)
-	sample := make([]*Neighbor, amount)
-	for i := range sample {
-		sample[i] = allNeighbors[randomIndexes[i]]
-	}
-	return sample
-}
-
 func (m *Manager) getNeighborsByID(ids []identity.ID) []*Neighbor {
 	result := make([]*Neighbor, 0, len(ids))
+	if len(ids) == 0 {
+		return result
+	}
+
 	m.neighborsMutex.RLock()
 	defer m.neighborsMutex.RUnlock()
 	for _, id := range ids {
@@ -225,7 +201,10 @@ func (m *Manager) getNeighborsByID(ids []identity.ID) []*Neighbor {
 }
 
 func (m *Manager) send(b []byte, to ...identity.ID) {
-	neighbors := m.getNeighborsByIDOrRandom(to...)
+	neighbors := m.getNeighborsByID(to)
+	if len(neighbors) == 0 {
+		neighbors = m.AllNeighbors()
+	}
 
 	for _, nbr := range neighbors {
 		if _, err := nbr.Write(b); err != nil {
@@ -355,6 +334,7 @@ func (m *Manager) processPacketMessage(data []byte, nbr *Neighbor) {
 	packet := new(pb.Message)
 	if err := proto.Unmarshal(data[1:], packet); err != nil {
 		m.log.Debugw("error processing packet", "err", err)
+		return
 	}
 	m.events.MessageReceived.Trigger(&MessageReceivedEvent{Data: packet.GetData(), Peer: nbr.Peer})
 }
@@ -363,16 +343,19 @@ func (m *Manager) processMessageRequest(data []byte, nbr *Neighbor) {
 	packet := new(pb.MessageRequest)
 	if err := proto.Unmarshal(data[1:], packet); err != nil {
 		m.log.Debugw("invalid packet", "err", err)
+		return
 	}
 
 	msgID, _, err := tangle.MessageIDFromBytes(packet.GetId())
 	if err != nil {
 		m.log.Debugw("invalid message id:", "err", err)
+		return
 	}
 
 	msgBytes, err := m.loadMessageFunc(msgID)
 	if err != nil {
 		m.log.Debugw("error loading message", "msg-id", msgID, "err", err)
+		return
 	}
 
 	// send the loaded message directly to the neighbor

packages/jsonmodels/tangle.go

@@ -35,7 +35,9 @@ type MessageMetadata struct {
 	StructureDetails   *StructureDetails `json:"structureDetails,omitempty"`
 	BranchID           string            `json:"branchID"`
 	Scheduled          bool              `json:"scheduled"`
+	ScheduledTime      int64             `json:"scheduledTime"`
 	Booked             bool              `json:"booked"`
+	BookedTime         int64             `json:"bookedTime"`
 	Eligible           bool              `json:"eligible"`
 	Invalid            bool              `json:"invalid"`
 	Finalized          bool              `json:"finalized"`