initial commit

.dockerignore

@@ -0,0 +1,2 @@
+.vscode
+test-data

.gitignore

@@ -0,0 +1,2 @@
+.vscode
+test-data/

Dockerfile

@@ -0,0 +1,9 @@
+FROM golang:1.18-alpine AS builder
+COPY . /build
+WORKDIR /build
+RUN go get .
+RUN go build -v -o /tmp/consul-raft-reader
+
+FROM alpine:3.16.2 AS final
+COPY --from=builder /tmp/consul-raft-reader /usr/local/bin/consul-raft-reader
+ENTRYPOINT ["/usr/local/bin/consul-raft-reader"]

Makefile

@@ -0,0 +1,25 @@
+VERSION="0.1.0"
+DOCKER_IMAGE="nvanthao/consul-raft-reader"
+
+.PHONY: tidy
+tidy: 
+	@echo "--> Tidy modules"
+	@go mod tidy
+
+.PHONY: docker-build-local
+docker-build-local:
+	@echo "--> Build docker image"
+	@docker build -t $(DOCKER_IMAGE) .
+
+.PHONY: docker-publish
+docker-publish: docker-build-local
+	@echo "---> Tag docker image"
+	@docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(VERSION)
+	@docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):latest
+	@echo "--> Publish docker image"
+	@docker push -a $(DOCKER_IMAGE)
+
+.PHONY: install
+install:
+	@echo "--> Install binary"
+	@go install .

README.md

@@ -0,0 +1,103 @@
+# consul-raft-reader
+consul-raft-reader is a CLI app to help understanding content of `raft.db` used in Consul.
+
+# Install
+
+```
+go install github.com/nvanthao/consul-raft-reader@latest
+```
+
+Or with Docker
+
+```
+docker pull nvanthao/consul-raft-reader
+```
+
+# Usage
+
+## Print logs of Raft file
+
+```
+consul-raft-reader print --start 1 --end 10 raft.db
+```
+
+Sample output
+
+```
+Index: 1 Term: 1 Log Type: LogConfiguration 
+Index: 2 Term: 2 Log Type: LogNoop 
+Index: 3 Term: 2 Log Type: LogBarrier 
+Index: 4 Term: 2 Log Type: LogCommand Message Type: Autopilot 
+Index: 5 Term: 2 Log Type: LogCommand Message Type: ConnectCA 
+Index: 6 Term: 2 Log Type: LogCommand Message Type: ConnectCA 
+Index: 7 Term: 2 Log Type: LogCommand Message Type: Register 
+Index: 8 Term: 2 Log Type: LogCommand Message Type: ConnectCA 
+Index: 9 Term: 2 Log Type: LogCommand Message Type: ConnectCA 
+Index: 10 Term: 2 Log Type: LogCommand Message Type: ConnectCA 
+```
+
+## Read log detail
+
+```
+consul-raft-reader read --index 4 raft.db
+```
+
+Sample output
+
+```
+Index: 4 
+Term: 2 
+Log Type: LogCommand 
+Message Type: Autopilot
+Data:
+{
+  "CAS": false,
+  "Config": {
+    "CleanupDeadServers": true,
+    "CreateIndex": 0,
+    "DisableUpgradeMigration": false,
+    "LastContactThreshold": 200000000,
+    "MaxTrailingLogs": 250,
+    "MinQuorum": 0,
+    "ModifyIndex": 0,
+    "RedundancyZoneTag": "",
+    "ServerStabilizationTime": 10000000000,
+    "UpgradeVersionTag": ""
+  },
+  "Datacenter": "",
+  "Token": ""
+}
+```
+
+## View basic stats
+
+```
+consul-raft-reader stats raft.db
+```
+
+Sample output
+
+```
+First Index: 1 
+Last Index: 217 
+Current Term:  
+Last Vote Term:  
+Last Vote Candidate: 172.22.0.2:8300 
+=== COUNT MESSAGE TYPES === 
+CoordinateUpdate: 173 
+Register: 10 
+ConnectCA: 5 
+SystemMetadata: 2 
+FederationState: 1 
+Tombstone: 1 
+Autopilot: 1 
+Intention: 1 
+```
+
+## Run with Docker
+
+```
+docker run -v $PWD/raft.db:/var/raft.db nvanthao/consul-raft-reader print --start 1 --end 10 /var/raft.db 
+docker run -v $PWD/raft.db:/var/raft.db nvanthao/consul-raft-reader read --index 4 /var/raft.db 
+docker run -v $PWD/raft.db:/var/raft.db nvanthao/consul-raft-reader stats /var/raft.db 
+```

app/store.go

@@ -0,0 +1,197 @@
+package app
+
+import (
+	"encoding/json"
+	"fmt"
+	"path/filepath"
+	"sort"
+
+	"github.com/hashicorp/consul/agent/structs"
+	"github.com/hashicorp/raft"
+	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
+	"go.etcd.io/bbolt"
+)
+
+var logTypes = map[raft.LogType]string{
+	raft.LogCommand:              "LogCommand",
+	raft.LogNoop:                 "LogNoop",
+	raft.LogAddPeerDeprecated:    "LogAddPeerDeprecated",
+	raft.LogRemovePeerDeprecated: "LogRemovePeerDeprecated",
+	raft.LogBarrier:              "LogBarrier",
+	raft.LogConfiguration:        "LogConfiguration",
+}
+
+type Store struct {
+	store      *raftboltdb.BoltStore
+	FirstIndex uint64
+	LastIndex  uint64
+}
+
+type RaftLog struct {
+	Term    uint64
+	Index   uint64
+	Type    string
+	MsgType string
+}
+
+type kv struct {
+	Key   string
+	Value int
+}
+
+// This function will read the Raft log at given index
+func (s *Store) Read(index uint64) error {
+	if index < s.FirstIndex || index > s.LastIndex {
+		return fmt.Errorf("index %d out of range [%d, %d]", index, s.FirstIndex, s.LastIndex)
+	}
+
+	var log raft.Log
+	err := s.store.GetLog(index, &log)
+
+	if err != nil {
+		return fmt.Errorf("unable to read log at index %d", index)
+	}
+
+	fmt.Printf("Index: %d \n", log.Index)
+	fmt.Printf("Term: %d \n", log.Term)
+	fmt.Printf("Log Type: %s \n", logTypes[log.Type])
+	if log.Type == raft.LogCommand {
+		fmt.Printf("Message Type: %s\n", getMessageType(log.Data[0]))
+	}
+	if len(log.Data) > 1 {
+		fmt.Println("Data:")
+		fmt.Println(getLogCommand(log.Data))
+	}
+
+	return nil
+}
+
+func (s *Store) Print(start, end uint64) {
+	logs := s.getLogs(start, end)
+	for _, log := range logs {
+		fmt.Printf("Index: %d ", log.Index)
+		fmt.Printf("Term: %d ", log.Term)
+		fmt.Printf("Log Type: %s ", log.Type)
+		if log.MsgType != "" {
+			fmt.Printf("Message Type: %s ", log.MsgType)
+		}
+		fmt.Println()
+	}
+}
+
+func (s *Store) Stats() {
+	fmt.Printf("First Index: %d \n", s.FirstIndex)
+	fmt.Printf("Last Index: %d \n", s.LastIndex)
+	fmt.Printf("Current Term: %s \n", s.getBucketValue("CurrentTerm"))
+	fmt.Printf("Last Vote Term: %s \n", s.getBucketValue("LastVoteTerm"))
+	fmt.Printf("Last Vote Candidate: %s \n", s.getBucketValue("LastVoteCand"))
+	fmt.Printf("=== COUNT MESSAGE TYPES === \n")
+	logs := s.getLogs(s.FirstIndex, s.LastIndex)
+	counts := countMsgType(logs)
+	ss := sortMapByValue(counts)
+	for _, s := range ss {
+		fmt.Printf("%s: %d \n", s.Key, s.Value)
+	}
+}
+
+func NewStore(file string) (*Store, error) {
+	store, err := raftboltdb.New(raftboltdb.Options{
+		BoltOptions: &bbolt.Options{
+			NoFreelistSync: true,
+		},
+		Path: filepath.Join(file),
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	FirstIndex, _ := store.FirstIndex()
+	LastIndex, _ := store.LastIndex()
+
+	return &Store{
+		store:      store,
+		FirstIndex: FirstIndex,
+		LastIndex:  LastIndex}, nil
+}
+
+// this function is trying to guess the command type from the log data
+// and default to a string(data) if it can't guess
+func getLogCommand(data []byte) string {
+	var st map[string]interface{}
+
+	// first we decode the data into our map[string]interface{}
+	if err := structs.Decode(data[1:], &st); err != nil {
+		return string(data[1:])
+	}
+
+	// then we encode it to a pretty json string
+	b, err := json.MarshalIndent(st, "", "  ")
+	if err != nil {
+		return string(data[1:])
+	}
+
+	return string(b)
+}
+
+// get logs from a range
+func (s *Store) getLogs(start, end uint64) (logs []RaftLog) {
+	for i := start; i <= end; i++ {
+		var log raft.Log
+		err := s.store.GetLog(i, &log)
+
+		// we don't worry much about not being able to read a log here
+		if err != nil {
+			continue
+		}
+		var msgType string
+		if log.Type == raft.LogCommand {
+			msgType = getMessageType(log.Data[0])
+		}
+
+		logs = append(logs, RaftLog{
+			Term:    log.Term,
+			Index:   log.Index,
+			Type:    logTypes[log.Type],
+			MsgType: msgType,
+		})
+	}
+	return
+}
+
+func getMessageType(msgType byte) string {
+	if msgType == 134 {
+		return "CoordinateUpdate"
+	}
+	return structs.MessageType(msgType).String()
+}
+
+func (s *Store) getBucketValue(key string) string {
+	value, err := s.store.Get([]byte(key))
+	if err != nil {
+		return "Unknown"
+	}
+	return string(value)
+}
+
+func countMsgType(logs []RaftLog) map[string]int {
+	memo := make(map[string]int)
+	for _, log := range logs {
+		if log.MsgType != "" {
+			memo[log.MsgType]++
+		}
+	}
+	return memo
+}
+
+func sortMapByValue(m map[string]int) []kv {
+	var ss []kv
+	for k, v := range m {
+		ss = append(ss, kv{k, v})
+	}
+
+	sort.Slice(ss, func(i, j int) bool {
+		return ss[i].Value > ss[j].Value
+	})
+
+	return ss
+}