Add support for v2 host announcements

.github/workflows/test.yml

@@ -108,7 +108,7 @@ jobs:
           go-test-args: "-race;-timeout=20m;-tags=netgo,v2"
 
   success: # Use in branch rulesets to ensure all matrix jobs completed successfully
-    needs: [test-sqlite, test-mysql]
+    needs: [test-sqlite, test-mysql, test-sqlite-v2, test-mysql-v2]
     runs-on: ubuntu-latest
     steps:
       - run: echo "Success!"

api/bus.go

@@ -5,6 +5,7 @@ import (
 
 	rhpv2 "go.sia.tech/core/rhp/v2"
 	rhpv3 "go.sia.tech/core/rhp/v3"
+	rhpv4 "go.sia.tech/core/rhp/v4"
 	"go.sia.tech/core/types"
 )
 
@@ -96,5 +97,6 @@ type (
 		ScanError  string               `json:"scanError,omitempty"`
 		Settings   rhpv2.HostSettings   `json:"settings,omitempty"`
 		PriceTable rhpv3.HostPriceTable `json:"priceTable,omitempty"`
+		V2Settings rhpv4.HostSettings   `json:"v2Settings,omitempty"`
 	}
 )

api/host.go

@@ -4,11 +4,13 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
 	"strings"
 	"time"
 
 	rhpv2 "go.sia.tech/core/rhp/v2"
 	rhpv3 "go.sia.tech/core/rhp/v3"
+	rhpv4 "go.sia.tech/core/rhp/v4"
 	"go.sia.tech/core/types"
 )
 
@@ -112,13 +114,13 @@ type (
 		Blocked           bool                 `json:"blocked"`
 		Checks            map[string]HostCheck `json:"checks"`
 		StoredData        uint64               `json:"storedData"`
-		ResolvedAddresses []string             `json:"resolvedAddresses"`
-		Subnets           []string             `json:"subnets"`
+		V2SiamuxAddresses []string             `json:"v2SiamuxAddresses"`
 	}
 
 	HostInfo struct {
-		PublicKey  types.PublicKey `json:"publicKey"`
-		SiamuxAddr string          `json:"siamuxAddr"`
+		PublicKey         types.PublicKey `json:"publicKey"`
+		SiamuxAddr        string          `json:"siamuxAddr"`
+		V2SiamuxAddresses []string        `json:"v2SiamuxAddresses"`
 	}
 
 	HostInteractions struct {
@@ -135,13 +137,12 @@ type (
 	}
 
 	HostScan struct {
-		HostKey           types.PublicKey      `json:"hostKey"`
-		PriceTable        rhpv3.HostPriceTable `json:"priceTable"`
-		Settings          rhpv2.HostSettings   `json:"settings"`
-		ResolvedAddresses []string             `json:"resolvedAddresses"`
-		Subnets           []string             `json:"subnets"`
-		Success           bool                 `json:"success"`
-		Timestamp         time.Time            `json:"timestamp"`
+		HostKey    types.PublicKey      `json:"hostKey"`
+		PriceTable rhpv3.HostPriceTable `json:"priceTable,omitempty"`
+		Settings   rhpv2.HostSettings   `json:"settings,omitempty"`
+		V2Settings rhpv4.HostSettings   `json:"v2Settings,omitempty"`
+		Success    bool                 `json:"success"`
+		Timestamp  time.Time            `json:"timestamp"`
 	}
 
 	HostPriceTable struct {
@@ -220,6 +221,18 @@ func (h Host) IsOnline() bool {
 	return h.Interactions.LastScanSuccess || h.Interactions.SecondToLastScanSuccess
 }
 
+func (h Host) V2SiamuxAddr() string {
+	// NOTE: eventually this can be smarter about picking an address but right now
+	// we just prioritize IPv4 over IPv6
+	sort.Slice(h.V2SiamuxAddresses, func(i, j int) bool {
+		return len(h.V2SiamuxAddresses[i]) < len(h.V2SiamuxAddresses[j])
+	})
+	if len(h.V2SiamuxAddresses) > 0 {
+		return h.V2SiamuxAddresses[0]
+	}
+	return ""
+}
+
 func (sb HostScoreBreakdown) String() string {
 	return fmt.Sprintf("Age: %v, Col: %v, Int: %v, SR: %v, UT: %v, V: %v, Pr: %v", sb.Age, sb.Collateral, sb.Interactions, sb.StorageRemaining, sb.Uptime, sb.Version, sb.Prices)
 }

autopilot/contractor/contractor.go

@@ -863,8 +863,7 @@ func performContractChecks(ctx *mCtx, alerter alerts.Alerter, bus Bus, cc contra
 		}
 
 		// extend logger
-		logger = logger.With("addresses", host.ResolvedAddresses).
-			With("blocked", host.Blocked)
+		logger = logger.With("blocked", host.Blocked)
 
 		// check if host is blocked
 		if host.Blocked {
@@ -1082,8 +1081,7 @@ func performContractFormations(ctx *mCtx, bus Bus, cr contractReviser, ipFilter
 		}
 
 		// prepare a logger
-		logger := logger.With("hostKey", candidate.host.PublicKey).
-			With("addresses", candidate.host.ResolvedAddresses)
+		logger := logger.With("hostKey", candidate.host.PublicKey)
 
 		// check if we already have a contract with a host on that address
 		if ctx.ShouldFilterRedundantIPs() && ipFilter.HasRedundantIP(candidate.host) {
@@ -1220,10 +1218,7 @@ func performContractMaintenance(ctx *mCtx, alerter alerts.Alerter, bus Bus, chur
 	}
 
 	// STEP 2: perform contract maintenance
-	ipFilter := &hostSet{
-		logger:          logger.Named("ipFilter"),
-		subnetToHostKey: make(map[string]string),
-	}
+	ipFilter := newHostSet(logger.Named("ipFilter"))
 	keptContracts, churnReasons, err := performContractChecks(ctx, alerter, bus, cc, cr, ipFilter, logger)
 	if err != nil {
 		return false, err

autopilot/contractor/hostset.go

@@ -3,46 +3,79 @@ package contractor
 import (
 	"context"
 	"errors"
-	"time"
 
+	"go.sia.tech/core/types"
 	"go.sia.tech/renterd/api"
 	"go.sia.tech/renterd/internal/utils"
 	"go.uber.org/zap"
 )
 
-var (
-	errHostTooManySubnets = errors.New("host has more than two subnets")
-)
-
 type (
 	hostSet struct {
-		subnetToHostKey map[string]string
+		resolvedAddresses map[types.PublicKey][]string
+		subnetToHostKey   map[string]string
 
 		logger *zap.SugaredLogger
 	}
 )
 
+func newHostSet(l *zap.SugaredLogger) *hostSet {
+	return &hostSet{
+		resolvedAddresses: make(map[types.PublicKey][]string),
+		subnetToHostKey:   make(map[string]string),
+		logger:            l,
+	}
+}
+
+func (hs *hostSet) resolveHostIP(host api.Host) ([]string, error) {
+	resolvedAddresses := hs.resolvedAddresses[host.PublicKey]
+	if len(resolvedAddresses) > 0 {
+		return resolvedAddresses, nil
+	}
+	// resolve host IP
+	// NOTE: we ignore errors here since failing to resolve an address is either
+	// 1. not the host's faul, so we give it the benefit of the doubt
+	// 2. the host is unreachable of incorrectly announced, in which case the scans will fail
+	//
+	var hostAddrs []string
+	if host.NetAddress != "" {
+		hostAddrs = append(hostAddrs, host.NetAddress)
+	}
+	hostAddrs = append(hostAddrs, host.V2SiamuxAddresses...)
+	resolvedAddresses, _, err := utils.ResolveHostIPs(context.Background(), hostAddrs)
+	if err != nil {
+		return nil, err
+	}
+
+	// update cache
+	hs.resolvedAddresses[host.PublicKey] = resolvedAddresses
+	return resolvedAddresses, nil
+}
+
 func (hs *hostSet) HasRedundantIP(host api.Host) bool {
-	// compat code for hosts that have been scanned before ResolvedAddresses
-	// were introduced
-	if len(host.ResolvedAddresses) == 0 {
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-		defer cancel()
-		host.ResolvedAddresses, _, _ = utils.ResolveHostIP(ctx, host.NetAddress)
+	logger := hs.logger.Named("hasRedundantIP").
+		With("hostKey", host.PublicKey).
+		With("netAddress", host.NetAddress).
+		With("v2SiamuxAddresses", host.V2SiamuxAddresses)
+
+	resolvedAddresses, err := hs.resolveHostIP(host)
+	if errors.Is(err, utils.ErrHostTooManyAddresses) {
+		logger.Errorf("host has more than 2 subnets, treating its IP %v as redundant", host.PublicKey, utils.ErrHostTooManyAddresses)
+		return true
+	} else if err != nil {
+		logger.With(zap.Error(err)).Error("failed to resolve host ip - treating it as redundant")
+		return true
 	}
 
-	subnets, err := utils.AddressesToSubnets(host.ResolvedAddresses)
+	subnets, err := utils.AddressesToSubnets(resolvedAddresses)
 	if err != nil {
-		hs.logger.Errorf("failed to parse host %v subnets: %v", host.PublicKey, err)
+		logger.With(zap.Error(err)).Errorf("failed to parse host subnets")
 		return true
 	}
 	// validate host subnets
 	if len(subnets) == 0 {
-		hs.logger.Errorf("host %v has no subnet, treating its IP %v as redundant", host.PublicKey, host.NetAddress)
-		return true
-	} else if len(subnets) > 2 {
-		hs.logger.Errorf("host %v has more than 2 subnets, treating its IP %v as redundant", host.PublicKey, errHostTooManySubnets)
-		return true
+		logger.Warnf("host has no subnets")
+		return false
 	}
 
 	// check if we know about this subnet
@@ -61,7 +94,12 @@ func (hs *hostSet) HasRedundantIP(host api.Host) bool {
 }
 
 func (hs *hostSet) Add(host api.Host) {
-	subnets, err := utils.AddressesToSubnets(host.ResolvedAddresses)
+	addresses, err := hs.resolveHostIP(host)
+	if err != nil {
+		hs.logger.Errorf("failed to resolve host %v addresses: %v", host.PublicKey, err)
+		return
+	}
+	subnets, err := utils.AddressesToSubnets(addresses)
 	if err != nil {
 		hs.logger.Errorf("failed to parse host %v subnets: %v", host.PublicKey, err)
 		return

autopilot/contractor/hostset_test.go

@@ -9,24 +9,23 @@ import (
 )
 
 func TestHostSet(t *testing.T) {
-	hs := &hostSet{
-		subnetToHostKey: make(map[string]string),
-		logger:          zap.NewNop().Sugar(),
-	}
+	hs := newHostSet(zap.NewNop().Sugar())
 
 	// Host with no subnets
 	host1 := api.Host{
 		PublicKey:         types.GeneratePrivateKey().PublicKey(),
-		ResolvedAddresses: []string{},
+		NetAddress:        "",
+		V2SiamuxAddresses: []string{},
 	}
-	if !hs.HasRedundantIP(host1) {
-		t.Fatalf("Expected host with no subnets to be considered redundant")
+	if hs.HasRedundantIP(host1) {
+		t.Fatalf("Expected host with no subnets to not be redundant")
 	}
 
 	// Host with more than 2 subnets
 	host2 := api.Host{
 		PublicKey:         types.GeneratePrivateKey().PublicKey(),
-		ResolvedAddresses: []string{"192.168.1.1", "10.0.0.1", "172.16.0.1"},
+		NetAddress:        "1.1.1.1:1111",
+		V2SiamuxAddresses: []string{"2.2.2.2:2222", "3.3.3.3:3333"},
 	}
 	if !hs.HasRedundantIP(host2) {
 		t.Fatalf("Expected host with more than 2 subnets to be considered redundant")
@@ -35,7 +34,8 @@ func TestHostSet(t *testing.T) {
 	// New host with unique subnet
 	host3 := api.Host{
 		PublicKey:         types.GeneratePrivateKey().PublicKey(),
-		ResolvedAddresses: []string{"192.168.2.1"},
+		NetAddress:        "",
+		V2SiamuxAddresses: []string{"4.4.4.4:4444"},
 	}
 	if hs.HasRedundantIP(host3) {
 		t.Fatal("Expected new host with unique subnet to not be considered redundant")
@@ -45,7 +45,8 @@ func TestHostSet(t *testing.T) {
 	// New host with same subnet but different public key
 	host4 := api.Host{
 		PublicKey:         types.GeneratePrivateKey().PublicKey(),
-		ResolvedAddresses: []string{"192.168.2.1"},
+		NetAddress:        "",
+		V2SiamuxAddresses: []string{"4.4.4.4:4444"},
 	}
 	if !hs.HasRedundantIP(host4) {
 		t.Fatal("Expected host with same subnet but different public key to be considered redundant")
@@ -59,7 +60,8 @@ func TestHostSet(t *testing.T) {
 	// Host with two valid subnets
 	host5 := api.Host{
 		PublicKey:         types.GeneratePrivateKey().PublicKey(),
-		ResolvedAddresses: []string{"192.168.3.1", "10.0.0.1"},
+		NetAddress:        "",
+		V2SiamuxAddresses: []string{"5.5.5.5:5555", "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:5555"},
 	}
 	if hs.HasRedundantIP(host5) {
 		t.Fatal("Expected host with two valid subnets to not be considered redundant")
@@ -69,9 +71,18 @@ func TestHostSet(t *testing.T) {
 	// New host with one overlapping subnet
 	host6 := api.Host{
 		PublicKey:         types.GeneratePrivateKey().PublicKey(),
-		ResolvedAddresses: []string{"10.0.0.1", "172.16.0.1"},
+		NetAddress:        "",
+		V2SiamuxAddresses: []string{"6.6.6.6:6666", "7.7.7.7:7777"},
 	}
 	if !hs.HasRedundantIP(host6) {
 		t.Fatal("Expected host with one overlapping subnet to be considered redundant")
 	}
+	host7 := api.Host{
+		PublicKey:         types.GeneratePrivateKey().PublicKey(),
+		NetAddress:        "6.6.6.6:6666",
+		V2SiamuxAddresses: []string{"8.8.8.8:8888"},
+	}
+	if !hs.HasRedundantIP(host7) {
+		t.Fatal("Expected host with one overlapping subnet to be considered redundant")
+	}
 }