diff --git a/README.md b/README.md index ca95f1e06..6b6ffa4b5 100644 --- a/README.md +++ b/README.md @@ -517,42 +517,6 @@ your node's consensus state using the following endpoint: - `GET /api/bus/consensus/state` -### Config - -The configuration can be updated through the UI or by using the following -endpoints: - -- `GET /api/autopilot/config` -- `PUT /api/autopilot/config` - -The autopilot will not perform any tasks until it is configured. An example -configuration can be found below. Especially the `contracts` section is -important, make sure the `amount` is set to the amount of hosts with which you -want to form a contract. The `allowance` is the amount of money the autopilot -can spend per period, make sure it is not set to zero or contracts won't get -formed. - -```json -{ - "hosts": { - "allowRedundantIPs": false, - "maxDowntimeHours": 1440, - "maxConsecutiveScanFailures": 20, - "scoreOverrides": {} - }, - "contracts": { - "set": "autopilot", - "amount": 50, - "allowance": "10000000000000000000000000000", - "period": 6048, - "renewWindow": 2016, - "download": 1099511627776, // 1TiB - "upload": 1099511627776, // 1TiB - "storage": 1099511627776 // 1TiB - } -} -``` - ### Blocklist Unfortunately the Sia blockchain is subject to hosts that announced themselves diff --git a/api/autopilot.go b/api/autopilot.go index 26dc74a82..94e24fc80 100644 --- a/api/autopilot.go +++ b/api/autopilot.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "go.sia.tech/core/types" "go.sia.tech/renterd/internal/utils" ) @@ -12,30 +11,28 @@ const ( // BlocksPerDay defines the amount of blocks that are mined in a day (one // block every 10 minutes roughly) BlocksPerDay = 144 - - // DefaultAutopilotID is the id of the autopilot. - DefaultAutopilotID = "autopilot" ) var ( - // ErrAutopilotNotFound is returned when an autopilot can't be found. - ErrAutopilotNotFound = errors.New("couldn't find autopilot") - - // ErrMaxDowntimeHoursTooHigh is returned if the autopilot config is updated + // ErrMaxDowntimeHoursTooHigh is returned if the contracts config is updated // with a value that exceeds the maximum of 99 years. ErrMaxDowntimeHoursTooHigh = errors.New("MaxDowntimeHours is too high, exceeds max value of 99 years") + + // ErrInvalidReleaseVersion is returned if the version is an invalid release + // string. + ErrInvalidReleaseVersion = errors.New("invalid release version") ) type ( - // Autopilot contains the autopilot's config and current period. + // Autopilot contains its configuration as well as the current period. Autopilot struct { - ID string `json:"id"` - Config AutopilotConfig `json:"config"` - CurrentPeriod uint64 `json:"currentPeriod"` + CurrentPeriod uint64 `json:"currentPeriod"` + AutopilotConfig } - // AutopilotConfig contains all autopilot configuration. + // AutopilotConfig contains configuration settings for the autopilot. AutopilotConfig struct { + Enabled bool `json:"enabled"` Contracts ContractsConfig `json:"contracts"` Hosts HostsConfig `json:"hosts"` } @@ -54,19 +51,34 @@ type ( // HostsConfig contains all hosts settings used in the autopilot. HostsConfig struct { - AllowRedundantIPs bool `json:"allowRedundantIPs"` - MaxDowntimeHours uint64 `json:"maxDowntimeHours"` - MinProtocolVersion string `json:"minProtocolVersion"` - MaxConsecutiveScanFailures uint64 `json:"maxConsecutiveScanFailures"` - ScoreOverrides map[types.PublicKey]float64 `json:"scoreOverrides"` + AllowRedundantIPs bool `json:"allowRedundantIPs"` + MaxConsecutiveScanFailures uint64 `json:"maxConsecutiveScanFailures"` + MaxDowntimeHours uint64 `json:"maxDowntimeHours"` + MinProtocolVersion string `json:"minProtocolVersion"` } ) -// EndHeight of a contract formed using the AutopilotConfig given the current -// period. -func (ap *Autopilot) EndHeight() uint64 { - return ap.CurrentPeriod + ap.Config.Contracts.Period + ap.Config.Contracts.RenewWindow -} +var ( + DefaultAutopilotConfig = AutopilotConfig{ + Enabled: false, + Contracts: ContractsConfig{ + Set: "autopilot", + Amount: 50, + Period: 144 * 7 * 6, + RenewWindow: 144 * 7 * 2, + Download: 1e12, // 1 TB + Upload: 1e12, // 1 TB + Storage: 4e12, // 4 TB + Prune: false, + }, + Hosts: HostsConfig{ + AllowRedundantIPs: false, + MaxConsecutiveScanFailures: 10, + MaxDowntimeHours: 24 * 7 * 2, + MinProtocolVersion: "1.6.0", + }, + } +) type ( // AutopilotTriggerRequest is the request object used by the /trigger @@ -84,8 +96,7 @@ type ( // AutopilotStateResponse is the response type for the /autopilot/state // endpoint. AutopilotStateResponse struct { - ID string `json:"id"` - Configured bool `json:"configured"` + Enabled bool `json:"enabled"` Migrating bool `json:"migrating"` MigratingLastStart TimeRFC3339 `json:"migratingLastStart"` Pruning bool `json:"pruning"` @@ -128,11 +139,24 @@ type ( } ) -func (c AutopilotConfig) Validate() error { - if c.Hosts.MaxDowntimeHours > 99*365*24 { +func (ap Autopilot) EndHeight() uint64 { + return ap.CurrentPeriod + ap.Contracts.Period + ap.Contracts.RenewWindow +} + +func (cc ContractsConfig) Validate() error { + if cc.Period == 0 { + return errors.New("period must be greater than 0") + } else if cc.RenewWindow == 0 { + return errors.New("renewWindow must be greater than 0") + } + return nil +} + +func (hc HostsConfig) Validate() error { + if hc.MaxDowntimeHours > 99*365*24 { return ErrMaxDowntimeHoursTooHigh - } else if c.Hosts.MinProtocolVersion != "" && !utils.IsVersion(c.Hosts.MinProtocolVersion) { - return fmt.Errorf("invalid min protocol version '%s'", c.Hosts.MinProtocolVersion) + } else if hc.MinProtocolVersion != "" && !utils.IsVersion(hc.MinProtocolVersion) { + return fmt.Errorf("%w: '%s'", ErrInvalidReleaseVersion, hc.MinProtocolVersion) } return nil } diff --git a/api/bus.go b/api/bus.go index 929cec02d..75eef7622 100644 --- a/api/bus.go +++ b/api/bus.go @@ -97,4 +97,12 @@ type ( Settings rhpv2.HostSettings `json:"settings,omitempty"` PriceTable rhpv3.HostPriceTable `json:"priceTable,omitempty"` } + + // UpdateAutopilotRequest is the request type for the /autopilot endpoint. + UpdateAutopilotRequest struct { + Enabled *bool `json:"enabled"` + Contracts *ContractsConfig `json:"contracts"` + CurrentPeriod *uint64 `json:"currentPeriod"` + Hosts *HostsConfig `json:"hosts"` + } ) diff --git a/api/host.go b/api/host.go index 3ca2cfe3f..9e10ec6c3 100644 --- a/api/host.go +++ b/api/host.go @@ -60,7 +60,6 @@ type ( HostsRequest struct { Offset int `json:"offset"` Limit int `json:"limit"` - AutopilotID string `json:"autopilotID"` FilterMode string `json:"filterMode"` UsabilityMode string `json:"usabilityMode"` AddressContains string `json:"addressContains"` @@ -88,7 +87,6 @@ type ( // Option types. type ( HostOptions struct { - AutopilotID string AddressContains string FilterMode string UsabilityMode string @@ -101,19 +99,19 @@ type ( type ( Host struct { - KnownSince time.Time `json:"knownSince"` - LastAnnouncement time.Time `json:"lastAnnouncement"` - PublicKey types.PublicKey `json:"publicKey"` - NetAddress string `json:"netAddress"` - PriceTable HostPriceTable `json:"priceTable"` - Settings rhpv2.HostSettings `json:"settings"` - Interactions HostInteractions `json:"interactions"` - Scanned bool `json:"scanned"` - Blocked bool `json:"blocked"` - Checks map[string]HostCheck `json:"checks"` - StoredData uint64 `json:"storedData"` - ResolvedAddresses []string `json:"resolvedAddresses"` - Subnets []string `json:"subnets"` + KnownSince time.Time `json:"knownSince"` + LastAnnouncement time.Time `json:"lastAnnouncement"` + PublicKey types.PublicKey `json:"publicKey"` + NetAddress string `json:"netAddress"` + PriceTable HostPriceTable `json:"priceTable"` + Settings rhpv2.HostSettings `json:"settings"` + Interactions HostInteractions `json:"interactions"` + Scanned bool `json:"scanned"` + Blocked bool `json:"blocked"` + Checks HostChecks `json:"checks,omitempty"` + StoredData uint64 `json:"storedData"` + ResolvedAddresses []string `json:"resolvedAddresses"` + Subnets []string `json:"subnets"` } HostInfo struct { @@ -156,7 +154,7 @@ type ( PriceTable HostPriceTable `json:"priceTable"` } - HostCheck struct { + HostChecks struct { GougingBreakdown HostGougingBreakdown `json:"gougingBreakdown"` ScoreBreakdown HostScoreBreakdown `json:"scoreBreakdown"` UsabilityBreakdown HostUsabilityBreakdown `json:"usabilityBreakdown"` @@ -192,8 +190,8 @@ type ( } ) -func (hc HostCheck) MarshalJSON() ([]byte, error) { - type check HostCheck +func (hc HostChecks) MarshalJSON() ([]byte, error) { + type check HostChecks return json.Marshal(struct { check Score float64 `json:"score"` diff --git a/api/host_test.go b/api/host_test.go index e2711723e..3e8f4b09a 100644 --- a/api/host_test.go +++ b/api/host_test.go @@ -7,7 +7,7 @@ import ( ) func TestMarshalHostScoreBreakdownJSON(t *testing.T) { - hc := HostCheck{ + hc := HostChecks{ ScoreBreakdown: HostScoreBreakdown{ Age: 1.1, Collateral: 1.1, diff --git a/api/prometheus.go b/api/prometheus.go index dea6b6145..4dc497847 100644 --- a/api/prometheus.go +++ b/api/prometheus.go @@ -50,9 +50,9 @@ func (asr AutopilotStateResponse) PrometheusMetric() (metrics []prometheus.Metri Value: float64(asr.UptimeMS), }, { - Name: "renterd_autopilot_state_configured", + Name: "renterd_autopilot_state_enabled", Labels: labels, - Value: boolToFloat(asr.Configured), + Value: boolToFloat(asr.Enabled), }, { Name: "renterd_autopilot_state_migrating", diff --git a/autopilot/autopilot.go b/autopilot/autopilot.go index b9768d589..61565c0a8 100644 --- a/autopilot/autopilot.go +++ b/autopilot/autopilot.go @@ -6,7 +6,6 @@ import ( "fmt" "net/http" "runtime" - "strings" "sync" "time" @@ -30,12 +29,12 @@ type Bus interface { alerts.Alerter webhooks.Broadcaster - // Accounts + // accounts Accounts(ctx context.Context, owner string) (accounts []api.Account, err error) - // Autopilots - Autopilot(ctx context.Context, id string) (autopilot api.Autopilot, err error) - UpdateAutopilot(ctx context.Context, autopilot api.Autopilot) error + // autopilot + Autopilot(ctx context.Context) (api.Autopilot, error) + UpdateCurrentPeriod(ctx context.Context, period uint64) error // consensus ConsensusNetwork(ctx context.Context) (consensus.Network, error) @@ -59,7 +58,7 @@ type Bus interface { Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) RemoveOfflineHosts(ctx context.Context, maxConsecutiveScanFailures uint64, maxDowntime time.Duration) (uint64, error) - UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) error + UpdateHostCheck(ctx context.Context, hostKey types.PublicKey, hostCheck api.HostChecks) error // metrics RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error @@ -95,8 +94,6 @@ type Bus interface { } type Autopilot struct { - id string - alerts alerts.Alerter bus Bus logger *zap.SugaredLogger @@ -126,11 +123,10 @@ type Autopilot struct { // New initializes an Autopilot. func New(cfg config.Autopilot, bus Bus, workers []Worker, logger *zap.Logger) (_ *Autopilot, err error) { - logger = logger.Named("autopilot").Named(cfg.ID) + logger = logger.Named("autopilot") shutdownCtx, shutdownCtxCancel := context.WithCancel(context.Background()) ap := &Autopilot{ - alerts: alerts.WithOrigin(bus, fmt.Sprintf("autopilot.%s", cfg.ID)), - id: cfg.ID, + alerts: alerts.WithOrigin(bus, "autopilot"), bus: bus, logger: logger.Sugar(), workers: newWorkerPool(workers), @@ -154,22 +150,16 @@ func New(cfg config.Autopilot, bus Bus, workers []Worker, logger *zap.Logger) (_ return ap, nil } -func (ap *Autopilot) Config(ctx context.Context) (api.Autopilot, error) { - return ap.bus.Autopilot(ctx, ap.id) -} - // Handler returns an HTTP handler that serves the autopilot api. func (ap *Autopilot) Handler() http.Handler { return jape.Mux(map[string]jape.Handler{ - "GET /config": ap.configHandlerGET, - "PUT /config": ap.configHandlerPUT, - "POST /config": ap.configHandlerPOST, - "GET /state": ap.stateHandlerGET, - "POST /trigger": ap.triggerHandlerPOST, + "POST /config/evaluate": ap.configEvaluateHandlerPOST, + "GET /state": ap.stateHandlerGET, + "POST /trigger": ap.triggerHandlerPOST, }) } -func (ap *Autopilot) configHandlerPOST(jc jape.Context) { +func (ap *Autopilot) configEvaluateHandlerPOST(jc jape.Context) { ctx := jc.Request.Context() // decode request @@ -260,25 +250,25 @@ func (ap *Autopilot) Run() { } } - // block until the autopilot is configured - if configured, interrupted := ap.blockUntilConfigured(ap.ticker.C); !configured { + // block until the autopilot is enabled + if enabled, interrupted := ap.blockUntilEnabled(ap.ticker.C); !enabled { if interrupted { close(tickerFired) return } - ap.logger.Info("autopilot stopped before it was able to confirm it was configured in the bus") + ap.logger.Info("autopilot stopped before it was able to confirm it was enabled in the bus") return } - // fetch configuration - autopilot, err := ap.Config(ap.shutdownCtx) + // fetch autopilot + autopilot, err := ap.bus.Autopilot(ap.shutdownCtx) if err != nil { - ap.logger.Errorf("aborting maintenance, failed to fetch autopilot config", zap.Error(err)) + ap.logger.Errorf("aborting maintenance, failed to fetch autopilot", zap.Error(err)) return } // update the scanner with the hosts config - ap.s.UpdateHostsConfig(autopilot.Config.Hosts) + ap.s.UpdateHostsConfig(autopilot.Hosts) // Log worker id chosen for this maintenance iteration. workerID, err := w.ID(ap.shutdownCtx) @@ -295,14 +285,14 @@ func (ap *Autopilot) Run() { } // build maintenance state - state, err := ap.buildState(ap.shutdownCtx) + buildState, err := ap.buildState(ap.shutdownCtx) if err != nil { ap.logger.Errorf("aborting maintenance, failed to build state, err: %v", err) return } // perform maintenance - setChanged, err := ap.c.PerformContractMaintenance(ap.shutdownCtx, state) + setChanged, err := ap.c.PerformContractMaintenance(ap.shutdownCtx, buildState) if err != nil && utils.IsErr(err, context.Canceled) { return } else if err != nil { @@ -320,7 +310,7 @@ func (ap *Autopilot) Run() { ap.m.tryPerformMigrations(ap.workers) // pruning - if autopilot.Config.Contracts.Prune { + if autopilot.Contracts.Prune { ap.tryPerformPruning() } else { ap.logger.Info("pruning disabled") @@ -382,27 +372,20 @@ func (ap *Autopilot) Uptime() (dur time.Duration) { return } -func (ap *Autopilot) blockUntilConfigured(interrupt <-chan time.Time) (configured, interrupted bool) { +func (ap *Autopilot) blockUntilEnabled(interrupt <-chan time.Time) (enabled, interrupted bool) { ticker := time.NewTicker(time.Second) defer ticker.Stop() var once sync.Once for { - // try and fetch the config - ctx, cancel := context.WithTimeout(ap.shutdownCtx, 30*time.Second) - _, err := ap.bus.Autopilot(ctx, ap.id) - cancel() - - // if the config was not found, or we were unable to fetch it, keep blocking - if utils.IsErr(err, context.Canceled) { - return - } else if utils.IsErr(err, api.ErrAutopilotNotFound) { - once.Do(func() { ap.logger.Info("autopilot is waiting to be configured...") }) - } else if err != nil { - ap.logger.Errorf("autopilot is unable to fetch its configuration from the bus, err: %v", err) + autopilot, err := ap.bus.Autopilot(ap.shutdownCtx) + if err != nil && !errors.Is(err, context.Canceled) { + ap.logger.Errorf("unable to fetch autopilot from the bus, err: %v", err) } - if err != nil { + + if err != nil || !autopilot.Enabled { + once.Do(func() { ap.logger.Info("autopilot is waiting to be enabled...") }) select { case <-ap.shutdownCtx.Done(): return false, false @@ -553,9 +536,9 @@ func (ap *Autopilot) performWalletMaintenance(ctx context.Context) error { ap.logger.Info("performing wallet maintenance") - autopilot, err := ap.Config(ctx) + autopilot, err := ap.bus.Autopilot(ctx) if err != nil { - return fmt.Errorf("failed to fetch autopilot config: %w", err) + return fmt.Errorf("failed to fetch autopilot: %w", err) } w, err := ap.bus.Wallet(ctx) if err != nil { @@ -565,7 +548,7 @@ func (ap *Autopilot) performWalletMaintenance(ctx context.Context) error { // convenience variables b := ap.bus l := ap.logger - cfg := autopilot.Config + cfg := autopilot.AutopilotConfig renewWindow := cfg.Contracts.RenewWindow // no contracts - nothing to do @@ -625,56 +608,6 @@ func (ap *Autopilot) performWalletMaintenance(ctx context.Context) error { return nil } -func (ap *Autopilot) configHandlerGET(jc jape.Context) { - autopilot, err := ap.bus.Autopilot(jc.Request.Context(), ap.id) - if utils.IsErr(err, api.ErrAutopilotNotFound) { - jc.Error(errors.New("autopilot is not configured yet"), http.StatusNotFound) - return - } - - if jc.Check("failed to get autopilot config", err) == nil { - jc.Encode(autopilot.Config) - } -} - -func (ap *Autopilot) configHandlerPUT(jc jape.Context) { - // decode and validate the config - var cfg api.AutopilotConfig - if jc.Decode(&cfg) != nil { - return - } else if err := cfg.Validate(); jc.Check("invalid autopilot config", err) != nil { - return - } - - // fetch the autopilot and update its config - var contractSetChanged bool - autopilot, err := ap.bus.Autopilot(jc.Request.Context(), ap.id) - if utils.IsErr(err, api.ErrAutopilotNotFound) { - autopilot = api.Autopilot{ID: ap.id, Config: cfg} - } else if err != nil { - jc.Error(err, http.StatusInternalServerError) - return - } else { - if autopilot.Config.Contracts.Set != cfg.Contracts.Set { - contractSetChanged = true - } - autopilot.Config = cfg - } - - // update the autopilot - if jc.Check("failed to update autopilot config", ap.bus.UpdateAutopilot(jc.Request.Context(), autopilot)) != nil { - return - } - - // update the scanner with the hosts config - ap.s.UpdateHostsConfig(cfg.Hosts) - - // interrupt migrations if necessary - if contractSetChanged { - ap.m.SignalMaintenanceFinished() - } -} - func (ap *Autopilot) triggerHandlerPOST(jc jape.Context) { var req api.AutopilotTriggerRequest if jc.Decode(&req) != nil { @@ -691,15 +624,15 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { ap.mu.Unlock() migrating, mLastStart := ap.m.Status() scanning, sLastStart := ap.s.Status() - _, err := ap.bus.Autopilot(jc.Request.Context(), ap.id) - if err != nil && !strings.Contains(err.Error(), api.ErrAutopilotNotFound.Error()) { + + autopilot, err := ap.bus.Autopilot(jc.Request.Context()) + if err != nil { jc.Error(err, http.StatusInternalServerError) return } jc.Encode(api.AutopilotStateResponse{ - ID: ap.id, - Configured: err == nil, + Enabled: autopilot.Enabled, Migrating: migrating, MigratingLastStart: api.TimeRFC3339(mLastStart), Pruning: pruning, @@ -719,16 +652,18 @@ func (ap *Autopilot) stateHandlerGET(jc jape.Context) { } func (ap *Autopilot) buildState(ctx context.Context) (*contractor.MaintenanceState, error) { - // fetch the autopilot from the bus - autopilot, err := ap.Config(ctx) + // fetch autopilot + autopilot, err := ap.bus.Autopilot(ctx) if err != nil { - return nil, err + return nil, fmt.Errorf("could not fetch autopilot, err: %v", err) } // fetch consensus state cs, err := ap.bus.ConsensusState(ctx) if err != nil { return nil, fmt.Errorf("could not fetch consensus state, err: %v", err) + } else if !cs.Synced { + return nil, errors.New("consensus not synced") } // fetch upload settings @@ -763,23 +698,24 @@ func (ap *Autopilot) buildState(ctx context.Context) (*contractor.MaintenanceSta } // update current period if necessary - if cs.Synced { + if cs.BlockHeight > 0 { if autopilot.CurrentPeriod == 0 { - autopilot.CurrentPeriod = cs.BlockHeight - err := ap.bus.UpdateAutopilot(ctx, autopilot) + err := ap.bus.UpdateCurrentPeriod(ctx, cs.BlockHeight) if err != nil { return nil, err } + autopilot.CurrentPeriod = cs.BlockHeight ap.logger.Infof("initialised current period to %d", autopilot.CurrentPeriod) - } else if nextPeriod := computeNextPeriod(cs.BlockHeight, autopilot.CurrentPeriod, autopilot.Config.Contracts.Period); nextPeriod != autopilot.CurrentPeriod { - prevPeriod := autopilot.CurrentPeriod - autopilot.CurrentPeriod = nextPeriod - err := ap.bus.UpdateAutopilot(ctx, autopilot) + } else if nextPeriod := computeNextPeriod(cs.BlockHeight, autopilot.CurrentPeriod, autopilot.Contracts.Period); nextPeriod != autopilot.CurrentPeriod { + err := ap.bus.UpdateCurrentPeriod(ctx, nextPeriod) if err != nil { return nil, err } - ap.logger.Infof("updated current period from %d to %d", prevPeriod, nextPeriod) + ap.logger.Infof("updated current period from %d to %d", autopilot.CurrentPeriod, nextPeriod) + autopilot.CurrentPeriod = nextPeriod } + } else if !skipContractFormations { + skipContractFormations = true } return &contractor.MaintenanceState{ diff --git a/autopilot/client.go b/autopilot/client.go index 05592662b..68aae4223 100644 --- a/autopilot/client.go +++ b/autopilot/client.go @@ -20,17 +20,6 @@ func NewClient(addr, password string) *Client { }} } -// Config returns the autopilot config. -func (c *Client) Config() (cfg api.AutopilotConfig, err error) { - err = c.c.GET("/config", &cfg) - return -} - -// UpdateConfig updates the autopilot config. -func (c *Client) UpdateConfig(cfg api.AutopilotConfig) error { - return c.c.PUT("/config", cfg) -} - // State returns the current state of the autopilot. func (c *Client) State() (state api.AutopilotStateResponse, err error) { err = c.c.GET("/state", &state) @@ -47,7 +36,7 @@ func (c *Client) Trigger(forceScan bool) (_ bool, err error) { // EvalutateConfig evaluates an autopilot config using the given gouging and // redundancy settings. func (c *Client) EvaluateConfig(ctx context.Context, cfg api.AutopilotConfig, gs api.GougingSettings, rs api.RedundancySettings) (resp api.ConfigEvaluationResponse, err error) { - err = c.c.WithContext(ctx).POST("/config", api.ConfigEvaluationRequest{ + err = c.c.WithContext(ctx).POST("/config/evaluate", api.ConfigEvaluationRequest{ AutopilotConfig: cfg, GougingSettings: gs, RedundancySettings: rs, diff --git a/autopilot/contract_pruning.go b/autopilot/contract_pruning.go index 0189430ec..4ba150fa9 100644 --- a/autopilot/contract_pruning.go +++ b/autopilot/contract_pruning.go @@ -63,13 +63,13 @@ func (ap *Autopilot) fetchPrunableContracts() (prunable []api.ContractPrunableDa } // fetch autopilot - autopilot, err := ap.bus.Autopilot(ctx, ap.id) + autopilot, err := ap.bus.Autopilot(ctx) if err != nil { return nil, err } // fetch contract set contracts - csc, err := ap.bus.Contracts(ctx, api.ContractsOpts{ContractSet: autopilot.Config.Contracts.Set}) + csc, err := ap.bus.Contracts(ctx, api.ContractsOpts{ContractSet: autopilot.Contracts.Set}) if err != nil { return nil, err } diff --git a/autopilot/contractor/contract_test.go b/autopilot/contractor/contract_test.go deleted file mode 100644 index 9b71470fe..000000000 --- a/autopilot/contractor/contract_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package contractor - -import ( - "reflect" - "testing" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" -) - -func TestSortContractsForMaintenance(t *testing.T) { - set := "testset" - cfg := api.ContractsConfig{ - Set: set, - } - - // empty but in set - c1 := contract{ - ContractMetadata: api.ContractMetadata{ - ID: types.FileContractID{1}, - Size: 0, - ContractSets: []string{set}, - }, - } - // some data and in set - c2 := contract{ - ContractMetadata: api.ContractMetadata{ - ID: types.FileContractID{2}, - Size: 10, - ContractSets: []string{set}, - }, - } - // same as c2 - sort should be stable - c3 := contract{ - ContractMetadata: api.ContractMetadata{ - ID: types.FileContractID{3}, - Size: 10, - ContractSets: []string{set}, - }, - } - // more data but not in set - c4 := contract{ - ContractMetadata: api.ContractMetadata{ - ID: types.FileContractID{4}, - Size: 20, - }, - } - // even more data but not in set - c5 := contract{ - ContractMetadata: api.ContractMetadata{ - ID: types.FileContractID{5}, - Size: 30, - }, - } - - contracts := []contract{c1, c2, c3, c4, c5} - sortContractsForMaintenance(cfg, contracts) - - if !reflect.DeepEqual(contracts, []contract{c2, c3, c1, c5, c4}) { - t.Fatal("unexpected sort order") - } -} diff --git a/autopilot/contractor/contractor.go b/autopilot/contractor/contractor.go index 266558ae2..292dd76e0 100644 --- a/autopilot/contractor/contractor.go +++ b/autopilot/contractor/contractor.go @@ -91,7 +91,7 @@ type Bus interface { Hosts(ctx context.Context, opts api.HostOptions) ([]api.Host, error) RecordContractSetChurnMetric(ctx context.Context, metrics ...api.ContractSetChurnMetric) error UpdateContractSet(ctx context.Context, set string, toAdd, toRemove []types.FileContractID) error - UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) error + UpdateHostCheck(ctx context.Context, hostKey types.PublicKey, hostCheck api.HostChecks) error } type HostScanner interface { @@ -322,7 +322,7 @@ func (c *Contractor) renewContract(ctx *mCtx, contract contract, host api.Host, // sanity check the endheight is not the same on renewals endHeight := ctx.EndHeight() if endHeight <= rev.ProofHeight { - logger.Infow("invalid renewal endheight", "oldEndheight", rev.EndHeight(), "newEndHeight", endHeight, "period", ctx.state.Period, "bh", cs.BlockHeight) + logger.Infow("invalid renewal endheight", "oldEndheight", rev.EndHeight(), "newEndHeight", endHeight, "period", ctx.state.ContractsConfig().Period, "bh", cs.BlockHeight) return api.ContractMetadata{}, false, fmt.Errorf("renewal endheight should surpass the current contract endheight, %v <= %v", endHeight, rev.EndHeight()) } @@ -811,7 +811,7 @@ func performContractChecks(ctx *mCtx, alerter alerts.Alerter, bus Bus, cc contra logger.With("contracts", len(contracts)).Info("checking existing contracts") var renewed, refreshed int for _, c := range contracts { - inSet := c.InSet(ctx.Set()) + inSet := c.InSet(ctx.ContractSet()) logger := logger.With("contractID", c.ID). With("inSet", inSet). @@ -881,8 +881,7 @@ func performContractChecks(ctx *mCtx, alerter alerts.Alerter, bus Bus, cc contra } // get check - check, ok := host.Checks[ctx.ApID()] - if !ok { + if host.Checks == (api.HostChecks{}) { logger.Warn("missing host check") churnReasons[c.ID] = api.ErrUsabilityHostNotFound.Error() continue @@ -891,15 +890,15 @@ func performContractChecks(ctx *mCtx, alerter alerts.Alerter, bus Bus, cc contra // NOTE: if we have a contract with a host that is not scanned, we either // added the host and contract manually or reset the host scans. In that case, // we ignore the fact that the host is not scanned for now to avoid churn. - if inSet && check.UsabilityBreakdown.NotCompletingScan { + if inSet && host.Checks.UsabilityBreakdown.NotCompletingScan { keepContract(c.ContractMetadata, host) logger.Info("ignoring contract with unscanned host") continue // no more checks until host is scanned } // check usability - if !check.UsabilityBreakdown.IsUsable() { - reasons := strings.Join(check.UsabilityBreakdown.UnusableReasons(), ",") + if !host.Checks.UsabilityBreakdown.IsUsable() { + reasons := strings.Join(host.Checks.UsabilityBreakdown.UnusableReasons(), ",") logger.With("reasons", reasons).Info("unusable host") churnReasons[c.ID] = reasons continue @@ -1031,7 +1030,6 @@ func performContractFormations(ctx *mCtx, bus Bus, cr contractReviser, ipFilter usedHosts[c.HostKey] = struct{}{} } allHosts, err := bus.Hosts(ctx, api.HostOptions{ - AutopilotID: ctx.ApID(), FilterMode: api.HostFilterModeAllowed, UsabilityMode: api.UsabilityFilterModeUsable, }) @@ -1043,18 +1041,18 @@ func performContractFormations(ctx *mCtx, bus Bus, cr contractReviser, ipFilter var candidates scoredHosts for _, host := range allHosts { logger := logger.With("hostKey", host.PublicKey) - hc, ok := host.Checks[ctx.ApID()] - if !ok { - logger.Warn("missing host check") + if host.Checks == (api.HostChecks{}) { + logger.Warnf("missing host check %v", host.PublicKey) continue - } else if _, used := usedHosts[host.PublicKey]; used { + } + if _, used := usedHosts[host.PublicKey]; used { logger.Debug("host already used") continue - } else if score := hc.ScoreBreakdown.Score(); score == 0 { + } else if score := host.Checks.ScoreBreakdown.Score(); score == 0 { logger.Error("host has a score of 0") continue } - candidates = append(candidates, newScoredHost(host, hc.ScoreBreakdown)) + candidates = append(candidates, newScoredHost(host, host.Checks.ScoreBreakdown)) } logger = logger.With("candidates", len(candidates)) @@ -1140,7 +1138,7 @@ func performHostChecks(ctx *mCtx, bus Bus, logger *zap.SugaredLogger) error { for _, h := range scoredHosts { h.host.PriceTable.HostBlockHeight = cs.BlockHeight // ignore HostBlockHeight hc := checkHost(ctx.GougingChecker(cs), h, minScore) - if err := bus.UpdateHostCheck(ctx, ctx.ApID(), h.host.PublicKey, *hc); err != nil { + if err := bus.UpdateHostCheck(ctx, h.host.PublicKey, *hc); err != nil { return fmt.Errorf("failed to update host check for host %v: %w", h.host.PublicKey, err) } usabilityBreakdown.track(hc.UsabilityBreakdown) diff --git a/autopilot/contractor/evaluate.go b/autopilot/contractor/evaluate.go index 70ce3ea5c..cb186fbd5 100644 --- a/autopilot/contractor/evaluate.go +++ b/autopilot/contractor/evaluate.go @@ -8,7 +8,7 @@ import ( "go.sia.tech/renterd/internal/gouging" ) -var ErrMissingRequiredFields = errors.New("missing required fields in configuration, both allowance and amount must be set") +var ErrMissingRequiredFields = errors.New("missing required fields in configuration, amount must be set") func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, period uint64, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (usables uint64) { gc := gouging.NewChecker(gs, cs, &period, &cfg.Contracts.RenewWindow) @@ -25,7 +25,7 @@ func countUsableHosts(cfg api.AutopilotConfig, cs api.ConsensusState, period uin // are too strict for the number of contracts required by 'cfg', it will provide // a recommendation on how to loosen it. func EvaluateConfig(cfg api.AutopilotConfig, cs api.ConsensusState, rs api.RedundancySettings, gs api.GougingSettings, hosts []api.Host) (resp api.ConfigEvaluationResponse, _ error) { - // we need an allowance and a target amount of contracts to evaluate + // we need an amount of contracts to evaluate if cfg.Contracts.Amount == 0 { return api.ConfigEvaluationResponse{}, ErrMissingRequiredFields } diff --git a/autopilot/contractor/evaluate_test.go b/autopilot/contractor/evaluate_test.go index 750bb1621..15531e6c6 100644 --- a/autopilot/contractor/evaluate_test.go +++ b/autopilot/contractor/evaluate_test.go @@ -40,7 +40,6 @@ func TestOptimiseGougingSetting(t *testing.T) { LastAnnouncement: time.Unix(0, 0), Scanned: true, Blocked: false, - Checks: nil, }) } diff --git a/autopilot/contractor/hostfilter.go b/autopilot/contractor/hostfilter.go index 4a3e93b04..78b857513 100644 --- a/autopilot/contractor/hostfilter.go +++ b/autopilot/contractor/hostfilter.go @@ -221,7 +221,7 @@ func isUpForRenewal(cfg api.AutopilotConfig, r api.Revision, blockHeight uint64) } // checkHost performs a series of checks on the host. -func checkHost(gc gouging.Checker, sh scoredHost, minScore float64) *api.HostCheck { +func checkHost(gc gouging.Checker, sh scoredHost, minScore float64) *api.HostChecks { h := sh.host // prepare host breakdown fields @@ -258,7 +258,7 @@ func checkHost(gc gouging.Checker, sh scoredHost, minScore float64) *api.HostChe } } - return &api.HostCheck{ + return &api.HostChecks{ UsabilityBreakdown: ub, GougingBreakdown: gb, ScoreBreakdown: sh.sb, diff --git a/autopilot/contractor/hosts_test.go b/autopilot/contractor/hosts_test.go deleted file mode 100644 index 9ca1fa6bd..000000000 --- a/autopilot/contractor/hosts_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package contractor - -import ( - "math" - "testing" - - "go.sia.tech/core/types" - "go.sia.tech/renterd/api" - "lukechampine.com/frand" -) - -func TestScoredHostsRandSelectByScore(t *testing.T) { - hostToScores := map[types.PublicKey]float64{ - {1}: math.SmallestNonzeroFloat64, - {2}: .1, - {3}: .2, - } - - var hosts scoredHosts - for hk, score := range hostToScores { - hosts = append(hosts, scoredHost{score: score, host: api.Host{PublicKey: hk}}) - } - - for i := 0; i < 1000; i++ { - seen := make(map[types.PublicKey]struct{}) - for _, h := range hosts.randSelectByScore(3) { - // assert we get non-normalized scores - if hostToScores[h.host.PublicKey] != h.score { - t.Fatal("unexpected") - } - - // assert we never select the same host twice - if _, seen := seen[h.host.PublicKey]; seen { - t.Fatal("unexpected") - } - seen[h.host.PublicKey] = struct{}{} - } - - // assert min float is never selected - frand.Shuffle(len(hosts), func(i, j int) { hosts[i], hosts[j] = hosts[j], hosts[i] }) - if hosts.randSelectByScore(1)[0].score == math.SmallestNonzeroFloat64 { - t.Fatal("unexpected") - } - } - - // assert we can pass any value for n - if len(hosts.randSelectByScore(0)) != 0 { - t.Fatal("unexpected") - } else if len(hosts.randSelectByScore(-1)) != 0 { - t.Fatal("unexpected") - } else if len(hosts.randSelectByScore(4)) != 3 { - t.Fatal("unexpected") - } - - // assert select is random on equal inputs, we calculate the chi-square - // statistic and assert it's less than critical value of 10.828 (1 degree of - // freedom, using alpha of 0.001) - var counts [2]int - hosts = scoredHosts{ - {score: .1, host: api.Host{PublicKey: types.PublicKey{1}}}, - {score: .1, host: api.Host{PublicKey: types.PublicKey{2}}}, - } - nRuns := 1e5 - for i := 0; i < int(nRuns); i++ { - if hosts.randSelectByScore(1)[0].host.PublicKey == (types.PublicKey{1}) { - counts[0]++ - } else { - counts[1]++ - } - } - var chi2 float64 - for i := 0; i < 2; i++ { - chi2 += math.Pow(float64(counts[i])-nRuns/2, 2) / (nRuns / 2) - } - if chi2 > 10.828 { - t.Fatal("unexpected", counts[0], counts[1], chi2) - } -} diff --git a/autopilot/contractor/hostscore_test.go b/autopilot/contractor/hostscore_test.go index 1407d5de9..5660dfe67 100644 --- a/autopilot/contractor/hostscore_test.go +++ b/autopilot/contractor/hostscore_test.go @@ -21,8 +21,6 @@ var cfg = api.AutopilotConfig{ Download: 1e12, // 1 TB Upload: 1e12, // 1 TB Storage: 4e12, // 4 TB - - Set: api.DefaultAutopilotID, }, Hosts: api.HostsConfig{ MaxDowntimeHours: 24 * 7 * 2, diff --git a/autopilot/contractor/state.go b/autopilot/contractor/state.go index 673c0171d..ceb95b802 100644 --- a/autopilot/contractor/state.go +++ b/autopilot/contractor/state.go @@ -37,12 +37,8 @@ func newMaintenanceCtx(ctx context.Context, state *MaintenanceState) *mCtx { } } -func (ctx *mCtx) ApID() string { - return ctx.state.AP.ID -} - func (ctx *mCtx) AutopilotConfig() api.AutopilotConfig { - return ctx.state.AP.Config + return ctx.state.AP.AutopilotConfig } func (ctx *mCtx) ContractsConfig() api.ContractsConfig { @@ -50,7 +46,7 @@ func (ctx *mCtx) ContractsConfig() api.ContractsConfig { } func (ctx *mCtx) ContractSet() string { - return ctx.state.AP.Config.Contracts.Set + return ctx.state.ContractsConfig().Set } func (ctx *mCtx) Deadline() (deadline time.Time, ok bool) { @@ -62,7 +58,7 @@ func (ctx *mCtx) Done() <-chan struct{} { } func (ctx *mCtx) EndHeight() uint64 { - return ctx.state.AP.EndHeight() + return ctx.state.AP.CurrentPeriod + ctx.state.AP.Contracts.Period + ctx.state.AP.Contracts.RenewWindow } func (ctx *mCtx) Err() error { @@ -81,19 +77,19 @@ func (ctx *mCtx) HostScore(h api.Host) (sb api.HostScoreBreakdown, err error) { err = errors.New("panic while scoring host") } }() - return hostScore(ctx.state.AP.Config, ctx.state.GS, h, ctx.state.RS.Redundancy()), nil + return hostScore(ctx.AutopilotConfig(), ctx.state.GS, h, ctx.state.RS.Redundancy()), nil } func (ctx *mCtx) Period() uint64 { - return ctx.state.Period() + return ctx.state.AP.Contracts.Period } func (ctx *mCtx) RenewWindow() uint64 { - return ctx.state.AP.Config.Contracts.RenewWindow + return ctx.state.AP.Contracts.RenewWindow } func (ctx *mCtx) ShouldFilterRedundantIPs() bool { - return !ctx.state.AP.Config.Hosts.AllowRedundantIPs + return !ctx.state.AP.Hosts.AllowRedundantIPs } func (ctx *mCtx) Value(key interface{}) interface{} { @@ -101,11 +97,7 @@ func (ctx *mCtx) Value(key interface{}) interface{} { } func (ctx *mCtx) WantedContracts() uint64 { - return ctx.state.AP.Config.Contracts.Amount -} - -func (ctx *mCtx) Set() string { - return ctx.state.ContractsConfig().Set + return ctx.state.AP.Contracts.Amount } func (ctx *mCtx) SortContractsForMaintenance(contracts []contract) { @@ -113,9 +105,5 @@ func (ctx *mCtx) SortContractsForMaintenance(contracts []contract) { } func (state *MaintenanceState) ContractsConfig() api.ContractsConfig { - return state.AP.Config.Contracts -} - -func (state *MaintenanceState) Period() uint64 { - return state.AP.Config.Contracts.Period + return state.AP.Contracts } diff --git a/autopilot/migrator.go b/autopilot/migrator.go index 01e0c9f20..ec038db83 100644 --- a/autopilot/migrator.go +++ b/autopilot/migrator.go @@ -179,12 +179,12 @@ func (m *migrator) performMigrations(p *workerPool) { } // fetch currently configured set - autopilot, err := m.ap.Config(m.ap.shutdownCtx) + ap, err := m.ap.bus.Autopilot(m.ap.shutdownCtx) if err != nil { - m.logger.Errorf("failed to fetch autopilot config: %w", err) + m.logger.Errorf("failed to fetch autopilot: %w", err) return } - set := autopilot.Config.Contracts.Set + set := ap.Contracts.Set if set == "" { m.logger.Error("could not perform migrations, no contract set configured") return diff --git a/bus/bus.go b/bus/bus.go index 9c9c67ff4..e9d8770cd 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -181,10 +181,10 @@ type ( SaveAccounts(context.Context, []api.Account) error } - // An AutopilotStore stores autopilots. + // A AutopilotStore stores autopilot state. AutopilotStore interface { - Autopilot(ctx context.Context, id string) (api.Autopilot, error) - Autopilots(ctx context.Context) ([]api.Autopilot, error) + Autopilot(ctx context.Context) (api.Autopilot, error) + InitAutopilot(ctx context.Context) error UpdateAutopilot(ctx context.Context, ap api.Autopilot) error } @@ -211,7 +211,7 @@ type ( ResetLostSectors(ctx context.Context, hk types.PublicKey) error UpdateHostAllowlistEntries(ctx context.Context, add, remove []types.PublicKey, clear bool) error UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error - UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, check api.HostCheck) error + UpdateHostCheck(ctx context.Context, hk types.PublicKey, check api.HostChecks) error UsableHosts(ctx context.Context) ([]sql.HostInfo, error) } @@ -366,6 +366,12 @@ func New(ctx context.Context, cfg config.Bus, masterKey [32]byte, am AlertManage rhp4Client: rhp4.New(dialer), } + // ensure autopilot state + err = store.InitAutopilot(ctx) + if err != nil { + return nil, err + } + // create contract locker b.contractLocker = ibus.NewContractLocker() @@ -396,11 +402,8 @@ func (b *Bus) Handler() http.Handler { "POST /alerts/dismiss": b.handlePOSTAlertsDismiss, "POST /alerts/register": b.handlePOSTAlertsRegister, - "GET /autopilots": b.autopilotsListHandlerGET, - "GET /autopilot/:id": b.autopilotsHandlerGET, - "PUT /autopilot/:id": b.autopilotsHandlerPUT, - - "PUT /autopilot/:id/host/:hostkey/check": b.autopilotHostCheckHandlerPUT, + "GET /autopilot": b.autopilotHandlerGET, + "PUT /autopilot": b.autopilotHandlerPUT, "GET /buckets": b.bucketsHandlerGET, "POST /buckets": b.bucketsHandlerPOST, @@ -445,6 +448,7 @@ func (b *Bus) Handler() http.Handler { "PUT /hosts/blocklist": b.hostsBlocklistHandlerPUT, "POST /hosts/remove": b.hostsRemoveHandlerPOST, "GET /host/:hostkey": b.hostsPubkeyHandlerGET, + "PUT /host/:hostkey/check": b.hostsCheckHandlerPUT, "POST /host/:hostkey/resetlostsectors": b.hostsResetLostSectorsPOST, "POST /host/:hostkey/scan": b.hostsScanHandlerPOST, diff --git a/bus/client/autopilot.go b/bus/client/autopilot.go new file mode 100644 index 000000000..87426a087 --- /dev/null +++ b/bus/client/autopilot.go @@ -0,0 +1,50 @@ +package client + +import ( + "context" + + "go.sia.tech/renterd/api" +) + +type UpdateAutopilotOption func(*api.UpdateAutopilotRequest) + +func WithAutopilotEnabled(enabled bool) UpdateAutopilotOption { + return func(req *api.UpdateAutopilotRequest) { + req.Enabled = &enabled + } +} +func WithContractsConfig(cfg api.ContractsConfig) UpdateAutopilotOption { + return func(req *api.UpdateAutopilotRequest) { + req.Contracts = &cfg + } +} +func WithCurrentPeriod(currentPeriod uint64) UpdateAutopilotOption { + return func(req *api.UpdateAutopilotRequest) { + req.CurrentPeriod = ¤tPeriod + } +} +func WithHostsConfig(cfg api.HostsConfig) UpdateAutopilotOption { + return func(req *api.UpdateAutopilotRequest) { + req.Hosts = &cfg + } +} + +// Autopilot returns the autopilot. +func (c *Client) Autopilot(ctx context.Context) (ap api.Autopilot, err error) { + err = c.c.WithContext(ctx).GET("/autopilot", &ap) + return +} + +// UpdateAutopilot updates the autopilot. +func (c *Client) UpdateAutopilot(ctx context.Context, opts ...UpdateAutopilotOption) error { + var req api.UpdateAutopilotRequest + for _, opt := range opts { + opt(&req) + } + return c.c.WithContext(ctx).PUT("/autopilot", req) +} + +// UpdateCurrentPeriod updates the current period. +func (c *Client) UpdateCurrentPeriod(ctx context.Context, currentPeriod uint64) error { + return c.UpdateAutopilot(ctx, WithCurrentPeriod(currentPeriod)) +} diff --git a/bus/client/autopilots.go b/bus/client/autopilots.go deleted file mode 100644 index 22c7cddfb..000000000 --- a/bus/client/autopilots.go +++ /dev/null @@ -1,26 +0,0 @@ -package client - -import ( - "context" - "fmt" - - "go.sia.tech/renterd/api" -) - -// Autopilot returns the autopilot with the given ID. -func (c *Client) Autopilot(ctx context.Context, id string) (autopilot api.Autopilot, err error) { - err = c.c.WithContext(ctx).GET(fmt.Sprintf("/autopilot/%s", id), &autopilot) - return -} - -// Autopilots returns all autopilots in the autopilots store. -func (c *Client) Autopilots(ctx context.Context) (autopilots []api.Autopilot, err error) { - err = c.c.WithContext(ctx).GET("/autopilots", &autopilots) - return -} - -// UpdateAutopilot updates the given autopilot in the store. -func (c *Client) UpdateAutopilot(ctx context.Context, autopilot api.Autopilot) (err error) { - err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s", autopilot.ID), autopilot) - return -} diff --git a/bus/client/hosts.go b/bus/client/hosts.go index 344e7a660..72324ba17 100644 --- a/bus/client/hosts.go +++ b/bus/client/hosts.go @@ -18,7 +18,6 @@ func (c *Client) Host(ctx context.Context, hostKey types.PublicKey) (h api.Host, // Hosts returns all hosts that match certain search criteria. func (c *Client) Hosts(ctx context.Context, opts api.HostOptions) (hosts []api.Host, err error) { err = c.c.WithContext(ctx).POST("/hosts", api.HostsRequest{ - AutopilotID: opts.AutopilotID, Offset: opts.Offset, Limit: opts.Limit, FilterMode: opts.FilterMode, @@ -71,8 +70,8 @@ func (c *Client) UpdateHostBlocklist(ctx context.Context, add, remove []string, // UpdateHostCheck updates the host with the most recent check performed by the // autopilot with given id. -func (c *Client) UpdateHostCheck(ctx context.Context, autopilotID string, hostKey types.PublicKey, hostCheck api.HostCheck) (err error) { - err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/autopilot/%s/host/%s/check", autopilotID, hostKey), hostCheck) +func (c *Client) UpdateHostCheck(ctx context.Context, hostKey types.PublicKey, hostCheck api.HostChecks) (err error) { + err = c.c.WithContext(ctx).PUT(fmt.Sprintf("/host/%s/check", hostKey), hostCheck) return } diff --git a/bus/routes.go b/bus/routes.go index 0050329e9..d0b3c9c30 100644 --- a/bus/routes.go +++ b/bus/routes.go @@ -523,11 +523,6 @@ func (b *Bus) hostsHandlerPOST(jc jape.Context) { return } - if req.AutopilotID == "" && req.UsabilityMode != api.UsabilityFilterModeAll { - jc.Error(errors.New("need to specify autopilot id when usability mode isn't 'all'"), http.StatusBadRequest) - return - } - // validate the filter mode switch req.FilterMode { case api.HostFilterModeAllowed: @@ -553,7 +548,6 @@ func (b *Bus) hostsHandlerPOST(jc jape.Context) { } hosts, err := b.store.Hosts(jc.Request.Context(), api.HostOptions{ - AutopilotID: req.AutopilotID, FilterMode: req.FilterMode, UsabilityMode: req.UsabilityMode, AddressContains: req.AddressContains, @@ -1754,6 +1748,61 @@ func (b *Bus) slabsPartialHandlerPOST(jc jape.Context) { }) } +func (b *Bus) autopilotHandlerGET(jc jape.Context) { + ap, err := b.store.Autopilot(jc.Request.Context()) + if jc.Check("failed to fetch autopilot", err) != nil { + return + } + jc.Encode(ap) +} + +func (b *Bus) autopilotHandlerPUT(jc jape.Context) { + // decode request + var req api.UpdateAutopilotRequest + if jc.Decode(&req) != nil { + return + } else if req == (api.UpdateAutopilotRequest{}) { + jc.Error(errors.New("request body is empty"), http.StatusBadRequest) + return + } + + // fetch the autopilot + ap, err := b.store.Autopilot(jc.Request.Context()) + if jc.Check("failed to fetch autopilot", err) != nil { + return + } + + // update the contracts config + if req.Contracts != nil { + if err := req.Contracts.Validate(); err != nil { + jc.Error(fmt.Errorf("failed to update autopilot, contracts config is invalid: %w", err), http.StatusBadRequest) + return + } + ap.Contracts = *req.Contracts + } + + // update the hosts config + if req.Hosts != nil { + if err := req.Hosts.Validate(); err != nil { + jc.Error(fmt.Errorf("failed to update autopilot, hosts config is invalid: %w", err), http.StatusBadRequest) + return + } + ap.Hosts = *req.Hosts + } + + // enable/disable the autopilot + if req.Enabled != nil { + ap.Enabled = *req.Enabled + } + + // update the current period + if req.CurrentPeriod != nil { + ap.CurrentPeriod = *req.CurrentPeriod + } + + jc.Check("failed to update autopilot", b.store.UpdateAutopilot(jc.Request.Context(), ap)) +} + func (b *Bus) contractIDAncestorsHandler(jc jape.Context) { var fcid types.FileContractID if jc.DecodeParam("id", &fcid) != nil { @@ -1936,69 +1985,18 @@ func (b *Bus) accountsHandlerPOST(jc jape.Context) { } } -func (b *Bus) autopilotsListHandlerGET(jc jape.Context) { - if autopilots, err := b.store.Autopilots(jc.Request.Context()); jc.Check("failed to fetch autopilots", err) == nil { - jc.Encode(autopilots) - } -} - -func (b *Bus) autopilotsHandlerGET(jc jape.Context) { - var id string - if jc.DecodeParam("id", &id) != nil { - return - } - ap, err := b.store.Autopilot(jc.Request.Context(), id) - if errors.Is(err, api.ErrAutopilotNotFound) { - jc.Error(err, http.StatusNotFound) - return - } - if jc.Check("couldn't load object", err) != nil { - return - } - - jc.Encode(ap) -} - -func (b *Bus) autopilotsHandlerPUT(jc jape.Context) { - var id string - if jc.DecodeParam("id", &id) != nil { - return - } - - var ap api.Autopilot - if jc.Decode(&ap) != nil { - return - } - - if ap.ID != id { - jc.Error(errors.New("id in path and body don't match"), http.StatusBadRequest) - return - } - - if jc.Check("failed to update autopilot", b.store.UpdateAutopilot(jc.Request.Context(), ap)) == nil { - b.pinMgr.TriggerUpdate() - } -} - -func (b *Bus) autopilotHostCheckHandlerPUT(jc jape.Context) { - var id string - if jc.DecodeParam("id", &id) != nil { - return - } +func (b *Bus) hostsCheckHandlerPUT(jc jape.Context) { var hk types.PublicKey if jc.DecodeParam("hostkey", &hk) != nil { return } - var hc api.HostCheck + var hc api.HostChecks if jc.Check("failed to decode host check", jc.Decode(&hc)) != nil { return } - err := b.store.UpdateHostCheck(jc.Request.Context(), id, hk, hc) - if errors.Is(err, api.ErrAutopilotNotFound) { - jc.Error(err, http.StatusNotFound) - return - } else if jc.Check("failed to update host", err) != nil { + err := b.store.UpdateHostCheck(jc.Request.Context(), hk, hc) + if jc.Check("failed to update host check", err) != nil { return } } diff --git a/cmd/renterd/config.go b/cmd/renterd/config.go index a1793bc00..00069ddb7 100644 --- a/cmd/renterd/config.go +++ b/cmd/renterd/config.go @@ -17,7 +17,6 @@ import ( "go.sia.tech/core/types" "go.sia.tech/coreutils/chain" "go.sia.tech/coreutils/wallet" - "go.sia.tech/renterd/api" "go.sia.tech/renterd/config" "golang.org/x/term" "gopkg.in/yaml.v3" @@ -110,7 +109,6 @@ func defaultConfig() config.Config { Autopilot: config.Autopilot{ Enabled: true, - ID: api.DefaultAutopilotID, RevisionSubmissionBuffer: 150, // 144 + 6 blocks leeway Heartbeat: 30 * time.Minute, MigrationHealthCutoff: 0.75, diff --git a/config/config.go b/config/config.go index 5eecee84f..13b5e20e6 100644 --- a/config/config.go +++ b/config/config.go @@ -136,7 +136,6 @@ type ( // Autopilot contains the configuration for an autopilot. Autopilot struct { Enabled bool `yaml:"enabled,omitempty"` - ID string `yaml:"id,omitempty"` Heartbeat time.Duration `yaml:"heartbeat,omitempty"` MigrationHealthCutoff float64 `yaml:"migrationHealthCutoff,omitempty"` RevisionBroadcastInterval time.Duration `yaml:"revisionBroadcastInterval,omitempty"` diff --git a/internal/bus/pinmanager.go b/internal/bus/pinmanager.go index b2a94f140..d31fbb4e4 100644 --- a/internal/bus/pinmanager.go +++ b/internal/bus/pinmanager.go @@ -27,9 +27,6 @@ type ( } Store interface { - Autopilot(ctx context.Context, id string) (api.Autopilot, error) - UpdateAutopilot(ctx context.Context, ap api.Autopilot) error - GougingSettings(ctx context.Context) (api.GougingSettings, error) UpdateGougingSettings(ctx context.Context, gs api.GougingSettings) error diff --git a/internal/bus/pinmanager_test.go b/internal/bus/pinmanager_test.go index fa941f822..efa2f220d 100644 --- a/internal/bus/pinmanager_test.go +++ b/internal/bus/pinmanager_test.go @@ -19,7 +19,6 @@ import ( ) const ( - testAutopilotID = "default" testUpdateInterval = 100 * time.Millisecond ) @@ -106,28 +105,16 @@ func (e *mockExplorer) setUnreachable(unreachable bool) { } type mockPinStore struct { - mu sync.Mutex - gs api.GougingSettings - ps api.PinnedSettings - autopilots map[string]api.Autopilot + mu sync.Mutex + gs api.GougingSettings + ps api.PinnedSettings } func newTestStore() *mockPinStore { - s := &mockPinStore{ - autopilots: make(map[string]api.Autopilot), - gs: api.DefaultGougingSettings, - ps: api.DefaultPinnedSettings, + return &mockPinStore{ + gs: api.DefaultGougingSettings, + ps: api.DefaultPinnedSettings, } - - // add default autopilot - s.autopilots[testAutopilotID] = api.Autopilot{ - ID: testAutopilotID, - Config: api.AutopilotConfig{ - Contracts: api.ContractsConfig{}, - }, - } - - return s } func (ms *mockPinStore) GougingSettings(ctx context.Context) (api.GougingSettings, error) { @@ -165,19 +152,6 @@ func (ms *mockPinStore) UpdatePinnedSettings(ctx context.Context, ps api.PinnedS return nil } -func (ms *mockPinStore) Autopilot(ctx context.Context, id string) (api.Autopilot, error) { - ms.mu.Lock() - defer ms.mu.Unlock() - return ms.autopilots[id], nil -} - -func (ms *mockPinStore) UpdateAutopilot(ctx context.Context, autopilot api.Autopilot) error { - ms.mu.Lock() - defer ms.mu.Unlock() - ms.autopilots[autopilot.ID] = autopilot - return nil -} - func TestPinManager(t *testing.T) { // mock dependencies a := &mockAlerter{} diff --git a/internal/sql/migrations.go b/internal/sql/migrations.go index 1487ea3d7..7d36f43f5 100644 --- a/internal/sql/migrations.go +++ b/internal/sql/migrations.go @@ -4,10 +4,15 @@ import ( "bytes" "context" "embed" + "encoding/json" + "errors" "fmt" "strings" "unicode/utf8" + dsql "database/sql" + + "go.sia.tech/renterd/api" "go.sia.tech/renterd/internal/utils" "go.sia.tech/renterd/object" "go.uber.org/zap" @@ -29,6 +34,7 @@ type ( MainMigrator interface { Migrator + InitAutopilot(ctx context.Context, tx Tx) error InsertDirectories(ctx context.Context, tx Tx, bucket, path string) (int64, error) MakeDirsForPath(ctx context.Context, tx Tx, path string) (int64, error) UpdateSetting(ctx context.Context, tx Tx, key, value string) error @@ -356,6 +362,66 @@ var ( return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00027_remove_directories", log) }, }, + { + ID: "00028_autopilot", + Migrate: func(tx Tx) error { + // remove all references to the autopilots table, without dropping the table + if err := performMigration(ctx, tx, migrationsFs, dbIdentifier, "00028_autopilot_1", log); err != nil { + return fmt.Errorf("failed to migrate: %v", err) + } + + // make sure the autopilot config is initialized + if err := m.InitAutopilot(ctx, tx); err != nil { + return fmt.Errorf("failed to initialize autopilot: %w", err) + } + + // fetch existing autopilot and override the blank config + var cfgraw []byte + var period uint64 + var cfg api.AutopilotConfig + err := tx.QueryRow(ctx, `SELECT config, current_period FROM autopilots WHERE identifier = "autopilot"`).Scan(&cfgraw, &period) + if errors.Is(dsql.ErrNoRows, err) { + log.Warn("existing autopilot not found, the autopilot will be recreated with default values and the period will be reset") + } else if err := json.Unmarshal(cfgraw, &cfg); err != nil { + log.Warnf("existing autopilot config not valid JSON, err %v", err) + } else { + var enabled bool + if err := cfg.Contracts.Validate(); err != nil { + log.Warnf("existing contracts config is invalid, autopilot will be disabled, err: %v", err) + } else if err := cfg.Hosts.Validate(); err != nil { + log.Warnf("existing hosts config is invalid, autopilot will be disabled, err: %v", err) + } else { + enabled = true + } + res, err := tx.Exec(ctx, `UPDATE autopilot_config SET enabled = ?, current_period = ?, contracts_set = ?, contracts_amount = ?, contracts_period = ?, contracts_renew_window = ?, contracts_download = ?, contracts_upload = ?, contracts_storage = ?, contracts_prune = ?, hosts_allow_redundant_ips = ?, hosts_max_downtime_hours = ?, hosts_min_protocol_version = ?, hosts_max_consecutive_scan_failures = ? WHERE id = ?`, + enabled, + period, + cfg.Contracts.Set, + cfg.Contracts.Amount, + cfg.Contracts.Period, + cfg.Contracts.RenewWindow, + cfg.Contracts.Download, + cfg.Contracts.Upload, + cfg.Contracts.Storage, + cfg.Contracts.Prune, + cfg.Hosts.AllowRedundantIPs, + cfg.Hosts.MaxDowntimeHours, + cfg.Hosts.MinProtocolVersion, + cfg.Hosts.MaxConsecutiveScanFailures, + AutopilotID) + if err != nil { + return fmt.Errorf("failed to update autopilot config: %w", err) + } else if n, err := res.RowsAffected(); err != nil { + return fmt.Errorf("failed to fetch rows affected: %w", err) + } else if n == 0 { + return fmt.Errorf("failed to override blank autopilot config not found") + } + } + + // drop autopilots table + return performMigration(ctx, tx, migrationsFs, dbIdentifier, "00028_autopilot_2", log) + }, + }, } } MetricsMigrations = func(ctx context.Context, migrationsFs embed.FS, log *zap.SugaredLogger) []Migration { diff --git a/internal/sql/sql.go b/internal/sql/sql.go index 02891fae3..5d07fd4ac 100644 --- a/internal/sql/sql.go +++ b/internal/sql/sql.go @@ -21,6 +21,7 @@ const ( factor = 1.8 // factor ^ retryAttempts = backoff time in milliseconds maxBackoff = 15 * time.Second + AutopilotID = 1 ConsensusInfoID = 1 ) diff --git a/internal/test/e2e/autopilot_test.go b/internal/test/e2e/autopilot_test.go new file mode 100644 index 000000000..ac358a1b9 --- /dev/null +++ b/internal/test/e2e/autopilot_test.go @@ -0,0 +1,72 @@ +package e2e + +import ( + "context" + "strings" + "testing" + + "go.sia.tech/renterd/api" + "go.sia.tech/renterd/bus/client" + "go.sia.tech/renterd/internal/test" + "go.sia.tech/renterd/internal/utils" +) + +func TestAutopilot(t *testing.T) { + // create test cluster + cluster := newTestCluster(t, clusterOptsDefault) + defer cluster.Shutdown() + tt := cluster.tt + b := cluster.Bus + + // assert autopilot is enabled by default + ap, err := b.Autopilot(context.Background()) + tt.OK(err) + if !ap.Enabled { + t.Fatal("autopilot should be enabled by default") + } + + // assert hosts and contracts config are defaulted + if ap.Contracts != test.AutopilotConfig.Contracts { + t.Fatalf("contracts config should be defaulted, got %v", ap.Contracts) + } else if ap.Hosts != test.AutopilotConfig.Hosts { + t.Fatalf("hosts config should be defaulted, got %v", ap.Hosts) + } + + // assert h config is validated + h := ap.Hosts + h.MaxDowntimeHours = 99*365*24 + 1 // exceed by one + if err := b.UpdateAutopilot(context.Background(), client.WithHostsConfig(h)); !utils.IsErr(err, api.ErrMaxDowntimeHoursTooHigh) { + t.Fatal("unexpected", err) + } + h.MaxDowntimeHours = 99 * 365 * 24 // allowed max + tt.OK(b.UpdateAutopilot(context.Background(), client.WithHostsConfig(h))) + + h.MinProtocolVersion = "not a version" + if err := b.UpdateAutopilot(context.Background(), client.WithHostsConfig(h)); !utils.IsErr(err, api.ErrInvalidReleaseVersion) { + t.Fatal("unexpected") + } + + // assert c config is validated + c := ap.Contracts + c.Period = 0 // invalid period + if err := b.UpdateAutopilot(context.Background(), client.WithContractsConfig(c)); err == nil || !strings.Contains(err.Error(), "period must be greater than 0") { + t.Fatal("unexpected", err) + } + c.Period = 1 // valid period + c.RenewWindow = 0 // invalid renew window + if err := b.UpdateAutopilot(context.Background(), client.WithContractsConfig(c)); err == nil || !strings.Contains(err.Error(), "renewWindow must be greater than 0") { + t.Fatal("unexpected", err) + } + c.RenewWindow = 1 // valid renew window + if err := b.UpdateAutopilot(context.Background(), client.WithContractsConfig(c)); err != nil { + t.Fatal(err) + } + + // assert we can disable the autopilot + tt.OK(b.UpdateAutopilot(context.Background(), client.WithAutopilotEnabled(false))) + ap, err = b.Autopilot(context.Background()) + tt.OK(err) + if ap.Enabled { + t.Fatal("autopilot should be disabled") + } +} diff --git a/internal/test/e2e/blocklist_test.go b/internal/test/e2e/blocklist_test.go index b2c55fdea..cac0add7f 100644 --- a/internal/test/e2e/blocklist_test.go +++ b/internal/test/e2e/blocklist_test.go @@ -23,7 +23,7 @@ func TestBlocklist(t *testing.T) { tt := cluster.tt // fetch contracts - opts := api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set} + opts := api.ContractsOpts{ContractSet: test.ContractSet} contracts, err := b.Contracts(ctx, opts) tt.OK(err) if len(contracts) != 3 { @@ -59,7 +59,7 @@ func TestBlocklist(t *testing.T) { // assert h1 is no longer in the contract set tt.Retry(100, 100*time.Millisecond, func() error { - contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) + contracts, err := b.Contracts(ctx, api.ContractsOpts{ContractSet: test.ContractSet}) tt.OK(err) if len(contracts) != 1 { return fmt.Errorf("unexpected number of contracts in set '%v', %v != 1", opts.ContractSet, len(contracts)) diff --git a/internal/test/e2e/cluster.go b/internal/test/e2e/cluster.go index 20cb02c97..5f4821cdb 100644 --- a/internal/test/e2e/cluster.go +++ b/internal/test/e2e/cluster.go @@ -29,6 +29,7 @@ import ( "go.sia.tech/renterd/api" "go.sia.tech/renterd/autopilot" "go.sia.tech/renterd/bus" + "go.sia.tech/renterd/bus/client" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/internal/utils" @@ -76,7 +77,6 @@ type TestCluster struct { genesisBlock types.Block bs bus.Store cm *chain.Manager - apID string dbName string dir string logger *zap.Logger @@ -181,23 +181,6 @@ func (c *TestCluster) Reboot(t *testing.T) *TestCluster { return newCluster } -// AutopilotConfig returns the autopilot's config and current period. -func (c *TestCluster) AutopilotConfig(ctx context.Context) (api.AutopilotConfig, uint64) { - c.tt.Helper() - ap, err := c.Bus.Autopilot(ctx, c.apID) - c.tt.OK(err) - return ap.Config, ap.CurrentPeriod -} - -// UpdateAutopilotConfig updates the cluster's autopilot with given config. -func (c *TestCluster) UpdateAutopilotConfig(ctx context.Context, cfg api.AutopilotConfig) { - c.tt.Helper() - c.tt.OK(c.Bus.UpdateAutopilot(ctx, api.Autopilot{ - ID: c.apID, - Config: cfg, - })) -} - type testClusterOptions struct { dbName string dir string @@ -209,11 +192,11 @@ type testClusterOptions struct { skipRunningAutopilot bool walletKey *types.PrivateKey - autopilotCfg *config.Autopilot - autopilotSettings *api.AutopilotConfig - cm *chain.Manager - busCfg *config.Bus - workerCfg *config.Worker + autopilotCfg *config.Autopilot + autopilotConfig *api.AutopilotConfig + cm *chain.Manager + busCfg *config.Bus + workerCfg *config.Worker } // newTestLogger creates a console logger used for testing. @@ -236,6 +219,8 @@ func newTestLogger(enable bool) *zap.Logger { // newTestCluster creates a new cluster without hosts with a funded bus. func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { + t.Helper() + // Skip any test that requires a cluster when running short tests. tt := test.NewTT(t) @@ -279,9 +264,9 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { if opts.uploadPacking { enableUploadPacking = opts.uploadPacking } - apSettings := test.AutopilotConfig - if opts.autopilotSettings != nil { - apSettings = *opts.autopilotSettings + apConfig := test.AutopilotConfig + if opts.autopilotConfig != nil { + apConfig = *opts.autopilotConfig } if opts.dbName != "" { dbCfg.Database.MySQL.Database = opts.dbName @@ -406,7 +391,6 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { autopilotShutdownFns = append(autopilotShutdownFns, ap.Shutdown) cluster := &TestCluster{ - apID: apCfg.ID, dir: dir, dbName: dbCfg.Database.MySQL.Database, logger: logger, @@ -468,10 +452,13 @@ func newTestCluster(t *testing.T, opts testClusterOptions) *TestCluster { // Update the autopilot to use test settings if !opts.skipSettingAutopilot { - tt.OK(busClient.UpdateAutopilot(ctx, api.Autopilot{ - ID: apCfg.ID, - Config: apSettings, - })) + tt.OKAll( + busClient.UpdateAutopilot(ctx, + client.WithContractsConfig(apConfig.Contracts), + client.WithHostsConfig(apConfig.Hosts), + client.WithAutopilotEnabled(true), + ), + ) } // Build upload settings. @@ -671,12 +658,12 @@ func (c *TestCluster) MineToRenewWindow() { cs, err := c.Bus.ConsensusState(context.Background()) c.tt.OK(err) - ap, err := c.Bus.Autopilot(context.Background(), c.apID) + ap, err := c.Bus.Autopilot(context.Background()) c.tt.OK(err) - renewWindowStart := ap.CurrentPeriod + ap.Config.Contracts.Period + renewWindowStart := ap.CurrentPeriod + ap.Contracts.Period if cs.BlockHeight >= renewWindowStart { - c.tt.Fatalf("already in renew window: bh: %v, currentPeriod: %v, periodLength: %v, renewWindow: %v", cs.BlockHeight, ap.CurrentPeriod, ap.Config.Contracts.Period, renewWindowStart) + c.tt.Fatalf("already in renew window: bh: %v, currentPeriod: %v, periodLength: %v, renewWindow: %v", cs.BlockHeight, ap.CurrentPeriod, ap.Contracts.Period, renewWindowStart) } c.MineBlocks(renewWindowStart - cs.BlockHeight) } @@ -1076,7 +1063,6 @@ func testWorkerCfg() config.Worker { func testApCfg() config.Autopilot { return config.Autopilot{ Heartbeat: time.Second, - ID: api.DefaultAutopilotID, MigrationHealthCutoff: 0.99, MigratorParallelSlabsPerWorker: 1, RevisionSubmissionBuffer: 0, diff --git a/internal/test/e2e/cluster_test.go b/internal/test/e2e/cluster_test.go index 3fc4278eb..a9d3d38bf 100644 --- a/internal/test/e2e/cluster_test.go +++ b/internal/test/e2e/cluster_test.go @@ -28,6 +28,7 @@ import ( "go.sia.tech/renterd/alerts" "go.sia.tech/renterd/api" "go.sia.tech/renterd/autopilot/contractor" + "go.sia.tech/renterd/bus/client" "go.sia.tech/renterd/config" "go.sia.tech/renterd/internal/test" "go.sia.tech/renterd/internal/utils" @@ -183,31 +184,31 @@ func TestNewTestCluster(t *testing.T) { defer cluster.Shutdown() tt := cluster.tt - // See if autopilot is running by triggering the loop. - _, err := cluster.Autopilot.Trigger(false) - tt.OK(err) - - // Add a host. + // add a host & wait for contracts to form cluster.AddHosts(1) - - // Wait for contracts to form. - var contract api.ContractMetadata contracts := cluster.WaitForContracts() - contract = contracts[0] + if len(contracts) != 1 { + t.Fatal("expected 1 contract, got", len(contracts)) + } + contract := contracts[0] + + // fetch autopilot + ap, err := cluster.Bus.Autopilot(context.Background()) + tt.OK(err) + + // fetch revision revision, err := cluster.Bus.ContractRevision(context.Background(), contract.ID) tt.OK(err) - // Verify startHeight and endHeight of the contract. - cfg, currentPeriod := cluster.AutopilotConfig(context.Background()) - expectedEndHeight := currentPeriod + cfg.Contracts.Period + cfg.Contracts.RenewWindow - if contract.EndHeight() != expectedEndHeight || revision.EndHeight() != expectedEndHeight { - t.Fatal("wrong endHeight", contract.EndHeight(), revision.EndHeight()) + // verify startHeight and endHeight of the contract. + if contract.EndHeight() != ap.EndHeight() || revision.EndHeight() != ap.EndHeight() { + t.Fatal("wrong endHeight", contract.EndHeight(), revision.EndHeight(), ap.EndHeight()) } else if contract.InitialRenterFunds.IsZero() || contract.ContractPrice.IsZero() { t.Fatal("InitialRenterFunds and ContractPrice shouldn't be zero") } // Wait for contract set to form - cluster.WaitForContractSetContracts(cfg.Contracts.Set, int(cfg.Contracts.Amount)) + cluster.WaitForContractSetContracts(test.ContractSet, int(ap.Contracts.Amount)) // Mine blocks until contracts start renewing. cluster.MineToRenewWindow() @@ -277,14 +278,14 @@ func TestNewTestCluster(t *testing.T) { hi, err := cluster.Bus.Host(context.Background(), host.PublicKey) if err != nil { t.Fatal(err) - } else if checks := hi.Checks[testApCfg().ID]; checks == (api.HostCheck{}) { + } else if hi.Checks == (api.HostChecks{}) { t.Fatal("host check not found") - } else if checks.ScoreBreakdown.Score() == 0 { - js, _ := json.MarshalIndent(checks.ScoreBreakdown, "", " ") + } else if hi.Checks.ScoreBreakdown.Score() == 0 { + js, _ := json.MarshalIndent(hi.Checks.ScoreBreakdown, "", " ") t.Fatalf("score shouldn't be 0 because that means one of the fields was 0: %s", string(js)) - } else if !checks.UsabilityBreakdown.IsUsable() { + } else if !hi.Checks.UsabilityBreakdown.IsUsable() { t.Fatal("host should be usable") - } else if len(checks.UsabilityBreakdown.UnusableReasons()) != 0 { + } else if len(hi.Checks.UsabilityBreakdown.UnusableReasons()) != 0 { t.Fatal("usable hosts don't have any reasons set") } else if reflect.DeepEqual(hi, api.Host{}) { t.Fatal("host wasn't set") @@ -300,14 +301,14 @@ func TestNewTestCluster(t *testing.T) { allHosts := make(map[types.PublicKey]struct{}) for _, hi := range hostInfos { - if checks := hi.Checks[testApCfg().ID]; checks == (api.HostCheck{}) { + if hi.Checks == (api.HostChecks{}) { t.Fatal("host check not found") - } else if checks.ScoreBreakdown.Score() == 0 { - js, _ := json.MarshalIndent(checks.ScoreBreakdown, "", " ") + } else if hi.Checks.ScoreBreakdown.Score() == 0 { + js, _ := json.MarshalIndent(hi.Checks.ScoreBreakdown, "", " ") t.Fatalf("score shouldn't be 0 because that means one of the fields was 0: %s", string(js)) - } else if !checks.UsabilityBreakdown.IsUsable() { + } else if !hi.Checks.UsabilityBreakdown.IsUsable() { t.Fatal("host should be usable") - } else if len(checks.UsabilityBreakdown.UnusableReasons()) != 0 { + } else if len(hi.Checks.UsabilityBreakdown.UnusableReasons()) != 0 { t.Fatal("usable hosts don't have any reasons set") } else if reflect.DeepEqual(hi, api.Host{}) { t.Fatal("host wasn't set") @@ -316,7 +317,6 @@ func TestNewTestCluster(t *testing.T) { } hostInfosUnusable, err := cluster.Bus.Hosts(context.Background(), api.HostOptions{ - AutopilotID: testApCfg().ID, FilterMode: api.UsabilityFilterModeAll, UsabilityMode: api.UsabilityFilterModeUnusable, }) @@ -326,7 +326,6 @@ func TestNewTestCluster(t *testing.T) { } hostInfosUsable, err := cluster.Bus.Hosts(context.Background(), api.HostOptions{ - AutopilotID: testApCfg().ID, FilterMode: api.UsabilityFilterModeAll, UsabilityMode: api.UsabilityFilterModeUsable, }) @@ -341,9 +340,7 @@ func TestNewTestCluster(t *testing.T) { // Fetch the autopilot state state, err := cluster.Autopilot.State() tt.OK(err) - if state.ID != api.DefaultAutopilotID { - t.Fatal("autopilot should have default id", state.ID) - } else if time.Time(state.StartTime).IsZero() { + if time.Time(state.StartTime).IsZero() { t.Fatal("autopilot should have start time") } else if time.Time(state.MigratingLastStart).IsZero() { t.Fatal("autopilot should have completed a migration") @@ -351,8 +348,8 @@ func TestNewTestCluster(t *testing.T) { t.Fatal("autopilot should have completed a scan") } else if state.UptimeMS == 0 { t.Fatal("uptime should be set") - } else if !state.Configured { - t.Fatal("autopilot should be configured") + } else if !state.Enabled { + t.Fatal("autopilot should be enabled") } // Fetch host @@ -769,7 +766,7 @@ func TestUploadDownloadExtended(t *testing.T) { tt.OKAll(w.UploadObject(context.Background(), bytes.NewReader(file2), testBucket, "fileś/file2", api.UploadObjectOptions{})) // fetch all entries from the worker - resp, err := cluster.Bus.Objects(context.Background(), "fileś/", api.ListObjectOptions{ + resp, err := b.Objects(context.Background(), "fileś/", api.ListObjectOptions{ Bucket: testBucket, Delimiter: "/", }) @@ -785,7 +782,7 @@ func TestUploadDownloadExtended(t *testing.T) { } // fetch entries in /fileś starting with "file" - res, err := cluster.Bus.Objects(context.Background(), "fileś/file", api.ListObjectOptions{ + res, err := b.Objects(context.Background(), "fileś/file", api.ListObjectOptions{ Bucket: testBucket, Delimiter: "/", }) @@ -795,7 +792,7 @@ func TestUploadDownloadExtended(t *testing.T) { } // fetch entries in /fileś starting with "foo" - res, err = cluster.Bus.Objects(context.Background(), "fileś/foo", api.ListObjectOptions{ + res, err = b.Objects(context.Background(), "fileś/foo", api.ListObjectOptions{ Bucket: testBucket, Delimiter: "/", }) @@ -821,7 +818,7 @@ func TestUploadDownloadExtended(t *testing.T) { {}, // any bucket {Bucket: testBucket}, // specific bucket } { - info, err := cluster.Bus.ObjectsStats(context.Background(), opts) + info, err := b.ObjectsStats(context.Background(), opts) tt.OK(err) objectsSize := uint64(len(file1) + len(file2) + len(small) + len(large)) if info.TotalObjectsSize != objectsSize { @@ -855,35 +852,6 @@ func TestUploadDownloadExtended(t *testing.T) { t.Fatal("unexpected") } } - - // update the bus setting and specify a non-existing contract set - cfg, _ := cluster.AutopilotConfig(context.Background()) - cfg.Contracts.Set = t.Name() - cluster.UpdateAutopilotConfig(context.Background(), cfg) - tt.OK(b.UpdateContractSet(context.Background(), t.Name(), nil, nil)) - - // assert there are no contracts in the set - csc, err := b.Contracts(context.Background(), api.ContractsOpts{ContractSet: t.Name()}) - tt.OK(err) - if len(csc) != 0 { - t.Fatalf("expected no contracts, got %v", len(csc)) - } - - // download the data again - for _, data := range [][]byte{small, large} { - path := fmt.Sprintf("data_%v", len(data)) - - var buffer bytes.Buffer - tt.OK(w.DownloadObject(context.Background(), &buffer, testBucket, path, api.DownloadObjectOptions{})) - - // assert it matches - if !bytes.Equal(data, buffer.Bytes()) { - t.Fatal("unexpected") - } - - // delete the object - tt.OK(w.DeleteObject(context.Background(), testBucket, path)) - } } // TestUploadDownloadSpending is an integration test that verifies the upload @@ -996,7 +964,7 @@ func TestUploadDownloadSpending(t *testing.T) { } // fetch contract set contracts - contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{ContractSet: test.AutopilotConfig.Contracts.Set}) + contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{ContractSet: test.ContractSet}) tt.OK(err) currentSet := make(map[types.FileContractID]struct{}) for _, c := range contracts { @@ -2286,7 +2254,14 @@ func TestWalletFormUnconfirmed(t *testing.T) { } // enable the autopilot by configuring it - cluster.UpdateAutopilotConfig(context.Background(), test.AutopilotConfig) + tt.OK( + b.UpdateAutopilot( + context.Background(), + client.WithContractsConfig(test.AutopilotConfig.Contracts), + client.WithHostsConfig(test.AutopilotConfig.Hosts), + client.WithAutopilotEnabled(true), + ), + ) // wait for a contract to form contractsFormed := cluster.WaitForContracts() @@ -2849,7 +2824,6 @@ func TestConsensusResync(t *testing.T) { network, genesis := testNetwork() store, state, err := chain.NewDBStore(chain.NewMemDB(), network, genesis) tt.OK(err) - newCluster := newTestCluster(t, testClusterOptions{ cm: chain.NewManager(store, state), dir: cluster.dir, diff --git a/internal/test/e2e/contracts_test.go b/internal/test/e2e/contracts_test.go index f21e0dcc2..e72f8a2b0 100644 --- a/internal/test/e2e/contracts_test.go +++ b/internal/test/e2e/contracts_test.go @@ -9,23 +9,23 @@ import ( "go.sia.tech/core/types" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/bus/client" "go.sia.tech/renterd/internal/test" ) func TestFormContract(t *testing.T) { // configure the autopilot not to form any contracts - apSettings := test.AutopilotConfig - apSettings.Contracts.Amount = 0 + apCfg := test.AutopilotConfig + apCfg.Contracts.Amount = 0 // create cluster opts := clusterOptsDefault - opts.autopilotSettings = &apSettings + opts.autopilotConfig = &apCfg cluster := newTestCluster(t, opts) defer cluster.Shutdown() // convenience variables b := cluster.Bus - a := cluster.Autopilot tt := cluster.tt // add a host @@ -35,7 +35,7 @@ func TestFormContract(t *testing.T) { // form a contract using the bus wallet, _ := b.Wallet(context.Background()) - ap, err := b.Autopilot(context.Background(), api.DefaultAutopilotID) + ap, err := b.Autopilot(context.Background()) tt.OK(err) contract, err := b.FormContract(context.Background(), wallet.Address, types.Siacoins(1), h.PublicKey, h.NetAddress, types.Siacoins(1), ap.EndHeight()) tt.OK(err) @@ -44,16 +44,12 @@ func TestFormContract(t *testing.T) { _, err = b.Contract(context.Background(), contract.ID) tt.OK(err) - // fetch autopilot config - old, err := b.Autopilot(context.Background(), api.DefaultAutopilotID) - tt.OK(err) - // mine to the renew window cluster.MineToRenewWindow() // wait until autopilot updated the current period tt.Retry(100, 100*time.Millisecond, func() error { - if curr, _ := b.Autopilot(context.Background(), api.DefaultAutopilotID); curr.CurrentPeriod == old.CurrentPeriod { + if curr, _ := b.Autopilot(context.Background()); curr.CurrentPeriod == ap.CurrentPeriod { return errors.New("autopilot didn't update the current period") } return nil @@ -62,8 +58,9 @@ func TestFormContract(t *testing.T) { // update autopilot config to allow for 1 contract, this won't form a // contract but will ensure we don't skip contract maintenance, which should // renew the contract we formed - apSettings.Contracts.Amount = 1 - tt.OK(a.UpdateConfig(apSettings)) + contracts := ap.Contracts + contracts.Amount = 1 + tt.OK(b.UpdateAutopilot(context.Background(), client.WithContractsConfig(contracts))) // assert the contract gets renewed and thus maintained var renewalID types.FileContractID diff --git a/internal/test/e2e/gouging_test.go b/internal/test/e2e/gouging_test.go index 3ebdd672e..caae37c59 100644 --- a/internal/test/e2e/gouging_test.go +++ b/internal/test/e2e/gouging_test.go @@ -11,6 +11,7 @@ import ( rhpv2 "go.sia.tech/core/rhp/v2" "go.sia.tech/renterd/api" + "go.sia.tech/renterd/bus/client" "go.sia.tech/renterd/internal/test" "lukechampine.com/frand" ) @@ -42,7 +43,7 @@ func TestGouging(t *testing.T) { // assert that the current period is greater than the period tt.Retry(10, time.Second, func() error { - if ap, _ := b.Autopilot(context.Background(), api.DefaultAutopilotID); ap.CurrentPeriod <= cfg.Period { + if ap, _ := b.Autopilot(context.Background()); ap.CurrentPeriod <= cfg.Period { return errors.New("current period is not greater than period") } return nil @@ -151,15 +152,13 @@ func TestHostMinVersion(t *testing.T) { tt := cluster.tt // set min version to a high value - cfg := test.AutopilotConfig - cfg.Hosts.MinProtocolVersion = "99.99.99" - cluster.UpdateAutopilotConfig(context.Background(), cfg) + hosts := test.AutopilotConfig.Hosts + hosts.MinProtocolVersion = "99.99.99" + tt.OK(cluster.Bus.UpdateAutopilot(context.Background(), client.WithHostsConfig(hosts))) // contracts in set should drop to 0 tt.Retry(100, 100*time.Millisecond, func() error { - contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{ - ContractSet: test.AutopilotConfig.Contracts.Set, - }) + contracts, err := cluster.Bus.Contracts(context.Background(), api.ContractsOpts{ContractSet: test.ContractSet}) tt.OK(err) if len(contracts) != 0 { return fmt.Errorf("expected 0 contracts, got %v", len(contracts)) diff --git a/internal/test/e2e/metrics_test.go b/internal/test/e2e/metrics_test.go index e9f384cce..c86e71552 100644 --- a/internal/test/e2e/metrics_test.go +++ b/internal/test/e2e/metrics_test.go @@ -24,8 +24,8 @@ func TestMetrics(t *testing.T) { // create a test cluster cluster := newTestCluster(t, testClusterOptions{ - hosts: test.RedundancySettings.TotalShards, - autopilotSettings: &apCfg, + hosts: test.RedundancySettings.TotalShards, + autopilotConfig: &apCfg, }) defer cluster.Shutdown() diff --git a/internal/test/e2e/migrations_test.go b/internal/test/e2e/migrations_test.go index c0ff74f35..e6a78a397 100644 --- a/internal/test/e2e/migrations_test.go +++ b/internal/test/e2e/migrations_test.go @@ -27,8 +27,8 @@ func TestMigrations(t *testing.T) { // create a new test cluster cluster := newTestCluster(t, testClusterOptions{ - autopilotSettings: &cfg, - hosts: int(cfg.Contracts.Amount), + autopilotConfig: &cfg, + hosts: int(cfg.Contracts.Amount), }) defer cluster.Shutdown() diff --git a/internal/test/e2e/pruning_test.go b/internal/test/e2e/pruning_test.go index 8aa3aab16..3c2190837 100644 --- a/internal/test/e2e/pruning_test.go +++ b/internal/test/e2e/pruning_test.go @@ -76,17 +76,6 @@ func TestHostPruning(t *testing.T) { } return nil }) - - // assert validation on MaxDowntimeHours - ap, err := b.Autopilot(context.Background(), api.DefaultAutopilotID) - tt.OK(err) - - ap.Config.Hosts.MaxDowntimeHours = 99*365*24 + 1 // exceed by one - if err = b.UpdateAutopilot(context.Background(), api.Autopilot{ID: t.Name(), Config: ap.Config}); !strings.Contains(err.Error(), api.ErrMaxDowntimeHoursTooHigh.Error()) { - t.Fatal(err) - } - ap.Config.Hosts.MaxDowntimeHours = 99 * 365 * 24 // allowed max - tt.OK(b.UpdateAutopilot(context.Background(), api.Autopilot{ID: t.Name(), Config: ap.Config})) } func TestSectorPruning(t *testing.T) { @@ -106,7 +95,6 @@ func TestSectorPruning(t *testing.T) { } // convenience variables - cfg := test.AutopilotConfig rs := test.RedundancySettings w := cluster.Worker b := cluster.Bus @@ -121,7 +109,7 @@ func TestSectorPruning(t *testing.T) { cluster.WaitForAccounts() // wait until we have a contract set - cluster.WaitForContractSetContracts(cfg.Contracts.Set, rs.TotalShards) + cluster.WaitForContractSetContracts(test.ContractSet, rs.TotalShards) // add several objects for i := 0; i < numObjects; i++ { diff --git a/stores/autopilot.go b/stores/autopilot.go index 9c557c3d7..44f05838e 100644 --- a/stores/autopilot.go +++ b/stores/autopilot.go @@ -2,36 +2,26 @@ package stores import ( "context" - "errors" "go.sia.tech/renterd/api" sql "go.sia.tech/renterd/stores/sql" ) -func (s *SQLStore) Autopilots(ctx context.Context) (aps []api.Autopilot, _ error) { - err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) (err error) { - aps, err = tx.Autopilots(ctx) +func (s *SQLStore) Autopilot(ctx context.Context) (ap api.Autopilot, err error) { + s.db.Transaction(ctx, func(tx sql.DatabaseTx) (err error) { + ap, err = tx.Autopilot(ctx) return }) - return aps, err + return } -func (s *SQLStore) Autopilot(ctx context.Context, id string) (ap api.Autopilot, _ error) { - err := s.db.Transaction(ctx, func(tx sql.DatabaseTx) (err error) { - ap, err = tx.Autopilot(ctx, id) - return +func (s *SQLStore) InitAutopilot(ctx context.Context) error { + return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { + return tx.InitAutopilot(ctx) }) - return ap, err } func (s *SQLStore) UpdateAutopilot(ctx context.Context, ap api.Autopilot) error { - // validate autopilot - if ap.ID == "" { - return errors.New("autopilot ID cannot be empty") - } - if err := ap.Config.Validate(); err != nil { - return err - } return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { return tx.UpdateAutopilot(ctx, ap) }) diff --git a/stores/autopilot_test.go b/stores/autopilot_test.go deleted file mode 100644 index a1eecde04..000000000 --- a/stores/autopilot_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package stores - -import ( - "context" - "reflect" - "testing" - - rhpv2 "go.sia.tech/core/rhp/v2" - "go.sia.tech/renterd/api" -) - -func TestAutopilotStore(t *testing.T) { - ss := newTestSQLStore(t, defaultTestSQLStoreConfig) - defer ss.Close() - - // assert we have no autopilots - autopilots, err := ss.Autopilots(context.Background()) - if err != nil { - t.Fatal(err) - } else if len(autopilots) != 0 { - t.Fatal("expected number of autopilots", len(autopilots)) - } - - // create a cfg - cfg := api.AutopilotConfig{ - Contracts: api.ContractsConfig{ - Amount: 3, - Period: 144, - RenewWindow: 72, - - Download: rhpv2.SectorSize * 500, - Upload: rhpv2.SectorSize * 500, - Storage: rhpv2.SectorSize * 5e3, - - Set: testContractSet, - }, - Hosts: api.HostsConfig{ - MaxDowntimeHours: 10, - MaxConsecutiveScanFailures: 10, - AllowRedundantIPs: true, // allow for integration tests by default - }, - } - - // add an autopilot with that config - err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: t.Name(), Config: cfg}) - if err != nil { - t.Fatal(err) - } - - // assert we have one autopilot - autopilots, err = ss.Autopilots(context.Background()) - if err != nil { - t.Fatal(err) - } else if len(autopilots) != 1 { - t.Fatal("expected number of autopilots", len(autopilots)) - } - autopilot := autopilots[0] - - // assert config - if !reflect.DeepEqual(autopilot.Config, cfg) { - t.Fatal("expected autopilot config to be default config") - } - if autopilot.CurrentPeriod != 0 { - t.Fatal("expected current period to be 0") - } - - // update the autopilot and set a new current period and update the config - autopilot.CurrentPeriod = 1 - autopilot.Config.Contracts.Amount = 99 - err = ss.UpdateAutopilot(context.Background(), autopilot) - if err != nil { - t.Fatal(err) - } - - // fetch it and assert it was updated - updated, err := ss.Autopilot(context.Background(), t.Name()) - if err != nil { - t.Fatal(err) - } - if updated.CurrentPeriod != 1 { - t.Fatal("expected current period to be 1") - } - if updated.Config.Contracts.Amount != 99 { - t.Fatal("expected amount to be 99") - } - - // update the autopilot with the same config and assert it does not fail - err = ss.UpdateAutopilot(context.Background(), updated) - if err != nil { - t.Fatal(err) - } -} diff --git a/stores/hostdb.go b/stores/hostdb.go index b1da3c11b..c7cfb6a40 100644 --- a/stores/hostdb.go +++ b/stores/hostdb.go @@ -18,7 +18,6 @@ var ( // Host returns information about a host. func (s *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, error) { hosts, err := s.Hosts(ctx, api.HostOptions{ - AutopilotID: "", AddressContains: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, @@ -35,9 +34,9 @@ func (s *SQLStore) Host(ctx context.Context, hostKey types.PublicKey) (api.Host, } } -func (s *SQLStore) UpdateHostCheck(ctx context.Context, autopilotID string, hk types.PublicKey, hc api.HostCheck) (err error) { +func (s *SQLStore) UpdateHostCheck(ctx context.Context, hk types.PublicKey, hc api.HostChecks) (err error) { return s.db.Transaction(ctx, func(tx sql.DatabaseTx) error { - return tx.UpdateHostCheck(ctx, autopilotID, hk, hc) + return tx.UpdateHostCheck(ctx, hk, hc) }) } diff --git a/stores/hostdb_test.go b/stores/hostdb_test.go index 2e95e78df..5bb9e4b0d 100644 --- a/stores/hostdb_test.go +++ b/stores/hostdb_test.go @@ -41,7 +41,6 @@ func TestSQLHostDB(t *testing.T) { // Assert it's returned allHosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -74,7 +73,6 @@ func TestSQLHostDB(t *testing.T) { // Same thing again but with hosts. hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -142,7 +140,6 @@ func TestHosts(t *testing.T) { // search all hosts his, err := ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -158,7 +155,6 @@ func TestHosts(t *testing.T) { // assert offset & limit are taken into account his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -172,7 +168,6 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected") } his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -186,7 +181,6 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected") } his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -202,7 +196,6 @@ func TestHosts(t *testing.T) { // assert address and key filters are taken into account if hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "com:1001", @@ -213,7 +206,6 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected", len(hosts), err) } if hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -224,7 +216,6 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected", len(hosts), err) } if hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "com:1002", @@ -235,7 +226,6 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected", len(hosts), err) } if hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "com:1002", @@ -252,7 +242,6 @@ func TestHosts(t *testing.T) { t.Fatal(err) } his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAllowed, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -268,7 +257,6 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected", his[0].PublicKey, his[1].PublicKey) } his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeBlocked, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -288,75 +276,31 @@ func TestHosts(t *testing.T) { t.Fatal(err) } - // add two autopilots - ap1 := "ap1" - err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: ap1}) - if err != nil { - t.Fatal(err) - } - ap2 := "ap2" - err = ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: ap2}) - if err != nil { - t.Fatal(err) - } - - // add host checks, h1 gets ap1 and h2 gets both + // add host checks h1c := newTestHostCheck() h1c.ScoreBreakdown.Age = .1 - err = ss.UpdateHostCheck(context.Background(), ap1, hk1, h1c) - if err != nil { - t.Fatal(err) - } - h2c1 := newTestHostCheck() - h2c1.ScoreBreakdown.Age = .21 - err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1) + err = ss.UpdateHostCheck(context.Background(), hk1, h1c) if err != nil { t.Fatal(err) } - h2c2 := newTestHostCheck() - h2c2.ScoreBreakdown.Age = .22 - err = ss.UpdateHostCheck(context.Background(), ap2, hk2, h2c2) + h2c := newTestHostCheck() + h2c.ScoreBreakdown.Age = .21 + err = ss.UpdateHostCheck(context.Background(), hk2, h2c) if err != nil { t.Fatal(err) } - // assert there are currently 3 checks + // assert number of host checks checkCount := func() int64 { t.Helper() return ss.Count("host_checks") } - if cnt := checkCount(); cnt != 3 { + if cnt := checkCount(); cnt != 2 { t.Fatal("unexpected", cnt) } // fetch all hosts his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", - FilterMode: api.HostFilterModeAll, - UsabilityMode: api.UsabilityFilterModeAll, - AddressContains: "", - KeyIn: nil, - Offset: 0, - Limit: -1, - }) - if err != nil { - t.Fatal(err) - } else if len(his) != 3 { - t.Fatal("unexpected", len(his)) - } - - // assert h1 and h2 have the expected checks - if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { - t.Fatal("unexpected", c1, ok) - } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { - t.Fatal("unexpected", c2, ok) - } else if c3, ok := his[1].Checks[ap2]; !ok || c3 != h2c2 { - t.Fatal("unexpected", c3, ok) - } - - // assert autopilot filter is taken into account - his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: ap1, FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -371,22 +315,19 @@ func TestHosts(t *testing.T) { } // assert h1 and h2 have the expected checks - if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { - t.Fatal("unexpected", c1, ok, his[0]) - } else if c2, ok := his[1].Checks[ap1]; !ok || c2 != h2c1 { - t.Fatal("unexpected", c2, ok) - } else if _, ok := his[1].Checks[ap2]; ok { - t.Fatal("unexpected") + if his[0].Checks != h1c { + t.Fatal("unexpected", his[0].Checks) + } else if his[1].Checks != h2c { + t.Fatal("unexpected", his[1].Checks) } // assert usability filter is taken into account - h2c1.UsabilityBreakdown.RedundantIP = true - err = ss.UpdateHostCheck(context.Background(), ap1, hk2, h2c1) + h2c.UsabilityBreakdown.RedundantIP = true + err = ss.UpdateHostCheck(context.Background(), hk2, h2c) if err != nil { t.Fatal(err) } his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: ap1, FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeUsable, AddressContains: "", @@ -400,13 +341,7 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected", len(his)) } - // assert h1 has the expected checks - if c1, ok := his[0].Checks[ap1]; !ok || c1 != h1c { - t.Fatal("unexpected", c1, ok) - } - his, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: ap1, FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeUnusable, AddressContains: "", @@ -422,28 +357,12 @@ func TestHosts(t *testing.T) { t.Fatal("unexpected") } - // assert only ap1 check is there - if _, ok := his[0].Checks[ap1]; !ok { - t.Fatal("unexpected") - } else if _, ok := his[0].Checks[ap2]; ok { - t.Fatal("unexpected") - } - // assert cascade delete on host _, err = ss.DB().Exec(context.Background(), "DELETE FROM hosts WHERE public_key = ?", sql.PublicKey(types.PublicKey{1})) if err != nil { t.Fatal(err) } - if cnt := checkCount(); cnt != 2 { - t.Fatal("unexpected", cnt) - } - - // assert cascade delete on autopilot - _, err = ss.DB().Exec(context.Background(), "DELETE FROM autopilots WHERE identifier IN (?,?)", ap1, ap2) - if err != nil { - t.Fatal(err) - } - if cnt := checkCount(); cnt != 0 { + if cnt := checkCount(); cnt != 1 { t.Fatal("unexpected", cnt) } } @@ -453,12 +372,6 @@ func TestUsableHosts(t *testing.T) { defer ss.Close() ctx := context.Background() - // add autopilot - err := ss.UpdateAutopilot(context.Background(), api.Autopilot{ID: api.DefaultAutopilotID}) - if err != nil { - t.Fatal(err) - } - // prepare hosts & contracts // // h1: usable @@ -487,7 +400,7 @@ func TestUsableHosts(t *testing.T) { // add host check if i != 3 { hc := newTestHostCheck() - err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, hk, hc) + err = ss.UpdateHostCheck(context.Background(), hk, hc) if err != nil { t.Fatal(err) } @@ -576,7 +489,7 @@ func TestUsableHosts(t *testing.T) { // add host check for h3 hc := newTestHostCheck() - err = ss.UpdateHostCheck(context.Background(), api.DefaultAutopilotID, types.PublicKey{3}, hc) + err = ss.UpdateHostCheck(context.Background(), types.PublicKey{3}, hc) if err != nil { t.Fatal(err) } @@ -879,7 +792,6 @@ func TestSQLHostAllowlist(t *testing.T) { numHosts := func() int { t.Helper() hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAllowed, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -967,7 +879,6 @@ func TestSQLHostAllowlist(t *testing.T) { assertHosts := func(total, allowed, blocked int) error { t.Helper() hosts, err := ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAll, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -982,7 +893,6 @@ func TestSQLHostAllowlist(t *testing.T) { return fmt.Errorf("invalid number of hosts: %v", len(hosts)) } hosts, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAllowed, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -997,7 +907,6 @@ func TestSQLHostAllowlist(t *testing.T) { return fmt.Errorf("invalid number of hosts: %v", len(hosts)) } hosts, err = ss.Hosts(context.Background(), api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeBlocked, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -1079,7 +988,6 @@ func TestSQLHostBlocklist(t *testing.T) { numHosts := func() int { t.Helper() hosts, err := ss.Hosts(ctx, api.HostOptions{ - AutopilotID: "", FilterMode: api.HostFilterModeAllowed, UsabilityMode: api.UsabilityFilterModeAll, AddressContains: "", @@ -1355,8 +1263,8 @@ func newTestScan(hk types.PublicKey, scanTime time.Time, settings rhpv2.HostSett } } -func newTestHostCheck() api.HostCheck { - return api.HostCheck{ +func newTestHostCheck() api.HostChecks { + return api.HostChecks{ GougingBreakdown: api.HostGougingBreakdown{ ContractErr: "foo", diff --git a/stores/sql/database.go b/stores/sql/database.go index 83893d04a..067d37be0 100644 --- a/stores/sql/database.go +++ b/stores/sql/database.go @@ -71,12 +71,8 @@ type ( // archived ones. ArchiveContract(ctx context.Context, fcid types.FileContractID, reason string) error - // Autopilot returns the autopilot with the given ID. Returns - // api.ErrAutopilotNotFound if the autopilot doesn't exist. - Autopilot(ctx context.Context, id string) (api.Autopilot, error) - - // Autopilots returns all autopilots. - Autopilots(ctx context.Context) ([]api.Autopilot, error) + // Autopilot returns the autopilot. + Autopilot(ctx context.Context) (api.Autopilot, error) // BanPeer temporarily bans one or more IPs. The addr should either be a // single IP with port (e.g. 1.2.3.4:5678) or a CIDR subnet (e.g. @@ -170,6 +166,9 @@ type ( // HostBlocklist returns the list of host addresses on the blocklist. HostBlocklist(ctx context.Context) ([]string, error) + // InitAutopilot initializes the autopilot state in the database. + InitAutopilot(ctx context.Context) error + // InsertBufferedSlab inserts a buffered slab into the database. This // includes the creation of a buffered slab as well as the corresponding // regular slab it is linked to. It returns the ID of the buffered slab @@ -320,8 +319,7 @@ type ( // UnspentSiacoinElements returns all wallet outputs in the database. UnspentSiacoinElements(ctx context.Context) ([]types.SiacoinElement, error) - // UpdateAutopilot updates the autopilot with the provided one or - // creates a new one if it doesn't exist yet. + // UpdateAutopilot updates the autopilot in the database. UpdateAutopilot(ctx context.Context, ap api.Autopilot) error // UpdateBucketPolicy updates the policy of the bucket with the provided @@ -343,7 +341,7 @@ type ( UpdateHostBlocklistEntries(ctx context.Context, add, remove []string, clear bool) error // UpdateHostCheck updates the host check for the given host. - UpdateHostCheck(ctx context.Context, autopilot string, hk types.PublicKey, hc api.HostCheck) error + UpdateHostCheck(ctx context.Context, hk types.PublicKey, hc api.HostChecks) error // UpdatePeerInfo updates the metadata for the specified peer. UpdatePeerInfo(ctx context.Context, addr string, fn func(*syncer.PeerInfo)) error diff --git a/stores/sql/main.go b/stores/sql/main.go index 8f90eea10..d545b58c6 100644 --- a/stores/sql/main.go +++ b/stores/sql/main.go @@ -30,9 +30,8 @@ import ( ) var ( - ErrNegativeOffset = errors.New("offset can not be negative") - ErrMissingAutopilotID = errors.New("missing autopilot id") - ErrSettingNotFound = errors.New("setting not found") + ErrNegativeOffset = errors.New("offset can not be negative") + ErrSettingNotFound = errors.New("setting not found") ) // helper types @@ -194,33 +193,46 @@ func ArchiveContract(ctx context.Context, tx sql.Tx, fcid types.FileContractID, return nil } -func Autopilot(ctx context.Context, tx sql.Tx, id string) (api.Autopilot, error) { - row := tx.QueryRow(ctx, "SELECT identifier, config, current_period FROM autopilots WHERE identifier = ?", id) - ap, err := scanAutopilot(row) - if errors.Is(err, dsql.ErrNoRows) { - return api.Autopilot{}, api.ErrAutopilotNotFound - } else if err != nil { - return api.Autopilot{}, fmt.Errorf("failed to fetch autopilot: %w", err) - } - return ap, nil +func Autopilot(ctx context.Context, tx sql.Tx) (ap api.Autopilot, err error) { + err = tx.QueryRow(ctx, ` +SELECT + enabled, + current_period, + contracts_set, + contracts_amount, + contracts_period, + contracts_renew_window, + contracts_download, + contracts_upload, + contracts_storage, + contracts_prune, + hosts_allow_redundant_ips, + hosts_max_downtime_hours, + hosts_min_protocol_version, + hosts_max_consecutive_scan_failures +FROM autopilot_config +WHERE id = ?`, sql.AutopilotID).Scan( + &ap.Enabled, + &ap.CurrentPeriod, + &ap.Contracts.Set, + &ap.Contracts.Amount, + &ap.Contracts.Period, + &ap.Contracts.RenewWindow, + &ap.Contracts.Download, + &ap.Contracts.Upload, + &ap.Contracts.Storage, + &ap.Contracts.Prune, + &ap.Hosts.AllowRedundantIPs, + &ap.Hosts.MaxDowntimeHours, + &ap.Hosts.MinProtocolVersion, + &ap.Hosts.MaxConsecutiveScanFailures, + ) + return } -func Autopilots(ctx context.Context, tx sql.Tx) ([]api.Autopilot, error) { - rows, err := tx.Query(ctx, "SELECT identifier, config, current_period FROM autopilots") - if err != nil { - return nil, fmt.Errorf("failed to fetch autopilots: %w", err) - } - defer rows.Close() - - var autopilots []api.Autopilot - for rows.Next() { - ap, err := scanAutopilot(rows) - if err != nil { - return nil, fmt.Errorf("failed to scan autopilot: %w", err) - } - autopilots = append(autopilots, ap) - } - return autopilots, nil +func AutopilotPeriod(ctx context.Context, tx sql.Tx) (period uint64, err error) { + err = tx.QueryRow(ctx, `SELECT current_period FROM autopilot_config WHERE id = ?`, sql.AutopilotID).Scan(&period) + return } func Bucket(ctx context.Context, tx sql.Tx, bucket string) (api.Bucket, error) { @@ -694,8 +706,6 @@ func HostBlocklist(ctx context.Context, tx sql.Tx) ([]string, error) { func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, error) { if opts.Offset < 0 { return nil, ErrNegativeOffset - } else if opts.AutopilotID == "" && opts.UsabilityMode != "" && opts.UsabilityMode != api.UsabilityFilterModeAll { - return nil, fmt.Errorf("%w: have to specify autopilot id when filter mode isn't 'all'", ErrMissingAutopilotID) } var hasAllowlist, hasBlocklist bool @@ -717,17 +727,6 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er var whereExprs []string var args []any - // fetch autopilot id - var autopilotID int64 - if opts.AutopilotID != "" { - if err := tx.QueryRow(ctx, "SELECT id FROM autopilots WHERE identifier = ?", opts.AutopilotID). - Scan(&autopilotID); errors.Is(err, dsql.ErrNoRows) { - return nil, api.ErrAutopilotNotFound - } else if err != nil { - return nil, fmt.Errorf("failed to fetch autopilot id: %w", err) - } - } - // filter allowlist/blocklist switch opts.FilterMode { case api.HostFilterModeAllowed: @@ -769,17 +768,12 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er } // filter usability - whereApExpr := "" - if opts.AutopilotID != "" { - whereApExpr = "AND hc.db_autopilot_id = ?" - + if opts.UsabilityMode != api.UsabilityFilterModeAll { switch opts.UsabilityMode { case api.UsabilityFilterModeUsable: - whereExprs = append(whereExprs, fmt.Sprintf("EXISTS (SELECT 1 FROM hosts h2 INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND h2.id = h.id WHERE (hc.usability_blocked = 0 AND hc.usability_offline = 0 AND hc.usability_low_score = 0 AND hc.usability_redundant_ip = 0 AND hc.usability_gouging = 0 AND hc.usability_not_accepting_contracts = 0 AND hc.usability_not_announced = 0 AND hc.usability_not_completing_scan = 0) %s)", whereApExpr)) - args = append(args, autopilotID) + whereExprs = append(whereExprs, "EXISTS (SELECT 1 FROM hosts h2 INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND h2.id = h.id WHERE (hc.usability_blocked = 0 AND hc.usability_offline = 0 AND hc.usability_low_score = 0 AND hc.usability_redundant_ip = 0 AND hc.usability_gouging = 0 AND hc.usability_not_accepting_contracts = 0 AND hc.usability_not_announced = 0 AND hc.usability_not_completing_scan = 0))") case api.UsabilityFilterModeUnusable: - whereExprs = append(whereExprs, fmt.Sprintf("EXISTS (SELECT 1 FROM hosts h2 INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND h2.id = h.id WHERE (hc.usability_blocked = 1 OR hc.usability_offline = 1 OR hc.usability_low_score = 1 OR hc.usability_redundant_ip = 1 OR hc.usability_gouging = 1 OR hc.usability_not_accepting_contracts = 1 OR hc.usability_not_announced = 1 OR hc.usability_not_completing_scan = 1) %s)", whereApExpr)) - args = append(args, autopilotID) + whereExprs = append(whereExprs, "EXISTS (SELECT 1 FROM hosts h2 INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND h2.id = h.id WHERE (hc.usability_blocked = 1 OR hc.usability_offline = 1 OR hc.usability_low_score = 1 OR hc.usability_redundant_ip = 1 OR hc.usability_gouging = 1 OR hc.usability_not_accepting_contracts = 1 OR hc.usability_not_announced = 1 OR hc.usability_not_completing_scan = 1))") } } @@ -834,15 +828,57 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er } rows, err = tx.Query(ctx, fmt.Sprintf(` - SELECT h.id, h.created_at, h.last_announcement, h.public_key, h.net_address, h.price_table, h.price_table_expiry, - h.settings, h.total_scans, h.last_scan, h.last_scan_success, h.second_to_last_scan_success, - h.uptime, h.downtime, h.successful_interactions, h.failed_interactions, COALESCE(h.lost_sectors, 0), - h.scanned, h.resolved_addresses, %s - FROM hosts h - %s - %s - %s - `, blockedExpr, whereExpr, orderByExpr, offsetLimitStr), args...) +SELECT + h.created_at, + h.last_announcement, + h.public_key, + h.net_address, + + h.price_table, + h.price_table_expiry, + h.settings, + + h.total_scans, + h.last_scan, + h.last_scan_success, + h.second_to_last_scan_success, + h.uptime, + h.downtime, + h.successful_interactions, + h.failed_interactions, + COALESCE(h.lost_sectors, 0), + h.scanned, + h.resolved_addresses, + + %s, + + COALESCE(hc.usability_blocked, 0), + COALESCE(hc.usability_offline, 0), + COALESCE(hc.usability_low_score, 0), + COALESCE(hc.usability_redundant_ip, 0), + COALESCE(hc.usability_gouging, 0), + COALESCE(hc.usability_not_accepting_contracts, 0), + COALESCE(hc.usability_not_announced, 0), + COALESCE(hc.usability_not_completing_scan, 0), + + COALESCE(hc.score_age,0), + COALESCE(hc.score_collateral,0), + COALESCE(hc.score_interactions,0), + COALESCE(hc.score_storage_remaining,0), + COALESCE(hc.score_uptime,0), + COALESCE(hc.score_version,0), + COALESCE(hc.score_prices,0), + + COALESCE(hc.gouging_contract_err, ""), + COALESCE(hc.gouging_download_err, ""), + COALESCE(hc.gouging_gouging_err, ""), + COALESCE(hc.gouging_prune_err, ""), + COALESCE(hc.gouging_upload_err, "") +FROM hosts h +LEFT JOIN host_checks hc ON hc.db_host_id = h.id +%s +%s +%s`, blockedExpr, whereExpr, orderByExpr, offsetLimitStr), args...) if err != nil { return nil, fmt.Errorf("failed to fetch hosts: %w", err) } @@ -851,16 +887,18 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er var hosts []api.Host for rows.Next() { var h api.Host - var hostID int64 var pte dsql.NullTime var resolvedAddresses string - err := rows.Scan(&hostID, &h.KnownSince, &h.LastAnnouncement, (*PublicKey)(&h.PublicKey), + err := rows.Scan(&h.KnownSince, &h.LastAnnouncement, (*PublicKey)(&h.PublicKey), &h.NetAddress, (*PriceTable)(&h.PriceTable.HostPriceTable), &pte, (*HostSettings)(&h.Settings), &h.Interactions.TotalScans, (*UnixTimeMS)(&h.Interactions.LastScan), &h.Interactions.LastScanSuccess, &h.Interactions.SecondToLastScanSuccess, (*DurationMS)(&h.Interactions.Uptime), (*DurationMS)(&h.Interactions.Downtime), &h.Interactions.SuccessfulInteractions, &h.Interactions.FailedInteractions, &h.Interactions.LostSectors, - &h.Scanned, &resolvedAddresses, &h.Blocked, - ) + &h.Scanned, &resolvedAddresses, &h.Blocked, &h.Checks.UsabilityBreakdown.Blocked, &h.Checks.UsabilityBreakdown.Offline, &h.Checks.UsabilityBreakdown.LowScore, &h.Checks.UsabilityBreakdown.RedundantIP, + &h.Checks.UsabilityBreakdown.Gouging, &h.Checks.UsabilityBreakdown.NotAcceptingContracts, &h.Checks.UsabilityBreakdown.NotAnnounced, &h.Checks.UsabilityBreakdown.NotCompletingScan, + &h.Checks.ScoreBreakdown.Age, &h.Checks.ScoreBreakdown.Collateral, &h.Checks.ScoreBreakdown.Interactions, &h.Checks.ScoreBreakdown.StorageRemaining, &h.Checks.ScoreBreakdown.Uptime, + &h.Checks.ScoreBreakdown.Version, &h.Checks.ScoreBreakdown.Prices, &h.Checks.GougingBreakdown.ContractErr, &h.Checks.GougingBreakdown.DownloadErr, &h.Checks.GougingBreakdown.GougingErr, + &h.Checks.GougingBreakdown.PruneErr, &h.Checks.GougingBreakdown.UploadErr) if err != nil { return nil, fmt.Errorf("failed to scan host: %w", err) } @@ -876,57 +914,6 @@ func Hosts(ctx context.Context, tx sql.Tx, opts api.HostOptions) ([]api.Host, er h.StoredData = storedDataMap[h.PublicKey] hosts = append(hosts, h) } - - // query host checks - var apExpr string - if opts.AutopilotID != "" { - apExpr = "WHERE ap.identifier = ?" - args = append(args, opts.AutopilotID) - } - rows, err = tx.Query(ctx, fmt.Sprintf(` - SELECT h.public_key, ap.identifier, hc.usability_blocked, hc.usability_offline, hc.usability_low_score, hc.usability_redundant_ip, - hc.usability_gouging, usability_not_accepting_contracts, hc.usability_not_announced, hc.usability_not_completing_scan, - hc.score_age, hc.score_collateral, hc.score_interactions, hc.score_storage_remaining, hc.score_uptime, - hc.score_version, hc.score_prices, hc.gouging_contract_err, hc.gouging_download_err, hc.gouging_gouging_err, - hc.gouging_prune_err, hc.gouging_upload_err - FROM ( - SELECT h.id, h.public_key - FROM hosts h - %s - %s - ) AS h - INNER JOIN host_checks hc ON hc.db_host_id = h.id - INNER JOIN autopilots ap ON hc.db_autopilot_id = ap.id - %s - `, whereExpr, offsetLimitStr, apExpr), args...) - if err != nil { - return nil, fmt.Errorf("failed to fetch host checks: %w", err) - } - defer rows.Close() - - hostChecks := make(map[types.PublicKey]map[string]api.HostCheck) - for rows.Next() { - var ap string - var pk PublicKey - var hc api.HostCheck - err := rows.Scan(&pk, &ap, &hc.UsabilityBreakdown.Blocked, &hc.UsabilityBreakdown.Offline, &hc.UsabilityBreakdown.LowScore, &hc.UsabilityBreakdown.RedundantIP, - &hc.UsabilityBreakdown.Gouging, &hc.UsabilityBreakdown.NotAcceptingContracts, &hc.UsabilityBreakdown.NotAnnounced, &hc.UsabilityBreakdown.NotCompletingScan, - &hc.ScoreBreakdown.Age, &hc.ScoreBreakdown.Collateral, &hc.ScoreBreakdown.Interactions, &hc.ScoreBreakdown.StorageRemaining, &hc.ScoreBreakdown.Uptime, - &hc.ScoreBreakdown.Version, &hc.ScoreBreakdown.Prices, &hc.GougingBreakdown.ContractErr, &hc.GougingBreakdown.DownloadErr, &hc.GougingBreakdown.GougingErr, - &hc.GougingBreakdown.PruneErr, &hc.GougingBreakdown.UploadErr) - if err != nil { - return nil, fmt.Errorf("failed to scan host: %w", err) - } - if _, ok := hostChecks[types.PublicKey(pk)]; !ok { - hostChecks[types.PublicKey(pk)] = make(map[string]api.HostCheck) - } - hostChecks[types.PublicKey(pk)][ap] = hc - } - - // fill in hosts - for i := range hosts { - hosts[i].Checks = hostChecks[hosts[i].PublicKey] - } return hosts, nil } @@ -2064,6 +2051,42 @@ WHERE fcid = ?`, return nil } +func UpdateAutopilot(ctx context.Context, tx sql.Tx, ap api.Autopilot) error { + _, err := tx.Exec(ctx, ` +UPDATE autopilot_config +SET enabled = ?, + current_period = ?, + contracts_set = ?, + contracts_amount = ?, + contracts_period = ?, + contracts_renew_window = ?, + contracts_download = ?, + contracts_upload = ?, + contracts_storage = ?, + contracts_prune = ?, + hosts_allow_redundant_ips = ?, + hosts_max_downtime_hours = ?, + hosts_min_protocol_version = ?, + hosts_max_consecutive_scan_failures = ? +WHERE id = ?`, + ap.Enabled, + ap.CurrentPeriod, + ap.Contracts.Set, + ap.Contracts.Amount, + ap.Contracts.Period, + ap.Contracts.RenewWindow, + ap.Contracts.Download, + ap.Contracts.Upload, + ap.Contracts.Storage, + ap.Contracts.Prune, + ap.Hosts.AllowRedundantIPs, + ap.Hosts.MaxDowntimeHours, + ap.Hosts.MinProtocolVersion, + ap.Hosts.MaxConsecutiveScanFailures, + sql.AutopilotID) + return err +} + func UpdatePeerInfo(ctx context.Context, tx sql.Tx, addr string, fn func(*syncer.PeerInfo)) error { info, err := PeerInfo(ctx, tx, addr) if err != nil { @@ -2210,18 +2233,12 @@ func UsableHosts(ctx context.Context, tx sql.Tx) ([]HostInfo, error) { whereExprs = append(whereExprs, "NOT EXISTS (SELECT 1 FROM host_blocklist_entry_hosts hbeh WHERE hbeh.db_host_id = h.id)") } - // fetch autopilot - var autopilotID int64 - if err := tx.QueryRow(ctx, "SELECT id FROM autopilots WHERE identifier = ?", api.DefaultAutopilotID).Scan(&autopilotID); err != nil { - return nil, fmt.Errorf("failed to fetch autopilot id: %w", err) - } - // only include usable hosts whereExprs = append(whereExprs, ` EXISTS ( SELECT 1 FROM hosts h2 - INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND hc.db_autopilot_id = ? AND h2.id = h.id + INNER JOIN host_checks hc ON hc.db_host_id = h2.id AND h2.id = h.id WHERE hc.usability_blocked = 0 AND hc.usability_offline = 0 AND @@ -2243,9 +2260,9 @@ EXISTS ( h.settings FROM hosts h INNER JOIN contracts c on c.host_id = h.id and c.archival_reason IS NULL - INNER JOIN host_checks hc on hc.db_host_id = h.id and hc.db_autopilot_id = ? + INNER JOIN host_checks hc on hc.db_host_id = h.id WHERE %s - GROUP by h.id`, strings.Join(whereExprs, "AND")), autopilotID, autopilotID) + GROUP by h.id`, strings.Join(whereExprs, " AND "))) if err != nil { return nil, fmt.Errorf("failed to fetch hosts: %w", err) } @@ -2310,14 +2327,6 @@ func WalletEventCount(ctx context.Context, tx sql.Tx) (count uint64, err error) return uint64(n), nil } -func scanAutopilot(s Scanner) (api.Autopilot, error) { - var a api.Autopilot - if err := s.Scan(&a.ID, (*AutopilotConfig)(&a.Config), &a.CurrentPeriod); err != nil { - return api.Autopilot{}, err - } - return a, nil -} - func scanBucket(s Scanner) (api.Bucket, error) { var createdAt time.Time var name, policy string diff --git a/stores/sql/mysql/main.go b/stores/sql/mysql/main.go index c0aa13591..d233d8489 100644 --- a/stores/sql/mysql/main.go +++ b/stores/sql/mysql/main.go @@ -71,6 +71,11 @@ func (b *MainDatabase) LoadSlabBuffers(ctx context.Context) ([]ssql.LoadedSlabBu return ssql.LoadSlabBuffers(ctx, b.db) } +func (b *MainDatabase) InitAutopilot(ctx context.Context, tx sql.Tx) error { + mtx := b.wrapTxn(tx) + return mtx.InitAutopilot(ctx) +} + func (b *MainDatabase) InsertDirectories(ctx context.Context, tx sql.Tx, bucket, path string) (int64, error) { mtx := b.wrapTxn(tx) return mtx.InsertDirectoriesDeprecated(ctx, bucket, path) @@ -193,12 +198,8 @@ func (tx *MainDatabaseTx) ArchiveContract(ctx context.Context, fcid types.FileCo return ssql.ArchiveContract(ctx, tx, fcid, reason) } -func (tx *MainDatabaseTx) Autopilot(ctx context.Context, id string) (api.Autopilot, error) { - return ssql.Autopilot(ctx, tx, id) -} - -func (tx *MainDatabaseTx) Autopilots(ctx context.Context) ([]api.Autopilot, error) { - return ssql.Autopilots(ctx, tx) +func (tx *MainDatabaseTx) Autopilot(ctx context.Context) (api.Autopilot, error) { + return ssql.Autopilot(ctx, tx) } func (tx *MainDatabaseTx) BanPeer(ctx context.Context, addr string, duration time.Duration, reason string) error { @@ -406,6 +407,42 @@ func (tx *MainDatabaseTx) Hosts(ctx context.Context, opts api.HostOptions) ([]ap return ssql.Hosts(ctx, tx, opts) } +func (tx *MainDatabaseTx) InitAutopilot(ctx context.Context) error { + _, err := tx.Exec(ctx, ` +INSERT IGNORE INTO autopilot_config ( + id, + created_at, + current_period, + contracts_set, + contracts_amount, + contracts_period, + contracts_renew_window, + contracts_download, + contracts_upload, + contracts_storage, + contracts_prune, + hosts_allow_redundant_ips, + hosts_max_consecutive_scan_failures, + hosts_max_downtime_hours, + hosts_min_protocol_version +) VALUES (?, ?, 0, "autopilot", ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`, + sql.AutopilotID, + time.Now(), + api.DefaultAutopilotConfig.Contracts.Amount, + api.DefaultAutopilotConfig.Contracts.Period, + api.DefaultAutopilotConfig.Contracts.RenewWindow, + api.DefaultAutopilotConfig.Contracts.Download, + api.DefaultAutopilotConfig.Contracts.Upload, + api.DefaultAutopilotConfig.Contracts.Storage, + api.DefaultAutopilotConfig.Contracts.Prune, + api.DefaultAutopilotConfig.Hosts.AllowRedundantIPs, + api.DefaultAutopilotConfig.Hosts.MaxConsecutiveScanFailures, + api.DefaultAutopilotConfig.Hosts.MaxDowntimeHours, + api.DefaultAutopilotConfig.Hosts.MinProtocolVersion, + ) + return err +} + func (tx *MainDatabaseTx) InsertBufferedSlab(ctx context.Context, fileName string, contractSetID int64, ec object.EncryptionKey, minShards, totalShards uint8) (int64, error) { return ssql.InsertBufferedSlab(ctx, tx, fileName, contractSetID, ec, minShards, totalShards) } @@ -919,14 +956,7 @@ func (tx *MainDatabaseTx) UnspentSiacoinElements(ctx context.Context) (elements } func (tx *MainDatabaseTx) UpdateAutopilot(ctx context.Context, ap api.Autopilot) error { - _, err := tx.Exec(ctx, ` - INSERT INTO autopilots (created_at, identifier, config, current_period) - VALUES (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - config = VALUES(config), - current_period = VALUES(current_period) - `, time.Now(), ap.ID, (*ssql.AutopilotConfig)(&ap.Config), ap.CurrentPeriod) - return err + return ssql.UpdateAutopilot(ctx, tx, ap) } func (tx *MainDatabaseTx) UpdateBucketPolicy(ctx context.Context, bucket string, bp api.BucketPolicy) error { @@ -1100,18 +1130,17 @@ func (tx *MainDatabaseTx) UpdateHostBlocklistEntries(ctx context.Context, add, r return nil } -func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, hk types.PublicKey, hc api.HostCheck) error { +func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, hk types.PublicKey, hc api.HostChecks) error { _, err := tx.Exec(ctx, ` - INSERT INTO host_checks (created_at, db_autopilot_id, db_host_id, usability_blocked, usability_offline, usability_low_score, + INSERT INTO host_checks (created_at, db_host_id, usability_blocked, usability_offline, usability_low_score, usability_redundant_ip, usability_gouging, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, score_age, score_collateral, score_interactions, score_storage_remaining, score_uptime, score_version, score_prices, gouging_contract_err, gouging_download_err, gouging_gouging_err, gouging_prune_err, gouging_upload_err) VALUES (?, - (SELECT id FROM autopilots WHERE identifier = ?), (SELECT id FROM hosts WHERE public_key = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE - created_at = VALUES(created_at), db_autopilot_id = VALUES(db_autopilot_id), db_host_id = VALUES(db_host_id), + created_at = VALUES(created_at), db_host_id = VALUES(db_host_id), usability_blocked = VALUES(usability_blocked), usability_offline = VALUES(usability_offline), usability_low_score = VALUES(usability_low_score), usability_redundant_ip = VALUES(usability_redundant_ip), usability_gouging = VALUES(usability_gouging), usability_not_accepting_contracts = VALUES(usability_not_accepting_contracts), usability_not_announced = VALUES(usability_not_announced), usability_not_completing_scan = VALUES(usability_not_completing_scan), @@ -1119,7 +1148,7 @@ func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, score_storage_remaining = VALUES(score_storage_remaining), score_uptime = VALUES(score_uptime), score_version = VALUES(score_version), score_prices = VALUES(score_prices), gouging_contract_err = VALUES(gouging_contract_err), gouging_download_err = VALUES(gouging_download_err), gouging_gouging_err = VALUES(gouging_gouging_err), gouging_prune_err = VALUES(gouging_prune_err), gouging_upload_err = VALUES(gouging_upload_err) - `, time.Now(), autopilot, ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, + `, time.Now(), ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, hc.ScoreBreakdown.Age, hc.ScoreBreakdown.Collateral, hc.ScoreBreakdown.Interactions, hc.ScoreBreakdown.StorageRemaining, hc.ScoreBreakdown.Uptime, hc.ScoreBreakdown.Version, hc.ScoreBreakdown.Prices, hc.GougingBreakdown.ContractErr, hc.GougingBreakdown.DownloadErr, hc.GougingBreakdown.GougingErr, hc.GougingBreakdown.PruneErr, hc.GougingBreakdown.UploadErr, diff --git a/stores/sql/mysql/migrations/main/migration_00028_autopilot_1.sql b/stores/sql/mysql/migrations/main/migration_00028_autopilot_1.sql new file mode 100644 index 000000000..4950d8e54 --- /dev/null +++ b/stores/sql/mysql/migrations/main/migration_00028_autopilot_1.sql @@ -0,0 +1,32 @@ +-- remove references to autopilots table +ALTER TABLE host_checks DROP FOREIGN KEY fk_host_checks_autopilot; +ALTER TABLE host_checks DROP FOREIGN KEY fk_host_checks_host; +ALTER TABLE host_checks DROP COLUMN db_autopilot_id; +ALTER TABLE host_checks DROP INDEX idx_host_checks_id; +ALTER TABLE host_checks ADD UNIQUE INDEX idx_host_checks_id (db_host_id); +ALTER TABLE host_checks ADD CONSTRAINT fk_host_checks_host FOREIGN KEY (db_host_id) REFERENCES hosts(id) ON DELETE CASCADE; + +-- create autopilot table & insert blank state object +CREATE TABLE `autopilot_config` ( + `id` bigint unsigned NOT NULL DEFAULT 1, + `created_at` datetime(3) DEFAULT NULL, + `current_period` bigint unsigned DEFAULT 0, + `enabled` boolean NOT NULL DEFAULT false, + + `contracts_set` varchar(191) DEFAULT NULL, + `contracts_amount` bigint unsigned DEFAULT NULL, + `contracts_period` bigint unsigned DEFAULT NULL, + `contracts_renew_window` bigint unsigned DEFAULT NULL, + `contracts_download` bigint unsigned DEFAULT NULL, + `contracts_upload` bigint unsigned DEFAULT NULL, + `contracts_storage` bigint unsigned DEFAULT NULL, + `contracts_prune` boolean NOT NULL DEFAULT false, + + `hosts_allow_redundant_ips` boolean NOT NULL DEFAULT false, + `hosts_max_downtime_hours` bigint unsigned DEFAULT NULL, + `hosts_min_protocol_version` varchar(191) DEFAULT NULL, + `hosts_max_consecutive_scan_failures` bigint unsigned DEFAULT NULL, + + PRIMARY KEY (`id`), + CHECK (`id` = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/sql/mysql/migrations/main/migration_00028_autopilot_2.sql b/stores/sql/mysql/migrations/main/migration_00028_autopilot_2.sql new file mode 100644 index 000000000..810aa14b9 --- /dev/null +++ b/stores/sql/mysql/migrations/main/migration_00028_autopilot_2.sql @@ -0,0 +1 @@ +DROP TABLE `autopilots`; diff --git a/stores/sql/mysql/migrations/main/schema.sql b/stores/sql/mysql/migrations/main/schema.sql index a415b11e9..3926dc6a1 100644 --- a/stores/sql/mysql/migrations/main/schema.sql +++ b/stores/sql/mysql/migrations/main/schema.sql @@ -1,14 +1,3 @@ --- dbAutopilot -CREATE TABLE `autopilots` ( - `id` bigint unsigned NOT NULL AUTO_INCREMENT, - `created_at` datetime(3) DEFAULT NULL, - `identifier` varchar(191) NOT NULL, - `config` longtext, - `current_period` bigint unsigned DEFAULT '0', - PRIMARY KEY (`id`), - UNIQUE KEY `identifier` (`identifier`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; - -- dbBucket CREATE TABLE `buckets` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT, @@ -369,7 +358,6 @@ CREATE TABLE `host_checks` ( `id` bigint unsigned NOT NULL AUTO_INCREMENT, `created_at` datetime(3) DEFAULT NULL, - `db_autopilot_id` bigint unsigned NOT NULL, `db_host_id` bigint unsigned NOT NULL, `usability_blocked` boolean NOT NULL DEFAULT false, @@ -396,7 +384,7 @@ CREATE TABLE `host_checks` ( `gouging_upload_err` text, PRIMARY KEY (`id`), - UNIQUE KEY `idx_host_checks_id` (`db_autopilot_id`, `db_host_id`), + UNIQUE KEY `idx_host_checks_id` (`db_host_id`), INDEX `idx_host_checks_usability_blocked` (`usability_blocked`), INDEX `idx_host_checks_usability_offline` (`usability_offline`), INDEX `idx_host_checks_usability_low_score` (`usability_low_score`), @@ -413,7 +401,6 @@ CREATE TABLE `host_checks` ( INDEX `idx_host_checks_score_version` (`score_version`), INDEX `idx_host_checks_score_prices` (`score_prices`), - CONSTRAINT `fk_host_checks_autopilot` FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, CONSTRAINT `fk_host_checks_host` FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; @@ -477,3 +464,27 @@ CREATE TABLE `wallet_outputs` ( UNIQUE KEY `output_id` (`output_id`), KEY `idx_wallet_outputs_maturity_height` (`maturity_height`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; + +CREATE TABLE `autopilot_config` ( + `id` bigint unsigned NOT NULL DEFAULT 1, + `created_at` datetime(3) DEFAULT NULL, + `current_period` bigint unsigned DEFAULT 0, + `enabled` boolean NOT NULL DEFAULT false, + + `contracts_set` varchar(191) DEFAULT NULL, + `contracts_amount` bigint unsigned DEFAULT NULL, + `contracts_period` bigint unsigned DEFAULT NULL, + `contracts_renew_window` bigint unsigned DEFAULT NULL, + `contracts_download` bigint unsigned DEFAULT NULL, + `contracts_upload` bigint unsigned DEFAULT NULL, + `contracts_storage` bigint unsigned DEFAULT NULL, + `contracts_prune` boolean NOT NULL DEFAULT false, + + `hosts_allow_redundant_ips` boolean NOT NULL DEFAULT false, + `hosts_max_downtime_hours` bigint unsigned DEFAULT NULL, + `hosts_min_protocol_version` varchar(191) DEFAULT NULL, + `hosts_max_consecutive_scan_failures` bigint unsigned DEFAULT NULL, + + PRIMARY KEY (`id`), + CHECK (`id` = 1) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; diff --git a/stores/sql/sqlite/main.go b/stores/sql/sqlite/main.go index 72abc0764..383bb1439 100644 --- a/stores/sql/sqlite/main.go +++ b/stores/sql/sqlite/main.go @@ -70,6 +70,11 @@ func (b *MainDatabase) LoadSlabBuffers(ctx context.Context) ([]ssql.LoadedSlabBu return ssql.LoadSlabBuffers(ctx, b.db) } +func (b *MainDatabase) InitAutopilot(ctx context.Context, tx sql.Tx) error { + mtx := b.wrapTxn(tx) + return mtx.InitAutopilot(ctx) +} + func (b *MainDatabase) InsertDirectories(ctx context.Context, tx sql.Tx, bucket, path string) (int64, error) { mtx := b.wrapTxn(tx) return mtx.InsertDirectories(ctx, bucket, path) @@ -192,12 +197,8 @@ func (tx *MainDatabaseTx) ArchiveContract(ctx context.Context, fcid types.FileCo return ssql.ArchiveContract(ctx, tx, fcid, reason) } -func (tx *MainDatabaseTx) Autopilot(ctx context.Context, id string) (api.Autopilot, error) { - return ssql.Autopilot(ctx, tx, id) -} - -func (tx *MainDatabaseTx) Autopilots(ctx context.Context) ([]api.Autopilot, error) { - return ssql.Autopilots(ctx, tx) +func (tx *MainDatabaseTx) Autopilot(ctx context.Context) (api.Autopilot, error) { + return ssql.Autopilot(ctx, tx) } func (tx *MainDatabaseTx) BanPeer(ctx context.Context, addr string, duration time.Duration, reason string) error { @@ -395,6 +396,42 @@ func (tx *MainDatabaseTx) Hosts(ctx context.Context, opts api.HostOptions) ([]ap return ssql.Hosts(ctx, tx, opts) } +func (tx *MainDatabaseTx) InitAutopilot(ctx context.Context) error { + _, err := tx.Exec(ctx, ` +INSERT OR IGNORE INTO autopilot_config ( + id, + created_at, + current_period, + contracts_set, + contracts_amount, + contracts_period, + contracts_renew_window, + contracts_download, + contracts_upload, + contracts_storage, + contracts_prune, + hosts_allow_redundant_ips, + hosts_max_consecutive_scan_failures, + hosts_max_downtime_hours, + hosts_min_protocol_version +) VALUES (?, ?, 0, "autopilot", ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);`, + sql.AutopilotID, + time.Now(), + api.DefaultAutopilotConfig.Contracts.Amount, + api.DefaultAutopilotConfig.Contracts.Period, + api.DefaultAutopilotConfig.Contracts.RenewWindow, + api.DefaultAutopilotConfig.Contracts.Download, + api.DefaultAutopilotConfig.Contracts.Upload, + api.DefaultAutopilotConfig.Contracts.Storage, + api.DefaultAutopilotConfig.Contracts.Prune, + api.DefaultAutopilotConfig.Hosts.AllowRedundantIPs, + api.DefaultAutopilotConfig.Hosts.MaxConsecutiveScanFailures, + api.DefaultAutopilotConfig.Hosts.MaxDowntimeHours, + api.DefaultAutopilotConfig.Hosts.MinProtocolVersion, + ) + return err +} + func (tx *MainDatabaseTx) InsertBufferedSlab(ctx context.Context, fileName string, contractSetID int64, ec object.EncryptionKey, minShards, totalShards uint8) (int64, error) { return ssql.InsertBufferedSlab(ctx, tx, fileName, contractSetID, ec, minShards, totalShards) } @@ -983,14 +1020,7 @@ func (tx *MainDatabaseTx) UnspentSiacoinElements(ctx context.Context) (elements } func (tx *MainDatabaseTx) UpdateAutopilot(ctx context.Context, ap api.Autopilot) error { - _, err := tx.Exec(ctx, ` - INSERT INTO autopilots (created_at, identifier, config, current_period) - VALUES (?, ?, ?, ?) - ON CONFLICT(identifier) DO UPDATE SET - config = EXCLUDED.config, - current_period = EXCLUDED.current_period - `, time.Now(), ap.ID, (*ssql.AutopilotConfig)(&ap.Config), ap.CurrentPeriod) - return err + return ssql.UpdateAutopilot(ctx, tx, ap) } func (tx *MainDatabaseTx) UpdateBucketPolicy(ctx context.Context, bucket string, policy api.BucketPolicy) error { @@ -1107,18 +1137,17 @@ func (tx *MainDatabaseTx) UpdateHostBlocklistEntries(ctx context.Context, add, r return nil } -func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, hk types.PublicKey, hc api.HostCheck) error { +func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, hk types.PublicKey, hc api.HostChecks) error { _, err := tx.Exec(ctx, ` - INSERT INTO host_checks (created_at, db_autopilot_id, db_host_id, usability_blocked, usability_offline, usability_low_score, + INSERT INTO host_checks (created_at, db_host_id, usability_blocked, usability_offline, usability_low_score, usability_redundant_ip, usability_gouging, usability_not_accepting_contracts, usability_not_announced, usability_not_completing_scan, score_age, score_collateral, score_interactions, score_storage_remaining, score_uptime, score_version, score_prices, gouging_contract_err, gouging_download_err, gouging_gouging_err, gouging_prune_err, gouging_upload_err) VALUES (?, - (SELECT id FROM autopilots WHERE identifier = ?), (SELECT id FROM hosts WHERE public_key = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (db_autopilot_id, db_host_id) DO UPDATE SET - created_at = EXCLUDED.created_at, db_autopilot_id = EXCLUDED.db_autopilot_id, db_host_id = EXCLUDED.db_host_id, + ON CONFLICT (db_host_id) DO UPDATE SET + created_at = EXCLUDED.created_at, db_host_id = EXCLUDED.db_host_id, usability_blocked = EXCLUDED.usability_blocked, usability_offline = EXCLUDED.usability_offline, usability_low_score = EXCLUDED.usability_low_score, usability_redundant_ip = EXCLUDED.usability_redundant_ip, usability_gouging = EXCLUDED.usability_gouging, usability_not_accepting_contracts = EXCLUDED.usability_not_accepting_contracts, usability_not_announced = EXCLUDED.usability_not_announced, usability_not_completing_scan = EXCLUDED.usability_not_completing_scan, @@ -1126,7 +1155,7 @@ func (tx *MainDatabaseTx) UpdateHostCheck(ctx context.Context, autopilot string, score_storage_remaining = EXCLUDED.score_storage_remaining, score_uptime = EXCLUDED.score_uptime, score_version = EXCLUDED.score_version, score_prices = EXCLUDED.score_prices, gouging_contract_err = EXCLUDED.gouging_contract_err, gouging_download_err = EXCLUDED.gouging_download_err, gouging_gouging_err = EXCLUDED.gouging_gouging_err, gouging_prune_err = EXCLUDED.gouging_prune_err, gouging_upload_err = EXCLUDED.gouging_upload_err - `, time.Now(), autopilot, ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, + `, time.Now(), ssql.PublicKey(hk), hc.UsabilityBreakdown.Blocked, hc.UsabilityBreakdown.Offline, hc.UsabilityBreakdown.LowScore, hc.UsabilityBreakdown.RedundantIP, hc.UsabilityBreakdown.Gouging, hc.UsabilityBreakdown.NotAcceptingContracts, hc.UsabilityBreakdown.NotAnnounced, hc.UsabilityBreakdown.NotCompletingScan, hc.ScoreBreakdown.Age, hc.ScoreBreakdown.Collateral, hc.ScoreBreakdown.Interactions, hc.ScoreBreakdown.StorageRemaining, hc.ScoreBreakdown.Uptime, hc.ScoreBreakdown.Version, hc.ScoreBreakdown.Prices, hc.GougingBreakdown.ContractErr, hc.GougingBreakdown.DownloadErr, hc.GougingBreakdown.GougingErr, hc.GougingBreakdown.PruneErr, hc.GougingBreakdown.UploadErr, diff --git a/stores/sql/sqlite/migrations/main/migration_00028_autopilot.sql b/stores/sql/sqlite/migrations/main/migration_00028_autopilot.sql new file mode 100644 index 000000000..810aa14b9 --- /dev/null +++ b/stores/sql/sqlite/migrations/main/migration_00028_autopilot.sql @@ -0,0 +1 @@ +DROP TABLE `autopilots`; diff --git a/stores/sql/sqlite/migrations/main/migration_00028_autopilot_1.sql b/stores/sql/sqlite/migrations/main/migration_00028_autopilot_1.sql new file mode 100644 index 000000000..e263a2e09 --- /dev/null +++ b/stores/sql/sqlite/migrations/main/migration_00028_autopilot_1.sql @@ -0,0 +1,8 @@ +-- recreate host_checks +CREATE TABLE `host_checks_temp` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +INSERT INTO `host_checks_temp` SELECT `id`, `created_at`, `db_host_id`, `usability_blocked`, `usability_offline`, `usability_low_score`, `usability_redundant_ip`, `usability_gouging`, `usability_not_accepting_contracts`, `usability_not_announced`, `usability_not_completing_scan`, `score_age`, `score_collateral`, `score_interactions`, `score_storage_remaining`, `score_uptime`, `score_version`, `score_prices`, `gouging_contract_err`, `gouging_download_err`, `gouging_gouging_err`, `gouging_prune_err`, `gouging_upload_err` FROM `host_checks`; +DROP TABLE `host_checks`; +ALTER TABLE `host_checks_temp` RENAME TO `host_checks`; + +-- create autopilot table & insert blank state object +CREATE TABLE autopilot_config (id INTEGER PRIMARY KEY CHECK (id = 1), created_at datetime, current_period integer DEFAULT 0, enabled integer NOT NULL DEFAULT 0, contracts_set text, contracts_amount integer, contracts_period integer, contracts_renew_window integer, contracts_download integer, contracts_upload integer, contracts_storage integer, contracts_prune integer NOT NULL DEFAULT 0, hosts_allow_redundant_ips integer NOT NULL DEFAULT 0, hosts_max_downtime_hours integer, hosts_min_protocol_version text, hosts_max_consecutive_scan_failures integer); \ No newline at end of file diff --git a/stores/sql/sqlite/migrations/main/schema.sql b/stores/sql/sqlite/migrations/main/schema.sql index 555a699f6..6efb093b7 100644 --- a/stores/sql/sqlite/migrations/main/schema.sql +++ b/stores/sql/sqlite/migrations/main/schema.sql @@ -119,9 +119,6 @@ CREATE TABLE `ephemeral_accounts` (`id` integer PRIMARY KEY AUTOINCREMENT,`creat CREATE INDEX `idx_ephemeral_accounts_requires_sync` ON `ephemeral_accounts`(`requires_sync`); CREATE INDEX `idx_ephemeral_accounts_owner` ON `ephemeral_accounts`(`owner`); --- dbAutopilot -CREATE TABLE `autopilots` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`identifier` text NOT NULL UNIQUE,`config` text,`current_period` integer DEFAULT 0); - -- dbWebhook CREATE TABLE `webhooks` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`module` text NOT NULL,`event` text NOT NULL,`url` text NOT NULL,`headers` text DEFAULT ('{}')); CREATE UNIQUE INDEX `idx_module_event_url` ON `webhooks`(`module`,`event`,`url`); @@ -131,8 +128,8 @@ CREATE TABLE `object_user_metadata` (`id` integer PRIMARY KEY AUTOINCREMENT,`cre CREATE UNIQUE INDEX `idx_object_user_metadata_key` ON `object_user_metadata`(`db_object_id`,`db_multipart_upload_id`,`key`); -- dbHostCheck -CREATE TABLE `host_checks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_autopilot_id` INTEGER NOT NULL, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_autopilot_id`) REFERENCES `autopilots` (`id`) ON DELETE CASCADE, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); -CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_autopilot_id`, `db_host_id`); +CREATE TABLE `host_checks` (`id` INTEGER PRIMARY KEY AUTOINCREMENT, `created_at` datetime, `db_host_id` INTEGER NOT NULL, `usability_blocked` INTEGER NOT NULL DEFAULT 0, `usability_offline` INTEGER NOT NULL DEFAULT 0, `usability_low_score` INTEGER NOT NULL DEFAULT 0, `usability_redundant_ip` INTEGER NOT NULL DEFAULT 0, `usability_gouging` INTEGER NOT NULL DEFAULT 0, `usability_not_accepting_contracts` INTEGER NOT NULL DEFAULT 0, `usability_not_announced` INTEGER NOT NULL DEFAULT 0, `usability_not_completing_scan` INTEGER NOT NULL DEFAULT 0, `score_age` REAL NOT NULL, `score_collateral` REAL NOT NULL, `score_interactions` REAL NOT NULL, `score_storage_remaining` REAL NOT NULL, `score_uptime` REAL NOT NULL, `score_version` REAL NOT NULL, `score_prices` REAL NOT NULL, `gouging_contract_err` TEXT, `gouging_download_err` TEXT, `gouging_gouging_err` TEXT, `gouging_prune_err` TEXT, `gouging_upload_err` TEXT, FOREIGN KEY (`db_host_id`) REFERENCES `hosts` (`id`) ON DELETE CASCADE); +CREATE UNIQUE INDEX `idx_host_checks_id` ON `host_checks` (`db_host_id`); CREATE INDEX `idx_host_checks_usability_blocked` ON `host_checks` (`usability_blocked`); CREATE INDEX `idx_host_checks_usability_offline` ON `host_checks` (`usability_offline`); CREATE INDEX `idx_host_checks_usability_low_score` ON `host_checks` (`usability_low_score`); @@ -170,3 +167,6 @@ CREATE INDEX `idx_wallet_events_block_id_height` ON `wallet_events`(`block_id`,` CREATE TABLE `wallet_outputs` (`id` integer PRIMARY KEY AUTOINCREMENT,`created_at` datetime,`output_id` blob NOT NULL,`leaf_index` integer,`merkle_proof` longblob NOT NULL,`value` text,`address` blob,`maturity_height` integer); CREATE UNIQUE INDEX `idx_wallet_outputs_output_id` ON `wallet_outputs`(`output_id`); CREATE INDEX `idx_wallet_outputs_maturity_height` ON `wallet_outputs`(`maturity_height`); + +-- dbAutopilot +CREATE TABLE autopilot_config (id INTEGER PRIMARY KEY CHECK (id = 1), created_at datetime, current_period integer DEFAULT 0, enabled integer NOT NULL DEFAULT 0, contracts_set text, contracts_amount integer, contracts_period integer, contracts_renew_window integer, contracts_download integer, contracts_upload integer, contracts_storage integer, contracts_prune integer NOT NULL DEFAULT 0, hosts_allow_redundant_ips integer NOT NULL DEFAULT 0, hosts_max_downtime_hours integer, hosts_min_protocol_version text, hosts_max_consecutive_scan_failures integer); \ No newline at end of file diff --git a/stores/sql/types.go b/stores/sql/types.go index d436aa1bd..6418651bb 100644 --- a/stores/sql/types.go +++ b/stores/sql/types.go @@ -16,7 +16,6 @@ import ( rhpv3 "go.sia.tech/core/rhp/v3" "go.sia.tech/core/types" "go.sia.tech/coreutils/wallet" - "go.sia.tech/renterd/api" "go.sia.tech/renterd/object" ) @@ -29,23 +28,22 @@ var ( ) type ( - AutopilotConfig api.AutopilotConfig - BCurrency types.Currency - BigInt big.Int - BusSetting string - Currency types.Currency - FileContractID types.FileContractID - Hash256 types.Hash256 - MerkleProof struct{ Hashes []types.Hash256 } - NullableString string - HostSettings rhpv2.HostSettings - PriceTable rhpv3.HostPriceTable - PublicKey types.PublicKey - EncryptionKey object.EncryptionKey - Uint64Str uint64 - UnixTimeMS time.Time - DurationMS time.Duration - Unsigned64 uint64 + BCurrency types.Currency + BigInt big.Int + BusSetting string + Currency types.Currency + FileContractID types.FileContractID + Hash256 types.Hash256 + MerkleProof struct{ Hashes []types.Hash256 } + NullableString string + HostSettings rhpv2.HostSettings + PriceTable rhpv3.HostPriceTable + PublicKey types.PublicKey + EncryptionKey object.EncryptionKey + Uint64Str uint64 + UnixTimeMS time.Time + DurationMS time.Duration + Unsigned64 uint64 StateElement struct { ID Hash256 @@ -60,7 +58,6 @@ type scannerValuer interface { } var ( - _ scannerValuer = (*AutopilotConfig)(nil) _ scannerValuer = (*BCurrency)(nil) _ scannerValuer = (*BigInt)(nil) _ scannerValuer = (*BusSetting)(nil) @@ -78,25 +75,6 @@ var ( _ scannerValuer = (*Unsigned64)(nil) ) -// Scan scan value into AutopilotConfig, implements sql.Scanner interface. -func (cfg *AutopilotConfig) Scan(value interface{}) error { - var bytes []byte - switch value := value.(type) { - case string: - bytes = []byte(value) - case []byte: - bytes = value - default: - return fmt.Errorf("failed to unmarshal AutopilotConfig value: %v %T", value, value) - } - return json.Unmarshal(bytes, cfg) -} - -// Value returns a AutopilotConfig value, implements driver.Valuer interface. -func (cfg AutopilotConfig) Value() (driver.Value, error) { - return json.Marshal(cfg) -} - // Scan implements the sql.Scanner interface. func (sc *BCurrency) Scan(src any) error { buf, ok := src.([]byte)