feat: module allowlist

LOCAL_DEVELOPMENT.md

@@ -79,3 +79,5 @@ There are two commands that can be used to run existing tests: `./stack unit-tes
 ## Notes on tooling
 
 Things should work right out-of-the-box, no extra configuration should be needed as Doppler provides the environment variables that are required with the current setup.
+
+ß

pkg/data/types.go

@@ -124,6 +124,11 @@ type JobOfferContainer struct {
 	State      uint8    `json:"state"`
 	JobOffer   JobOffer `json:"job_offer"`
 }
+type EnabledModules struct {
+	ModuleID string
+	Version  string
+	Enabled  bool
+}
 
 // posted to the solver by a resource provider
 type ResourceOffer struct {
@@ -140,7 +145,8 @@ type ResourceOffer struct {
 	Spec MachineSpec `json:"spec"`
 	// the module ID's that this resource provider can run
 	// an empty list means ALL modules
-	Modules []string `json:"modules"`
+	Modules []EnabledModules `json:"modules"`
+
 	// tells the solver how to match these prices
 	// for RP this will normally be FixedPrice
 	// we expect the default pricing to be filled in

pkg/data/utils.go

@@ -5,10 +5,9 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/lilypad-tech/lilypad/pkg/web3/bindings/controller"
 	"github.com/ethereum/go-ethereum/common"
-
 	mdag "github.com/ipfs/go-merkledag"
+	"github.com/lilypad-tech/lilypad/pkg/web3/bindings/controller"
 )
 
 // CalculateCID takes an interface, serializes it to JSON, and returns its IPFS CID

pkg/http/utils.go

@@ -11,9 +11,10 @@ import (
 	"net/http"
 	"net/url"
 	"strings"
+	"time"
 
-	"github.com/lilypad-tech/lilypad/pkg/web3"
 	"github.com/hashicorp/go-retryablehttp"
+	"github.com/lilypad-tech/lilypad/pkg/web3"
 	"github.com/rs/zerolog/log"
 )
 
@@ -397,3 +398,31 @@ func newRetryClient() *retryablehttp.Client {
 	}
 	return retryClient
 }
+
+func GetRequestWithTimeout(url string, timeout time.Duration) ([]byte, error) {
+	// A HTTP client with the specified timeout
+	client := &http.Client{
+		Timeout: timeout,
+	}
+
+	// Create the HTTP request
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("creating request failed: %s", err)
+	}
+
+	// Perform the HTTP request
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("executing request failed: %s", err)
+	}
+	defer resp.Body.Close()
+
+	// Read the response body
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("reading response body failed: %s", err)
+	}
+
+	return body, nil
+}

pkg/jobcreator/controller.go

@@ -3,6 +3,7 @@ package jobcreator
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/lilypad-tech/lilypad/pkg/data"
@@ -25,13 +26,19 @@ type JobCreatorController struct {
 	loop                  *system.ControlLoop
 	log                   *system.ServiceLogger
 	jobOfferSubscriptions []JobOfferSubscriber
+	moduleAllowlist       []string
+	enableAllowlist       bool
 }
 
-// the background "even if we have not heard of an event" loop
-// i.e. things will not wait 10 seconds - the control loop
-// reacts to events in the system - this 10 second background
-// loop is just for in case we miss any events
-const CONTROL_LOOP_INTERVAL = 10 * time.Second
+const (
+	ALLOWLIST_URL = "https://raw.githubusercontent.com/lilypad-tech/module-allowlist/main/allowlist.txt"
+
+	// the background "even if we have not heard of an event" loop
+	// i.e. things will not wait 10 seconds - the control loop
+	// reacts to events in the system - this 10 second background
+	// loop is just for in case we miss any events
+	CONTROL_LOOP_INTERVAL = 10 * time.Second
+)
 
 func NewJobCreatorController(
 	options JobCreatorOptions,
@@ -141,6 +148,7 @@ func (controller *JobCreatorController) subscribeToWeb3() error {
 	controller.web3Events.Storage.SubscribeDealStateChange(func(ev storage.StorageDealStateChange) {
 		deal, err := controller.solverClient.GetDeal(ev.DealId)
 		if err != nil {
+			err = fmt.Errorf("module allowlist error: %s", err)
 			controller.log.Error("error getting deal", err)
 			return
 		}
@@ -156,6 +164,34 @@ func (controller *JobCreatorController) subscribeToWeb3() error {
 
 func (controller *JobCreatorController) Start(ctx context.Context, cm *system.CleanupManager) chan error {
 	errorChan := make(chan error)
+
+	// Check the enableAllowlist flag to determine whether to manage the module allowlist
+	if controller.enableAllowlist {
+		// Initial fetch of the module allowlist
+		err := controller.UpdateModuleAllowlist()
+		if err != nil {
+			controller.log.Error("failed to fetch module allowlist", err)
+		}
+
+		// Periodic update logic here using a time.Ticker
+		ticker := time.NewTicker(1 * time.Hour)
+		go func() {
+			for {
+				select {
+				case <-ticker.C:
+					err := controller.UpdateModuleAllowlist()
+					if err != nil {
+						controller.log.Error("periodic module allowlist update failed", err)
+					}
+				case <-ctx.Done():
+					ticker.Stop()
+					return
+				}
+			}
+		}()
+	}
+
+	// Setup subscriptions and other initialization tasks
 	err := controller.subscribeToSolver()
 	if err != nil {
 		errorChan <- err
@@ -166,6 +202,11 @@ func (controller *JobCreatorController) Start(ctx context.Context, cm *system.Cl
 		errorChan <- err
 		return errorChan
 	}
+	err = controller.allowlistApproved()
+	if err != nil {
+		errorChan <- err
+		return errorChan
+	}
 
 	// this connects the websocket client
 	err = controller.solverClient.Start(ctx, cm)
@@ -201,6 +242,38 @@ func (controller *JobCreatorController) Start(ctx context.Context, cm *system.Cl
 	return errorChan
 }
 
+func (controller *JobCreatorController) UpdateModuleAllowlist() error {
+	// Call the new helper function with a 5-second timeout!
+	body, err := http.GetRequestWithTimeout(ALLOWLIST_URL, 5*time.Second)
+	if err != nil {
+		return fmt.Errorf("failed to fetch module allowlist: %s", err)
+	}
+
+	// Update the internal cache of the allowlist
+	controller.moduleAllowlist = strings.Split(strings.TrimSpace(string(body)), "\n")
+	return nil
+}
+
+func (controller *JobCreatorController) allowlistApproved() error {
+	controller.web3Events.Storage.SubscribeDealStateChange(func(ev storage.StorageDealStateChange) {
+		deal, err := controller.solverClient.GetDeal(ev.DealId)
+		if err != nil {
+
+			controller.log.Error("module allowlist error", err)
+			return
+		}
+
+		// error handling out of the allowlist function- this is a critical error!!
+		if deal.JobCreator != controller.web3SDK.GetAddress().String() {
+			return
+		}
+		controller.log.Debug("StorageDealStateChange", data.GetAgreementStateString(ev.State))
+		system.DumpObjectDebug(ev)
+		controller.loop.Trigger()
+	})
+	return nil
+}
+
 /*
  *
  *
@@ -288,6 +361,7 @@ func (controller *JobCreatorController) agreeToDeals() error {
 // list the deals that have results posted but we have not yet checked
 // we do this synchronously to prevent us racing with large result sets
 // also we are the client so have a lower chance of there being a chunky backlog
+
 func (controller *JobCreatorController) checkResults() error {
 	// load all deals in ResultsSubmitted state and don't have either results checked or accepted txs
 	completedDeals, err := controller.solverClient.GetDealsWithFilter(

pkg/options/resource-provider.go

@@ -43,7 +43,11 @@ func GetDefaultResourceProviderOfferOptions() resourceprovider.ResourceProviderO
 		Specs: []data.MachineSpec{},
 		// if an RP wants to only run certain modules they list them here
 		// XXX SECURITY: enforce that they are specified with specific git hashes!
+
+		// Define the default
+
 		Modules: GetDefaultServeOptionStringArray("OFFER_MODULES", []string{}),
+
 		// this is the default pricing mode for an RP
 		Mode: GetDefaultPricingMode(data.FixedPrice),
 		// this is the default pricing for a module unless it has a specific price
@@ -103,6 +107,15 @@ func AddResourceProviderPowCliFlags(cmd *cobra.Command, options *resourceprovide
 	)
 }
 
+// add the enable module allowlist flag- Grab the path- check if the file exists, check the execution
+func AddResourceProviderAllowlistCliFlags(cmd *cobra.Command, options *resourceprovider.ResourceProviderEnabledModules) {
+	cmd.PersistentFlags().BoolVar(
+		&options.EnableAllowlist, "enable-allowlist", options.EnableAllowlist,
+		`Enable the module allowlist (ENABLE_ALLOWLIST)`,
+	)
+}
+
+// If err, throw error at user, and if all that works all to the rp offers
 func AddResourceProviderCliFlags(cmd *cobra.Command, options *resourceprovider.ResourceProviderOptions) {
 	AddBacalhauCliFlags(cmd, &options.Bacalhau)
 	AddWeb3CliFlags(cmd, &options.Web3)
@@ -158,6 +171,8 @@ func CheckResourceProviderOptions(options resourceprovider.ResourceProviderOptio
 	return nil
 }
 
+// Add another check here
+
 func ProcessResourceProviderOfferOptions(options resourceprovider.ResourceProviderOfferOptions, network string) (resourceprovider.ResourceProviderOfferOptions, error) {
 	newServicesOptions, err := ProcessServicesOptions(options.Services, network)
 	if err != nil {

pkg/resourceprovider/controller.go

@@ -266,6 +266,7 @@ func (controller *ResourceProviderController) ensureResourceOffers() error {
 		// check if the resource offer already exists
 		// if it does then we need to update it
 		// if it doesn't then we need to add it
+
 		_, ok := existingResourceOffersMap[index]
 		if !ok {
 			addResourceOffers = append(addResourceOffers, controller.getResourceOffer(index, spec))
@@ -343,6 +344,8 @@ func (controller *ResourceProviderController) agreeToDeals() error {
 		controller.log.Info("updated deal with agree tx", txHash)
 	}
 
+	//
+
 	return err
 
 }

pkg/resourceprovider/resourceprovider.go

@@ -60,11 +60,18 @@ type ResourceProviderPowOptions struct {
 	CudaBlockSize int
 }
 
+// Enabled Modules
+
+type ResourceProviderEnabledModules struct {
+	EnableAllowlist bool
+}
+
 type ResourceProviderOptions struct {
-	Bacalhau bacalhau.BacalhauExecutorOptions
-	Offers   ResourceProviderOfferOptions
-	Web3     web3.Web3Options
-	Pow      ResourceProviderPowOptions
+	Bacalhau  bacalhau.BacalhauExecutorOptions
+	Offers    ResourceProviderOfferOptions
+	Web3      web3.Web3Options
+	Pow       ResourceProviderPowOptions
+	Allowlist ResourceProviderAllowlistOptions
 }
 
 type ResourceProvider struct {