A few issues after e2e testing some edge cases #132 (#133)

- CI/CD jobs that take too long were failing -> need to add an eta update mechanism
- Re announcing CI/CD jobs -> only keep one loop + fix logic that was re-announcing some jobs
- Network FQDN is a global variable -> solve that by relying on config (need updated in config-compiler and go-specs)

cli/app/config_cmd.go

@@ -84,7 +84,6 @@ func configCommand() *cli.Command {
 					&cli.StringFlag{
 						Name:    "network",
 						Aliases: []string{"n", "fqdn"},
-						Value:   "example.com",
 					},
 					&cli.IntFlag{
 						Name:    "p2p-port",

cli/app/gen_config.go

@@ -119,7 +119,7 @@ func generateSourceConfig(ctx *cli.Context) error {
 	ports.Lite = ports.Main + 5
 	ports.Ipfs = ports.Main + 10
 
-	if bundle != nil && ports.Main != 4242 {
+	if bundle != nil && ports.Main == 4242 {
 		ports.Main = bundle.Ports.Main
 		if bundle.Ports.Lite != 0 {
 			ports.Lite = bundle.Ports.Lite
@@ -157,7 +157,7 @@ func generateSourceConfig(ctx *cli.Context) error {
 	}
 
 	var location *seer.Location
-	if bundle != nil {
+	if bundle != nil && bundle.Location != nil {
 		location = bundle.Location
 	} else {
 		location, err = estimateGPSLocation()

cli/app/helpers.go

@@ -3,12 +3,9 @@ package app
 import (
 	"errors"
 	"fmt"
-	"regexp"
 	"strings"
 
 	"github.com/libp2p/go-libp2p/core/pnet"
-	domainSpecs "github.com/taubyte/go-specs/domain"
-	"github.com/taubyte/tau/config"
 )
 
 var (
@@ -44,21 +41,26 @@ func deleteEmpty(s []string) []string {
 	return r
 }
 
-func setNetworkDomains(conf *config.Source) {
-	domainSpecs.WhiteListedDomains = conf.Domains.Whitelist.Postfix
-	domainSpecs.TaubyteServiceDomain = regexp.MustCompile(convertToServiceRegex(conf.NetworkFqdn))
-	domainSpecs.SpecialDomain = regexp.MustCompile(conf.Domains.Generated)
-	domainSpecs.TaubyteHooksDomain = regexp.MustCompile(fmt.Sprintf(`https://patrick.tau.%s`, conf.NetworkFqdn))
+func convertToPostfixRegex(url string) string {
+	urls := strings.Split(url, ".")
+	pRegex := `^([^.]+\.)?`
+	var network string
+	for _, _url := range urls {
+		network += fmt.Sprintf(`\.%s`, _url)
+	}
+
+	pRegex += network[2:] + "$" // skip the first "\."
+	return pRegex
 }
 
-func convertToServiceRegex(url string) string {
+func convertToProtocolsRegex(url string) string {
 	urls := strings.Split(url, ".")
-	serviceRegex := `^([^.]+\.)?tau`
+	pRegex := `^([^.]+\.)?tau`
 	var network string
 	for _, _url := range urls {
 		network += fmt.Sprintf(`\.%s`, _url)
 	}
 
-	serviceRegex += network + "$"
-	return serviceRegex
+	pRegex += network + "$"
+	return pRegex
 }

cli/app/parse_config.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 
 	"github.com/libp2p/go-libp2p/core/crypto"
 	"github.com/libp2p/go-libp2p/core/peer"
@@ -61,20 +62,26 @@ func parseSourceConfig(ctx *cli.Context, shape string) (string, *config.Node, *c
 	}
 
 	protocol := &config.Node{
-		Root:            root,
-		Shape:           shape,
-		P2PAnnounce:     src.P2PAnnounce,
-		P2PListen:       src.P2PListen,
-		Ports:           src.Ports.ToMap(),
-		Location:        src.Location,
-		NetworkFqdn:     src.NetworkFqdn,
-		GeneratedDomain: src.Domains.Generated,
-		ServicesDomain:  convertToServiceRegex(src.NetworkFqdn),
-		HttpListen:      "0.0.0.0:443",
-		Protocols:       src.Protocols,
-		Plugins:         src.Plugins,
-		Peers:           src.Peers,
-		DevMode:         ctx.Bool("dev-mode"),
+		Root:                  root,
+		Shape:                 shape,
+		P2PAnnounce:           src.P2PAnnounce,
+		P2PListen:             src.P2PListen,
+		Ports:                 src.Ports.ToMap(),
+		Location:              src.Location,
+		NetworkFqdn:           src.NetworkFqdn,
+		GeneratedDomain:       src.Domains.Generated,
+		GeneratedDomainRegExp: regexp.MustCompile(convertToPostfixRegex(src.Domains.Generated)),
+		ProtocolsDomainRegExp: regexp.MustCompile(convertToProtocolsRegex(src.NetworkFqdn)),
+		AliasDomainsRegExp:    make([]*regexp.Regexp, 0),
+		HttpListen:            "0.0.0.0:443",
+		Protocols:             src.Protocols,
+		Plugins:               src.Plugins,
+		Peers:                 src.Peers,
+		DevMode:               ctx.Bool("dev-mode"),
+	}
+
+	for _, d := range src.Domains.Aliases {
+		protocol.AliasDomainsRegExp = append(protocol.AliasDomainsRegExp, regexp.MustCompile(convertToPostfixRegex(d)))
 	}
 
 	if len(src.Privatekey) == 0 {

cli/app/service_regex_test.go

@@ -8,11 +8,11 @@ import (
 
 var (
 	testUrl       = "example.test.com"
-	expectedRegex = `^[^.]+\.tau\.example\.test\.com`
+	expectedRegex = `^([^.]+\.)?tau\.example\.test\.com$`
 )
 
-func TestServiceRegex(t *testing.T) {
-	url := convertToServiceRegex(testUrl)
+func TestProtocolsRegex(t *testing.T) {
+	url := convertToProtocolsRegex(testUrl)
 	assert.Equal(t, url, expectedRegex)
 
 }

cli/app/show_config.go

@@ -35,8 +35,8 @@ func displayConfig(pid string, config *config.Node) error {
 
 	domt := table.NewWriter()
 	domt.AppendRows([]table.Row{
-		{"Generated", fmt.Sprintf("Match(`%s`)", config.GeneratedDomain)},
-		{"Services/Protocols", fmt.Sprintf("Match(`%s`)", config.ServicesDomain)},
+		{"Generated", config.GeneratedDomain},
+		{"Aliases", fmt.Sprint(config.AliasDomains)},
 	})
 	domt.SetStyle(table.StyleLight)
 

cli/app/start_cmd.go

@@ -32,7 +32,7 @@ func startCommand() *cli.Command {
 
 		Action: func(ctx *cli.Context) error {
 			shape := ctx.String("shape")
-			_, protocolConfig, sourceConfig, err := parseSourceConfig(ctx, shape)
+			_, protocolConfig, _, err := parseSourceConfig(ctx, shape)
 			if err != nil {
 				return fmt.Errorf("parsing config failed with: %s", err)
 			}
@@ -52,7 +52,6 @@ func startCommand() *cli.Command {
 				cmd.CombinedOutput()
 			}
 
-			setNetworkDomains(sourceConfig)
 			return node.Start(ctx.Context, protocolConfig)
 		},
 	}

clients/p2p/monkey/tests/p2p_test.go

@@ -21,7 +21,7 @@ func TestMonkeyClient(t *testing.T) {
 		return &starfish{Jobs: make(map[string]*patrick.Job, 0)}, nil
 	}
 
-	protocolCommon.LocalPatrick = true
+	protocolCommon.MockedPatrick = true
 
 	u := dreamland.New(dreamland.UniverseConfig{Name: t.Name()})
 	defer u.Stop()

clients/p2p/patrick/client.go

@@ -10,18 +10,19 @@ import (
 )
 
 func New(ctx context.Context, node peer.Node) (iface.Client, error) {
-	var (
-		c   Client
-		err error
-	)
-	if c.client, err = client.New(node, protocolsCommon.PatrickProtocol); err != nil {
+	c := &Client{
+		node: node,
+	}
+
+	var err error
+	if c.Client, err = client.New(c.node, protocolsCommon.PatrickProtocol); err != nil {
 		logger.Error("API client creation failed:", err)
 		return nil, err
 	}
 
-	return &c, nil
+	return c, nil
 }
 
 func (c *Client) Close() {
-	c.client.Close()
+	c.Client.Close()
 }

clients/p2p/patrick/jobs.go

@@ -10,15 +10,15 @@ import (
 )
 
 func (client *Client) Lock(jid string, eta uint32) error {
-	if _, err := client.client.Send("patrick", command.Body{"action": "lock", "jid": jid, "eta": eta}); err != nil {
+	if _, err := client.Send("patrick", command.Body{"action": "lock", "jid": jid, "eta": eta}); err != nil {
 		return fmt.Errorf("failed send lock with error: %w", err)
 	}
 
 	return nil
 }
 
 func (client *Client) IsLocked(jid string) (bool, error) {
-	resp, err := client.client.Send("patrick", command.Body{"action": "isLocked", "jid": jid})
+	resp, err := client.Send("patrick", command.Body{"action": "isLocked", "jid": jid})
 	if err != nil {
 		return false, fmt.Errorf("failed send isLocked with error: %w", err)
 	}
@@ -28,35 +28,41 @@ func (client *Client) IsLocked(jid string) (bool, error) {
 		return false, err
 	}
 
-	return locked, nil
+	by, err := maps.String(resp, "locked-by")
+	if err != nil {
+		return false, err
+	}
+
+	return locked && (by == client.node.ID().String()), nil
 }
 
+// TODO: delete
 func (client *Client) Unlock(jid string) error {
-	if _, err := client.client.Send("patrick", command.Body{"action": "unlock", "jid": jid}); err != nil {
+	if _, err := client.Send("patrick", command.Body{"action": "unlock", "jid": jid}); err != nil {
 		return fmt.Errorf("failed send unlock with error: %w", err)
 	}
 
 	return nil
 }
 
 func (client *Client) Done(jid string, cid_log map[string]string, assetCid map[string]string) error {
-	if _, err := client.client.Send("patrick", command.Body{"action": "done", "jid": jid, "cid": cid_log, "assetCid": assetCid}); err != nil {
+	if _, err := client.Send("patrick", command.Body{"action": "done", "jid": jid, "cid": cid_log, "assetCid": assetCid}); err != nil {
 		return fmt.Errorf("failed sending done with error: %w", err)
 	}
 
 	return nil
 }
 
 func (client *Client) Failed(jid string, cid_log map[string]string, assetCid map[string]string) error {
-	if _, err := client.client.Send("patrick", command.Body{"action": "failed", "jid": jid, "cid": cid_log, "assetCid": assetCid}); err != nil {
+	if _, err := client.Send("patrick", command.Body{"action": "failed", "jid": jid, "cid": cid_log, "assetCid": assetCid}); err != nil {
 		return fmt.Errorf("failed sending failed with error: %w", err)
 	}
 
 	return nil
 }
 
 func (client *Client) Cancel(jid string, cid_log map[string]string) (interface{}, error) {
-	resp, err := client.client.Send("patrick", command.Body{"action": "cancel", "jid": jid, "cid": cid_log})
+	resp, err := client.Send("patrick", command.Body{"action": "cancel", "jid": jid, "cid": cid_log})
 	if err != nil {
 		return nil, fmt.Errorf("failed sending cancel with error: %w", err)
 	}
@@ -65,7 +71,7 @@ func (client *Client) Cancel(jid string, cid_log map[string]string) (interface{}
 }
 
 func (client *Client) List() ([]string, error) {
-	resp, err := client.client.Send("patrick", command.Body{"action": "list", "jid": ""})
+	resp, err := client.Send("patrick", command.Body{"action": "list", "jid": ""})
 	if err != nil {
 		return nil, fmt.Errorf("failed sending list with error: %w", err)
 	}
@@ -79,7 +85,7 @@ func (client *Client) List() ([]string, error) {
 }
 
 func (client *Client) Timeout(jid string) error {
-	if _, err := client.client.Send("patrick", command.Body{"action": "timeout", "jid": jid}); err != nil {
+	if _, err := client.Send("patrick", command.Body{"action": "timeout", "jid": jid}); err != nil {
 		return fmt.Errorf("failed sending timeout with error: %w", err)
 	}
 
@@ -88,7 +94,7 @@ func (client *Client) Timeout(jid string) error {
 
 func (client *Client) Get(jid string) (*iface.Job, error) {
 	var job iface.Job
-	resp, err := client.client.Send("patrick", command.Body{"action": "info", "jid": jid})
+	resp, err := client.Send("patrick", command.Body{"action": "info", "jid": jid})
 	if err != nil {
 		return nil, fmt.Errorf("failed sending list with error: %w", err)
 	}

clients/p2p/patrick/types.go

@@ -2,13 +2,15 @@ package patrick
 
 import (
 	iface "github.com/taubyte/go-interfaces/services/patrick"
+	"github.com/taubyte/p2p/peer"
 	client "github.com/taubyte/p2p/streams/client"
 )
 
 var _ iface.Client = &Client{}
 
 type Client struct {
-	client *client.Client
+	*client.Client
+	node peer.Node
 }
 
 type Peer struct {

config/protocols.go

@@ -2,6 +2,7 @@ package config
 
 import (
 	"errors"
+	"regexp"
 
 	"github.com/taubyte/go-interfaces/kvdb"
 	"github.com/taubyte/go-interfaces/p2p/keypair"
@@ -27,9 +28,14 @@ type Node struct {
 	Ports           map[string]int // TODO: use a struct
 	Location        *seerIface.Location
 	NetworkFqdn     string
-	HttpListen      string
 	GeneratedDomain string
-	ServicesDomain  string //TODO: delete if not useful
+	AliasDomains    []string
+
+	HttpListen string
+
+	AliasDomainsRegExp    []*regexp.Regexp
+	GeneratedDomainRegExp *regexp.Regexp
+	ProtocolsDomainRegExp *regexp.Regexp
 
 	Node       peer.Node
 	PrivateKey []byte

config/tau.go

@@ -54,9 +54,9 @@ type Plugins struct {
 }
 
 type Domains struct {
-	Key       DVKey            `yaml:"key"`
-	Whitelist DomainsWhiteList `yaml:"whitelist"`
-	Generated string           `yaml:"generated"`
+	Key       DVKey    `yaml:"key"`
+	Aliases   []string `yaml:"aliases"`
+	Generated string   `yaml:"generated"`
 }
 
 type DomainsWhiteList struct {

go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/spf13/afero v1.9.5
 	github.com/taubyte/builder v0.2.1
 	github.com/taubyte/cli-common v0.1.1
-	github.com/taubyte/config-compiler v0.4.6
+	github.com/taubyte/config-compiler v0.4.7
 	github.com/taubyte/domain-validation v1.0.1
 	github.com/taubyte/go-interfaces v0.2.14-0.20230928164739-cb43412ebf90
 	github.com/taubyte/go-project-schema v0.9.3
@@ -44,14 +44,15 @@ require (
 	github.com/taubyte/go-seer v1.0.6
 	github.com/taubyte/go-simple-container v0.4.4
 	github.com/taubyte/go-simple-git v0.2.5
-	github.com/taubyte/go-specs v0.10.8-0.20230912140105-e8d804edc77c
+	github.com/taubyte/go-specs v0.10.8
 	github.com/taubyte/http v0.10.5
 	github.com/taubyte/p2p v0.11.0
 	github.com/taubyte/utils v0.1.7
 	github.com/taubyte/vm v1.0.4
 	github.com/taubyte/vm-core-plugins v0.3.4
 	github.com/taubyte/vm-orbit v1.0.0
 	github.com/urfave/cli/v2 v2.25.7
+	go.uber.org/zap v1.24.0
 	go4.org v0.0.0-20230225012048-214862532bf5
 	golang.org/x/crypto v0.19.0
 	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
@@ -251,7 +252,6 @@ require (
 	go.uber.org/dig v1.17.0 // indirect
 	go.uber.org/fx v1.20.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/text v0.14.0 // indirect