Handle CAA requests in Seer and prefix logs with tau #130 (#131)

- added tau to logs
- fixed autocert caa issue + did some cleanup
- cleanup logic of handling dns requests

cli/app/helpers.go

@@ -53,12 +53,12 @@ func setNetworkDomains(conf *config.Source) {
 
 func convertToServiceRegex(url string) string {
 	urls := strings.Split(url, ".")
-	serviceRegex := `^[^.]+\.tau`
+	serviceRegex := `^([^.]+\.)?tau`
 	var network string
 	for _, _url := range urls {
 		network += fmt.Sprintf(`\.%s`, _url)
 	}
 
-	serviceRegex += network
+	serviceRegex += network + "$"
 	return serviceRegex
 }

clients/p2p/auth/vars.go

@@ -2,4 +2,4 @@ package auth
 
 import "github.com/ipfs/go-log/v2"
 
-var logger = log.Logger("auth.api.p2p")
+var logger = log.Logger("tau.auth.api.p2p")

clients/p2p/hoarder/vars.go

@@ -4,4 +4,4 @@ import (
 	"github.com/ipfs/go-log/v2"
 )
 
-var logger = log.Logger("hoarder.p2p.client")
+var logger = log.Logger("tau.hoarder.p2p.client")

clients/p2p/monkey/client.go

@@ -11,7 +11,7 @@ import (
 	protocolCommon "github.com/taubyte/tau/protocols/common"
 )
 
-var logger = log.Logger("monkey.p2p.client")
+var logger = log.Logger("tau.monkey.p2p.client")
 
 var _ iface.Client = &Client{}
 

clients/p2p/patrick/vars.go

@@ -4,4 +4,4 @@ import (
 	"github.com/ipfs/go-log/v2"
 )
 
-var logger = log.Logger("patrick.p2p.client")
+var logger = log.Logger("tau.patrick.p2p.client")

clients/p2p/seer/vars.go

@@ -10,5 +10,5 @@ import (
 var (
 	DefaultGeoBeaconInterval = 5 * time.Minute
 	ErrorGeoBeaconStopped    = errors.New("geoBeacon Stopped")
-	logger                   = log.Logger("seer.p2p.client")
+	logger                   = log.Logger("tau.seer.p2p.client")
 )

clients/p2p/tns/vars.go

@@ -9,5 +9,5 @@ import (
 var (
 	CacheFetchRetryWait       = 1 * time.Second
 	MaximumCacheFetchInterval = 1 * time.Second
-	logger                    = log.Logger("tns.api.p2p")
+	logger                    = log.Logger("tau.tns.api.p2p")
 )

go.mod

@@ -5,6 +5,7 @@ go 1.19
 require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/avast/retry-go v3.0.0+incompatible
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/foxcpp/go-mockdns v1.0.0
 	github.com/fxamacker/cbor/v2 v2.4.0
 	github.com/go-git/go-git/v5 v5.4.2
@@ -52,11 +53,12 @@ require (
 	github.com/taubyte/vm-orbit v1.0.0
 	github.com/urfave/cli/v2 v2.25.7
 	go4.org v0.0.0-20230225012048-214862532bf5
-	golang.org/x/crypto v0.11.0
+	golang.org/x/crypto v0.19.0
 	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1
 	golang.org/x/net v0.12.0
 	golang.org/x/oauth2 v0.10.0
-	golang.org/x/sys v0.10.0
+	golang.org/x/sys v0.17.0
+	golang.org/x/term v0.17.0
 	gopkg.in/go-playground/webhooks.v5 v5.17.0
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -94,7 +96,6 @@ require (
 	github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect
 	github.com/deckarep/golang-set/v2 v2.3.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
-	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/docker v20.10.14+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
@@ -253,8 +254,7 @@ require (
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/term v0.10.0 // indirect
-	golang.org/x/text v0.11.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.11.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -1040,8 +1040,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm
 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
+golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1240,16 +1240,16 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
-golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
+golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1261,8 +1261,8 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

libdream/vars.go

@@ -43,7 +43,7 @@ var (
 	maxUniverses     = 100
 	portsPerUniverse = 100
 
-	logger = log.Logger("dreamland")
+	logger = log.Logger("tau.dreamland")
 
 	universes      map[string]*Universe
 	universesLock  sync.RWMutex

pkgs/http-auto/methods.go

@@ -95,15 +95,14 @@ func New(node, clientNode peer.Node, opts ...options.Option) (*Service, error) {
 // TODO: do a domain validation
 func (s *Service) Start() {
 	go func() {
-		// TODO: run a go-routine that restart the service if certificate expires
 		m := &autocert.Manager{
 			Prompt: autocert.AcceptTOS,
 			Cache:  s.certStore,
 		}
 
 		cfg := &tls.Config{
 			GetCertificate: func(hello *tls.ClientHelloInfo) (cert *tls.Certificate, err error) {
-				logger.Debugf("Looking for a static certificate for %s", hello.ServerName)
+				logger.Debugf("GetCertificate for %s from %s %v", hello.ServerName, hello.Conn.RemoteAddr(), hello.SupportedProtos)
 				hello.ServerName = strings.ToLower(hello.ServerName)
 
 				// Make sure its registered inside tns first and get projectID
@@ -113,8 +112,7 @@ func (s *Service) Start() {
 					if !s.customDomainChecker(hello) {
 						return nil, fmt.Errorf("customDomainChecker for %s was false", hello.ServerName)
 					}
-				} else if !domainSpecs.TaubyteServiceDomain.MatchString(hello.ServerName) &&
-					!domainSpecs.TaubyteHooksDomain.MatchString(hello.ServerName) {
+				} else if !domainSpecs.TaubyteServiceDomain.MatchString(hello.ServerName) {
 					projectId, err := s.validateFromTns(hello.ServerName)
 					if err != nil {
 						return nil, fmt.Errorf("failed validateFromTns for %s with %v", hello.ServerName, err)
@@ -133,14 +131,18 @@ func (s *Service) Start() {
 					}
 				}
 
+				logger.Debugf("looking for certificate for %s", hello.ServerName)
 				cert, err = s.authClient.GetStaticCertificate(hello.ServerName)
 				if err != nil {
-					logger.Debugf("Getting certificate for `%s` predefined", hello.ServerName)
+					logger.Debugf("autocert will handle certificate for `%s`", hello.ServerName)
 					cert, err = m.GetCertificate(hello)
 					if err != nil {
-						logger.Errorf("Getting certificate for `%s` failed: %s", hello.ServerName, err.Error())
-						return nil, fmt.Errorf("failed autocert manager get certificate with %v", err)
+						logger.Errorf("autocert getting certificate for `%s` failed: %s", hello.ServerName, err.Error())
+						return nil, err
 					}
+					logger.Debugf("autocert got certificate for `%s` %v", hello.ServerName, cert)
+				} else {
+					logger.Debugf("got certificate for `%s` %v", hello.ServerName, hello.SupportedProtos)
 				}
 
 				return cert, nil

pkgs/http-auto/vars.go

@@ -6,4 +6,4 @@ import (
 )
 
 var DefaultAllowedMethods = basicHttp.DefaultAllowedMethods
-var logger = logging.Logger("http.auto")
+var logger = logging.Logger("tau.http.auto")

protocols/auth/acme/store/store.go

@@ -14,7 +14,7 @@ import (
 	protocolCommon "github.com/taubyte/tau/protocols/common"
 )
 
-var logger = log.Logger("auth.acme.store")
+var logger = log.Logger("tau.auth.acme.store")
 
 // Store implements Store and Cache using taubyte acme service
 // NOTE: Must periodically chewck the validity of the certificate by a go-routine. If

protocols/auth/api_acme.go

@@ -11,25 +11,23 @@ import (
 	"github.com/taubyte/p2p/streams/command"
 	cr "github.com/taubyte/p2p/streams/command/response"
 	"github.com/taubyte/utils/maps"
+	"golang.org/x/crypto/acme/autocert"
 )
 
-// ErrCacheMiss is returned when a certificate is not found in cache.
-var ErrCacheMiss = errors.New("acme/autocert: certificate cache miss")
-
 // https://golang.org/pkg/crypto/x509/#example_Certificate_Verify
 
 // TODO: validate fqdn
 func (srv *AuthService) setACMECertificate(ctx context.Context, fqdn string, certificate []byte) error {
-	logger.Debugf("Set certificate for `%s`", fqdn)
-	defer logger.Debugf("Set certificate for `%s` done", fqdn)
+	logger.Debugf("Set acme certificate for `%s`", fqdn)
+	defer logger.Debugf("Set acme certificate for `%s` done", fqdn)
 
 	err := srv.db.Put(ctx, "/acme/"+base64.StdEncoding.EncodeToString([]byte(fqdn))+"/certificate/pem", certificate)
 	if err != nil {
-		logger.Errorf("Set certificate for `%s` failed with: %s", fqdn, err.Error())
+		logger.Errorf("Set acme certificate for `%s` failed with: %s", fqdn, err.Error())
 		return err
 	}
 
-	logger.Debugf("Set certificate for `%s` = %v", fqdn, certificate)
+	logger.Debugf("Set acme certificate for `%s` = %v", fqdn, certificate)
 
 	return nil
 }
@@ -52,27 +50,24 @@ func (srv *AuthService) setACMEStaticCertificate(ctx context.Context, fqdn strin
 // TODO: validate fqdn
 // LATER: validate peer has access to it
 func (srv *AuthService) getACMECertificate(ctx context.Context, fqdn string) ([]byte, error) {
-	logger.Debugf("Get certificate for `%s`", fqdn)
-	defer logger.Debugf("Get certificate for `%s` done", fqdn)
+	logger.Debugf("Get acme certificate for `%s`", fqdn)
+	defer logger.Debugf("Get acme certificate for `%s` done", fqdn)
 
 	key := "/acme/" + base64.StdEncoding.EncodeToString([]byte(fqdn)) + "/certificate/pem"
 	certificate, err := srv.db.Get(ctx, key)
 	if err != nil {
-		certificate, err = srv.getACMEStaticCertificate(ctx, fqdn)
-		if err != nil {
-			logger.Error("Get certificate for " + fqdn + " returned " + err.Error())
-			return nil, ErrCacheMiss
-		}
+		logger.Error("Get acme certificate for " + fqdn + " returned " + err.Error())
+		return nil, autocert.ErrCacheMiss
 	}
 
 	if certificate == nil {
 		// cleanup entry
 		logger.Error(fqdn + " : Found empty certificate!")
 		srv.db.Delete(ctx, key)
-		return nil, ErrCacheMiss
+		return nil, autocert.ErrCacheMiss
 	}
 
-	logger.Debugf("Get certificate for `%s`: %v", fqdn, certificate)
+	logger.Debugf("Get acme certificate for `%s`: %v", fqdn, certificate)
 
 	return certificate, nil
 }
@@ -89,15 +84,15 @@ func (srv *AuthService) getACMEStaticCertificate(ctx context.Context, fqdn strin
 		certificate, err = srv.db.Get(ctx, key)
 		if err != nil {
 			logger.Error("Get certificate for " + fqdn + " returned " + err.Error())
-			return nil, ErrCacheMiss
+			return nil, autocert.ErrCacheMiss
 		}
 	}
 
 	if certificate == nil {
 		// cleanup entry
 		logger.Error(fqdn + " : Found empty certificate!")
 		srv.db.Delete(ctx, key)
-		return nil, ErrCacheMiss
+		return nil, autocert.ErrCacheMiss
 	}
 
 	logger.Debugf("Get certificate for `%s`: %v", fqdn, certificate)
@@ -113,14 +108,14 @@ func (srv *AuthService) getACMECache(ctx context.Context, key string) ([]byte, e
 	key_base := "/acme/cache/" + base64.StdEncoding.EncodeToString([]byte(key))
 	data, err := srv.db.Get(ctx, key_base+"/data")
 	if err != nil {
-		return nil, ErrCacheMiss
+		return nil, autocert.ErrCacheMiss
 	}
 
 	if data == nil {
 		logger.Error(key + " : Found empty !")
 		srv.db.Delete(ctx, key_base+"/data")
 		srv.db.Delete(ctx, key_base+"/timestamp")
-		return nil, ErrCacheMiss
+		return nil, autocert.ErrCacheMiss
 	}
 
 	logger.Debugf("Get acme cache for `%s`: %v", key, data)

protocols/auth/hooks/hooks.go

@@ -14,7 +14,7 @@ import (
 
 type Data map[string]interface{}
 
-var logger = log.Logger("auth.hooks")
+var logger = log.Logger("tau.auth.hooks")
 
 type Hook interface {
 	Register(ctx context.Context) error

protocols/auth/projects/methods.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	logger = log.Logger("auth.service.api.projects")
+	logger = log.Logger("tau.auth.service.api.projects")
 )
 
 func (r *ProjectObject) Serialize() Data {

protocols/auth/repositories/repositories.go

@@ -14,7 +14,7 @@ import (
 
 var (
 	GitProviders = []string{"github"}
-	logger       = log.Logger("auth.service.api.repositories")
+	logger       = log.Logger("tau.auth.service.api.repositories")
 )
 
 func (r *GithubRepository) Serialize() Data {

protocols/auth/service.go

@@ -18,7 +18,7 @@ import (
 )
 
 var (
-	logger = log.Logger("auth.service")
+	logger = log.Logger("tau.auth.service")
 )
 
 func New(ctx context.Context, config *tauConfig.Node) (*AuthService, error) {

protocols/gateway/new.go

@@ -15,7 +15,7 @@ import (
 	protocolCommon "github.com/taubyte/tau/protocols/common"
 )
 
-var logger = log.Logger("gateway.service")
+var logger = log.Logger("tau.gateway.service")
 
 // TODO: Substrate Nodes should ping Gateway that they are alive, threshold should
 func New(ctx context.Context, config *tauConfig.Node) (gateway iface.Service, err error) {

protocols/hoarder/vars.go

@@ -9,6 +9,6 @@ import (
 const maxWaitTime = 15 * time.Second
 
 var (
-	logger              = log.Logger("hoarder.service")
+	logger              = log.Logger("tau.hoarder.service")
 	RebroadCastInterval = 5
 )

protocols/monkey/jobs/run.go

@@ -8,7 +8,7 @@ import (
 )
 
 var (
-	logger = log.Logger("monkey.jobs.client")
+	logger = log.Logger("tau.monkey.jobs.client")
 )
 
 func (c *Context) Run(ctx context.Context, ctxC context.CancelFunc) (err error) {

protocols/monkey/service.go

@@ -22,7 +22,7 @@ import (
 	protocolCommon "github.com/taubyte/tau/protocols/common"
 )
 
-var logger = log.Logger("monkey.service")
+var logger = log.Logger("tau.monkey.service")
 
 func (srv *Service) subscribe() error {
 	return srv.node.PubSubSubscribe(

protocols/patrick/service.go

@@ -22,7 +22,7 @@ import (
 
 var (
 	BootstrapTime                   = 10 * time.Second
-	logger                          = log.Logger("patrick.Service")
+	logger                          = log.Logger("tau.patrick.Service")
 	DefaultReAnnounceJobTime        = 7 * time.Minute
 	DefaultReAnnounceFailedJobsTime = 7 * time.Minute
 )