Merge pull request #220 from tooolbox/fix/perf-and-utility

Perf and utility

api/api.go

@@ -6,13 +6,16 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"net/http/pprof"
+	"runtime"
 
 	"github.com/ajvb/kala/api/middleware"
 	"github.com/ajvb/kala/job"
 
-	"github.com/codegangsta/negroni"
 	"github.com/gorilla/mux"
+	"github.com/phyber/negroni-gzip/gzip"
 	log "github.com/sirupsen/logrus"
+	"github.com/urfave/negroni"
 )
 
 const (
@@ -319,14 +322,31 @@ func SetupApiRoutes(r *mux.Router, cache job.JobCache, db job.JobDB, defaultOwne
 	r.HandleFunc(ApiUrlPrefix+"stats/", HandleKalaStatsRequest(cache)).Methods("GET")
 }
 
-func MakeServer(listenAddr string, cache job.JobCache, db job.JobDB, defaultOwner string) *http.Server {
+func MakeServer(listenAddr string, cache job.JobCache, db job.JobDB, defaultOwner string, profile bool) *http.Server {
 	r := mux.NewRouter()
 	// Allows for the use for /job as well as /job/
 	r.StrictSlash(true)
 	SetupApiRoutes(r, cache, db, defaultOwner)
 	r.PathPrefix("/webui/").Handler(http.StripPrefix("/webui/", http.FileServer(http.Dir("./webui/"))))
-	n := negroni.New(negroni.NewRecovery(), &middleware.Logger{log.Logger{}})
+
+	if profile {
+		runtime.SetMutexProfileFraction(5)
+		// Register pprof handlers
+		r.HandleFunc("/debug/pprof/", pprof.Index)
+		r.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		r.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		r.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		r.HandleFunc("/debug/pprof/trace", pprof.Trace)
+		r.Handle("/debug/pprof/goroutine", pprof.Handler("goroutine"))
+		r.Handle("/debug/pprof/heap", pprof.Handler("heap"))
+		r.Handle("/debug/pprof/threadcreate", pprof.Handler("threadcreate"))
+		r.Handle("/debug/pprof/block", pprof.Handler("block"))
+		r.Handle("/debug/pprof/mutex", pprof.Handler("mutex"))
+	}
+
+	n := negroni.New(negroni.NewRecovery(), &middleware.Logger{log.Logger{}}, gzip.Gzip(gzip.DefaultCompression))
 	n.UseHandler(r)
+
 	return &http.Server{
 		Addr:    listenAddr,
 		Handler: n,

api/middleware/logger.go

@@ -4,8 +4,8 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/codegangsta/negroni"
 	log "github.com/sirupsen/logrus"
+	"github.com/urfave/negroni"
 )
 
 // Logger is a middleware handler that logs the request as it goes in and the response as it goes out.

cmd/app.go

@@ -1,12 +1,12 @@
 package cmd
 
 import (
-	"fmt"
-	"strings"
-	"time"
 	"crypto/tls"
 	"crypto/x509"
+	"fmt"
 	"io/ioutil"
+	"strings"
+	"time"
 
 	"github.com/ajvb/kala/api"
 	"github.com/ajvb/kala/job"
@@ -17,9 +17,9 @@ import (
 	"github.com/ajvb/kala/job/storage/postgres"
 	"github.com/ajvb/kala/job/storage/redis"
 
-	"github.com/codegangsta/cli"
 	redislib "github.com/garyburd/redigo/redis"
 	log "github.com/sirupsen/logrus"
+	"github.com/urfave/cli"
 	"gopkg.in/mgo.v2"
 )
 
@@ -47,11 +47,12 @@ func init() {
 					Command: cmd,
 				}
 
-				err := j.RunCmd()
+				out, err := j.RunCmd()
 				if err != nil {
 					log.Fatalf("Command Failed with err: %s", err)
 				} else {
 					fmt.Println("Command Succeeded!")
+					fmt.Println("Output: ", out)
 				}
 			},
 		},
@@ -139,6 +140,10 @@ func init() {
 					Value: -1,
 					Usage: "Sets the jobstat-ttl in minutes. The default -1 value indicates JobStat entries will be kept forever",
 				},
+				cli.BoolFlag{
+					Name:  "profile",
+					Usage: "Activate pprof handlers",
+				},
 			},
 			Action: func(c *cli.Context) {
 				if c.Bool("v") {
@@ -190,26 +195,27 @@ func init() {
 					db = postgres.New(dsn)
 				case "mysql", "mariadb":
 					dsn := fmt.Sprintf("%s:%s@%s", c.String("jobDBUsername"), c.String("jobDBPassword"), c.String("jobDBAddress"))
+					log.Debug("Mysql/Maria DSN: ", dsn)
 					if c.String("jobDBTlsCAPath") != "" {
 						// https://godoc.org/github.com/go-sql-driver/mysql#RegisterTLSConfig
 						rootCertPool := x509.NewCertPool()
 						pem, err := ioutil.ReadFile(c.String("jobDBTlsCAPath"))
 						if err != nil {
-								log.Fatal(err)
+							log.Fatal(err)
 						}
 						if ok := rootCertPool.AppendCertsFromPEM(pem); !ok {
-								log.Fatal("Failed to append PEM.")
+							log.Fatal("Failed to append PEM.")
 						}
 						clientCert := make([]tls.Certificate, 0, 1)
 						certs, err := tls.LoadX509KeyPair(c.String("jobDBTlsCertPath"), c.String("jobDBTlsKeyPath"))
 						if err != nil {
-								log.Fatal(err)
+							log.Fatal(err)
 						}
 						clientCert = append(clientCert, certs)
 						db = mysql.New(dsn, &tls.Config{
-							RootCAs: rootCertPool,
+							RootCAs:      rootCertPool,
 							Certificates: clientCert,
-							ServerName: c.String("jobDBTlsServerName"),
+							ServerName:   c.String("jobDBTlsServerName"),
 						})
 					} else {
 						db = mysql.New(dsn, nil)
@@ -228,7 +234,7 @@ func init() {
 				cache.Start(time.Duration(c.Int("persist-every"))*time.Second, time.Duration(c.Int("jobstat-ttl"))*time.Minute)
 
 				log.Infof("Starting server on port %s", connectionString)
-				srv := api.MakeServer(connectionString, cache, db, c.String("default-owner"))
+				srv := api.MakeServer(connectionString, cache, db, c.String("default-owner"), c.Bool("profile"))
 				log.Fatal(srv.ListenAndServe())
 			},
 		},

go.mod

@@ -4,31 +4,33 @@ go 1.13
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.3.0
+	github.com/armon/go-metrics v0.3.3 // indirect
 	github.com/boltdb/bolt v1.3.1-0.20170131192018-e9cf4fae01b5
-	github.com/codegangsta/cli v1.8.1-0.20150711215404-bca61c476e3c
-	github.com/codegangsta/negroni v0.1.1-0.20150319171304-c7477ad8e330
 	github.com/cornelk/hashmap v1.0.1
-	github.com/dchest/siphash v1.1.0
 	github.com/garyburd/redigo v1.0.1-0.20170208211623-48545177e92a
 	github.com/go-sql-driver/mysql v1.4.1
-	github.com/gorilla/context v0.0.0-20141217160251-215affda49ad
+	github.com/gorilla/context v0.0.0-20141217160251-215affda49ad // indirect
 	github.com/gorilla/mux v0.0.0-20150808061613-ffb3f683aad4
 	github.com/hashicorp/consul v0.9.1-0.20170720214845-3117d9ec2b86
-	github.com/hashicorp/go-cleanhttp v0.0.0-20170211013415-3573b8b52aa7
-	github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90
-	github.com/hashicorp/serf v0.8.2-0.20170714182601-bbeddf0b3ab3
+	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect
+	github.com/hashicorp/go-uuid v1.0.2 // indirect
+	github.com/hashicorp/memberlist v0.2.2 // indirect
+	github.com/hashicorp/serf v0.8.2-0.20170714182601-bbeddf0b3ab3 // indirect
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/lestrrat-go/tcputil v0.0.0-20180223003554-d3c7f98154fb // indirect
 	github.com/lestrrat-go/test-mysqld v0.0.0-20190527004737-6c91be710371
 	github.com/lib/pq v1.0.0
 	github.com/mattn/go-shellwords v1.0.0
-	github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747
+	github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 // indirect
 	github.com/mixer/clock v0.0.0-20190507173039-c311c17adb1f
 	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d
-	github.com/pkg/errors v0.8.1 // indirect
+	github.com/phyber/negroni-gzip v0.0.0-20180113114010-ef6356a5d029
 	github.com/rafaeljusto/redigomock v0.0.0-20170720131524-7ae0511314e9
-	github.com/sirupsen/logrus v0.11.3-0.20170215164324-7f4b1adc7917
-	github.com/stretchr/testify v0.0.0-20150812162424-f5520455607c
-	golang.org/x/sys v0.0.0-20161023150541-c200b10b5d5e
+	github.com/sirupsen/logrus v1.4.2
+	github.com/stretchr/testify v1.4.0
+	github.com/urfave/cli v1.22.4
+	github.com/urfave/negroni v1.0.0
+	google.golang.org/appengine v1.6.6 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528
 )