Add more extensive benchmarking.

Author: adewes

Makefile

@@ -3,6 +3,7 @@
 SHELL := /bin/bash
 
 GOFLAGS ?= -ldflags=\"-extldflags=-static\" $(GOFLAGS:)
+BENCHARGS ?= -run=NONE -bench=.
 
 export KIEBITZ_TEST = yes
 
@@ -32,7 +33,7 @@ test-races: test-setup
 	KIEBITZ_SETTINGS=$(KIEBITZ_TEST_SETTINGS) go test -race $(testargs) `go list ./...`
 
 bench: test-setup
-	KIEBITZ_SETTINGS=$(KIEBITZ_TEST_SETTINGS) go test -run=NONE -bench=. $(GOFLAGS) `go list ./... | grep -v api/`
+	KIEBITZ_SETTINGS=$(KIEBITZ_TEST_SETTINGS) go test $(BENCHARGS) $(GOFLAGS) `go list ./... | grep -v api/`
 
 clean:
 	@go clean $(GOFLAGS) -i ./...

appointments.go

@@ -304,6 +304,29 @@ type SignedAppointment struct {
 	PublicKey   []byte       `json:"publicKey"`
 }
 
+func MakeAppointment(timestamp time.Time, slots, duration int64) (*Appointment, error) {
+	if id, err := crypto.RandomBytes(32); err != nil {
+		return nil, err
+	} else {
+		slotData := make([]*Slot, slots)
+		for i, _ := range slotData {
+			sd := &Slot{}
+			if id, err := crypto.RandomBytes(32); err != nil {
+				return nil, err
+			} else {
+				sd.ID = id
+			}
+			slotData[i] = sd
+		}
+		return &Appointment{
+			Timestamp: timestamp,
+			Duration:  duration,
+			ID:        id,
+			SlotData:  slotData,
+		}, nil
+	}
+}
+
 type Appointment struct {
 	Timestamp  time.Time              `json:"timestamp"`
 	Duration   int64                  `json:"duration"`

databases/redis.go

@@ -49,22 +49,16 @@ type RedisLock struct {
 	dLock       *redislock.Lock
 }
 
-func MakeRedisLock(ctx context.Context, lockKey string, client redis.UniversalClient) (RedisLock, error) {
-
-	rL := RedisLock{
+func MakeRedisLock(ctx context.Context, lockKey string, client redis.UniversalClient) *RedisLock {
+	return &RedisLock{
 		client:      client,
 		lockKey:     lockKey,
 		ctx:         ctx,
 		dLockClient: redislock.New(client),
 	}
-
-	return rL, nil
 }
 
-// Makes sure, that RedisLock implements services.Lock
-var _ services.Lock = RedisLock{}
-
-func (r RedisLock) Lock() error {
+func (r *RedisLock) Lock() error {
 
 	lock, err := r.dLockClient.Obtain(r.ctx, r.lockKey, 100*time.Millisecond, nil)
 
@@ -76,9 +70,9 @@ func (r RedisLock) Lock() error {
 	return nil
 }
 
-func (r RedisLock) Release() error {
+func (r *RedisLock) Release() error {
 	if r.dLock == nil {
-		services.Log.Error("Unable to release lock, which was never locked.")
+		services.Log.Errorf("Unable to release lock %s, which was never locked.", r.lockKey)
 		return errors.New("no lock")
 	}
 
@@ -206,22 +200,13 @@ func (d *Redis) Reset() error {
 
 func (d *Redis) Lock(lockKey string) (services.Lock, error) {
 
-	redisLock, err := MakeRedisLock(d.ctx, lockKey, d.client)
-
-	if err != nil {
-		return nil, err
-	}
-
-	err = redisLock.Lock()
+	redisLock := MakeRedisLock(d.ctx, lockKey, d.client)
 
-	if err != nil {
+	if err := redisLock.Lock(); err != nil {
 		return nil, err
 	}
 
-	var sLock services.Lock
-	sLock = redisLock
-
-	return sLock, nil
+	return redisLock, nil
 
 }
 

helpers/client.go

@@ -41,7 +41,8 @@ type Response struct {
 	body []byte
 }
 
-func (r *Response) read() error {
+func (r *Response) Read() error {
+
 	if r.body != nil {
 		return nil // already read the body
 	}
@@ -74,7 +75,7 @@ func (r *Response) CoerceResult(target interface{}) error {
 func (r *Response) JSON() (map[string]interface{}, error) {
 	var value map[string]interface{}
 
-	if err := r.read(); err != nil {
+	if err := r.Read(); err != nil {
 		return nil, err
 	}
 
@@ -86,7 +87,7 @@ func (r *Response) JSON() (map[string]interface{}, error) {
 }
 
 func (r *Response) Bytes() ([]byte, error) {
-	if err := r.read(); err != nil {
+	if err := r.Read(); err != nil {
 		return nil, err
 	}
 	return r.body, nil
@@ -150,6 +151,10 @@ func Request(url, method string, params interface{}, key *crypto.Key, client *ht
 
 	req, err := http.NewRequest("POST", url, reader)
 
+	// this is important, Golang won't close requests otherwise...
+	// https://github.com/golang/go/issues/28012
+	req.Close = true
+
 	if err != nil {
 		return nil, err
 	}
@@ -159,7 +164,10 @@ func Request(url, method string, params interface{}, key *crypto.Key, client *ht
 	if resp, err := client.Do(req); err != nil {
 		return nil, err
 	} else {
-		return &Response{Response: resp}, nil
+		response := &Response{Response: resp}
+		// we always read the body as otherwise the connection can stay open
+		// and will not be freed properly...
+		return response, response.Read()
 	}
 
 }
@@ -260,7 +268,7 @@ func (a *AppointmentsClient) GetStats(params *services.GetStatsParams) (*Respons
 }
 
 func (a *AppointmentsClient) GetAppointmentsByZipCode(params *services.GetAppointmentsByZipCodeParams) (*Response, error) {
-	return nil, nil
+	return a.requester("getAppointmentsByZipCode", params, nil)
 }
 
 func (a *AppointmentsClient) GetProviderAppointments(params *services.GetProviderAppointmentsParams) (*Response, error) {

http/server.go

@@ -18,10 +18,14 @@ package http
 
 import (
 	"context"
+	cryptoTls "crypto/tls"
 	"fmt"
 	"github.com/kiebitz-oss/services"
 	"github.com/kiebitz-oss/services/metrics"
+	kiebitzNet "github.com/kiebitz-oss/services/net"
+	"github.com/kiebitz-oss/services/tls"
 	"github.com/prometheus/client_golang/prometheus"
+	"net"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -48,10 +52,17 @@ type Route struct {
 
 type H map[string]interface{}
 
+type Hooks struct {
+	Finished Handler
+}
+
 type HTTPServer struct {
-	settings      *services.HTTPServerSettings
+	settings      *HTTPServerSettings
+	tlsConfig     *cryptoTls.Config
+	listener      net.Listener
 	mutex         sync.Mutex
 	running       bool
+	hooks         *Hooks
 	err           error
 	server        *http.Server
 	routeGroups   []*RouteGroup
@@ -81,7 +92,7 @@ func initializeRouteGroup(routeGroup *RouteGroup) error {
 	return nil
 }
 
-func MakeHTTPServer(settings *services.HTTPServerSettings, routeGroups []*RouteGroup, metricsPrefix string) (*HTTPServer, error) {
+func MakeHTTPServer(settings *HTTPServerSettings, routeGroups []*RouteGroup, metricsPrefix string) (*HTTPServer, error) {
 
 	for _, routeGroup := range routeGroups {
 		if err := initializeRouteGroup(routeGroup); err != nil {
@@ -90,12 +101,19 @@ func MakeHTTPServer(settings *services.HTTPServerSettings, routeGroups []*RouteG
 	}
 
 	s := &HTTPServer{
-		metricsPrefix: metricsPrefix,
 		settings:      settings,
 		routeGroups:   routeGroups,
 		mutex:         sync.Mutex{},
+		hooks:         &Hooks{},
+		metricsPrefix: metricsPrefix,
 		server: &http.Server{
 			Addr: settings.BindAddress,
+			// we disable HTTP/2 for all servers as there seems to be a bug in the Golang
+			// HTTP/2 implementation that causes EOF errors when reading from the server
+			// response, which in turn causes trouble with our proxy server when terminating
+			// This can be re-eneabled once the bug is fixed upstream...
+			// more info: https://github.com/golang/go/issues/46071
+			TLSNextProto: make(map[string]func(*http.Server, *cryptoTls.Conn, http.Handler)),
 		},
 	}
 
@@ -105,6 +123,18 @@ func MakeHTTPServer(settings *services.HTTPServerSettings, routeGroups []*RouteG
 	return s, nil
 }
 
+func (h *HTTPServer) SetHooks(hooks *Hooks) {
+	h.hooks = hooks
+}
+
+func (h *HTTPServer) SetListener(listener net.Listener) {
+	h.listener = listener
+}
+
+func (h *HTTPServer) SetTLSConfig(config *cryptoTls.Config) {
+	h.tlsConfig = config
+}
+
 func handleRouteGroup(context *Context, group *RouteGroup, handlers []Handler) {
 
 	for i, route := range group.Routes {
@@ -123,14 +153,16 @@ func handleRouteGroup(context *Context, group *RouteGroup, handlers []Handler) {
 				}
 			}
 		}
+		if context.Aborted {
+			break
+		}
 	}
 
 	for _, subgroup := range group.Subgroups {
 		handleRouteGroup(context, subgroup, append(handlers, group.Handlers...))
 	}
 
 }
-
 func (s *HTTPServer) ServeHTTP(writer http.ResponseWriter, request *http.Request) {
 
 	startHandleTime := time.Now()
@@ -161,6 +193,7 @@ func (s *HTTPServer) ServeHTTP(writer http.ResponseWriter, request *http.Request
 	}
 
 	s.httpDurations.WithLabelValues(u.Path, statusCodeString).Observe(handleDuration.Seconds())
+
 }
 
 func (s *HTTPServer) Start() error {
@@ -180,16 +213,37 @@ func (s *HTTPServer) Start() error {
 
 	var listener func() error
 
-	if s.settings.TLS != nil {
-		// we listen via a TLS connection
-		listener = func() error {
-			return s.server.ListenAndServeTLS(s.settings.TLS.CertificateFile, s.settings.TLS.KeyFile)
+	useTLS := false
+	if s.settings.TLS != nil && s.tlsConfig == nil {
+
+		var err error
+		s.tlsConfig, err = tls.TLSServerConfig(s.settings.TLS)
+
+		if err != nil {
+			return err
+		}
+	}
+
+	if s.tlsConfig != nil {
+		useTLS = true
+		s.server.TLSConfig = s.tlsConfig
+	}
+
+	if s.listener == nil {
+		if listener, err := net.Listen("tcp", s.settings.BindAddress); err != nil {
+			return err
+		} else if s.settings.TCPRateLimits != nil {
+			s.listener = kiebitzNet.MakeRateLimitedListener(listener, s.settings.TCPRateLimits)
+		} else {
+			s.listener = listener
 		}
-	} else {
-		// we listen without TLS (e.g. when we're behind a reverse proxy with TLS termination)
-		listener = func() error {
-			return s.server.ListenAndServe()
+	}
+
+	listener = func() error {
+		if useTLS {
+			return s.server.ServeTLS(s.listener, "", "")
 		}
+		return s.server.Serve(s.listener)
 	}
 
 	go func() {

http/settings.go

@@ -0,0 +1,29 @@
+// Kiebitz - Privacy-Friendly Appointment Scheduling
+// Copyright (C) 2021-2021 The Kiebitz Authors
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package http
+
+import (
+	"github.com/kiebitz-oss/services"
+	"github.com/kiebitz-oss/services/net"
+)
+
+// Settings for the JSON-RPC server
+type HTTPServerSettings struct {
+	TLS           *services.TLSSettings `json:"tls"`
+	BindAddress   string                `json:"bind_address"`
+	TCPRateLimits []*net.RateLimit      `json:"tcp_rate_limits"`
+}

jsonrpc/server.go

@@ -110,7 +110,7 @@ func MakeJSONRPCServer(settings *services.JSONRPCServerSettings, handler Handler
 		},
 	}
 
-	httpServerSettings := &services.HTTPServerSettings{
+	httpServerSettings := &http.HTTPServerSettings{
 		TLS:         settings.TLS,
 		BindAddress: settings.BindAddress,
 	}