Skip to content

Commit

Permalink
Merge branch 'release/1.17.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
@mwnicks
mwnicks committed Mar 24, 2021
2 parents 9bde42b + 3bcfec4 commit c7a5639
Show file tree
Hide file tree
Showing 8 changed files with 111 additions and 109 deletions.
4 changes: 2 additions & 2 deletions dp-api-router.nomad
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ job "dp-api-router" {
service {
name = "dp-api-router"
port = "http"
tags = ["web"]
tags = ["web", "system_job"]
check {
type = "http"
path = "/health"
Expand Down Expand Up @@ -115,7 +115,7 @@ job "dp-api-router" {
service {
name = "dp-api-router"
port = "http"
tags = ["publishing"]
tags = ["publishing", "system_job"]
check {
type = "http"
path = "/health"
Expand Down
2 changes: 1 addition & 1 deletion interceptor/interceptor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ const (
)

var (
re = regexp.MustCompile(`^(.+:\/\/)(.+)(\/v\d)$`)
re = regexp.MustCompile(`^(.+://)(.+)(/v\d)$`)
)

// RoundTrip intercepts the response body and post processes to add the correct enviornment
Expand Down
31 changes: 22 additions & 9 deletions interceptor/interceptor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ func TestUnitInterceptor(t *testing.T) {

b, _ := ioutil.ReadAll(resp.Body)

err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 0)
})

Expand All @@ -52,6 +54,8 @@ func TestUnitInterceptor(t *testing.T) {

b, _ := ioutil.ReadAll(resp.Body)

err = resp.Body.Close()
So(err, ShouldBeNil)
So(string(b), ShouldEqual, `{"links":{"self":{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}}}`+"\n")
})

Expand All @@ -66,8 +70,9 @@ func TestUnitInterceptor(t *testing.T) {

b, _ := ioutil.ReadAll(resp.Body)

err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 76)

So(string(b), ShouldEqual, `{"links":{"self":{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}}}`+"\n")
})

Expand All @@ -82,8 +87,9 @@ func TestUnitInterceptor(t *testing.T) {

b, _ := ioutil.ReadAll(resp.Body)

err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 102)

So(string(b), ShouldEqual, `{"@context":"context.json","links":{"self":{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}}}`+"\n")
})

Expand All @@ -98,8 +104,9 @@ func TestUnitInterceptor(t *testing.T) {

b, _ := ioutil.ReadAll(resp.Body)

err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 77)

So(string(b), ShouldEqual, `{"downloads":{"csv":{"href":"https://download.beta.ons.gov.uk/myfile.csv"}}}`+"\n")
})

Expand All @@ -113,8 +120,9 @@ func TestUnitInterceptor(t *testing.T) {
So(err, ShouldBeNil)

b, _ := ioutil.ReadAll(resp.Body)
err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 77)

So(string(b), ShouldEqual, `{"dimensions":[{"href":"https://api.beta.ons.gov.uk/v1/codelists/1234567"}]}`+"\n")
})

Expand All @@ -128,8 +136,9 @@ func TestUnitInterceptor(t *testing.T) {
So(err, ShouldBeNil)

b, _ := ioutil.ReadAll(resp.Body)
err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 88)

So(string(b), ShouldEqual, `{"items":[{"links":{"self":{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}}}]}`+"\n")
})

Expand All @@ -143,8 +152,9 @@ func TestUnitInterceptor(t *testing.T) {
So(err, ShouldBeNil)

b, _ := ioutil.ReadAll(resp.Body)
err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 83)

So(string(b), ShouldEqual, `{"links":{"instances":[{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}]}}`+"\n")
})

Expand All @@ -158,8 +168,9 @@ func TestUnitInterceptor(t *testing.T) {
So(err, ShouldBeNil)

b, _ := ioutil.ReadAll(resp.Body)
err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 91)

So(string(b), ShouldEqual, `{"dimensions":{"time":{"option":{"href":"https://api.beta.ons.gov.uk/v1/datasets/time"}}}}`+"\n")
})

Expand All @@ -173,8 +184,9 @@ func TestUnitInterceptor(t *testing.T) {
So(err, ShouldBeNil)

b, _ := ioutil.ReadAll(resp.Body)
err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 116)

So(string(b), ShouldEqual, `{"dimensions":{"time":{"option":{"href":"https://api.beta.ons.gov.uk/v1/datasets/time?hello=world&mobile=phone"}}}}`+"\n")
})

Expand All @@ -189,8 +201,9 @@ func TestUnitInterceptor(t *testing.T) {

b, _ := ioutil.ReadAll(resp.Body)

err = resp.Body.Close()
So(err, ShouldBeNil)
So(len(b), ShouldEqual, 154)

So(string(b), ShouldEqual, `[{"links":{"self":{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}}},{"links":{"self":{"href":"https://api.beta.ons.gov.uk/v1/datasets/12345"}}}]`+"\n")
})
}
17 changes: 10 additions & 7 deletions middleware/audit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ func AuditHandler(auditProducer *event.AvroProducer,

// Retrieve Identity from Zebedee, which is stored in context.
// if it fails, try to audit with the statusCode before returning
ctx, statusCode, err := retrieveIdentity(w, r, idClient, zebedeeURL)
ctx, statusCode, err := retrieveIdentity(w, r, idClient)
if err != nil {
// error already handled in retrieveIdentity. Try to audit it.
auditEvent.StatusCode = int32(statusCode)
Expand Down Expand Up @@ -156,7 +156,11 @@ func AuditHandler(auditProducer *event.AvroProducer,

// Copy the intercepted body and header to the original response writer
w.WriteHeader(rec.statusCode)
io.Copy(w, rec.body)
_, err = io.Copy(w, rec.body)
if err != nil {
handleError(r.Context(), w, r, http.StatusInternalServerError, "failed to copy intercepted response to original response writer", err, log.Data{"event": auditEvent})
return
}

// Finally send the outbound audit message
auditProducer.Send(eventBytes)
Expand Down Expand Up @@ -207,10 +211,9 @@ func GenerateAuditEvent(req *http.Request) *event.Audit {
return auditEvent
}

// retrieveIdentity requests the user and caller identity from Zebedee, using the provided client and url.
func retrieveIdentity(w http.ResponseWriter, req *http.Request, idClient *clientsidentity.Client, zebedeeURL string) (ctx context.Context, status int, err error) {
// retrieveIdentity requests the user and caller identity from Zebedee, using the provided client.
func retrieveIdentity(w http.ResponseWriter, req *http.Request, idClient *clientsidentity.Client) (ctx context.Context, status int, err error) {
ctx = req.Context()
log.Event(ctx, "executing identity check for auditing purposes", log.INFO)

florenceToken, err := getFlorenceToken(ctx, req)
if err != nil {
Expand All @@ -224,7 +227,7 @@ func retrieveIdentity(w http.ResponseWriter, req *http.Request, idClient *client
return ctx, http.StatusInternalServerError, err
}

// CheckRequest performs the call to Zebedee GET /identity and stores the values in context
// CheckRequest performs the call to Zebedee GET /identity and stores the values in context
ctx, statusCode, authFailure, err := idClient.CheckRequest(req, florenceToken, serviceAuthToken)
logData := log.Data{"auth_status_code": statusCode}
if err != nil {
Expand Down Expand Up @@ -291,7 +294,7 @@ func getServiceAuthToken(ctx context.Context, req *http.Request) (string, error)
return authToken, err
}

// Now is a time.Now wrapper
// Now is a time.Now wrapper specifically for testing purposes, and should not me unlambda'd - despite what golangci-lint says
var Now = func() time.Time {
return time.Now()
}
33 changes: 16 additions & 17 deletions middleware/audit_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ import (
"github.com/ONSdigital/dp-api-router/middleware"
"github.com/ONSdigital/dp-api-router/schema"

"github.com/ONSdigital/dp-kafka/v2"
kafka "github.com/ONSdigital/dp-kafka/v2"
"github.com/ONSdigital/dp-kafka/v2/kafkatest"
dphttp "github.com/ONSdigital/dp-net/http"
dprequest "github.com/ONSdigital/dp-net/request"
Expand All @@ -43,7 +43,6 @@ var (
testTimeMillisOutbound int64 = 1587884752456
testBody = []byte{1, 2, 3, 4}
errMarshal = errors.New("avro marshal error")
errDo = errors.New("identity client Do failed")
)

// valid identity response for testing
Expand All @@ -52,10 +51,11 @@ var testIdentityResponse = &dprequest.IdentityResponse{
}

// utility function to generate handlers for testing, which return the provided status code and body
func testHandler(statusCode int, body []byte) http.Handler {
func testHandler(statusCode int, body []byte, c C) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
w.WriteHeader(statusCode)
w.Write(body)
_, err := w.Write(body)
c.So(err, ShouldBeNil)
})
}

Expand Down Expand Up @@ -89,7 +89,6 @@ func createHTTPClientMock(retCode int, retBody interface{}) *dphttp.ClienterMock
return []string{}
},
SetPathsWithNoRetriesFunc: func([]string) {
return
},
DoFunc: func(ctx context.Context, req *http.Request) (*http.Response, error) {
body, _ := json.Marshal(retBody)
Expand Down Expand Up @@ -215,7 +214,7 @@ func TestAuditHandlerHeaders(t *testing.T) {

Convey("And a valid audit handler with successful downstream", func(c C) {
p, a := createValidAuditHandler()
auditHandler := a(testHandler(http.StatusOK, testBody))
auditHandler := a(testHandler(http.StatusOK, testBody, c))

// execute request and wait for 2 audit events
auditEvents := serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 2)
Expand Down Expand Up @@ -248,7 +247,7 @@ func TestAuditHandlerHeaders(t *testing.T) {

Convey("And a failing audit handler with successful downstream", func(c C) {
p, a := createFailingAuditHandler()
auditHandler := a(testHandler(http.StatusOK, testBody))
auditHandler := a(testHandler(http.StatusOK, testBody, c))

// execute request and don't expect audit events
serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 0)
Expand All @@ -270,7 +269,7 @@ func TestAuditHandlerHeaders(t *testing.T) {

Convey("And a valid audit handler with successful downstream", func(c C) {
p, a := createValidAuditHandler()
auditHandler := a(testHandler(http.StatusOK, testBody))
auditHandler := a(testHandler(http.StatusOK, testBody, c))

// execute request and wait for 2 audit events
auditEvents := serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 2)
Expand Down Expand Up @@ -303,7 +302,7 @@ func TestAuditHandlerHeaders(t *testing.T) {

Convey("And a failing audit handler with successful downstream", func(c C) {
p, a := createFailingAuditHandler()
auditHandler := a(testHandler(http.StatusOK, testBody))
auditHandler := a(testHandler(http.StatusOK, testBody, c))

// execute request and don't expect audit events
serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 0)
Expand All @@ -328,7 +327,7 @@ func TestAuditHandlerHeaders(t *testing.T) {

Convey("And a valid audit handler with successful downstream", func(c C) {
p, a := createValidAuditHandler()
auditHandler := a(testHandler(http.StatusOK, testBody))
auditHandler := a(testHandler(http.StatusOK, testBody, c))

// execute request and wait for 2 audit events
auditEvents := serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 2)
Expand Down Expand Up @@ -387,7 +386,7 @@ func TestAuditHandler(t *testing.T) {

Convey("And a valid audit handler with unsuccessful (Forbidden) downstream", func(c C) {
p, a := createValidAuditHandler()
auditHandler := a(testHandler(http.StatusForbidden, testBody))
auditHandler := a(testHandler(http.StatusForbidden, testBody, c))

// execute request and wait for 2 audit events
auditEvents := serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 2)
Expand Down Expand Up @@ -420,7 +419,7 @@ func TestAuditHandler(t *testing.T) {

Convey("And a failing audit handler with unsuccessful (Forbidden) downstream", func(c C) {
p, a := createFailingAuditHandler()
auditHandler := a(testHandler(http.StatusForbidden, testBody))
auditHandler := a(testHandler(http.StatusForbidden, testBody, c))

// execute request and don't expect audit events
serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 0)
Expand Down Expand Up @@ -455,7 +454,7 @@ func TestAuditHandler(t *testing.T) {
p := kafkatest.NewMessageProducer(true)
a := event.NewAvroProducer(p.Channels().Output, failingMarshaller)
enableZebedeeAudit := true
auditHandler := middleware.AuditHandler(a, cliMock, testZebedeeURL, testVersionPrefix, enableZebedeeAudit, nil)(testHandler(http.StatusForbidden, testBody))
auditHandler := middleware.AuditHandler(a, cliMock, testZebedeeURL, testVersionPrefix, enableZebedeeAudit, nil)(testHandler(http.StatusForbidden, testBody, c))

// execute request and expect only 1 audit event
auditEvents := serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 1)
Expand Down Expand Up @@ -484,7 +483,7 @@ func TestAuditIgnoreSkip(t *testing.T) {
Convey("And a valid audit handler without downstream", func(c C) {

p, a := createValidAuditHandler()
auditHandler := a(testHandler(http.StatusForbidden, testBody))
auditHandler := a(testHandler(http.StatusForbidden, testBody, c))

// execute request and don't wait for audit events
serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 0)
Expand All @@ -506,7 +505,7 @@ func TestAuditIgnoreSkip(t *testing.T) {
Convey("And a valid audit handler without downstream", func(c C) {

p, a := createValidAuditHandler()
auditHandler := a(testHandler(http.StatusForbidden, testBody))
auditHandler := a(testHandler(http.StatusForbidden, testBody, c))

// execute request and wait for 2 audit events
auditEvents := serveAndCaptureAudit(c, w, req, auditHandler, p.Channels().Output, 2)
Expand Down Expand Up @@ -542,7 +541,7 @@ func TestSkipZebedeeAudit(t *testing.T) {
},
}
auditMiddleware := middleware.AuditHandler(auditProducer, cliMock, testZebedeeURL, testVersionPrefix, enableZebedeeAudit, routerMock)
auditHandler := auditMiddleware(testHandler(http.StatusOK, testBody))
auditHandler := auditMiddleware(testHandler(http.StatusOK, testBody, c))

Convey("When the handler receives a Zebedee request", func(c C) {
req, err := http.NewRequest(http.MethodGet, "/data", nil)
Expand Down Expand Up @@ -581,7 +580,7 @@ func TestSkipZebedeeAudit(t *testing.T) {
},
}
auditMiddleware := middleware.AuditHandler(auditProducer, cliMock, testZebedeeURL, testVersionPrefix, enableZebedeeAudit, routerMock)
auditHandler := auditMiddleware(testHandler(http.StatusOK, testBody))
auditHandler := auditMiddleware(testHandler(http.StatusOK, testBody, c))

Convey("When the handler receives a request for a known route (not zebedee)", func(c C) {
req, err := http.NewRequest(http.MethodGet, "/v1/datasets?q1=v1&q2=v2", nil)
Expand Down
Loading

0 comments on commit c7a5639

Please sign in to comment.