From de3473930f5e943a80cb946b003f9c64095cdcba Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Wed, 26 Aug 2020 09:40:27 -0300 Subject: [PATCH 01/24] Upload msg attachments on WhatsApp, then cache the returned media ids --- config.go | 2 + handlers/whatsapp/whatsapp.go | 128 ++++++++++++++++++++++++---------- 2 files changed, 94 insertions(+), 36 deletions(-) diff --git a/config.go b/config.go index 62500f423..dd3d4d533 100644 --- a/config.go +++ b/config.go @@ -22,6 +22,7 @@ type Config struct { AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` FacebookApplicationSecret string `help:"the Facebook app secret"` FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` + WhatsAppMediaExpiration int `help:"the time in seconds for expire WhatsApp media ids in Redis"` MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` LibratoUsername string `help:"the username that will be used to authenticate to Librato"` LibratoToken string `help:"the token that will be used to authenticate to Librato"` @@ -57,6 +58,7 @@ func NewConfig() *Config { AWSSecretAccessKey: "missing_aws_secret_access_key", FacebookApplicationSecret: "missing_facebook_app_secret", FacebookWebhookSecret: "missing_facebook_webhook_secret", + WhatsAppMediaExpiration: 86400, MaxWorkers: 32, LogLevel: "error", Version: "Dev", diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index b82de2a5b..269d312bb 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/garyburd/redigo/redis" + "github.com/sirupsen/logrus" "net/http" "net/url" "strconv" @@ -276,8 +278,8 @@ func (h *handler) BuildDownloadMediaRequest(ctx context.Context, b courier.Backe // set the access token as the authorization header req, _ := http.NewRequest(http.MethodGet, attachmentURL, nil) - req.Header = buildAuthorizationHeader(req.Header, channel, token) req.Header.Set("User-Agent", utils.HTTPUserAgent) + setWhatsAppAuthHeader(&req.Header, channel) return req, nil } @@ -325,7 +327,8 @@ type mtTextPayload struct { } type mediaObject struct { - Link string `json:"link" validate:"required"` + ID string `json:"id,omitempty"` + Link string `json:"link,omitempty"` Caption string `json:"caption,omitempty"` } @@ -430,52 +433,50 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat if len(msg.Attachments()) > 0 { for attachmentCount, attachment := range msg.Attachments() { - mimeType, s3url := handlers.SplitAttachment(attachment) - + mimeType, mediaURL := handlers.SplitAttachment(attachment) + mediaID, err := h.fetchMediaID(msg.Channel(), mimeType, mediaURL) + if err == nil && mediaID != "" { + mediaURL = "" + } + mediaPayload := &mediaObject{ID: mediaID, Link: mediaURL} externalID := "" if strings.HasPrefix(mimeType, "audio") { payload := mtAudioPayload{ To: msg.URN().Path(), Type: "audio", } - payload.Audio = &mediaObject{Link: s3url} - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) - + payload.Audio = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else if strings.HasPrefix(mimeType, "application") { payload := mtDocumentPayload{ To: msg.URN().Path(), Type: "document", } - if attachmentCount == 0 { - payload.Document = &mediaObject{Link: s3url, Caption: msg.Text()} - } else { - payload.Document = &mediaObject{Link: s3url} + mediaPayload.Caption = msg.Text() } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) - + payload.Document = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else if strings.HasPrefix(mimeType, "image") { payload := mtImagePayload{ To: msg.URN().Path(), Type: "image", } if attachmentCount == 0 { - payload.Image = &mediaObject{Link: s3url, Caption: msg.Text()} - } else { - payload.Image = &mediaObject{Link: s3url} + mediaPayload.Caption = msg.Text() } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + payload.Image = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else if strings.HasPrefix(mimeType, "video") { payload := mtVideoPayload{ To: msg.URN().Path(), Type: "video", } if attachmentCount == 0 { - payload.Video = &mediaObject{Link: s3url, Caption: msg.Text()} - } else { - payload.Video = &mediaObject{Link: s3url} + mediaPayload.Caption = msg.Text() } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + payload.Video = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else { duration := time.Since(start) err = fmt.Errorf("unknown attachment mime type: %s", mimeType) @@ -525,7 +526,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat for _, v := range templating.Variables { payload.HSM.LocalizableParams = append(payload.HSM.LocalizableParams, LocalizableParam{Default: v}) } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else { payload := templatePayload{ @@ -544,7 +545,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } payload.Template.Components = append(payload.Template.Components, *component) - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } // add logs to our status @@ -564,7 +565,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat Type: "text", } payload.Text.Body = part - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) // add logs to our status for _, log := range logs { @@ -600,7 +601,61 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } -func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload interface{}) (string, string, []*courier.ChannelLog, error) { +// fetchMediaID tries to fetch the id for the uploaded media, setting the result in redis. +func (h *handler) fetchMediaID(channel courier.Channel, mimeType, mediaURL string) (string, error) { + // check on cache first + rc := h.Backend().RedisPool().Get() + defer rc.Close() + + cacheKey := fmt.Sprintf("whatsapp_media:%s:%s", channel.UUID().String(), mediaURL) + mediaID, err := redis.String(rc.Do("GET", cacheKey)) + if err == nil { + return mediaID, nil + } else if err != redis.ErrNil { + return "", err + } + + // download media + req, err := http.NewRequest("GET", mediaURL, nil) + if err != nil { + return "", err + } + res, err := utils.MakeHTTPRequest(req) + if err != nil { + return "", err + } + + // upload media to WhatsApp + baseURL := channel.StringConfigForKey(courier.ConfigBaseURL, "") + req, err = http.NewRequest("POST", baseURL + "/v1/media", bytes.NewReader(res.Body)) + if err != nil { + return "", err + } + setWhatsAppAuthHeader(&req.Header, channel) + req.Header.Add("Content-Type", mimeType) + res, err = utils.MakeHTTPRequest(req) + if err != nil { + return "", err + } + + // take uploaded media id + mediaID, err = jsonparser.GetString(res.Body, "media", "[0]", "id") + if err != nil { + return "", err + } + + // put on cache + rc = h.Backend().RedisPool().Get() + defer rc.Close() + + _, err = rc.Do("SET", cacheKey, mediaID, "EX", h.Server().Config().WhatsAppMediaExpiration) + if err != nil { + logrus.WithError(err).Error("error setting the media id to redis") + } + return mediaID, nil +} + +func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) @@ -610,7 +665,7 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i return "", "", []*courier.ChannelLog{log}, err } req, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) - req.Header = buildWhatsAppRequestHeader(msg.Channel(), token) + req.Header = buildWhatsAppHeaders(msg.Channel()) rr, err := utils.MakeHTTPRequest(req) log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) errPayload := &mtErrorPayload{} @@ -624,7 +679,7 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i } // check contact baseURL := fmt.Sprintf("%s://%s", sendPath.Scheme, sendPath.Host) - rrCheck, err := checkWhatsAppContact(msg.Channel(), baseURL, token, msg.URN()) + rrCheck, err := checkWhatsAppContact(msg.Channel(), baseURL, msg.URN()) if rrCheck == nil { elapsed := time.Now().Sub(start) @@ -680,7 +735,7 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i } // try send msg again reqRetry, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) - reqRetry.Header = buildWhatsAppRequestHeader(msg.Channel(), token) + reqRetry.Header = buildWhatsAppHeaders(msg.Channel()) if retryParam != "" { reqRetry.URL.RawQuery = fmt.Sprintf("%s=1", retryParam) @@ -698,22 +753,23 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i return "", externalID, []*courier.ChannelLog{log}, err } -func buildAuthorizationHeader(header http.Header, channel courier.Channel, token string) http.Header { +func setWhatsAppAuthHeader(header *http.Header, channel courier.Channel) { + authToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") + if channel.ChannelType() == channelTypeD3 { - header.Set(d3AuthorizationKey, token) + header.Set(d3AuthorizationKey, authToken) } else { - header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken)) } - return header } -func buildWhatsAppRequestHeader(channel courier.Channel, token string) http.Header { +func buildWhatsAppHeaders(channel courier.Channel) http.Header { header := http.Header{ "Content-Type": []string{"application/json"}, "Accept": []string{"application/json"}, "User-Agent": []string{utils.HTTPUserAgent}, } - header = buildAuthorizationHeader(header, channel, token) + setWhatsAppAuthHeader(&header, channel) return header } @@ -740,7 +796,7 @@ type mtContactCheckPayload struct { ForceCheck bool `json:"force_check"` } -func checkWhatsAppContact(channel courier.Channel, baseURL string, token string, urn urns.URN) (*utils.RequestResponse, error) { +func checkWhatsAppContact(channel courier.Channel, baseURL string, urn urns.URN) (*utils.RequestResponse, error) { payload := mtContactCheckPayload{ Blocking: "wait", Contacts: []string{fmt.Sprintf("+%s", urn.Path())}, @@ -753,7 +809,7 @@ func checkWhatsAppContact(channel courier.Channel, baseURL string, token string, } sendURL := fmt.Sprintf("%s/v1/contacts", baseURL) req, _ := http.NewRequest(http.MethodPost, sendURL, bytes.NewReader(reqBody)) - req.Header = buildWhatsAppRequestHeader(channel, token) + req.Header = buildWhatsAppHeaders(channel) rr, err := utils.MakeHTTPRequest(req) if err != nil { From 5238e514ead0eabef708280098ea0103f08cd70e Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Wed, 26 Aug 2020 09:41:26 -0300 Subject: [PATCH 02/24] Add tests for WhatsApp cached medias --- handlers/whatsapp/whatsapp_test.go | 96 ++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 9d5e3b611..857a47ed4 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -3,7 +3,9 @@ package whatsapp import ( "context" "encoding/json" + "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -529,6 +531,76 @@ var defaultSendTestCases = []ChannelSendTestCase{ }, } +var mediaCacheSendTestCases = []ChannelSendTestCase{ + {Label: "Media Upload Error", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/media", + Body: "media bytes", + }: MockedResponse{ + Status: 401, + Body: `{ "errors": [{"code":1005,"title":"Access denied","details":"Invalid credentials."}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + BodyContains: `/document.pdf`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Media Upload OK", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/media", + Body: "media bytes", + }: MockedResponse{ + Status: 200, + Body: `{ "media" : [{"id": "f043afd0-f0ae-4b9c-ab3d-696fb4c8cd68"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"document","document":{"id":"f043afd0-f0ae-4b9c-ab3d-696fb4c8cd68","caption":"document caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Cached Media", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"document","document":{"id":"f043afd0-f0ae-4b9c-ab3d-696fb4c8cd68","caption":"document caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, +} + var hsmSupportSendTestCases = []ChannelSendTestCase{ {Label: "Template Send", Text: "templated message", @@ -541,6 +613,20 @@ var hsmSupportSendTestCases = []ChannelSendTestCase{ }, } +func mockAttachmentURLs(mediaServer *httptest.Server, testCases []ChannelSendTestCase) []ChannelSendTestCase { + casesWithMockedUrls := make([]ChannelSendTestCase, len(testCases)) + + for i, testCase := range testCases { + mockedCase := testCase + + for j, attachment := range testCase.Attachments { + mockedCase.Attachments[j] = strings.Replace(attachment, "https://foo.bar", mediaServer.URL, 1) + } + casesWithMockedUrls[i] = mockedCase + } + return casesWithMockedUrls +} + func TestSending(t *testing.T) { var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "WA", "250788383383", "US", map[string]interface{}{ @@ -567,4 +653,14 @@ func TestSending(t *testing.T) { RunChannelSendTestCases(t, defaultChannel, newWAHandler(courier.ChannelType("WA"), "WhatsApp"), defaultSendTestCases, nil) RunChannelSendTestCases(t, hsmSupportChannel, newWAHandler(courier.ChannelType("WA"), "WhatsApp"), hsmSupportSendTestCases, nil) RunChannelSendTestCases(t, d3Channel, newWAHandler(courier.ChannelType("D3"), "360Dialog"), defaultSendTestCases, nil) + + mediaServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + res.WriteHeader(200) + res.Write([]byte("media bytes")) + })) + defer mediaServer.Close() + mediaCacheSendTestCases := mockAttachmentURLs(mediaServer, mediaCacheSendTestCases) + + RunChannelSendTestCases(t, defaultChannel, newWAHandler(courier.ChannelType("WA"), "WhatsApp"), mediaCacheSendTestCases, nil) } From 1514a26704497bd25580ec7656c75db21ef2a156 Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Wed, 26 Aug 2020 13:06:49 -0300 Subject: [PATCH 03/24] Log errors on fetching media id in WhatsApp handler --- handlers/whatsapp/whatsapp.go | 55 +++++++++++++++++++++++++---------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 269d312bb..858064751 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "github.com/garyburd/redigo/redis" - "github.com/sirupsen/logrus" "net/http" "net/url" "strconv" @@ -434,7 +433,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat for attachmentCount, attachment := range msg.Attachments() { mimeType, mediaURL := handlers.SplitAttachment(attachment) - mediaID, err := h.fetchMediaID(msg.Channel(), mimeType, mediaURL) + mediaID, mediaLogs, err := h.fetchMediaID(msg, mimeType, mediaURL) if err == nil && mediaID != "" { mediaURL = "" } @@ -483,6 +482,11 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat logs = []*courier.ChannelLog{courier.NewChannelLogFromError("Error sending message", msg.Channel(), msg.ID(), duration, err)} } + // add media logs to our status + for _, log := range mediaLogs { + status.AddLog(log) + } + // add logs to our status for _, log := range logs { status.AddLog(log) @@ -602,46 +606,64 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } // fetchMediaID tries to fetch the id for the uploaded media, setting the result in redis. -func (h *handler) fetchMediaID(channel courier.Channel, mimeType, mediaURL string) (string, error) { +func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (string, []*courier.ChannelLog, error) { + var logs []*courier.ChannelLog + start := time.Now() + // check on cache first rc := h.Backend().RedisPool().Get() defer rc.Close() - cacheKey := fmt.Sprintf("whatsapp_media:%s:%s", channel.UUID().String(), mediaURL) + cacheKey := fmt.Sprintf("whatsapp_media:%s:%s", msg.Channel().UUID().String(), mediaURL) mediaID, err := redis.String(rc.Do("GET", cacheKey)) if err == nil { - return mediaID, nil + return mediaID, logs, nil } else if err != redis.ErrNil { - return "", err + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error reading the media id from redis", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err } // download media req, err := http.NewRequest("GET", mediaURL, nil) if err != nil { - return "", err + return "", logs, err } res, err := utils.MakeHTTPRequest(req) if err != nil { - return "", err + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error downloading the media", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err } // upload media to WhatsApp - baseURL := channel.StringConfigForKey(courier.ConfigBaseURL, "") + baseURL := msg.Channel().StringConfigForKey(courier.ConfigBaseURL, "") req, err = http.NewRequest("POST", baseURL + "/v1/media", bytes.NewReader(res.Body)) if err != nil { - return "", err + return "", logs, err } - setWhatsAppAuthHeader(&req.Header, channel) + setWhatsAppAuthHeader(&req.Header, msg.Channel()) req.Header.Add("Content-Type", mimeType) res, err = utils.MakeHTTPRequest(req) if err != nil { - return "", err + if res != nil { + err = errors.Wrap(err, string(res.Body)) + } + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error uploading the media to WhatsApp", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err } // take uploaded media id mediaID, err = jsonparser.GetString(res.Body, "media", "[0]", "id") if err != nil { - return "", err + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error reading the media id from response", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err } // put on cache @@ -650,9 +672,12 @@ func (h *handler) fetchMediaID(channel courier.Channel, mimeType, mediaURL strin _, err = rc.Do("SET", cacheKey, mediaID, "EX", h.Server().Config().WhatsAppMediaExpiration) if err != nil { - logrus.WithError(err).Error("error setting the media id to redis") + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error setting the media id to redis", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err } - return mediaID, nil + return mediaID, logs, nil } func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { From 9e127ecfb1b51775ba9459cb68f9ed58136af375 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Fri, 31 Jul 2020 15:44:34 -0700 Subject: [PATCH 04/24] WIP Add discord handler Using the external handler as a starting point, we now have a discord handler. We need to write some tests though... --- cmd/courier/main.go | 1 + handlers/discord/discord.go | 346 +++++++++++++++++++++ handlers/discord/discord_test.go | 503 +++++++++++++++++++++++++++++++ 3 files changed, 850 insertions(+) create mode 100644 handlers/discord/discord.go create mode 100644 handlers/discord/discord_test.go diff --git a/cmd/courier/main.go b/cmd/courier/main.go index 23914af28..67c3d7bcf 100644 --- a/cmd/courier/main.go +++ b/cmd/courier/main.go @@ -20,6 +20,7 @@ import ( _ "github.com/nyaruka/courier/handlers/clickatell" _ "github.com/nyaruka/courier/handlers/clicksend" _ "github.com/nyaruka/courier/handlers/dart" + _ "github.com/nyaruka/courier/handlers/discord" _ "github.com/nyaruka/courier/handlers/dmark" _ "github.com/nyaruka/courier/handlers/external" _ "github.com/nyaruka/courier/handlers/facebook" diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go new file mode 100644 index 000000000..e27971350 --- /dev/null +++ b/handlers/discord/discord.go @@ -0,0 +1,346 @@ +package discord + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "strings" + + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/urns" + "github.com/pkg/errors" +) + +const ( + contentURLEncoded = "urlencoded" + contentJSON = "json" + + configMOFromField = "mo_from_field" + configMOTextField = "mo_text_field" + configMODateField = "mo_date_field" + + configMOResponseContentType = "mo_response_content_type" + configMOResponse = "mo_response" +) + +var defaultFromFields = []string{"from", "sender"} +var defaultTextFields = []string{"text"} +var defaultDateFields = []string{"date", "time"} + +var contentTypeMappings = map[string]string{ + contentURLEncoded: "application/x-www-form-urlencoded", + contentJSON: "application/json", +} + +func init() { + courier.RegisterHandler(newHandler()) +} + +type handler struct { + handlers.BaseHandler +} + +func newHandler() courier.ChannelHandler { + return &handler{handlers.NewBaseHandler(courier.ChannelType("DS"), "Discord")} +} + +// Initialize is called by the engine once everything is loaded +func (h *handler) Initialize(s courier.Server) error { + h.SetServer(s) + s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveMessage) + s.AddHandlerRoute(h, http.MethodGet, "receive", h.receiveMessage) + + sentHandler := h.buildStatusHandler("sent") + s.AddHandlerRoute(h, http.MethodGet, "sent", sentHandler) + s.AddHandlerRoute(h, http.MethodPost, "sent", sentHandler) + + deliveredHandler := h.buildStatusHandler("delivered") + s.AddHandlerRoute(h, http.MethodGet, "delivered", deliveredHandler) + s.AddHandlerRoute(h, http.MethodPost, "delivered", deliveredHandler) + + failedHandler := h.buildStatusHandler("failed") + s.AddHandlerRoute(h, http.MethodGet, "failed", failedHandler) + s.AddHandlerRoute(h, http.MethodPost, "failed", failedHandler) + + s.AddHandlerRoute(h, http.MethodPost, "stopped", h.receiveStopContact) + s.AddHandlerRoute(h, http.MethodGet, "stopped", h.receiveStopContact) + + return nil +} + +type stopContactForm struct { + From string `validate:"required" name:"from"` +} + +func (h *handler) receiveStopContact(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + form := &stopContactForm{} + err := handlers.DecodeAndValidateForm(form, r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // create our URN + urn := urns.NilURN + urn, err = urns.NewURNFromParts(channel.Schemes()[0], form.From, "", "") + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + urn = urn.Normalize("") + + // create a stop channel event + channelEvent := h.Backend().NewChannelEvent(channel, courier.StopContact, urn) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + if err != nil { + return nil, err + } + return []courier.Event{channelEvent}, courier.WriteChannelEventSuccess(ctx, w, r, channelEvent) +} + +// utility function to grab the form value for either the passed in name (if non-empty) or the first set +// value from defaultNames +func getFormField(form url.Values, defaultNames []string, name string) string { + if name != "" { + values, found := form[name] + if found { + return values[0] + } + } + + for _, name := range defaultNames { + values, found := form[name] + if found { + return values[0] + } + } + + return "" +} + +// receiveMessage is our HTTP handler function for incoming messages +func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + var err error + + var from, dateString, text string + + // parse our form + err = r.ParseForm() + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, errors.Wrapf(err, "invalid request")) + } + + from = getFormField(r.Form, defaultFromFields, channel.StringConfigForKey(configMOFromField, "")) + text = getFormField(r.Form, defaultTextFields, channel.StringConfigForKey(configMOTextField, "")) + dateString = getFormField(r.Form, defaultDateFields, channel.StringConfigForKey(configMODateField, "")) + + // must have from field + if from == "" { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("must have one of 'sender' or 'from' set")) + } + + // if we have a date, parse it + date := time.Now() + if dateString != "" { + date, err = time.Parse(time.RFC3339Nano, dateString) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("invalid date format, must be RFC 3339")) + } + } + + // create our URN + urn := urns.NilURN + urn, err = urns.NewURNFromParts(channel.Schemes()[0], from, "", "") + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // build our msg + msg := h.Backend().NewIncomingMsg(channel, urn, text).WithReceivedOn(date) + + // and finally write our message + return handlers.WriteMsgsAndResponse(ctx, h, []courier.Msg{msg}, w, r) +} + +// WriteMsgSuccessResponse writes our response in TWIML format +func (h *handler) WriteMsgSuccessResponse(ctx context.Context, w http.ResponseWriter, r *http.Request, msgs []courier.Msg) error { + moResponse := msgs[0].Channel().StringConfigForKey(configMOResponse, "") + if moResponse == "" { + return courier.WriteMsgSuccess(ctx, w, r, msgs) + } + moResponseContentType := msgs[0].Channel().StringConfigForKey(configMOResponseContentType, "") + if moResponseContentType != "" { + w.Header().Set("Content-Type", moResponseContentType) + } + w.WriteHeader(200) + _, err := fmt.Fprint(w, moResponse) + return err +} + +// buildStatusHandler deals with building a handler that takes what status is received in the URL +func (h *handler) buildStatusHandler(status string) courier.ChannelHandleFunc { + return func(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + return h.receiveStatus(ctx, status, channel, w, r) + } +} + +type statusForm struct { + ID int64 `name:"id" validate:"required"` +} + +var statusMappings = map[string]courier.MsgStatusValue{ + "failed": courier.MsgFailed, + "sent": courier.MsgSent, + "delivered": courier.MsgDelivered, +} + +// receiveStatus is our HTTP handler function for status updates +func (h *handler) receiveStatus(ctx context.Context, statusString string, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + form := &statusForm{} + err := handlers.DecodeAndValidateForm(form, r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // get our status + msgStatus, found := statusMappings[strings.ToLower(statusString)] + if !found { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown status '%s', must be one failed, sent or delivered", statusString)) + } + + // write our status + status := h.Backend().NewMsgStatusForID(channel, courier.NewMsgID(form.ID), msgStatus) + return handlers.WriteMsgStatusAndResponse(ctx, h, channel, status, w, r) +} + +// SendMsg sends the passed in message, returning any error +func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { + sendURL := msg.Channel().StringConfigForKey(courier.ConfigSendURL, "") + if sendURL == "" { + return nil, fmt.Errorf("no send url set for DS channel") + } + + // figure out what encoding to tell kannel to send as + sendMethod := http.MethodPost + // sendBody := msg.Channel().StringConfigForKey(courier.ConfigSendBody, "") + contentType := contentJSON + contentTypeHeader := contentTypeMappings[contentType] + if contentTypeHeader == "" { + contentTypeHeader = contentType + } + + status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) + parts := handlers.SplitMsgByChannel(msg.Channel(), handlers.GetTextAndAttachments(msg), 160) + for i, part := range parts { + // build our request + form := map[string]string{ + "id": msg.ID().String(), + "text": part, + "to": msg.URN().Path(), + "to_no_plus": strings.TrimPrefix(msg.URN().Path(), "+"), + "from": msg.Channel().Address(), + "from_no_plus": strings.TrimPrefix(msg.Channel().Address(), "+"), + "channel": msg.Channel().UUID().String(), + } + + formEncoded := encodeVariables(form, contentType) + + // put quick replies on last message part + url := replaceVariables(sendURL, formEncoded) + + var body io.Reader + if sendMethod == http.MethodPost || sendMethod == http.MethodPut { + formEncoded = encodeVariables(form, contentType) + + if i == len(parts)-1 { + formEncoded["quick_replies"] = buildQuickRepliesResponse(msg.QuickReplies(), sendMethod, contentType) + } else { + formEncoded["quick_replies"] = buildQuickRepliesResponse([]string{}, sendMethod, contentType) + } + body = strings.NewReader(replaceVariables(sendBody, formEncoded)) + } + + req, err := http.NewRequest(sendMethod, url, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentTypeHeader) + + authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + + rr, err := utils.MakeHTTPRequest(req) + + // record our status and log + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + status.AddLog(log) + if err != nil { + return status, nil + } + // If we don't have an error, set the message as wired and move on + status.SetStatus(courier.MsgWired) + + } + + return status, nil +} + +type quickReplyXMLItem struct { + XMLName xml.Name `xml:"item"` + Value string `xml:",chardata"` +} + +func buildQuickRepliesResponse(quickReplies []string, sendMethod string, contentType string) string { + if quickReplies == nil { + quickReplies = []string{} + } + if (sendMethod == http.MethodPost || sendMethod == http.MethodPut) && contentType == contentJSON { + marshalled, _ := json.Marshal(quickReplies) + return string(marshalled) + } + response := bytes.Buffer{} + + for _, reply := range quickReplies { + reply = url.QueryEscape(reply) + response.WriteString(fmt.Sprintf("&quick_reply=%s", reply)) + } + return response.String() +} + +func encodeVariables(variables map[string]string, contentType string) map[string]string { + encoded := make(map[string]string) + + for k, v := range variables { + // encode according to our content type + switch contentType { + case contentJSON: + marshalled, _ := json.Marshal(v) + v = string(marshalled) + + case contentURLEncoded: + v = url.QueryEscape(v) + + } + encoded[k] = v + } + return encoded +} + +func replaceVariables(text string, variables map[string]string) string { + for k, v := range variables { + text = strings.Replace(text, fmt.Sprintf("{{%s}}", k), v, -1) + } + return text +} + +// const defaultSendBody = `id={{id}}&text={{text}}&to={{to}}&to_no_plus={{to_no_plus}}&from={{from}}&from_no_plus={{from_no_plus}}&channel={{channel}}` +const sendBody = `{"id": {{id}}, "text": {{text}}, "to":{{to}}, "channel": {{channel}}}` diff --git a/handlers/discord/discord_test.go b/handlers/discord/discord_test.go new file mode 100644 index 000000000..24e8e02e6 --- /dev/null +++ b/handlers/discord/discord_test.go @@ -0,0 +1,503 @@ +package discord + +import ( + "net/http/httptest" + "testing" + "time" + + "net/http" + + "github.com/nyaruka/courier" + . "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" +) + +var ( + receiveValidMessage = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join" + receiveValidMessageFrom = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?from=%2B2349067554729&text=Join" + receiveValidNoPlus = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?from=2349067554729&text=Join" + receiveValidMessageWithDate = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join&date=2017-06-23T12:30:00.500Z" + receiveValidMessageWithTime = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join&time=2017-06-23T12:30:00Z" + receiveNoParams = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/" + invalidURN = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=MTN&text=Join" + receiveNoSender = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?text=Join" + receiveInvalidDate = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join&time=20170623T123000Z" + failedNoParams = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/failed/" + failedValid = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/failed/?id=12345" + sentValid = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/sent/?id=12345" + invalidStatus = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/wired/" + deliveredValid = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/delivered/?id=12345" + deliveredValidPost = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/delivered/" + stoppedEvent = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/stopped/?from=%2B2349067554729" + stoppedEventPost = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/stopped/" + stoppedEventInvalidURN = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/stopped/?from=MTN" +) + +var testChannels = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", nil), +} + +var gmChannels = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "GM", nil), +} + +var handleTestCases = []ChannelHandleTestCase{ + {Label: "Receive Valid Message", URL: receiveValidMessage, Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + {Label: "Receive Valid Post", URL: receiveNoParams, Data: "sender=%2B2349067554729&text=Join", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + {Label: "Receive Valid From", URL: receiveValidMessageFrom, Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + {Label: "Receive Country Parse", URL: receiveValidNoPlus, Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + {Label: "Receive Valid Message With Date", URL: receiveValidMessageWithDate, Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729"), Date: Tp(time.Date(2017, 6, 23, 12, 30, 0, int(500*time.Millisecond), time.UTC))}, + {Label: "Receive Valid Message With Time", URL: receiveValidMessageWithTime, Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729"), Date: Tp(time.Date(2017, 6, 23, 12, 30, 0, 0, time.UTC))}, + {Label: "Invalid URN", URL: invalidURN, Data: "empty", Status: 400, Response: "phone number supplied is not a number"}, + {Label: "Receive No Params", URL: receiveNoParams, Data: "empty", Status: 400, Response: "must have one of 'sender' or 'from' set"}, + {Label: "Receive No Sender", URL: receiveNoSender, Data: "empty", Status: 400, Response: "must have one of 'sender' or 'from' set"}, + {Label: "Receive Invalid Date", URL: receiveInvalidDate, Data: "empty", Status: 400, Response: "invalid date format, must be RFC 3339"}, + {Label: "Failed No Params", URL: failedNoParams, Status: 400, Response: "field 'id' required"}, + {Label: "Failed Valid", URL: failedValid, Status: 200, Response: `"status":"F"`}, + {Label: "Invalid Status", URL: invalidStatus, Status: 404, Response: `page not found`}, + {Label: "Sent Valid", URL: sentValid, Status: 200, Response: `"status":"S"`}, + {Label: "Delivered Valid", URL: deliveredValid, Status: 200, Data: "nothing", Response: `"status":"D"`}, + {Label: "Delivered Valid Post", URL: deliveredValidPost, Data: "id=12345", Status: 200, Response: `"status":"D"`}, + {Label: "Stopped Event", URL: stoppedEvent, Status: 200, Data: "nothing", Response: "Accepted"}, + {Label: "Stopped Event Post", URL: stoppedEventPost, Data: "from=%2B2349067554729", Status: 200, Response: "Accepted"}, + {Label: "Stopped Event Invalid URN", URL: stoppedEventInvalidURN, Data: "empty", Status: 400, Response: "phone number supplied is not a number"}, + {Label: "Stopped event No Params", URL: stoppedEventPost, Status: 400, Response: "field 'from' required"}, +} + +var testSOAPReceiveChannels = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + configTextXPath: "//content", + configFromXPath: "//source", + configMOResponse: "0", + configMOResponseContentType: "text/xml", + })} + +var handleSOAPReceiveTestCases = []ChannelHandleTestCase{ + {Label: "Receive Valid Post SOAP", URL: receiveNoParams, Data: `2349067554729Join`, + Status: 200, Response: "0", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + {Label: "Receive Invalid SOAP", URL: receiveNoParams, Data: ``, + Status: 400, Response: "missing from"}, +} + +var gmTestCases = []ChannelHandleTestCase{ + {Label: "Receive Non Plus Message", URL: "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=2207222333&text=Join", Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2207222333")}, +} + +var customChannels = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + configMOFromField: "from_number", + configMODateField: "timestamp", + configMOTextField: "messageText", + })} + +var customTestCases = []ChannelHandleTestCase{ + {Label: "Receive Custom Message", URL: "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?from_number=12067799192&messageText=Join×tamp=2017-06-23T12:30:00Z", Data: "empty", Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+12067799192"), Date: Tp(time.Date(2017, 6, 23, 12, 30, 0, 0, time.UTC))}, + {Label: "Receive Custom Missing", URL: "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sent_from=12067799192&messageText=Join", Data: "empty", Status: 400, Response: "must have one of 'sender' or 'from' set"}, +} + +func TestHandler(t *testing.T) { + RunChannelTestCases(t, testChannels, newHandler(), handleTestCases) + RunChannelTestCases(t, testSOAPReceiveChannels, newHandler(), handleSOAPReceiveTestCases) + RunChannelTestCases(t, gmChannels, newHandler(), gmTestCases) + RunChannelTestCases(t, customChannels, newHandler(), customTestCases) +} + +func BenchmarkHandler(b *testing.B) { + RunChannelBenchmarks(b, testChannels, newHandler(), handleTestCases) + RunChannelBenchmarks(b, testSOAPReceiveChannels, newHandler(), handleSOAPReceiveTestCases) +} + +// setSendURL takes care of setting the send_url to our test server host +func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { + // this is actually a path, which we'll combine with the test server URL + sendURL := c.StringConfigForKey("send_path", "") + sendURL, _ = utils.AddURLPath(s.URL, sendURL) + c.(*courier.MockChannel).SetConfig(courier.ConfigSendURL, sendURL) +} + +var longSendTestCases = []ChannelSendTestCase{ + {Label: "Long Send", + Text: "This is a long message that will be longer than 30....... characters", URN: "tel:+250788383383", + QuickReplies: []string{"One"}, + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + URLParams: map[string]string{"text": "characters", "to": "+250788383383", "from": "2020", "quick_reply": "One"}, + SendPrep: setSendURL}, +} + +var getSendSmartEncodingTestCases = []ChannelSendTestCase{ + {Label: "Smart Encoding", + Text: "Fancy “Smart” Quotes", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + URLParams: map[string]string{"text": `Fancy "Smart" Quotes`, "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, +} + +var postSendSmartEncodingTestCases = []ChannelSendTestCase{ + {Label: "Smart Encoding", + Text: "Fancy “Smart” Quotes", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + PostParams: map[string]string{"text": `Fancy "Smart" Quotes`, "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, +} + +var getSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + URLParams: map[string]string{"text": "Simple Message", "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, + {Label: "Unicode Send", + Text: "☺", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + URLParams: map[string]string{"text": "☺", "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, + {Label: "Error Sending", + Text: "Error Message", URN: "tel:+250788383383", + Status: "E", + ResponseBody: "1: Unknown channel", ResponseStatus: 401, + URLParams: map[string]string{"text": `Error Message`, "to": "+250788383383"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, + {Label: "Send Attachment", + Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", + ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, + URLParams: map[string]string{"text": "My pic!\nhttps://foo.bar/image.jpg", "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, +} + +var postSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + PostParams: map[string]string{"text": "Simple Message", "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, + {Label: "Unicode Send", + Text: "☺", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + PostParams: map[string]string{"text": "☺", "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, + {Label: "Error Sending", + Text: "Error Message", URN: "tel:+250788383383", + Status: "E", + ResponseBody: "1: Unknown channel", ResponseStatus: 401, + PostParams: map[string]string{"text": `Error Message`, "to": "+250788383383"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, + {Label: "Send Attachment", + Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", + ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, + PostParams: map[string]string{"text": "My pic!\nhttps://foo.bar/image.jpg", "to": "+250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, +} + +var postSendCustomContentTypeTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + PostParams: map[string]string{"text": "Simple Message", "to": "250788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"}, + SendPrep: setSendURL}, +} + +var jsonSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: `{ "to":"+250788383383", "text":"Simple Message", "from":"2020", "quick_replies":[] }`, + Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, + SendPrep: setSendURL}, + {Label: "Unicode Send", + Text: `☺ "hi!"`, URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: `{ "to":"+250788383383", "text":"☺ \"hi!\"", "from":"2020", "quick_replies":[] }`, + Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, + SendPrep: setSendURL}, + {Label: "Error Sending", + Text: "Error Message", URN: "tel:+250788383383", + Status: "E", + ResponseBody: "1: Unknown channel", ResponseStatus: 401, + RequestBody: `{ "to":"+250788383383", "text":"Error Message", "from":"2020", "quick_replies":[] }`, + Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, + SendPrep: setSendURL}, + {Label: "Send Attachment", + Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", + ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, + RequestBody: `{ "to":"+250788383383", "text":"My pic!\nhttps://foo.bar/image.jpg", "from":"2020", "quick_replies":[] }`, + Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, + SendPrep: setSendURL}, + {Label: "Send Quick Replies", + Text: "Some message", URN: "tel:+250788383383", QuickReplies: []string{"One", "Two", "Three"}, + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: `{ "to":"+250788383383", "text":"Some message", "from":"2020", "quick_replies":["One","Two","Three"] }`, + Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, + SendPrep: setSendURL}, +} + +var jsonLongSendTestCases = []ChannelSendTestCase{ + {Label: "Send Quick Replies", + Text: "This is a long message that will be longer than 30....... characters", URN: "tel:+250788383383", + QuickReplies: []string{"One", "Two", "Three"}, + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: `{ "to":"+250788383383", "text":"characters", "from":"2020", "quick_replies":["One","Two","Three"] }`, + Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, + SendPrep: setSendURL}, +} + +var xmlSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: `+250788383383Simple Message2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Unicode Send", + Text: `☺`, URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: `+250788383383☺2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Error Sending", + Text: "Error Message", URN: "tel:+250788383383", + Status: "E", + ResponseBody: "1: Unknown channel", ResponseStatus: 401, + RequestBody: `+250788383383Error Message2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Send Attachment", + Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", + ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, + RequestBody: `+250788383383My pic! https://foo.bar/image.jpg2020` + + ``, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Send Quick Replies", + Text: "Some message", URN: "tel:+250788383383", QuickReplies: []string{"One", "Two", "Three"}, + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: "+250788383383Some message2020" + + "OneTwoThree", + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, +} + +var xmlLongSendTestCases = []ChannelSendTestCase{ + {Label: "Send Quick Replies", + Text: "This is a long message that will be longer than 30....... characters", URN: "tel:+250788383383", + QuickReplies: []string{"One", "Two", "Three"}, + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + RequestBody: "+250788383383characters2020" + + "OneTwoThree", + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, +} + +var xmlSendWithResponseContentTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0", ResponseStatus: 200, + RequestBody: `+250788383383Simple Message2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Unicode Send", + Text: `☺`, URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0", ResponseStatus: 200, + RequestBody: `+250788383383☺2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Error Sending", + Text: "Error Message", URN: "tel:+250788383383", + Status: "E", + ResponseBody: "0", ResponseStatus: 401, + RequestBody: `+250788383383Error Message2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Error Sending with 200 status code", + Text: "Error Message", URN: "tel:+250788383383", + Status: "E", + ResponseBody: "1", ResponseStatus: 200, + RequestBody: `+250788383383Error Message2020`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Send Attachment", + Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", + ResponseBody: `0`, ResponseStatus: 200, + RequestBody: `+250788383383My pic! https://foo.bar/image.jpg2020` + + ``, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, + {Label: "Send Quick Replies", + Text: "Some message", URN: "tel:+250788383383", QuickReplies: []string{"One", "Two", "Three"}, + Status: "W", + ResponseBody: "0", ResponseStatus: 200, + RequestBody: `+250788383383Some message2020` + + `OneTwoThree`, + Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, + SendPrep: setSendURL}, +} + +var nationalGetSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "tel:+250788383383", + Status: "W", + ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, + URLParams: map[string]string{"text": "Simple Message", "to": "788383383", "from": "2020"}, + Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, + SendPrep: setSendURL}, +} + +func TestSending(t *testing.T) { + var getChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + courier.ConfigSendMethod: http.MethodGet}) + + var getSmartChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + configEncoding: encodingSmart, + courier.ConfigSendMethod: http.MethodGet}) + + var postChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: "to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + courier.ConfigSendMethod: http.MethodPost}) + + var postChannelCustomContentType = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: "to={{to_no_plus}}&text={{text}}&from={{from_no_plus}}{{quick_replies}}", + courier.ConfigContentType: "application/x-www-form-urlencoded; charset=utf-8", + courier.ConfigSendMethod: http.MethodPost}) + + var postSmartChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: "to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + configEncoding: encodingSmart, + courier.ConfigSendMethod: http.MethodPost}) + + var jsonChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendAuthorization: "Token ABCDEF", + }) + + var xmlChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, + courier.ConfigContentType: contentXML, + courier.ConfigSendMethod: http.MethodPut, + }) + + var xmlChannelWithResponseContent = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, + configMTResponseCheck: "0", + courier.ConfigContentType: contentXML, + courier.ConfigSendMethod: http.MethodPut, + }) + + RunChannelSendTestCases(t, getChannel, newHandler(), getSendTestCases, nil) + RunChannelSendTestCases(t, getSmartChannel, newHandler(), getSendTestCases, nil) + RunChannelSendTestCases(t, getSmartChannel, newHandler(), getSendSmartEncodingTestCases, nil) + RunChannelSendTestCases(t, postChannel, newHandler(), postSendTestCases, nil) + RunChannelSendTestCases(t, postChannelCustomContentType, newHandler(), postSendCustomContentTypeTestCases, nil) + RunChannelSendTestCases(t, postSmartChannel, newHandler(), postSendTestCases, nil) + RunChannelSendTestCases(t, postSmartChannel, newHandler(), postSendSmartEncodingTestCases, nil) + RunChannelSendTestCases(t, jsonChannel, newHandler(), jsonSendTestCases, nil) + RunChannelSendTestCases(t, xmlChannel, newHandler(), xmlSendTestCases, nil) + RunChannelSendTestCases(t, xmlChannelWithResponseContent, newHandler(), xmlSendWithResponseContentTestCases, nil) + + var getChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "max_length": 30, + "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + courier.ConfigSendMethod: http.MethodGet}) + + var getChannel30StrLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "max_length": "30", + "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + courier.ConfigSendMethod: http.MethodGet}) + + var jsonChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendAuthorization: "Token ABCDEF", + }) + + var xmlChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, + courier.ConfigContentType: contentXML, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendAuthorization: "Token ABCDEF", + }) + + RunChannelSendTestCases(t, getChannel30IntLength, newHandler(), longSendTestCases, nil) + RunChannelSendTestCases(t, getChannel30StrLength, newHandler(), longSendTestCases, nil) + RunChannelSendTestCases(t, jsonChannel30IntLength, newHandler(), jsonLongSendTestCases, nil) + RunChannelSendTestCases(t, xmlChannel30IntLength, newHandler(), xmlLongSendTestCases, nil) + + var nationalChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", + "use_national": true, + courier.ConfigSendMethod: http.MethodGet}) + + RunChannelSendTestCases(t, nationalChannel, newHandler(), nationalGetSendTestCases, nil) + +} From 4198363b13fd4a6079430126977b2a6260b2e9c5 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Mon, 10 Aug 2020 18:42:57 -0700 Subject: [PATCH 05/24] Add test for discord As the discord channel type is largely the same as the external one, the tests are largely derived (though much simpler) from the external tests as well. --- go.mod | 2 + handlers/discord/discord.go | 60 +--- handlers/discord/discord_test.go | 488 +------------------------------ 3 files changed, 25 insertions(+), 525 deletions(-) diff --git a/go.mod b/go.mod index 37b3c739d..0b028a186 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/nyaruka/courier +replace github.com/nyaruka/gocommon => /home/awen/go/src/github.com/nyaruka/gocommon + require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go index e27971350..8ed56ad44 100644 --- a/handlers/discord/discord.go +++ b/handlers/discord/discord.go @@ -1,10 +1,8 @@ package discord import ( - "bytes" "context" "encoding/json" - "encoding/xml" "fmt" "io" "net/http" @@ -90,7 +88,7 @@ func (h *handler) receiveStopContact(ctx context.Context, channel courier.Channe // create our URN urn := urns.NilURN - urn, err = urns.NewURNFromParts(channel.Schemes()[0], form.From, "", "") + urn, err = urns.NewURNFromParts(urns.DiscordScheme, form.From, "", "") if err != nil { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } @@ -129,7 +127,7 @@ func getFormField(form url.Values, defaultNames []string, name string) string { func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { var err error - var from, dateString, text string + var from, text string // parse our form err = r.ParseForm() @@ -139,25 +137,21 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w from = getFormField(r.Form, defaultFromFields, channel.StringConfigForKey(configMOFromField, "")) text = getFormField(r.Form, defaultTextFields, channel.StringConfigForKey(configMOTextField, "")) - dateString = getFormField(r.Form, defaultDateFields, channel.StringConfigForKey(configMODateField, "")) // must have from field if from == "" { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("must have one of 'sender' or 'from' set")) } + if text == "" { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("must have 'text' set")) + } // if we have a date, parse it date := time.Now() - if dateString != "" { - date, err = time.Parse(time.RFC3339Nano, dateString) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("invalid date format, must be RFC 3339")) - } - } // create our URN urn := urns.NilURN - urn, err = urns.NewURNFromParts(channel.Schemes()[0], from, "", "") + urn, err = urns.NewURNFromParts(urns.DiscordScheme, from, "", "") if err != nil { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } @@ -171,17 +165,7 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w // WriteMsgSuccessResponse writes our response in TWIML format func (h *handler) WriteMsgSuccessResponse(ctx context.Context, w http.ResponseWriter, r *http.Request, msgs []courier.Msg) error { - moResponse := msgs[0].Channel().StringConfigForKey(configMOResponse, "") - if moResponse == "" { - return courier.WriteMsgSuccess(ctx, w, r, msgs) - } - moResponseContentType := msgs[0].Channel().StringConfigForKey(configMOResponseContentType, "") - if moResponseContentType != "" { - w.Header().Set("Content-Type", moResponseContentType) - } - w.WriteHeader(200) - _, err := fmt.Fprint(w, moResponse) - return err + return courier.WriteMsgSuccess(ctx, w, r, msgs) } // buildStatusHandler deals with building a handler that takes what status is received in the URL @@ -238,7 +222,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) parts := handlers.SplitMsgByChannel(msg.Channel(), handlers.GetTextAndAttachments(msg), 160) - for i, part := range parts { + for _, part := range parts { // build our request form := map[string]string{ "id": msg.ID().String(), @@ -252,18 +236,12 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat formEncoded := encodeVariables(form, contentType) - // put quick replies on last message part url := replaceVariables(sendURL, formEncoded) var body io.Reader if sendMethod == http.MethodPost || sendMethod == http.MethodPut { formEncoded = encodeVariables(form, contentType) - if i == len(parts)-1 { - formEncoded["quick_replies"] = buildQuickRepliesResponse(msg.QuickReplies(), sendMethod, contentType) - } else { - formEncoded["quick_replies"] = buildQuickRepliesResponse([]string{}, sendMethod, contentType) - } body = strings.NewReader(replaceVariables(sendBody, formEncoded)) } @@ -294,28 +272,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } -type quickReplyXMLItem struct { - XMLName xml.Name `xml:"item"` - Value string `xml:",chardata"` -} - -func buildQuickRepliesResponse(quickReplies []string, sendMethod string, contentType string) string { - if quickReplies == nil { - quickReplies = []string{} - } - if (sendMethod == http.MethodPost || sendMethod == http.MethodPut) && contentType == contentJSON { - marshalled, _ := json.Marshal(quickReplies) - return string(marshalled) - } - response := bytes.Buffer{} - - for _, reply := range quickReplies { - reply = url.QueryEscape(reply) - response.WriteString(fmt.Sprintf("&quick_reply=%s", reply)) - } - return response.String() -} - func encodeVariables(variables map[string]string, contentType string) map[string]string { encoded := make(map[string]string) diff --git a/handlers/discord/discord_test.go b/handlers/discord/discord_test.go index 24e8e02e6..6a29139a1 100644 --- a/handlers/discord/discord_test.go +++ b/handlers/discord/discord_test.go @@ -3,501 +3,43 @@ package discord import ( "net/http/httptest" "testing" - "time" - - "net/http" "github.com/nyaruka/courier" . "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" ) -var ( - receiveValidMessage = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join" - receiveValidMessageFrom = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?from=%2B2349067554729&text=Join" - receiveValidNoPlus = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?from=2349067554729&text=Join" - receiveValidMessageWithDate = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join&date=2017-06-23T12:30:00.500Z" - receiveValidMessageWithTime = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join&time=2017-06-23T12:30:00Z" - receiveNoParams = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/" - invalidURN = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=MTN&text=Join" - receiveNoSender = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?text=Join" - receiveInvalidDate = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=%2B2349067554729&text=Join&time=20170623T123000Z" - failedNoParams = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/failed/" - failedValid = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/failed/?id=12345" - sentValid = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/sent/?id=12345" - invalidStatus = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/wired/" - deliveredValid = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/delivered/?id=12345" - deliveredValidPost = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/delivered/" - stoppedEvent = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/stopped/?from=%2B2349067554729" - stoppedEventPost = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/stopped/" - stoppedEventInvalidURN = "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/stopped/?from=MTN" -) - -var testChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", nil), -} - -var gmChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "GM", nil), -} - -var handleTestCases = []ChannelHandleTestCase{ - {Label: "Receive Valid Message", URL: receiveValidMessage, Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, - {Label: "Receive Valid Post", URL: receiveNoParams, Data: "sender=%2B2349067554729&text=Join", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, - {Label: "Receive Valid From", URL: receiveValidMessageFrom, Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, - {Label: "Receive Country Parse", URL: receiveValidNoPlus, Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, - {Label: "Receive Valid Message With Date", URL: receiveValidMessageWithDate, Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2349067554729"), Date: Tp(time.Date(2017, 6, 23, 12, 30, 0, int(500*time.Millisecond), time.UTC))}, - {Label: "Receive Valid Message With Time", URL: receiveValidMessageWithTime, Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2349067554729"), Date: Tp(time.Date(2017, 6, 23, 12, 30, 0, 0, time.UTC))}, - {Label: "Invalid URN", URL: invalidURN, Data: "empty", Status: 400, Response: "phone number supplied is not a number"}, - {Label: "Receive No Params", URL: receiveNoParams, Data: "empty", Status: 400, Response: "must have one of 'sender' or 'from' set"}, - {Label: "Receive No Sender", URL: receiveNoSender, Data: "empty", Status: 400, Response: "must have one of 'sender' or 'from' set"}, - {Label: "Receive Invalid Date", URL: receiveInvalidDate, Data: "empty", Status: 400, Response: "invalid date format, must be RFC 3339"}, - {Label: "Failed No Params", URL: failedNoParams, Status: 400, Response: "field 'id' required"}, - {Label: "Failed Valid", URL: failedValid, Status: 200, Response: `"status":"F"`}, - {Label: "Invalid Status", URL: invalidStatus, Status: 404, Response: `page not found`}, - {Label: "Sent Valid", URL: sentValid, Status: 200, Response: `"status":"S"`}, - {Label: "Delivered Valid", URL: deliveredValid, Status: 200, Data: "nothing", Response: `"status":"D"`}, - {Label: "Delivered Valid Post", URL: deliveredValidPost, Data: "id=12345", Status: 200, Response: `"status":"D"`}, - {Label: "Stopped Event", URL: stoppedEvent, Status: 200, Data: "nothing", Response: "Accepted"}, - {Label: "Stopped Event Post", URL: stoppedEventPost, Data: "from=%2B2349067554729", Status: 200, Response: "Accepted"}, - {Label: "Stopped Event Invalid URN", URL: stoppedEventInvalidURN, Data: "empty", Status: 400, Response: "phone number supplied is not a number"}, - {Label: "Stopped event No Params", URL: stoppedEventPost, Status: 400, Response: "field 'from' required"}, +func TestHandler(t *testing.T) { + RunChannelTestCases(t, testChannels, newHandler(), testCases) } -var testSOAPReceiveChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - configTextXPath: "//content", - configFromXPath: "//source", - configMOResponse: "0", - configMOResponseContentType: "text/xml", - })} - -var handleSOAPReceiveTestCases = []ChannelHandleTestCase{ - {Label: "Receive Valid Post SOAP", URL: receiveNoParams, Data: `2349067554729Join`, - Status: 200, Response: "0", - Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, - {Label: "Receive Invalid SOAP", URL: receiveNoParams, Data: ``, - Status: 400, Response: "missing from"}, +func BenchmarkHandler(b *testing.B) { + RunChannelBenchmarks(b, testChannels, newHandler(), testCases) } -var gmTestCases = []ChannelHandleTestCase{ - {Label: "Receive Non Plus Message", URL: "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sender=2207222333&text=Join", Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+2207222333")}, +var testChannels = []courier.Channel{ + courier.NewMockChannel("bac782c2-7aeb-4389-92f5-97887744f573", "DS", "discord", "US", map[string]interface{}{}), } -var customChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - configMOFromField: "from_number", - configMODateField: "timestamp", - configMOTextField: "messageText", - })} - -var customTestCases = []ChannelHandleTestCase{ - {Label: "Receive Custom Message", URL: "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?from_number=12067799192&messageText=Join×tamp=2017-06-23T12:30:00Z", Data: "empty", Status: 200, Response: "Accepted", - Text: Sp("Join"), URN: Sp("tel:+12067799192"), Date: Tp(time.Date(2017, 6, 23, 12, 30, 0, 0, time.UTC))}, - {Label: "Receive Custom Missing", URL: "/c/ex/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive/?sent_from=12067799192&messageText=Join", Data: "empty", Status: 400, Response: "must have one of 'sender' or 'from' set"}, +var testCases = []ChannelHandleTestCase{ + {Label: "Recieve Message", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802&text=hello`, Status: 200, Text: Sp("hello"), URN: Sp("discord:694634743521607802")}, + {Label: "Invalid ID", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=somebody&text=hello`, Status: 400, Response: "Error"}, + {Label: "Garbage Body", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `sdfaskdfajsdkfajsdfaksdf`, Status: 400, Response: "Error"}, + {Label: "Missing Text", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802`, Status: 400, Response: "Error"}, } -func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), handleTestCases) - RunChannelTestCases(t, testSOAPReceiveChannels, newHandler(), handleSOAPReceiveTestCases) - RunChannelTestCases(t, gmChannels, newHandler(), gmTestCases) - RunChannelTestCases(t, customChannels, newHandler(), customTestCases) -} - -func BenchmarkHandler(b *testing.B) { - RunChannelBenchmarks(b, testChannels, newHandler(), handleTestCases) - RunChannelBenchmarks(b, testSOAPReceiveChannels, newHandler(), handleSOAPReceiveTestCases) +var sendTestCases = []ChannelSendTestCase{ + {Label: "Simple Send", Text: "Hello World", URN: "discord:694634743521607802", Path: "/discord/rp/send", ResponseBody: "", ResponseStatus: 200, SendPrep: setSendURL}, } // setSendURL takes care of setting the send_url to our test server host func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { // this is actually a path, which we'll combine with the test server URL - sendURL := c.StringConfigForKey("send_path", "") + sendURL := c.StringConfigForKey("send_path", "/discord/rp/send") sendURL, _ = utils.AddURLPath(s.URL, sendURL) c.(*courier.MockChannel).SetConfig(courier.ConfigSendURL, sendURL) } - -var longSendTestCases = []ChannelSendTestCase{ - {Label: "Long Send", - Text: "This is a long message that will be longer than 30....... characters", URN: "tel:+250788383383", - QuickReplies: []string{"One"}, - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - URLParams: map[string]string{"text": "characters", "to": "+250788383383", "from": "2020", "quick_reply": "One"}, - SendPrep: setSendURL}, -} - -var getSendSmartEncodingTestCases = []ChannelSendTestCase{ - {Label: "Smart Encoding", - Text: "Fancy “Smart” Quotes", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - URLParams: map[string]string{"text": `Fancy "Smart" Quotes`, "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, -} - -var postSendSmartEncodingTestCases = []ChannelSendTestCase{ - {Label: "Smart Encoding", - Text: "Fancy “Smart” Quotes", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - PostParams: map[string]string{"text": `Fancy "Smart" Quotes`, "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, -} - -var getSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - URLParams: map[string]string{"text": "Simple Message", "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, - {Label: "Unicode Send", - Text: "☺", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - URLParams: map[string]string{"text": "☺", "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, - {Label: "Error Sending", - Text: "Error Message", URN: "tel:+250788383383", - Status: "E", - ResponseBody: "1: Unknown channel", ResponseStatus: 401, - URLParams: map[string]string{"text": `Error Message`, "to": "+250788383383"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, - {Label: "Send Attachment", - Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", - ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, - URLParams: map[string]string{"text": "My pic!\nhttps://foo.bar/image.jpg", "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, -} - -var postSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - PostParams: map[string]string{"text": "Simple Message", "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, - {Label: "Unicode Send", - Text: "☺", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - PostParams: map[string]string{"text": "☺", "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, - {Label: "Error Sending", - Text: "Error Message", URN: "tel:+250788383383", - Status: "E", - ResponseBody: "1: Unknown channel", ResponseStatus: 401, - PostParams: map[string]string{"text": `Error Message`, "to": "+250788383383"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, - {Label: "Send Attachment", - Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", - ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, - PostParams: map[string]string{"text": "My pic!\nhttps://foo.bar/image.jpg", "to": "+250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, -} - -var postSendCustomContentTypeTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - PostParams: map[string]string{"text": "Simple Message", "to": "250788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded; charset=utf-8"}, - SendPrep: setSendURL}, -} - -var jsonSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: `{ "to":"+250788383383", "text":"Simple Message", "from":"2020", "quick_replies":[] }`, - Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, - SendPrep: setSendURL}, - {Label: "Unicode Send", - Text: `☺ "hi!"`, URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: `{ "to":"+250788383383", "text":"☺ \"hi!\"", "from":"2020", "quick_replies":[] }`, - Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, - SendPrep: setSendURL}, - {Label: "Error Sending", - Text: "Error Message", URN: "tel:+250788383383", - Status: "E", - ResponseBody: "1: Unknown channel", ResponseStatus: 401, - RequestBody: `{ "to":"+250788383383", "text":"Error Message", "from":"2020", "quick_replies":[] }`, - Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, - SendPrep: setSendURL}, - {Label: "Send Attachment", - Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", - ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, - RequestBody: `{ "to":"+250788383383", "text":"My pic!\nhttps://foo.bar/image.jpg", "from":"2020", "quick_replies":[] }`, - Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, - SendPrep: setSendURL}, - {Label: "Send Quick Replies", - Text: "Some message", URN: "tel:+250788383383", QuickReplies: []string{"One", "Two", "Three"}, - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: `{ "to":"+250788383383", "text":"Some message", "from":"2020", "quick_replies":["One","Two","Three"] }`, - Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, - SendPrep: setSendURL}, -} - -var jsonLongSendTestCases = []ChannelSendTestCase{ - {Label: "Send Quick Replies", - Text: "This is a long message that will be longer than 30....... characters", URN: "tel:+250788383383", - QuickReplies: []string{"One", "Two", "Three"}, - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: `{ "to":"+250788383383", "text":"characters", "from":"2020", "quick_replies":["One","Two","Three"] }`, - Headers: map[string]string{"Authorization": "Token ABCDEF", "Content-Type": "application/json"}, - SendPrep: setSendURL}, -} - -var xmlSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: `+250788383383Simple Message2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Unicode Send", - Text: `☺`, URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: `+250788383383☺2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Error Sending", - Text: "Error Message", URN: "tel:+250788383383", - Status: "E", - ResponseBody: "1: Unknown channel", ResponseStatus: 401, - RequestBody: `+250788383383Error Message2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Send Attachment", - Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", - ResponseBody: `0: Accepted for delivery`, ResponseStatus: 200, - RequestBody: `+250788383383My pic! https://foo.bar/image.jpg2020` + - ``, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Send Quick Replies", - Text: "Some message", URN: "tel:+250788383383", QuickReplies: []string{"One", "Two", "Three"}, - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: "+250788383383Some message2020" + - "OneTwoThree", - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, -} - -var xmlLongSendTestCases = []ChannelSendTestCase{ - {Label: "Send Quick Replies", - Text: "This is a long message that will be longer than 30....... characters", URN: "tel:+250788383383", - QuickReplies: []string{"One", "Two", "Three"}, - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - RequestBody: "+250788383383characters2020" + - "OneTwoThree", - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, -} - -var xmlSendWithResponseContentTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0", ResponseStatus: 200, - RequestBody: `+250788383383Simple Message2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Unicode Send", - Text: `☺`, URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0", ResponseStatus: 200, - RequestBody: `+250788383383☺2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Error Sending", - Text: "Error Message", URN: "tel:+250788383383", - Status: "E", - ResponseBody: "0", ResponseStatus: 401, - RequestBody: `+250788383383Error Message2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Error Sending with 200 status code", - Text: "Error Message", URN: "tel:+250788383383", - Status: "E", - ResponseBody: "1", ResponseStatus: 200, - RequestBody: `+250788383383Error Message2020`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Send Attachment", - Text: "My pic!", URN: "tel:+250788383383", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", - ResponseBody: `0`, ResponseStatus: 200, - RequestBody: `+250788383383My pic! https://foo.bar/image.jpg2020` + - ``, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, - {Label: "Send Quick Replies", - Text: "Some message", URN: "tel:+250788383383", QuickReplies: []string{"One", "Two", "Three"}, - Status: "W", - ResponseBody: "0", ResponseStatus: 200, - RequestBody: `+250788383383Some message2020` + - `OneTwoThree`, - Headers: map[string]string{"Content-Type": "text/xml; charset=utf-8"}, - SendPrep: setSendURL}, -} - -var nationalGetSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "tel:+250788383383", - Status: "W", - ResponseBody: "0: Accepted for delivery", ResponseStatus: 200, - URLParams: map[string]string{"text": "Simple Message", "to": "788383383", "from": "2020"}, - Headers: map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, - SendPrep: setSendURL}, -} - func TestSending(t *testing.T) { - var getChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - courier.ConfigSendMethod: http.MethodGet}) - - var getSmartChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - configEncoding: encodingSmart, - courier.ConfigSendMethod: http.MethodGet}) - - var postChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: "to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - courier.ConfigSendMethod: http.MethodPost}) - - var postChannelCustomContentType = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: "to={{to_no_plus}}&text={{text}}&from={{from_no_plus}}{{quick_replies}}", - courier.ConfigContentType: "application/x-www-form-urlencoded; charset=utf-8", - courier.ConfigSendMethod: http.MethodPost}) - - var postSmartChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: "to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - configEncoding: encodingSmart, - courier.ConfigSendMethod: http.MethodPost}) - - var jsonChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", - }) - - var xmlChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, - courier.ConfigContentType: contentXML, - courier.ConfigSendMethod: http.MethodPut, - }) - - var xmlChannelWithResponseContent = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, - configMTResponseCheck: "0", - courier.ConfigContentType: contentXML, - courier.ConfigSendMethod: http.MethodPut, - }) - - RunChannelSendTestCases(t, getChannel, newHandler(), getSendTestCases, nil) - RunChannelSendTestCases(t, getSmartChannel, newHandler(), getSendTestCases, nil) - RunChannelSendTestCases(t, getSmartChannel, newHandler(), getSendSmartEncodingTestCases, nil) - RunChannelSendTestCases(t, postChannel, newHandler(), postSendTestCases, nil) - RunChannelSendTestCases(t, postChannelCustomContentType, newHandler(), postSendCustomContentTypeTestCases, nil) - RunChannelSendTestCases(t, postSmartChannel, newHandler(), postSendTestCases, nil) - RunChannelSendTestCases(t, postSmartChannel, newHandler(), postSendSmartEncodingTestCases, nil) - RunChannelSendTestCases(t, jsonChannel, newHandler(), jsonSendTestCases, nil) - RunChannelSendTestCases(t, xmlChannel, newHandler(), xmlSendTestCases, nil) - RunChannelSendTestCases(t, xmlChannelWithResponseContent, newHandler(), xmlSendWithResponseContentTestCases, nil) - - var getChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "max_length": 30, - "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - courier.ConfigSendMethod: http.MethodGet}) - - var getChannel30StrLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "max_length": "30", - "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - courier.ConfigSendMethod: http.MethodGet}) - - var jsonChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", - }) - - var xmlChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, - courier.ConfigContentType: contentXML, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", - }) - - RunChannelSendTestCases(t, getChannel30IntLength, newHandler(), longSendTestCases, nil) - RunChannelSendTestCases(t, getChannel30StrLength, newHandler(), longSendTestCases, nil) - RunChannelSendTestCases(t, jsonChannel30IntLength, newHandler(), jsonLongSendTestCases, nil) - RunChannelSendTestCases(t, xmlChannel30IntLength, newHandler(), xmlLongSendTestCases, nil) - - var nationalChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", - map[string]interface{}{ - "send_path": "?to={{to}}&text={{text}}&from={{from}}{{quick_replies}}", - "use_national": true, - courier.ConfigSendMethod: http.MethodGet}) - - RunChannelSendTestCases(t, nationalChannel, newHandler(), nationalGetSendTestCases, nil) + RunChannelSendTestCases(t, testChannels[0], newHandler(), sendTestCases, nil) } From dfb597e32734eb93633023f2bd8cfbec15660623 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Mon, 10 Aug 2020 18:58:00 -0700 Subject: [PATCH 06/24] Update go.mod with repo path for fork --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 0b028a186..cf73bfda1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/nyaruka/courier -replace github.com/nyaruka/gocommon => /home/awen/go/src/github.com/nyaruka/gocommon +replace github.com/nyaruka/gocommon => github.com/awensaunders/gocommon v1.2.1-0.20200804204232-b6e8c3fe0df1 require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 From 676b5c23839b15e2171e3ef96055cfd8ff7b0935 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Wed, 26 Aug 2020 22:45:27 -0700 Subject: [PATCH 07/24] Add attachment support for Discord, improve coverage We want to be able to send attachments to our discord proxy, so now we can. Also use a more sensible JSON serialization logic as we don't need to worry about multiple content types... --- handlers/discord/discord.go | 158 +++++++++++-------------------- handlers/discord/discord_test.go | 5 +- 2 files changed, 57 insertions(+), 106 deletions(-) diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go index 8ed56ad44..6f34b4e46 100644 --- a/handlers/discord/discord.go +++ b/handlers/discord/discord.go @@ -1,6 +1,7 @@ package discord import ( + "bytes" "context" "encoding/json" "fmt" @@ -55,23 +56,16 @@ func newHandler() courier.ChannelHandler { func (h *handler) Initialize(s courier.Server) error { h.SetServer(s) s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveMessage) - s.AddHandlerRoute(h, http.MethodGet, "receive", h.receiveMessage) sentHandler := h.buildStatusHandler("sent") - s.AddHandlerRoute(h, http.MethodGet, "sent", sentHandler) s.AddHandlerRoute(h, http.MethodPost, "sent", sentHandler) deliveredHandler := h.buildStatusHandler("delivered") - s.AddHandlerRoute(h, http.MethodGet, "delivered", deliveredHandler) s.AddHandlerRoute(h, http.MethodPost, "delivered", deliveredHandler) failedHandler := h.buildStatusHandler("failed") - s.AddHandlerRoute(h, http.MethodGet, "failed", failedHandler) s.AddHandlerRoute(h, http.MethodPost, "failed", failedHandler) - s.AddHandlerRoute(h, http.MethodPost, "stopped", h.receiveStopContact) - s.AddHandlerRoute(h, http.MethodGet, "stopped", h.receiveStopContact) - return nil } @@ -79,30 +73,6 @@ type stopContactForm struct { From string `validate:"required" name:"from"` } -func (h *handler) receiveStopContact(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { - form := &stopContactForm{} - err := handlers.DecodeAndValidateForm(form, r) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - // create our URN - urn := urns.NilURN - urn, err = urns.NewURNFromParts(urns.DiscordScheme, form.From, "", "") - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - urn = urn.Normalize("") - - // create a stop channel event - channelEvent := h.Backend().NewChannelEvent(channel, courier.StopContact, urn) - err = h.Backend().WriteChannelEvent(ctx, channelEvent) - if err != nil { - return nil, err - } - return []courier.Event{channelEvent}, courier.WriteChannelEventSuccess(ctx, w, r, channelEvent) -} - // utility function to grab the form value for either the passed in name (if non-empty) or the first set // value from defaultNames func getFormField(form url.Values, defaultNames []string, name string) string { @@ -216,87 +186,65 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat // sendBody := msg.Channel().StringConfigForKey(courier.ConfigSendBody, "") contentType := contentJSON contentTypeHeader := contentTypeMappings[contentType] - if contentTypeHeader == "" { - contentTypeHeader = contentType - } status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - parts := handlers.SplitMsgByChannel(msg.Channel(), handlers.GetTextAndAttachments(msg), 160) - for _, part := range parts { - // build our request - form := map[string]string{ - "id": msg.ID().String(), - "text": part, - "to": msg.URN().Path(), - "to_no_plus": strings.TrimPrefix(msg.URN().Path(), "+"), - "from": msg.Channel().Address(), - "from_no_plus": strings.TrimPrefix(msg.Channel().Address(), "+"), - "channel": msg.Channel().UUID().String(), - } - - formEncoded := encodeVariables(form, contentType) - - url := replaceVariables(sendURL, formEncoded) - - var body io.Reader - if sendMethod == http.MethodPost || sendMethod == http.MethodPut { - formEncoded = encodeVariables(form, contentType) - - body = strings.NewReader(replaceVariables(sendBody, formEncoded)) - } - - req, err := http.NewRequest(sendMethod, url, body) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", contentTypeHeader) - - authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") - if authorization != "" { - req.Header.Set("Authorization", authorization) - } - - rr, err := utils.MakeHTTPRequest(req) - - // record our status and log - log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) - status.AddLog(log) - if err != nil { - return status, nil - } - // If we don't have an error, set the message as wired and move on - status.SetStatus(courier.MsgWired) - + fmt.Println("Content Type:", contentType) + fmt.Println("Hellooooooooo") + fmt.Println(msg.Attachments()) + attachmentURLs := []string{} + for _, attachment := range msg.Attachments() { + _, attachmentURL := handlers.SplitAttachment(attachment) + attachmentURLs = append(attachmentURLs, attachmentURL) + } + attachmentString := fmt.Sprint(attachmentURLs) + fmt.Println(attachmentString) + // build our request + type OutputMessage struct { + ID string `json:"id"` + Text string `json:"text"` + To string `json:"to"` + Channel string `json:"channel"` + Attachments []string `json:"attachments"` + } + + ourMessage := OutputMessage{ + ID: msg.ID().String(), + Text: msg.Text(), + To: msg.URN().Path(), + Channel: msg.Channel().UUID().String(), + Attachments: attachmentURLs, + } + + var body io.Reader + marshalled, err := json.Marshal(ourMessage) + if err != nil { + return nil, err } + body = bytes.NewReader(marshalled) + fmt.Print("Body:") + fmt.Println(body) - return status, nil -} - -func encodeVariables(variables map[string]string, contentType string) map[string]string { - encoded := make(map[string]string) + req, err := http.NewRequest(sendMethod, sendURL, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentTypeHeader) - for k, v := range variables { - // encode according to our content type - switch contentType { - case contentJSON: - marshalled, _ := json.Marshal(v) - v = string(marshalled) + authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") + if authorization != "" { + req.Header.Set("Authorization", authorization) + } - case contentURLEncoded: - v = url.QueryEscape(v) + rr, err := utils.MakeHTTPRequest(req) - } - encoded[k] = v + // record our status and log + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + status.AddLog(log) + if err != nil { + return status, nil } - return encoded -} + // If we don't have an error, set the message as wired and move on + status.SetStatus(courier.MsgWired) -func replaceVariables(text string, variables map[string]string) string { - for k, v := range variables { - text = strings.Replace(text, fmt.Sprintf("{{%s}}", k), v, -1) - } - return text + return status, nil } - -// const defaultSendBody = `id={{id}}&text={{text}}&to={{to}}&to_no_plus={{to_no_plus}}&from={{from}}&from_no_plus={{from_no_plus}}&channel={{channel}}` -const sendBody = `{"id": {{id}}, "text": {{text}}, "to":{{to}}, "channel": {{channel}}}` diff --git a/handlers/discord/discord_test.go b/handlers/discord/discord_test.go index 6a29139a1..b49151051 100644 --- a/handlers/discord/discord_test.go +++ b/handlers/discord/discord_test.go @@ -26,10 +26,13 @@ var testCases = []ChannelHandleTestCase{ {Label: "Invalid ID", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=somebody&text=hello`, Status: 400, Response: "Error"}, {Label: "Garbage Body", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `sdfaskdfajsdkfajsdfaksdf`, Status: 400, Response: "Error"}, {Label: "Missing Text", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802`, Status: 400, Response: "Error"}, + {Label: "Message Sent Handler", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/sent/", Data: `id=12345`, Status: 200, Response: `"status":"S"`}, + {Label: "Message Sent Handler Garbage", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/sent/", Data: `nothing`, Status: 400}, } var sendTestCases = []ChannelSendTestCase{ - {Label: "Simple Send", Text: "Hello World", URN: "discord:694634743521607802", Path: "/discord/rp/send", ResponseBody: "", ResponseStatus: 200, SendPrep: setSendURL}, + {Label: "Simple Send", Text: "Hello World", URN: "discord:694634743521607802", Path: "/discord/rp/send", ResponseStatus: 200, RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":[]}`, SendPrep: setSendURL}, + {Label: "Simple Send", Text: "Hello World", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "discord:694634743521607802", Path: "/discord/rp/send", RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":["https://foo.bar/image.jpg"]}`, ResponseStatus: 200, SendPrep: setSendURL}, } // setSendURL takes care of setting the send_url to our test server host From 69bd1521f83295d721e4e797f22644ec911179b9 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Wed, 26 Aug 2020 23:09:17 -0700 Subject: [PATCH 08/24] Delete printf debugging --- handlers/discord/discord.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go index 6f34b4e46..c87b397c3 100644 --- a/handlers/discord/discord.go +++ b/handlers/discord/discord.go @@ -188,16 +188,11 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat contentTypeHeader := contentTypeMappings[contentType] status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - fmt.Println("Content Type:", contentType) - fmt.Println("Hellooooooooo") - fmt.Println(msg.Attachments()) attachmentURLs := []string{} for _, attachment := range msg.Attachments() { _, attachmentURL := handlers.SplitAttachment(attachment) attachmentURLs = append(attachmentURLs, attachmentURL) } - attachmentString := fmt.Sprint(attachmentURLs) - fmt.Println(attachmentString) // build our request type OutputMessage struct { ID string `json:"id"` @@ -221,8 +216,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return nil, err } body = bytes.NewReader(marshalled) - fmt.Print("Body:") - fmt.Println(body) req, err := http.NewRequest(sendMethod, sendURL, body) if err != nil { From 66089f9a50e0d46532050478f4ac225d80453073 Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Mon, 31 Aug 2020 08:26:38 -0300 Subject: [PATCH 09/24] Add WhatsApp medias that failed to upload on cache --- config.go | 96 +++++++++++++++--------------- handlers/whatsapp/whatsapp.go | 12 +++- handlers/whatsapp/whatsapp_test.go | 29 +++++++-- 3 files changed, 81 insertions(+), 56 deletions(-) diff --git a/config.go b/config.go index dd3d4d533..857a47d05 100644 --- a/config.go +++ b/config.go @@ -4,32 +4,33 @@ import "github.com/nyaruka/ezconf" // Config is our top level configuration object type Config struct { - Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` - SentryDSN string `help:"the DSN used for logging errors to Sentry"` - Domain string `help:"the domain courier is exposed on"` - Address string `help:"the network interface address courier will bind to"` - Port int `help:"the port courier will listen on"` - DB string `help:"URL describing how to connect to the RapidPro database"` - Redis string `help:"URL describing how to connect to Redis"` - SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` - S3Endpoint string `help:"the S3 endpoint we will write attachments to"` - S3Region string `help:"the S3 region we will write attachments to"` - S3MediaBucket string `help:"the S3 bucket we will write attachments to"` - S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` - S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` - S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` - AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` - AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - FacebookApplicationSecret string `help:"the Facebook app secret"` - FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` - WhatsAppMediaExpiration int `help:"the time in seconds for expire WhatsApp media ids in Redis"` - MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` - LibratoUsername string `help:"the username that will be used to authenticate to Librato"` - LibratoToken string `help:"the token that will be used to authenticate to Librato"` - StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` - StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` - LogLevel string `help:"the logging level courier should use"` - Version string `help:"the version that will be used in request and response headers"` + Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` + SentryDSN string `help:"the DSN used for logging errors to Sentry"` + Domain string `help:"the domain courier is exposed on"` + Address string `help:"the network interface address courier will bind to"` + Port int `help:"the port courier will listen on"` + DB string `help:"URL describing how to connect to the RapidPro database"` + Redis string `help:"URL describing how to connect to Redis"` + SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` + S3Endpoint string `help:"the S3 endpoint we will write attachments to"` + S3Region string `help:"the S3 region we will write attachments to"` + S3MediaBucket string `help:"the S3 bucket we will write attachments to"` + S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` + S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` + S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` + AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` + AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` + FacebookApplicationSecret string `help:"the Facebook app secret"` + FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` + WhatsAppMediaExpiration int `help:"the time in seconds for expire WhatsApp media ids in Redis"` + WhatsAppFailedMediaExpiration int `help:"the time in seconds for expire media URLs in Redis that the upload failed, then a new upload attempt will occur"` + MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` + LibratoUsername string `help:"the username that will be used to authenticate to Librato"` + LibratoToken string `help:"the token that will be used to authenticate to Librato"` + StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` + StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` + LogLevel string `help:"the logging level courier should use"` + Version string `help:"the version that will be used in request and response headers"` // IncludeChannels is the list of channels to enable, empty means include all IncludeChannels []string @@ -41,27 +42,28 @@ type Config struct { // NewConfig returns a new default configuration object func NewConfig() *Config { return &Config{ - Backend: "rapidpro", - Domain: "localhost", - Address: "", - Port: 8080, - DB: "postgres://temba:temba@localhost/temba?sslmode=disable", - Redis: "redis://localhost:6379/15", - SpoolDir: "/var/spool/courier", - S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", - S3MediaBucket: "courier-media", - S3MediaPrefix: "/media/", - S3DisableSSL: false, - S3ForcePathStyle: false, - AWSAccessKeyID: "missing_aws_access_key_id", - AWSSecretAccessKey: "missing_aws_secret_access_key", - FacebookApplicationSecret: "missing_facebook_app_secret", - FacebookWebhookSecret: "missing_facebook_webhook_secret", - WhatsAppMediaExpiration: 86400, - MaxWorkers: 32, - LogLevel: "error", - Version: "Dev", + Backend: "rapidpro", + Domain: "localhost", + Address: "", + Port: 8080, + DB: "postgres://temba:temba@localhost/temba?sslmode=disable", + Redis: "redis://localhost:6379/15", + SpoolDir: "/var/spool/courier", + S3Endpoint: "https://s3.amazonaws.com", + S3Region: "us-east-1", + S3MediaBucket: "courier-media", + S3MediaPrefix: "/media/", + S3DisableSSL: false, + S3ForcePathStyle: false, + AWSAccessKeyID: "missing_aws_access_key_id", + AWSSecretAccessKey: "missing_aws_secret_access_key", + FacebookApplicationSecret: "missing_facebook_app_secret", + FacebookWebhookSecret: "missing_facebook_webhook_secret", + WhatsAppMediaExpiration: 86400, // 1d + WhatsAppFailedMediaExpiration: 1800, // 30min + MaxWorkers: 32, + LogLevel: "error", + Version: "Dev", } } diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 858064751..355257d70 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -625,6 +625,12 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return "", logs, err } + // check on failure cache + failureCacheKey := fmt.Sprintf("whatsapp_failed_media:%s:%s", msg.Channel().UUID().String(), mediaURL) + if failed, _ := redis.Bool(rc.Do("GET", failureCacheKey)); failed { + return "", logs, errors.New("ignoring media that previously failed to upload") + } + // download media req, err := http.NewRequest("GET", mediaURL, nil) if err != nil { @@ -648,6 +654,9 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri req.Header.Add("Content-Type", mimeType) res, err = utils.MakeHTTPRequest(req) if err != nil { + // put on failure cache + rc.Do("SET", failureCacheKey, true, "EX", h.Server().Config().WhatsAppFailedMediaExpiration) + if res != nil { err = errors.Wrap(err, string(res.Body)) } @@ -667,9 +676,6 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri } // put on cache - rc = h.Backend().RedisPool().Get() - defer rc.Close() - _, err = rc.Do("SET", cacheKey, mediaID, "EX", h.Server().Config().WhatsAppMediaExpiration) if err != nil { elapsed := time.Now().Sub(start) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 857a47ed4..78a304cdb 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -557,11 +557,28 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ }, SendPrep: setSendURL, }, - {Label: "Media Upload OK", + {Label: "Previous Media Upload Error", Text: "document caption", URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + BodyContains: `/document.pdf`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Media Upload OK", + Text: "video caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"video/mp4:https://foo.bar/video.mp4"}, Responses: map[MockedRequest]MockedResponse{ MockedRequest{ Method: "POST", @@ -569,12 +586,12 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: "media bytes", }: MockedResponse{ Status: 200, - Body: `{ "media" : [{"id": "f043afd0-f0ae-4b9c-ab3d-696fb4c8cd68"}] }`, + Body: `{ "media" : [{"id": "36c484d1-1283-4b94-988d-7276bdec4de2"}] }`, }, MockedRequest{ Method: "POST", Path: "/v1/messages", - Body: `{"to":"250788123123","type":"document","document":{"id":"f043afd0-f0ae-4b9c-ab3d-696fb4c8cd68","caption":"document caption"}}`, + Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, }: MockedResponse{ Status: 201, Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, @@ -583,15 +600,15 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL, }, {Label: "Cached Media", - Text: "document caption", + Text: "video caption", URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", - Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Attachments: []string{"video/mp4:https://foo.bar/video.mp4"}, Responses: map[MockedRequest]MockedResponse{ MockedRequest{ Method: "POST", Path: "/v1/messages", - Body: `{"to":"250788123123","type":"document","document":{"id":"f043afd0-f0ae-4b9c-ab3d-696fb4c8cd68","caption":"document caption"}}`, + Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, }: MockedResponse{ Status: 201, Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, From 85feed2edf0357918d3fd004f9406351b894aedc Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Thu, 3 Sep 2020 15:15:16 -0700 Subject: [PATCH 10/24] Remove go.mod override for gocommon Now that gocommon has the discord work upstreamed, we can remove the override that pointed to the fork where those changes were made. --- go.mod | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.mod b/go.mod index cf73bfda1..37b3c739d 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,5 @@ module github.com/nyaruka/courier -replace github.com/nyaruka/gocommon => github.com/awensaunders/gocommon v1.2.1-0.20200804204232-b6e8c3fe0df1 - require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect From e7fefbac909a0daf3c7a44c76545ee80a32a438f Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Thu, 3 Sep 2020 15:28:15 -0700 Subject: [PATCH 11/24] Update go.mod Now that the gocommon discord changes are upstreamed, we need to bump the dep in order for things to work. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 37b3c739d..5e3145c79 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.2.0 + github.com/nyaruka/gocommon v1.4.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/nyaruka/phonenumbers v1.0.44 // indirect From c1f7202ec34f9b81a0a43ac4a84929f5c1e547c6 Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Tue, 8 Sep 2020 16:14:05 -0300 Subject: [PATCH 12/24] Use hashes in WhatsApp medias caching --- handlers/whatsapp/whatsapp.go | 66 ++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 12 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 355257d70..4aa5f4ad5 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -28,6 +28,9 @@ const ( channelTypeWa = "WA" channelTypeD3 = "D3" + + mediaCacheKeyPattern = "whatsapp_media:%s" + failureMediaCacheKeyPattern = "whatsapp_failed_media:%s" ) var ( @@ -610,14 +613,14 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri var logs []*courier.ChannelLog start := time.Now() - // check on cache first + // check in cache first rc := h.Backend().RedisPool().Get() defer rc.Close() - cacheKey := fmt.Sprintf("whatsapp_media:%s:%s", msg.Channel().UUID().String(), mediaURL) - mediaID, err := redis.String(rc.Do("GET", cacheKey)) + cacheKey := fmt.Sprintf(mediaCacheKeyPattern, msg.Channel().UUID().String()) + expiration, err := getMediaExpirationFromCache(rc, cacheKey, mediaURL) if err == nil { - return mediaID, logs, nil + return expiration.MediaID, logs, nil } else if err != redis.ErrNil { elapsed := time.Now().Sub(start) log := courier.NewChannelLogFromError("error reading the media id from redis", msg.Channel(), msg.ID(), elapsed, err) @@ -625,9 +628,9 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return "", logs, err } - // check on failure cache - failureCacheKey := fmt.Sprintf("whatsapp_failed_media:%s:%s", msg.Channel().UUID().String(), mediaURL) - if failed, _ := redis.Bool(rc.Do("GET", failureCacheKey)); failed { + // check in failure cache + failureCacheKey := fmt.Sprintf(failureMediaCacheKeyPattern, msg.Channel().UUID().String()) + if expiration, _ := getMediaExpirationFromCache(rc, failureCacheKey, mediaURL); expiration != nil { return "", logs, errors.New("ignoring media that previously failed to upload") } @@ -654,8 +657,8 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri req.Header.Add("Content-Type", mimeType) res, err = utils.MakeHTTPRequest(req) if err != nil { - // put on failure cache - rc.Do("SET", failureCacheKey, true, "EX", h.Server().Config().WhatsAppFailedMediaExpiration) + // put in failure cache + setMediaExpirationInCache(rc, failureCacheKey, mediaURL, "", h.Server().Config().WhatsAppFailedMediaExpiration) if res != nil { err = errors.Wrap(err, string(res.Body)) @@ -667,7 +670,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri } // take uploaded media id - mediaID, err = jsonparser.GetString(res.Body, "media", "[0]", "id") + mediaID, err := jsonparser.GetString(res.Body, "media", "[0]", "id") if err != nil { elapsed := time.Now().Sub(start) log := courier.NewChannelLogFromError("error reading the media id from response", msg.Channel(), msg.ID(), elapsed, err) @@ -675,8 +678,8 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return "", logs, err } - // put on cache - _, err = rc.Do("SET", cacheKey, mediaID, "EX", h.Server().Config().WhatsAppMediaExpiration) + // put in cache + err = setMediaExpirationInCache(rc, cacheKey, mediaURL, mediaID, h.Server().Config().WhatsAppMediaExpiration) if err != nil { elapsed := time.Now().Sub(start) log := courier.NewChannelLogFromError("error setting the media id to redis", msg.Channel(), msg.ID(), elapsed, err) @@ -686,6 +689,45 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return mediaID, logs, nil } +type mediaExpiration struct { + ExpiresOn time.Time `json:"expires_on"` + MediaID string `json:"media_id,omitempty"` +} + +func getMediaExpirationFromCache(rc redis.Conn, cacheKey, mediaURL string) (*mediaExpiration, error) { + expirationJSON, err := redis.String(rc.Do("HGET", cacheKey, mediaURL)) + if err != nil { + return nil, err + } + + expiration := &mediaExpiration{} + err = json.Unmarshal([]byte(expirationJSON), expiration) + if err != nil { + return nil, err + } + + if expiration.ExpiresOn.Before(time.Now()) { + rc.Do("HDEL", cacheKey, mediaURL) + return nil, redis.ErrNil + } + return expiration, nil +} + +func setMediaExpirationInCache(rc redis.Conn, cacheKey, mediaURL, mediaID string, expiration int) error { + expiresOn := time.Now().Add(time.Second * time.Duration(expiration)) + expirationJSON, err := json.Marshal(mediaExpiration{ExpiresOn: expiresOn, MediaID: mediaID}) + if err != nil { + return err + } + + _, err = rc.Do("HSET", cacheKey, mediaURL, string(expirationJSON)) + if err != nil { + return err + } + + return nil +} + func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) From 3367a276a485ba69984258fe04f7ae558fb7590c Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Tue, 6 Oct 2020 15:02:51 -0700 Subject: [PATCH 13/24] Add quickreply support to discord handler Even though discord doesn't currently support quick replies, we can still pass this over to the proxy so that we don't need to change courier when it does. In the meantime, the proxy can do a partial attempt to support single emoji quick replies in the form of reactions, which is perfectly servicable for some purposes. --- handlers/discord/discord.go | 22 ++++++++++++---------- handlers/discord/discord_test.go | 5 +++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go index c87b397c3..e4ded589f 100644 --- a/handlers/discord/discord.go +++ b/handlers/discord/discord.go @@ -195,19 +195,21 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } // build our request type OutputMessage struct { - ID string `json:"id"` - Text string `json:"text"` - To string `json:"to"` - Channel string `json:"channel"` - Attachments []string `json:"attachments"` + ID string `json:"id"` + Text string `json:"text"` + To string `json:"to"` + Channel string `json:"channel"` + Attachments []string `json:"attachments"` + QuickReplies []string `json:"quick_replies"` } ourMessage := OutputMessage{ - ID: msg.ID().String(), - Text: msg.Text(), - To: msg.URN().Path(), - Channel: msg.Channel().UUID().String(), - Attachments: attachmentURLs, + ID: msg.ID().String(), + Text: msg.Text(), + To: msg.URN().Path(), + Channel: msg.Channel().UUID().String(), + Attachments: attachmentURLs, + QuickReplies: msg.QuickReplies(), } var body io.Reader diff --git a/handlers/discord/discord_test.go b/handlers/discord/discord_test.go index b49151051..43586ac0c 100644 --- a/handlers/discord/discord_test.go +++ b/handlers/discord/discord_test.go @@ -31,8 +31,9 @@ var testCases = []ChannelHandleTestCase{ } var sendTestCases = []ChannelSendTestCase{ - {Label: "Simple Send", Text: "Hello World", URN: "discord:694634743521607802", Path: "/discord/rp/send", ResponseStatus: 200, RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":[]}`, SendPrep: setSendURL}, - {Label: "Simple Send", Text: "Hello World", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "discord:694634743521607802", Path: "/discord/rp/send", RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":["https://foo.bar/image.jpg"]}`, ResponseStatus: 200, SendPrep: setSendURL}, + {Label: "Simple Send", Text: "Hello World", URN: "discord:694634743521607802", Path: "/discord/rp/send", ResponseStatus: 200, RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":[],"quick_replies":null}`, SendPrep: setSendURL}, + {Label: "Simple Send", Text: "Hello World", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "discord:694634743521607802", Path: "/discord/rp/send", RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":["https://foo.bar/image.jpg"],"quick_replies":null}`, ResponseStatus: 200, SendPrep: setSendURL}, + {Label: "Simple Send with attachements and Quick Replies", Text: "Hello World", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, QuickReplies: []string{"hello", "world"}, URN: "discord:694634743521607802", Path: "/discord/rp/send", RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":["https://foo.bar/image.jpg"],"quick_replies":["hello","world"]}`, ResponseStatus: 200, SendPrep: setSendURL}, } // setSendURL takes care of setting the send_url to our test server host From 3977b0a7a76a9b88bd1d43c6bba3b3481b9d7d14 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Tue, 6 Oct 2020 16:06:51 -0700 Subject: [PATCH 14/24] Delete unnecessary config in Discord handler We don't need the discord proxy handler to be particularly configurable in terms of the mapping from various different config variables, because it's totally pointless if we control both sides of the API, which being a standalone proxy, we do. This deletes some of the unneccessary configurability. --- handlers/discord/discord.go | 37 ++++++------------------------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go index e4ded589f..b939ba44d 100644 --- a/handlers/discord/discord.go +++ b/handlers/discord/discord.go @@ -20,26 +20,10 @@ import ( ) const ( - contentURLEncoded = "urlencoded" - contentJSON = "json" - - configMOFromField = "mo_from_field" - configMOTextField = "mo_text_field" - configMODateField = "mo_date_field" - - configMOResponseContentType = "mo_response_content_type" - configMOResponse = "mo_response" + jsonMimeTypeType = "application/json" + urlEncodedMimeType = "application/x-www-form-urlencoded" ) -var defaultFromFields = []string{"from", "sender"} -var defaultTextFields = []string{"text"} -var defaultDateFields = []string{"date", "time"} - -var contentTypeMappings = map[string]string{ - contentURLEncoded: "application/x-www-form-urlencoded", - contentJSON: "application/json", -} - func init() { courier.RegisterHandler(newHandler()) } @@ -75,21 +59,13 @@ type stopContactForm struct { // utility function to grab the form value for either the passed in name (if non-empty) or the first set // value from defaultNames -func getFormField(form url.Values, defaultNames []string, name string) string { +func getFormField(form url.Values, name string) string { if name != "" { values, found := form[name] if found { return values[0] } } - - for _, name := range defaultNames { - values, found := form[name] - if found { - return values[0] - } - } - return "" } @@ -105,8 +81,8 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, errors.Wrapf(err, "invalid request")) } - from = getFormField(r.Form, defaultFromFields, channel.StringConfigForKey(configMOFromField, "")) - text = getFormField(r.Form, defaultTextFields, channel.StringConfigForKey(configMOTextField, "")) + from = getFormField(r.Form, "from") + text = getFormField(r.Form, "text") // must have from field if from == "" { @@ -184,8 +160,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat // figure out what encoding to tell kannel to send as sendMethod := http.MethodPost // sendBody := msg.Channel().StringConfigForKey(courier.ConfigSendBody, "") - contentType := contentJSON - contentTypeHeader := contentTypeMappings[contentType] + contentTypeHeader := jsonMimeTypeType status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) attachmentURLs := []string{} From c208ec44223edd473bd372ef8e78838a0894b4d3 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 12 Oct 2020 13:46:49 -0500 Subject: [PATCH 15/24] Update to latest gocommon 1.5.3 and golang 1.15 --- .github/workflows/ci.yml | 2 +- go.mod | 8 ++++---- go.sum | 33 +++++++++++++++++++++++---------- handlers/viber/viber.go | 4 ++-- handlers/viber/viber_test.go | 3 +-- 5 files changed, 31 insertions(+), 19 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d69dcf20..5144a4880 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: '1.14.x' + go-version: '1.15.x' jobs: test: name: Test diff --git a/go.mod b/go.mod index dec9b85a6..f490817e6 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/nyaruka/courier require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect - github.com/aws/aws-sdk-go v1.34.17 + github.com/aws/aws-sdk-go v1.34.31 github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/dghubble/oauth1 v0.4.0 @@ -22,16 +22,16 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.5.1 + github.com/nyaruka/gocommon v1.5.3 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.1 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/go-playground/assert.v1 v1.2.1 gopkg.in/go-playground/validator.v9 v9.11.0 gopkg.in/h2non/filetype.v1 v1.0.5 ) -go 1.13 +go 1.15 diff --git a/go.sum b/go.sum index e356a0d72..7c4941daf 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/aws/aws-sdk-go v1.34.17 h1:9OzUgRrLmYm2mbfFx4v+2nBEg+Cvape1cvn9C3RNWTE= -github.com/aws/aws-sdk-go v1.34.17/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.31 h1:408wh5EHKzxyby8JpYfnn1w3fsF26AIU0o1kbJoRy7E= +github.com/aws/aws-sdk-go v1.34.31/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= @@ -38,8 +38,10 @@ github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= @@ -61,14 +63,14 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.5.1 h1:2R6uo6EVSTHOerupAmVm6h5fyufO189dlv/5gwHj3lM= -github.com/nyaruka/gocommon v1.5.1/go.mod h1:6XoaOsVk6z+294hM6pZxX3fDgT2IyLV8hFU4FoQz9Aw= +github.com/nyaruka/gocommon v1.5.3 h1:oWk3s5Ykho2uQMypR0LtW2vuDrcK1rIdvEW1X1VW4mw= +github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= -github.com/nyaruka/phonenumbers v1.0.57 h1:V4FNPs061PSUOEzQaLH0+pfzEdqoiMH/QJWryx/0hfs= -github.com/nyaruka/phonenumbers v1.0.57/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/phonenumbers v1.0.58 h1:IAlGDA4wuGQXe2lwOQvkZfBvA1DlAik+MX5k9k5C2IU= +github.com/nyaruka/phonenumbers v1.0.58/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -84,14 +86,21 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA= +golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= @@ -108,3 +117,7 @@ gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/viber/viber.go b/handlers/viber/viber.go index 6b52442ae..daf2d6a2c 100644 --- a/handlers/viber/viber.go +++ b/handlers/viber/viber.go @@ -171,7 +171,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return []courier.Event{channelEvent}, courier.WriteChannelEventSuccess(ctx, w, r, channelEvent) case "failed": - msgStatus := h.Backend().NewMsgStatusForExternalID(channel, string(payload.MessageToken), courier.MsgFailed) + msgStatus := h.Backend().NewMsgStatusForExternalID(channel, fmt.Sprintf("%d", payload.MessageToken), courier.MsgFailed) return handlers.WriteMsgStatusAndResponse(ctx, h, channel, msgStatus, w, r) case "delivered": @@ -248,7 +248,7 @@ func writeWelcomeMessageResponse(w http.ResponseWriter, channel courier.Channel, AuthToken: authToken, Text: msgText, Type: "text", - TrackingData: string(event.EventID()), + TrackingData: fmt.Sprintf("%d", event.EventID()), } responseBody := &bytes.Buffer{} diff --git a/handlers/viber/viber_test.go b/handlers/viber/viber_test.go index d712cb497..c725ad960 100644 --- a/handlers/viber/viber_test.go +++ b/handlers/viber/viber_test.go @@ -409,7 +409,6 @@ var ( } }` - receiveInvalidMessageType = `{ "event": "message", "timestamp": 1481142112807, @@ -487,7 +486,7 @@ var testWelcomeMessageCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: receiveURL, Data: validMsg, Status: 200, Response: "Accepted", Text: Sp("incoming msg"), URN: Sp("viber:xy5/5y6O81+/kbWHpLhBoA=="), ExternalID: Sp("4987381189870374000"), PrepRequest: addValidSignature}, - {Label: "Conversation Started", URL: receiveURL, Data: validConversationStarted, Status: 200, Response: `{"auth_token":"Token","text":"Welcome to VP, Please subscribe here for more.","type":"text","tracking_data":"\u0000"}`, PrepRequest: addValidSignature}, + {Label: "Conversation Started", URL: receiveURL, Data: validConversationStarted, Status: 200, Response: `{"auth_token":"Token","text":"Welcome to VP, Please subscribe here for more.","type":"text","tracking_data":"0"}`, PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { From 0a63e52d37a96db6d89c3eadcad4808ceecc844b Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 12 Oct 2020 14:34:36 -0500 Subject: [PATCH 16/24] Update CHANGELOG.md for v5.7.9 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8da2b69a8..1234f539e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v5.7.9 +---------- + * Update to latest gocommon 1.5.3 and golang 1.15 + * Add session status from mailroom to MT message sent to external channel API call + * Remove incoming message prefix for Play Mobile free accounts + v5.7.8 ---------- * deal with empty message in FreshChat incoming requests From fcd13204da497e798e0cf113c023ec50afedf6f4 Mon Sep 17 00:00:00 2001 From: Awen Saunders Date: Tue, 13 Oct 2020 17:31:22 -0700 Subject: [PATCH 17/24] Add support for attachments from discord-proxy We already supported attachments coming from courier to the discord proxy, now we support them the other way round too. --- handlers/discord/discord.go | 4 ++++ handlers/discord/discord_test.go | 1 + 2 files changed, 5 insertions(+) diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go index b939ba44d..999f4ce6d 100644 --- a/handlers/discord/discord.go +++ b/handlers/discord/discord.go @@ -105,6 +105,10 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w // build our msg msg := h.Backend().NewIncomingMsg(channel, urn, text).WithReceivedOn(date) + for _, attachment := range r.Form["attachments"] { + msg.WithAttachment(attachment) + } + // and finally write our message return handlers.WriteMsgsAndResponse(ctx, h, []courier.Msg{msg}, w, r) } diff --git a/handlers/discord/discord_test.go b/handlers/discord/discord_test.go index 43586ac0c..49bc6b3d9 100644 --- a/handlers/discord/discord_test.go +++ b/handlers/discord/discord_test.go @@ -23,6 +23,7 @@ var testChannels = []courier.Channel{ var testCases = []ChannelHandleTestCase{ {Label: "Recieve Message", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802&text=hello`, Status: 200, Text: Sp("hello"), URN: Sp("discord:694634743521607802")}, + {Label: "Recieve Message with attachment", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802&text=hello&attachments=https://test.test/foo.png`, Status: 200, Text: Sp("hello"), URN: Sp("discord:694634743521607802"), Attachments: []string{"https://test.test/foo.png"}}, {Label: "Invalid ID", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=somebody&text=hello`, Status: 400, Response: "Error"}, {Label: "Garbage Body", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `sdfaskdfajsdkfajsdfaksdf`, Status: 400, Response: "Error"}, {Label: "Missing Text", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802`, Status: 400, Response: "Error"}, From 1db08546e997a589b82a58850e55643d7b410d99 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Tue, 20 Oct 2020 12:46:59 +0200 Subject: [PATCH 18/24] Support receiving Multipart form data requests for EX channels --- handlers/external/external.go | 8 +++++++- handlers/external/external_test.go | 3 +++ handlers/test.go | 28 +++++++++++++++++++++++----- handlers/vk/vk.go | 21 +++++++++++---------- 4 files changed, 44 insertions(+), 16 deletions(-) diff --git a/handlers/external/external.go b/handlers/external/external.go index 6498ebd6f..0bc8b1b8f 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -173,7 +173,13 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w text = textNode.InnerText() } else { // parse our form - err := r.ParseForm() + contentType := r.Header.Get("Content-Type") + var err error + if strings.Contains(contentType, "multipart/form-data") { + err = r.ParseMultipartForm(10000000) + } else { + err = r.ParseForm() + } if err != nil { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, errors.Wrapf(err, "invalid request")) } diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index 315a3a8f1..1c4b84685 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -46,6 +46,9 @@ var handleTestCases = []ChannelHandleTestCase{ Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, {Label: "Receive Valid Post", URL: receiveNoParams, Data: "sender=%2B2349067554729&text=Join", Status: 200, Response: "Accepted", Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + + {Label: "Receive Valid Post multipart form", URL: receiveNoParams, MultipartFormFields: map[string]string{"sender": "2349067554729", "text": "Join"}, Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, {Label: "Receive Valid From", URL: receiveValidMessageFrom, Data: "empty", Status: 200, Response: "Accepted", Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, {Label: "Receive Country Parse", URL: receiveValidNoPlus, Data: "empty", Status: 200, Response: "Accepted", diff --git a/handlers/test.go b/handlers/test.go index a99a8ba83..6f139d194 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "io/ioutil" + "mime/multipart" "net/http" "net/http/httptest" "strings" @@ -33,6 +34,8 @@ type ChannelHandleTestCase struct { Response string Headers map[string]string + MultipartFormFields map[string]string + Name *string Text *string URN *string @@ -127,7 +130,7 @@ func ensureTestServerUp(host string) { } // utility method to make a request to a handler URL -func testHandlerRequest(tb testing.TB, s courier.Server, path string, headers map[string]string, data string, expectedStatus int, expectedBody *string, requestPrepFunc RequestPrepFunc) string { +func testHandlerRequest(tb testing.TB, s courier.Server, path string, headers map[string]string, data string, multipartFormFields map[string]string, expectedStatus int, expectedBody *string, requestPrepFunc RequestPrepFunc) string { var req *http.Request var err error url := fmt.Sprintf("https://%s%s", s.Config().Domain, path) @@ -144,6 +147,21 @@ func testHandlerRequest(tb testing.TB, s courier.Server, path string, headers ma contentType = "application/xml" } req.Header.Set("Content-Type", contentType) + } else if multipartFormFields != nil { + var body bytes.Buffer + bodyMultipartWriter := multipart.NewWriter(&body) + for k, v := range multipartFormFields { + fieldWriter, err := bodyMultipartWriter.CreateFormField(k) + require.Nil(tb, err) + _, err = fieldWriter.Write([]byte(v)) + require.Nil(tb, err) + } + contentType := fmt.Sprintf("multipart/form-data;boundary=%v", bodyMultipartWriter.Boundary()) + bodyMultipartWriter.Close() + + req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(body.Bytes())) + require.Nil(tb, err) + req.Header.Set("Content-Type", contentType) } else { req, err = http.NewRequest(http.MethodGet, url, nil) } @@ -350,7 +368,7 @@ func RunChannelTestCases(t *testing.T, channels []courier.Channel, handler couri mb.ClearQueueMsgs() mb.ClearSeenExternalIDs() - testHandlerRequest(t, s, testCase.URL, testCase.Headers, testCase.Data, testCase.Status, &testCase.Response, testCase.PrepRequest) + testHandlerRequest(t, s, testCase.URL, testCase.Headers, testCase.Data, testCase.MultipartFormFields, testCase.Status, &testCase.Response, testCase.PrepRequest) // pop our message off and test against it contactName := mb.GetLastContactName() @@ -433,14 +451,14 @@ func RunChannelTestCases(t *testing.T, channels []courier.Channel, handler couri t.Run("Queue Error", func(t *testing.T) { mb.SetErrorOnQueue(true) defer mb.SetErrorOnQueue(false) - testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, 400, Sp("unable to queue message"), validCase.PrepRequest) + testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, validCase.MultipartFormFields, 400, Sp("unable to queue message"), validCase.PrepRequest) }) } if !validCase.NoInvalidChannelCheck { t.Run("Receive With Invalid Channel", func(t *testing.T) { mb.ClearChannels() - testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, 400, Sp("channel not found"), validCase.PrepRequest) + testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, validCase.MultipartFormFields, 400, Sp("channel not found"), validCase.PrepRequest) }) } } @@ -461,7 +479,7 @@ func RunChannelBenchmarks(b *testing.B, channels []courier.Channel, handler cour b.Run(testCase.Label, func(b *testing.B) { for i := 0; i < b.N; i++ { - testHandlerRequest(b, s, testCase.URL, testCase.Headers, testCase.Data, testCase.Status, nil, testCase.PrepRequest) + testHandlerRequest(b, s, testCase.URL, testCase.Headers, testCase.Data, testCase.MultipartFormFields, testCase.Status, nil, testCase.PrepRequest) } }) } diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index 7ab505602..e82ceeaa3 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -5,12 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/buger/jsonparser" - "github.com/go-errors/errors" - "github.com/nyaruka/courier" - "github.com/nyaruka/courier/handlers" - "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/urns" "io" "io/ioutil" "mime/multipart" @@ -19,6 +13,13 @@ import ( "strconv" "strings" "time" + + "github.com/buger/jsonparser" + "github.com/go-errors/errors" + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/urns" ) var ( @@ -270,7 +271,7 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w // DescribeURN handles VK contact details func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - req, err := http.NewRequest(http.MethodPost, apiBaseURL + actionGetUser, nil) + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionGetUser, nil) if err != nil { return nil, err @@ -383,7 +384,7 @@ func takeFirstAttachmentUrl(payload moNewMessagePayload) string { func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - req, err := http.NewRequest(http.MethodPost, apiBaseURL + actionSendMessage, nil) + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) if err != nil { return status, errors.New("Cannot create send message request") @@ -460,7 +461,7 @@ func handleMediaUploadAndGetAttachment(channel courier.Channel, mediaType, media // initialize server URL to upload photos if URLPhotoUploadServer == "" { - if serverURL, err := getUploadServerURL(channel, apiBaseURL + actionGetPhotoUploadServer); err == nil { + if serverURL, err := getUploadServerURL(channel, apiBaseURL+actionGetPhotoUploadServer); err == nil { URLPhotoUploadServer = serverURL } } @@ -480,7 +481,7 @@ func handleMediaUploadAndGetAttachment(channel courier.Channel, mediaType, media return "", err } serverId := strconv.FormatInt(payload.ServerId, 10) - info, err := saveUploadedMediaInfo(channel, apiBaseURL + actionSaveUploadedPhotoInfo, serverId, payload.Hash, uploadKey, payload.Photo) + info, err := saveUploadedMediaInfo(channel, apiBaseURL+actionSaveUploadedPhotoInfo, serverId, payload.Hash, uploadKey, payload.Photo) if err != nil { return "", err From 058005d476faa96ece4f49dae2afe9b0fce8363e Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Tue, 20 Oct 2020 08:13:44 -0700 Subject: [PATCH 19/24] Update CHANGELOG.md for v5.7.10 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1234f539e..0326dbf66 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v5.7.10 +---------- + * Support receiving Multipart form data requests for EX channels + v5.7.9 ---------- * Update to latest gocommon 1.5.3 and golang 1.15 From ee5aec0005a0b14f1b09222bf7e1190e251c472b Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Thu, 22 Oct 2020 18:57:44 -0300 Subject: [PATCH 20/24] Use rcache module for cache WhatsApp medias --- go.mod | 2 +- go.sum | 5 +++ handlers/whatsapp/whatsapp.go | 76 ++++++++++------------------------- 3 files changed, 27 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index f490817e6..27a9afbc9 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.5.3 + github.com/nyaruka/gocommon v1.6.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 7c4941daf..5cb8862d4 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,9 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= +github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -65,6 +68,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.5.3 h1:oWk3s5Ykho2uQMypR0LtW2vuDrcK1rIdvEW1X1VW4mw= github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= +github.com/nyaruka/gocommon v1.6.0 h1:vlZdWvDbBu1iOKDMS4eF4QFXOHy1oGRr8yInbm84MWU= +github.com/nyaruka/gocommon v1.6.0/go.mod h1:r5UqoAdoP9VLb/wmtF1O0v73PQc79tZaVjbXlO16PUA= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 4aa5f4ad5..0f96766c2 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -5,7 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/garyburd/redigo/redis" "net/http" "net/url" "strconv" @@ -16,6 +15,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/rcache" "github.com/nyaruka/gocommon/urns" "github.com/pkg/errors" ) @@ -29,8 +29,8 @@ const ( channelTypeWa = "WA" channelTypeD3 = "D3" - mediaCacheKeyPattern = "whatsapp_media:%s" - failureMediaCacheKeyPattern = "whatsapp_failed_media:%s" + mediaCacheKeyPattern = "whatsapp_media_%s" + failureMediaCacheKeyPattern = "whatsapp_failed_media_%s" ) var ( @@ -618,19 +618,24 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri defer rc.Close() cacheKey := fmt.Sprintf(mediaCacheKeyPattern, msg.Channel().UUID().String()) - expiration, err := getMediaExpirationFromCache(rc, cacheKey, mediaURL) - if err == nil { - return expiration.MediaID, logs, nil - } else if err != redis.ErrNil { + mediaID, err := rcache.Get(rc, cacheKey, mediaURL) + if err != nil { elapsed := time.Now().Sub(start) - log := courier.NewChannelLogFromError("error reading the media id from redis", msg.Channel(), msg.ID(), elapsed, err) + log := courier.NewChannelLogFromError("error reading media id from redis", msg.Channel(), msg.ID(), elapsed, err) logs = append(logs, log) return "", logs, err + } else if mediaID != "" { + return mediaID, logs, nil } // check in failure cache failureCacheKey := fmt.Sprintf(failureMediaCacheKeyPattern, msg.Channel().UUID().String()) - if expiration, _ := getMediaExpirationFromCache(rc, failureCacheKey, mediaURL); expiration != nil { + if failed, err := rcache.Get(rc, failureCacheKey, mediaURL); err != nil { + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error reading failed media from redis", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } else if failed == "true" { return "", logs, errors.New("ignoring media that previously failed to upload") } @@ -642,7 +647,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri res, err := utils.MakeHTTPRequest(req) if err != nil { elapsed := time.Now().Sub(start) - log := courier.NewChannelLogFromError("error downloading the media", msg.Channel(), msg.ID(), elapsed, err) + log := courier.NewChannelLogFromError("error downloading media", msg.Channel(), msg.ID(), elapsed, err) logs = append(logs, log) return "", logs, err } @@ -658,76 +663,37 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri res, err = utils.MakeHTTPRequest(req) if err != nil { // put in failure cache - setMediaExpirationInCache(rc, failureCacheKey, mediaURL, "", h.Server().Config().WhatsAppFailedMediaExpiration) + rcache.Set(rc, failureCacheKey, mediaURL, "true") if res != nil { err = errors.Wrap(err, string(res.Body)) } elapsed := time.Now().Sub(start) - log := courier.NewChannelLogFromError("error uploading the media to WhatsApp", msg.Channel(), msg.ID(), elapsed, err) + log := courier.NewChannelLogFromError("error uploading media to WhatsApp", msg.Channel(), msg.ID(), elapsed, err) logs = append(logs, log) return "", logs, err } // take uploaded media id - mediaID, err := jsonparser.GetString(res.Body, "media", "[0]", "id") + mediaID, err = jsonparser.GetString(res.Body, "media", "[0]", "id") if err != nil { elapsed := time.Now().Sub(start) - log := courier.NewChannelLogFromError("error reading the media id from response", msg.Channel(), msg.ID(), elapsed, err) + log := courier.NewChannelLogFromError("error reading media id from response", msg.Channel(), msg.ID(), elapsed, err) logs = append(logs, log) return "", logs, err } // put in cache - err = setMediaExpirationInCache(rc, cacheKey, mediaURL, mediaID, h.Server().Config().WhatsAppMediaExpiration) + err = rcache.Set(rc, cacheKey, mediaURL, mediaID) if err != nil { elapsed := time.Now().Sub(start) - log := courier.NewChannelLogFromError("error setting the media id to redis", msg.Channel(), msg.ID(), elapsed, err) + log := courier.NewChannelLogFromError("error setting media id to redis", msg.Channel(), msg.ID(), elapsed, err) logs = append(logs, log) return "", logs, err } return mediaID, logs, nil } -type mediaExpiration struct { - ExpiresOn time.Time `json:"expires_on"` - MediaID string `json:"media_id,omitempty"` -} - -func getMediaExpirationFromCache(rc redis.Conn, cacheKey, mediaURL string) (*mediaExpiration, error) { - expirationJSON, err := redis.String(rc.Do("HGET", cacheKey, mediaURL)) - if err != nil { - return nil, err - } - - expiration := &mediaExpiration{} - err = json.Unmarshal([]byte(expirationJSON), expiration) - if err != nil { - return nil, err - } - - if expiration.ExpiresOn.Before(time.Now()) { - rc.Do("HDEL", cacheKey, mediaURL) - return nil, redis.ErrNil - } - return expiration, nil -} - -func setMediaExpirationInCache(rc redis.Conn, cacheKey, mediaURL, mediaID string, expiration int) error { - expiresOn := time.Now().Add(time.Second * time.Duration(expiration)) - expirationJSON, err := json.Marshal(mediaExpiration{ExpiresOn: expiresOn, MediaID: mediaID}) - if err != nil { - return err - } - - _, err = rc.Do("HSET", cacheKey, mediaURL, string(expirationJSON)) - if err != nil { - return err - } - - return nil -} - func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) From 9242c22213864d88599346f09bb59bcc31856037 Mon Sep 17 00:00:00 2001 From: Allan Lima Date: Fri, 23 Oct 2020 09:14:35 -0300 Subject: [PATCH 21/24] Switch from github.com/garyburd/redigo/redis to github.com/gomodule/redigo/redis --- backend.go | 2 +- backends/rapidpro/backend.go | 2 +- backends/rapidpro/backend_test.go | 2 +- backends/rapidpro/msg.go | 2 +- backends/rapidpro/task.go | 2 +- celery/celery.go | 2 +- celery/celery_test.go | 2 +- cmd/fuzzer/main.go | 2 +- go.mod | 2 +- go.sum | 5 ----- handlers/hormuud/hormuud.go | 2 +- handlers/jiochat/jiochat.go | 2 +- handlers/mtarget/mtarget.go | 2 +- handlers/wechat/wechat.go | 2 +- queue/queue.go | 2 +- queue/queue_test.go | 2 +- test.go | 2 +- 17 files changed, 16 insertions(+), 21 deletions(-) diff --git a/backend.go b/backend.go index 56d119e62..2e0626f6b 100644 --- a/backend.go +++ b/backend.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/gocommon/urns" ) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 668bd4761..861d1a1f7 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -13,7 +13,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" "github.com/nyaruka/courier/batch" diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 3d017689e..529bab1bc 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -21,7 +21,7 @@ import ( "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/null" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" ) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 912abcf34..e5550d105 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -18,7 +18,7 @@ import ( "mime" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/lib/pq" "github.com/nyaruka/courier" "github.com/nyaruka/courier/queue" diff --git a/backends/rapidpro/task.go b/backends/rapidpro/task.go index 232a7fa73..d96d60e7d 100644 --- a/backends/rapidpro/task.go +++ b/backends/rapidpro/task.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" ) diff --git a/celery/celery.go b/celery/celery.go index a80a536d3..90851aa65 100644 --- a/celery/celery.go +++ b/celery/celery.go @@ -6,7 +6,7 @@ import ( "github.com/nyaruka/gocommon/uuids" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) // allows queuing a task to celery (with a redis backend) diff --git a/celery/celery_test.go b/celery/celery_test.go index e2ba824ee..b8baf60f7 100644 --- a/celery/celery_test.go +++ b/celery/celery_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) func getPool() *redis.Pool { diff --git a/cmd/fuzzer/main.go b/cmd/fuzzer/main.go index fa4b8819a..21e9376f4 100644 --- a/cmd/fuzzer/main.go +++ b/cmd/fuzzer/main.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" ) diff --git a/go.mod b/go.mod index 27a9afbc9..6f69bb39a 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require ( github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/dghubble/oauth1 v0.4.0 github.com/evalphobia/logrus_sentry v0.4.6 - github.com/garyburd/redigo v1.5.0 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 github.com/go-playground/locales v0.11.2 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible + github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 github.com/kr/pretty v0.1.0 // indirect diff --git a/go.sum b/go.sum index 5cb8862d4..91fa97fe9 100644 --- a/go.sum +++ b/go.sum @@ -18,8 +18,6 @@ github.com/evalphobia/logrus_sentry v0.4.6 h1:825MLGu+SW5H8hMXGeBI7TwX7vgJLd9hz0 github.com/evalphobia/logrus_sentry v0.4.6/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/garyburd/redigo v1.5.0 h1:OcZhiwwjKtBe7TO4TlXpj/1E3I2RVg1uLxwMT4VFF5w= -github.com/garyburd/redigo v1.5.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= @@ -36,7 +34,6 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= @@ -66,8 +63,6 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.5.3 h1:oWk3s5Ykho2uQMypR0LtW2vuDrcK1rIdvEW1X1VW4mw= -github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= github.com/nyaruka/gocommon v1.6.0 h1:vlZdWvDbBu1iOKDMS4eF4QFXOHy1oGRr8yInbm84MWU= github.com/nyaruka/gocommon v1.6.0/go.mod h1:r5UqoAdoP9VLb/wmtF1O0v73PQc79tZaVjbXlO16PUA= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= diff --git a/handlers/hormuud/hormuud.go b/handlers/hormuud/hormuud.go index 2ed84486e..ba3921858 100644 --- a/handlers/hormuud/hormuud.go +++ b/handlers/hormuud/hormuud.go @@ -11,7 +11,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/jiochat/jiochat.go b/handlers/jiochat/jiochat.go index e1f718897..6e986d9c0 100644 --- a/handlers/jiochat/jiochat.go +++ b/handlers/jiochat/jiochat.go @@ -14,7 +14,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/mtarget/mtarget.go b/handlers/mtarget/mtarget.go index fe391290f..7ad6ecce5 100644 --- a/handlers/mtarget/mtarget.go +++ b/handlers/mtarget/mtarget.go @@ -10,7 +10,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/wechat/wechat.go b/handlers/wechat/wechat.go index 582204717..661a6401d 100644 --- a/handlers/wechat/wechat.go +++ b/handlers/wechat/wechat.go @@ -14,7 +14,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/queue/queue.go b/queue/queue.go index 05f08bac1..7aa4bebaa 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/sirupsen/logrus" ) diff --git a/queue/queue_test.go b/queue/queue_test.go index eb5d5db52..157793a0e 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" ) diff --git a/test.go b/test.go index c35054aff..2298d0836 100644 --- a/test.go +++ b/test.go @@ -13,7 +13,7 @@ import ( "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" _ "github.com/lib/pq" // postgres driver ) From 483765b00117e81d60d817b83ac834b70f9b1704 Mon Sep 17 00:00:00 2001 From: Nic Pottier Date: Fri, 23 Oct 2020 08:00:29 -0700 Subject: [PATCH 22/24] Update CHANGELOG.md for v5.7.11 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0326dbf66..bcfbcfa35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v5.7.11 +---------- + * Cache media ids for WhatsApp attachments + v5.7.10 ---------- * Support receiving Multipart form data requests for EX channels From 24743e90be1db298c0d768d58e2e77afb2c0db26 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 23 Oct 2020 15:38:27 -0500 Subject: [PATCH 23/24] Remove no longer needed codecov token from CI workflow --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5144a4880..6b92884ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: if: success() uses: codecov/codecov-action@v1 with: - token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true release: name: Release From ada51a6e64b4bae54479f9701261ac9ab10116c3 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 29 Oct 2020 14:46:40 -0500 Subject: [PATCH 24/24] Update to gocommon v1.6.1 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6f69bb39a..0f2d7a5bd 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.6.0 + github.com/nyaruka/gocommon v1.6.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index 91fa97fe9..6e0447af8 100644 --- a/go.sum +++ b/go.sum @@ -63,8 +63,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.6.0 h1:vlZdWvDbBu1iOKDMS4eF4QFXOHy1oGRr8yInbm84MWU= -github.com/nyaruka/gocommon v1.6.0/go.mod h1:r5UqoAdoP9VLb/wmtF1O0v73PQc79tZaVjbXlO16PUA= +github.com/nyaruka/gocommon v1.6.1 h1:pUScZMXtIR8CSePZL2iwM8rt2d60gCezTZ8bgymYFVY= +github.com/nyaruka/gocommon v1.6.1/go.mod h1:r5UqoAdoP9VLb/wmtF1O0v73PQc79tZaVjbXlO16PUA= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE=