diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d69dcf20..6b92884ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: '1.14.x' + go-version: '1.15.x' jobs: test: name: Test @@ -38,7 +38,7 @@ jobs: if: success() uses: codecov/codecov-action@v1 with: - token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true release: name: Release diff --git a/CHANGELOG.md b/CHANGELOG.md index 8da2b69a8..bcfbcfa35 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +v5.7.11 +---------- + * Cache media ids for WhatsApp attachments + +v5.7.10 +---------- + * Support receiving Multipart form data requests for EX channels + +v5.7.9 +---------- + * Update to latest gocommon 1.5.3 and golang 1.15 + * Add session status from mailroom to MT message sent to external channel API call + * Remove incoming message prefix for Play Mobile free accounts + v5.7.8 ---------- * deal with empty message in FreshChat incoming requests diff --git a/backend.go b/backend.go index 56d119e62..2e0626f6b 100644 --- a/backend.go +++ b/backend.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/gocommon/urns" ) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 668bd4761..861d1a1f7 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -13,7 +13,7 @@ import ( "time" "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" "github.com/nyaruka/courier/batch" diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 3d017689e..529bab1bc 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -21,7 +21,7 @@ import ( "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/null" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/sirupsen/logrus" "github.com/stretchr/testify/suite" ) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 912abcf34..e5550d105 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -18,7 +18,7 @@ import ( "mime" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/lib/pq" "github.com/nyaruka/courier" "github.com/nyaruka/courier/queue" diff --git a/backends/rapidpro/task.go b/backends/rapidpro/task.go index 232a7fa73..d96d60e7d 100644 --- a/backends/rapidpro/task.go +++ b/backends/rapidpro/task.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" ) diff --git a/celery/celery.go b/celery/celery.go index a80a536d3..90851aa65 100644 --- a/celery/celery.go +++ b/celery/celery.go @@ -6,7 +6,7 @@ import ( "github.com/nyaruka/gocommon/uuids" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) // allows queuing a task to celery (with a redis backend) diff --git a/celery/celery_test.go b/celery/celery_test.go index e2ba824ee..b8baf60f7 100644 --- a/celery/celery_test.go +++ b/celery/celery_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" ) func getPool() *redis.Pool { diff --git a/cmd/courier/main.go b/cmd/courier/main.go index 7f7afb708..b654771d4 100644 --- a/cmd/courier/main.go +++ b/cmd/courier/main.go @@ -21,6 +21,7 @@ import ( _ "github.com/nyaruka/courier/handlers/clickmobile" _ "github.com/nyaruka/courier/handlers/clicksend" _ "github.com/nyaruka/courier/handlers/dart" + _ "github.com/nyaruka/courier/handlers/discord" _ "github.com/nyaruka/courier/handlers/dmark" _ "github.com/nyaruka/courier/handlers/external" _ "github.com/nyaruka/courier/handlers/facebook" diff --git a/cmd/fuzzer/main.go b/cmd/fuzzer/main.go index fa4b8819a..21e9376f4 100644 --- a/cmd/fuzzer/main.go +++ b/cmd/fuzzer/main.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" ) diff --git a/go.mod b/go.mod index f890cec8c..0f2d7a5bd 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require ( github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/dghubble/oauth1 v0.4.0 github.com/evalphobia/logrus_sentry v0.4.6 - github.com/garyburd/redigo v1.5.0 github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 github.com/go-playground/locales v0.11.2 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible + github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 github.com/kr/pretty v0.1.0 // indirect @@ -34,4 +34,4 @@ require ( gopkg.in/h2non/filetype.v1 v1.0.5 ) -go 1.13 +go 1.15 diff --git a/go.sum b/go.sum index 28794ef8f..6e0447af8 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,7 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/aws/aws-sdk-go v1.34.17 h1:9OzUgRrLmYm2mbfFx4v+2nBEg+Cvape1cvn9C3RNWTE= -github.com/aws/aws-sdk-go v1.34.17/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.31 h1:408wh5EHKzxyby8JpYfnn1w3fsF26AIU0o1kbJoRy7E= github.com/aws/aws-sdk-go v1.34.31/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= @@ -19,8 +18,6 @@ github.com/evalphobia/logrus_sentry v0.4.6 h1:825MLGu+SW5H8hMXGeBI7TwX7vgJLd9hz0 github.com/evalphobia/logrus_sentry v0.4.6/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/garyburd/redigo v1.5.0 h1:OcZhiwwjKtBe7TO4TlXpj/1E3I2RVg1uLxwMT4VFF5w= -github.com/garyburd/redigo v1.5.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= @@ -37,12 +34,13 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= -github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= -github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= @@ -65,16 +63,13 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.5.1 h1:2R6uo6EVSTHOerupAmVm6h5fyufO189dlv/5gwHj3lM= -github.com/nyaruka/gocommon v1.5.1/go.mod h1:6XoaOsVk6z+294hM6pZxX3fDgT2IyLV8hFU4FoQz9Aw= github.com/nyaruka/gocommon v1.6.1 h1:pUScZMXtIR8CSePZL2iwM8rt2d60gCezTZ8bgymYFVY= github.com/nyaruka/gocommon v1.6.1/go.mod h1:r5UqoAdoP9VLb/wmtF1O0v73PQc79tZaVjbXlO16PUA= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= -github.com/nyaruka/phonenumbers v1.0.57 h1:V4FNPs061PSUOEzQaLH0+pfzEdqoiMH/QJWryx/0hfs= -github.com/nyaruka/phonenumbers v1.0.57/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/phonenumbers v1.0.58 h1:IAlGDA4wuGQXe2lwOQvkZfBvA1DlAik+MX5k9k5C2IU= github.com/nyaruka/phonenumbers v1.0.58/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -91,19 +86,20 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321 h1:lleNcKRbcaC8MqgLwghIkzZ2JBQAb7QQ9MiwRt1BisA= golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -121,5 +117,7 @@ gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/discord/discord.go b/handlers/discord/discord.go new file mode 100644 index 000000000..999f4ce6d --- /dev/null +++ b/handlers/discord/discord.go @@ -0,0 +1,224 @@ +package discord + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "strings" + + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/urns" + "github.com/pkg/errors" +) + +const ( + jsonMimeTypeType = "application/json" + urlEncodedMimeType = "application/x-www-form-urlencoded" +) + +func init() { + courier.RegisterHandler(newHandler()) +} + +type handler struct { + handlers.BaseHandler +} + +func newHandler() courier.ChannelHandler { + return &handler{handlers.NewBaseHandler(courier.ChannelType("DS"), "Discord")} +} + +// Initialize is called by the engine once everything is loaded +func (h *handler) Initialize(s courier.Server) error { + h.SetServer(s) + s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveMessage) + + sentHandler := h.buildStatusHandler("sent") + s.AddHandlerRoute(h, http.MethodPost, "sent", sentHandler) + + deliveredHandler := h.buildStatusHandler("delivered") + s.AddHandlerRoute(h, http.MethodPost, "delivered", deliveredHandler) + + failedHandler := h.buildStatusHandler("failed") + s.AddHandlerRoute(h, http.MethodPost, "failed", failedHandler) + + return nil +} + +type stopContactForm struct { + From string `validate:"required" name:"from"` +} + +// utility function to grab the form value for either the passed in name (if non-empty) or the first set +// value from defaultNames +func getFormField(form url.Values, name string) string { + if name != "" { + values, found := form[name] + if found { + return values[0] + } + } + return "" +} + +// receiveMessage is our HTTP handler function for incoming messages +func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + var err error + + var from, text string + + // parse our form + err = r.ParseForm() + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, errors.Wrapf(err, "invalid request")) + } + + from = getFormField(r.Form, "from") + text = getFormField(r.Form, "text") + + // must have from field + if from == "" { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("must have one of 'sender' or 'from' set")) + } + if text == "" { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("must have 'text' set")) + } + + // if we have a date, parse it + date := time.Now() + + // create our URN + urn := urns.NilURN + urn, err = urns.NewURNFromParts(urns.DiscordScheme, from, "", "") + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // build our msg + msg := h.Backend().NewIncomingMsg(channel, urn, text).WithReceivedOn(date) + + for _, attachment := range r.Form["attachments"] { + msg.WithAttachment(attachment) + } + + // and finally write our message + return handlers.WriteMsgsAndResponse(ctx, h, []courier.Msg{msg}, w, r) +} + +// WriteMsgSuccessResponse writes our response in TWIML format +func (h *handler) WriteMsgSuccessResponse(ctx context.Context, w http.ResponseWriter, r *http.Request, msgs []courier.Msg) error { + return courier.WriteMsgSuccess(ctx, w, r, msgs) +} + +// buildStatusHandler deals with building a handler that takes what status is received in the URL +func (h *handler) buildStatusHandler(status string) courier.ChannelHandleFunc { + return func(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + return h.receiveStatus(ctx, status, channel, w, r) + } +} + +type statusForm struct { + ID int64 `name:"id" validate:"required"` +} + +var statusMappings = map[string]courier.MsgStatusValue{ + "failed": courier.MsgFailed, + "sent": courier.MsgSent, + "delivered": courier.MsgDelivered, +} + +// receiveStatus is our HTTP handler function for status updates +func (h *handler) receiveStatus(ctx context.Context, statusString string, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + form := &statusForm{} + err := handlers.DecodeAndValidateForm(form, r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // get our status + msgStatus, found := statusMappings[strings.ToLower(statusString)] + if !found { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown status '%s', must be one failed, sent or delivered", statusString)) + } + + // write our status + status := h.Backend().NewMsgStatusForID(channel, courier.NewMsgID(form.ID), msgStatus) + return handlers.WriteMsgStatusAndResponse(ctx, h, channel, status, w, r) +} + +// SendMsg sends the passed in message, returning any error +func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { + sendURL := msg.Channel().StringConfigForKey(courier.ConfigSendURL, "") + if sendURL == "" { + return nil, fmt.Errorf("no send url set for DS channel") + } + + // figure out what encoding to tell kannel to send as + sendMethod := http.MethodPost + // sendBody := msg.Channel().StringConfigForKey(courier.ConfigSendBody, "") + contentTypeHeader := jsonMimeTypeType + + status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) + attachmentURLs := []string{} + for _, attachment := range msg.Attachments() { + _, attachmentURL := handlers.SplitAttachment(attachment) + attachmentURLs = append(attachmentURLs, attachmentURL) + } + // build our request + type OutputMessage struct { + ID string `json:"id"` + Text string `json:"text"` + To string `json:"to"` + Channel string `json:"channel"` + Attachments []string `json:"attachments"` + QuickReplies []string `json:"quick_replies"` + } + + ourMessage := OutputMessage{ + ID: msg.ID().String(), + Text: msg.Text(), + To: msg.URN().Path(), + Channel: msg.Channel().UUID().String(), + Attachments: attachmentURLs, + QuickReplies: msg.QuickReplies(), + } + + var body io.Reader + marshalled, err := json.Marshal(ourMessage) + if err != nil { + return nil, err + } + body = bytes.NewReader(marshalled) + + req, err := http.NewRequest(sendMethod, sendURL, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentTypeHeader) + + authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + + rr, err := utils.MakeHTTPRequest(req) + + // record our status and log + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + status.AddLog(log) + if err != nil { + return status, nil + } + // If we don't have an error, set the message as wired and move on + status.SetStatus(courier.MsgWired) + + return status, nil +} diff --git a/handlers/discord/discord_test.go b/handlers/discord/discord_test.go new file mode 100644 index 000000000..49bc6b3d9 --- /dev/null +++ b/handlers/discord/discord_test.go @@ -0,0 +1,50 @@ +package discord + +import ( + "net/http/httptest" + "testing" + + "github.com/nyaruka/courier" + . "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" +) + +func TestHandler(t *testing.T) { + RunChannelTestCases(t, testChannels, newHandler(), testCases) +} + +func BenchmarkHandler(b *testing.B) { + RunChannelBenchmarks(b, testChannels, newHandler(), testCases) +} + +var testChannels = []courier.Channel{ + courier.NewMockChannel("bac782c2-7aeb-4389-92f5-97887744f573", "DS", "discord", "US", map[string]interface{}{}), +} + +var testCases = []ChannelHandleTestCase{ + {Label: "Recieve Message", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802&text=hello`, Status: 200, Text: Sp("hello"), URN: Sp("discord:694634743521607802")}, + {Label: "Recieve Message with attachment", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802&text=hello&attachments=https://test.test/foo.png`, Status: 200, Text: Sp("hello"), URN: Sp("discord:694634743521607802"), Attachments: []string{"https://test.test/foo.png"}}, + {Label: "Invalid ID", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=somebody&text=hello`, Status: 400, Response: "Error"}, + {Label: "Garbage Body", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `sdfaskdfajsdkfajsdfaksdf`, Status: 400, Response: "Error"}, + {Label: "Missing Text", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/receive", Data: `from=694634743521607802`, Status: 400, Response: "Error"}, + {Label: "Message Sent Handler", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/sent/", Data: `id=12345`, Status: 200, Response: `"status":"S"`}, + {Label: "Message Sent Handler Garbage", URL: "/c/ds/bac782c2-7aeb-4389-92f5-97887744f573/sent/", Data: `nothing`, Status: 400}, +} + +var sendTestCases = []ChannelSendTestCase{ + {Label: "Simple Send", Text: "Hello World", URN: "discord:694634743521607802", Path: "/discord/rp/send", ResponseStatus: 200, RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":[],"quick_replies":null}`, SendPrep: setSendURL}, + {Label: "Simple Send", Text: "Hello World", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "discord:694634743521607802", Path: "/discord/rp/send", RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":["https://foo.bar/image.jpg"],"quick_replies":null}`, ResponseStatus: 200, SendPrep: setSendURL}, + {Label: "Simple Send with attachements and Quick Replies", Text: "Hello World", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, QuickReplies: []string{"hello", "world"}, URN: "discord:694634743521607802", Path: "/discord/rp/send", RequestBody: `{"id":"10","text":"Hello World","to":"694634743521607802","channel":"bac782c2-7aeb-4389-92f5-97887744f573","attachments":["https://foo.bar/image.jpg"],"quick_replies":["hello","world"]}`, ResponseStatus: 200, SendPrep: setSendURL}, +} + +// setSendURL takes care of setting the send_url to our test server host +func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { + // this is actually a path, which we'll combine with the test server URL + sendURL := c.StringConfigForKey("send_path", "/discord/rp/send") + sendURL, _ = utils.AddURLPath(s.URL, sendURL) + c.(*courier.MockChannel).SetConfig(courier.ConfigSendURL, sendURL) +} +func TestSending(t *testing.T) { + + RunChannelSendTestCases(t, testChannels[0], newHandler(), sendTestCases, nil) +} diff --git a/handlers/external/external.go b/handlers/external/external.go index 6e886538a..a9c000226 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -173,7 +173,13 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w text = textNode.InnerText() } else { // parse our form - err := r.ParseForm() + contentType := r.Header.Get("Content-Type") + var err error + if strings.Contains(contentType, "multipart/form-data") { + err = r.ParseMultipartForm(10000000) + } else { + err = r.ParseForm() + } if err != nil { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, errors.Wrapf(err, "invalid request")) } diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index 315a3a8f1..1c4b84685 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -46,6 +46,9 @@ var handleTestCases = []ChannelHandleTestCase{ Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, {Label: "Receive Valid Post", URL: receiveNoParams, Data: "sender=%2B2349067554729&text=Join", Status: 200, Response: "Accepted", Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, + + {Label: "Receive Valid Post multipart form", URL: receiveNoParams, MultipartFormFields: map[string]string{"sender": "2349067554729", "text": "Join"}, Status: 200, Response: "Accepted", + Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, {Label: "Receive Valid From", URL: receiveValidMessageFrom, Data: "empty", Status: 200, Response: "Accepted", Text: Sp("Join"), URN: Sp("tel:+2349067554729")}, {Label: "Receive Country Parse", URL: receiveValidNoPlus, Data: "empty", Status: 200, Response: "Accepted", diff --git a/handlers/hormuud/hormuud.go b/handlers/hormuud/hormuud.go index 2ed84486e..ba3921858 100644 --- a/handlers/hormuud/hormuud.go +++ b/handlers/hormuud/hormuud.go @@ -11,7 +11,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/jiochat/jiochat.go b/handlers/jiochat/jiochat.go index e1f718897..6e986d9c0 100644 --- a/handlers/jiochat/jiochat.go +++ b/handlers/jiochat/jiochat.go @@ -14,7 +14,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/mtarget/mtarget.go b/handlers/mtarget/mtarget.go index fe391290f..7ad6ecce5 100644 --- a/handlers/mtarget/mtarget.go +++ b/handlers/mtarget/mtarget.go @@ -10,7 +10,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/test.go b/handlers/test.go index a99a8ba83..6f139d194 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "io/ioutil" + "mime/multipart" "net/http" "net/http/httptest" "strings" @@ -33,6 +34,8 @@ type ChannelHandleTestCase struct { Response string Headers map[string]string + MultipartFormFields map[string]string + Name *string Text *string URN *string @@ -127,7 +130,7 @@ func ensureTestServerUp(host string) { } // utility method to make a request to a handler URL -func testHandlerRequest(tb testing.TB, s courier.Server, path string, headers map[string]string, data string, expectedStatus int, expectedBody *string, requestPrepFunc RequestPrepFunc) string { +func testHandlerRequest(tb testing.TB, s courier.Server, path string, headers map[string]string, data string, multipartFormFields map[string]string, expectedStatus int, expectedBody *string, requestPrepFunc RequestPrepFunc) string { var req *http.Request var err error url := fmt.Sprintf("https://%s%s", s.Config().Domain, path) @@ -144,6 +147,21 @@ func testHandlerRequest(tb testing.TB, s courier.Server, path string, headers ma contentType = "application/xml" } req.Header.Set("Content-Type", contentType) + } else if multipartFormFields != nil { + var body bytes.Buffer + bodyMultipartWriter := multipart.NewWriter(&body) + for k, v := range multipartFormFields { + fieldWriter, err := bodyMultipartWriter.CreateFormField(k) + require.Nil(tb, err) + _, err = fieldWriter.Write([]byte(v)) + require.Nil(tb, err) + } + contentType := fmt.Sprintf("multipart/form-data;boundary=%v", bodyMultipartWriter.Boundary()) + bodyMultipartWriter.Close() + + req, err = http.NewRequest(http.MethodPost, url, bytes.NewReader(body.Bytes())) + require.Nil(tb, err) + req.Header.Set("Content-Type", contentType) } else { req, err = http.NewRequest(http.MethodGet, url, nil) } @@ -350,7 +368,7 @@ func RunChannelTestCases(t *testing.T, channels []courier.Channel, handler couri mb.ClearQueueMsgs() mb.ClearSeenExternalIDs() - testHandlerRequest(t, s, testCase.URL, testCase.Headers, testCase.Data, testCase.Status, &testCase.Response, testCase.PrepRequest) + testHandlerRequest(t, s, testCase.URL, testCase.Headers, testCase.Data, testCase.MultipartFormFields, testCase.Status, &testCase.Response, testCase.PrepRequest) // pop our message off and test against it contactName := mb.GetLastContactName() @@ -433,14 +451,14 @@ func RunChannelTestCases(t *testing.T, channels []courier.Channel, handler couri t.Run("Queue Error", func(t *testing.T) { mb.SetErrorOnQueue(true) defer mb.SetErrorOnQueue(false) - testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, 400, Sp("unable to queue message"), validCase.PrepRequest) + testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, validCase.MultipartFormFields, 400, Sp("unable to queue message"), validCase.PrepRequest) }) } if !validCase.NoInvalidChannelCheck { t.Run("Receive With Invalid Channel", func(t *testing.T) { mb.ClearChannels() - testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, 400, Sp("channel not found"), validCase.PrepRequest) + testHandlerRequest(t, s, validCase.URL, validCase.Headers, validCase.Data, validCase.MultipartFormFields, 400, Sp("channel not found"), validCase.PrepRequest) }) } } @@ -461,7 +479,7 @@ func RunChannelBenchmarks(b *testing.B, channels []courier.Channel, handler cour b.Run(testCase.Label, func(b *testing.B) { for i := 0; i < b.N; i++ { - testHandlerRequest(b, s, testCase.URL, testCase.Headers, testCase.Data, testCase.Status, nil, testCase.PrepRequest) + testHandlerRequest(b, s, testCase.URL, testCase.Headers, testCase.Data, testCase.MultipartFormFields, testCase.Status, nil, testCase.PrepRequest) } }) } diff --git a/handlers/viber/viber.go b/handlers/viber/viber.go index 6b52442ae..daf2d6a2c 100644 --- a/handlers/viber/viber.go +++ b/handlers/viber/viber.go @@ -171,7 +171,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return []courier.Event{channelEvent}, courier.WriteChannelEventSuccess(ctx, w, r, channelEvent) case "failed": - msgStatus := h.Backend().NewMsgStatusForExternalID(channel, string(payload.MessageToken), courier.MsgFailed) + msgStatus := h.Backend().NewMsgStatusForExternalID(channel, fmt.Sprintf("%d", payload.MessageToken), courier.MsgFailed) return handlers.WriteMsgStatusAndResponse(ctx, h, channel, msgStatus, w, r) case "delivered": @@ -248,7 +248,7 @@ func writeWelcomeMessageResponse(w http.ResponseWriter, channel courier.Channel, AuthToken: authToken, Text: msgText, Type: "text", - TrackingData: string(event.EventID()), + TrackingData: fmt.Sprintf("%d", event.EventID()), } responseBody := &bytes.Buffer{} diff --git a/handlers/viber/viber_test.go b/handlers/viber/viber_test.go index d712cb497..c725ad960 100644 --- a/handlers/viber/viber_test.go +++ b/handlers/viber/viber_test.go @@ -409,7 +409,6 @@ var ( } }` - receiveInvalidMessageType = `{ "event": "message", "timestamp": 1481142112807, @@ -487,7 +486,7 @@ var testWelcomeMessageCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: receiveURL, Data: validMsg, Status: 200, Response: "Accepted", Text: Sp("incoming msg"), URN: Sp("viber:xy5/5y6O81+/kbWHpLhBoA=="), ExternalID: Sp("4987381189870374000"), PrepRequest: addValidSignature}, - {Label: "Conversation Started", URL: receiveURL, Data: validConversationStarted, Status: 200, Response: `{"auth_token":"Token","text":"Welcome to VP, Please subscribe here for more.","type":"text","tracking_data":"\u0000"}`, PrepRequest: addValidSignature}, + {Label: "Conversation Started", URL: receiveURL, Data: validConversationStarted, Status: 200, Response: `{"auth_token":"Token","text":"Welcome to VP, Please subscribe here for more.","type":"text","tracking_data":"0"}`, PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index 7ab505602..e82ceeaa3 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -5,12 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/buger/jsonparser" - "github.com/go-errors/errors" - "github.com/nyaruka/courier" - "github.com/nyaruka/courier/handlers" - "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/urns" "io" "io/ioutil" "mime/multipart" @@ -19,6 +13,13 @@ import ( "strconv" "strings" "time" + + "github.com/buger/jsonparser" + "github.com/go-errors/errors" + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/urns" ) var ( @@ -270,7 +271,7 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w // DescribeURN handles VK contact details func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - req, err := http.NewRequest(http.MethodPost, apiBaseURL + actionGetUser, nil) + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionGetUser, nil) if err != nil { return nil, err @@ -383,7 +384,7 @@ func takeFirstAttachmentUrl(payload moNewMessagePayload) string { func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - req, err := http.NewRequest(http.MethodPost, apiBaseURL + actionSendMessage, nil) + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) if err != nil { return status, errors.New("Cannot create send message request") @@ -460,7 +461,7 @@ func handleMediaUploadAndGetAttachment(channel courier.Channel, mediaType, media // initialize server URL to upload photos if URLPhotoUploadServer == "" { - if serverURL, err := getUploadServerURL(channel, apiBaseURL + actionGetPhotoUploadServer); err == nil { + if serverURL, err := getUploadServerURL(channel, apiBaseURL+actionGetPhotoUploadServer); err == nil { URLPhotoUploadServer = serverURL } } @@ -480,7 +481,7 @@ func handleMediaUploadAndGetAttachment(channel courier.Channel, mediaType, media return "", err } serverId := strconv.FormatInt(payload.ServerId, 10) - info, err := saveUploadedMediaInfo(channel, apiBaseURL + actionSaveUploadedPhotoInfo, serverId, payload.Hash, uploadKey, payload.Photo) + info, err := saveUploadedMediaInfo(channel, apiBaseURL+actionSaveUploadedPhotoInfo, serverId, payload.Hash, uploadKey, payload.Photo) if err != nil { return "", err diff --git a/handlers/wechat/wechat.go b/handlers/wechat/wechat.go index 582204717..661a6401d 100644 --- a/handlers/wechat/wechat.go +++ b/handlers/wechat/wechat.go @@ -14,7 +14,7 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index b82de2a5b..0f96766c2 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -15,6 +15,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/rcache" "github.com/nyaruka/gocommon/urns" "github.com/pkg/errors" ) @@ -27,6 +28,9 @@ const ( channelTypeWa = "WA" channelTypeD3 = "D3" + + mediaCacheKeyPattern = "whatsapp_media_%s" + failureMediaCacheKeyPattern = "whatsapp_failed_media_%s" ) var ( @@ -276,8 +280,8 @@ func (h *handler) BuildDownloadMediaRequest(ctx context.Context, b courier.Backe // set the access token as the authorization header req, _ := http.NewRequest(http.MethodGet, attachmentURL, nil) - req.Header = buildAuthorizationHeader(req.Header, channel, token) req.Header.Set("User-Agent", utils.HTTPUserAgent) + setWhatsAppAuthHeader(&req.Header, channel) return req, nil } @@ -325,7 +329,8 @@ type mtTextPayload struct { } type mediaObject struct { - Link string `json:"link" validate:"required"` + ID string `json:"id,omitempty"` + Link string `json:"link,omitempty"` Caption string `json:"caption,omitempty"` } @@ -430,58 +435,61 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat if len(msg.Attachments()) > 0 { for attachmentCount, attachment := range msg.Attachments() { - mimeType, s3url := handlers.SplitAttachment(attachment) - + mimeType, mediaURL := handlers.SplitAttachment(attachment) + mediaID, mediaLogs, err := h.fetchMediaID(msg, mimeType, mediaURL) + if err == nil && mediaID != "" { + mediaURL = "" + } + mediaPayload := &mediaObject{ID: mediaID, Link: mediaURL} externalID := "" if strings.HasPrefix(mimeType, "audio") { payload := mtAudioPayload{ To: msg.URN().Path(), Type: "audio", } - payload.Audio = &mediaObject{Link: s3url} - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) - + payload.Audio = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else if strings.HasPrefix(mimeType, "application") { payload := mtDocumentPayload{ To: msg.URN().Path(), Type: "document", } - if attachmentCount == 0 { - payload.Document = &mediaObject{Link: s3url, Caption: msg.Text()} - } else { - payload.Document = &mediaObject{Link: s3url} + mediaPayload.Caption = msg.Text() } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) - + payload.Document = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else if strings.HasPrefix(mimeType, "image") { payload := mtImagePayload{ To: msg.URN().Path(), Type: "image", } if attachmentCount == 0 { - payload.Image = &mediaObject{Link: s3url, Caption: msg.Text()} - } else { - payload.Image = &mediaObject{Link: s3url} + mediaPayload.Caption = msg.Text() } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + payload.Image = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else if strings.HasPrefix(mimeType, "video") { payload := mtVideoPayload{ To: msg.URN().Path(), Type: "video", } if attachmentCount == 0 { - payload.Video = &mediaObject{Link: s3url, Caption: msg.Text()} - } else { - payload.Video = &mediaObject{Link: s3url} + mediaPayload.Caption = msg.Text() } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + payload.Video = mediaPayload + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else { duration := time.Since(start) err = fmt.Errorf("unknown attachment mime type: %s", mimeType) logs = []*courier.ChannelLog{courier.NewChannelLogFromError("Error sending message", msg.Channel(), msg.ID(), duration, err)} } + // add media logs to our status + for _, log := range mediaLogs { + status.AddLog(log) + } + // add logs to our status for _, log := range logs { status.AddLog(log) @@ -525,7 +533,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat for _, v := range templating.Variables { payload.HSM.LocalizableParams = append(payload.HSM.LocalizableParams, LocalizableParam{Default: v}) } - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } else { payload := templatePayload{ @@ -544,7 +552,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } payload.Template.Components = append(payload.Template.Components, *component) - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) } // add logs to our status @@ -564,7 +572,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat Type: "text", } payload.Text.Body = part - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, token, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) // add logs to our status for _, log := range logs { @@ -600,7 +608,93 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } -func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload interface{}) (string, string, []*courier.ChannelLog, error) { +// fetchMediaID tries to fetch the id for the uploaded media, setting the result in redis. +func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (string, []*courier.ChannelLog, error) { + var logs []*courier.ChannelLog + start := time.Now() + + // check in cache first + rc := h.Backend().RedisPool().Get() + defer rc.Close() + + cacheKey := fmt.Sprintf(mediaCacheKeyPattern, msg.Channel().UUID().String()) + mediaID, err := rcache.Get(rc, cacheKey, mediaURL) + if err != nil { + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error reading media id from redis", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } else if mediaID != "" { + return mediaID, logs, nil + } + + // check in failure cache + failureCacheKey := fmt.Sprintf(failureMediaCacheKeyPattern, msg.Channel().UUID().String()) + if failed, err := rcache.Get(rc, failureCacheKey, mediaURL); err != nil { + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error reading failed media from redis", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } else if failed == "true" { + return "", logs, errors.New("ignoring media that previously failed to upload") + } + + // download media + req, err := http.NewRequest("GET", mediaURL, nil) + if err != nil { + return "", logs, err + } + res, err := utils.MakeHTTPRequest(req) + if err != nil { + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error downloading media", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } + + // upload media to WhatsApp + baseURL := msg.Channel().StringConfigForKey(courier.ConfigBaseURL, "") + req, err = http.NewRequest("POST", baseURL + "/v1/media", bytes.NewReader(res.Body)) + if err != nil { + return "", logs, err + } + setWhatsAppAuthHeader(&req.Header, msg.Channel()) + req.Header.Add("Content-Type", mimeType) + res, err = utils.MakeHTTPRequest(req) + if err != nil { + // put in failure cache + rcache.Set(rc, failureCacheKey, mediaURL, "true") + + if res != nil { + err = errors.Wrap(err, string(res.Body)) + } + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error uploading media to WhatsApp", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } + + // take uploaded media id + mediaID, err = jsonparser.GetString(res.Body, "media", "[0]", "id") + if err != nil { + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error reading media id from response", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } + + // put in cache + err = rcache.Set(rc, cacheKey, mediaURL, mediaID) + if err != nil { + elapsed := time.Now().Sub(start) + log := courier.NewChannelLogFromError("error setting media id to redis", msg.Channel(), msg.ID(), elapsed, err) + logs = append(logs, log) + return "", logs, err + } + return mediaID, logs, nil +} + +func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) @@ -610,7 +704,7 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i return "", "", []*courier.ChannelLog{log}, err } req, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) - req.Header = buildWhatsAppRequestHeader(msg.Channel(), token) + req.Header = buildWhatsAppHeaders(msg.Channel()) rr, err := utils.MakeHTTPRequest(req) log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) errPayload := &mtErrorPayload{} @@ -624,7 +718,7 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i } // check contact baseURL := fmt.Sprintf("%s://%s", sendPath.Scheme, sendPath.Host) - rrCheck, err := checkWhatsAppContact(msg.Channel(), baseURL, token, msg.URN()) + rrCheck, err := checkWhatsAppContact(msg.Channel(), baseURL, msg.URN()) if rrCheck == nil { elapsed := time.Now().Sub(start) @@ -680,7 +774,7 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i } // try send msg again reqRetry, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) - reqRetry.Header = buildWhatsAppRequestHeader(msg.Channel(), token) + reqRetry.Header = buildWhatsAppHeaders(msg.Channel()) if retryParam != "" { reqRetry.URL.RawQuery = fmt.Sprintf("%s=1", retryParam) @@ -698,22 +792,23 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, token string, payload i return "", externalID, []*courier.ChannelLog{log}, err } -func buildAuthorizationHeader(header http.Header, channel courier.Channel, token string) http.Header { +func setWhatsAppAuthHeader(header *http.Header, channel courier.Channel) { + authToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") + if channel.ChannelType() == channelTypeD3 { - header.Set(d3AuthorizationKey, token) + header.Set(d3AuthorizationKey, authToken) } else { - header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + header.Set("Authorization", fmt.Sprintf("Bearer %s", authToken)) } - return header } -func buildWhatsAppRequestHeader(channel courier.Channel, token string) http.Header { +func buildWhatsAppHeaders(channel courier.Channel) http.Header { header := http.Header{ "Content-Type": []string{"application/json"}, "Accept": []string{"application/json"}, "User-Agent": []string{utils.HTTPUserAgent}, } - header = buildAuthorizationHeader(header, channel, token) + setWhatsAppAuthHeader(&header, channel) return header } @@ -740,7 +835,7 @@ type mtContactCheckPayload struct { ForceCheck bool `json:"force_check"` } -func checkWhatsAppContact(channel courier.Channel, baseURL string, token string, urn urns.URN) (*utils.RequestResponse, error) { +func checkWhatsAppContact(channel courier.Channel, baseURL string, urn urns.URN) (*utils.RequestResponse, error) { payload := mtContactCheckPayload{ Blocking: "wait", Contacts: []string{fmt.Sprintf("+%s", urn.Path())}, @@ -753,7 +848,7 @@ func checkWhatsAppContact(channel courier.Channel, baseURL string, token string, } sendURL := fmt.Sprintf("%s/v1/contacts", baseURL) req, _ := http.NewRequest(http.MethodPost, sendURL, bytes.NewReader(reqBody)) - req.Header = buildWhatsAppRequestHeader(channel, token) + req.Header = buildWhatsAppHeaders(channel) rr, err := utils.MakeHTTPRequest(req) if err != nil { diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 9d5e3b611..78a304cdb 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -3,7 +3,9 @@ package whatsapp import ( "context" "encoding/json" + "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -529,6 +531,93 @@ var defaultSendTestCases = []ChannelSendTestCase{ }, } +var mediaCacheSendTestCases = []ChannelSendTestCase{ + {Label: "Media Upload Error", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/media", + Body: "media bytes", + }: MockedResponse{ + Status: 401, + Body: `{ "errors": [{"code":1005,"title":"Access denied","details":"Invalid credentials."}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + BodyContains: `/document.pdf`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Previous Media Upload Error", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + BodyContains: `/document.pdf`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Media Upload OK", + Text: "video caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"video/mp4:https://foo.bar/video.mp4"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/media", + Body: "media bytes", + }: MockedResponse{ + Status: 200, + Body: `{ "media" : [{"id": "36c484d1-1283-4b94-988d-7276bdec4de2"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Cached Media", + Text: "video caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"video/mp4:https://foo.bar/video.mp4"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, +} + var hsmSupportSendTestCases = []ChannelSendTestCase{ {Label: "Template Send", Text: "templated message", @@ -541,6 +630,20 @@ var hsmSupportSendTestCases = []ChannelSendTestCase{ }, } +func mockAttachmentURLs(mediaServer *httptest.Server, testCases []ChannelSendTestCase) []ChannelSendTestCase { + casesWithMockedUrls := make([]ChannelSendTestCase, len(testCases)) + + for i, testCase := range testCases { + mockedCase := testCase + + for j, attachment := range testCase.Attachments { + mockedCase.Attachments[j] = strings.Replace(attachment, "https://foo.bar", mediaServer.URL, 1) + } + casesWithMockedUrls[i] = mockedCase + } + return casesWithMockedUrls +} + func TestSending(t *testing.T) { var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "WA", "250788383383", "US", map[string]interface{}{ @@ -567,4 +670,14 @@ func TestSending(t *testing.T) { RunChannelSendTestCases(t, defaultChannel, newWAHandler(courier.ChannelType("WA"), "WhatsApp"), defaultSendTestCases, nil) RunChannelSendTestCases(t, hsmSupportChannel, newWAHandler(courier.ChannelType("WA"), "WhatsApp"), hsmSupportSendTestCases, nil) RunChannelSendTestCases(t, d3Channel, newWAHandler(courier.ChannelType("D3"), "360Dialog"), defaultSendTestCases, nil) + + mediaServer := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { + defer req.Body.Close() + res.WriteHeader(200) + res.Write([]byte("media bytes")) + })) + defer mediaServer.Close() + mediaCacheSendTestCases := mockAttachmentURLs(mediaServer, mediaCacheSendTestCases) + + RunChannelSendTestCases(t, defaultChannel, newWAHandler(courier.ChannelType("WA"), "WhatsApp"), mediaCacheSendTestCases, nil) } diff --git a/queue/queue.go b/queue/queue.go index 05f08bac1..7aa4bebaa 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -5,7 +5,7 @@ import ( "sync" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/sirupsen/logrus" ) diff --git a/queue/queue_test.go b/queue/queue_test.go index eb5d5db52..157793a0e 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" "github.com/stretchr/testify/assert" ) diff --git a/test.go b/test.go index c35054aff..2298d0836 100644 --- a/test.go +++ b/test.go @@ -13,7 +13,7 @@ import ( "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" - "github.com/garyburd/redigo/redis" + "github.com/gomodule/redigo/redis" _ "github.com/lib/pq" // postgres driver )