diff --git a/config/config.yaml b/config/config.yaml index b3e17549a..9bc391293 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -9,6 +9,7 @@ app: port: 8000 grpc: port: 8001 + metrics_port: 9000 identity_proxy_header: X-Shield-Email # full path prefixed with scheme where resources config yaml files are kept # e.g.: diff --git a/core/user/errors.go b/core/user/errors.go index 76e01272f..1f8dfb4e2 100644 --- a/core/user/errors.go +++ b/core/user/errors.go @@ -9,6 +9,7 @@ var ( ErrConflict = errors.New("user already exist") ErrEmptyKey = errors.New("empty key") ErrKeyAlreadyExists = errors.New("key already exist") + ErrKeyDoesNotExists = errors.New("key does not exist") ErrMissingEmail = errors.New("user email is missing") ErrInvalidUUID = errors.New("invalid syntax of uuid") ) diff --git a/go.mod b/go.mod index a159e2d79..106d5028b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,8 @@ module github.com/odpf/shield go 1.18 require ( + contrib.go.opencensus.io/exporter/ocagent v0.7.0 + contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/MakeNowJust/heredoc v1.0.0 github.com/abbot/go-http-auth v0.4.0 github.com/authzed/authzed-go v0.7.1-0.20221109204547-1aa903788b3b @@ -27,6 +29,7 @@ require ( github.com/mcuadros/go-defaults v1.2.0 github.com/mitchellh/mapstructure v1.5.0 github.com/newrelic/go-agent v3.20.2+incompatible + github.com/newrelic/newrelic-opencensus-exporter-go v0.4.0 github.com/odpf/salt v0.2.5-0.20221130085531-51c81815f7d6 github.com/ory/dockertest v3.3.5+incompatible github.com/pkg/errors v0.9.1 @@ -37,6 +40,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/stretchr/testify v1.8.1 github.com/tidwall/gjson v1.14.4 + go.opencensus.io v0.24.0 go.uber.org/zap v1.24.0 gocloud.dev v0.28.0 golang.org/x/exp v0.0.0-20230108222341-4b8118a2686a @@ -64,9 +68,12 @@ require ( github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect github.com/aymanbagabas/go-osc52 v1.2.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/briandowns/spinner v1.20.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect + github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/charmbracelet/glamour v0.6.0 // indirect github.com/cli/safeexec v1.0.1 // indirect github.com/containerd/continuity v0.3.0 // indirect @@ -79,6 +86,8 @@ require ( github.com/fatih/color v1.13.0 // indirect github.com/felixge/fgprof v0.9.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-kit/log v0.2.1 // indirect + github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-sql-driver/mysql v1.7.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/cel-go v0.13.0 // indirect @@ -106,10 +115,12 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/microcosm-cc/bluemonday v1.0.21 // indirect github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/muesli/reflow v0.3.0 // indirect github.com/muesli/termenv v0.13.0 // indirect + github.com/newrelic/newrelic-telemetry-sdk-go v0.2.0 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect @@ -117,6 +128,11 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.13.1 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.37.0 // indirect + github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/rivo/uniseg v0.4.3 // indirect github.com/rs/zerolog v1.28.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect @@ -133,10 +149,10 @@ require ( github.com/tidwall/pretty v1.2.1 // indirect github.com/yuin/goldmark v1.5.3 // indirect github.com/yuin/goldmark-emoji v1.0.1 // indirect - go.opencensus.io v0.24.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.5.0 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.4.0 // indirect golang.org/x/term v0.4.0 // indirect golang.org/x/text v0.6.0 // indirect diff --git a/go.sum b/go.sum index ecb49349a..6fcd5c5f7 100644 --- a/go.sum +++ b/go.sum @@ -392,6 +392,10 @@ cloud.google.com/go/workflows v1.8.0/go.mod h1:ysGhmEajwZxGn1OhGOGKsTXc5PyxOc0vf cloud.google.com/go/workflows v1.9.0/go.mod h1:ZGkj1aFIOd9c8Gerkjjq7OW7I5+l6cSvT3ujaO/WwSA= code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= contrib.go.opencensus.io/exporter/aws v0.0.0-20200617204711-c478e41e60e9/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= +contrib.go.opencensus.io/exporter/ocagent v0.7.0 h1:BEfdCTXfMV30tLZD8c9n64V/tIZX5+9sXiuFLnrr1k8= +contrib.go.opencensus.io/exporter/ocagent v0.7.0/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= +contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= contrib.go.opencensus.io/exporter/stackdriver v0.13.14/go.mod h1:5pSSGY0Bhuk7waTHuDf4aQ8D2DrhgETRo9fy6k3Xlzc= contrib.go.opencensus.io/integrations/ocsql v0.1.7/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -641,6 +645,7 @@ github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= @@ -670,13 +675,16 @@ github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInq github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s= github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/glamour v0.3.0/go.mod h1:TzF0koPZhqq0YVBNL100cPHznAAjVj7fksX2RInwjGw= github.com/charmbracelet/glamour v0.6.0 h1:wi8fse3Y7nfcabbbDuwolqTqMQPMnVPeZhDM273bISc= @@ -994,11 +1002,13 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2CSIqUrmQPqA0gdRIlnLEY0gK5JGjh37zN5U= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -1289,6 +1299,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.1/go.mod h1:G+WkljZi4mflcqVxYSgvt8MNctRQHjEH8ubKtt1Ka3w= @@ -1612,6 +1623,7 @@ github.com/mattn/go-sqlite3 v1.14.7/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mcuadros/go-defaults v1.2.0 h1:FODb8WSf0uGaY8elWJAkoLL0Ri6AlZ1bFlenk56oZtc= @@ -1698,6 +1710,10 @@ github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= github.com/newrelic/go-agent v3.20.2+incompatible h1:kO1pT79OwgW3KqJzEDUWgg8eam41FKjewMs8mqSRLJk= github.com/newrelic/go-agent v3.20.2+incompatible/go.mod h1:a8Fv1b/fYhFSReoTU6HDkTYIMZeSVNffmoS726Y0LzQ= +github.com/newrelic/newrelic-opencensus-exporter-go v0.4.0 h1:BjzhyzSrzc8/WtyZDWBF8XATW4M92EoZiy38kgL3gfo= +github.com/newrelic/newrelic-opencensus-exporter-go v0.4.0/go.mod h1:gSMlmRnmdRq5c2NTNuh+JtBGXSvnJDqIudxiXwQ07m0= +github.com/newrelic/newrelic-telemetry-sdk-go v0.2.0 h1:W8+lNIfAldCScGiikToSprbf3DCaMXk0VIM9l73BIpY= +github.com/newrelic/newrelic-telemetry-sdk-go v0.2.0/go.mod h1:G9MqE/cHGv3Hx3qpYhfuyFUsGx2DpVcGi1iJIqTg+JQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= @@ -1836,7 +1852,9 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1845,6 +1863,7 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -1860,6 +1879,8 @@ github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common/assets v0.1.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= github.com/prometheus/common/assets v0.2.0/go.mod h1:D17UVUE12bHbim7HzwUvtqm6gwBEaDQ0F+hIGbFbccI= @@ -1879,9 +1900,12 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/prometheus v0.35.0/go.mod h1:7HaLx5kEPKJ0GDgbODG0fZgXbQ8K/XjZNJXQmbmgQlY= github.com/prometheus/prometheus v0.40.5/go.mod h1:bxgdmtoSNLmmIVPGmeTJ3OiP67VmuY4yalE4ZP6L/j8= +github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= +github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/embedmd v0.0.0-20171029212350-c8060a0752a2/go.mod h1:7jOTMgqac46PZcF54q6l2hkLEG8op93fZu61KmxWDV4= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -2011,6 +2035,7 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= @@ -2109,6 +2134,7 @@ go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -2316,6 +2342,7 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -2433,6 +2460,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2586,6 +2614,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2758,6 +2787,7 @@ google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= diff --git a/internal/api/v1beta1/user.go b/internal/api/v1beta1/user.go index 2bab0f571..c9b2f54aa 100644 --- a/internal/api/v1beta1/user.go +++ b/internal/api/v1beta1/user.go @@ -6,6 +6,8 @@ import ( "strings" grpczap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.opencensus.io/stats" + "go.opencensus.io/tag" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -13,6 +15,7 @@ import ( "github.com/odpf/shield/core/user" "github.com/odpf/shield/pkg/metadata" + "github.com/odpf/shield/pkg/telemetry" "github.com/odpf/shield/pkg/uuid" shieldv1beta1 "github.com/odpf/shield/proto/v1beta1" ) @@ -65,6 +68,7 @@ func (h Handler) ListUsers(ctx context.Context, request *shieldv1beta1.ListUsers func (h Handler) CreateUser(ctx context.Context, request *shieldv1beta1.CreateUserRequest) (*shieldv1beta1.CreateUserResponse, error) { logger := grpczap.Extract(ctx) + ctx, err := tag.New(ctx, tag.Insert(telemetry.KeyMethod, "CreateUser")) currentUserEmail, ok := user.GetEmailFromContext(ctx) if !ok { @@ -103,6 +107,14 @@ func (h Handler) CreateUser(ctx context.Context, request *shieldv1beta1.CreateUs switch { case errors.Is(err, user.ErrConflict): return nil, grpcConflictError + case errors.Is(errors.Unwrap(err), user.ErrKeyDoesNotExists): + missingKey := strings.Split(err.Error(), ":") + if len(missingKey) == 2 { + ctx, _ = tag.New(ctx, tag.Upsert(telemetry.KeyMissingKey, missingKey[1])) + } + stats.Record(ctx, telemetry.MMissingMetadataKeys.M(1)) + + return nil, grpcBadBodyError default: return nil, grpcInternalServerError } diff --git a/internal/proxy/hook/authz/authz.go b/internal/proxy/hook/authz/authz.go index 2b44ada4a..48b97ae1d 100644 --- a/internal/proxy/hook/authz/authz.go +++ b/internal/proxy/hook/authz/authz.go @@ -7,6 +7,8 @@ import ( "strings" "github.com/mitchellh/mapstructure" + "go.opencensus.io/stats" + "go.opencensus.io/tag" "github.com/odpf/salt/log" "github.com/odpf/shield/core/namespace" @@ -17,6 +19,7 @@ import ( "github.com/odpf/shield/internal/proxy/hook" "github.com/odpf/shield/internal/proxy/middleware" "github.com/odpf/shield/pkg/body_extractor" + "github.com/odpf/shield/pkg/telemetry" ) type ResourceService interface { @@ -82,6 +85,20 @@ func (a Authz) ServeHook(res *http.Response, err error) (*http.Response, error) return a.escape.ServeHook(res, err) } + isResourceCreated := false + attributes := map[string]interface{}{} + + defer func(isResourceCreated bool, ctx context.Context, attributes map[string]interface{}) { + if !isResourceCreated { + requestDetail := fmt.Sprintf("[status: %d, request: %s, attributes: %s ]", res.StatusCode, res.Request.Method+"@"+res.Request.URL.Host, attributes) + ctx, err := tag.New(ctx, tag.Insert(telemetry.KeyMethod, "ServeHook"), tag.Insert(telemetry.KeyRequestDetails, requestDetail)) + if err != nil { + a.log.Debug("failed to add metrics tags: ", err.Error()) + } + stats.Record(ctx, telemetry.MResourceFailedToCreate.M(1)) + } + }(isResourceCreated, res.Request.Context(), attributes) + ruleFromRequest, ok := hook.ExtractRule(res.Request) if !ok { return a.next.ServeHook(res, nil) @@ -101,7 +118,6 @@ func (a Authz) ServeHook(res *http.Response, err error) (*http.Response, error) return a.next.ServeHook(res, fmt.Errorf("namespace variable not defined in rules")) } - attributes := map[string]interface{}{} attributes["namespace"] = ruleFromRequest.Backend.Namespace identityProxyHeaderValue := res.Request.Header.Get(a.identityProxyHeaderKey) @@ -208,12 +224,22 @@ func (a Authz) ServeHook(res *http.Response, err error) (*http.Response, error) a.log.Error(err.Error()) return a.escape.ServeHook(res, fmt.Errorf(err.Error())) } + + isResourceCreated = true a.log.Info(fmt.Sprintf("Resource %s created with ID %s", newResource.URN, newResource.Idxa)) for _, rel := range config.Relations { subjectId, err := getAttributesValues(attributes[rel.SubjectIDAttribute]) if err != nil { a.log.Error(fmt.Sprintf("cannot create relation: %s not found in attributes", rel.SubjectIDAttribute)) + + relationDetail := fmt.Sprintf("%s to %s %s on %s", rel.Role, rel.SubjectPrincipal, rel.SubjectIDAttribute, newResource.Name) + ctx, err := tag.New(res.Request.Context(), tag.Insert(telemetry.KeyMethod, "ServeHook"), tag.Insert(telemetry.KeyRelationDetails, relationDetail)) + if err != nil { + a.log.Debug("failed to add metrics tags: ", err.Error()) + } + stats.Record(ctx, telemetry.MRelationFailedToCreate.M(1)) + continue } @@ -230,6 +256,14 @@ func (a Authz) ServeHook(res *http.Response, err error) (*http.Response, error) }) if err != nil { a.log.Error(err.Error()) + + relationDetail := fmt.Sprintf("%s to %s %s on %s", rel.Role, rel.SubjectPrincipal, rel.SubjectIDAttribute, newResource.Name) + ctx, err := tag.New(res.Request.Context(), tag.Insert(telemetry.KeyMethod, "ServeHook"), tag.Insert(telemetry.KeyRelationDetails, relationDetail)) + if err != nil { + a.log.Debug("failed to add metrics tags: ", err.Error()) + } + stats.Record(ctx, telemetry.MRelationFailedToCreate.M(1)) + return a.escape.ServeHook(res, fmt.Errorf(err.Error())) } diff --git a/internal/server/config.go b/internal/server/config.go index f0496403b..a6a7ed30f 100644 --- a/internal/server/config.go +++ b/internal/server/config.go @@ -1,6 +1,10 @@ package server -import "fmt" +import ( + "fmt" + + "github.com/odpf/shield/pkg/telemetry" +) type GRPCConfig struct { Port int `mapstructure:"port" default:"8081"` @@ -17,6 +21,9 @@ type Config struct { // GRPC Config GRPC GRPCConfig `mapstructure:"grpc"` + //metrics port + MetricsPort int `yaml:"metrics_port" mapstructure:"metrics_port" default:"9000"` + // the network interface to listen on Host string `yaml:"host" mapstructure:"host" default:"127.0.0.1"` @@ -43,4 +50,6 @@ type Config struct { // ResourcesPathSecretSecret could be a env name, file path or actual value required // to access ResourcesPathSecretPath files ResourcesConfigPathSecret string `yaml:"resources_config_path_secret" mapstructure:"resources_config_path_secret"` + + TelemetryConfig telemetry.Config `yaml:"telemetry_config" mapstructure:"telemetry_config"` } diff --git a/internal/server/server.go b/internal/server/server.go index 23f020fb6..6333c166b 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -20,6 +20,7 @@ import ( "github.com/odpf/shield/internal/api/v1beta1" "github.com/odpf/shield/internal/server/grpc_interceptors" "github.com/odpf/shield/internal/server/health" + "github.com/odpf/shield/pkg/telemetry" shieldv1beta1 "github.com/odpf/shield/proto/v1beta1" "go.uber.org/zap" "google.golang.org/grpc" @@ -76,7 +77,15 @@ func Serve( return err } - logger.Info("[shield] api server starting", "http-port", cfg.Port, "grpc-port", cfg.GRPC.Port) + pe, err := telemetry.SetupOpenCensus(ctx, cfg.TelemetryConfig) + if err != nil { + logger.Error("failed to setup OpenCensus", "err", err) + } + + httpMuxMetrics := http.NewServeMux() + httpMuxMetrics.Handle("/metrics", pe) + + logger.Info("[shield] api server starting", "http-port", cfg.Port, "grpc-port", cfg.GRPC.Port, "metrics-port", cfg.MetricsPort) if err := mux.Serve( ctx, @@ -86,6 +95,12 @@ func Serve( WriteTimeout: 120 * time.Second, MaxHeaderBytes: 1 << 20, }), + mux.WithHTTPTarget(fmt.Sprintf(":%d", cfg.MetricsPort), &http.Server{ + Handler: httpMuxMetrics, + ReadTimeout: 120 * time.Second, + WriteTimeout: 120 * time.Second, + MaxHeaderBytes: 1 << 20, + }), mux.WithGRPCTarget(fmt.Sprintf(":%d", cfg.GRPC.Port), grpcServer), mux.WithGracePeriod(5*time.Second), ); !errors.Is(err, context.Canceled) { diff --git a/internal/store/postgres/user_repository.go b/internal/store/postgres/user_repository.go index 8322a39e6..9f551fe81 100644 --- a/internal/store/postgres/user_repository.go +++ b/internal/store/postgres/user_repository.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "strings" "time" @@ -228,6 +229,14 @@ func (r UserRepository) Create(ctx context.Context, usr user.User) (user.User, e switch { case errors.Is(err, errDuplicateKey): return user.User{}, user.ErrConflict + case errors.Is(err, errForeignKeyViolation): + re := regexp.MustCompile(`\(([^)]+)\) `) + match := re.FindStringSubmatch(err.Error()) + if len(match) > 1 { + return user.User{}, fmt.Errorf("%w:%s", user.ErrKeyDoesNotExists, match[1]) + } + return user.User{}, user.ErrKeyDoesNotExists + default: tx.Rollback() return user.User{}, err diff --git a/pkg/telemetry/opencensus.go b/pkg/telemetry/opencensus.go new file mode 100644 index 000000000..0523b7999 --- /dev/null +++ b/pkg/telemetry/opencensus.go @@ -0,0 +1,74 @@ +package telemetry + +import ( + "context" + + "contrib.go.opencensus.io/exporter/prometheus" + "go.opencensus.io/stats" + "go.opencensus.io/stats/view" + "go.opencensus.io/tag" +) + +var ( + // The number of new metadata keys + MMissingMetadataKeys = stats.Int64("metadata-keys/counter", "The number of missing metadata keys", "1") + MResourceFailedToCreate = stats.Int64("resource-failed-to-create/counter", "The number of resources failed to be created", "1") + MRelationFailedToCreate = stats.Int64("relation-failed-to-create/counter", "The number of relations failed to be created", "1") +) + +var ( + KeyMethod, _ = tag.NewKey("method") + KeyMissingKey, _ = tag.NewKey("missing-key") + KeyRequestDetails, _ = tag.NewKey("request-details") + KeyRelationDetails, _ = tag.NewKey("relation-details") +) + +var ( + MissingMetadataKeysView = &view.View{ + Name: "metadata-keys/counter", + Measure: MMissingMetadataKeys, + Description: "The number of missing metadata keys", + + Aggregation: view.Count(), + TagKeys: []tag.Key{KeyMethod, KeyMissingKey}} + + ResourceFailedToCreateView = &view.View{ + Name: "resource-failed-to-create/counter", + Measure: MResourceFailedToCreate, + Description: "The number of resources failed to be created", + + Aggregation: view.Count(), + TagKeys: []tag.Key{KeyMethod, KeyRequestDetails}} + + RelationFailedToCreateView = &view.View{ + Name: "relation-failed-to-create/counter", + Measure: MRelationFailedToCreate, + Description: "The number of relations failed to be created", + + Aggregation: view.Count(), + TagKeys: []tag.Key{KeyMethod, KeyRelationDetails}} +) + +func SetupOpenCensus(ctx context.Context, cfg Config) (*prometheus.Exporter, error) { + if err := setupViews(); err != nil { + return nil, err + } + + pe, err := prometheus.NewExporter(prometheus.Options{ + Namespace: cfg.ServiceName, + }) + if err != nil { + return nil, err + } + view.RegisterExporter(pe) + return pe, nil +} + +func setupViews() error { + err := view.Register(MissingMetadataKeysView, ResourceFailedToCreateView, RelationFailedToCreateView) + if err != nil { + return err + } + + return nil +} diff --git a/pkg/telemetry/telemetry.go b/pkg/telemetry/telemetry.go new file mode 100644 index 000000000..160867026 --- /dev/null +++ b/pkg/telemetry/telemetry.go @@ -0,0 +1,6 @@ +package telemetry + +type Config struct { + // OpenCensus exporter configurations. + ServiceName string `yaml:"service_name" mapstructure:"service_name"` +}