From a3a7611e99e1802ae7f49e91d5cf53497dcbd8c1 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Tue, 3 Dec 2024 17:29:48 -0800 Subject: [PATCH] Improve observability; add sqlite retries to deal with sqlite_busy errors (#352) * Use the pragma `sqlite_busy` to add automatic retries on sqlite busy error * I was seeing high rates of sqlitebusy errors when deployed on K8s on Azure and this was causing learning to happen much less frequently then it should * Clean up the prometheus metrics to support dashboards for monitoring impact. The key metrics we want are * Number of accepted suggestions * Number of executed cells * Number of learned examples * Cleanup up prometheus metrics so we can better monitor why learning isn't happening; this was used to track down sqlite busy errors * Support monitoring with Datadog * With Datadog structured logging we need to use the field "level" for severity or else it won't automatically parse that field * Udpate the configuration to allow the user to control what fields are used in the encoder --- app/go.mod | 31 ++++---- app/go.sum | 37 ++++++++++ app/pkg/agent/agent.go | 24 +++++++ app/pkg/analyze/fsql/schema.sql | 2 +- app/pkg/analyze/session_manager.go | 17 +++-- app/pkg/application/app.go | 52 ++++++++++++-- app/pkg/config/config.go | 9 +++ app/pkg/learn/learner.go | 29 ++++++-- app/pkg/server/server.go | 18 +++-- ...devcontainer-01JDDK3R7CDQP3ZBEM1TQRK466.md | 51 -------------- .../en/docs/operator-manual/datadog.md | 70 +++++++++++++++++++ 11 files changed, 251 insertions(+), 89 deletions(-) delete mode 100644 developer_guides/devcontainer-01JDDK3R7CDQP3ZBEM1TQRK466.md create mode 100644 docs/content/en/docs/operator-manual/datadog.md diff --git a/app/go.mod b/app/go.mod index a5c4b84..156b422 100644 --- a/app/go.mod +++ b/app/go.mod @@ -54,7 +54,7 @@ require ( gonum.org/v1/gonum v0.15.0 google.golang.org/api v0.189.0 google.golang.org/grpc v1.64.1 - google.golang.org/protobuf v1.34.2 + google.golang.org/protobuf v1.35.2 gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.27.3 k8s.io/client-go v1.5.2 @@ -81,8 +81,9 @@ require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect + github.com/bufbuild/connect-go v1.10.0 // indirect github.com/bytedance/sonic v1.11.3 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect github.com/chenzhuoyu/iasm v0.9.1 // indirect github.com/cli/go-gh v1.2.1 // indirect @@ -100,6 +101,7 @@ require ( github.com/docker/docker v26.1.2+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/easyCZ/connect-go-prometheus v0.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect @@ -136,7 +138,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/compress v1.17.4 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -147,12 +149,13 @@ require ( github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect @@ -160,10 +163,10 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_golang v1.12.0 // indirect - github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/client_golang v1.20.5 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.60.1 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/replicate/replicate-go v0.21.0 // indirect github.com/rivo/uniseg v0.4.7 // indirect @@ -207,12 +210,12 @@ require ( go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.7.0 // indirect - golang.org/x/crypto v0.25.0 // indirect - golang.org/x/net v0.27.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect + golang.org/x/crypto v0.27.0 // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.23.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.27.0 // indirect + golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/genproto v0.0.0-20240722135656-d784300faade // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade // indirect diff --git a/app/go.sum b/app/go.sum index 8208fbb..a000413 100644 --- a/app/go.sum +++ b/app/go.sum @@ -112,6 +112,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bufbuild/connect-go v1.10.0 h1:QAJ3G9A1OYQW2Jbk3DeoJbkCxuKArrvZgDt47mjdTbg= +github.com/bufbuild/connect-go v1.10.0/go.mod h1:CAIePUgkDR5pAFaylSMtNK45ANQjp9JvpluG20rhpV8= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/bytedance/sonic v1.5.0/go.mod h1:ED5hyg4y6t3/9Ku1R6dU/4KyJ48DZ4jPhfY1O2AihPM= @@ -121,11 +123,14 @@ github.com/bytedance/sonic v1.11.3/go.mod h1:iZcSUejdk5aukTND/Eu/ivjQuEL0Cu9/rf5 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/chenzhuoyu/base64x v0.0.0-20211019084208-fb5309c8db06/go.mod h1:DH46F32mSOjUmXrMHnKwZdA8wcEefY7UVqBKYGjpdQY= @@ -185,6 +190,8 @@ github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNk github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/easyCZ/connect-go-prometheus v0.0.1 h1:FBY/pP3p5glNMBOChsB3YjN6CdCCtDRKZ8buTlaHGS0= +github.com/easyCZ/connect-go-prometheus v0.0.1/go.mod h1:i0QRR0j2RBYS0ZV40AnW5SG0eJmUgtINUILwyI+tzXk= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -239,6 +246,7 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -420,6 +428,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= @@ -457,6 +467,8 @@ github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/maxence-charriere/go-app/v9 v9.8.0 h1:rDfLNvxIKXyjpRS76P45kn9Xj8IumwfoqpsEJYxfd+E= github.com/maxence-charriere/go-app/v9 v9.8.0/go.mod h1:gzgFoeaDuoNHw9MbJraTCKIoKtZ/SoIfOIHHn2FOffc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -472,6 +484,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0= github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -510,23 +524,31 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.0 h1:C+UIj/QWtmqY13Arb8kwMt5j34/0Z2iKamrJ+ryC0Gg= github.com/prometheus/client_golang v1.12.0/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= +github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a h1:CmF68hwI0XsOQ5UwlBopMi2Ow4Pbg32akc4KIVCOm+Y= github.com/prometheus/client_model v0.2.1-0.20210607210712-147c58e9608a/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/replicate/replicate-go v0.21.0 h1:37eWInQH7rRr14Epj5axfh/Iko+vsOqsS1JKm9DEhr4= @@ -707,6 +729,8 @@ golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30= golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -784,6 +808,8 @@ golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -794,6 +820,8 @@ golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs= +golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -808,6 +836,8 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -867,6 +897,8 @@ golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -875,6 +907,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.22.0 h1:BbsgPEJULsl2fV/AT3v15Mjva5yXKQDyKf+TbDz7QJk= +golang.org/x/term v0.24.0 h1:Mh5cbb+Zk2hqqXNO7S1iTjEphVL+jb8ZWaqh/g+JWkM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -890,6 +923,8 @@ golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1053,6 +1088,8 @@ google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFW google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= +google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/app/pkg/agent/agent.go b/app/pkg/agent/agent.go index 5ff2b2b..8955370 100644 --- a/app/pkg/agent/agent.go +++ b/app/pkg/agent/agent.go @@ -6,6 +6,9 @@ import ( "strings" "sync" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/jlewi/foyle/app/pkg/runme/ulid" "github.com/jlewi/foyle/protos/go/foyle/v1alpha1/v1alpha1connect" @@ -49,6 +52,19 @@ const ( temperature = 0.9 ) +var ( + executedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "cells_executed_total", + Help: "Total number of executed cells broken down by status"}, + []string{"status"}, + ) + + acceptedCounter = promauto.NewCounter(prometheus.CounterOpts{ + Name: "cells_accepted_total", + Help: "Total number of suggested cells accepted broken down by type"}, + ) +) + // Agent is the agent. type Agent struct { v1alpha1.UnimplementedGenerateServiceServer @@ -625,6 +641,14 @@ func (a *Agent) LogEvents(ctx context.Context, req *connect.Request[v1alpha1.Log func() { _, span := tp.Start(ctx, "LogEvent", trace.WithAttributes(attribute.String("eventType", event.Type.String()), attribute.String("contextId", event.ContextId), attribute.String("selectedCellId", event.SelectedId))) defer span.End() + + switch event.GetType() { + case v1alpha1.LogEventType_ACCEPTED: + + acceptedCounter.Inc() + case v1alpha1.LogEventType_EXECUTE: + executedCounter.WithLabelValues(event.GetExecuteStatus().String()).Inc() + } // N.B we can't use zap.Object to log the event because it contains runme protos which don't have the zap marshaler bindings. log.Info("LogEvent", "eventId", event.GetEventId(), "eventType", event.Type, "contextId", event.ContextId, "selectedCellId", event.SelectedId, logs.ZapProto("event", event)) }() diff --git a/app/pkg/analyze/fsql/schema.sql b/app/pkg/analyze/fsql/schema.sql index 38d3106..26d11c8 100644 --- a/app/pkg/analyze/fsql/schema.sql +++ b/app/pkg/analyze/fsql/schema.sql @@ -33,4 +33,4 @@ CREATE TABLE IF NOT EXISTS results ( -- The JSON serialization of the proto. proto_json TEXT NOT NULL -); \ No newline at end of file +); diff --git a/app/pkg/analyze/session_manager.go b/app/pkg/analyze/session_manager.go index dc87917..bb6dd46 100644 --- a/app/pkg/analyze/session_manager.go +++ b/app/pkg/analyze/session_manager.go @@ -8,6 +8,9 @@ import ( "os" "path/filepath" + "github.com/go-logr/zapr" + "go.uber.org/zap" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -69,6 +72,15 @@ func NewSessionsManager(db *sql.DB) (*SessionsManager, error) { return nil, err } + // Set busy_timeout using PRAGMA. This is to deal with frequent sqlite busy errors when deployed on + // Azure. + // This is in milliseconds + if _, err := db.Exec("PRAGMA busy_timeout = 10000;"); err != nil { + return nil, errors.Wrapf(err, "Failed to set busy timeout for the database") + } + log := zapr.NewLogger(zap.L()) + log.Info("sqlite busy_timeout set", "timeout", 5000) + // Create the dbtx from the actual database queries := fsql.New(db) @@ -115,7 +127,6 @@ func (db *SessionsManager) Update(ctx context.Context, contextID string, updateF tx, err := db.db.BeginTx(ctx, &sql.TxOptions{}) if err != nil { - // DO NOT COMMIT sessCounter.WithLabelValues("failedstart").Inc() return errors.Wrapf(err, "Failed to start transaction") } @@ -132,7 +143,6 @@ func (db *SessionsManager) Update(ctx context.Context, contextID string, updateF if err != nil { logDBErrors(ctx, err) if err != sql.ErrNoRows { - // DO NOT COMMIT sessCounter.WithLabelValues("failedget").Inc() return errors.Wrapf(err, "Failed to get session with id %v", contextID) } @@ -144,7 +154,6 @@ func (db *SessionsManager) Update(ctx context.Context, contextID string, updateF } } - // DO NOT COMMIT sessCounter.WithLabelValues("callupdatefunc").Inc() if err := updateFunc(session); err != nil { @@ -172,7 +181,6 @@ func (db *SessionsManager) Update(ctx context.Context, contextID string, updateF NumGenerateTraces: newRow.NumGenerateTraces, } - // DO NOT COMMIT sessCounter.WithLabelValues("callupdatesession").Inc() if err := queries.UpdateSession(ctx, update); err != nil { logDBErrors(ctx, err) @@ -199,7 +207,6 @@ func (db *SessionsManager) Update(ctx context.Context, contextID string, updateF return err } - // DO NOT COMMIT sessCounter.WithLabelValues("done").Inc() return nil } diff --git a/app/pkg/application/app.go b/app/pkg/application/app.go index d47d811..cb523b2 100644 --- a/app/pkg/application/app.go +++ b/app/pkg/application/app.go @@ -280,9 +280,27 @@ func (a *App) createCoreForConsole(paths []string) (zapcore.Core, error) { // Use the keys used by cloud logging // https://cloud.google.com/logging/docs/structured-logging - c.LevelKey = "severity" - c.TimeKey = "time" - c.MessageKey = "message" + logFields := a.Config.Logging.LogFields + if logFields == nil { + logFields = &config.LogFields{} + } + if logFields.Level != "" { + c.LevelKey = logFields.Level + } else { + c.LevelKey = "severity" + } + + if logFields.Time != "" { + c.TimeKey = logFields.Time + } else { + c.TimeKey = "time" + } + + if logFields.Message != "" { + c.MessageKey = logFields.Message + } else { + c.MessageKey = "message" + } lvl := a.Config.GetLogLevel() zapLvl := zap.NewAtomicLevel() @@ -333,10 +351,32 @@ func (a *App) createJSONCoreLogger(paths []string) (zapcore.Core, error) { c := zap.NewProductionEncoderConfig() // Use the keys used by cloud logging // https://cloud.google.com/logging/docs/structured-logging - c.LevelKey = "severity" - c.TimeKey = "time" - c.MessageKey = "message" + logFields := a.Config.Logging.LogFields + if logFields == nil { + logFields = &config.LogFields{} + } + if logFields.Level != "" { + c.LevelKey = logFields.Level + } else { + c.LevelKey = "severity" + } + + if logFields.Time != "" { + c.TimeKey = logFields.Time + } else { + c.TimeKey = "time" + } + + if logFields.Message != "" { + c.MessageKey = logFields.Message + } else { + c.MessageKey = "message" + } + // We attach the function key to the logs because that is useful for identifying the function that generated the log. + // N.B are logs processing depends on this field being present in the logs. This is one reason + // why we don't allow it to be customized to match the field expected by a logging backend like Datadog + // or Cloud Logging c.FunctionKey = "function" jsonEncoder := zapcore.NewJSONEncoder(c) diff --git a/app/pkg/config/config.go b/app/pkg/config/config.go index ff0594f..a62ebec 100644 --- a/app/pkg/config/config.go +++ b/app/pkg/config/config.go @@ -201,6 +201,15 @@ type Logging struct { // MaxDelaySeconds is the maximum delay in seconds to wait before processing the logs. MaxDelaySeconds int `json:"maxDelaySeconds,omitempty" yaml:"maxDelaySeconds,omitempty"` + + LogFields *LogFields `json:"logFields,omitempty" yaml:"logFields,omitempty"` +} + +// LogFields is the fields to use when logging to structured logging +type LogFields struct { + Level string `json:"level,omitempty" yaml:"level,omitempty"` + Time string `json:"time,omitempty" yaml:"time,omitempty"` + Message string `json:"message,omitempty" yaml:"message,omitempty"` } type LogSink struct { diff --git a/app/pkg/learn/learner.go b/app/pkg/learn/learner.go index 501b8c2..836e73b 100644 --- a/app/pkg/learn/learner.go +++ b/app/pkg/learn/learner.go @@ -41,9 +41,17 @@ const ( var ( enqueuedCounter = promauto.NewCounter(prometheus.CounterOpts{ Name: "learner_enqueued_total", - Help: "Total number of enqueued blocks", + Help: "Total number of enqueued sessions for learning", }) + sessFiltered = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "learner_not_learnable", + Help: "Number of sessions that aren't learnable", + }, + []string{"status"}, + ) + sessProcessed = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "learner_sessions_processed", @@ -51,6 +59,14 @@ var ( }, []string{"status"}, ) + + learnedCounter = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "learned_examples_total", + Help: "Number of examples learned", + }, + []string{"status"}, + ) ) // Learner handles the learn loop to learn from past mistakes. @@ -298,7 +314,10 @@ func (l *Learner) Reconcile(ctx context.Context, id string) error { if len(writeErrors.Causes) > 0 { writeErrors.Final = errors.New("Not all examples could be successfully reconciled") + learnedCounter.WithLabelValues("error").Inc() return writeErrors + } else { + learnedCounter.WithLabelValues("success").Inc() } return nil } @@ -381,26 +400,26 @@ func isLearnable(session *logspb.Session) bool { if execEvent == nil { // Since the cell wasn't successfully executed we don't learn from it - sessProcessed.WithLabelValues("noexec").Inc() + sessFiltered.WithLabelValues("noexec").Inc() return false } log := zapr.NewLogger(zap.L()) if session.GetFullContext() == nil { - sessProcessed.WithLabelValues("nocontext").Inc() + sessFiltered.WithLabelValues("nocontext").Inc() log.Error(errors.New("Session missing fullcontext"), "contextId", session.GetContextId()) return false } if session.GetFullContext().GetNotebook() == nil { - sessProcessed.WithLabelValues("nonotebook").Inc() + sessFiltered.WithLabelValues("nonotebook").Inc() log.Error(errors.New("Session missing notebook"), "contextId", session.GetContextId()) return false } if session.GetFullContext().GetSelected() == 0 { // If its the first cell we can't learn from it because what would we use as context to predict it? - sessProcessed.WithLabelValues("firstcell").Inc() + sessFiltered.WithLabelValues("firstcell").Inc() return false } return true diff --git a/app/pkg/server/server.go b/app/pkg/server/server.go index afc802e..a8e2271 100644 --- a/app/pkg/server/server.go +++ b/app/pkg/server/server.go @@ -127,13 +127,17 @@ func (s *Server) createGinEngine() error { apiPrefix := s.config.APIPrefix() - // Set up the connect-rpc handlers for the EvalServer + // Set up the connect-rpc handlers + + // Create the OTEL interceptor otelInterceptor, err := otelconnect.NewInterceptor() if err != nil { return errors.Wrapf(err, "Failed to create otel interceptor") } - path, handler := v1alpha1connect.NewEvalServiceHandler(s.evalServer, connect.WithInterceptors(otelInterceptor)) + interceptors := []connect.Interceptor{otelInterceptor} + + path, handler := v1alpha1connect.NewEvalServiceHandler(s.evalServer, connect.WithInterceptors(interceptors...)) log.Info("Setting up eval service", "path", path) // Since we want to add the prefix apiPrefix we need to strip it before passing it to the connect-rpc handler @@ -141,19 +145,19 @@ func (s *Server) createGinEngine() error { // support prefixes. router.Any(apiPrefix+"/"+path+"*any", gin.WrapH(http.StripPrefix("/"+apiPrefix, handler))) - generatePath, generateHandler := v1alpha1connect.NewGenerateServiceHandler(s, connect.WithInterceptors(otelInterceptor)) + generatePath, generateHandler := v1alpha1connect.NewGenerateServiceHandler(s, connect.WithInterceptors(interceptors...)) log.Info("Setting up generate service", "path", apiPrefix+"/"+generatePath) router.Any(apiPrefix+"/"+generatePath+"*any", gin.WrapH(http.StripPrefix("/"+apiPrefix, generateHandler))) - aiSvcPath, aiSvcHandler := v1alpha1connect.NewAIServiceHandler(s.agent, connect.WithInterceptors(otelInterceptor)) + aiSvcPath, aiSvcHandler := v1alpha1connect.NewAIServiceHandler(s.agent, connect.WithInterceptors(interceptors...)) log.Info("Setting up AI service", "path", apiPrefix+"/"+aiSvcPath) router.Any(apiPrefix+"/"+aiSvcPath+"*any", gin.WrapH(http.StripPrefix("/"+apiPrefix, aiSvcHandler))) - logsSvcPath, logsSvcHandler := logspbconnect.NewLogsServiceHandler(s.logsCrud, connect.WithInterceptors(otelInterceptor)) + logsSvcPath, logsSvcHandler := logspbconnect.NewLogsServiceHandler(s.logsCrud, connect.WithInterceptors(interceptors...)) log.Info("Setting up logs service", "path", apiPrefix+"/"+logsSvcPath) router.Any(apiPrefix+"/"+logsSvcPath+"*any", gin.WrapH(http.StripPrefix("/"+apiPrefix, logsSvcHandler))) - sessSvcPath, sessSvcHandler := logspbconnect.NewSessionsServiceHandler(s.sessManager, connect.WithInterceptors(otelInterceptor)) + sessSvcPath, sessSvcHandler := logspbconnect.NewSessionsServiceHandler(s.sessManager, connect.WithInterceptors(interceptors...)) log.Info("Setting up sessions service", "path", apiPrefix+"/"+sessSvcPath) router.Any(apiPrefix+"/"+sessSvcPath+"*any", gin.WrapH(http.StripPrefix("/"+apiPrefix, sessSvcHandler))) @@ -161,7 +165,7 @@ func (s *Server) createGinEngine() error { if err != nil { return errors.Wrapf(err, "Failed to create ConvertersService") } - cvtSvcPath, cvtSvcHandler := logspbconnect.NewConversionServiceHandler(cSvc, connect.WithInterceptors(otelInterceptor)) + cvtSvcPath, cvtSvcHandler := logspbconnect.NewConversionServiceHandler(cSvc, connect.WithInterceptors(interceptors...)) log.Info("Setting up conversion service", "path", apiPrefix+"/"+cvtSvcPath) router.Any(apiPrefix+"/"+cvtSvcPath+"*any", gin.WrapH(http.StripPrefix("/"+apiPrefix, cvtSvcHandler))) diff --git a/developer_guides/devcontainer-01JDDK3R7CDQP3ZBEM1TQRK466.md b/developer_guides/devcontainer-01JDDK3R7CDQP3ZBEM1TQRK466.md deleted file mode 100644 index 51b802a..0000000 --- a/developer_guides/devcontainer-01JDDK3R7CDQP3ZBEM1TQRK466.md +++ /dev/null @@ -1,51 +0,0 @@ ---- -runme: - document: - relativePath: devcontainer.md - session: - id: 01JDDK3R7CDQP3ZBEM1TQRK466 - updated: 2024-11-23 15:10:17-08:00 ---- - -# Building a dev instance of a container - -* Use ko to build a docker image with some changes you want to test without doing a release - -```bash {"id":"01JDDG5YXKP5JT14J1A8X6S6JB"} -# TODO(jeremy): Should we pute these in a different repository -cd ../app -# TODO(jeremy): WHy do we have to set ko_config_path; why isn't it picked up automatically? -KO_DOCKER_REPO=ghcr.io/jlewi/foyle \ - ko build --base-import-paths ./ - -# Ran on 2024-11-23 15:09:59-08:00 for 13.913s exited with 0 -2024/11/23 15:10:01 Using base cgr.dev/chainguard/static:latest@sha256:5ff428f8a48241b93a4174dbbc135a4ffb2381a9e10bdbbc5b9db145645886d5 for github.com/jlewi/foyle/app -2024/11/23 15:10:02 Using build config foyle for github.com/jlewi/foyle/app -2024/11/23 15:10:04 git is in a dirty state -Please check in your pipeline what can be changing the following files: -AM app/.ko.yaml - M app/Dockerfile - M app/pkg/analyze/session_manager.go - M manifests/statefulset.yaml -?? developer_guides/devcontainer-01JDDK3R7CDQP3ZBEM1TQRK466.md -?? developer_guides/devcontainer.md - -2024/11/23 15:10:04 Building github.com/jlewi/foyle/app for linux/amd64 -2024/11/23 15:10:10 Publishing ghcr.io/jlewi/foyle/app:latest -2024/11/23 15:10:10 existing blob: sha256:31395f960e91fef49bf76d6ef18ab0b06f538025931d1cc57d6e729487da71c4 -2024/11/23 15:10:10 existing blob: sha256:250c06f7c38e52dc77e5c7586c3e40280dc7ff9bb9007c396e06d96736cf8542 -2024/11/23 15:10:11 pushed blob: sha256:b63296218d0e42ae111b040e4f88e45416291572612eee7b905300f52ba7b92a -2024/11/23 15:10:11 pushed blob: sha256:80953368c6fb45c3f2761dde20c75785aa31343a7b2c1307ce36e4a7bd82da93 -2024/11/23 15:10:11 pushed blob: sha256:e0be50e8b3b26f747fb8923cabac200955fcd31b2a710febd812226e3ce4b2ad -2024/11/23 15:10:12 ghcr.io/jlewi/foyle/app:sha256-9be786a0993fc946df13618a771ec90b5c7f9d082e37ba665dbf2a8541682ae0.sbom: digest: sha256:6e7f4b619c5d31f2495d701c9bd80c8fb06580efbd37be0a7b5f8324e7889bc5 size: 375 -2024/11/23 15:10:12 Published SBOM ghcr.io/jlewi/foyle/app:sha256-9be786a0993fc946df13618a771ec90b5c7f9d082e37ba665dbf2a8541682ae0.sbom -2024/11/23 15:10:13 pushed blob: sha256:1733c90cb8f756f25ef803fb450f8c26c2be9cb63e629cd2cde5538e54dddd14 -2024/11/23 15:10:13 ghcr.io/jlewi/foyle/app:latest: digest: sha256:9be786a0993fc946df13618a771ec90b5c7f9d082e37ba665dbf2a8541682ae0 size: 1337 -2024/11/23 15:10:13 Published ghcr.io/jlewi/foyle/app@sha256:9be786a0993fc946df13618a771ec90b5c7f9d082e37ba665dbf2a8541682ae0 -ghcr.io/jlewi/foyle/app@sha256:9be786a0993fc946df13618a771ec90b5c7f9d082e37ba665dbf2a8541682ae0 - -``` - -It looks like the output of your previous `ko build` command indicated that your Git repository is in a dirty state with untracked files. Before proceeding, it's a good idea to either commit or clean up these changes. Here are the next steps you can take: - -1. Check the status of your Git repository to see what files are causing the dirty state. \ No newline at end of file diff --git a/docs/content/en/docs/operator-manual/datadog.md b/docs/content/en/docs/operator-manual/datadog.md new file mode 100644 index 0000000..32e0517 --- /dev/null +++ b/docs/content/en/docs/operator-manual/datadog.md @@ -0,0 +1,70 @@ +--- +title: "Datadog" +description: "Use Datadog to monitor Foyle" +weight: 7 +--- + +## What You'll Learn + +How to use Datadog to monitor Foyle. + +## Datadog Logging + + +If you are running Foyle in a Kubernetes cluster with the Datadog agent then you can +do the following to have your logs show up in Datadog. + +Add Datadog's universal tagging labels to the Foyle statefulset and the pod template e.g + + +``` +labels: + tags.datadoghq.com/env: staging + tags.datadoghq.com/service: foyle + tags.datadoghq.com/version: "0.1" + +``` + +Modify your logging configuration to tell Foyle to use the "level" field to store the logging +level. Ensure that `json` is set to `true`. + +``` +logging: + level: info + ... + # Specify the names of the fields to match what Datadog uses + # https://docs.datadoghq.com/logs/log_configuration/parsing/?tab=matchers + logFields: + level: level + + sinks: + - json: true +``` + +## Prometheus (OpenMetrics) + +To configure Datadog to scrape Foyle's OpenMetrics (Prometheus endpoint) add the annotation +for the OpenMetrics check to the annotations of the pod template spec + +```yaml +apiVersion: apps/v1 +kind: StatefulSet +spec: + ... + template: + metadata: + annotations: + ad.datadoghq.com/foyle.checks: | + { + "openmetrics": { + "init_config": {}, + "instances": [ + { + "openmetrics_endpoint": "http://%%host%%:%%port%%/metrics", + "namespace": "foyle", + "metrics": [".*"] + } + ] + } + } +``` \ No newline at end of file