diff --git a/go.mod b/go.mod
index 34566607..f9d28313 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
github.com/qiniu/x v1.13.10
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.9.0
+ github.com/tmc/langchaingo v0.1.12
github.com/xanzy/go-gitlab v0.109.0
golang.org/x/mod v0.21.0
k8s.io/api v0.28.3
@@ -38,10 +39,11 @@ require (
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.19 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.17 // indirect
github.com/beorn7/perks v1.0.1 // indirect
- github.com/cespare/xxhash/v2 v2.2.0 // indirect
+ github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/distribution/reference v0.6.0 // indirect
+ github.com/dlclark/regexp2 v1.10.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
@@ -50,10 +52,10 @@ require (
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
- github.com/go-openapi/swag v0.22.3 // indirect
+ github.com/go-openapi/swag v0.22.4 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
- github.com/golang/protobuf v1.5.3 // indirect
+ github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/go-github/v56 v56.0.0 // indirect
@@ -72,13 +74,12 @@ require (
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/moby/sys/user v0.3.0 // indirect
github.com/moby/sys/userns v0.1.0 // indirect
- github.com/moby/term v0.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
- github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
+ github.com/pkoukk/tiktoken-go v0.1.6 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.54.0 // indirect
@@ -92,12 +93,12 @@ require (
go.opentelemetry.io/otel/sdk v1.29.0 // indirect
go.opentelemetry.io/otel/trace v1.29.0 // indirect
golang.org/x/net v0.28.0 // indirect
- golang.org/x/oauth2 v0.19.0 // indirect
+ golang.org/x/oauth2 v0.20.0 // indirect
golang.org/x/sync v0.8.0 // indirect
golang.org/x/sys v0.25.0 // indirect
golang.org/x/term v0.23.0 // indirect
golang.org/x/text v0.17.0 // indirect
- golang.org/x/time v0.3.0 // indirect
+ golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
diff --git a/go.sum b/go.sum
index af0927d2..c3c6b8ef 100644
--- a/go.sum
+++ b/go.sum
@@ -1,7 +1,7 @@
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
-github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Carlji/prow v1.0.0 h1:URdiAovXck7M+xO/B9+3idKBy8LeFGsifZyDuprbRmE=
github.com/Carlji/prow v1.0.0/go.mod h1:ET3UlJAy73TpAdmnvfFZAHfWxqB1rVIFkAgSwMHzMic=
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
@@ -34,10 +34,11 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0 h1:yUmoVv70H3J4UOqxqsee39+KlXxNEDfTbAp8c/qULKk=
github.com/bradleyfalzon/ghinstallation/v2 v2.8.0/go.mod h1:fmPmvCiBWhJla3zDv9ZTQSZc8AbwyRnGW1yg5ep1Pcs=
+github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
@@ -46,6 +47,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/dlclark/regexp2 v1.10.0 h1:+/GIL799phkJqYW+3YbOd8LCcbHzT0Pbo8zl70MHsq0=
+github.com/dlclark/regexp2 v1.10.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4=
github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
@@ -54,8 +57,8 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g=
github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
+github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -68,21 +71,20 @@ github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn
github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU=
+github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
@@ -165,6 +167,8 @@ github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQ
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw=
+github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
@@ -177,8 +181,8 @@ github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
github.com/qiniu/x v1.13.10 h1:J4Z3XugYzAq85SlyAfqlKVrbf05glMbAOh+QncsDQpE=
github.com/qiniu/x v1.13.10/go.mod h1:INZ2TSWSJVWO/RuELQROERcslBwVgFG7MkTfEdaQz9E=
-github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
-github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
+github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -195,6 +199,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tmc/langchaingo v0.1.12 h1:yXwSu54f3b1IKw0jJ5/DWu+qFVH1NBblwC0xddBzGJE=
+github.com/tmc/langchaingo v0.1.12/go.mod h1:cd62xD6h+ouk8k/QQFhOsjRYBSA1JJ5UVKXSIgm7Ni4=
github.com/xanzy/go-gitlab v0.109.0 h1:RcRme5w8VpLXTSTTMZdVoQWY37qTJWg+gwdQl4aAttE=
github.com/xanzy/go-gitlab v0.109.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -228,8 +234,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE=
golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg=
-golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg=
-golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8=
+golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo=
+golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -247,8 +253,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc=
golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
@@ -259,15 +265,13 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao=
+google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw=
google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd h1:BBOTEWLuuEGQy9n1y9MhVJ9Qt0BDu21X8qZs71/uPZo=
google.golang.org/genproto/googleapis/api v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:fO8wJzT2zbQbAjbIoos1285VfEIYKDDY+Dt+WpTkh6g=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd h1:6TEm2ZxXoQmFWFlt1vNxvVOa1Q0dXFQD1m/rYjXmS0E=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240822170219-fc7c04adadcd/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc=
google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/internal/linters/agent.go b/internal/linters/agent.go
index 52476d32..08eb9c5e 100644
--- a/internal/linters/agent.go
+++ b/internal/linters/agent.go
@@ -27,8 +27,11 @@ import (
"github.com/qiniu/reviewbot/config"
"github.com/qiniu/reviewbot/internal/cache"
"github.com/qiniu/reviewbot/internal/lintersutil"
+ "github.com/qiniu/reviewbot/internal/llm"
"github.com/qiniu/reviewbot/internal/runner"
"github.com/qiniu/reviewbot/internal/storage"
+ "github.com/qiniu/x/log"
+ "github.com/tmc/langchaingo/llms"
)
// issueCache is the issue references cache.
@@ -59,6 +62,8 @@ type Agent struct {
GenLogViewURL func() string
// IssueReferences is the compiled issue references config for the linter.
IssueReferences []config.CompiledIssueReference
+ // ModelClient is the LLM model client.
+ ModelClient llms.Model
}
// getMsgFormat returns the message format based on report type.
@@ -140,6 +145,11 @@ func (a *Agent) ApplyTypedMessageByIssueReferences(ctx context.Context, lintResu
}
}
if !processed {
+ resp, err := llm.QueryForReference(ctx, a.ModelClient, output.Message)
+ if err != nil {
+ log.Errorf("failed to query LLM server: %v", err)
+ }
+ output.Message += fmt.Sprintf(ReferenceFooter, resp)
newOutputs = append(newOutputs, output)
}
}
@@ -155,5 +165,6 @@ func (a *Agent) ApplyTypedMessageByIssueReferences(ctx context.Context, lintResu
const ReferenceFooter = `
Details
+
%s
`
diff --git a/internal/linters/agent_test.go b/internal/linters/agent_test.go
index f71fb9b1..dc693d6d 100644
--- a/internal/linters/agent_test.go
+++ b/internal/linters/agent_test.go
@@ -2,6 +2,7 @@ package linters
import (
"context"
+ "fmt"
"reflect"
"regexp"
"testing"
@@ -126,7 +127,7 @@ func TestApplyTypedMessageByIssueReferences(t *testing.T) {
expectedOutput: map[string][]LinterOutput{
"file4.go": {
{
- Message: "regular lint message without issue reference",
+ Message: "regular lint message without issue reference" + fmt.Sprintf(ReferenceFooter, ""),
Line: 40,
},
},
@@ -180,7 +181,7 @@ func TestApplyTypedMessageByIssueReferences(t *testing.T) {
},
"file6.go": {
{
- Message: "no issue reference",
+ Message: "no issue reference" + fmt.Sprintf(ReferenceFooter, ""),
Line: 60,
},
},
diff --git a/internal/llm/llm.go b/internal/llm/llm.go
new file mode 100644
index 00000000..ebab8632
--- /dev/null
+++ b/internal/llm/llm.go
@@ -0,0 +1,100 @@
+package llm
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "github.com/tmc/langchaingo/llms"
+)
+
+type Config struct {
+ Provider string
+ APIKey string
+ Model string
+ ServerURL string
+}
+
+var ErrUnsupportedProvider = errors.New("unsupported llm provider")
+var ErrModelIsNil = errors.New("model is nil")
+
+func New(ctx context.Context, config Config) (llms.Model, error) {
+ switch config.Provider {
+ case "openai":
+ return initOpenAIClient(config)
+ case "ollama":
+ return initOllamaClient(config)
+ default:
+ return nil, ErrUnsupportedProvider
+ }
+}
+
+func Query(ctx context.Context, model llms.Model, query string, extraContext string) (string, error) {
+ ragQuery := fmt.Sprintf(ragTemplateStr, query, extraContext)
+ respText, err := llms.GenerateFromSinglePrompt(ctx, model, ragQuery)
+ if err != nil {
+ return "", err
+ }
+ return respText, nil
+}
+
+const ragTemplateStr = `
+I will ask you a question and will provide some additional context information.
+Assume this context information is factual and correct, as part of internal
+documentation.
+If the question relates to the context, answer it using the context.
+If the question does not relate to the context, answer it as normal.
+
+For example, let's say the context has nothing in it about tropical flowers;
+then if I ask you about tropical flowers, just answer what you know about them
+without referring to the context.
+
+For example, if the context does mention minerology and I ask you about that,
+provide information from the context along with general knowledge.
+
+Question:
+%s
+
+Context:
+%s
+`
+
+func QueryForReference(ctx context.Context, model llms.Model, linterOutput string) (string, error) {
+ if model == nil {
+ return "", ErrModelIsNil
+ }
+ ragQuery := fmt.Sprintf(referenceTemplateStr, linterOutput)
+ respText, err := llms.GenerateFromSinglePrompt(ctx, model, ragQuery)
+ if err != nil {
+ return "", err
+ }
+ return respText, nil
+}
+
+const referenceTemplateStr = `
+You are a lint expert, you can explain in detail the meaning of the lint result according to the content of the given context.
+You will follow the format of the example in to answer in Chinese, firstly, you will explain the lint, secondly, you will give the incorrect usage (it can be a code example or text description), and finally, you will give the correct usage (it can be a code example or a text description), and finally add a blank line before the result.
+
+
+
+### lint 解释
+- 该 lint 出现表示一个变量被赋值后又被重新赋值，但在此过程中没有使用其原始值。这种情况通常表明代码中可能存在冗余，或者可能是逻辑错误，因为原始值在此期间没有被实际使用。
+### 错误用法
+ //
+ func main() {
+ x := 10
+ x = 20 // 重新赋值，但未使用原始值
+
+ fmt.Println("x =", x) // 仅打印新值20
+ }
+
+
+### 正确用法
+- 方案一：使用赋值变量
+- 方案二 ：移除冗余赋值
+
+
+Context:
+%s
+
+`
diff --git a/internal/llm/ollama.go b/internal/llm/ollama.go
new file mode 100644
index 00000000..9e7bc433
--- /dev/null
+++ b/internal/llm/ollama.go
@@ -0,0 +1,29 @@
+package llm
+
+import (
+ "errors"
+
+ "github.com/tmc/langchaingo/llms"
+ "github.com/tmc/langchaingo/llms/ollama"
+)
+
+var ErrServerURLRequired = errors.New("server URL is required")
+var ErrModelRequired = errors.New("model is required")
+
+func initOllamaClient(config Config) (llms.Model, error) {
+ if config.ServerURL == "" {
+ return nil, ErrServerURLRequired
+ }
+ if config.Model == "" {
+ return nil, ErrModelRequired
+ }
+ opts := []ollama.Option{
+ ollama.WithServerURL(config.ServerURL),
+ ollama.WithModel(config.Model),
+ }
+ m, err := ollama.New(opts...)
+ if err != nil {
+ return nil, err
+ }
+ return m, nil
+}
diff --git a/internal/llm/openapi.go b/internal/llm/openapi.go
new file mode 100644
index 00000000..41c81ac2
--- /dev/null
+++ b/internal/llm/openapi.go
@@ -0,0 +1,26 @@
+package llm
+
+import (
+ "errors"
+
+ "github.com/tmc/langchaingo/llms"
+ "github.com/tmc/langchaingo/llms/openai"
+)
+
+var ErrAPIKeyRequired = errors.New("API key is required")
+
+func initOpenAIClient(config Config) (llms.Model, error) {
+ if config.APIKey == "" {
+ return nil, ErrAPIKeyRequired
+ }
+ opts := []openai.Option{
+ openai.WithModel(config.Model),
+ openai.WithToken(config.APIKey),
+ openai.WithBaseURL(config.ServerURL),
+ }
+ m, err := openai.New(opts...)
+ if err != nil {
+ return nil, err
+ }
+ return m, nil
+}
diff --git a/main.go b/main.go
index e45c90fb..0d5f9c22 100644
--- a/main.go
+++ b/main.go
@@ -31,6 +31,7 @@ import (
"github.com/google/go-github/v57/github"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/qiniu/reviewbot/config"
+ "github.com/qiniu/reviewbot/internal/llm"
"github.com/qiniu/reviewbot/internal/storage"
"github.com/qiniu/reviewbot/internal/version"
"github.com/qiniu/x/log"
@@ -78,11 +79,19 @@ type options struct {
serverAddr string
// kube config file
kubeConfig string
+
+ // llm related
+ llmProvider string
+ llmModel string
+ llmServerURL string
+ llmAPIKey string
}
var (
- errAppNotSet = errors.New("app-private-key is required when using github app")
- errWebHookNotSet = errors.New("webhook-secret is required")
+ errAppNotSet = errors.New("app-private-key is required when using github app")
+ errWebHookNotSet = errors.New("webhook-secret is required")
+ errLLMKeyNotSet = errors.New("llm api key is not set")
+ errLLMServerNotSet = errors.New("llm model or server url is not set")
)
func (o options) Validate() error {
@@ -93,6 +102,19 @@ func (o options) Validate() error {
if o.webhookSecret == "" {
return errWebHookNotSet
}
+
+ if o.llmProvider != "" {
+ switch o.llmProvider {
+ case "openai":
+ if o.llmAPIKey == "" {
+ return errLLMKeyNotSet
+ }
+ case "ollama":
+ if o.llmModel == "" || o.llmServerURL == "" {
+ return errLLMServerNotSet
+ }
+ }
+ }
return nil
}
@@ -120,6 +142,12 @@ func gatherOptions() options {
fs.StringVar(&o.gitLabPersonalAccessToken, "gitlab.personal-access-token", "", "personal gitlab access token")
fs.StringVar(&o.gitLabHost, "gitlab.host", "", "gitlab server")
+ // llm related
+ fs.StringVar(&o.llmProvider, "llm.provider", "", "llm provider")
+ fs.StringVar(&o.llmModel, "llm.model", "", "llm model")
+ fs.StringVar(&o.llmServerURL, "llm.server-url", "", "llm server url")
+ fs.StringVar(&o.llmAPIKey, "llm.api-key", "", "llm api key")
+
err := fs.Parse(os.Args[1:])
if err != nil {
log.Fatalf("failed to parse flags: %v", err)
@@ -232,6 +260,13 @@ func main() {
}
}
+ modelConfig := llm.Config{
+ Provider: o.llmProvider,
+ APIKey: o.llmAPIKey,
+ Model: o.llmModel,
+ ServerURL: o.llmServerURL,
+ }
+
s := &Server{
webhookSecret: []byte(o.webhookSecret),
gitClientFactory: v2,
@@ -243,6 +278,7 @@ func main() {
gitLabHost: o.gitLabHost,
gitLabPersonalAccessToken: o.gitLabPersonalAccessToken,
gitHubPersonalAccessToken: o.gitHubPersonalAccessToken,
+ modelConfig: modelConfig,
}
// github app
@@ -254,6 +290,7 @@ func main() {
}
}
+ s.initLLMModel()
go s.initDockerRunner()
go s.initKubernetesRunner()
s.initCustomLinters()
diff --git a/server.go b/server.go
index ab3427d8..17d62bb6 100644
--- a/server.go
+++ b/server.go
@@ -39,9 +39,11 @@ import (
"github.com/qiniu/reviewbot/config"
"github.com/qiniu/reviewbot/internal/linters"
"github.com/qiniu/reviewbot/internal/lintersutil"
+ "github.com/qiniu/reviewbot/internal/llm"
"github.com/qiniu/reviewbot/internal/runner"
"github.com/qiniu/reviewbot/internal/storage"
"github.com/qiniu/x/log"
+ "github.com/tmc/langchaingo/llms"
"github.com/xanzy/go-gitlab"
gitv2 "sigs.k8s.io/prow/pkg/git/v2"
)
@@ -78,6 +80,9 @@ type Server struct {
// gitHubAppPrivateKey string
gitHubAppAuth *GitHubAppAuth
gitHubPersonalAccessToken string
+
+ modelConfig llm.Config
+ modelClient llms.Model
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@@ -113,6 +118,18 @@ func (s *Server) serveGitHub(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "Event received. Have a nice day.")
switch event := event.(type) {
+ case *github.IssueCommentEvent:
+ go func() {
+ if err := s.processIssueCommentEvent(ctx, event); err != nil {
+ log.Errorf("process issue comment event: %v", err)
+ }
+ }()
+ case *github.PullRequestReviewCommentEvent:
+ go func() {
+ if err := s.processPullRequestReviewCommentEvent(ctx, event); err != nil {
+ log.Errorf("process pull request review comment event: %v", err)
+ }
+ }()
case *github.PullRequestEvent:
go func() {
if err := s.processPullRequestEvent(ctx, event); err != nil {
@@ -310,6 +327,9 @@ func (s *Server) handleCodeRequestEvent(ctx context.Context, info *codeRequestIn
// set issue references
agent.IssueReferences = s.config.GetCompiledIssueReferences(name)
+ // set model client
+ agent.ModelClient = s.modelClient
+
// run linter finally
if err := fn(ctx, agent); err != nil {
if errors.Is(err, context.Canceled) {
@@ -349,6 +369,14 @@ func (s *Server) withCancel(ctx context.Context, info *codeRequestInfo, fn func(
return fn(ctx)
}
+func (s *Server) initLLMModel() {
+ modelClient, err := llm.New(context.Background(), s.modelConfig)
+ if err != nil {
+ log.Fatalf("failed to init rag server: %v", err)
+ }
+ s.modelClient = modelClient
+}
+
func (s *Server) initCustomLinters() {
for linterName, customLinter := range s.config.CustomLinters {
linters.RegisterPullRequestHandler(linterName, linters.GeneralLinterHandler)
@@ -482,6 +510,79 @@ func (s *Server) pullImageWithRetry(ctx context.Context, image string, dockerRun
}
}
+func (s *Server) processIssueCommentEvent(ctx context.Context, event *github.IssueCommentEvent) error {
+ log := lintersutil.FromContext(ctx)
+ if event.GetAction() != "created" {
+ log.Debugf("skipping action %s\n", event.GetAction())
+ return nil
+ }
+ if !strings.Contains(*event.Comment.Body, "reviewbot") {
+ return nil
+ }
+
+ query := event.GetComment().GetBody()
+ resp, err := llm.Query(ctx, s.modelClient, query, "")
+ if err != nil {
+ return err
+ }
+ log.Infof("query success,got resp: %s", resp)
+
+ replyComment := &github.IssueComment{
+ Body: github.String(resp),
+ }
+ installationID := event.GetInstallation().GetID()
+ _, _, err = s.GithubClient(installationID).Issues.CreateComment(ctx, event.GetRepo().GetOwner().GetLogin(), event.GetRepo().GetName(), event.GetIssue().GetNumber(), replyComment)
+ if err != nil {
+ return err
+ }
+ log.Infof("create comment success: %v", replyComment)
+ return nil
+}
+
+func (s *Server) processPullRequestReviewCommentEvent(ctx context.Context, event *github.PullRequestReviewCommentEvent) error {
+ log := lintersutil.FromContext(ctx)
+ if event.GetAction() != "created" {
+ log.Debugf("skipping action %s\n", event.GetAction())
+ return nil
+ }
+
+ query := event.GetComment().GetBody()
+ path := event.GetComment().GetPath()
+ position := event.GetComment().GetPosition()
+ diffHunk := event.GetComment().GetDiffHunk()
+ inReplyTo := event.GetComment().GetInReplyTo()
+
+ if !strings.Contains(query, "reviewbot") {
+ log.Debugf("skipping reviewbot comment\n")
+ return nil
+ }
+
+ installationID := event.GetInstallation().GetID()
+ githubClient := s.GithubClient(installationID)
+ historyComments, err := prepareCommentContext(ctx, event, githubClient, path, position)
+ if err != nil {
+ return err
+ }
+ queryContext := "diffHunk: " + diffHunk + "\n" + "history comments: " + historyComments
+
+ // send query to llm model
+ resp, err := llm.Query(ctx, s.modelClient, query, queryContext)
+ if err != nil {
+ return err
+ }
+ log.Infof("query success,got resp: %s", resp)
+
+ // reply comment
+ comment, _, err := githubClient.PullRequests.CreateCommentInReplyTo(ctx, event.GetRepo().GetOwner().GetLogin(), event.GetRepo().GetName(), event.GetPullRequest().GetNumber(), resp, inReplyTo)
+ if err != nil {
+ log.Errorf("failed to create comment: %v", err)
+ return err
+ }
+
+ log.Infof("create comment success: %v", comment)
+ return nil
+}
+
func (s *Server) processPullRequestEvent(ctx context.Context, event *github.PullRequestEvent) error {
log := lintersutil.FromContext(ctx)
if event.GetAction() != "opened" && event.GetAction() != "reopened" && event.GetAction() != "synchronize" {
@@ -646,3 +747,24 @@ func checkKubectlInstalled() error {
cmd := exec.Command("kubectl", "version", "--client")
return cmd.Run()
}
+
+func prepareCommentContext(ctx context.Context, event *github.PullRequestReviewCommentEvent, githubClient *github.Client, path string, position int) (string, error) {
+ // get all comments
+ prComments, _, err := githubClient.PullRequests.ListComments(ctx, event.GetRepo().GetOwner().GetLogin(), event.GetRepo().GetName(), event.GetPullRequest().GetNumber(), nil)
+ if err != nil {
+ return "", err
+ }
+
+ // filter comments by path and position
+ var comments []string
+ if path != "" && position != 0 {
+ for _, comment := range prComments {
+ if comment.GetPath() == path && comment.GetPosition() == position {
+ comments = append(comments, comment.GetBody())
+ }
+ }
+ }
+ log.Infof("filter comments success: %v", comments)
+
+ return strings.Join(comments, "\n"), nil
+}