diff --git a/go.mod b/go.mod index 40d76b6acf..fa81ddb98c 100644 --- a/go.mod +++ b/go.mod @@ -107,10 +107,10 @@ require ( golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 golang.org/x/oauth2 v0.24.0 golang.org/x/sync v0.10.0 - google.golang.org/api v0.211.0 + google.golang.org/api v0.212.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 google.golang.org/grpc v1.68.1 - google.golang.org/protobuf v1.35.2 + google.golang.org/protobuf v1.36.0 ) require ( @@ -124,9 +124,9 @@ require ( require ( cel.dev/expr v0.16.1 // indirect cloud.google.com/go v0.116.0 // indirect - cloud.google.com/go/auth v0.12.1 // indirect + cloud.google.com/go/auth v0.13.0 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect - cloud.google.com/go/compute/metadata v0.5.2 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect cloud.google.com/go/iam v1.2.2 // indirect cloud.google.com/go/monitoring v1.21.2 // indirect dario.cat/mergo v1.0.1 // indirect diff --git a/go.sum b/go.sum index 94508de14c..4b1e768310 100644 --- a/go.sum +++ b/go.sum @@ -34,8 +34,8 @@ cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE= cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U= -cloud.google.com/go/auth v0.12.1 h1:n2Bj25BUMM0nvE9D2XLTiImanwZhO3DkfWSYS/SAJP4= -cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4= +cloud.google.com/go/auth v0.13.0 h1:8Fu8TZy167JkW8Tj3q7dIkr2v4cndv41ouecJx0PAHs= +cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/EMxX6Q= cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU= cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -50,8 +50,8 @@ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTB cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw= cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= -cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo= -cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= cloud.google.com/go/datacatalog v1.23.0 h1:9F2zIbWNNmtrSkPIyGRQNsIugG5VgVVFip6+tXSdWLg= cloud.google.com/go/datacatalog v1.23.0/go.mod h1:9Wamq8TDfL2680Sav7q3zEhBJSPBrDxJU8WtPJ25dBM= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= @@ -1772,8 +1772,8 @@ google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7 google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= -google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg= -google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0= +google.golang.org/api v0.212.0 h1:BcRj3MJfHF3FYD29rk7u9kuu1SyfGqfHcA0hSwKqkHg= +google.golang.org/api v0.212.0/go.mod h1:gICpLlpp12/E8mycRMzgy3SQ9cFh2XnVJ6vJi/kQbvI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1922,8 +1922,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ= +google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= diff --git a/warehouse/archive/archiver.go b/warehouse/archive/archiver.go index f611f60001..81b4548973 100644 --- a/warehouse/archive/archiver.go +++ b/warehouse/archive/archiver.go @@ -431,18 +431,12 @@ func (a *Archiver) getStagingFilesData( FROM %s WHERE - source_id = $1 - AND destination_id = $2 - AND id >= $3 - and id <= $4;`, + upload_id = $1;`, pq.QuoteIdentifier(warehouseutils.WarehouseStagingFilesTable), ) stagingFileRows, err := txn.QueryContext(ctx, stmt, - u.sourceID, - u.destID, - u.startStagingFileId, - u.endStagingFileId, + u.uploadID, ) if err != nil { return nil, fmt.Errorf("cannot query staging files data: %w", err) diff --git a/warehouse/archive/testdata/dump.sql b/warehouse/archive/testdata/dump.sql index 18853a361e..927916da38 100644 --- a/warehouse/archive/testdata/dump.sql +++ b/warehouse/archive/testdata/dump.sql @@ -43,13 +43,13 @@ VALUES 1, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, 231, NOW(), NOW(), NOW(), - NOW(), '{}', NULL + NOW(), '{}', 1 ), ( 2, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz', '{}', 'test-sourceID', 'test-destinationID', 'succeeded', 1, 231, NOW(), NOW(), NOW(), - NOW(), '{}', NULL + NOW(), '{}', 2 ), ( 3, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz', diff --git a/warehouse/archive/testdata/storage.json b/warehouse/archive/testdata/storage.json index 1db2b7fea9..7610a01652 100644 --- a/warehouse/archive/testdata/storage.json +++ b/warehouse/archive/testdata/storage.json @@ -1,6 +1,6 @@ { - "test-prefix/wh-staging-files.test-sourceID.test-destinationID.1.unix_time.json.gz": "{\"id\":1,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n", - "test-prefix/wh-staging-files.test-sourceID.test-destinationID.2.unix_time.json.gz": "{\"id\":2,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n", + "test-prefix/wh-staging-files.test-sourceID.test-destinationID.1.unix_time.json.gz": "{\"id\":1,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":1}\n", + "test-prefix/wh-staging-files.test-sourceID.test-destinationID.2.unix_time.json.gz": "{\"id\":2,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":2}\n", "test-prefix/wh-staging-files.test-sourceID.test-destinationID.3.unix_time.json.gz": "{\"id\":3,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":3}\n", "test-prefix/wh-staging-files.test-sourceID.test-destinationID.4.unix_time.json.gz": "{\"id\":4,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":4}\n" -} \ No newline at end of file +}