From 188ce1c5ed658b652af58cedd2bd4c0897470c6e Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Sat, 8 Jul 2023 00:09:23 +1000 Subject: [PATCH] Enforce per-query chunks limit earlier when streaming chunks from ingesters to queriers (#5369) * Add initial attempt at enforcing chunks limit earlier. * Update unit tests to match new behaviour. * Use changes from https://github.com/grafana/mimir-prometheus/pull/512 * Add changelog entry. * Fix linting issue. * Simplify methods not expected to be used. --- CHANGELOG.md | 1 + go.mod | 2 +- go.sum | 4 +- pkg/distributor/distributor_test.go | 69 ++++- pkg/distributor/query.go | 5 + pkg/distributor/query_test.go | 96 +++--- pkg/ingester/client/ingester.pb.go | 274 ++++++++++-------- pkg/ingester/client/ingester.proto | 1 + pkg/ingester/client/streaming.go | 6 +- pkg/ingester/client/streaming_test.go | 2 +- pkg/ingester/ingester.go | 8 +- pkg/ingester/ingester_test.go | 6 +- .../prometheus/storage/interface.go | 6 + .../prometheus/prometheus/storage/merge.go | 55 +++- .../prometheus/prometheus/storage/series.go | 17 ++ .../prometheus/prometheus/tsdb/querier.go | 4 + vendor/modules.txt | 4 +- 17 files changed, 367 insertions(+), 193 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08bdbd2e03a..51d0ce30e6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ * [ENHANCEMENT] Querier: add `cortex_querier_queries_rejected_total` metric that counts the number of queries rejected due to hitting a limit (eg. max series per query or max chunks per query). #5316 #5440 * [ENHANCEMENT] Querier: add experimental `-querier.minimize-ingester-requests-hedging-delay` option to initiate requests to further ingesters when request minimisation is enabled and not all initial requests have completed. #5368 * [ENHANCEMENT] Clarify docs for `-ingester.client.*` flags to make it clear that these are used by both queriers and distributors. #5375 +* [ENHANCEMENT] Querier: enforce `max-chunks-per-query` limit earlier in query processing when streaming chunks from ingesters to queriers to avoid unnecessarily consuming resources for queries that will be aborted. #5369 * [ENHANCEMENT] Ingester: added `cortex_ingester_shipper_last_successful_upload_timestamp_seconds` metric tracking the last successful TSDB block uploaded to the bucket (unix timestamp in seconds). #5396 * [BUGFIX] Ingester: Handle when previous ring state is leaving and the number of tokens has changed. #5204 * [BUGFIX] Querier: fix issue where queries that use the `timestamp()` function fail with `execution: attempted to read series at index 0 from stream, but the stream has already been exhausted` if streaming chunks from ingesters to queriers is enabled. #5370 diff --git a/go.mod b/go.mod index 1229c5393a6..8e8727724b8 100644 --- a/go.mod +++ b/go.mod @@ -248,7 +248,7 @@ require ( ) // Using a fork of Prometheus with Mimir-specific changes. -replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20230706135548-245a68172fbe +replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20230706234245-f3697f524295 // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: diff --git a/go.sum b/go.sum index 93971eeace0..9bcae86f4cf 100644 --- a/go.sum +++ b/go.sum @@ -873,8 +873,8 @@ github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9 h1:WB3bGH2f1UN6 github.com/grafana/gomemcache v0.0.0-20230316202710-a081dae0aba9/go.mod h1:PGk3RjYHpxMM8HFPhKKo+vve3DdlPUELZLSDEFehPuU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe h1:yIXAAbLswn7VNWBIvM71O2QsgfgW9fRXZNR0DXe6pDU= github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= -github.com/grafana/mimir-prometheus v0.0.0-20230706135548-245a68172fbe h1:8c4p0PhhhD5G4Ueessf+BF2cS5KYXbryPUFvBdKhYm0= -github.com/grafana/mimir-prometheus v0.0.0-20230706135548-245a68172fbe/go.mod h1:fkSyvELGooBfCA+cH2ziQvMpbQFdyopvJLEBEmNg2zk= +github.com/grafana/mimir-prometheus v0.0.0-20230706234245-f3697f524295 h1:cPynDsMdfh+qDDnAkJy27J5mMTBpYpqHVFVi2/d/o2E= +github.com/grafana/mimir-prometheus v0.0.0-20230706234245-f3697f524295/go.mod h1:fkSyvELGooBfCA+cH2ziQvMpbQFdyopvJLEBEmNg2zk= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956 h1:em1oddjXL8c1tL0iFdtVtPloq2hRPen2MJQKoAWpxu0= github.com/grafana/opentracing-contrib-go-stdlib v0.0.0-20230509071955-f410e79da956/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6 h1:A3dhViTeFDSQcGOXuUi6ukCQSMyDtDISBp2z6OOo2YM= diff --git a/pkg/distributor/distributor_test.go b/pkg/distributor/distributor_test.go index d732bf865ac..bc6e529062c 100644 --- a/pkg/distributor/distributor_test.go +++ b/pkg/distributor/distributor_test.go @@ -2912,6 +2912,7 @@ type prepConfig struct { ingestersSeriesCountTotal uint64 ingesterZones []string labelNamesStreamZonesResponseDelay map[string]time.Duration + preferStreamingChunks bool timeOut bool } @@ -3022,6 +3023,8 @@ func prepare(t *testing.T, cfg prepConfig) ([]*Distributor, []mockIngester, []*p distributorCfg.DefaultLimits.MaxInflightPushRequestsBytes = cfg.maxInflightRequestsBytes distributorCfg.DefaultLimits.MaxIngestionRate = cfg.maxIngestionRate distributorCfg.ShuffleShardingLookbackPeriod = time.Hour + distributorCfg.PreferStreamingChunks = cfg.preferStreamingChunks + distributorCfg.StreamingChunksPerIngesterSeriesBufferSize = 128 cfg.limits.IngestionTenantShardSize = cfg.shuffleShardSize @@ -3493,12 +3496,25 @@ func (i *mockIngester) QueryStream(_ context.Context, req *client.QueryRequest, return nil, err } - results := []*client.QueryStreamResponse{} + nonStreamingResponses := []*client.QueryStreamResponse{} + streamingLabelResponses := []*client.QueryStreamResponse{} + streamingChunkResponses := []*client.QueryStreamResponse{} + + series := make([]*mimirpb.PreallocTimeseries, 0, len(i.timeseries)) + for _, ts := range i.timeseries { if !match(ts.Labels, matchers) { continue } + series = append(series, ts) + } + + slices.SortFunc(series, func(a, b *mimirpb.PreallocTimeseries) bool { + return labels.Compare(mimirpb.FromLabelAdaptersToLabels(a.Labels), mimirpb.FromLabelAdaptersToLabels(b.Labels)) < 0 + }) + + for seriesIndex, ts := range series { c, err := chunk.NewForEncoding(chunk.PrometheusXorChunk) if err != nil { return nil, err @@ -3574,15 +3590,48 @@ func (i *mockIngester) QueryStream(_ context.Context, req *client.QueryRequest, } } - results = append(results, &client.QueryStreamResponse{ - Chunkseries: []client.TimeSeriesChunk{ - { - Labels: ts.Labels, - Chunks: wireChunks, + if req.StreamingChunksBatchSize > 0 { + streamingLabelResponses = append(streamingLabelResponses, &client.QueryStreamResponse{ + StreamingSeries: []client.QueryStreamSeries{ + { + Labels: ts.Labels, + ChunkCount: int64(len(wireChunks)), + }, }, - }, - }) + }) + + streamingChunkResponses = append(streamingChunkResponses, &client.QueryStreamResponse{ + StreamingSeriesChunks: []client.QueryStreamSeriesChunks{ + { + SeriesIndex: uint64(seriesIndex), + Chunks: wireChunks, + }, + }, + }) + } else { + nonStreamingResponses = append(nonStreamingResponses, &client.QueryStreamResponse{ + Chunkseries: []client.TimeSeriesChunk{ + { + Labels: ts.Labels, + Chunks: wireChunks, + }, + }, + }) + } } + + var results []*client.QueryStreamResponse + + if req.StreamingChunksBatchSize > 0 { + endOfLabelsMessage := &client.QueryStreamResponse{ + IsEndOfSeriesStream: true, + } + results = append(streamingLabelResponses, endOfLabelsMessage) + results = append(results, streamingChunkResponses...) + } else { + results = nonStreamingResponses + } + return &stream{ results: results, }, nil @@ -3874,6 +3923,10 @@ func (s *stream) Recv() (*client.QueryStreamResponse, error) { return result, nil } +func (s *stream) Context() context.Context { + return context.Background() +} + func (i *mockIngester) AllUserStats(context.Context, *client.UserStatsRequest, ...grpc.CallOption) (*client.UsersStatsResponse, error) { return &i.stats, nil } diff --git a/pkg/distributor/query.go b/pkg/distributor/query.go index 5f8b4951c9e..a72cccb732b 100644 --- a/pkg/distributor/query.go +++ b/pkg/distributor/query.go @@ -283,6 +283,11 @@ func (d *Distributor) queryIngesterStream(ctx context.Context, replicationSet ri return ingesterQueryResult{}, limitErr } + // We enforce the chunk count limit here, but enforce the chunk bytes limit while streaming the chunks themselves. + if chunkLimitErr := queryLimiter.AddChunks(int(s.ChunkCount)); chunkLimitErr != nil { + return ingesterQueryResult{}, chunkLimitErr + } + labelsBatch = append(labelsBatch, mimirpb.FromLabelAdaptersToLabels(s.Labels)) } diff --git a/pkg/distributor/query_test.go b/pkg/distributor/query_test.go index e9e989145a8..f045c0850fd 100644 --- a/pkg/distributor/query_test.go +++ b/pkg/distributor/query_test.go @@ -34,56 +34,66 @@ import ( func TestDistributor_QueryStream_ShouldReturnErrorIfMaxChunksPerQueryLimitIsReached(t *testing.T) { const maxChunksLimit = 30 // Chunks are duplicated due to replication factor. - ctx := user.InjectOrgID(context.Background(), "user") - limits := &validation.Limits{} - flagext.DefaultValues(limits) - limits.MaxChunksPerQuery = maxChunksLimit + for _, streamingEnabled := range []bool{true, false} { + t.Run(fmt.Sprintf("streaming enabled: %v", streamingEnabled), func(t *testing.T) { + ctx := user.InjectOrgID(context.Background(), "user") + limits := &validation.Limits{} + flagext.DefaultValues(limits) + limits.MaxChunksPerQuery = maxChunksLimit + + // Prepare distributors. + ds, _, _ := prepare(t, prepConfig{ + numIngesters: 3, + happyIngesters: 3, + numDistributors: 1, + limits: limits, + preferStreamingChunks: streamingEnabled, + }) + + // Push a number of series below the max chunks limit. Each series has 1 sample, + // so expect 1 chunk per series when querying back. + initialSeries := maxChunksLimit / 3 + writeReq := makeWriteRequest(0, initialSeries, 0, false, false) + writeRes, err := ds[0].Push(ctx, writeReq) + require.Equal(t, &mimirpb.WriteResponse{}, writeRes) + require.Nil(t, err) + + allSeriesMatchers := []*labels.Matcher{ + labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), + } - // Prepare distributors. - ds, _, _ := prepare(t, prepConfig{ - numIngesters: 3, - happyIngesters: 3, - numDistributors: 1, - limits: limits, - }) + ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, maxChunksLimit, stats.NewQueryMetrics(prometheus.NewPedanticRegistry()))) - ctx = limiter.AddQueryLimiterToContext(ctx, limiter.NewQueryLimiter(0, 0, maxChunksLimit, stats.NewQueryMetrics(prometheus.NewPedanticRegistry()))) + // Since the number of series (and thus chunks) is equal to the limit (but doesn't + // exceed it), we expect a query running on all series to succeed. + queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.NoError(t, err) - // Push a number of series below the max chunks limit. Each series has 1 sample, - // so expect 1 chunk per series when querying back. - initialSeries := maxChunksLimit / 3 - writeReq := makeWriteRequest(0, initialSeries, 0, false, false) - writeRes, err := ds[0].Push(ctx, writeReq) - assert.Equal(t, &mimirpb.WriteResponse{}, writeRes) - assert.Nil(t, err) + if streamingEnabled { + require.Len(t, queryRes.StreamingSeries, initialSeries) + } else { + require.Len(t, queryRes.Chunkseries, initialSeries) + } - allSeriesMatchers := []*labels.Matcher{ - labels.MustNewMatcher(labels.MatchRegexp, model.MetricNameLabel, ".+"), - } + // Push more series to exceed the limit once we'll query back all series. + writeReq = &mimirpb.WriteRequest{} + for i := 0; i < maxChunksLimit; i++ { + writeReq.Timeseries = append(writeReq.Timeseries, + makeWriteRequestTimeseries([]mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: fmt.Sprintf("another_series_%d", i)}}, 0, 0), + ) + } - // Since the number of series (and thus chunks) is equal to the limit (but doesn't - // exceed it), we expect a query running on all series to succeed. - queryRes, err := ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.NoError(t, err) - assert.Len(t, queryRes.Chunkseries, initialSeries) + writeRes, err = ds[0].Push(ctx, writeReq) + require.Equal(t, &mimirpb.WriteResponse{}, writeRes) + require.Nil(t, err) - // Push more series to exceed the limit once we'll query back all series. - writeReq = &mimirpb.WriteRequest{} - for i := 0; i < maxChunksLimit; i++ { - writeReq.Timeseries = append(writeReq.Timeseries, - makeWriteRequestTimeseries([]mimirpb.LabelAdapter{{Name: model.MetricNameLabel, Value: fmt.Sprintf("another_series_%d", i)}}, 0, 0), - ) + // Since the number of series (and thus chunks) is exceeding to the limit, we expect + // a query running on all series to fail. + _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) + require.Error(t, err) + require.ErrorContains(t, err, "the query exceeded the maximum number of chunks") + }) } - - writeRes, err = ds[0].Push(ctx, writeReq) - assert.Equal(t, &mimirpb.WriteResponse{}, writeRes) - assert.Nil(t, err) - - // Since the number of series (and thus chunks) is exceeding to the limit, we expect - // a query running on all series to fail. - _, err = ds[0].QueryStream(ctx, math.MinInt32, math.MaxInt32, allSeriesMatchers...) - require.Error(t, err) - assert.ErrorContains(t, err, "the query exceeded the maximum number of chunks") } func TestDistributor_QueryStream_ShouldReturnErrorIfMaxSeriesPerQueryLimitIsReached(t *testing.T) { diff --git a/pkg/ingester/client/ingester.pb.go b/pkg/ingester/client/ingester.pb.go index 6ebede48207..86fe0d8f960 100644 --- a/pkg/ingester/client/ingester.pb.go +++ b/pkg/ingester/client/ingester.pb.go @@ -926,7 +926,8 @@ func (m *QueryStreamResponse) GetStreamingSeriesChunks() []QueryStreamSeriesChun } type QueryStreamSeries struct { - Labels []github_com_grafana_mimir_pkg_mimirpb.LabelAdapter `protobuf:"bytes,1,rep,name=labels,proto3,customtype=github.com/grafana/mimir/pkg/mimirpb.LabelAdapter" json:"labels"` + Labels []github_com_grafana_mimir_pkg_mimirpb.LabelAdapter `protobuf:"bytes,1,rep,name=labels,proto3,customtype=github.com/grafana/mimir/pkg/mimirpb.LabelAdapter" json:"labels"` + ChunkCount int64 `protobuf:"varint,2,opt,name=chunk_count,json=chunkCount,proto3" json:"chunk_count,omitempty"` } func (m *QueryStreamSeries) Reset() { *m = QueryStreamSeries{} } @@ -961,6 +962,13 @@ func (m *QueryStreamSeries) XXX_DiscardUnknown() { var xxx_messageInfo_QueryStreamSeries proto.InternalMessageInfo +func (m *QueryStreamSeries) GetChunkCount() int64 { + if m != nil { + return m.ChunkCount + } + return 0 +} + type QueryStreamSeriesChunks struct { SeriesIndex uint64 `protobuf:"varint,1,opt,name=series_index,json=seriesIndex,proto3" json:"series_index,omitempty"` Chunks []Chunk `protobuf:"bytes,2,rep,name=chunks,proto3" json:"chunks"` @@ -1986,127 +1994,127 @@ func init() { func init() { proto.RegisterFile("ingester.proto", fileDescriptor_60f6df4f3586b478) } var fileDescriptor_60f6df4f3586b478 = []byte{ - // 1906 bytes of a gzipped FileDescriptorProto + // 1919 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xbc, 0x58, 0xcd, 0x6f, 0x1b, 0xc7, 0x15, 0xe7, 0xf0, 0x4b, 0xe2, 0x23, 0x45, 0xad, 0x86, 0x96, 0xc9, 0xd0, 0x35, 0xa5, 0x6c, 0xe1, 0x94, 0x4d, 0x13, 0xca, 0x5f, 0x2d, 0x9c, 0x20, 0x45, 0x40, 0x49, 0xb4, 0x45, 0xdb, 0x24, 0x9d, 0x25, 0x95, 0xb8, 0x05, 0x82, 0xc5, 0x92, 0x1c, 0x49, 0x0b, 0x73, 0x97, 0xcc, 0xce, 0xb2, 0x90, - 0xd2, 0x4b, 0x81, 0xde, 0x8b, 0xde, 0x7a, 0x2b, 0xd0, 0x5b, 0xd1, 0x53, 0xd1, 0x4b, 0x6f, 0x3d, - 0xe7, 0x12, 0xc0, 0xc7, 0xa0, 0x07, 0xa3, 0x96, 0x7b, 0x68, 0x6f, 0x01, 0xfa, 0x0f, 0x14, 0x3b, - 0x33, 0xfb, 0xc9, 0x95, 0xa5, 0x04, 0xb6, 0x4f, 0xe4, 0xbc, 0x8f, 0xdf, 0xbc, 0xf7, 0xe6, 0xbd, - 0x37, 0x6f, 0x07, 0x8a, 0xba, 0x79, 0x48, 0xa8, 0x4d, 0xac, 0xc6, 0xcc, 0x9a, 0xda, 0x53, 0x9c, - 0x1d, 0x4d, 0x2d, 0x9b, 0x1c, 0x57, 0xdf, 0x3f, 0xd4, 0xed, 0xa3, 0xf9, 0xb0, 0x31, 0x9a, 0x1a, - 0x5b, 0x87, 0xd3, 0xc3, 0xe9, 0x16, 0x63, 0x0f, 0xe7, 0x07, 0x6c, 0xc5, 0x16, 0xec, 0x1f, 0x57, - 0xab, 0x5e, 0x0f, 0x8a, 0x5b, 0xda, 0x81, 0x66, 0x6a, 0x5b, 0x86, 0x6e, 0xe8, 0xd6, 0xd6, 0xec, - 0xc9, 0x21, 0xff, 0x37, 0x1b, 0xf2, 0x5f, 0xae, 0x21, 0x77, 0xa1, 0xfa, 0x50, 0x1b, 0x92, 0x49, + 0x72, 0x2a, 0xd0, 0x7b, 0xd1, 0x5b, 0x2f, 0x45, 0x81, 0xde, 0x8a, 0x9e, 0x8a, 0x5e, 0x7a, 0xeb, + 0x39, 0x97, 0x00, 0x3e, 0x06, 0x3d, 0x18, 0xb5, 0xdc, 0x43, 0x7b, 0x0b, 0xd0, 0x7f, 0xa0, 0xd8, + 0x99, 0xd9, 0x4f, 0xae, 0x2c, 0x25, 0xb0, 0x7d, 0x22, 0xe7, 0xbd, 0x37, 0xbf, 0x79, 0xef, 0xcd, + 0xfb, 0xda, 0x81, 0xa2, 0x6e, 0x1e, 0x12, 0x6a, 0x13, 0xab, 0x31, 0xb3, 0xa6, 0xf6, 0x14, 0x67, + 0x47, 0x53, 0xcb, 0x26, 0xc7, 0xd5, 0xf7, 0x0f, 0x75, 0xfb, 0x68, 0x3e, 0x6c, 0x8c, 0xa6, 0xc6, + 0xd6, 0xe1, 0xf4, 0x70, 0xba, 0xc5, 0xd8, 0xc3, 0xf9, 0x01, 0x5b, 0xb1, 0x05, 0xfb, 0xc7, 0xb7, + 0x55, 0xaf, 0x07, 0xc5, 0x2d, 0xed, 0x40, 0x33, 0xb5, 0x2d, 0x43, 0x37, 0x74, 0x6b, 0x6b, 0xf6, + 0xe4, 0x90, 0xff, 0x9b, 0x0d, 0xf9, 0x2f, 0xdf, 0x21, 0x77, 0xa1, 0xfa, 0x50, 0x1b, 0x92, 0x49, 0x57, 0x33, 0x08, 0x6d, 0x9a, 0xe3, 0x4f, 0xb5, 0xc9, 0x9c, 0x50, 0x85, 0x7c, 0x31, 0x27, 0xd4, 0xc6, 0xd7, 0x61, 0xd9, 0xd0, 0xec, 0xd1, 0x11, 0xb1, 0x68, 0x05, 0x6d, 0xa6, 0xea, 0xf9, 0x9b, - 0x97, 0x1a, 0xdc, 0xb2, 0x06, 0xd3, 0xea, 0x70, 0xa6, 0xe2, 0x49, 0xc9, 0x7b, 0x70, 0x25, 0x16, - 0x8f, 0xce, 0xa6, 0x26, 0x25, 0xf8, 0xc7, 0x90, 0xd1, 0x6d, 0x62, 0xb8, 0x68, 0xa5, 0x10, 0x9a, - 0x90, 0xe5, 0x12, 0xf2, 0x2e, 0xe4, 0x03, 0x54, 0x7c, 0x15, 0x60, 0xe2, 0x2c, 0x55, 0x53, 0x33, - 0x48, 0x05, 0x6d, 0xa2, 0x7a, 0x4e, 0xc9, 0x4d, 0xdc, 0xad, 0xf0, 0x65, 0xc8, 0xfe, 0x8a, 0x09, - 0x56, 0x92, 0x9b, 0xa9, 0x7a, 0x4e, 0x11, 0x2b, 0xf9, 0x2f, 0x08, 0xae, 0x06, 0x60, 0x76, 0x34, - 0x6b, 0xac, 0x9b, 0xda, 0x44, 0xb7, 0x4f, 0x5c, 0x1f, 0x37, 0x20, 0xef, 0x03, 0x73, 0xc3, 0x72, - 0x0a, 0x78, 0xc8, 0x34, 0x14, 0x84, 0xe4, 0x45, 0x82, 0x80, 0x7f, 0x06, 0x85, 0xd1, 0x74, 0x6e, - 0xda, 0xaa, 0x41, 0xec, 0xa3, 0xe9, 0xb8, 0x92, 0xda, 0x44, 0xf5, 0xa2, 0xef, 0xec, 0x8e, 0xc3, - 0xeb, 0x30, 0x96, 0x92, 0x1f, 0xf9, 0x0b, 0x79, 0x1f, 0x6a, 0x67, 0xd9, 0x2a, 0xe2, 0x77, 0x2b, - 0x1c, 0xbf, 0xab, 0x8b, 0xf1, 0xeb, 0x13, 0x4b, 0x27, 0x94, 0x6d, 0xe1, 0x46, 0xf2, 0x19, 0x82, - 0xf5, 0x58, 0x81, 0xf3, 0x82, 0xaa, 0x01, 0xe6, 0x6c, 0x16, 0x4c, 0x95, 0x32, 0x4d, 0x11, 0x83, - 0x5b, 0x2f, 0xdd, 0x7a, 0x81, 0xda, 0x32, 0x6d, 0xeb, 0x44, 0x91, 0x26, 0x11, 0x72, 0x75, 0x67, - 0xd1, 0x34, 0x26, 0x8a, 0x25, 0x48, 0x3d, 0x21, 0x27, 0xc2, 0x26, 0xe7, 0x2f, 0xbe, 0x04, 0x19, - 0x66, 0x47, 0x25, 0xb9, 0x89, 0xea, 0x69, 0x85, 0x2f, 0x3e, 0x4c, 0xde, 0x41, 0xf2, 0xd7, 0x08, - 0xf2, 0x0a, 0xd1, 0xc6, 0xee, 0x91, 0x36, 0x60, 0xe9, 0x8b, 0x39, 0x37, 0x36, 0x92, 0xb5, 0x9f, - 0xcc, 0x89, 0xe5, 0x9e, 0xbc, 0xe2, 0x0a, 0xe1, 0xc7, 0x50, 0xd6, 0x46, 0x23, 0x32, 0xb3, 0xc9, - 0x58, 0xb5, 0x44, 0xa8, 0x55, 0xfb, 0x64, 0x26, 0x9c, 0x2d, 0xde, 0xdc, 0x74, 0xf5, 0x03, 0xbb, - 0x34, 0xdc, 0x43, 0x19, 0x9c, 0xcc, 0x88, 0xb2, 0xee, 0x02, 0x04, 0xa9, 0x54, 0xbe, 0x0d, 0x85, - 0x20, 0x01, 0xe7, 0x61, 0xa9, 0xdf, 0xec, 0x3c, 0x7a, 0xd8, 0xea, 0x4b, 0x09, 0x5c, 0x86, 0x52, - 0x7f, 0xa0, 0xb4, 0x9a, 0x9d, 0xd6, 0xae, 0xfa, 0xb8, 0xa7, 0xa8, 0x3b, 0x7b, 0xfb, 0xdd, 0x07, - 0x7d, 0x09, 0xc9, 0x1f, 0x3b, 0x5a, 0x9a, 0x07, 0x85, 0xb7, 0x60, 0xc9, 0x22, 0x74, 0x3e, 0xb1, - 0x5d, 0x7f, 0xd6, 0x23, 0xfe, 0x70, 0x39, 0xc5, 0x95, 0x92, 0x4f, 0x00, 0xf7, 0x6d, 0x8b, 0x68, - 0x46, 0x08, 0x66, 0x1b, 0x8a, 0xa3, 0xa3, 0xb9, 0xf9, 0x84, 0x8c, 0xdd, 0xa3, 0xe4, 0x68, 0x57, - 0x5c, 0x34, 0xae, 0xb3, 0xc3, 0x65, 0xf8, 0x61, 0x28, 0x2b, 0xa3, 0xe0, 0xd2, 0xa9, 0x16, 0x27, - 0x6a, 0x27, 0xaa, 0x6e, 0x8e, 0xc9, 0x31, 0x3b, 0x8a, 0x94, 0x02, 0x8c, 0xd4, 0x76, 0x28, 0xf2, - 0x5f, 0x11, 0x94, 0x62, 0x70, 0xf0, 0x01, 0x64, 0xd9, 0xe1, 0x47, 0x4b, 0x7f, 0x36, 0xe4, 0xb9, - 0xf2, 0x48, 0xd3, 0xad, 0xed, 0x0f, 0xbe, 0x7a, 0xb6, 0x91, 0xf8, 0xe7, 0xb3, 0x8d, 0x1b, 0x17, - 0xe9, 0x63, 0x5c, 0xaf, 0x39, 0xd6, 0x66, 0x36, 0xb1, 0x14, 0x81, 0x8e, 0x6f, 0x40, 0x96, 0x59, - 0xec, 0xe6, 0x69, 0x29, 0xc6, 0xb9, 0xed, 0xb4, 0xb3, 0x8f, 0x22, 0x04, 0xe5, 0x3f, 0x24, 0x21, - 0x1f, 0xe0, 0xe2, 0x1a, 0xe4, 0x0d, 0xdd, 0x54, 0x6d, 0xdd, 0x20, 0x2a, 0x2b, 0x35, 0xc7, 0xc7, - 0x9c, 0xa1, 0x9b, 0x03, 0xdd, 0x20, 0x1d, 0xca, 0xf8, 0xda, 0xb1, 0xc7, 0x4f, 0x0a, 0xbe, 0x76, - 0x2c, 0xf8, 0xd7, 0x21, 0xed, 0x24, 0x8f, 0x28, 0xfb, 0x1f, 0xc4, 0x18, 0xd0, 0x68, 0x99, 0xa3, - 0xe9, 0x58, 0x37, 0x0f, 0x15, 0x26, 0x89, 0x1f, 0x41, 0x7a, 0xac, 0xd9, 0x5a, 0x25, 0xbd, 0x89, - 0xea, 0x85, 0xed, 0x8f, 0x44, 0x14, 0x6e, 0x5f, 0x28, 0x0a, 0xfb, 0x26, 0xd5, 0x0e, 0xc8, 0xf6, - 0x89, 0x4d, 0xfa, 0x13, 0x7d, 0x44, 0x14, 0x86, 0x24, 0xef, 0xc2, 0xb2, 0xbb, 0x87, 0x93, 0x74, - 0xfb, 0xdd, 0x07, 0xdd, 0xde, 0x67, 0x5d, 0x29, 0x81, 0x97, 0x20, 0xf5, 0xb8, 0xa7, 0x48, 0x08, - 0xaf, 0x40, 0x6e, 0xaf, 0xdd, 0x1f, 0xf4, 0xee, 0x29, 0xcd, 0x8e, 0x94, 0xc4, 0x25, 0x58, 0xbd, - 0xfb, 0xb0, 0xd7, 0x1c, 0xa8, 0x3e, 0x31, 0x25, 0xff, 0x1b, 0x41, 0x21, 0x58, 0x32, 0xf8, 0x3d, - 0xc0, 0xd4, 0xd6, 0x2c, 0x9b, 0x39, 0x4f, 0x6d, 0xcd, 0x98, 0xf9, 0x11, 0x92, 0x18, 0x67, 0xe0, - 0x32, 0x3a, 0x14, 0xd7, 0x41, 0x22, 0xe6, 0x38, 0x2c, 0xcb, 0xa3, 0x55, 0x24, 0xe6, 0x38, 0x28, - 0x19, 0xec, 0xb1, 0xa9, 0x0b, 0xf5, 0xd8, 0x9f, 0xc3, 0x15, 0xca, 0x02, 0xaa, 0x9b, 0x87, 0x2a, - 0x3f, 0x48, 0x75, 0xe8, 0x30, 0x55, 0xaa, 0x7f, 0x49, 0x2a, 0x63, 0xd6, 0x23, 0x2a, 0x9e, 0x08, - 0x0b, 0x3b, 0xdd, 0x76, 0x04, 0xfa, 0xfa, 0x97, 0xe4, 0x7e, 0x7a, 0x39, 0x2d, 0x65, 0x94, 0xcc, - 0x91, 0x6e, 0xda, 0x54, 0xfe, 0x13, 0x82, 0x4b, 0xad, 0x63, 0x62, 0xcc, 0x26, 0x9a, 0xf5, 0x46, - 0xdc, 0xbd, 0xb1, 0xe0, 0xee, 0x7a, 0x9c, 0xbb, 0x34, 0x70, 0xb1, 0x3e, 0x80, 0x95, 0x50, 0xb1, - 0xe3, 0x0f, 0x01, 0xd8, 0x4e, 0x71, 0x7d, 0x6e, 0x36, 0x6c, 0x38, 0xdb, 0xf1, 0xd2, 0x13, 0xd9, - 0x1e, 0x90, 0x96, 0xff, 0x97, 0x84, 0x12, 0x43, 0x73, 0xbb, 0x84, 0xc0, 0xfc, 0x18, 0xf2, 0x3c, - 0x94, 0x41, 0xd0, 0xb2, 0x6b, 0x9a, 0x0f, 0x19, 0xac, 0xa2, 0xa0, 0x46, 0xc4, 0xa8, 0xe4, 0x77, - 0x31, 0x0a, 0xdf, 0x07, 0xc9, 0x3f, 0x51, 0x81, 0xc0, 0x83, 0xf3, 0x56, 0xa8, 0xdd, 0x71, 0x9b, - 0x43, 0x30, 0xab, 0x9e, 0xa2, 0xe8, 0x36, 0xb7, 0xa1, 0xac, 0x53, 0xd5, 0x39, 0x8d, 0xe9, 0x81, - 0xc0, 0x52, 0xb9, 0x0c, 0xab, 0xb1, 0x65, 0xa5, 0xa4, 0xd3, 0x96, 0x39, 0xee, 0x1d, 0x70, 0x79, - 0x0e, 0x89, 0x3f, 0x87, 0x72, 0xd4, 0x02, 0x91, 0x5a, 0x95, 0x0c, 0x33, 0x64, 0xe3, 0x4c, 0x43, - 0x44, 0x7e, 0x71, 0x73, 0xd6, 0x23, 0xe6, 0x70, 0xa6, 0xfc, 0x6b, 0x58, 0x5b, 0xd0, 0x7b, 0x53, - 0x7d, 0x51, 0xd6, 0xa1, 0x7c, 0x86, 0xd1, 0xf8, 0x6d, 0x28, 0x08, 0x67, 0x79, 0x53, 0x47, 0xac, - 0x76, 0xf2, 0x9c, 0xc6, 0xba, 0x3a, 0xfe, 0x49, 0xa4, 0xab, 0xae, 0x78, 0xb3, 0x4c, 0x4c, 0x3f, - 0xed, 0xc3, 0x7a, 0xa4, 0x9a, 0x5e, 0x41, 0xca, 0xfe, 0x03, 0x01, 0x0e, 0x4e, 0x89, 0xa2, 0x42, - 0xcf, 0x99, 0x60, 0xe2, 0x0b, 0x38, 0xf9, 0x1d, 0x0a, 0x38, 0x75, 0x6e, 0x01, 0x3b, 0x09, 0x75, - 0x81, 0x02, 0xbe, 0x03, 0xa5, 0x90, 0xfd, 0x22, 0x26, 0x6f, 0x43, 0x21, 0x30, 0x63, 0xb9, 0xf3, - 0x67, 0xde, 0x1f, 0x94, 0xa8, 0xfc, 0x47, 0x04, 0x6b, 0xfe, 0x50, 0xfd, 0x66, 0x7b, 0xd3, 0x85, - 0x5c, 0xfb, 0xa9, 0x38, 0x1a, 0x61, 0x9f, 0xf0, 0xec, 0xbc, 0xc1, 0x5a, 0xbe, 0x0f, 0xd2, 0x3e, - 0x25, 0x56, 0xdf, 0xd6, 0x6c, 0xcf, 0xab, 0xe8, 0xe8, 0x8c, 0x2e, 0x38, 0x3a, 0xff, 0x1d, 0xc1, - 0x5a, 0x00, 0x4c, 0x98, 0x70, 0xcd, 0xfd, 0xb0, 0xd2, 0xa7, 0xa6, 0x6a, 0x69, 0x36, 0xcf, 0x10, - 0xa4, 0xac, 0x78, 0x54, 0x45, 0xb3, 0x89, 0x93, 0x44, 0xe6, 0xdc, 0xf0, 0xe7, 0x5b, 0x27, 0xfd, - 0x73, 0xe6, 0xdc, 0x2d, 0xd1, 0xf7, 0x00, 0x6b, 0x33, 0x5d, 0x8d, 0x20, 0xa5, 0x18, 0x92, 0xa4, - 0xcd, 0xf4, 0x76, 0x08, 0xac, 0x01, 0x25, 0x6b, 0x3e, 0x21, 0x51, 0xf1, 0x34, 0x13, 0x5f, 0x73, - 0x58, 0x21, 0x79, 0xf9, 0x73, 0x28, 0x39, 0x86, 0xb7, 0x77, 0xc3, 0xa6, 0x97, 0x61, 0x69, 0x4e, - 0x89, 0xa5, 0xea, 0x63, 0x91, 0xd5, 0x59, 0x67, 0xd9, 0x1e, 0xe3, 0xf7, 0xc5, 0xac, 0x90, 0x64, - 0x67, 0xe3, 0xb5, 0xc6, 0x05, 0xe7, 0xc5, 0x20, 0x70, 0x0f, 0xb0, 0xc3, 0xa2, 0x61, 0xf4, 0x1b, - 0x90, 0xa1, 0x0e, 0x21, 0x3a, 0x01, 0xc6, 0x58, 0xa2, 0x70, 0x49, 0xf9, 0x6f, 0x08, 0x6a, 0x1d, - 0x62, 0x5b, 0xfa, 0x88, 0xde, 0x9d, 0x5a, 0xe1, 0x54, 0x78, 0xcd, 0x29, 0x79, 0x07, 0x0a, 0x6e, - 0xae, 0xa9, 0x94, 0xd8, 0x2f, 0xbf, 0x32, 0xf3, 0xae, 0x68, 0x9f, 0xd8, 0xf2, 0x03, 0xd8, 0x38, - 0xd3, 0x66, 0x11, 0x8a, 0x3a, 0x64, 0x0d, 0x26, 0x22, 0x62, 0x21, 0xf9, 0x0d, 0x89, 0xab, 0x2a, - 0x82, 0x2f, 0x57, 0xe0, 0xb2, 0x00, 0xeb, 0x10, 0x5b, 0x73, 0xa2, 0x2b, 0x1c, 0x97, 0x7b, 0x50, - 0x5e, 0xe0, 0x08, 0xf8, 0xdb, 0xb0, 0x6c, 0x08, 0x9a, 0xd8, 0xa0, 0x12, 0xdd, 0xc0, 0xd3, 0xf1, - 0x24, 0xe5, 0xff, 0x22, 0x58, 0x8d, 0x5c, 0xb7, 0x4e, 0xbc, 0x0e, 0xac, 0xa9, 0xa1, 0xba, 0x4f, - 0x05, 0x7e, 0x6a, 0x14, 0x1d, 0x7a, 0x5b, 0x90, 0xdb, 0xe3, 0x60, 0xee, 0x24, 0x43, 0xb9, 0xe3, - 0x5f, 0x36, 0xa9, 0xd7, 0x3a, 0x84, 0xfb, 0xd7, 0x45, 0xfa, 0xfc, 0xeb, 0xe2, 0x6b, 0x04, 0x19, - 0xee, 0xe1, 0xeb, 0xca, 0x9f, 0x2a, 0x2c, 0x13, 0x31, 0x0c, 0xb3, 0xb2, 0xcd, 0x28, 0xde, 0xfa, - 0x35, 0x8c, 0xde, 0x4d, 0x58, 0x09, 0x65, 0xda, 0xf7, 0x78, 0x45, 0x51, 0xa1, 0x10, 0xe4, 0xe0, - 0x6b, 0xe2, 0x8b, 0x82, 0x77, 0xc3, 0x35, 0x57, 0x9b, 0xb1, 0xd9, 0xe7, 0x27, 0xff, 0x8c, 0xc0, - 0x90, 0x66, 0xd7, 0x20, 0x3f, 0x74, 0xf6, 0xdf, 0xff, 0x6a, 0x4e, 0x31, 0x22, 0x5f, 0xc8, 0xbf, - 0x45, 0x50, 0xf4, 0xf3, 0xeb, 0xae, 0x3e, 0x21, 0xaf, 0x22, 0xbd, 0xaa, 0xb0, 0x7c, 0xa0, 0x4f, - 0x08, 0xb3, 0x81, 0x6f, 0xe7, 0xad, 0x1d, 0xdb, 0xfc, 0x38, 0xf3, 0x48, 0xbd, 0x5b, 0x87, 0x7c, - 0xa0, 0xa1, 0x3b, 0x5f, 0x24, 0xed, 0xae, 0xda, 0x69, 0x75, 0x7a, 0xca, 0x2f, 0xa4, 0x04, 0x06, - 0xc8, 0x36, 0x77, 0x06, 0xed, 0x4f, 0x5b, 0x12, 0x7a, 0xf7, 0x3e, 0xe4, 0x3c, 0x67, 0x71, 0x0e, - 0x32, 0xad, 0x4f, 0xf6, 0x9b, 0x0f, 0xa5, 0x84, 0xa3, 0xd2, 0xed, 0x0d, 0x54, 0xbe, 0x44, 0x78, - 0x15, 0xf2, 0x4a, 0xeb, 0x5e, 0xeb, 0xb1, 0xda, 0x69, 0x0e, 0x76, 0xf6, 0xa4, 0x24, 0xc6, 0x50, - 0xe4, 0x84, 0x6e, 0x4f, 0xd0, 0x52, 0x37, 0x7f, 0xb7, 0x04, 0xcb, 0xae, 0x37, 0xf8, 0x03, 0x48, - 0x3f, 0x9a, 0xd3, 0x23, 0x7c, 0xd9, 0xaf, 0x84, 0xcf, 0x2c, 0xdd, 0x26, 0xa2, 0xb2, 0xab, 0xe5, - 0x05, 0x3a, 0xaf, 0x6b, 0x39, 0x81, 0x77, 0x21, 0x1f, 0x98, 0xa8, 0x70, 0xec, 0x1b, 0x43, 0xf5, - 0x4a, 0xcc, 0xc4, 0xe8, 0x63, 0x5c, 0x47, 0xb8, 0x07, 0x45, 0xc6, 0x72, 0x27, 0x26, 0x8a, 0xbd, - 0x0f, 0xc6, 0xb8, 0x4f, 0x92, 0xea, 0xd5, 0x33, 0xb8, 0x9e, 0x59, 0x7b, 0xe1, 0x77, 0xb3, 0x6a, - 0xdc, 0x13, 0x5b, 0xd4, 0xb8, 0x98, 0xc1, 0x44, 0x4e, 0xe0, 0x16, 0x80, 0x7f, 0xad, 0xe3, 0xb7, - 0x42, 0xc2, 0xc1, 0x51, 0xa4, 0x5a, 0x8d, 0x63, 0x79, 0x30, 0xdb, 0x90, 0xf3, 0x2e, 0x27, 0x5c, - 0x89, 0xb9, 0xaf, 0x38, 0xc8, 0xd9, 0x37, 0x99, 0x9c, 0xc0, 0x77, 0xa1, 0xd0, 0x9c, 0x4c, 0x2e, - 0x02, 0x53, 0x0d, 0x72, 0x68, 0x14, 0x67, 0xe2, 0x35, 0xea, 0xe8, 0x7d, 0x80, 0xdf, 0xf1, 0xaa, - 0xea, 0xa5, 0x97, 0x5c, 0xf5, 0x47, 0xe7, 0xca, 0x79, 0xbb, 0x0d, 0x60, 0x35, 0x72, 0x2d, 0xe0, - 0x5a, 0x44, 0x3b, 0x72, 0x93, 0x54, 0x37, 0xce, 0xe4, 0x7b, 0xa8, 0x43, 0x31, 0x48, 0x86, 0x9f, - 0x58, 0xb1, 0xbc, 0x78, 0x08, 0xd1, 0xf7, 0xdc, 0xea, 0x0f, 0x5f, 0x2a, 0x13, 0xc8, 0xca, 0x27, - 0x70, 0x39, 0xfe, 0x25, 0x12, 0x5f, 0x8b, 0xc9, 0x99, 0xc5, 0x57, 0xd5, 0xea, 0x3b, 0xe7, 0x89, - 0xf9, 0x9b, 0x6d, 0x7f, 0xf4, 0xf4, 0x79, 0x2d, 0xf1, 0xcd, 0xf3, 0x5a, 0xe2, 0xdb, 0xe7, 0x35, - 0xf4, 0x9b, 0xd3, 0x1a, 0xfa, 0xf3, 0x69, 0x0d, 0x7d, 0x75, 0x5a, 0x43, 0x4f, 0x4f, 0x6b, 0xe8, - 0x5f, 0xa7, 0x35, 0xf4, 0x9f, 0xd3, 0x5a, 0xe2, 0xdb, 0xd3, 0x1a, 0xfa, 0xfd, 0x8b, 0x5a, 0xe2, - 0xe9, 0x8b, 0x5a, 0xe2, 0x9b, 0x17, 0xb5, 0xc4, 0x2f, 0xb3, 0xa3, 0x89, 0x4e, 0x4c, 0x7b, 0x98, - 0x65, 0x0f, 0xd9, 0xb7, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0xd1, 0x8b, 0x35, 0x44, 0x43, 0x17, - 0x00, 0x00, + 0x97, 0x1a, 0x5c, 0xb3, 0x06, 0xdb, 0xd5, 0xe1, 0x4c, 0xc5, 0x93, 0x92, 0xf7, 0xe0, 0x4a, 0x2c, + 0x1e, 0x9d, 0x4d, 0x4d, 0x4a, 0xf0, 0x8f, 0x21, 0xa3, 0xdb, 0xc4, 0x70, 0xd1, 0x4a, 0x21, 0x34, + 0x21, 0xcb, 0x25, 0xe4, 0x5d, 0xc8, 0x07, 0xa8, 0xf8, 0x2a, 0xc0, 0xc4, 0x59, 0xaa, 0xa6, 0x66, + 0x90, 0x0a, 0xda, 0x44, 0xf5, 0x9c, 0x92, 0x9b, 0xb8, 0x47, 0xe1, 0xcb, 0x90, 0xfd, 0x15, 0x13, + 0xac, 0x24, 0x37, 0x53, 0xf5, 0x9c, 0x22, 0x56, 0xf2, 0x5f, 0x10, 0x5c, 0x0d, 0xc0, 0xec, 0x68, + 0xd6, 0x58, 0x37, 0xb5, 0x89, 0x6e, 0x9f, 0xb8, 0x36, 0x6e, 0x40, 0xde, 0x07, 0xe6, 0x8a, 0xe5, + 0x14, 0xf0, 0x90, 0x69, 0xc8, 0x09, 0xc9, 0x8b, 0x38, 0x01, 0xff, 0x0c, 0x0a, 0xa3, 0xe9, 0xdc, + 0xb4, 0x55, 0x83, 0xd8, 0x47, 0xd3, 0x71, 0x25, 0xb5, 0x89, 0xea, 0x45, 0xdf, 0xd8, 0x1d, 0x87, + 0xd7, 0x61, 0x2c, 0x25, 0x3f, 0xf2, 0x17, 0xf2, 0x3e, 0xd4, 0xce, 0xd2, 0x55, 0xf8, 0xef, 0x56, + 0xd8, 0x7f, 0x57, 0x17, 0xfd, 0xd7, 0x27, 0x96, 0x4e, 0x28, 0x3b, 0xc2, 0xf5, 0xe4, 0x33, 0x04, + 0xeb, 0xb1, 0x02, 0xe7, 0x39, 0x55, 0x03, 0xcc, 0xd9, 0xcc, 0x99, 0x2a, 0x65, 0x3b, 0x85, 0x0f, + 0x6e, 0xbd, 0xf4, 0xe8, 0x05, 0x6a, 0xcb, 0xb4, 0xad, 0x13, 0x45, 0x9a, 0x44, 0xc8, 0xd5, 0x9d, + 0x45, 0xd5, 0x98, 0x28, 0x96, 0x20, 0xf5, 0x84, 0x9c, 0x08, 0x9d, 0x9c, 0xbf, 0xf8, 0x12, 0x64, + 0x98, 0x1e, 0x95, 0xe4, 0x26, 0xaa, 0xa7, 0x15, 0xbe, 0xf8, 0x30, 0x79, 0x07, 0xc9, 0x5f, 0x23, + 0xc8, 0x2b, 0x44, 0x1b, 0xbb, 0x57, 0xda, 0x80, 0xa5, 0x2f, 0xe6, 0x5c, 0xd9, 0x48, 0xd4, 0x7e, + 0x32, 0x27, 0x96, 0x7b, 0xf3, 0x8a, 0x2b, 0x84, 0x1f, 0x43, 0x59, 0x1b, 0x8d, 0xc8, 0xcc, 0x26, + 0x63, 0xd5, 0x12, 0xae, 0x56, 0xed, 0x93, 0x99, 0x30, 0xb6, 0x78, 0x73, 0xd3, 0xdd, 0x1f, 0x38, + 0xa5, 0xe1, 0x5e, 0xca, 0xe0, 0x64, 0x46, 0x94, 0x75, 0x17, 0x20, 0x48, 0xa5, 0xf2, 0x6d, 0x28, + 0x04, 0x09, 0x38, 0x0f, 0x4b, 0xfd, 0x66, 0xe7, 0xd1, 0xc3, 0x56, 0x5f, 0x4a, 0xe0, 0x32, 0x94, + 0xfa, 0x03, 0xa5, 0xd5, 0xec, 0xb4, 0x76, 0xd5, 0xc7, 0x3d, 0x45, 0xdd, 0xd9, 0xdb, 0xef, 0x3e, + 0xe8, 0x4b, 0x48, 0xfe, 0xd8, 0xd9, 0xa5, 0x79, 0x50, 0x78, 0x0b, 0x96, 0x2c, 0x42, 0xe7, 0x13, + 0xdb, 0xb5, 0x67, 0x3d, 0x62, 0x0f, 0x97, 0x53, 0x5c, 0x29, 0xf9, 0x04, 0x70, 0xdf, 0xb6, 0x88, + 0x66, 0x84, 0x60, 0xb6, 0xa1, 0x38, 0x3a, 0x9a, 0x9b, 0x4f, 0xc8, 0xd8, 0xbd, 0x4a, 0x8e, 0x76, + 0xc5, 0x45, 0xe3, 0x7b, 0x76, 0xb8, 0x0c, 0xbf, 0x0c, 0x65, 0x65, 0x14, 0x5c, 0x3a, 0xd9, 0xe2, + 0x78, 0xed, 0x44, 0xd5, 0xcd, 0x31, 0x39, 0x66, 0x57, 0x91, 0x52, 0x80, 0x91, 0xda, 0x0e, 0x45, + 0xfe, 0x2b, 0x82, 0x52, 0x0c, 0x0e, 0x3e, 0x80, 0x2c, 0xbb, 0xfc, 0x68, 0xea, 0xcf, 0x86, 0x3c, + 0x56, 0x1e, 0x69, 0xba, 0xb5, 0xfd, 0xc1, 0x57, 0xcf, 0x36, 0x12, 0xff, 0x7c, 0xb6, 0x71, 0xe3, + 0x22, 0x75, 0x8c, 0xef, 0x6b, 0x8e, 0xb5, 0x99, 0x4d, 0x2c, 0x45, 0xa0, 0xe3, 0x1b, 0x90, 0x65, + 0x1a, 0xbb, 0x71, 0x5a, 0x8a, 0x31, 0x6e, 0x3b, 0xed, 0x9c, 0xa3, 0x08, 0x41, 0xf9, 0xf7, 0x49, + 0xc8, 0x07, 0xb8, 0xb8, 0x06, 0x79, 0x43, 0x37, 0x55, 0x5b, 0x37, 0x88, 0xca, 0x52, 0xcd, 0xb1, + 0x31, 0x67, 0xe8, 0xe6, 0x40, 0x37, 0x48, 0x87, 0x32, 0xbe, 0x76, 0xec, 0xf1, 0x93, 0x82, 0xaf, + 0x1d, 0x0b, 0xfe, 0x75, 0x48, 0x3b, 0xc1, 0x23, 0xd2, 0xfe, 0x07, 0x31, 0x0a, 0x34, 0x5a, 0xe6, + 0x68, 0x3a, 0xd6, 0xcd, 0x43, 0x85, 0x49, 0xe2, 0x47, 0x90, 0x1e, 0x6b, 0xb6, 0x56, 0x49, 0x6f, + 0xa2, 0x7a, 0x61, 0xfb, 0x23, 0xe1, 0x85, 0xdb, 0x17, 0xf2, 0xc2, 0xbe, 0x49, 0xb5, 0x03, 0xb2, + 0x7d, 0x62, 0x93, 0xfe, 0x44, 0x1f, 0x11, 0x85, 0x21, 0xc9, 0xbb, 0xb0, 0xec, 0x9e, 0xe1, 0x04, + 0xdd, 0x7e, 0xf7, 0x41, 0xb7, 0xf7, 0x59, 0x57, 0x4a, 0xe0, 0x25, 0x48, 0x3d, 0xee, 0x29, 0x12, + 0xc2, 0x2b, 0x90, 0xdb, 0x6b, 0xf7, 0x07, 0xbd, 0x7b, 0x4a, 0xb3, 0x23, 0x25, 0x71, 0x09, 0x56, + 0xef, 0x3e, 0xec, 0x35, 0x07, 0xaa, 0x4f, 0x4c, 0xc9, 0xff, 0x46, 0x50, 0x08, 0xa6, 0x0c, 0x7e, + 0x0f, 0x30, 0xb5, 0x35, 0xcb, 0x66, 0xc6, 0x53, 0x5b, 0x33, 0x66, 0xbe, 0x87, 0x24, 0xc6, 0x19, + 0xb8, 0x8c, 0x0e, 0xc5, 0x75, 0x90, 0x88, 0x39, 0x0e, 0xcb, 0x72, 0x6f, 0x15, 0x89, 0x39, 0x0e, + 0x4a, 0x06, 0x6b, 0x6c, 0xea, 0x42, 0x35, 0xf6, 0xe7, 0x70, 0x85, 0x32, 0x87, 0xea, 0xe6, 0xa1, + 0xca, 0x2f, 0x52, 0x1d, 0x3a, 0x4c, 0x95, 0xea, 0x5f, 0x92, 0xca, 0x98, 0xd5, 0x88, 0x8a, 0x27, + 0xc2, 0xdc, 0x4e, 0xb7, 0x1d, 0x81, 0xbe, 0xfe, 0x25, 0xb9, 0x9f, 0x5e, 0x4e, 0x4b, 0x19, 0x25, + 0x73, 0xa4, 0x9b, 0x36, 0x95, 0xff, 0x84, 0xe0, 0x52, 0xeb, 0x98, 0x18, 0xb3, 0x89, 0x66, 0xbd, + 0x11, 0x73, 0x6f, 0x2c, 0x98, 0xbb, 0x1e, 0x67, 0x2e, 0x0d, 0x34, 0xd6, 0x07, 0xb0, 0x12, 0x4a, + 0x76, 0xfc, 0x21, 0x00, 0x3b, 0x29, 0xae, 0xce, 0xcd, 0x86, 0x0d, 0xe7, 0x38, 0x9e, 0x7a, 0x22, + 0xda, 0x03, 0xd2, 0xf2, 0xff, 0x92, 0x50, 0x62, 0x68, 0x6e, 0x95, 0x10, 0x98, 0x1f, 0x43, 0x9e, + 0xbb, 0x32, 0x08, 0x5a, 0x76, 0x55, 0xf3, 0x21, 0x83, 0x59, 0x14, 0xdc, 0x11, 0x51, 0x2a, 0xf9, + 0x5d, 0x94, 0xc2, 0xf7, 0x41, 0xf2, 0x6f, 0x54, 0x20, 0x70, 0xe7, 0xbc, 0x15, 0x2a, 0x77, 0x5c, + 0xe7, 0x10, 0xcc, 0xaa, 0xb7, 0x51, 0x54, 0x9b, 0xdb, 0x50, 0xd6, 0xa9, 0xea, 0xdc, 0xc6, 0xf4, + 0x40, 0x60, 0xa9, 0x5c, 0x86, 0xe5, 0xd8, 0xb2, 0x52, 0xd2, 0x69, 0xcb, 0x1c, 0xf7, 0x0e, 0xb8, + 0x3c, 0x87, 0xc4, 0x9f, 0x43, 0x39, 0xaa, 0x81, 0x08, 0xad, 0x4a, 0x86, 0x29, 0xb2, 0x71, 0xa6, + 0x22, 0x22, 0xbe, 0xb8, 0x3a, 0xeb, 0x11, 0x75, 0x38, 0x53, 0xfe, 0x03, 0x82, 0xb5, 0x85, 0x8d, + 0x6f, 0xac, 0x30, 0x6e, 0x88, 0xbb, 0x55, 0xd9, 0xc4, 0xe1, 0x56, 0x6e, 0x46, 0x62, 0x2d, 0x5b, + 0xd6, 0xa1, 0x7c, 0x86, 0x59, 0xf8, 0x6d, 0x28, 0x08, 0x77, 0xf0, 0xb2, 0x8f, 0x58, 0x76, 0xe5, + 0x39, 0x8d, 0xd5, 0x7d, 0xfc, 0x93, 0x48, 0xdd, 0x5d, 0xf1, 0xa6, 0x9d, 0x98, 0x8a, 0xdb, 0x87, + 0xf5, 0x48, 0xbe, 0xbd, 0x82, 0xa0, 0xfe, 0x07, 0x02, 0x1c, 0x9c, 0x23, 0x45, 0x0e, 0x9f, 0x33, + 0xe3, 0xc4, 0xa7, 0x78, 0xf2, 0x3b, 0xa4, 0x78, 0xea, 0xdc, 0x14, 0x77, 0x42, 0xee, 0x02, 0x29, + 0x7e, 0x07, 0x4a, 0x21, 0xfd, 0x85, 0x4f, 0xde, 0x86, 0x42, 0x60, 0x0a, 0x73, 0x27, 0xd4, 0xbc, + 0x3f, 0x4a, 0x51, 0xf9, 0x8f, 0x08, 0xd6, 0xfc, 0xb1, 0xfb, 0xcd, 0x56, 0xaf, 0x0b, 0x99, 0xf6, + 0x53, 0x71, 0x35, 0x42, 0x3f, 0x61, 0xd9, 0x79, 0xa3, 0xb7, 0x7c, 0x1f, 0xa4, 0x7d, 0x4a, 0xac, + 0xbe, 0xad, 0xd9, 0x9e, 0x55, 0xd1, 0xe1, 0x1a, 0x5d, 0x70, 0xb8, 0xfe, 0x3b, 0x82, 0xb5, 0x00, + 0x98, 0x50, 0xe1, 0x9a, 0xfb, 0xe9, 0xa5, 0x4f, 0x4d, 0xd5, 0xd2, 0x6c, 0x1e, 0x21, 0x48, 0x59, + 0xf1, 0xa8, 0x8a, 0x66, 0x13, 0x27, 0x88, 0xcc, 0xb9, 0xe1, 0x4f, 0xc0, 0x4e, 0xf8, 0xe7, 0xcc, + 0xb9, 0x9b, 0xc3, 0xef, 0x01, 0xd6, 0x66, 0xba, 0x1a, 0x41, 0x4a, 0x31, 0x24, 0x49, 0x9b, 0xe9, + 0xed, 0x10, 0x58, 0x03, 0x4a, 0xd6, 0x7c, 0x42, 0xa2, 0xe2, 0x69, 0x26, 0xbe, 0xe6, 0xb0, 0x42, + 0xf2, 0xf2, 0xe7, 0x50, 0x72, 0x14, 0x6f, 0xef, 0x86, 0x55, 0x2f, 0xc3, 0xd2, 0x9c, 0x12, 0x4b, + 0xd5, 0xc7, 0x22, 0xaa, 0xb3, 0xce, 0xb2, 0x3d, 0xc6, 0xef, 0x8b, 0x69, 0x22, 0xc9, 0xee, 0xc6, + 0x2b, 0x9e, 0x0b, 0xc6, 0x8b, 0x51, 0xe1, 0x1e, 0x60, 0x87, 0x45, 0xc3, 0xe8, 0x37, 0x20, 0x43, + 0x1d, 0x42, 0x74, 0x46, 0x8c, 0xd1, 0x44, 0xe1, 0x92, 0xf2, 0xdf, 0x10, 0xd4, 0x3a, 0xc4, 0xb6, + 0xf4, 0x11, 0xbd, 0x3b, 0xb5, 0xc2, 0xa1, 0xf0, 0x9a, 0x43, 0xf2, 0x0e, 0x14, 0xdc, 0x58, 0x53, + 0x29, 0xb1, 0x5f, 0xde, 0x54, 0xf3, 0xae, 0x68, 0x9f, 0xd8, 0xf2, 0x03, 0xd8, 0x38, 0x53, 0x67, + 0xe1, 0x8a, 0x3a, 0x64, 0x0d, 0x26, 0x22, 0x7c, 0x21, 0xf9, 0x05, 0x89, 0x6f, 0x55, 0x04, 0x5f, + 0xae, 0xc0, 0x65, 0x01, 0xd6, 0x21, 0xb6, 0xe6, 0x78, 0x57, 0x18, 0x2e, 0xf7, 0xa0, 0xbc, 0xc0, + 0x11, 0xf0, 0xb7, 0x61, 0xd9, 0x10, 0x34, 0x71, 0x40, 0x25, 0x7a, 0x80, 0xb7, 0xc7, 0x93, 0x94, + 0xff, 0x8b, 0x60, 0x35, 0xd2, 0x90, 0x1d, 0x7f, 0x1d, 0x58, 0x53, 0x43, 0x75, 0x1f, 0x13, 0xfc, + 0xd0, 0x28, 0x3a, 0xf4, 0xb6, 0x20, 0xb7, 0xc7, 0xc1, 0xd8, 0x49, 0x86, 0x62, 0xc7, 0xef, 0x46, + 0xa9, 0xd7, 0xda, 0x8d, 0xfc, 0x76, 0x91, 0x3e, 0xbf, 0x5d, 0x7c, 0x8d, 0x20, 0xc3, 0x2d, 0x7c, + 0x5d, 0xf1, 0x53, 0x85, 0x65, 0x22, 0xc6, 0x65, 0x96, 0xb6, 0x19, 0xc5, 0x5b, 0xbf, 0x86, 0xe1, + 0xbc, 0x09, 0x2b, 0xa1, 0x48, 0xfb, 0x1e, 0xef, 0x2c, 0x2a, 0x14, 0x82, 0x1c, 0x7c, 0x4d, 0x7c, + 0x73, 0xf0, 0x6a, 0xb8, 0xe6, 0xee, 0x66, 0x6c, 0xf6, 0x81, 0xca, 0x3f, 0x34, 0x30, 0xa4, 0x59, + 0x1b, 0xe4, 0x97, 0xce, 0xfe, 0xfb, 0xdf, 0xd5, 0x29, 0x46, 0xe4, 0x0b, 0xf9, 0x37, 0x08, 0x8a, + 0x7e, 0x7c, 0xdd, 0xd5, 0x27, 0xe4, 0x55, 0x84, 0x57, 0x15, 0x96, 0x0f, 0xf4, 0x09, 0x61, 0x3a, + 0xf0, 0xe3, 0xbc, 0xb5, 0xa3, 0x9b, 0xef, 0x67, 0xee, 0xa9, 0x77, 0xeb, 0x90, 0x0f, 0x14, 0x74, + 0xe7, 0x9b, 0xa5, 0xdd, 0x55, 0x3b, 0xad, 0x4e, 0x4f, 0xf9, 0x85, 0x94, 0xc0, 0x00, 0xd9, 0xe6, + 0xce, 0xa0, 0xfd, 0x69, 0x4b, 0x42, 0xef, 0xde, 0x87, 0x9c, 0x67, 0x2c, 0xce, 0x41, 0xa6, 0xf5, + 0xc9, 0x7e, 0xf3, 0xa1, 0x94, 0x70, 0xb6, 0x74, 0x7b, 0x03, 0x95, 0x2f, 0x11, 0x5e, 0x85, 0xbc, + 0xd2, 0xba, 0xd7, 0x7a, 0xac, 0x76, 0x9a, 0x83, 0x9d, 0x3d, 0x29, 0x89, 0x31, 0x14, 0x39, 0xa1, + 0xdb, 0x13, 0xb4, 0xd4, 0xcd, 0xdf, 0x2e, 0xc1, 0xb2, 0x6b, 0x0d, 0xfe, 0x00, 0xd2, 0x8f, 0xe6, + 0xf4, 0x08, 0x5f, 0xf6, 0x33, 0xe1, 0x33, 0x4b, 0xb7, 0x89, 0xc8, 0xec, 0x6a, 0x79, 0x81, 0xce, + 0xf3, 0x5a, 0x4e, 0xe0, 0x5d, 0xc8, 0x07, 0x26, 0x2a, 0x1c, 0xfb, 0x0a, 0x51, 0xbd, 0x12, 0x33, + 0x53, 0xfa, 0x18, 0xd7, 0x11, 0xee, 0x41, 0x91, 0xb1, 0xdc, 0x89, 0x89, 0x62, 0xef, 0x93, 0x32, + 0xee, 0xa3, 0xa5, 0x7a, 0xf5, 0x0c, 0xae, 0xa7, 0xd6, 0x5e, 0xf8, 0x65, 0xad, 0x1a, 0xf7, 0x08, + 0x17, 0x55, 0x2e, 0x66, 0x30, 0x91, 0x13, 0xb8, 0x05, 0xe0, 0xb7, 0x75, 0xfc, 0x56, 0x48, 0x38, + 0x38, 0x8a, 0x54, 0xab, 0x71, 0x2c, 0x0f, 0x66, 0x1b, 0x72, 0x5e, 0x73, 0xc2, 0x95, 0x98, 0x7e, + 0xc5, 0x41, 0xce, 0xee, 0x64, 0x72, 0x02, 0xdf, 0x85, 0x42, 0x73, 0x32, 0xb9, 0x08, 0x4c, 0x35, + 0xc8, 0xa1, 0x51, 0x9c, 0x89, 0x57, 0xa8, 0xa3, 0xfd, 0x00, 0xbf, 0xe3, 0x65, 0xd5, 0x4b, 0x9b, + 0x5c, 0xf5, 0x47, 0xe7, 0xca, 0x79, 0xa7, 0x0d, 0x60, 0x35, 0xd2, 0x16, 0x70, 0x2d, 0xb2, 0x3b, + 0xd2, 0x49, 0xaa, 0x1b, 0x67, 0xf2, 0x3d, 0xd4, 0xa1, 0x18, 0x24, 0xc3, 0x8f, 0xb0, 0x58, 0x5e, + 0xbc, 0x84, 0xe8, 0x8b, 0x6f, 0xf5, 0x87, 0x2f, 0x95, 0x09, 0x44, 0xe5, 0x13, 0xb8, 0x1c, 0xff, + 0x56, 0x89, 0xaf, 0xc5, 0xc4, 0xcc, 0xe2, 0xbb, 0x6b, 0xf5, 0x9d, 0xf3, 0xc4, 0xfc, 0xc3, 0xb6, + 0x3f, 0x7a, 0xfa, 0xbc, 0x96, 0xf8, 0xe6, 0x79, 0x2d, 0xf1, 0xed, 0xf3, 0x1a, 0xfa, 0xf5, 0x69, + 0x0d, 0xfd, 0xf9, 0xb4, 0x86, 0xbe, 0x3a, 0xad, 0xa1, 0xa7, 0xa7, 0x35, 0xf4, 0xaf, 0xd3, 0x1a, + 0xfa, 0xcf, 0x69, 0x2d, 0xf1, 0xed, 0x69, 0x0d, 0xfd, 0xee, 0x45, 0x2d, 0xf1, 0xf4, 0x45, 0x2d, + 0xf1, 0xcd, 0x8b, 0x5a, 0xe2, 0x97, 0xd9, 0xd1, 0x44, 0x27, 0xa6, 0x3d, 0xcc, 0xb2, 0xa7, 0xee, + 0x5b, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xae, 0xb9, 0xf1, 0x3c, 0x65, 0x17, 0x00, 0x00, } func (x CountMethod) String() string { @@ -2681,6 +2689,9 @@ func (this *QueryStreamSeries) Equal(that interface{}) bool { return false } } + if this.ChunkCount != that1.ChunkCount { + return false + } return true } func (this *QueryStreamSeriesChunks) Equal(that interface{}) bool { @@ -3499,9 +3510,10 @@ func (this *QueryStreamSeries) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 5) + s := make([]string, 0, 6) s = append(s, "&client.QueryStreamSeries{") s = append(s, "Labels: "+fmt.Sprintf("%#v", this.Labels)+",\n") + s = append(s, "ChunkCount: "+fmt.Sprintf("%#v", this.ChunkCount)+",\n") s = append(s, "}") return strings.Join(s, "") } @@ -5020,6 +5032,11 @@ func (m *QueryStreamSeries) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ChunkCount != 0 { + i = encodeVarintIngester(dAtA, i, uint64(m.ChunkCount)) + i-- + dAtA[i] = 0x10 + } if len(m.Labels) > 0 { for iNdEx := len(m.Labels) - 1; iNdEx >= 0; iNdEx-- { { @@ -6141,6 +6158,9 @@ func (m *QueryStreamSeries) Size() (n int) { n += 1 + l + sovIngester(uint64(l)) } } + if m.ChunkCount != 0 { + n += 1 + sovIngester(uint64(m.ChunkCount)) + } return n } @@ -6749,6 +6769,7 @@ func (this *QueryStreamSeries) String() string { } s := strings.Join([]string{`&QueryStreamSeries{`, `Labels:` + fmt.Sprintf("%v", this.Labels) + `,`, + `ChunkCount:` + fmt.Sprintf("%v", this.ChunkCount) + `,`, `}`, }, "") return s @@ -8968,6 +8989,25 @@ func (m *QueryStreamSeries) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ChunkCount", wireType) + } + m.ChunkCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowIngester + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ChunkCount |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipIngester(dAtA[iNdEx:]) diff --git a/pkg/ingester/client/ingester.proto b/pkg/ingester/client/ingester.proto index b7844e8c4c2..98abcceac9e 100644 --- a/pkg/ingester/client/ingester.proto +++ b/pkg/ingester/client/ingester.proto @@ -153,6 +153,7 @@ message QueryStreamResponse { message QueryStreamSeries { repeated cortexpb.LabelPair labels = 1 [(gogoproto.nullable) = false, (gogoproto.customtype) = "github.com/grafana/mimir/pkg/mimirpb.LabelAdapter"]; + int64 chunk_count = 2; } message QueryStreamSeriesChunks { diff --git a/pkg/ingester/client/streaming.go b/pkg/ingester/client/streaming.go index 7c1c4798c2c..dea7216cd7a 100644 --- a/pkg/ingester/client/streaming.go +++ b/pkg/ingester/client/streaming.go @@ -132,11 +132,7 @@ func (s *SeriesChunksStreamReader) StartBuffering() { } } - if err := s.queryLimiter.AddChunks(chunkCount); err != nil { - onError(err) - return - } - + // The chunk count limit is enforced earlier, while we're reading series labels, so we don't need to do that here. if err := s.queryLimiter.AddChunkBytes(chunkBytes); err != nil { onError(err) return diff --git a/pkg/ingester/client/streaming_test.go b/pkg/ingester/client/streaming_test.go index db33635190c..87a12ef29b3 100644 --- a/pkg/ingester/client/streaming_test.go +++ b/pkg/ingester/client/streaming_test.go @@ -243,7 +243,7 @@ func TestSeriesChunksStreamReader_ChunksLimits(t *testing.T) { "query selects too many chunks": { maxChunks: 2, maxChunkBytes: 200, - expectedError: "the query exceeded the maximum number of chunks (limit: 2 chunks) (err-mimir-max-chunks-per-query). Consider reducing the time range and/or number of series selected by the query. One way to reduce the number of selected series is to add more label matchers to the query. Otherwise, to adjust the related per-tenant limit, configure -querier.max-fetched-chunks-per-query, or contact your service administrator.", + expectedError: "", // The limit on the number of chunks is enforced earlier, in distributor.queryIngesterStream() }, "query selects too many chunk bytes": { maxChunks: 4, diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 019d62f0e6e..7c39126107e 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1814,8 +1814,14 @@ func (i *Ingester) sendStreamingQuerySeries(q storage.ChunkQuerier, from, throug lastSeriesNode.series = append(lastSeriesNode.series, series) seriesCount++ + chunkCount, err := series.ChunkCount() + if err != nil { + return nil, 0, err + } + seriesInBatch = append(seriesInBatch, client.QueryStreamSeries{ - Labels: mimirpb.FromLabelsToLabelAdapters(series.Labels()), + Labels: mimirpb.FromLabelsToLabelAdapters(series.Labels()), + ChunkCount: int64(chunkCount), }) if len(seriesInBatch) >= queryStreamBatchSize { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 438754e5f05..0fbb92b69a1 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -3122,9 +3122,9 @@ func TestIngester_QueryStream_StreamingWithManySamples(t *testing.T) { seriesLabelsMsg := client.QueryStreamResponse{ StreamingSeries: []client.QueryStreamSeries{ - {Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "foo", "l", "1"))}, - {Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "foo", "l", "2"))}, - {Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "foo", "l", "3"))}, + {Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "foo", "l", "1")), ChunkCount: 834}, + {Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "foo", "l", "2")), ChunkCount: 8334}, + {Labels: mimirpb.FromLabelsToLabelAdapters(labels.FromStrings(labels.MetricName, "foo", "l", "3")), ChunkCount: 4167}, }, IsEndOfSeriesStream: true, } diff --git a/vendor/github.com/prometheus/prometheus/storage/interface.go b/vendor/github.com/prometheus/prometheus/storage/interface.go index 2a440cc93ce..74ddc5acadd 100644 --- a/vendor/github.com/prometheus/prometheus/storage/interface.go +++ b/vendor/github.com/prometheus/prometheus/storage/interface.go @@ -415,6 +415,12 @@ type ChunkSeriesSet interface { type ChunkSeries interface { Labels ChunkIterable + + // ChunkCount returns the number of chunks available from this ChunkSeries. + // + // This value is used by Mimir's ingesters to report the number of chunks expected to be returned by a query, + // which is used by queriers to enforce the 'max chunks per query' limit. + ChunkCount() (int, error) } // Labels represents an item that has labels e.g. time series. diff --git a/vendor/github.com/prometheus/prometheus/storage/merge.go b/vendor/github.com/prometheus/prometheus/storage/merge.go index c0665d720bc..a196b0bc0dd 100644 --- a/vendor/github.com/prometheus/prometheus/storage/merge.go +++ b/vendor/github.com/prometheus/prometheus/storage/merge.go @@ -658,22 +658,42 @@ func NewCompactingChunkSeriesMerger(mergeFunc VerticalSeriesMergeFunc) VerticalC if len(series) == 0 { return nil } + + chunkIteratorFn := func(chunks.Iterator) chunks.Iterator { + iterators := make([]chunks.Iterator, 0, len(series)) + for _, s := range series { + iterators = append(iterators, s.Iterator(nil)) + } + return &compactChunkIterator{ + mergeFunc: mergeFunc, + iterators: iterators, + } + } + return &ChunkSeriesEntry{ - Lset: series[0].Labels(), - ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator { - iterators := make([]chunks.Iterator, 0, len(series)) - for _, s := range series { - iterators = append(iterators, s.Iterator(nil)) - } - return &compactChunkIterator{ - mergeFunc: mergeFunc, - iterators: iterators, - } + Lset: series[0].Labels(), + ChunkIteratorFn: chunkIteratorFn, + ChunkCountFn: func() (int, error) { + // This method is expensive, but we don't expect to ever actually use this on the ingester query path in Mimir - + // it's just here to ensure things don't break if this assumption ever changes. + // Ingesters return uncompacted chunks to queriers, so this method is never called. + return countChunks(chunkIteratorFn) }, } } } +func countChunks(chunkIteratorFn func(chunks.Iterator) chunks.Iterator) (int, error) { + chunkCount := 0 + it := chunkIteratorFn(nil) + + for it.Next() { + chunkCount++ + } + + return chunkCount, it.Err() +} + // compactChunkIterator is responsible to compact chunks from different iterators of the same time series into single chainSeries. // If time-overlapping chunks are found, they are encoded and passed to series merge and encoded again into one bigger chunk. // TODO(bwplotka): Currently merge will compact overlapping chunks with bigger chunk, without limit. Split it: https://github.com/prometheus/tsdb/issues/670 @@ -801,6 +821,7 @@ func NewConcatenatingChunkSeriesMerger() VerticalChunkSeriesMergeFunc { if len(series) == 0 { return nil } + return &ChunkSeriesEntry{ Lset: series[0].Labels(), ChunkIteratorFn: func(chunks.Iterator) chunks.Iterator { @@ -812,6 +833,20 @@ func NewConcatenatingChunkSeriesMerger() VerticalChunkSeriesMergeFunc { iterators: iterators, } }, + ChunkCountFn: func() (int, error) { + chunkCount := 0 + + for _, series := range series { + c, err := series.ChunkCount() + if err != nil { + return 0, err + } + + chunkCount += c + } + + return chunkCount, nil + }, } } } diff --git a/vendor/github.com/prometheus/prometheus/storage/series.go b/vendor/github.com/prometheus/prometheus/storage/series.go index b73f1e35ce9..1d1aa820e97 100644 --- a/vendor/github.com/prometheus/prometheus/storage/series.go +++ b/vendor/github.com/prometheus/prometheus/storage/series.go @@ -35,11 +35,13 @@ func (s *SeriesEntry) Iterator(it chunkenc.Iterator) chunkenc.Iterator { return type ChunkSeriesEntry struct { Lset labels.Labels + ChunkCountFn func() (int, error) ChunkIteratorFn func(chunks.Iterator) chunks.Iterator } func (s *ChunkSeriesEntry) Labels() labels.Labels { return s.Lset } func (s *ChunkSeriesEntry) Iterator(it chunks.Iterator) chunks.Iterator { return s.ChunkIteratorFn(it) } +func (s *ChunkSeriesEntry) ChunkCount() (int, error) { return s.ChunkCountFn() } // NewListSeries returns series entry with iterator that allows to iterate over provided samples. func NewListSeries(lset labels.Labels, s []tsdbutil.Sample) *SeriesEntry { @@ -78,6 +80,7 @@ func NewListChunkSeriesFromSamples(lset labels.Labels, samples ...[]tsdbutil.Sam } return NewListChunkSeriesIterator(chks...) }, + ChunkCountFn: func() (int, error) { return len(samples), nil }, // We create one chunk per slice of samples. } } @@ -399,6 +402,20 @@ func (s *seriesToChunkEncoder) Iterator(it chunks.Iterator) chunks.Iterator { return NewListChunkSeriesIterator(chks...) } +func (s *seriesToChunkEncoder) ChunkCount() (int, error) { + // This method is expensive, but we don't expect to ever actually use this on the ingester query path in Mimir - + // it's just here to ensure things don't break if this assumption ever changes. + + chunkCount := 0 + it := s.Iterator(nil) + + for it.Next() { + chunkCount++ + } + + return chunkCount, it.Err() +} + func appendChunk(chks []chunks.Meta, mint, maxt int64, chk chunkenc.Chunk) []chunks.Meta { if chk != nil { chks = append(chks, chunks.Meta{ diff --git a/vendor/github.com/prometheus/prometheus/tsdb/querier.go b/vendor/github.com/prometheus/prometheus/tsdb/querier.go index 854687298cd..dad55954925 100644 --- a/vendor/github.com/prometheus/prometheus/tsdb/querier.go +++ b/vendor/github.com/prometheus/prometheus/tsdb/querier.go @@ -649,6 +649,10 @@ func (s *chunkSeriesEntry) Iterator(it chunks.Iterator) chunks.Iterator { return pi } +func (s *chunkSeriesEntry) ChunkCount() (int, error) { + return len(s.chks), nil +} + // populateWithDelSeriesIterator allows to iterate over samples for the single series. type populateWithDelSeriesIterator struct { populateWithDelGenericSeriesIterator diff --git a/vendor/modules.txt b/vendor/modules.txt index e2abc7625f4..d638b230109 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -877,7 +877,7 @@ github.com/prometheus/exporter-toolkit/web github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/prometheus/prometheus v1.8.2-0.20220620125440-d7e7b8e04b5e => github.com/grafana/mimir-prometheus v0.0.0-20230706135548-245a68172fbe +# github.com/prometheus/prometheus v1.8.2-0.20220620125440-d7e7b8e04b5e => github.com/grafana/mimir-prometheus v0.0.0-20230706234245-f3697f524295 ## explicit; go 1.19 github.com/prometheus/prometheus/config github.com/prometheus/prometheus/discovery @@ -1474,7 +1474,7 @@ sigs.k8s.io/kustomize/kyaml/yaml/walk # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20230706135548-245a68172fbe +# github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20230706234245-f3697f524295 # github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220714140823-09ffed8adbbe # gopkg.in/yaml.v3 => github.com/colega/go-yaml-yaml v0.0.0-20220720105220-255a8d16d094 # github.com/grafana/regexp => github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6