diff --git a/api/proto/banyandb/database/v1/rpc.proto b/api/proto/banyandb/database/v1/rpc.proto index a3429b7df..44a48f574 100644 --- a/api/proto/banyandb/database/v1/rpc.proto +++ b/api/proto/banyandb/database/v1/rpc.proto @@ -483,3 +483,16 @@ service TopNAggregationRegistryService { // Exist doesn't expose an HTTP endpoint. Please use HEAD method to touch Get instead rpc Exist(TopNAggregationRegistryServiceExistRequest) returns (TopNAggregationRegistryServiceExistResponse); } + +message MeasureAggregateFunctionServiceSupportRequest {} + +message MeasureAggregateFunctionServiceSupportResponse { + repeated banyandb.database.v1.MeasureAggregateFunction measure_aggregate_function = 1; +} + +service MeasureAggregateFunctionService { + // Support doesn't need metadata, it's static and stateless. + rpc Support(MeasureAggregateFunctionServiceSupportRequest) returns (MeasureAggregateFunctionServiceSupportResponse) { + option (google.api.http) = {get: "/v1/measure-agg/schema/support"}; + } +} diff --git a/api/proto/banyandb/database/v1/schema.proto b/api/proto/banyandb/database/v1/schema.proto index 7941866c3..e96a8ac43 100644 --- a/api/proto/banyandb/database/v1/schema.proto +++ b/api/proto/banyandb/database/v1/schema.proto @@ -20,6 +20,7 @@ syntax = "proto3"; package banyandb.database.v1; import "banyandb/common/v1/common.proto"; +import "banyandb/model/v1/common.proto"; import "banyandb/model/v1/query.proto"; import "google/protobuf/timestamp.proto"; import "validate/validate.proto"; @@ -95,6 +96,8 @@ message FieldSpec { EncodingMethod encoding_method = 3 [(validate.rules).enum.defined_only = true]; // compression_method indicates how to compress data during writing CompressionMethod compression_method = 4 [(validate.rules).enum.defined_only = true]; + // aggregate_function indicates how to aggregate data + model.v1.AggregationFunction aggregate_function = 5; } // Measure intends to store data point @@ -114,6 +117,13 @@ message Measure { google.protobuf.Timestamp updated_at = 6; } +message MeasureAggregateFunction { + // type indicates the type of function argument + FieldType type = 1 [(validate.rules).enum.defined_only = true]; + // aggregate_function indicates specific function for measure data + model.v1.MeasureAggregate aggregate_function = 2; +} + // TopNAggregation generates offline TopN statistics for a measure's TopN approximation message TopNAggregation { // metadata is the identity of an aggregation diff --git a/api/proto/banyandb/model/v1/common.proto b/api/proto/banyandb/model/v1/common.proto index f639c39bb..b54b7a050 100644 --- a/api/proto/banyandb/model/v1/common.proto +++ b/api/proto/banyandb/model/v1/common.proto @@ -77,3 +77,29 @@ enum AggregationFunction { AGGREGATION_FUNCTION_COUNT = 4; AGGREGATION_FUNCTION_SUM = 5; } + +enum MeasureAggregate { + MEASURE_AGGREGATE_UNSPECIFIED = 0; + // Calculate the minimum value of delta measures. + MEASURE_AGGREGATE_MIN = 1; + // Calculate the maximum value of delta measures. + MEASURE_AGGREGATE_MAX = 2; + // Count the number of delta measures. + MEASURE_AGGREGATE_COUNT = 3; + // Calculate the sum value of delta measures. + MEASURE_AGGREGATE_SUM = 4; + // Calculate the average value of delta measures. + MEASURE_AGGREGATE_AVG = 5; + // Calculate the percentage of delta measures, where the input matches with the condition. + MEASURE_AGGREGATE_PERCENT = 6; + // Calculate the ratio for measures, where the input matches with the condition. + MEASURE_AGGREGATE_RATE = 7; + // Calculate the histogram for delta measures. + MEASURE_AGGREGATE_HISTOGRAM = 8; + // Calculate the {p99, p95, p90, p75, p50} for delta measures. + MEASURE_AGGREGATE_PERCENTILE2 = 9; + // Calculate the apdex for delta measures. + MEASURE_AGGREGATE_APDEX = 10; + // Same like PERCENTILE2, little different on algorithm. + MEASURE_AGGREGATE_PERCENTILE = 11; +} diff --git a/docs/api-reference.md b/docs/api-reference.md index 299e1ef91..d84f9b4f5 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -40,6 +40,7 @@ - [TagValue](#banyandb-model-v1-TagValue) - [AggregationFunction](#banyandb-model-v1-AggregationFunction) + - [MeasureAggregate](#banyandb-model-v1-MeasureAggregate) - [banyandb/model/v1/query.proto](#banyandb_model_v1_query-proto) - [Condition](#banyandb-model-v1-Condition) @@ -62,6 +63,7 @@ - [IndexRule](#banyandb-database-v1-IndexRule) - [IndexRuleBinding](#banyandb-database-v1-IndexRuleBinding) - [Measure](#banyandb-database-v1-Measure) + - [MeasureAggregateFunction](#banyandb-database-v1-MeasureAggregateFunction) - [Stream](#banyandb-database-v1-Stream) - [Subject](#banyandb-database-v1-Subject) - [TagFamilySpec](#banyandb-database-v1-TagFamilySpec) @@ -112,6 +114,8 @@ - [IndexRuleRegistryServiceListResponse](#banyandb-database-v1-IndexRuleRegistryServiceListResponse) - [IndexRuleRegistryServiceUpdateRequest](#banyandb-database-v1-IndexRuleRegistryServiceUpdateRequest) - [IndexRuleRegistryServiceUpdateResponse](#banyandb-database-v1-IndexRuleRegistryServiceUpdateResponse) + - [MeasureAggregateFunctionServiceSupportRequest](#banyandb-database-v1-MeasureAggregateFunctionServiceSupportRequest) + - [MeasureAggregateFunctionServiceSupportResponse](#banyandb-database-v1-MeasureAggregateFunctionServiceSupportResponse) - [MeasureRegistryServiceCreateRequest](#banyandb-database-v1-MeasureRegistryServiceCreateRequest) - [MeasureRegistryServiceCreateResponse](#banyandb-database-v1-MeasureRegistryServiceCreateResponse) - [MeasureRegistryServiceDeleteRequest](#banyandb-database-v1-MeasureRegistryServiceDeleteRequest) @@ -152,6 +156,7 @@ - [GroupRegistryService](#banyandb-database-v1-GroupRegistryService) - [IndexRuleBindingRegistryService](#banyandb-database-v1-IndexRuleBindingRegistryService) - [IndexRuleRegistryService](#banyandb-database-v1-IndexRuleRegistryService) + - [MeasureAggregateFunctionService](#banyandb-database-v1-MeasureAggregateFunctionService) - [MeasureRegistryService](#banyandb-database-v1-MeasureRegistryService) - [StreamRegistryService](#banyandb-database-v1-StreamRegistryService) - [TopNAggregationRegistryService](#banyandb-database-v1-TopNAggregationRegistryService) @@ -690,6 +695,28 @@ Trace is the top level message of a trace. | AGGREGATION_FUNCTION_SUM | 5 | | + + + +### MeasureAggregate + + +| Name | Number | Description | +| ---- | ------ | ----------- | +| MEASURE_AGGREGATE_UNSPECIFIED | 0 | | +| MEASURE_AGGREGATE_MIN | 1 | Calculate the minimum value of delta measures. | +| MEASURE_AGGREGATE_MAX | 2 | Calculate the maximum value of delta measures. | +| MEASURE_AGGREGATE_COUNT | 3 | Count the number of delta measures. | +| MEASURE_AGGREGATE_SUM | 4 | Calculate the sum value of delta measures. | +| MEASURE_AGGREGATE_AVG | 5 | Calculate the average value of delta measures. | +| MEASURE_AGGREGATE_PERCENT | 6 | Calculate the percentage of delta measures, where the input matches with the condition. | +| MEASURE_AGGREGATE_RATE | 7 | Calculate the ratio for measures, where the input matches with the condition. | +| MEASURE_AGGREGATE_HISTOGRAM | 8 | Calculate the histogram for delta measures. | +| MEASURE_AGGREGATE_PERCENTILE2 | 9 | Calculate the {p99, p95, p90, p75, p50} for delta measures. | +| MEASURE_AGGREGATE_APDEX | 10 | Calculate the apdex for delta measures. | +| MEASURE_AGGREGATE_PERCENTILE | 11 | Same like PERCENTILE2, little different on algorithm. | + + @@ -954,6 +981,7 @@ FieldSpec is the specification of field | field_type | [FieldType](#banyandb-database-v1-FieldType) | | field_type denotes the type of field value | | encoding_method | [EncodingMethod](#banyandb-database-v1-EncodingMethod) | | encoding_method indicates how to encode data during writing | | compression_method | [CompressionMethod](#banyandb-database-v1-CompressionMethod) | | compression_method indicates how to compress data during writing | +| aggregate_function | [banyandb.model.v1.AggregationFunction](#banyandb-model-v1-AggregationFunction) | | aggregate_function indicates how to aggregate data | @@ -1023,6 +1051,22 @@ Measure intends to store data point + + +### MeasureAggregateFunction + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| type | [FieldType](#banyandb-database-v1-FieldType) | | type indicates the type of function argument | +| aggregate_function | [banyandb.model.v1.MeasureAggregate](#banyandb-model-v1-MeasureAggregate) | | aggregate_function indicates specific function for measure data | + + + + + + ### Stream @@ -1717,6 +1761,31 @@ Type determine the index structure under the hood + + +### MeasureAggregateFunctionServiceSupportRequest + + + + + + + + + +### MeasureAggregateFunctionServiceSupportResponse + + + +| Field | Type | Label | Description | +| ----- | ---- | ----- | ----------- | +| measure_aggregate_function | [MeasureAggregateFunction](#banyandb-database-v1-MeasureAggregateFunction) | repeated | | + + + + + + ### MeasureRegistryServiceCreateRequest @@ -2301,6 +2370,16 @@ Type determine the index structure under the hood | Exist | [IndexRuleRegistryServiceExistRequest](#banyandb-database-v1-IndexRuleRegistryServiceExistRequest) | [IndexRuleRegistryServiceExistResponse](#banyandb-database-v1-IndexRuleRegistryServiceExistResponse) | Exist doesn't expose an HTTP endpoint. Please use HEAD method to touch Get instead | + + +### MeasureAggregateFunctionService + + +| Method Name | Request Type | Response Type | Description | +| ----------- | ------------ | ------------- | ------------| +| Support | [MeasureAggregateFunctionServiceSupportRequest](#banyandb-database-v1-MeasureAggregateFunctionServiceSupportRequest) | [MeasureAggregateFunctionServiceSupportResponse](#banyandb-database-v1-MeasureAggregateFunctionServiceSupportResponse) | Support doesn't need metadata, it's static and stateless. | + + ### MeasureRegistryService diff --git a/docs/interacting/data-lifecycle.md b/docs/interacting/data-lifecycle.md new file mode 100644 index 000000000..6163b7329 --- /dev/null +++ b/docs/interacting/data-lifecycle.md @@ -0,0 +1,102 @@ +# Data Lifecycle + +## [Measures](../concept/data-model.md#measures) and [Streams](../concept/data-model.md#streams) + +Due to the design of BanyanDB, the data in the `Measures and Streams` can not be deleted directly. +The data will be deleted automatically based on the [Groups](../concept/data-model.md#groups) `TTL` setting. + +The TTL means the `time to live` of the data in the group. +Each group has an internal trigger which is triggered by writing events. If there is no further data, the expired data can’t get removed. + +For example, the following command will create a group with a TTL of 7 days: +```shell +bydbctl group create -f - <