diff --git a/.vscode/config-schema.yaml b/.vscode/config-schema.yaml index 06601b93..d98a9626 100644 --- a/.vscode/config-schema.yaml +++ b/.vscode/config-schema.yaml @@ -36,6 +36,14 @@ properties: type: string labels: $ref: "#/definitions/labels" + values: + type: array + items: + type: object + additionalProperties: false + required: + - name + - help histograms: type: array items: diff --git a/README.md b/README.md index 17ff3e55..9c5e4c0e 100644 --- a/README.md +++ b/README.md @@ -759,10 +759,13 @@ perf_event_array: flush_interval: labels: [ - label ] +[ values: { name: "...", help: "..."}] ``` An example of `perf_map` can be found [here](examples/oomkill.yaml). +An example of `values` can be found [here](examples/complex-value.yaml). + #### `histogram` See [Histograms](#histograms) section for more details. diff --git a/config/config.go b/config/config.go index b7f8bc48..553fc91e 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,7 @@ type Counter struct { PerfEventArray bool `yaml:"perf_event_array"` FlushInterval time.Duration `yaml:"flush_interval"` Labels []Label `yaml:"labels"` + Values []Value `yaml:"values"` } // Histogram is a metric defining prometheus histogram @@ -75,6 +76,12 @@ type Decoder struct { AllowUnknown bool `yaml:"allow_unknown"` } +// Value describes a metric in when it's split across multiple u64 +type Value struct { + Name string `yaml:"name"` + Help string `yaml:"help"` +} + // HistogramBucketType is an enum to define how to interpret histogram type HistogramBucketType string diff --git a/examples/complex-value.bpf.c b/examples/complex-value.bpf.c new file mode 100644 index 00000000..a0eb6dc3 --- /dev/null +++ b/examples/complex-value.bpf.c @@ -0,0 +1,83 @@ +#include +#include +#include + +#define MKDEV(ma, mi) ((mi & 0xff) | (ma << 8) | ((mi & ~0xff) << 12)) + +/** + * commit d152c682f03c ("block: add an explicit ->disk backpointer to the + * request_queue") and commit f3fa33acca9f ("block: remove the ->rq_disk + * field in struct request") make some changes to `struct request` and + * `struct request_queue`. Now, to get the `struct gendisk *` field in a CO-RE + * way, we need both `struct request` and `struct request_queue`. + * see: + * https://github.com/torvalds/linux/commit/d152c682f03c + * https://github.com/torvalds/linux/commit/f3fa33acca9f + */ +struct request_queue___x { + struct gendisk *disk; +} __attribute__((preserve_access_index)); + +struct request___x { + struct request_queue___x *q; + struct gendisk *rq_disk; +} __attribute__((preserve_access_index)); + +struct key_t { + u32 dev; +}; + +struct value_t { + u64 count; + u64 bytes; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(max_entries, 1024); + __type(key, struct key_t); + __type(value, struct value_t); +} block_rq_completions SEC(".maps"); + +static __always_inline struct gendisk *get_disk(void *request) +{ + struct request___x *r = request; + + if (bpf_core_field_exists(r->rq_disk)) + return r->rq_disk; + return r->q->disk; +} + +static struct value_t *get_value(void *map, struct key_t *key) +{ + struct value_t *value = bpf_map_lookup_elem(map, key); + if (!value) { + struct value_t zero = { .count = 0, .bytes = 0 }; + bpf_map_update_elem(map, key, &zero, BPF_NOEXIST); + value = bpf_map_lookup_elem(map, key); + if (!value) { + return NULL; + } + } + + return value; +} + +SEC("tp_btf/block_rq_complete") +int BPF_PROG(block_rq_complete, struct request *rq, blk_status_t error, unsigned int nr_bytes) +{ + struct gendisk *disk = get_disk(rq); + struct key_t key = { .dev = disk ? MKDEV(disk->major, disk->first_minor) : 0 }; + struct value_t *value = get_value(&block_rq_completions, &key); + + if (!value) { + return 0; + } + + __sync_fetch_and_add(&value->count, 1); + __sync_fetch_and_add(&value->bytes, nr_bytes); + + return 0; +} + +char LICENSE[] SEC("license") = "GPL"; diff --git a/examples/complex-value.yaml b/examples/complex-value.yaml new file mode 100644 index 00000000..97556996 --- /dev/null +++ b/examples/complex-value.yaml @@ -0,0 +1,14 @@ +metrics: + counters: + - name: block_rq_completions + help: Block request completions split into count and bytes + labels: + - name: device + size: 4 + decoders: + - name: majorminor + values: + - name: block_rq_completions_total + help: Total number of block request completions + - name: block_rq_completed_bytes_total + help: Total number of bytes served by block requests completions diff --git a/exporter/exporter.go b/exporter/exporter.go index b3a54bac..9d9bf434 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -392,7 +392,13 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { e.perfEventArrayCollectors = append(e.perfEventArrayCollectors, perfSink) } - addDescs(cfg.Name, counter.Name, counter.Help, counter.Labels) + if counter.Values != nil { + for _, value := range counter.Values { + addDescs(cfg.Name, value.Name, value.Help, counter.Labels) + } + } else { + addDescs(cfg.Name, counter.Name, counter.Help, counter.Labels) + } } for _, histogram := range cfg.Metrics.Histograms { @@ -477,10 +483,14 @@ func (e *Exporter) collectCounters(ch chan<- prometheus.Metric) { aggregatedMapValues := aggregateMapValues(mapValues) - desc := e.descs[cfg.Name][counter.Name] - for _, metricValue := range aggregatedMapValues { - ch <- prometheus.MustNewConstMetric(desc, prometheus.CounterValue, metricValue.value, metricValue.labels...) + if counter.Values != nil { + for i, value := range counter.Values { + ch <- prometheus.MustNewConstMetric(e.descs[cfg.Name][value.Name], prometheus.CounterValue, metricValue.value[i], metricValue.labels...) + } + } else { + ch <- prometheus.MustNewConstMetric(e.descs[cfg.Name][counter.Name], prometheus.CounterValue, metricValue.value[0], metricValue.labels...) + } } } } @@ -531,7 +541,7 @@ func (e *Exporter) collectHistograms(ch chan<- prometheus.Metric) { break } - histograms[key].buckets[float64(leUint)] = uint64(metricValue.value) + histograms[key].buckets[float64(leUint)] = uint64(metricValue.value[0]) } if skip { @@ -673,29 +683,33 @@ func (e *Exporter) MapsHandler(w http.ResponseWriter, r *http.Request) { } func validateMaps(module *libbpfgo.Module, cfg config.Config) error { - maps := []string{} + sizes := map[string]int{} for _, counter := range cfg.Metrics.Counters { if counter.Name != "" && !counter.PerfEventArray { - maps = append(maps, counter.Name) + if counter.Values != nil { + sizes[counter.Name] = len(counter.Values) * 8 + } else { + sizes[counter.Name] = 8 + } } } for _, histogram := range cfg.Metrics.Histograms { if histogram.Name != "" { - maps = append(maps, histogram.Name) + sizes[histogram.Name] = 8 } } - for _, name := range maps { + for name, expected := range sizes { m, err := module.GetMap(name) if err != nil { return fmt.Errorf("failed to get map %q: %w", name, err) } valueSize := m.ValueSize() - if valueSize != 8 { - return fmt.Errorf("value size for map %q is not expected 8 bytes (u64), it is %d bytes", name, valueSize) + if valueSize != expected { + return fmt.Errorf("value size for map %q is not expected %d bytes (8 bytes per u64 value), it is %d bytes", name, expected, valueSize) } } @@ -721,7 +735,9 @@ func aggregateMapValues(values []metricValue) []aggregatedMetricValue { value: value.value, } } else { - existing.value += value.value + for i := range existing.value { + existing.value[i] += value.value[i] + } } } @@ -785,17 +801,18 @@ func readMapValues(m *libbpfgo.BPFMap, labels []config.Label) ([]metricValue, er return metricValues, nil } -func mapValue(m *libbpfgo.BPFMap, key []byte) (float64, error) { +func mapValue(m *libbpfgo.BPFMap, key []byte) ([]float64, error) { v, err := m.GetValue(unsafe.Pointer(&key[0])) if err != nil { - return 0.0, err + return []float64{0.0}, err } return decodeValue(v), nil } -func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([]float64, error) { - values := []float64{} +func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([][]float64, error) { + values := [][]float64{} + size := m.ValueSize() value, err := m.GetValue(unsafe.Pointer(&key[0])) @@ -811,8 +828,14 @@ func mapValuePerCPU(m *libbpfgo.BPFMap, key []byte) ([]float64, error) { } // Assuming counter's value type is always u64 -func decodeValue(value []byte) float64 { - return float64(util.GetHostByteOrder().Uint64(value)) +func decodeValue(value []byte) []float64 { + values := make([]float64, len(value)/8) + + for i := range values { + values[i] = float64(util.GetHostByteOrder().Uint64(value[i*8:])) + } + + return values } // metricValue is a row in a kernel map @@ -822,7 +845,7 @@ type metricValue struct { // labels are decoded from the raw key labels []string // value is the kernel map value - value float64 + value []float64 } // aggregatedMetricValue is a value after aggregation of equal label sets @@ -830,5 +853,5 @@ type aggregatedMetricValue struct { // labels are decoded from the raw key labels []string // value is the kernel map value - value float64 + value []float64 } diff --git a/exporter/exporter_test.go b/exporter/exporter_test.go index 2b3e83e6..5528764a 100644 --- a/exporter/exporter_test.go +++ b/exporter/exporter_test.go @@ -10,32 +10,32 @@ func TestAggregatedMetricValues(t *testing.T) { values := []metricValue{ { labels: []string{"foo"}, - value: 8, + value: []float64{8}, }, { labels: []string{"bar"}, - value: 1, + value: []float64{1}, }, { labels: []string{"foo"}, - value: 3, + value: []float64{3}, }, } aggregated := aggregateMapValues(values) sort.Slice(aggregated, func(i, j int) bool { - return aggregated[i].value > aggregated[j].value + return aggregated[i].value[0] > aggregated[j].value[0] }) expected := []aggregatedMetricValue{ { labels: []string{"foo"}, - value: 11, + value: []float64{11}, }, { labels: []string{"bar"}, - value: 1, + value: []float64{1}, }, }