Emit index usage and index size metrics from mysql integration (#19383)

azhou-datadog · web-flow · commit 778d9c780c47 · 2025-01-14T21:25:31.000Z
* Emit index usage and size metrics

* Add logs for debugging

* Use query executor to run index usage queries

* Set collection interval to 5 min

* linter, change log, and config validation

* clean up

* sort metadata.csv

* sync models

* magic number replaced with var

* linter

* make collection interval configurable

* fix spec.yaml

* sync models

* update readme, report size in bytes, don't exclude primary

* updated spec description
diff --git a/mysql/README.md b/mysql/README.md
@@ -91,6 +91,14 @@ mysql> GRANT SELECT ON performance_schema.* TO 'datadog'@'%';
 Query OK, 0 rows affected (0.00 sec)
 ```
 
+To collect index metrics, grant the `datadog` user an additional privilege:
+
+```shell
+
+mysql> "GRANT SELECT ON mysql.innodb_index_stats TO 'datadog'@'%'";
+Query OK, 0 rows affected (0.00 sec)
+```
+
 ### Configuration
 
 Follow the instructions below to configure this check for an Agent running on a host. For containerized environments, see the [Docker](?tab=docker#docker), [Kubernetes](?tab=kubernetes#kubernetes), or [ECS](?tab=ecs#ecs) sections.
diff --git a/mysql/assets/configuration/spec.yaml b/mysql/assets/configuration/spec.yaml
@@ -336,7 +336,6 @@ files:
                 value:
                   type: boolean
                   example: false
-
               - name: extra_performance_metrics
                 description: |
                   These metrics are reported if `performance_schema` is enabled in the MySQL instance
@@ -543,6 +542,31 @@ files:
                 value:
                   type: number
                   example: 10
+          - name: index_metrics
+            description: |
+              Configure collection of index metrics.
+              Metrics provided by the options:
+                - mysql.index.size (per index)
+                - mysql.index.reads (per index)
+                - mysql.index.updates (per index)
+                - mysql.index.deletes (per index)
+              Note that the index size metric requires the user defined for this instance to have 
+              SELECT privileges. Take a look at 
+              https://docs.datadoghq.com/integrations/mysql/?tab=host#prepare-mysql for further instructions.
+            options:
+              - name: enabled
+                description: |
+                  Enable collection of index metrics.
+                value:
+                  type: boolean
+                  example: true
+              - name: collection_interval
+                hidden: true
+                description: |
+                  Set the index metrics collection interval (in seconds). Defaults to 5 minutes.
+                value:
+                  type: number
+                  example: 300
           - name: aws
             description: |
               This block defines the configuration for AWS RDS and Aurora instances.
diff --git a/mysql/changelog.d/19383.added b/mysql/changelog.d/19383.added
@@ -0,0 +1 @@
+Emit index usage and index metrics from mysql integration
diff --git a/mysql/datadog_checks/mysql/config.py b/mysql/datadog_checks/mysql/config.py
@@ -47,6 +47,7 @@ def __init__(self, instance, init_config):
         self.settings_config = instance.get('collect_settings', {}) or {}
         self.activity_config = instance.get('query_activity', {}) or {}
         self.schemas_config: dict = instance.get('schemas_collection', {}) or {}
+        self.index_config: dict = instance.get('index_metrics', {}) or {}
 
         self.cloud_metadata = {}
         aws = instance.get('aws', {})
diff --git a/mysql/datadog_checks/mysql/config_models/instance.py b/mysql/datadog_checks/mysql/config_models/instance.py
@@ -67,6 +67,15 @@ class Gcp(BaseModel):
     project_id: Optional[str] = None
 
 
+class IndexMetrics(BaseModel):
+    model_config = ConfigDict(
+        arbitrary_types_allowed=True,
+        frozen=True,
+    )
+    collection_interval: Optional[float] = None
+    enabled: Optional[bool] = None
+
+
 class MetricPatterns(BaseModel):
     model_config = ConfigDict(
         arbitrary_types_allowed=True,
@@ -196,6 +205,7 @@ class InstanceConfig(BaseModel):
     empty_default_hostname: Optional[bool] = None
     gcp: Optional[Gcp] = None
     host: Optional[str] = None
+    index_metrics: Optional[IndexMetrics] = None
     log_unobfuscated_plans: Optional[bool] = None
     log_unobfuscated_queries: Optional[bool] = None
     max_custom_queries: Optional[int] = None
diff --git a/mysql/datadog_checks/mysql/data/conf.yaml.example b/mysql/datadog_checks/mysql/data/conf.yaml.example
@@ -499,6 +499,23 @@ instances:
         #
         # collection_interval: 10
 
+    ## Configure collection of index metrics.
+    ## Metrics provided by the options:
+    ##   - mysql.index.size (per index)
+    ##   - mysql.index.reads (per index)
+    ##   - mysql.index.updates (per index)
+    ##   - mysql.index.deletes (per index)
+    ## Note that the index size metric requires the user defined for this instance to have 
+    ## SELECT privileges. Take a look at 
+    ## https://docs.datadoghq.com/integrations/mysql/?tab=host#prepare-mysql for further instructions.
+    #
+    # index_metrics:
+
+        ## @param enabled - boolean - optional - default: true
+        ## Enable collection of index metrics.
+        #
+        # enabled: true
+
     ## This block defines the configuration for AWS RDS and Aurora instances.
     ##
     ## Complete this section if you have installed the Datadog AWS Integration
diff --git a/mysql/datadog_checks/mysql/index_metrics.py b/mysql/datadog_checks/mysql/index_metrics.py
@@ -0,0 +1,75 @@
+# (C) Datadog, Inc. 2025-present
+# All rights reserved
+# Licensed under a 3-clause BSD style license (see LICENSE)
+
+from datadog_checks.base import is_affirmative
+
+QUERY_INDEX_SIZE = {
+    'name': 'mysql.innodb_index_stats',
+    'query': """
+        SELECT
+            database_name,
+            table_name,
+            index_name,
+            stat_value * @@innodb_page_size AS index_size_bytes
+        FROM
+            mysql.innodb_index_stats
+        WHERE
+            stat_name = 'size'
+    """.strip(),
+    'columns': [
+        {'name': 'db', 'type': 'tag'},
+        {'name': 'table', 'type': 'tag'},
+        {'name': 'index', 'type': 'tag'},
+        {'name': 'mysql.index.size', 'type': 'gauge'},
+    ],
+}
+QUERY_INDEX_USAGE = {
+    'name': 'performance_schema.table_io_waits_summary_by_index_usage',
+    'query': """
+        SELECT
+            object_schema,
+            object_name,
+            index_name,
+            count_read,
+            count_update,
+            count_delete
+        FROM
+            performance_schema.table_io_waits_summary_by_index_usage
+        WHERE index_name IS NOT NULL
+        AND object_schema NOT IN ('mysql', 'performance_schema')
+    """.strip(),
+    'columns': [
+        {'name': 'db', 'type': 'tag'},
+        {'name': 'table', 'type': 'tag'},
+        {'name': 'index', 'type': 'tag'},
+        {'name': 'mysql.index.reads', 'type': 'gauge'},
+        {'name': 'mysql.index.updates', 'type': 'gauge'},
+        {'name': 'mysql.index.deletes', 'type': 'gauge'},
+    ],
+}
+
+DEFAULT_INDEX_METRIC_COLLECTION_INTERVAL = 300  # 5 minutes
+
+
+class MySqlIndexMetrics:
+    def __init__(self, config):
+        self._config = config
+
+    @property
+    def include_index_metrics(self) -> bool:
+        return is_affirmative(self._config.index_config.get('enabled', True))
+
+    @property
+    def collection_interval(self) -> int:
+        return int(self._config.index_config.get('collection_interval', DEFAULT_INDEX_METRIC_COLLECTION_INTERVAL))
+
+    @property
+    def queries(self):
+        # make a copy of the query to avoid modifying the original
+        # in case different instances have different collection intervals
+        usage_query = QUERY_INDEX_USAGE.copy()
+        size_query = QUERY_INDEX_SIZE.copy()
+        usage_query['collection_interval'] = self.collection_interval
+        size_query['collection_interval'] = self.collection_interval
+        return [size_query, usage_query]
diff --git a/mysql/datadog_checks/mysql/mysql.py b/mysql/datadog_checks/mysql/mysql.py
@@ -54,6 +54,7 @@
     TABLE_VARS,
     VARIABLES_VARS,
 )
+from .index_metrics import MySqlIndexMetrics
 from .innodb_metrics import InnoDBMetrics
 from .metadata import MySQLMetadata
 from .queries import (
@@ -130,6 +131,7 @@ def __init__(self, name, init_config, instances):
         self._statement_samples = MySQLStatementSamples(self, self._config, self._get_connection_args())
         self._mysql_metadata = MySQLMetadata(self, self._config, self._get_connection_args())
         self._query_activity = MySQLActivity(self, self._config, self._get_connection_args())
+        self._index_metrics = MySqlIndexMetrics(self._config)
         # _database_instance_emitted: limit the collection and transmission of the database instance metadata
         self._database_instance_emitted = TTLCache(
             maxsize=1,
@@ -378,7 +380,8 @@ def _get_runtime_queries(self, db):
 
         if self.performance_schema_enabled:
             queries.extend([QUERY_USER_CONNECTIONS])
-
+        if self._index_metrics.include_index_metrics:
+            queries.extend(self._index_metrics.queries)
         self._runtime_queries_cached = self._new_query_executor(queries)
         self._runtime_queries_cached.compile_queries()
         self.log.debug("initialized runtime queries")
diff --git a/mysql/metadata.csv b/mysql/metadata.csv
@@ -17,6 +17,10 @@ mysql.galera.wsrep_local_state,gauge,,,,Internal Galera cluster state number,0,m
 mysql.galera.wsrep_received,gauge,,,,Total number of write-sets received from other nodes.,0,mysql,mysql galera wsrep_received,
 mysql.galera.wsrep_received_bytes,gauge,,,,Total size (in bytes) of writesets received from other nodes.,0,mysql,mysql galera wsrep_received_bytes,
 mysql.galera.wsrep_replicated_bytes,gauge,,,,Total size (in bytes) of writesets sent to other nodes.,0,mysql,mysql galera wsrep_replicated_bytes,
+mysql.index.deletes,gauge,,operation,,The number of delete operations using an index. Resets to 0 upon database restart.,0,mysql,mysql index delete usage,
+mysql.index.reads,gauge,,operation,,The number of read operations using an index. Resets to 0 upon database restart.,0,mysql,mysql index read usage,
+mysql.index.size,gauge,,byte,,Size of index in bytes,0,mysql,mysql index size,
+mysql.index.updates,gauge,,operation,,The number of update operations using an index. Resets to 0 upon database restart.,0,mysql,mysql index update usage,
 mysql.info.schema.size,gauge,,mebibyte,,Size of schemas in MiB,0,mysql,mysql schema size,
 mysql.info.table.data_size,gauge,,mebibyte,,Size of tables data in MiB,0,mysql,mysql data table size,memory
 mysql.info.table.index_size,gauge,,mebibyte,,Size of tables index in MiB,0,mysql,mysql index table size,
diff --git a/mysql/tests/conftest.py b/mysql/tests/conftest.py
@@ -102,6 +102,7 @@ def instance_complex():
             'table_size_metrics': True,
             'system_table_size_metrics': True,
             'table_row_stats_metrics': True,
+            'index_metrics': True,
         },
         'tags': tags.METRIC_TAGS,
         'queries': [
@@ -405,6 +406,7 @@ def _add_dog_user(conn):
     cur.execute("GRANT PROCESS ON *.* TO 'dog'@'%'")
     cur.execute("GRANT REPLICATION CLIENT ON *.* TO 'dog'@'%'")
     cur.execute("GRANT SELECT ON performance_schema.* TO 'dog'@'%'")
+    cur.execute("GRANT SELECT ON mysql.innodb_index_stats TO 'dog'@'%'")
 
     # refactor try older mysql.user table first. if this fails, go to newer ALTER USER
     try:
diff --git a/mysql/tests/test_mysql.py b/mysql/tests/test_mysql.py
@@ -48,6 +48,8 @@ def test_minimal_config(aggregator, dd_run_check, instance_basic):
         + variables.INNODB_VARS
         + variables.BINLOG_VARS
         + variables.COMMON_PERFORMANCE_VARS
+        + variables.INDEX_SIZE_VARS
+        + variables.INDEX_USAGE_VARS
     )
 
     operation_time_metrics = (
@@ -73,7 +75,6 @@ def test_minimal_config(aggregator, dd_run_check, instance_basic):
             continue
         else:
             aggregator.assert_metric(mname, at_least=1)
-        aggregator.assert_metric(mname, at_least=1)
 
     optional_metrics = (
         variables.COMPLEX_STATUS_VARS
@@ -161,6 +162,8 @@ def _assert_complex_config(aggregator, service_check_tags, metric_tags, hostname
         + variables.STATEMENT_VARS
         + variables.TABLE_VARS
         + variables.ROW_TABLE_STATS_VARS
+        + variables.INDEX_SIZE_VARS
+        + variables.INDEX_USAGE_VARS
     )
 
     operation_time_metrics = variables.SIMPLE_OPERATION_TIME_METRICS + variables.COMPLEX_OPERATION_TIME_METRICS
@@ -233,6 +236,7 @@ def _assert_complex_config(aggregator, service_check_tags, metric_tags, hostname
     aggregator.assert_metric('alice.age', value=25)
     aggregator.assert_metric('bob.age', value=20)
 
+    _test_index_metrics(aggregator, variables.INDEX_USAGE_VARS + variables.INDEX_SIZE_VARS, metric_tags)
     # test optional metrics
     optional_metrics = (
         variables.OPTIONAL_REPLICATION_METRICS
@@ -304,6 +308,7 @@ def test_complex_config_replica(aggregator, dd_run_check, instance_complex):
         + variables.STATEMENT_VARS
         + variables.TABLE_VARS
         + variables.ROW_TABLE_STATS_VARS
+        + variables.INDEX_SIZE_VARS
     )
 
     operation_time_metrics = (
@@ -313,7 +318,9 @@ def test_complex_config_replica(aggregator, dd_run_check, instance_complex):
     )
 
     if MYSQL_VERSION_PARSED >= parse_version('5.6') and MYSQL_FLAVOR != 'mariadb':
-        testable_metrics.extend(variables.PERFORMANCE_VARS + variables.COMMON_PERFORMANCE_VARS)
+        testable_metrics.extend(
+            variables.PERFORMANCE_VARS + variables.COMMON_PERFORMANCE_VARS + variables.INDEX_USAGE_VARS
+        )
         operation_time_metrics.extend(
             variables.COMMON_PERFORMANCE_OPERATION_TIME_METRICS + variables.PERFORMANCE_OPERATION_TIME_METRICS
         )
@@ -458,6 +465,43 @@ def test_correct_hostname(dbm_enabled, reported_hostname, expected_hostname, agg
         aggregator.assert_metric(mname, hostname=expected_hostname, at_least=0)
 
 
+def _test_index_metrics(aggregator, index_metrics, metric_tags):
+    for mname in index_metrics:
+        if mname in ['mysql.index.reads', 'mysql.index.updates', 'mysql.index.deletes']:
+            aggregator.assert_metric(
+                mname,
+                tags=metric_tags + ['db:testdb', 'table:users', 'index:id'],
+                count=1,
+            )
+            aggregator.assert_metric(
+                mname,
+                tags=metric_tags
+                + [
+                    'db:datadog_test_schemas',
+                    'table:cities',
+                    'index:single_column_index',
+                ],
+                count=1,
+            )
+            aggregator.assert_metric(
+                mname,
+                tags=metric_tags + ['db:datadog_test_schemas', 'table:cities', 'index:two_columns_index'],
+                count=1,
+            )
+        if mname == 'mysql.index.size':
+            aggregator.assert_metric(mname, tags=metric_tags + ['db:testdb', 'table:users', 'index:id'], count=1)
+            aggregator.assert_metric(
+                mname,
+                tags=metric_tags + ['db:datadog_test_schemas', 'table:cities', 'index:single_column_index'],
+                count=1,
+            )
+            aggregator.assert_metric(
+                mname,
+                tags=metric_tags + ['db:datadog_test_schemas', 'table:cities', 'index:two_columns_index'],
+                count=1,
+            )
+
+
 def _test_optional_metrics(aggregator, optional_metrics):
     """
     Check optional metrics - They can either be present or not
diff --git a/mysql/tests/variables.py b/mysql/tests/variables.py
@@ -280,6 +280,15 @@
     'mysql.replication.group.transactions_rollback',
 ]
 
+INDEX_SIZE_VARS = [
+    'mysql.index.size',
+]
+INDEX_USAGE_VARS = [
+    'mysql.index.reads',
+    'mysql.index.updates',
+    'mysql.index.deletes',
+]
+
 SIMPLE_OPERATION_TIME_METRICS = [
     'status_metrics',
     'innodb_metrics',

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Emit index usage and index metrics from mysql integration`