[azuremonitorreceiver] Add the ability to pull Resource Metrics using AZ Query Batch API

.chloggen/add-batch-api-to-azure-monitor-receiver.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: azuremonitorreceiver
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow the Metrics Batch API to be used instead of the Azure Resource Manager API to run less queries and avoid being  throttled.
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: []
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: This is intended to be a Proof-of-Concept and includes a lot of replication. It is inteded to be used to start a conversation until ideas can be integrated and issues resolved.
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

cmd/configschema/go.mod

@@ -202,8 +202,10 @@ require (
 	github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery v1.2.0-beta.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 // indirect
 	github.com/Azure/azure-storage-queue-go v0.0.0-20230531184854-c06a8eff66fe // indirect
 	github.com/Azure/go-amqp v1.0.4 // indirect

cmd/otelcontribcol/go.mod

@@ -246,10 +246,12 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery v1.2.0-beta.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 // indirect
 	github.com/Azure/azure-storage-queue-go v0.0.0-20230531184854-c06a8eff66fe // indirect
 	github.com/Azure/go-amqp v1.0.4 // indirect

go.mod

@@ -214,10 +214,12 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery v1.2.0-beta.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v2 v2.2.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.2.1 // indirect
 	github.com/Azure/azure-storage-queue-go v0.0.0-20230531184854-c06a8eff66fe // indirect
 	github.com/Azure/go-amqp v1.0.4 // indirect

receiver/azuremonitorreceiver/config.go

@@ -26,6 +26,7 @@ var (
 	errMissingClientSecret   = errors.New(`ClientSecret" is not specified in config`)
 	errMissingFedTokenFile   = errors.New(`FederatedTokenFile is not specified in config`)
 	errInvalidCloud          = errors.New(`Cloud" is invalid`)
+	errInvalidRegion         = errors.New("`Region` is not specified in config`")
 
 	monitorServices = []string{
 		"Microsoft.EventGrid/eventSubscriptions",
@@ -244,6 +245,10 @@ type Config struct {
 	CacheResourcesDefinitions               float64                       `mapstructure:"cache_resources_definitions"`
 	MaximumNumberOfMetricsInACall           int                           `mapstructure:"maximum_number_of_metrics_in_a_call"`
 	AppendTagsAsAttributes                  bool                          `mapstructure:"append_tags_as_attributes"`
+	UseBatchApi                             bool                          `mapstructure:"use_batch_api"`
+	DiscoverSubscription                    bool                          `mapstructure:"discover_subscriptions"`
+	Region                                  string                        `mapstructure:"region"`
+	MaximumNumberOfDimensionsInACall        int                           `mapstructure:"maximum_number_of_dimensions_in_a_call"`
 }
 
 const (
@@ -253,7 +258,7 @@ const (
 
 // Validate validates the configuration by checking for missing or invalid fields
 func (c Config) Validate() (err error) {
-	if c.SubscriptionID == "" {
+	if c.SubscriptionID == "" && !c.DiscoverSubscription {
 		err = multierr.Append(err, errMissingSubscriptionID)
 	}
 
@@ -290,5 +295,9 @@ func (c Config) Validate() (err error) {
 		err = multierr.Append(err, errInvalidCloud)
 	}
 
+	if c.UseBatchApi && c.Region == "" && !c.DiscoverSubscription {
+		err = multierr.Append(err, errInvalidRegion)
+	}
+
 	return
 }

receiver/azuremonitorreceiver/factory.go

@@ -36,14 +36,15 @@ func createDefaultConfig() component.Config {
 	cfg.CollectionInterval = defaultCollectionInterval
 
 	return &Config{
-		ScraperControllerSettings:     cfg,
-		MetricsBuilderConfig:          metadata.DefaultMetricsBuilderConfig(),
-		CacheResources:                24 * 60 * 60,
-		CacheResourcesDefinitions:     24 * 60 * 60,
-		MaximumNumberOfMetricsInACall: 20,
-		Services:                      monitorServices,
-		Authentication:                servicePrincipal,
-		Cloud:                         defaultCloud,
+		ScraperControllerSettings:        cfg,
+		MetricsBuilderConfig:             metadata.DefaultMetricsBuilderConfig(),
+		CacheResources:                   24 * 60 * 60,
+		CacheResourcesDefinitions:        24 * 60 * 60,
+		MaximumNumberOfMetricsInACall:    20,
+		Services:                         monitorServices,
+		Authentication:                   servicePrincipal,
+		Cloud:                            defaultCloud,
+		MaximumNumberOfDimensionsInACall: 10,
 	}
 }
 
@@ -53,8 +54,16 @@ func createMetricsReceiver(_ context.Context, params receiver.CreateSettings, rC
 		return nil, errConfigNotAzureMonitor
 	}
 
-	azureScraper := newScraper(cfg, params)
-	scraper, err := scraperhelper.NewScraper(metadata.Type.String(), azureScraper.scrape, scraperhelper.WithStart(azureScraper.start))
+	var scraper scraperhelper.Scraper
+	var err error
+	if cfg.UseBatchApi {
+		azureBatchScraper := newBatchScraper(cfg, params)
+		scraper, err = scraperhelper.NewScraper(metadata.Type.String(), azureBatchScraper.scrape, scraperhelper.WithStart(azureBatchScraper.start))
+	} else {
+		azureScraper := newScraper(cfg, params)
+		scraper, err = scraperhelper.NewScraper(metadata.Type.String(), azureScraper.scrape, scraperhelper.WithStart(azureScraper.start))
+	}
+
 	if err != nil {
 		return nil, err
 	}

receiver/azuremonitorreceiver/go.mod

@@ -5,8 +5,10 @@ go 1.20
 require (
 	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1
+	github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery v1.2.0-beta.1
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armsubscriptions v1.3.0
 	github.com/google/go-cmp v0.6.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden v0.94.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.94.0