diff --git a/client/rest/http.go b/client/rest/http.go index 5fa256a..cc729fe 100644 --- a/client/rest/http.go +++ b/client/rest/http.go @@ -29,13 +29,14 @@ import ( "strings" "time" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/constants" ) func NewHTTPClient(proxyUrl string) (*http.Client, error) { transport := http.DefaultTransport.(*http.Transport).Clone() - transport.MaxConnsPerHost = 200 - transport.MaxIdleConnsPerHost = 200 + transport.MaxConnsPerHost = config.ColMaxConnsPerHost.Value().(int) + transport.MaxIdleConnsPerHost = config.ColMaxIdleConnsPerHost.Value().(int) transport.DisableKeepAlives = false // defaults to TLS 1.0 which is not favorable diff --git a/cmd/list-app-owners.go b/cmd/list-app-owners.go index 56df1b5..c155d82 100644 --- a/cmd/list-app-owners.go +++ b/cmd/list-app-owners.go @@ -26,6 +26,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -62,7 +63,7 @@ func listAppOwnersCmdImpl(cmd *cobra.Command, args []string) { func listAppOwners(ctx context.Context, client client.AzureClient, apps <-chan azureWrapper[models.App]) <-chan azureWrapper[models.AppOwners] { var ( out = make(chan azureWrapper[models.AppOwners]) - streams = pipeline.Demux(ctx.Done(), apps, 25) + streams = pipeline.Demux(ctx.Done(), apps, config.ColStreamCount.Value().(int)) wg sync.WaitGroup params = query.GraphParams{} ) diff --git a/cmd/list-app-role-assignments.go b/cmd/list-app-role-assignments.go index cbff4a7..675df4e 100644 --- a/cmd/list-app-role-assignments.go +++ b/cmd/list-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listAppRoleAssignments(ctx context.Context, client client.AzureClient, serv var ( out = make(chan interface{}) filteredSPs = make(chan models.ServicePrincipal) - streams = pipeline.Demux(ctx.Done(), filteredSPs, 25) + streams = pipeline.Demux(ctx.Done(), filteredSPs, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-automation-account-role-assignments.go b/cmd/list-automation-account-role-assignments.go index 3a15f9d..8627b22 100644 --- a/cmd/list-automation-account-role-assignments.go +++ b/cmd/list-automation-account-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listAutomationAccountRoleAssignments(ctx context.Context, client client.Azu var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-automation-accounts.go b/cmd/list-automation-accounts.go index 1887964..736650a 100644 --- a/cmd/list-automation-accounts.go +++ b/cmd/list-automation-accounts.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listAutomationAccounts(ctx context.Context, client client.AzureClient, subs var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-container-registries.go b/cmd/list-container-registries.go index dcdc379..3786159 100644 --- a/cmd/list-container-registries.go +++ b/cmd/list-container-registries.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listContainerRegistries(ctx context.Context, client client.AzureClient, sub var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-container-registry-role-assignments.go b/cmd/list-container-registry-role-assignments.go index 8c3a44a..7e8b696 100644 --- a/cmd/list-container-registry-role-assignments.go +++ b/cmd/list-container-registry-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listContainerRegistryRoleAssignments(ctx context.Context, client client.Azu var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-device-owners.go b/cmd/list-device-owners.go index 4fcc48d..39e9b9e 100644 --- a/cmd/list-device-owners.go +++ b/cmd/list-device-owners.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listDeviceOwners(ctx context.Context, client client.AzureClient, devices <- var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-function-app-role-assignments.go b/cmd/list-function-app-role-assignments.go index 56d5d9b..66dc6fa 100644 --- a/cmd/list-function-app-role-assignments.go +++ b/cmd/list-function-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listFunctionAppRoleAssignments(ctx context.Context, client client.AzureClie var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-function-apps.go b/cmd/list-function-apps.go index 7f961ab..a055412 100644 --- a/cmd/list-function-apps.go +++ b/cmd/list-function-apps.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listFunctionApps(ctx context.Context, client client.AzureClient, subscripti var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-group-members.go b/cmd/list-group-members.go index c465d6b..330fbac 100644 --- a/cmd/list-group-members.go +++ b/cmd/list-group-members.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -66,7 +67,7 @@ func listGroupMembers(ctx context.Context, client client.AzureClient, groups <-c var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup params = query.GraphParams{ Select: unique(listGroupMembersSelect), diff --git a/cmd/list-group-owners.go b/cmd/list-group-owners.go index ed077d7..2db015b 100644 --- a/cmd/list-group-owners.go +++ b/cmd/list-group-owners.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listGroupOwners(ctx context.Context, client client.AzureClient, groups <-ch var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup params = query.GraphParams{} ) diff --git a/cmd/list-key-vault-role-assignments.go b/cmd/list-key-vault-role-assignments.go index 0780c8b..c385d32 100644 --- a/cmd/list-key-vault-role-assignments.go +++ b/cmd/list-key-vault-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listKeyVaultRoleAssignments(ctx context.Context, client client.AzureClient, var ( out = make(chan azureWrapper[models.KeyVaultRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-key-vaults.go b/cmd/list-key-vaults.go index 6ea7641..8596a4e 100644 --- a/cmd/list-key-vaults.go +++ b/cmd/list-key-vaults.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listKeyVaults(ctx context.Context, client client.AzureClient, subscriptions var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-logic-app-role-assignments.go b/cmd/list-logic-app-role-assignments.go index 8768970..5af69f2 100644 --- a/cmd/list-logic-app-role-assignments.go +++ b/cmd/list-logic-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listLogicAppRoleAssignments(ctx context.Context, client client.AzureClient, var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-logic-apps.go b/cmd/list-logic-apps.go index e60c26c..8ce4d95 100644 --- a/cmd/list-logic-apps.go +++ b/cmd/list-logic-apps.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listLogicApps(ctx context.Context, client client.AzureClient, subscriptions var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-managed-cluster-role-assignments.go b/cmd/list-managed-cluster-role-assignments.go index 82e09a3..29713ab 100644 --- a/cmd/list-managed-cluster-role-assignments.go +++ b/cmd/list-managed-cluster-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listManagedClusterRoleAssignments(ctx context.Context, client client.AzureC var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-managed-clusters.go b/cmd/list-managed-clusters.go index 61fe220..490224b 100644 --- a/cmd/list-managed-clusters.go +++ b/cmd/list-managed-clusters.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listManagedClusters(ctx context.Context, client client.AzureClient, subscri var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-management-group-descendants.go b/cmd/list-management-group-descendants.go index 803b20f..3cbcee7 100644 --- a/cmd/list-management-group-descendants.go +++ b/cmd/list-management-group-descendants.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listManagementGroupDescendants(ctx context.Context, client client.AzureClie var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-management-group-role-assignments.go b/cmd/list-management-group-role-assignments.go index 97009b4..d003eb7 100644 --- a/cmd/list-management-group-role-assignments.go +++ b/cmd/list-management-group-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listManagementGroupRoleAssignments(ctx context.Context, client client.Azure var ( out = make(chan azureWrapper[models.ManagementGroupRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-resource-group-role-assignments.go b/cmd/list-resource-group-role-assignments.go index dc10820..5d89a56 100644 --- a/cmd/list-resource-group-role-assignments.go +++ b/cmd/list-resource-group-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listResourceGroupRoleAssignments(ctx context.Context, client client.AzureCl var ( out = make(chan azureWrapper[models.ResourceGroupRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-resource-groups.go b/cmd/list-resource-groups.go index c9b9516..586279c 100644 --- a/cmd/list-resource-groups.go +++ b/cmd/list-resource-groups.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listResourceGroups(ctx context.Context, client client.AzureClient, subscrip var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) @@ -98,7 +99,7 @@ func listResourceGroups(ctx context.Context, client client.AzureClient, subscrip } else { resourceGroup := models.ResourceGroup{ ResourceGroup: item.Ok, - SubscriptionId: "/subscriptions/"+id, + SubscriptionId: "/subscriptions/" + id, TenantId: client.TenantInfo().TenantId, } log.V(2).Info("found resource group", "resourceGroup", resourceGroup) diff --git a/cmd/list-role-assignments.go b/cmd/list-role-assignments.go index b3248a4..d9fa24b 100644 --- a/cmd/list-role-assignments.go +++ b/cmd/list-role-assignments.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listRoleAssignments(ctx context.Context, client client.AzureClient, roles < var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-service-principal-owners.go b/cmd/list-service-principal-owners.go index 7c13810..2f209ac 100644 --- a/cmd/list-service-principal-owners.go +++ b/cmd/list-service-principal-owners.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listServicePrincipalOwners(ctx context.Context, client client.AzureClient, var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-storage-account-role-assignments.go b/cmd/list-storage-account-role-assignments.go index a1b2546..af97baf 100644 --- a/cmd/list-storage-account-role-assignments.go +++ b/cmd/list-storage-account-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -65,7 +66,7 @@ func listStorageAccountRoleAssignments(ctx context.Context, client client.AzureC var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-storage-accounts.go b/cmd/list-storage-accounts.go index 000ae9c..541938b 100644 --- a/cmd/list-storage-accounts.go +++ b/cmd/list-storage-accounts.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -63,7 +64,7 @@ func listStorageAccounts(ctx context.Context, client client.AzureClient, subscri var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-storage-containers.go b/cmd/list-storage-containers.go index ff0a494..8578bf6 100644 --- a/cmd/list-storage-containers.go +++ b/cmd/list-storage-containers.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listStorageContainers(ctx context.Context, client client.AzureClient, stora // The error message with higher values for size is // "The request was throttled." // See issue #7: https://github.com/bloodhoundad/azurehound/issues/7 - streams = pipeline.Demux(ctx.Done(), ids, 2) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) @@ -104,7 +105,7 @@ func listStorageContainers(ctx context.Context, client client.AzureClient, stora storageContainer := models.StorageContainer{ StorageContainer: item.Ok, StorageAccountId: stAccount.(models.StorageAccount).StorageAccount.Id, - SubscriptionId: "/subscriptions/"+stAccount.(models.StorageAccount).SubscriptionId, + SubscriptionId: "/subscriptions/" + stAccount.(models.StorageAccount).SubscriptionId, ResourceGroupId: item.Ok.ResourceGroupId(), ResourceGroupName: item.Ok.ResourceGroupName(), TenantId: client.TenantInfo().TenantId, diff --git a/cmd/list-subscription-role-assignments.go b/cmd/list-subscription-role-assignments.go index b23421e..09ef036 100644 --- a/cmd/list-subscription-role-assignments.go +++ b/cmd/list-subscription-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listSubscriptionRoleAssignments(ctx context.Context, client client.AzureCli var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-virtual-machine-role-assignments.go b/cmd/list-virtual-machine-role-assignments.go index 389cafd..1057283 100644 --- a/cmd/list-virtual-machine-role-assignments.go +++ b/cmd/list-virtual-machine-role-assignments.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listVirtualMachineRoleAssignments(ctx context.Context, client client.AzureC var ( out = make(chan azureWrapper[models.VirtualMachineRoleAssignments]) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-virtual-machines.go b/cmd/list-virtual-machines.go index 6e4c77e..c0cc6ef 100644 --- a/cmd/list-virtual-machines.go +++ b/cmd/list-virtual-machines.go @@ -27,6 +27,7 @@ import ( "github.com/bloodhoundad/azurehound/v2/client" "github.com/bloodhoundad/azurehound/v2/client/query" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -64,7 +65,7 @@ func listVirtualMachines(ctx context.Context, client client.AzureClient, subscri var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-vm-scale-set-role-assignments.go b/cmd/list-vm-scale-set-role-assignments.go index 34841c9..343354e 100644 --- a/cmd/list-vm-scale-set-role-assignments.go +++ b/cmd/list-vm-scale-set-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listVMScaleSetRoleAssignments(ctx context.Context, client client.AzureClien var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-vm-scale-sets.go b/cmd/list-vm-scale-sets.go index 2ace333..e4c42e8 100644 --- a/cmd/list-vm-scale-sets.go +++ b/cmd/list-vm-scale-sets.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -69,7 +70,7 @@ func listVMScaleSets(ctx context.Context, client client.AzureClient, subscriptio var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) @@ -102,7 +103,7 @@ func listVMScaleSets(ctx context.Context, client client.AzureClient, subscriptio } else { vmScaleSet := models.VMScaleSet{ VMScaleSet: item.Ok, - SubscriptionId: "/subscriptions/"+id, + SubscriptionId: "/subscriptions/" + id, ResourceGroupId: item.Ok.ResourceGroupId(), TenantId: client.TenantInfo().TenantId, } diff --git a/cmd/list-web-app-role-assignments.go b/cmd/list-web-app-role-assignments.go index e4d41c2..a694928 100644 --- a/cmd/list-web-app-role-assignments.go +++ b/cmd/list-web-app-role-assignments.go @@ -27,6 +27,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -70,7 +71,7 @@ func listWebAppRoleAssignments(ctx context.Context, client client.AzureClient, w var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/list-web-apps.go b/cmd/list-web-apps.go index ec53067..dd8a845 100644 --- a/cmd/list-web-apps.go +++ b/cmd/list-web-apps.go @@ -26,6 +26,7 @@ import ( "time" "github.com/bloodhoundad/azurehound/v2/client" + "github.com/bloodhoundad/azurehound/v2/config" "github.com/bloodhoundad/azurehound/v2/enums" "github.com/bloodhoundad/azurehound/v2/models" "github.com/bloodhoundad/azurehound/v2/panicrecovery" @@ -68,7 +69,7 @@ func listWebApps(ctx context.Context, client client.AzureClient, subscriptions < var ( out = make(chan interface{}) ids = make(chan string) - streams = pipeline.Demux(ctx.Done(), ids, 25) + streams = pipeline.Demux(ctx.Done(), ids, config.ColStreamCount.Value().(int)) wg sync.WaitGroup ) diff --git a/cmd/start.go b/cmd/start.go index e817d42..01e450b 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -53,6 +53,7 @@ var ErrExceededRetryLimit = errors.New("exceeded max retry limit for ingest batc func init() { configs := append(config.AzureConfig, config.BloodHoundEnterpriseConfig...) + configs = append(configs, config.CollectionConfig...) config.Init(startCmd, configs) rootCmd.AddCommand(startCmd) } @@ -154,7 +155,7 @@ func start(ctx context.Context) { // Batch data out for ingestion stream := listAll(ctx, azClient) - batches := pipeline.Batch(ctx.Done(), stream, 256, 10*time.Second) + batches := pipeline.Batch(ctx.Done(), stream, config.ColBatchSize.Value().(int), 10*time.Second) hasIngestErr := ingest(ctx, *bheInstance, bheClient, batches) // Notify BHE instance of job end diff --git a/cmd/svc_windows.go b/cmd/svc_windows.go index 3831993..a3109f4 100644 --- a/cmd/svc_windows.go +++ b/cmd/svc_windows.go @@ -46,6 +46,7 @@ func (s *azurehoundSvc) Init(env svc.Environment) error { return err } else { log = *logr + config.CheckCollectionConfigSanity(log) if config.ConfigFileUsed() != "" { log.V(1).Info(fmt.Sprintf("Config File: %v", config.ConfigFileUsed())) diff --git a/cmd/utils.go b/cmd/utils.go index fd59f82..6d4792a 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -72,6 +72,7 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { return err } else { log = *logr + config.CheckCollectionConfigSanity(log) if config.ConfigFileUsed() != "" { log.V(1).Info(fmt.Sprintf("Config File: %v", config.ConfigFileUsed())) diff --git a/config/config.go b/config/config.go index 578262f..3e48f94 100644 --- a/config/config.go +++ b/config/config.go @@ -264,6 +264,50 @@ var ( Default: "", } + ColBatchSize = Config{ + Name: "batchSize", + Shorthand: "", + Usage: "The number of resources to send in a single batch sent to the server.", + Persistent: true, + Required: false, + Default: 100, + MinValue: 1, + MaxValue: 256, + } + + ColMaxConnsPerHost = Config{ + Name: "maxConnsPerHost", + Shorthand: "", + Usage: "The maximum number of connections made during collection.", + Persistent: true, + Required: false, + Default: 20, + MinValue: 1, + MaxValue: 200, + } + + ColMaxIdleConnsPerHost = Config{ + Name: "maxIdleConnsPerHost", + Shorthand: "", + Usage: "The maximum number of idle connections allowed during collection.", + Persistent: true, + Required: false, + Default: 20, + MinValue: 1, + MaxValue: 200, + } + + ColStreamCount = Config{ + Name: "streamCount", + Shorthand: "", + Usage: "The number of threads to use when collecting various resources.", + Persistent: true, + Required: false, + Default: 2, + MinValue: 1, + MaxValue: 25, + } + // Command specific configurations KeyVaultAccessTypes = Config{ Name: "access-types", @@ -314,6 +358,13 @@ var ( BHETokenId, BHEToken, } + + CollectionConfig = []Config{ + ColBatchSize, + ColMaxConnsPerHost, + ColMaxIdleConnsPerHost, + ColStreamCount, + } ) func ConfigFileUsed() string { diff --git a/config/internal/config.go b/config/internal/config.go index 83517ac..5cff72f 100644 --- a/config/internal/config.go +++ b/config/internal/config.go @@ -36,6 +36,8 @@ type Config struct { Required bool Persistent bool Default interface{} + MinValue int + MaxValue int } func (s Config) Value() interface{} { diff --git a/config/utils.go b/config/utils.go index 27c36c1..a7c36a1 100644 --- a/config/utils.go +++ b/config/utils.go @@ -24,6 +24,7 @@ import ( client "github.com/bloodhoundad/azurehound/v2/client/config" config "github.com/bloodhoundad/azurehound/v2/config/internal" "github.com/bloodhoundad/azurehound/v2/constants" + "github.com/go-logr/logr" ) var Init = config.Init @@ -49,6 +50,24 @@ func SetAzureDefaults() { } } +func CheckCollectionConfigSanity(log logr.Logger) { + useSaneIntValues(ColBatchSize, log) + useSaneIntValues(ColMaxConnsPerHost, log) + useSaneIntValues(ColMaxIdleConnsPerHost, log) + useSaneIntValues(ColStreamCount, log) +} + +func useSaneIntValues(c config.Config, log logr.Logger) { + val := c.Value().(int) + if val < c.MinValue { + log.V(1).Info(fmt.Sprintf("Provided value %d for config option %s is less than minimum value %d. Using default value %d.", val, c.Name, c.MinValue, c.Default)) + c.Set(c.Default) + } else if val > c.MaxValue { + log.V(1).Info(fmt.Sprintf("Provided value %d for config option %s is greater than maximum value %d. Using default value %d.", val, c.Name, c.MaxValue, c.Default)) + c.Set(c.Default) + } +} + func ValidateURL(input string) error { if parsedURL, err := url.Parse(input); err != nil { return err diff --git a/config/utils_test.go b/config/utils_test.go new file mode 100644 index 0000000..9fafccc --- /dev/null +++ b/config/utils_test.go @@ -0,0 +1,75 @@ +// Copyright (C) 2022 Specter Ops, Inc. +// +// This file is part of AzureHound. +// +// AzureHound is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// AzureHound is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +package config_test + +import ( + "testing" + + "github.com/bloodhoundad/azurehound/v2/config" + "github.com/bloodhoundad/azurehound/v2/logger" +) + +func TestCheckCollectionConfigSanity(t *testing.T) { + config.JsonLogs.Set(true) + + if logr, err := logger.GetLogger(); err != nil { + t.Errorf("Error creating logger: %v", err) + } else { + log := *logr + config.CheckCollectionConfigSanity(log) + + if config.ColBatchSize.Value().(int) != config.ColBatchSize.Default { + t.Errorf("ColBatchSize did not have the default value of %d. Actual: %d", config.ColBatchSize.Default, config.ColBatchSize.Value()) + } + + if config.ColMaxConnsPerHost.Value().(int) != config.ColMaxConnsPerHost.Default { + t.Errorf("ColMaxConnsPerHost did not have the default value of %d. Actual: %d", config.ColMaxConnsPerHost.Default, config.ColMaxConnsPerHost.Value()) + } + + if config.ColMaxIdleConnsPerHost.Value().(int) != config.ColMaxIdleConnsPerHost.Default { + t.Errorf("ColMaxIdleConnsPerHost did not have the default value of %d. Actual: %d", config.ColMaxIdleConnsPerHost.Default, config.ColMaxIdleConnsPerHost.Value()) + } + + if config.ColStreamCount.Value().(int) != config.ColStreamCount.Default { + t.Errorf("ColStreamCount did not have the default value of %d. Actual: %d", config.ColStreamCount.Default, config.ColStreamCount.Value()) + } + } +} + +func TestCheckCollectionConfigSanityOutOfBounds(t *testing.T) { + config.JsonLogs.Set(true) + + if logr, err := logger.GetLogger(); err != nil { + t.Errorf("Error creating logger: %v", err) + } else { + log := *logr + + config.ColBatchSize.Set(9999) + config.ColMaxConnsPerHost.Set(-9999) + + config.CheckCollectionConfigSanity(log) + + if config.ColBatchSize.Value().(int) != config.ColBatchSize.Default { + t.Errorf("ColBatchSize should have reverted to the default value of %d. Actual: %d", config.ColBatchSize.Default, config.ColBatchSize.Value()) + } + + if config.ColMaxConnsPerHost.Value().(int) != config.ColMaxConnsPerHost.Default { + t.Errorf("ColMaxConnsPerHost should have reverted to the default value of %d. Actual: %d", config.ColMaxConnsPerHost.Default, config.ColMaxConnsPerHost.Value()) + } + } +} diff --git a/models/azure/app_scope.go b/models/azure/app_scope.go index 45908ac..ff2da07 100644 --- a/models/azure/app_scope.go +++ b/models/azure/app_scope.go @@ -23,8 +23,8 @@ package azure // // This may be in both the following principal and scope scenarios: // -// A single principal and a single scope -// Multiple principals and multiple scopes. +// A single principal and a single scope +// Multiple principals and multiple scopes. type AppScope struct { Entity