Support pagination (#38)

Support pagination

provider/aws/docdb.go

@@ -72,7 +72,7 @@ func (r *DocumentDBManager) Detect() ([]DetectedDocumentDB, error) {
 
 	log.Info("Analyze documentDB")
 	detectedDocDB := []DetectedDocumentDB{}
-	instances, err := r.DescribeInstances()
+	instances, err := r.DescribeInstances(nil, nil)
 	if err != nil {
 		log.WithField("error", err).Error("could not describe documentDB instances")
 		return detectedDocDB, err
@@ -193,9 +193,10 @@ func (r *DocumentDBManager) GetPricingFilterInput(instance *docdb.DBInstance) *p
 }
 
 // DescribeInstances return list of documentDB instances
-func (r *DocumentDBManager) DescribeInstances() ([]*docdb.DBInstance, error) {
+func (r *DocumentDBManager) DescribeInstances(marker *string, instances []*docdb.DBInstance) ([]*docdb.DBInstance, error) {
 
 	input := &docdb.DescribeDBInstancesInput{
+		Marker: marker,
 		Filters: []*docdb.Filter{
 			&docdb.Filter{
 				Name:   awsClient.String("engine"),
@@ -209,10 +210,17 @@ func (r *DocumentDBManager) DescribeInstances() ([]*docdb.DBInstance, error) {
 		return nil, err
 	}
 
-	instances := []*docdb.DBInstance{}
+	if instances == nil {
+		instances = []*docdb.DBInstance{}
+	}
+
 	for _, instance := range resp.DBInstances {
 		instances = append(instances, instance)
 	}
 
+	if resp.Marker != nil {
+		r.DescribeInstances(marker, instances)
+	}
+
 	return instances, nil
 }

provider/aws/docdb_test.go

@@ -51,7 +51,7 @@ func TestDescribeDocdb(t *testing.T) {
 
 		docdbManager := aws.NewDocDBManager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		result, _ := docdbManager.DescribeInstances()
+		result, _ := docdbManager.DescribeInstances(nil, nil)
 
 		if len(result) != len(defaultDocdbMock.DBInstances) {
 			t.Fatalf("unexpected docdb tables count, got %d expected %d", len(result), len(defaultDocdbMock.DBInstances))
@@ -67,7 +67,7 @@ func TestDescribeDocdb(t *testing.T) {
 
 		docdbManager := aws.NewDocDBManager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		_, err := docdbManager.DescribeInstances()
+		_, err := docdbManager.DescribeInstances(nil, nil)
 
 		if err == nil {
 			t.Fatalf("unexpected describe table error, return empty")

provider/aws/ec2.go

@@ -71,7 +71,7 @@ func (r *EC2Manager) Detect() ([]DetectedEC2, error) {
 	log.Info("Analyze EC2")
 	detectedEC2 := []DetectedEC2{}
 
-	instances, err := r.DescribeInstances()
+	instances, err := r.DescribeInstances(nil, nil)
 	if err != nil {
 		return detectedEC2, err
 	}
@@ -238,9 +238,10 @@ func (r *EC2Manager) GetPricingFilterInput(instance *ec2.Instance) *pricing.GetP
 }
 
 // DescribeInstances return list of running instance
-func (r *EC2Manager) DescribeInstances() ([]*ec2.Instance, error) {
+func (r *EC2Manager) DescribeInstances(nextToken *string, instances []*ec2.Instance) ([]*ec2.Instance, error) {
 
 	input := &ec2.DescribeInstancesInput{
+		NextToken: nextToken,
 		Filters: []*ec2.Filter{
 			&ec2.Filter{
 				Name:   awsClient.String("instance-state-name"),
@@ -255,12 +256,19 @@ func (r *EC2Manager) DescribeInstances() ([]*ec2.Instance, error) {
 		return nil, err
 	}
 
-	instances := []*ec2.Instance{}
+	if instances == nil {
+		instances = []*ec2.Instance{}
+	}
+
 	for _, reservations := range resp.Reservations {
 		for _, instance := range reservations.Instances {
 			instances = append(instances, instance)
 		}
 	}
 
+	if resp.NextToken != nil {
+		r.DescribeInstances(resp.NextToken, instances)
+	}
+
 	return instances, nil
 }

provider/aws/ec2_test.go

@@ -51,7 +51,7 @@ func TestEC2DescribeInstances(t *testing.T) {
 
 		ec2Manager := aws.NewEC2Manager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		result, _ := ec2Manager.DescribeInstances()
+		result, _ := ec2Manager.DescribeInstances(nil, nil)
 
 		if len(result) != len(defaultEC2Mock.Reservations[0].Instances) {
 			t.Fatalf("unexpected ec2 instance count, got %d expected %d", len(result), len(defaultEC2Mock.Reservations[0].Instances))
@@ -67,7 +67,7 @@ func TestEC2DescribeInstances(t *testing.T) {
 
 		ec2Manager := aws.NewEC2Manager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		_, err := ec2Manager.DescribeInstances()
+		_, err := ec2Manager.DescribeInstances(nil, nil)
 
 		if err == nil {
 			t.Fatalf("unexpected describe Instances error, return empty")

provider/aws/ec2volumes.go

@@ -62,7 +62,7 @@ func (ev *EC2VolumeManager) Detect() ([]DetectedAWSEC2Volume, error) {
 	log.Info("analyze Volumes")
 
 	detected := []DetectedAWSEC2Volume{}
-	volumes, err := ev.Describe()
+	volumes, err := ev.Describe(nil, nil)
 
 	if err != nil {
 		log.WithField("error", err).Error("could not describe ec2 volumes")
@@ -164,9 +164,10 @@ func (ev *EC2VolumeManager) GetBasePricingFilterInput(vol *ec2.Volume, extraFilt
 }
 
 // Describe return list of volumes with available status
-func (ev *EC2VolumeManager) Describe() ([]*ec2.Volume, error) {
+func (ev *EC2VolumeManager) Describe(token *string, volumes []*ec2.Volume) ([]*ec2.Volume, error) {
 
 	input := &ec2.DescribeVolumesInput{
+		NextToken: token,
 		Filters: []*ec2.Filter{
 			{
 				Name:   awsClient.String("status"),
@@ -180,10 +181,15 @@ func (ev *EC2VolumeManager) Describe() ([]*ec2.Volume, error) {
 		return nil, err
 	}
 
-	volumes := []*ec2.Volume{}
+	if volumes == nil {
+		volumes = []*ec2.Volume{}
+	}
+
 	for _, vol := range resp.Volumes {
 		volumes = append(volumes, vol)
-
+	}
+	if resp.NextToken != nil {
+		ev.Describe(resp.NextToken, volumes)
 	}
 
 	return volumes, nil

provider/aws/ec2volumes_test.go

@@ -59,7 +59,7 @@ func TestDescribeVolumes(t *testing.T) {
 		}
 
 		volumeManager := aws.NewVolumesManager(&mockClient, mockStorage, pricingManager, "us-east-1")
-		response, err := volumeManager.Describe()
+		response, err := volumeManager.Describe(nil, nil)
 
 		if len(response) != 3 {
 			t.Fatalf("unexpected volume detected, got %d expected %d", len(response), 3)
@@ -78,7 +78,7 @@ func TestDescribeVolumes(t *testing.T) {
 		}
 
 		volumeManager := aws.NewVolumesManager(&mockClient, mockStorage, pricingManager, "us-east-1")
-		_, err := volumeManager.Describe()
+		_, err := volumeManager.Describe(nil, nil)
 
 		if err == nil {
 			t.Fatalf("unexpected describe Volumes error, return empty")

provider/aws/elasticache.go

@@ -72,7 +72,7 @@ func (r *ElasticacheManager) Detect() ([]DetectedElasticache, error) {
 	log.Info("Analyze elasticache")
 	detectedelasticache := []DetectedElasticache{}
 
-	instances, err := r.DescribeInstances()
+	instances, err := r.DescribeInstances(nil, nil)
 	if err != nil {
 		return detectedelasticache, err
 	}
@@ -190,21 +190,29 @@ func (r *ElasticacheManager) GetPricingFilterInput(instance *elasticache.CacheCl
 }
 
 // DescribeInstances return list of elasticache instances
-func (r *ElasticacheManager) DescribeInstances() ([]*elasticache.CacheCluster, error) {
+func (r *ElasticacheManager) DescribeInstances(Marker *string, elasticaches []*elasticache.CacheCluster) ([]*elasticache.CacheCluster, error) {
 
-	// limit := int64(20)
-	input := &elasticache.DescribeCacheClustersInput{}
+	input := &elasticache.DescribeCacheClustersInput{
+		Marker: Marker,
+	}
 
 	resp, err := r.client.DescribeCacheClusters(input)
 	if err != nil {
 		log.WithField("error", err).Error("could not describe rds instances")
 		return nil, err
 	}
 
-	instances := []*elasticache.CacheCluster{}
-	for _, instance := range resp.CacheClusters {
-		instances = append(instances, instance)
+	if elasticaches == nil {
+		elasticaches = []*elasticache.CacheCluster{}
+	}
+
+	for _, elasticache := range resp.CacheClusters {
+		elasticaches = append(elasticaches, elasticache)
+	}
+
+	if resp.Marker != nil {
+		r.DescribeInstances(resp.Marker, elasticaches)
 	}
 
-	return instances, nil
+	return elasticaches, nil
 }

provider/aws/elasticache_test.go

@@ -54,7 +54,7 @@ func TestDescribeCacheClusters(t *testing.T) {
 
 		rdsManager := aws.NewElasticacheManager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		result, _ := rdsManager.DescribeInstances()
+		result, _ := rdsManager.DescribeInstances(nil, nil)
 
 		if len(result) != len(defaultElasticacheMock.CacheClusters) {
 			t.Fatalf("unexpected elasticache instance count, got %d expected %d", len(result), len(defaultElasticacheMock.CacheClusters))
@@ -70,7 +70,7 @@ func TestDescribeCacheClusters(t *testing.T) {
 
 		rdsManager := aws.NewElasticacheManager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		_, err := rdsManager.DescribeInstances()
+		_, err := rdsManager.DescribeInstances(nil, nil)
 
 		if err == nil {
 			t.Fatalf("unexpected describe Instances error, return empty")

provider/aws/elb.go

@@ -69,7 +69,7 @@ func (r *ELBManager) Detect() ([]DetectedELB, error) {
 	log.Info("Analyze ELB")
 	detectedELB := []DetectedELB{}
 
-	instances, err := r.DescribeLoadbalancers()
+	instances, err := r.DescribeLoadbalancers(nil, nil)
 	if err != nil {
 		return detectedELB, err
 	}
@@ -199,20 +199,29 @@ func (r *ELBManager) GetPricingFilterInput() *pricing.GetProductsInput {
 }
 
 // DescribeLoadbalancers return list of load loadbalancers
-func (r *ELBManager) DescribeLoadbalancers() ([]*elb.LoadBalancerDescription, error) {
+func (r *ELBManager) DescribeLoadbalancers(marker *string, loadbalancers []*elb.LoadBalancerDescription) ([]*elb.LoadBalancerDescription, error) {
 
-	input := &elb.DescribeLoadBalancersInput{}
+	input := &elb.DescribeLoadBalancersInput{
+		Marker: marker,
+	}
 
 	resp, err := r.client.DescribeLoadBalancers(input)
 	if err != nil {
 		log.WithField("error", err).Error("could not describe elb instances")
 		return nil, err
 	}
 
-	instances := []*elb.LoadBalancerDescription{}
-	for _, instance := range resp.LoadBalancerDescriptions {
-		instances = append(instances, instance)
+	if loadbalancers == nil {
+		loadbalancers = []*elb.LoadBalancerDescription{}
+	}
+
+	for _, lb := range resp.LoadBalancerDescriptions {
+		loadbalancers = append(loadbalancers, lb)
+	}
+
+	if resp.NextMarker != nil {
+		r.DescribeLoadbalancers(resp.NextMarker, loadbalancers)
 	}
 
-	return instances, nil
+	return loadbalancers, nil
 }

provider/aws/elb_test.go

@@ -52,7 +52,7 @@ func TestDescribeLoadBalancers(t *testing.T) {
 
 		elbManager := aws.NewELBManager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		result, _ := elbManager.DescribeLoadbalancers()
+		result, _ := elbManager.DescribeLoadbalancers(nil, nil)
 
 		if len(result) != len(defaultELBMock.LoadBalancerDescriptions) {
 			t.Fatalf("unexpected elb instance count, got %d expected %d", len(result), len(defaultELBMock.LoadBalancerDescriptions))
@@ -68,7 +68,7 @@ func TestDescribeLoadBalancers(t *testing.T) {
 
 		elbManager := aws.NewELBManager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		_, err := elbManager.DescribeLoadbalancers()
+		_, err := elbManager.DescribeLoadbalancers(nil, nil)
 
 		if err == nil {
 			t.Fatalf("unexpected describe Instances error, return empty")

provider/aws/elbv2.go

@@ -70,7 +70,7 @@ func (r *ELBV2Manager) Detect() ([]DetectedELBV2, error) {
 	log.Info("Analyze ELBV2")
 	detectedELBV2 := []DetectedELBV2{}
 
-	instances, err := r.DescribeLoadbalancers()
+	instances, err := r.DescribeLoadbalancers(nil, nil)
 	if err != nil {
 		return detectedELBV2, err
 	}
@@ -207,20 +207,29 @@ func (r *ELBV2Manager) GetPricingFilterInput() *pricing.GetProductsInput {
 }
 
 // DescribeLoadbalancers return list of load loadbalancers
-func (r *ELBV2Manager) DescribeLoadbalancers() ([]*elbv2.LoadBalancer, error) {
+func (r *ELBV2Manager) DescribeLoadbalancers(marker *string, loadbalancers []*elbv2.LoadBalancer) ([]*elbv2.LoadBalancer, error) {
 
-	input := &elbv2.DescribeLoadBalancersInput{}
+	input := &elbv2.DescribeLoadBalancersInput{
+		Marker: marker,
+	}
 
 	resp, err := r.client.DescribeLoadBalancers(input)
 	if err != nil {
 		log.WithField("error", err).Error("could not describe elb instances")
 		return nil, err
 	}
 
-	instances := []*elbv2.LoadBalancer{}
-	for _, instance := range resp.LoadBalancers {
-		instances = append(instances, instance)
+	if loadbalancers == nil {
+		loadbalancers = []*elbv2.LoadBalancer{}
+	}
+
+	for _, lb := range resp.LoadBalancers {
+		loadbalancers = append(loadbalancers, lb)
+	}
+
+	if resp.NextMarker != nil {
+		r.DescribeLoadbalancers(resp.NextMarker, loadbalancers)
 	}
 
-	return instances, nil
+	return loadbalancers, nil
 }

provider/aws/elbv2_test.go

@@ -53,7 +53,7 @@ func TestDescribeLoadBalancersV2(t *testing.T) {
 
 		elbv2Manager := aws.NewELBV2Manager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		result, _ := elbv2Manager.DescribeLoadbalancers()
+		result, _ := elbv2Manager.DescribeLoadbalancers(nil, nil)
 
 		if len(result) != len(defaultELBV2Mock.LoadBalancers) {
 			t.Fatalf("unexpected elbv2 instance count, got %d expected %d", len(result), len(defaultELBV2Mock.LoadBalancers))
@@ -69,7 +69,7 @@ func TestDescribeLoadBalancersV2(t *testing.T) {
 
 		elbv2Manager := aws.NewELBV2Manager(&mockClient, mockStorage, nil, nil, metrics, "us-east-1")
 
-		_, err := elbv2Manager.DescribeLoadbalancers()
+		_, err := elbv2Manager.DescribeLoadbalancers(nil, nil)
 
 		if err == nil {
 			t.Fatalf("unexpected describe Instances error, return empty")