internal/host: add plugin host pagination

internal/host/plugin/host.go

@@ -196,3 +196,13 @@ func (agg *hostAgg) TableName() string {
 func (agg *hostAgg) GetPublicId() string {
 	return agg.PublicId
 }
+
+type deletedHost struct {
+	PublicId   string `gorm:"primary_key"`
+	DeleteTime *timestamp.Timestamp
+}
+
+// TableName returns the tablename to override the default gorm table name
+func (s *deletedHost) TableName() string {
+	return "host_plugin_host_deleted"
+}

internal/host/plugin/host_set_member_test.go

@@ -162,7 +162,7 @@ func TestHostSetMember_InsertDelete(t *testing.T) {
 	require.Len(t, hosts, 2)
 
 	// Base case the count by catalog ID
-	hosts, _, err = repo.ListHostsByCatalogId(ctx, blueCat.PublicId)
+	hosts, _, _, err = repo.listHosts(ctx, blueCat.PublicId)
 	require.NoError(t, err)
 	assert.Len(t, hosts, 4)
 
@@ -179,7 +179,7 @@ func TestHostSetMember_InsertDelete(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, 1, count)
 	assert.NoError(t, db.TestVerifyOplog(t, rw, blueHost1.PublicId, db.WithOperation(oplog.OpType_OP_TYPE_DELETE), db.WithCreateNotBefore(10*time.Second)))
-	hosts, _, err = repo.ListHostsByCatalogId(ctx, blueCat.PublicId)
+	hosts, _, _, err = repo.listHosts(ctx, blueCat.PublicId)
 	require.NoError(t, err)
 	require.Len(t, hosts, 3)
 
@@ -194,7 +194,7 @@ func TestHostSetMember_InsertDelete(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, 1, count)
 	assert.NoError(t, db.TestVerifyOplog(t, rw, blueHost2.PublicId, db.WithOperation(oplog.OpType_OP_TYPE_DELETE), db.WithCreateNotBefore(10*time.Second)))
-	hosts, _, err = repo.ListHostsByCatalogId(ctx, blueCat.PublicId)
+	hosts, _, _, err = repo.listHosts(ctx, blueCat.PublicId)
 	require.NoError(t, err)
 	require.Len(t, hosts, 2)
 
@@ -211,7 +211,7 @@ func TestHostSetMember_InsertDelete(t *testing.T) {
 	assert.Equal(t, 2, count)
 	assert.NoError(t, db.TestVerifyOplog(t, rw, blueHost3.PublicId, db.WithOperation(oplog.OpType_OP_TYPE_DELETE), db.WithCreateNotBefore(10*time.Second)))
 	assert.NoError(t, db.TestVerifyOplog(t, rw, blueHost4.PublicId, db.WithOperation(oplog.OpType_OP_TYPE_DELETE), db.WithCreateNotBefore(10*time.Second)))
-	hosts, _, err = repo.ListHostsByCatalogId(ctx, blueCat.PublicId)
+	hosts, _, _, err = repo.listHosts(ctx, blueCat.PublicId)
 	require.NoError(t, err)
 	require.Len(t, hosts, 0)
 }

internal/host/plugin/job_set_sync_test.go

@@ -220,7 +220,7 @@ func TestSetSyncJob_Run(t *testing.T) {
 	assert.Equal(1, r.numSets)
 	assert.Equal(1, r.numProcessed)
 	// Check the version number of the host(s)
-	hosts, _, err := hostRepo.ListHostsByCatalogId(ctx, hsa.CatalogId)
+	hosts, _, _, err := hostRepo.listHosts(ctx, hsa.CatalogId)
 	require.NoError(err)
 	assert.Len(hosts, 1)
 	for _, host := range hosts {
@@ -254,7 +254,7 @@ func TestSetSyncJob_Run(t *testing.T) {
 	assert.Equal(1, r.numSets)
 	assert.Equal(1, r.numProcessed)
 	// Check the version number of the host(s) again
-	hosts, _, err = hostRepo.ListHostsByCatalogId(ctx, hsa.CatalogId)
+	hosts, _, _, err = hostRepo.listHosts(ctx, hsa.CatalogId)
 	require.NoError(err)
 	assert.Len(hosts, 1)
 	for _, host := range hosts {

internal/host/plugin/query.go

@@ -53,5 +53,293 @@ set
   last_sync_time = current_timestamp,
   need_sync = false
 where public_id = ?
+`
+
+	estimateCountHosts = `
+select sum(reltuples::bigint) as estimate from pg_class where oid in ('host_plugin_host'::regclass)
+`
+
+	listHostsTemplate = `
+with hosts as (
+    select public_id,
+           catalog_id,
+           external_id,
+           external_name,
+           name,
+           description,
+           create_time,
+           update_time,
+           version
+      from host_plugin_host
+     where catalog_id = @catalog_id
+  order by create_time desc, public_id asc
+     limit %d
+),
+host_catalog as (
+  select project_id,
+         plugin_id,
+         public_id
+    from host_plugin_catalog
+   where public_id = @catalog_id
+),
+host_ip_addresses as (
+    select string_agg(distinct host(address), '|') as ip_addresses,
+           host_id
+      from host_ip_address
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_dns_names as (
+    select string_agg(distinct name, '|') as dns_names,
+           host_id
+      from host_dns_name
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_set_ids as (
+    select string_agg(distinct set_id, '|') as set_ids,
+           host_id
+      from host_plugin_set_member
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+final as (
+           select h.public_id,
+                  h.catalog_id,
+                  h.external_id,
+                  h.external_name,
+                  hc.project_id,
+                  hc.plugin_id,
+                  h.name,
+                  h.description,
+                  h.create_time,
+                  h.update_time,
+                  h.version,
+                  hia.ip_addresses,
+                  hdn.dns_names,
+                  hsi.set_ids
+             from hosts h
+             join host_catalog hc       on hc.public_id = h.catalog_id
+  left outer join host_ip_addresses hia on hia.host_id = h.public_id
+  left outer join host_dns_names hdn    on hdn.host_id = h.public_id
+  left outer join host_set_ids hsi      on hsi.host_id = h.public_id
+)
+  select *
+    from final
+order by create_time desc, public_id asc;
+`
+
+	listHostsPageTemplate = `
+with hosts as (
+    select public_id,
+           catalog_id,
+           external_id,
+           external_name,
+           name,
+           description,
+           create_time,
+           update_time,
+           version
+      from host_plugin_host
+     where catalog_id = @catalog_id
+       and (create_time, public_id) < (@last_item_create_time, @last_item_id)
+  order by create_time desc, public_id asc
+     limit %d
+),
+host_catalog as (
+  select project_id,
+         plugin_id,
+         public_id
+    from host_plugin_catalog
+   where public_id = @catalog_id
+),
+host_ip_addresses as (
+    select string_agg(distinct host(address), '|') as ip_addresses,
+           host_id
+      from host_ip_address
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_dns_names as (
+    select string_agg(distinct name, '|') as dns_names,
+           host_id
+      from host_dns_name
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_set_ids as (
+    select string_agg(distinct set_id, '|') as set_ids,
+           host_id
+      from host_plugin_set_member
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+final as (
+           select h.public_id,
+                  h.catalog_id,
+                  h.external_id,
+                  h.external_name,
+                  hc.project_id,
+                  hc.plugin_id,
+                  h.name,
+                  h.description,
+                  h.create_time,
+                  h.update_time,
+                  h.version,
+                  hia.ip_addresses,
+                  hdn.dns_names,
+                  hsi.set_ids
+             from hosts h
+             join host_catalog hc       on hc.public_id = h.catalog_id
+  left outer join host_ip_addresses hia on hia.host_id = h.public_id
+  left outer join host_dns_names hdn    on hdn.host_id = h.public_id
+  left outer join host_set_ids hsi      on hsi.host_id = h.public_id
+)
+  select *
+    from final
+order by create_time desc, public_id asc;
+`
+
+	listHostsRefreshTemplate = `
+with hosts as (
+    select public_id,
+           catalog_id,
+           external_id,
+           external_name,
+           name,
+           description,
+           create_time,
+           update_time,
+           version
+      from host_plugin_host
+     where catalog_id = @catalog_id
+       and update_time > @updated_after_time
+  order by update_time desc, public_id asc
+     limit %d
+),
+host_catalog as (
+  select project_id,
+         plugin_id,
+         public_id
+    from host_plugin_catalog
+   where public_id = @catalog_id
+),
+host_ip_addresses as (
+    select string_agg(distinct host(address), '|') as ip_addresses,
+           host_id
+      from host_ip_address
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_dns_names as (
+    select string_agg(distinct name, '|') as dns_names,
+           host_id
+      from host_dns_name
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_set_ids as (
+    select string_agg(distinct set_id, '|') as set_ids,
+           host_id
+      from host_plugin_set_member
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+final as (
+           select h.public_id,
+                  h.catalog_id,
+                  h.external_id,
+                  h.external_name,
+                  hc.project_id,
+                  hc.plugin_id,
+                  h.name,
+                  h.description,
+                  h.create_time,
+                  h.update_time,
+                  h.version,
+                  hia.ip_addresses,
+                  hdn.dns_names,
+                  hsi.set_ids
+             from hosts h
+             join host_catalog hc       on hc.public_id = h.catalog_id
+  left outer join host_ip_addresses hia on hia.host_id = h.public_id
+  left outer join host_dns_names hdn    on hdn.host_id = h.public_id
+  left outer join host_set_ids hsi      on hsi.host_id = h.public_id
+)
+  select *
+    from final
+order by update_time desc, public_id asc;
+`
+
+	listHostsRefreshPageTemplate = `
+with hosts as (
+    select public_id,
+           catalog_id,
+           external_id,
+           external_name,
+           name,
+           description,
+           create_time,
+           update_time,
+           version
+      from host_plugin_host
+     where catalog_id = @catalog_id
+       and update_time > @updated_after_time
+       and (update_time, public_id) < (@last_item_update_time, @last_item_id)
+  order by update_time desc, public_id asc
+     limit %d
+),
+host_catalog as (
+  select project_id,
+         plugin_id,
+         public_id
+    from host_plugin_catalog
+   where public_id = @catalog_id
+),
+host_ip_addresses as (
+    select string_agg(distinct host(address), '|') as ip_addresses,
+           host_id
+      from host_ip_address
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_dns_names as (
+    select string_agg(distinct name, '|') as dns_names,
+           host_id
+      from host_dns_name
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+host_set_ids as (
+    select string_agg(distinct set_id, '|') as set_ids,
+           host_id
+      from host_plugin_set_member
+     where host_id in (select public_id from hosts)
+  group by host_id
+),
+final as (
+           select h.public_id,
+                  h.catalog_id,
+                  h.external_id,
+                  h.external_name,
+                  hc.project_id,
+                  hc.plugin_id,
+                  h.name,
+                  h.description,
+                  h.create_time,
+                  h.update_time,
+                  h.version,
+                  hia.ip_addresses,
+                  hdn.dns_names,
+                  hsi.set_ids
+             from hosts h
+             join host_catalog hc       on hc.public_id = h.catalog_id
+  left outer join host_ip_addresses hia on hia.host_id = h.public_id
+  left outer join host_dns_names hdn    on hdn.host_id = h.public_id
+  left outer join host_set_ids hsi      on hsi.host_id = h.public_id
+)
+  select *
+    from final
+order by update_time desc, public_id asc;
 `
 )