From 95003e27ead6e775b00304021aa51281fadbac60 Mon Sep 17 00:00:00 2001 From: yachang Date: Tue, 25 Jun 2019 14:29:21 -0400 Subject: [PATCH 1/9] Update dedup.go --- cloud/bq/dedup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/dedup.go b/cloud/bq/dedup.go index 2b5cf350..a053e185 100644 --- a/cloud/bq/dedup.go +++ b/cloud/bq/dedup.go @@ -240,7 +240,7 @@ func Dedup(ctx context.Context, dsExt *dataset.Dataset, src string, destTable bq queryString = fmt.Sprintf(dedupTemplateNDT, src) case strings.HasPrefix(destTable.TableID(), "switch"): queryString = fmt.Sprintf(dedupTemplateSwitch, src) - case strings.HasPrefix(destTable.TableID(), "traceroute"): + case strings.HasPrefix(destTable.TableID(), "scamper"): queryString = fmt.Sprintf(dedupTemplateTraceroute, src) case strings.HasPrefix(destTable.TableID(), "tcpinfo"): queryString = fmt.Sprintf(dedupTemplateTCPInfo, src) From 45f25cd8ea801f8424f001e160fa994770f4b056 Mon Sep 17 00:00:00 2001 From: yachang Date: Tue, 25 Jun 2019 14:36:56 -0400 Subject: [PATCH 2/9] dedup --- cloud/bq/dedup.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cloud/bq/dedup.go b/cloud/bq/dedup.go index a053e185..94151936 100644 --- a/cloud/bq/dedup.go +++ b/cloud/bq/dedup.go @@ -177,11 +177,10 @@ var dedupTemplateSwitch = ` // as mlab-oti:batch.traceroute_20170601. var dedupTemplateTraceroute = ` #standardSQL - # Select single row based on test_id, client_ip, server_ip, src_ip, dest_ip + # Select single row based on TestTime, client_ip, server_ip SELECT * EXCEPT (row_number) FROM ( SELECT *, ROW_NUMBER() OVER ( - PARTITION BY CONCAT(test_id, connection_spec.client_ip, connection_spec.server_ip, - paris_traceroute_hop.src_ip, paris_traceroute_hop.dest_ip) + PARTITION BY CONCAT(TestTime, Source.IP, DESTINATION.IP) ) row_number FROM ` + "`%s`" + `) WHERE row_number = 1` From 6bd535357a17d493b65f1d0b93174e2179c7729f Mon Sep 17 00:00:00 2001 From: yachang Date: Wed, 26 Jun 2019 11:16:43 -0400 Subject: [PATCH 3/9] Update dedup.go --- cloud/bq/dedup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/dedup.go b/cloud/bq/dedup.go index 94151936..00aa2ee7 100644 --- a/cloud/bq/dedup.go +++ b/cloud/bq/dedup.go @@ -239,7 +239,7 @@ func Dedup(ctx context.Context, dsExt *dataset.Dataset, src string, destTable bq queryString = fmt.Sprintf(dedupTemplateNDT, src) case strings.HasPrefix(destTable.TableID(), "switch"): queryString = fmt.Sprintf(dedupTemplateSwitch, src) - case strings.HasPrefix(destTable.TableID(), "scamper"): + case strings.HasPrefix(destTable.TableID(), "traceroute"): queryString = fmt.Sprintf(dedupTemplateTraceroute, src) case strings.HasPrefix(destTable.TableID(), "tcpinfo"): queryString = fmt.Sprintf(dedupTemplateTCPInfo, src) From 72fdcd0ce8535cd0e0404a40895927b8f2934234 Mon Sep 17 00:00:00 2001 From: yachang Date: Thu, 27 Jun 2019 11:35:24 -0400 Subject: [PATCH 4/9] Update dedup.go --- cloud/bq/dedup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/dedup.go b/cloud/bq/dedup.go index 00aa2ee7..94151936 100644 --- a/cloud/bq/dedup.go +++ b/cloud/bq/dedup.go @@ -239,7 +239,7 @@ func Dedup(ctx context.Context, dsExt *dataset.Dataset, src string, destTable bq queryString = fmt.Sprintf(dedupTemplateNDT, src) case strings.HasPrefix(destTable.TableID(), "switch"): queryString = fmt.Sprintf(dedupTemplateSwitch, src) - case strings.HasPrefix(destTable.TableID(), "traceroute"): + case strings.HasPrefix(destTable.TableID(), "scamper"): queryString = fmt.Sprintf(dedupTemplateTraceroute, src) case strings.HasPrefix(destTable.TableID(), "tcpinfo"): queryString = fmt.Sprintf(dedupTemplateTCPInfo, src) From 1c8428585bd5400c44af6c3cae261a54fcdda6d5 Mon Sep 17 00:00:00 2001 From: yachang Date: Thu, 27 Jun 2019 13:50:44 -0400 Subject: [PATCH 5/9] Update dedup.go --- cloud/bq/dedup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/dedup.go b/cloud/bq/dedup.go index 94151936..ed321401 100644 --- a/cloud/bq/dedup.go +++ b/cloud/bq/dedup.go @@ -180,7 +180,7 @@ var dedupTemplateTraceroute = ` # Select single row based on TestTime, client_ip, server_ip SELECT * EXCEPT (row_number) FROM ( SELECT *, ROW_NUMBER() OVER ( - PARTITION BY CONCAT(TestTime, Source.IP, DESTINATION.IP) + PARTITION BY CONCAT(STRING(TestTime), Source.IP, DESTINATION.IP) ) row_number FROM ` + "`%s`" + `) WHERE row_number = 1` From 8ba3a429c664a969b5964f39f7a728897a533f87 Mon Sep 17 00:00:00 2001 From: yachang Date: Thu, 27 Jun 2019 15:31:01 -0400 Subject: [PATCH 6/9] Update sanity.go --- cloud/bq/sanity.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cloud/bq/sanity.go b/cloud/bq/sanity.go index d82aefd1..43a71b0e 100644 --- a/cloud/bq/sanity.go +++ b/cloud/bq/sanity.go @@ -147,7 +147,8 @@ func (at *AnnotatedTable) CheckIsRegular(ctx context.Context) error { } // GetTableDetail fetches more detailed info about a partition or table. -// Expects table to have test_id, and task_filename fields. +// Expects table to have test_id, and task_filename fields for legacy tables, +// but it is not true for new traceroute tables. func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.Table) (*Detail, error) { // If table is a partition, then we have to separate out the partition part for the query. parts := strings.Split(table.TableID(), "$") @@ -175,6 +176,13 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T FROM `+"`%s.%s`"+` %s -- where clause`, dataset, tableName, where) + + tracerouteQuery := fmt.Sprintf(` + #standardSQL + SELECT COUNT(DISTINCT ParseInfo.TaskFileName) AS TaskFileCount + FROM `+"`%s.%s`"+` + %s -- where clause`, + dataset, tableName, where) legacyNDTQuery := fmt.Sprintf(` #standardSQL @@ -190,6 +198,8 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T query = tcpinfoQuery } else if parts[0] == "legacy" { query = legacyNDTQuery + } else if parts[0] == "scamper" { + query = tracerouteQuery } err := dsExt.QueryAndParse(ctx, query, &detail) if err != nil { From 83d383fe3a565df54adb983e9aa1e18a88acdd20 Mon Sep 17 00:00:00 2001 From: yachang Date: Thu, 27 Jun 2019 15:35:54 -0400 Subject: [PATCH 7/9] Update sanity.go --- cloud/bq/sanity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/sanity.go b/cloud/bq/sanity.go index 43a71b0e..6e48dcdd 100644 --- a/cloud/bq/sanity.go +++ b/cloud/bq/sanity.go @@ -179,7 +179,7 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T tracerouteQuery := fmt.Sprintf(` #standardSQL - SELECT COUNT(DISTINCT ParseInfo.TaskFileName) AS TaskFileCount + SELECT COUNT(DISTINCT ParseInfo.TaskFileName) AS TestCount, COUNT(DISTINCT ParseInfo.TaskFileName) AS TaskFileCount FROM `+"`%s.%s`"+` %s -- where clause`, dataset, tableName, where) From 71a4ab3a8f5a930e24e8c77bd4c7694a5dcdb269 Mon Sep 17 00:00:00 2001 From: yachang Date: Fri, 28 Jun 2019 10:18:27 -0400 Subject: [PATCH 8/9] Update dedup.go --- cloud/bq/dedup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/dedup.go b/cloud/bq/dedup.go index ed321401..dd2a19d7 100644 --- a/cloud/bq/dedup.go +++ b/cloud/bq/dedup.go @@ -239,7 +239,7 @@ func Dedup(ctx context.Context, dsExt *dataset.Dataset, src string, destTable bq queryString = fmt.Sprintf(dedupTemplateNDT, src) case strings.HasPrefix(destTable.TableID(), "switch"): queryString = fmt.Sprintf(dedupTemplateSwitch, src) - case strings.HasPrefix(destTable.TableID(), "scamper"): + case strings.HasPrefix(destTable.TableID(), "traceroute"): queryString = fmt.Sprintf(dedupTemplateTraceroute, src) case strings.HasPrefix(destTable.TableID(), "tcpinfo"): queryString = fmt.Sprintf(dedupTemplateTCPInfo, src) From 1bbd5d45a031a43b029bc11ad11fd203f78db866 Mon Sep 17 00:00:00 2001 From: yachang Date: Fri, 28 Jun 2019 10:19:30 -0400 Subject: [PATCH 9/9] Update sanity.go --- cloud/bq/sanity.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/bq/sanity.go b/cloud/bq/sanity.go index 6e48dcdd..966c9699 100644 --- a/cloud/bq/sanity.go +++ b/cloud/bq/sanity.go @@ -198,7 +198,7 @@ func GetTableDetail(ctx context.Context, dsExt *dataset.Dataset, table bqiface.T query = tcpinfoQuery } else if parts[0] == "legacy" { query = legacyNDTQuery - } else if parts[0] == "scamper" { + } else if parts[0] == "traceroute" { query = tracerouteQuery } err := dsExt.QueryAndParse(ctx, query, &detail)