Branch that have jjneely#20 commit and is buildable from this repo

cmd/bucky-fill/main.go

@@ -9,8 +9,8 @@ import (
 	"strings"
 )
 
-import "github.com/jjneely/buckytools"
-import "github.com/jjneely/buckytools/fill"
+import "github.com/Civil/buckytools"
+import "github.com/Civil/buckytools/fill"
 
 var (
 	deleteSourceFiles bool

cmd/bucky-isempty/main.go

@@ -11,9 +11,9 @@ import (
 )
 
 import (
-	"github.com/jjneely/buckytools"
-	"github.com/jjneely/buckytools/metrics"
-	"github.com/jjneely/buckytools/whisper"
+	"github.com/Civil/buckytools"
+	"github.com/Civil/buckytools/metrics"
+	"github.com/Civil/buckytools/whisper"
 )
 
 // Command Line Flags

cmd/bucky-sparsify/main.go

@@ -12,8 +12,8 @@ import (
 )
 
 import (
-	"github.com/jjneely/buckytools"
-	"github.com/jjneely/buckytools/lock"
+	"github.com/Civil/buckytools"
+	"github.com/Civil/buckytools/lock"
 )
 
 // Command Line Flags

cmd/bucky/backfill.go

@@ -10,7 +10,7 @@ import (
 	"time"
 )
 
-// import "github.com/jjneely/buckytools/hashing"
+// import "github.com/Civil/buckytools/hashing"
 
 type MigrateWork struct {
 	oldName     string

cmd/bucky/cluster.go

@@ -6,7 +6,7 @@ import (
 	"net"
 )
 
-import "github.com/jjneely/buckytools/hashing"
+import "github.com/Civil/buckytools/hashing"
 
 type ClusterConfig struct {
 	// Port is the port remote buckyd daemons listen on

cmd/bucky/common.go

@@ -15,8 +15,8 @@ import (
 
 import "github.com/golang/snappy"
 
-import . "github.com/jjneely/buckytools/metrics"
-import "github.com/jjneely/buckytools/hashing"
+import . "github.com/Civil/buckytools/metrics"
+import "github.com/Civil/buckytools/hashing"
 
 // HostPort is a convenience variable for sub-commands.  This holds the
 // HOST:PORT to connect to if SetupHostname() is called in init()

cmd/bucky/help.go

@@ -5,7 +5,7 @@ import (
 	"os"
 )
 
-import . "github.com/jjneely/buckytools"
+import . "github.com/Civil/buckytools"
 
 func init() {
 	usage := "[sub-command]"

cmd/bucky/inconsistent.go

@@ -11,7 +11,7 @@ import (
 	"time"
 )
 
-// import "github.com/jjneely/buckytools/hashing"
+// import "github.com/Civil/buckytools/hashing"
 
 func init() {
 	usage := "[options]"

cmd/bucky/main.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 )
 
-import . "github.com/jjneely/buckytools"
+import . "github.com/Civil/buckytools"
 
 // We use STDIN and STDOUT as much as possible for handing lists, and
 // other data.  Status, errors, and other data not related to pushing

cmd/bucky/restore.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 )
 
-import . "github.com/jjneely/buckytools/metrics"
+import . "github.com/Civil/buckytools/metrics"
 
 var tarPrefix string
 

cmd/bucky/stat.go

@@ -11,7 +11,7 @@ import (
 	"time"
 )
 
-import . "github.com/jjneely/buckytools/metrics"
+import . "github.com/Civil/buckytools/metrics"
 
 func init() {
 	usage := "[options] <metric expression>"

cmd/bucky/tar.go

@@ -14,7 +14,7 @@ import (
 )
 
 import "github.com/golang/crypto/ssh/terminal"
-import "github.com/jjneely/buckytools/metrics"
+import "github.com/Civil/buckytools/metrics"
 
 var metricWorkers int
 var workerErrors bool

cmd/buckyd/main.go

@@ -10,9 +10,9 @@ import (
 	"strings"
 )
 
-import . "github.com/jjneely/buckytools"
-import "github.com/jjneely/buckytools/metrics"
-import "github.com/jjneely/buckytools/hashing"
+import . "github.com/Civil/buckytools"
+import "github.com/Civil/buckytools/metrics"
+import "github.com/Civil/buckytools/hashing"
 
 var metricsCache *metrics.MetricsCacheType
 var tmpDir string

cmd/buckyd/metrics.go

@@ -15,8 +15,8 @@ import (
 
 import "github.com/golang/snappy"
 
-import . "github.com/jjneely/buckytools/metrics"
-import "github.com/jjneely/buckytools/fill"
+import . "github.com/Civil/buckytools/metrics"
+import "github.com/Civil/buckytools/fill"
 
 // listMetrics retrieves a list of metrics on the localhost and sends
 // it to the client.

cmd/findhash/main.go

@@ -11,7 +11,7 @@ import (
 	"strings"
 )
 
-import "github.com/jjneely/buckytools/hashing"
+import "github.com/Civil/buckytools/hashing"
 import "github.com/pborman/uuid"
 
 func getConfig(file string) []string {

datapoints.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-import "github.com/jjneely/buckytools/whisper"
+import "github.com/Civil/buckytools/whisper"
 
 // FindValidDataPoints does a backwards walk through time to examine the
 // highest resolution data for each archive / time period.  We collect valid

fill/fill.go

@@ -6,7 +6,7 @@ import (
 	"time"
 )
 
-import "github.com/jjneely/buckytools/whisper"
+import "github.com/Civil/buckytools/whisper"
 
 // fillArchive() is a private function that fills data points from srcWSP
 // into dstWsp.  Used by FIll()
@@ -54,7 +54,7 @@ func fillArchive(srcWsp, dstWsp *whisper.Whisper, start, stop int) error {
 			}
 			tsStart += ts.Step()
 		}
-		dstWsp.UpdateMany(points)
+		dstWsp.UpdateManyWithRetention(points, v.MaxRetention())
 
 		stop = fromTime
 		if start >= stop {

fill/fill_test.go

@@ -11,7 +11,7 @@ import (
 	"time"
 )
 
-import "github.com/jjneely/buckytools/whisper"
+import "github.com/Civil/buckytools/whisper"
 
 func whisperCreateData(path string, ts []*whisper.TimeSeriesPoint) error {
 	os.Remove(path) // Don't care if it fails
@@ -113,7 +113,9 @@ func validateWhisper(path string, ts []*whisper.TimeSeriesPoint) error {
 	}
 	defer wsp.Close()
 
-	wspData, err := wsp.Fetch(0, int(time.Now().Unix()))
+	now := int(time.Now().Unix())
+	fromTime := now - wsp.Retentions()[0].MaxRetention()
+	wspData, err := wsp.Fetch(fromTime, now)
 	if err != nil {
 		return err
 	}
@@ -151,8 +153,11 @@ func fetchFromFile(path string) ([]*whisper.TimeSeriesPoint, error) {
 	}
 	defer wsp.Close()
 
+	now := int(time.Now().Unix())
+	fromTime := now - wsp.Retentions()[0].MaxRetention()
+
 	// Parse and fetch data from it
-	ts, err := wsp.Fetch(0, int(time.Now().Unix()))
+	ts, err := wsp.Fetch(fromTime, now)
 	if err != nil {
 		return tsp, err
 	}
@@ -164,7 +169,7 @@ func simulateFill(a, b []*whisper.TimeSeriesPoint) []*whisper.TimeSeriesPoint {
 	// Assume that we are simulating the fill operation on WSP DBs created
 	// with the above functions.
 	// This is a shallow copy operation.
-	dataMerged := make([]*whisper.TimeSeriesPoint, 30)
+	dataMerged := make([]*whisper.TimeSeriesPoint, len(b))
 	// copy everything in b over to our return value
 	copy(dataMerged, b)
 	gapstart := -1
@@ -240,6 +245,155 @@ func TestFill(t *testing.T) {
 	fmt.Println()
 }
 
+func whisperCreateDataMany(path string, ts []*whisper.TimeSeriesPoint) error {
+	os.Remove(path) // Don't care if it fails
+	retentions, err := whisper.ParseRetentionDefs("1m:30m,5m:60m,15m:150m")
+	if err != nil {
+		return err
+	}
+	wsp, err := whisper.Create(path, retentions, whisper.Sum, 0)
+	if err != nil {
+		return err
+	}
+	defer wsp.Close()
+
+	// Iterate through the slice so we can support null values
+	wsp.UpdateMany(ts)
+
+	return nil
+}
+
+func whisperCreateNullsManyArchives(path string) ([]*whisper.TimeSeriesPoint, error) {
+	values := []float64{
+		math.NaN(),
+		math.NaN(),
+		math.NaN(),
+		0.0,
+		7.0,
+		1.0,
+		7.0,
+		2.0,
+		9.0,
+		9.0,
+		9.0,
+		4.0,
+		3.0,
+		2.0,
+		7.0,
+		5.0,
+		1.0,
+		9.0,
+		4.0,
+		4.0,
+		1.0,
+		5.0,
+		5.0,
+		8.0,
+		4.0,
+		2.0,
+		6.0,
+		0.0,
+		math.NaN(),
+		math.NaN(),
+	}
+
+	now := int(time.Now().Unix())
+	ts := make([]*whisper.TimeSeriesPoint, 0, len(values))
+	for _, v := range values {
+		ts = append(ts, &whisper.TimeSeriesPoint{Value: v, Time: now})
+		now -= 60
+	}
+
+	os.Remove(path) // Don't care if it fails
+	retentions, err := whisper.ParseRetentionDefs("1m:30m,5m:60m,15m:150m")
+	if err != nil {
+		return ts, err
+	}
+	wsp, err := whisper.Create(path, retentions, whisper.Sum, 0)
+	if err != nil {
+		return ts, err
+	}
+	defer wsp.Close()
+
+	for _, point := range ts {
+		_ = wsp.Update(point.Value, point.Time)
+	}
+
+	return ts, nil
+}
+
+func TestTwoArchives(t *testing.T) {
+	dataC, err := whisperCreateNullsManyArchives("c1.wsp")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create an identical set of test data
+	err = whisperCreateDataMany("c2.wsp", dataC)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = whisperCreateDataMany("d1.wsp", dataC)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = whisperCreateDataMany("d2.wsp", dataC)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// whisper-fill.py needs to be in the PATH somewhere
+	log.Println("Running whisper-fill.py...")
+	c := exec.Command("whisper-fill.py", "c1.wsp", "d1.wsp")
+
+	reference_err := c.Run()
+	pythonFill, err := fetchFromFile("d1.wsp")
+
+	// Run my version
+	err = Files("c2.wsp", "d2.wsp", int(time.Now().Unix()))
+	if err != nil {
+		t.Error(err)
+	}
+	goFill, err := fetchFromFile("d2.wsp")
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Compare to what we think our version should be
+	simuFill := simulateFill(dataC, dataC)
+
+	err = validateWhisper("d2.wsp", simuFill)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Validate the reference if whisper-fill.py was found
+	if reference_err == nil {
+		err = validateWhisper("d1.wsp", simuFill)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+
+	if len(goFill) != len(pythonFill) {
+		t.Fatalf("length mismatch, python=%v, go=%v, expected=%v", len(goFill), len(pythonFill))
+	}
+
+	// Now try to print out a table of C, D, Python, Go, Simu
+	fmt.Printf("C     \tD     \tPython\tGo    \tSimu\n")
+	fmt.Printf("======\t======\t======\t======\t======\n")
+	for i := 0; i < len(goFill); i++ {
+		if reference_err != nil {
+			fmt.Printf("%6.1f\t%6.1f\t%6.1f\t%6.1f\t%6.1f\n", dataC[i].Value, dataC[i].Value, math.NaN(), goFill[i].Value, simuFill[i].Value)
+		} else {
+			fmt.Printf("%6.1f\t%6.1f\t%6.1f\t%6.1f\t%6.1f\n", dataC[i].Value, dataC[i].Value, pythonFill[i].Value, goFill[i].Value, simuFill[i].Value)
+		}
+	}
+	fmt.Println()
+}
+
 func TestReference(t *testing.T) {
 	// Create our random test data
 	dataA, err := whisperCreate("a1.wsp")