detect bad timestamp if more than an hour earlier than previous packet

main.go

@@ -19,6 +19,7 @@ import (
 	"io"
 	"log"
 	"os"
+	"time"
 
 	"github.com/assafmo/joincap/minheap"
 	humanize "github.com/dustin/go-humanize"
@@ -27,9 +28,11 @@ import (
 	flags "github.com/jessevdk/go-flags"
 )
 
-const version = "0.8.9"
+const version = "0.9.0"
 const maxSnaplen uint32 = 262144
 
+var priorTimestamp int64
+
 func main() {
 	err := joincap(os.Args)
 	if err != nil {
@@ -107,7 +110,6 @@ func joincap(args []string) error {
 			nextPacket, err := readNext(
 				earliestPacket.Reader,
 				earliestPacket.InputFile,
-				earliestPacket.MinimumLegalTimestamp,
 				cmdFlags.Verbose)
 			if err == io.EOF {
 				break
@@ -163,9 +165,12 @@ func initHeapWithInputFiles(inputFilePaths []string, minTimeHeap *minheap.Packet
 				)
 		}
 
-		nextPacket, err := readNext(reader, inputFile, 0, verbose)
+		nextPacket, err := readNext(reader, inputFile, verbose)
 		if err == nil {
 			heap.Push(minTimeHeap, nextPacket)
+			if nextPacket.Timestamp < priorTimestamp {
+				priorTimestamp = nextPacket.Timestamp
+			}
 		} else {
 			log.Printf("%s: %v before first packet (skipping this file)\n", inputFile.Name(), err)
 		}
@@ -180,7 +185,7 @@ func initHeapWithInputFiles(inputFilePaths []string, minTimeHeap *minheap.Packet
 	return linkType, nil
 }
 
-func readNext(reader *pcapgo.Reader, inputFile *os.File, minimumLegalTimestamp int64, verbose bool) (minheap.Packet, error) {
+func readNext(reader *pcapgo.Reader, inputFile *os.File, verbose bool) (minheap.Packet, error) {
 	for {
 		data, captureInfo, err := reader.ReadPacketData()
 		if err != nil {
@@ -198,10 +203,12 @@ func readNext(reader *pcapgo.Reader, inputFile *os.File, minimumLegalTimestamp i
 			// skip errors
 			continue
 		}
-		if minimumLegalTimestamp > 0 &&
-			captureInfo.Timestamp.UnixNano() < minimumLegalTimestamp {
+		if captureInfo.Timestamp.UnixNano()+int64(time.Nanosecond*time.Hour) < priorTimestamp {
 			if verbose {
-				log.Printf("%s: illegal packet timestamp %v (skipping this packet)\n", inputFile.Name(), captureInfo.Timestamp)
+				log.Printf("%s: illegal packet timestamp %v - more than an hour before the prior packet's timestamp %v (skipping this packet)\n",
+					inputFile.Name(),
+					captureInfo.Timestamp,
+					time.Unix(0, priorTimestamp).UTC())
 			}
 			// skip errors
 			continue
@@ -214,17 +221,12 @@ func readNext(reader *pcapgo.Reader, inputFile *os.File, minimumLegalTimestamp i
 			continue
 		}
 
-		if minimumLegalTimestamp == 0 {
-			minimumLegalTimestamp = captureInfo.Timestamp.UnixNano()
-		}
-
 		return minheap.Packet{
-			Timestamp:             captureInfo.Timestamp.UnixNano(),
-			MinimumLegalTimestamp: minimumLegalTimestamp,
-			CaptureInfo:           captureInfo,
-			Data:                  data,
-			Reader:                reader,
-			InputFile:             inputFile,
+			Timestamp:   captureInfo.Timestamp.UnixNano(),
+			CaptureInfo: captureInfo,
+			Data:        data,
+			Reader:      reader,
+			InputFile:   inputFile,
 		}, nil
 	}
 }
@@ -234,4 +236,6 @@ func write(writer *pcapgo.Writer, packetToWrite minheap.Packet, verbose bool) {
 	if err != nil && verbose { // skip errors
 		log.Printf("write error: %v (skipping this packet)\n", err)
 	}
+
+	priorTimestamp = packetToWrite.Timestamp
 }

main_test.go

@@ -437,9 +437,9 @@ func TestNormalOutputSnaplenOnBigInputSnaplen(t *testing.T) {
 	}
 }
 
-// TestIgnorePacketsWithTimeEarlierThanFirst packets with timestamp smaller than the
-// first packet should be ignored
-func TestIgnorePacketsWithTimeEarlierThanFirst(t *testing.T) {
+// TestIgnorePacketsWithTimeAnHourErlierThanPriorPacket packets with timestamp
+// more than an hour before prior packet should be ignored
+func TestIgnorePacketsWithTimeAnHourErlierThanPriorPacket(t *testing.T) {
 	outputFile, err := ioutil.TempFile("", "joincap_output_")
 	if err != nil {
 		t.Fatal(err)
@@ -451,12 +451,32 @@ func TestIgnorePacketsWithTimeEarlierThanFirst(t *testing.T) {
 		"-v", "-w", outputFile.Name(),
 		"pcap_examples/second_packet_time_is_too_small.pcap"})
 
-	// the second packet is edited to have 1970 date...
+	// the second packet is edited to be 68 minutes erlier than the first packet
 	if packetCount(t, outputFile.Name()) != packetCount(t, okPcap)-1 {
 		t.FailNow()
 	}
 }
 
+// TestPacketsWithTimeLessThanHourBeforePriorPacketAreOK packets with timestamp
+// less than an hour before prior packet are ok
+func TestPacketsWithTimeLessThanHourBeforePriorPacketAreOK(t *testing.T) {
+	outputFile, err := ioutil.TempFile("", "joincap_output_")
+	if err != nil {
+		t.Fatal(err)
+	}
+	outputFile.Close()
+	defer os.Remove(outputFile.Name())
+
+	joincap([]string{"joincap",
+		"-v", "-w", outputFile.Name(),
+		"pcap_examples/second_packet_time_is_smaller_but_not_too_small.pcap"})
+
+	// the second packet is edited to be 55 minutes erlier than the first packet
+	if packetCount(t, outputFile.Name()) != packetCount(t, okPcap) {
+		t.FailNow()
+	}
+}
+
 // TestPrintVersion tests that the version is printed okay
 func TestPrintVersion(t *testing.T) {
 	savedStdout := os.Stdout

minheap/minheap.go

@@ -10,12 +10,11 @@ import (
 // Packet is used by PacketHeap to order packets by timestamp
 // and by joincap to merge pcaps
 type Packet struct {
-	Timestamp             int64
-	MinimumLegalTimestamp int64
-	CaptureInfo           gopacket.CaptureInfo
-	Data                  []byte
-	Reader                *pcapgo.Reader
-	InputFile             *os.File
+	Timestamp   int64
+	CaptureInfo gopacket.CaptureInfo
+	Data        []byte
+	Reader      *pcapgo.Reader
+	InputFile   *os.File
 }
 
 // PacketHeap is a minimum heap of packets ordered by timestamp