Make extract module public (#54)

* Add non-platform stops as (inverted) parents to station

* Update comments describing graph

* Improve error message with value

* MustGet* helper methods

* Make extract module public

causes/causes.go

@@ -271,7 +271,7 @@ func NewInvalidFieldError(field string, value string, err error) *InvalidFieldEr
 }
 
 func (e *InvalidFieldError) Error() string {
-	return fmt.Sprintf("invalid value for field %s: '%s'", e.Field, e.Value)
+	return fmt.Sprintf("invalid value for field %s: '%s', reason: %s", e.Field, e.Value, e.cause.Error())
 }
 
 ////////////////////////////

cmd/gotransit/extract.go

@@ -8,8 +8,8 @@ import (
 
 	"github.com/interline-io/gotransit"
 	"github.com/interline-io/gotransit/copier"
+	"github.com/interline-io/gotransit/extract"
 	"github.com/interline-io/gotransit/gtdb"
-	"github.com/interline-io/gotransit/internal/extract"
 	"github.com/interline-io/gotransit/internal/log"
 )
 

ext.go

@@ -2,6 +2,7 @@ package gotransit
 
 import (
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/interline-io/gotransit/internal/log"
@@ -73,6 +74,32 @@ func NewReader(url string) (Reader, error) {
 	return GetReader("csv", url)
 }
 
+// MustOpenReaderOrPanic is a helper that returns an opened reader or panics.
+func MustOpenReaderOrPanic(path string) Reader {
+	r, err := NewReader(path)
+	if err != nil {
+		panic(fmt.Sprintf("No handler for reader '%s': %s", path, err.Error()))
+	}
+	if err := r.Open(); err != nil {
+		panic(fmt.Sprintf("Could not open reader '%s': %s", path, err.Error()))
+	}
+	return r
+}
+
+// MustOpenReaderOrExit is a helper that returns an opened a reader or exits.
+func MustOpenReaderOrExit(path string) Reader {
+	r, err := NewReader(path)
+	if err != nil {
+		fmt.Printf("No handler for reader '%s': %s", path, err.Error())
+		os.Exit(1)
+	}
+	if err := r.Open(); err != nil {
+		fmt.Printf("Could not open reader '%s': %s", path, err.Error())
+		os.Exit(1)
+	}
+	return r
+}
+
 // NewWriter uses the scheme prefix as the driver name, defaulting to csv.
 func NewWriter(dburl string) (Writer, error) {
 	url := strings.Split(dburl, "://")
@@ -82,6 +109,32 @@ func NewWriter(dburl string) (Writer, error) {
 	return GetWriter("csv", dburl)
 }
 
+// MustOpenWriterOrPanic is a helper that returns an opened writer or panics.
+func MustOpenWriterOrPanic(path string) Writer {
+	r, err := NewWriter(path)
+	if err != nil {
+		panic(fmt.Sprintf("No handler for reader '%s': %s", path, err.Error()))
+	}
+	if err := r.Open(); err != nil {
+		panic(fmt.Sprintf("Could not open reader '%s': %s", path, err.Error()))
+	}
+	return r
+}
+
+// MustOpenWriterOrExit is a helper that returns an opened a writer or exits.
+func MustOpenWriterOrExit(path string) Writer {
+	r, err := NewWriter(path)
+	if err != nil {
+		fmt.Printf("No handler for writer '%s': %s", path, err.Error())
+		os.Exit(1)
+	}
+	if err := r.Open(); err != nil {
+		fmt.Printf("Could not open writer '%s': %s", path, err.Error())
+		os.Exit(1)
+	}
+	return r
+}
+
 // GetReader returns a Reader for the URL.
 func GetReader(driver string, dburl string) (Reader, error) {
 	if f, ok := readerFactories[driver]; ok {

internal/extract/marker.go → extract/marker.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/interline-io/gotransit"
 	"github.com/interline-io/gotransit/internal/graph"
+	"github.com/interline-io/gotransit/internal/log"
 )
 
 /*
@@ -13,9 +14,10 @@ agency
 	route
 		trip / stop_time
 
-station
-	stop
-		trip / stop_time
+non-platform stops (inverted)
+	station
+		platform
+			trip / stop_time
 
 calendar / calendar_dates
 	trip
@@ -25,7 +27,7 @@ shape
 
 fare_attribute / fare_rule (inverted)
 	farezone (virtual)
-		stop
+		platform
 
 -------
 
@@ -88,7 +90,7 @@ func (em *Marker) Filter(reader gotransit.Reader, fm map[string][]string) error
 	// Find all children
 	result := map[*graph.Node]bool{}
 	em.graph.Search(foundNodes[:], false, func(n *graph.Node) {
-		// log.Trace("child: %s", n)
+		log.Trace("child: %s", n)
 		result[n] = true
 	})
 	// Now find parents of all found children
@@ -97,8 +99,10 @@ func (em *Marker) Filter(reader gotransit.Reader, fm map[string][]string) error
 		check2 = append(check2, k)
 	}
 	em.graph.Search(check2[:], true, func(n *graph.Node) {
+		log.Trace("parent: %s", n)
 		result[n] = true
 	})
 	em.found = result
+	// fmt.Printf("result: %#v\n", result)
 	return nil
 }

internal/extract/marker_test.go → extract/marker_test.go

@@ -16,7 +16,7 @@ func nn(filename, eid string) node {
 
 func TestExtract_Filter_BART(t *testing.T) {
 	em := NewMarker()
-	reader, err := gtcsv.NewReader("../../testdata/external/bart.zip")
+	reader, err := gtcsv.NewReader("../testdata/external/bart.zip")
 	if err != nil {
 		t.Error(err)
 	}
@@ -38,7 +38,7 @@ func TestExtract_Filter_BART(t *testing.T) {
 }
 
 func TestExtract_Filter_ExampleFeed(t *testing.T) {
-	reader, err := gtcsv.NewReader("../../testdata/extract-examples")
+	reader, err := gtcsv.NewReader("../testdata/extract-examples")
 	if err != nil {
 		t.Error(err)
 	}

gtfs_calendar.go

@@ -47,12 +47,12 @@ func (ent *Calendar) Errors() (errs []error) {
 	errs = ValidateTags(ent)
 	errs = append(errs, ent.BaseEntity.loadErrors...)
 	if ent.StartDate.IsZero() {
-		errs = append(errs, causes.NewInvalidFieldError("start_date", "", fmt.Errorf("start_date is empty")))
+		errs = append(errs, causes.NewInvalidFieldError("start_date", ent.StartDate.String(), fmt.Errorf("start_date is empty")))
 	}
 	if ent.EndDate.IsZero() {
-		errs = append(errs, causes.NewInvalidFieldError("end_date", "", fmt.Errorf("end_date is empty")))
+		errs = append(errs, causes.NewInvalidFieldError("end_date", ent.EndDate.String(), fmt.Errorf("end_date is empty")))
 	} else if ent.EndDate.Before(ent.StartDate) {
-		errs = append(errs, causes.NewInvalidFieldError("end_date", "", fmt.Errorf("end_date '%s' must come after start_date '%s'", ent.EndDate, ent.StartDate)))
+		errs = append(errs, causes.NewInvalidFieldError("end_date", ent.EndDate.String(), fmt.Errorf("end_date '%s' must come after start_date '%s'", ent.EndDate, ent.StartDate)))
 	}
 	return errs
 }

internal/graph/graph.go

@@ -1,27 +1,27 @@
-package graph 
+package graph
 
 type Node struct {
 	Filename string
-	ID string
+	ID       string
 }
 
 func NewNode(filename, id string) *Node {
 	return &Node{
 		Filename: filename,
-		ID: id,
+		ID:       id,
 	}
 }
 
 type EntityGraph struct {
-	Nodes map[Node]*Node
-	Parents map[Node][]*Node
+	Nodes    map[Node]*Node
+	Parents  map[Node][]*Node
 	Children map[Node][]*Node
 }
 
 func NewEntityGraph() *EntityGraph {
 	return &EntityGraph{
-		Nodes: map[Node]*Node{},
-		Parents: map[Node][]*Node{},
+		Nodes:    map[Node]*Node{},
+		Parents:  map[Node][]*Node{},
 		Children: map[Node][]*Node{},
 	}
 }
@@ -34,14 +34,14 @@ func (eg *EntityGraph) Node(n *Node) (*Node, bool) {
 func (eg *EntityGraph) AddNode(n *Node) (*Node, bool) {
 	if found, ok := eg.Nodes[*n]; ok {
 		return found, false
-	} 
+	}
 	eg.Nodes[*n] = n
 	return n, true
 }
 
 func (eg *EntityGraph) AddEdge(n1, n2 *Node) bool {
 	eg.Children[*n1] = append(eg.Children[*n1], n2) // deref
-	eg.Parents[*n2] = append(eg.Parents[*n2], n1) // deref
+	eg.Parents[*n2] = append(eg.Parents[*n2], n1)   // deref
 	return true
 }
 
@@ -70,7 +70,7 @@ func (eg *EntityGraph) Search(queue []*Node, up bool, f func(*Node)) {
 		} else {
 			edges, _ = eg.findParents(cur)
 		}
-		for i:=0;i<len(edges);i++ {
+		for i := 0; i < len(edges); i++ {
 			j := edges[i]
 			if !visited[j] {
 				queue = append(queue, j)

internal/graph/load.go

@@ -1,6 +1,8 @@
 package graph
 
-import "github.com/interline-io/gotransit"
+import (
+	"github.com/interline-io/gotransit"
+)
 
 // we just need EntityID / Filename
 type entity interface {
@@ -63,25 +65,35 @@ func BuildGraph(reader gotransit.Reader) (*EntityGraph, error) {
 	}
 	// Add Stops and link to parent stations
 	ps := map[string]string{}   // parent stations
+	cs := map[string][]string{} // non-platform stops in stations
 	fz := map[string][]string{} // farezones	1
 	for ent := range reader.Stops() {
 		en := entityNode(&ent)
 		eg.AddNode(en)
 		if ent.ParentStation.Key != "" {
 			ps[ent.StopID] = ent.ParentStation.Key
+			cs[ent.ParentStation.Key] = append(cs[ent.ParentStation.Key], ent.StopID)
 		}
-		if len(ent.ZoneID) > 0 {
+		if ent.ZoneID != "" {
 			fz[ent.ZoneID] = append(fz[ent.ZoneID], ent.StopID)
 		}
 	}
 	// Add stops to parent stops
-	for k, sid := range ps {
-		a, ok1 := eg.Node(NewNode("stops.txt", sid))
-		b, ok2 := eg.Node(NewNode("stops.txt", k))
+	for sid, parentid := range ps {
+		a, ok1 := eg.Node(NewNode("stops.txt", parentid))
+		b, ok2 := eg.Node(NewNode("stops.txt", sid))
 		if ok1 && ok2 {
 			eg.AddEdge(a, b)
 		}
+		// Add non-platform stops, inverted as parents of station
+		for _, npsid := range cs[parentid] {
+			c, ok3 := eg.Node(NewNode("stops.txt", npsid))
+			if ok1 && ok3 {
+				eg.AddEdge(c, a)
+			}
+		}
 	}
+
 	// Add stops to farezones
 	for k, sids := range fz {
 		fn, _ := eg.AddNode(NewNode("farezone", k))