timeframes.txt, networks.txt, route_networks.txt (#388)

* timeframes.txt, networks.txt, route_networks.txt

* Special case handling for routes.txt:network_id

* Change RouteNetworkIDCompatFilter to use AfterWrite

* Improve handling of multi-column unique checks

* Fix migrations

adapters/direct/reader.go

@@ -35,6 +35,9 @@ type Reader struct {
 	FareMediaList        []gtfs.FareMedia
 	FareProductList      []gtfs.FareProduct
 	RiderCategoryList    []gtfs.RiderCategory
+	TimeframeList        []gtfs.Timeframe
+	NetworkList          []gtfs.Network
+	RouteNetworkList     []gtfs.RouteNetwork
 	OtherList            []tt.Entity
 }
 
@@ -396,3 +399,36 @@ func (mr *Reader) RiderCategories() chan gtfs.RiderCategory {
 	}()
 	return out
 }
+
+func (mr *Reader) Timeframes() chan gtfs.Timeframe {
+	out := make(chan gtfs.Timeframe, bufferSize)
+	go func() {
+		for _, ent := range mr.TimeframeList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}
+
+func (mr *Reader) Networks() chan gtfs.Network {
+	out := make(chan gtfs.Network, bufferSize)
+	go func() {
+		for _, ent := range mr.NetworkList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}
+
+func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork {
+	out := make(chan gtfs.RouteNetwork, bufferSize)
+	go func() {
+		for _, ent := range mr.RouteNetworkList {
+			out <- ent
+		}
+		close(out)
+	}()
+	return out
+}

adapters/multireader/reader.go

@@ -166,6 +166,18 @@ func (mr *Reader) RiderCategories() chan gtfs.RiderCategory {
 	return readEntities(mr, func(r adapters.Reader) chan gtfs.RiderCategory { return r.RiderCategories() }, setFv[*gtfs.RiderCategory])
 }
 
+func (mr *Reader) Timeframes() chan gtfs.Timeframe {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.Timeframe { return r.Timeframes() }, setFv[*gtfs.Timeframe])
+}
+
+func (mr *Reader) Networks() chan gtfs.Network {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.Network { return r.Networks() }, setFv[*gtfs.Network])
+}
+
+func (mr *Reader) RouteNetworks() chan gtfs.RouteNetwork {
+	return readEntities(mr, func(r adapters.Reader) chan gtfs.RouteNetwork { return r.RouteNetworks() }, setFv[*gtfs.RouteNetwork])
+}
+
 type canSetFV interface {
 	SetFeedVersionID(int)
 }

causes/causes.go

@@ -226,6 +226,22 @@ func (e *DuplicateIDError) Error() string {
 
 ////////////////////////////
 
+// DuplicateIDError reports when a unique ID is used more than once in a file.
+type DuplicateKeyError struct {
+	bc
+}
+
+// NewDuplicateIDError returns a new DuplicateIDErrror
+func NewDuplicateKeyError(eid string) *DuplicateKeyError {
+	return &DuplicateKeyError{bc: bc{Value: eid}}
+}
+
+func (e *DuplicateKeyError) Error() string {
+	return fmt.Sprintf("entity with fields '%s' is present more than once", e.Value)
+}
+
+////////////////////////////
+
 // DuplicateServiceExceptionError reports when a (service_id,date) value is present more than once.
 type DuplicateServiceExceptionError struct {
 	ServiceID string

copier/copier.go

@@ -111,6 +111,8 @@ type Options struct {
 	CopyExtraFiles bool
 	// Simplify shapes
 	SimplifyShapes float64
+	// Convert route network_id to networks.txt/route_networks.txt
+	NormalizeNetworks bool
 	// DeduplicateStopTimes
 	DeduplicateJourneyPatterns bool
 	// Default error handler
@@ -120,7 +122,7 @@ type Options struct {
 	// Named extensions
 	Extensions []string
 	// Initialized extensions
-	extensions []Extension
+	extensions []any
 	// Error limit
 	ErrorLimit int
 
@@ -129,7 +131,7 @@ type Options struct {
 	sublogger zerolog.Logger
 }
 
-func (opts *Options) AddExtension(ext Extension) {
+func (opts *Options) AddExtension(ext any) {
 	opts.extensions = append(opts.extensions, ext)
 }
 
@@ -241,16 +243,14 @@ func NewCopier(reader adapters.Reader, writer adapters.Writer, opts Options) (*C
 
 	// Default set of validators
 	if !opts.NoValidators {
-		copier.AddValidator(&rules.EntityDuplicateCheck{}, 0)
+		copier.AddValidator(&rules.EntityDuplicateIDCheck{}, 0)
+		copier.AddValidator(&rules.EntityDuplicateKeyCheck{}, 0)
 		copier.AddValidator(&rules.ValidFarezoneCheck{}, 0)
 		copier.AddValidator(&rules.AgencyIDConditionallyRequiredCheck{}, 0)
 		copier.AddValidator(&rules.StopTimeSequenceCheck{}, 0)
 		copier.AddValidator(&rules.InconsistentTimezoneCheck{}, 0)
 		copier.AddValidator(&rules.ParentStationLocationTypeCheck{}, 0)
 		copier.AddValidator(&rules.CalendarDuplicateDates{}, 0)
-		copier.AddValidator(&rules.DuplicateFareLegRuleCheck{}, 0)
-		copier.AddValidator(&rules.DuplicateFareTransferRuleCheck{}, 0)
-		copier.AddValidator(&rules.DuplicateFareProductCheck{}, 0)
 	}
 
 	// Default extensions
@@ -267,6 +267,12 @@ func NewCopier(reader adapters.Reader, writer adapters.Writer, opts Options) (*C
 		// Simplify shapes.txt
 		copier.AddExtension(&filters.SimplifyShapeFilter{SimplifyValue: copier.SimplifyShapes})
 	}
+	if copier.NormalizeNetworks {
+		// Convert routes.txt network_id to networks.txt/route_networks.txt
+		copier.AddExtension(&filters.RouteNetworkIDFilter{})
+	} else {
+		copier.AddExtension(&filters.RouteNetworkIDCompatFilter{})
+	}
 	if copier.SimplifyCalendars && copier.NormalizeServiceIDs {
 		// Simplify calendar and calendar dates
 		copier.AddExtension(&filters.SimplifyCalendarFilter{})
@@ -352,7 +358,9 @@ func (copier *Copier) addExtension(ext interface{}, warning bool) error {
 		added = true
 	}
 	if !added {
-		return errors.New("extension does not satisfy any extension interfaces")
+		err := errors.New("extension does not satisfy any extension interfaces")
+		log.Error().Err(err).Msg(err.Error())
+		return err
 	}
 	return nil
 }
@@ -573,6 +581,9 @@ func (copier *Copier) Copy() *Result {
 		func() error { return batchCopy(copier, batchChan(r.FeedInfos(), bs, nil)) },
 		func() error { return batchCopy(copier, batchChan(r.Translations(), bs, nil)) },
 		func() error { return batchCopy(copier, batchChan(r.Attributions(), bs, nil)) },
+		func() error { return batchCopy(copier, batchChan(r.Timeframes(), bs, nil)) },
+		func() error { return batchCopy(copier, batchChan(r.Networks(), bs, nil)) },
+		func() error { return batchCopy(copier, batchChan(r.RouteNetworks(), bs, nil)) },
 		func() error { return batchCopy(copier, batchChan(r.Areas(), bs, nil)) },
 		func() error { return batchCopy(copier, batchChan(r.StopAreas(), bs, nil)) },
 		func() error { return batchCopy(copier, batchChan(r.RiderCategories(), bs, nil)) },

dmfr/feed_version_tables.go

@@ -94,6 +94,9 @@ func GetFeedVersionTables() FeedVersionTables {
 			"gtfs_fare_products",
 			"gtfs_fare_transfer_rules",
 			"gtfs_rider_categories",
+			"gtfs_route_networks",
+			"gtfs_networks",
+			"gtfs_timeframes",
 			"gtfs_areas",
 			"gtfs_pathways",
 			"gtfs_fare_attributes",

filters/route_network_id.go

@@ -0,0 +1,55 @@
+package filters
+
+import (
+	"github.com/interline-io/transitland-lib/gtfs"
+	"github.com/interline-io/transitland-lib/tt"
+)
+
+// RouteNetworkIDFilter converts routes.txt network_id into networks.txt/route_networks.txt
+type RouteNetworkIDFilter struct{}
+
+func (e *RouteNetworkIDFilter) Expand(ent tt.Entity, emap *tt.EntityMap) ([]tt.Entity, bool, error) {
+	// Check if route and has NetworkID set
+	v, ok := ent.(*gtfs.Route)
+	if !ok {
+		return nil, false, nil
+	}
+	if !v.NetworkID.Valid {
+		return nil, false, nil
+	}
+	// Expand into route + route_network + possible network
+	var ret []tt.Entity
+	ret = append(ret, ent)
+	if _, ok := emap.Get("networks.txt", v.NetworkID.Val); !ok {
+		n := gtfs.Network{}
+		n.NetworkID.Set(v.NetworkID.Val)
+		ret = append(ret, &n)
+	}
+	rn := gtfs.RouteNetwork{}
+	rn.NetworkID.Set(v.NetworkID.Val)
+	rn.RouteID.Set(v.RouteID.Val)
+	ret = append(ret, &rn)
+	return ret, true, nil
+}
+
+func (e *RouteNetworkIDFilter) Filter(ent tt.Entity, emap *tt.EntityMap) error {
+	// Unset any set NetworkID
+	if v, ok := ent.(*gtfs.Route); ok {
+		v.NetworkID = tt.String{}
+	}
+	return nil
+}
+
+////////////
+
+// RouteNetworkIDCompatFilter copies routes.txt:network_id IDs into networks.txt:network_id
+type RouteNetworkIDCompatFilter struct{}
+
+func (e *RouteNetworkIDCompatFilter) AfterWrite(eid string, ent tt.Entity, emap *tt.EntityMap) error {
+	if v, ok := ent.(*gtfs.Route); ok {
+		if v.NetworkID.Valid {
+			emap.Set("networks.txt", v.NetworkID.Val, v.NetworkID.Val)
+		}
+	}
+	return nil
+}

gtfs/fare_leg_rule.go

@@ -1,17 +1,22 @@
 package gtfs
 
 import (
+	"fmt"
+
 	"github.com/interline-io/transitland-lib/tt"
 )
 
 // FareLegRule fare_leg_rules.txt
 type FareLegRule struct {
-	LegGroupID    tt.String
-	FromAreaID    tt.String `target:"areas.txt"`
-	ToAreaID      tt.String `target:"areas.txt"`
-	NetworkID     tt.String `target:"routes.txt:network_id"`
-	FareProductID tt.String `csv:",required" target:"fare_products.txt:fare_product_id"`
-	TransferOnly  tt.Int    `enum:"0,1"` // interline ext
+	LegGroupID           tt.String
+	FromAreaID           tt.String `target:"areas.txt"`
+	ToAreaID             tt.String `target:"areas.txt"`
+	NetworkID            tt.String `target:"networks.txt"`
+	FareProductID        tt.String `csv:",required" target:"fare_products.txt:fare_product_id"`
+	FromTimeframeGroupID tt.String `target:"timeframes.txt:timeframe_group_id"`
+	ToTimeframeGroupID   tt.String `target:"timeframes.txt:timeframe_group_id"`
+	RulePriority         tt.Int    `range:"0,"`
+	TransferOnly         tt.Int    `enum:"0,1"` // interline ext
 	tt.BaseEntity
 }
 
@@ -30,3 +35,16 @@ func (ent *FareLegRule) TableName() string {
 func (ent *FareLegRule) GroupKey() (string, string) {
 	return "leg_group_id", ent.LegGroupID.Val
 }
+
+func (ent *FareLegRule) DuplicateKey() string {
+	key := fmt.Sprintf(
+		"fare_product_id:'%s' network_id:'%s' from_area_id:'%s' to_area_id:'%s' from_timeframe_group_id:'%s' to_timeframe_group_id:'%s'",
+		ent.FareProductID.Val,
+		ent.NetworkID.Val,
+		ent.FromAreaID.Val,
+		ent.ToAreaID.Val,
+		ent.FromTimeframeGroupID.Val,
+		ent.ToTimeframeGroupID.Val,
+	)
+	return key
+}

gtfs/fare_product.go

@@ -1,6 +1,8 @@
 package gtfs
 
 import (
+	"fmt"
+
 	"github.com/interline-io/transitland-lib/causes"
 	"github.com/interline-io/transitland-lib/tt"
 )
@@ -54,3 +56,12 @@ func (ent *FareProduct) ConditionalErrors() (errs []error) {
 	}
 	return errs
 }
+
+func (ent *FareProduct) DuplicateKey() string {
+	return fmt.Sprintf(
+		"fare_product_id:'%s' rider_category_id:'%s' fare_media_id:'%s'",
+		ent.FareProductID,
+		ent.RiderCategoryID,
+		ent.FareMediaID,
+	)
+}

gtfs/fare_rule.go

@@ -1,6 +1,8 @@
 package gtfs
 
 import (
+	"fmt"
+
 	"github.com/interline-io/transitland-lib/tt"
 )
 
@@ -23,3 +25,13 @@ func (ent *FareRule) Filename() string {
 func (ent *FareRule) TableName() string {
 	return "gtfs_fare_rules"
 }
+
+func (ent *FareRule) DuplicateKey() string {
+	return fmt.Sprintf(
+		"route_id:'%s' origin_id:'%s' destination_id:'%s' contains_id:'%s'",
+		ent.RouteID.Val,
+		ent.OriginID.Val,
+		ent.DestinationID.Val,
+		ent.ContainsID.Val,
+	)
+}

gtfs/fare_transfer_rule.go

@@ -54,3 +54,15 @@ func (ent *FareTransferRule) ConditionalErrors() (errs []error) {
 	}
 	return errs
 }
+
+func (ent *FareTransferRule) DuplicateKey() string {
+	return fmt.Sprintf(
+		"fare_product_id:'%s' from_leg_group_id:'%s' to_leg_group_id:'%s' filter_fare_product_id:'%s' transfer_count:%d duration_limit:%d",
+		ent.FareProductID.Val,
+		ent.FromLegGroupID.Val,
+		ent.ToLegGroupID.Val,
+		ent.FilterFareProductID.Val,
+		ent.TransferCount.Val,
+		ent.DurationLimit.Val,
+	)
+}

gtfs/gtfs.go

@@ -36,4 +36,7 @@ type Reader interface {
 	FareProducts() chan FareProduct
 	RiderCategories() chan RiderCategory
 	FareMedia() chan FareMedia
+	Timeframes() chan Timeframe
+	Networks() chan Network
+	RouteNetworks() chan RouteNetwork
 }

gtfs/network.go

@@ -0,0 +1,25 @@
+package gtfs
+
+import "github.com/interline-io/transitland-lib/tt"
+
+type Network struct {
+	NetworkID   tt.String
+	NetworkName tt.String
+	tt.BaseEntity
+}
+
+func (ent *Network) EntityKey() string {
+	return ent.NetworkID.Val
+}
+
+func (ent *Network) EntityID() string {
+	return entID(ent.ID, ent.NetworkID.Val)
+}
+
+func (ent *Network) Filename() string {
+	return "networks.txt"
+}
+
+func (ent *Network) TableName() string {
+	return "gtfs_networks"
+}