Make ShapeLine, similar to Service, for handling compound entities (#380

)

* Make ShapeLine, similar to Service, for handling compound entities

* Update tests

adapters/direct/reader.go

@@ -80,9 +80,18 @@ func (mr *Reader) StopTimesByTripID(...string) chan []gtfs.StopTime {
 
 // ShapesByShapeID .
 func (mr *Reader) ShapesByShapeID(...string) chan []gtfs.Shape {
-	c := make(chan []gtfs.Shape, 1000)
-	close(c)
-	return c
+	out := make(chan []gtfs.Shape, 1000)
+	go func() {
+		ents := map[string][]gtfs.Shape{}
+		for _, ent := range mr.ShapeList {
+			ents[ent.ShapeID.Val] = append(ents[ent.ShapeID.Val], ent)
+		}
+		for _, v := range ents {
+			out <- v
+		}
+		close(out)
+	}()
+	return out
 }
 
 // ShapeLinesByShapeID .

adapters/direct/writer.go

@@ -51,14 +51,16 @@ func (mw *Writer) NewReader() (adapters.Reader, error) {
 // AddEntity .
 func (mw *Writer) AddEntity(ent tt.Entity) (string, error) {
 	switch v := ent.(type) {
+	case *service.Service:
+		mw.Reader.CalendarList = append(mw.Reader.CalendarList, v.Calendar)
+	case *service.ShapeLine:
+		mw.Reader.ShapeList = append(mw.Reader.ShapeList, service.FlattenShape(*v)...)
 	case *gtfs.Stop:
 		mw.Reader.StopList = append(mw.Reader.StopList, *v)
 	case *gtfs.StopTime:
 		mw.Reader.StopTimeList = append(mw.Reader.StopTimeList, *v)
 	case *gtfs.Agency:
 		mw.Reader.AgencyList = append(mw.Reader.AgencyList, *v)
-	case *service.Service:
-		mw.Reader.CalendarList = append(mw.Reader.CalendarList, v.Calendar)
 	case *gtfs.Calendar:
 		mw.Reader.CalendarList = append(mw.Reader.CalendarList, *v)
 	case *gtfs.CalendarDate:

adapters/empty/reader.go

@@ -38,6 +38,11 @@ func (mr *Reader) StopTimesByTripID(ids ...string) chan []gtfs.StopTime {
 	return readNullEntities[[]gtfs.StopTime](mr)
 }
 
+// ShapesByShapeID .
+func (mr *Reader) ShapesByShapeID(...string) chan []gtfs.Shape {
+	return readNullEntities[[]gtfs.Shape](mr)
+}
+
 func (mr *Reader) Stops() chan gtfs.Stop {
 	return readNullEntities[gtfs.Stop](mr)
 }

adapters/multireader/reader.go

@@ -66,6 +66,10 @@ func (mr *Reader) StopTimesByTripID(ids ...string) chan []gtfs.StopTime {
 	return readEntities(mr, func(r adapters.Reader) chan []gtfs.StopTime { return r.StopTimesByTripID(ids...) }, nil)
 }
 
+func (mr *Reader) ShapesByShapeID(ids ...string) chan []gtfs.Shape {
+	return readEntities(mr, func(r adapters.Reader) chan []gtfs.Shape { return r.ShapesByShapeID(ids...) }, nil)
+}
+
 func (mr *Reader) Stops() chan gtfs.Stop {
 	return readEntities(mr, func(r adapters.Reader) chan gtfs.Stop { return r.Stops() }, setFv[*gtfs.Stop])
 }

adapters/reader.go

@@ -2,37 +2,24 @@ package adapters
 
 import "github.com/interline-io/transitland-lib/gtfs"
 
-// Reader defines an interface for reading entities from a GTFS feed.
+// Reader is the main interface for reading GTFS data
 type Reader interface {
+	EntityReader
+	GtfsReader
+}
+
+// EntityReader defines methods for opening a reader, validating its structure, and reading entities through reflection
+type EntityReader interface {
 	Open() error
 	Close() error
 	ValidateStructure() []error
-	StopTimesByTripID(...string) chan []gtfs.StopTime
 	String() string
-	// Entities
 	ReadEntities(c interface{}) error
-	Stops() chan gtfs.Stop
-	StopTimes() chan gtfs.StopTime
-	Agencies() chan gtfs.Agency
-	Calendars() chan gtfs.Calendar
-	CalendarDates() chan gtfs.CalendarDate
-	FareAttributes() chan gtfs.FareAttribute
-	FareRules() chan gtfs.FareRule
-	FeedInfos() chan gtfs.FeedInfo
-	Frequencies() chan gtfs.Frequency
-	Routes() chan gtfs.Route
-	Shapes() chan gtfs.Shape
-	Transfers() chan gtfs.Transfer
-	Pathways() chan gtfs.Pathway
-	Levels() chan gtfs.Level
-	Trips() chan gtfs.Trip
-	Translations() chan gtfs.Translation
-	Attributions() chan gtfs.Attribution
-	Areas() chan gtfs.Area
-	StopAreas() chan gtfs.StopArea
-	FareLegRules() chan gtfs.FareLegRule
-	FareTransferRules() chan gtfs.FareTransferRule
-	FareProducts() chan gtfs.FareProduct
-	RiderCategories() chan gtfs.RiderCategory
-	FareMedia() chan gtfs.FareMedia
+}
+
+// GtfsReader defines methods for accessing core GTFS entities
+type GtfsReader interface {
+	gtfs.Reader
+	StopTimesByTripID(...string) chan []gtfs.StopTime
+	ShapesByShapeID(...string) chan []gtfs.Shape
 }

copier/copier.go

@@ -800,7 +800,8 @@ func (copier *Copier) copyTransfers() error {
 // copyShapes writes Shapes
 func (copier *Copier) copyShapes() error {
 	// Not safe for batch copy (currently)
-	for ent := range copier.Reader.Shapes() {
+	for shapeEnts := range copier.Reader.ShapesByShapeID() {
+		ent := service.NewShapeLineFromShapes(shapeEnts)
 		sid := ent.EntityID()
 		if copier.SimplifyShapes > 0 {
 			simplifyValue := copier.SimplifyShapes / 1e6
@@ -1219,7 +1220,7 @@ func (copier *Copier) createMissingShape(shapeID string, stoptimes []gtfs.StopTi
 	for i := 0; i < len(line); i++ {
 		flatCoords = append(flatCoords, line[i].Lon, line[i].Lat, dists[i])
 	}
-	shape := gtfs.Shape{}
+	shape := service.ShapeLine{}
 	shape.Generated = true
 	shape.ShapeID.Set(shapeID)
 	shape.Geometry = tt.NewLineStringFromFlatCoords(flatCoords)

dmfr/feed.go

@@ -76,16 +76,6 @@ func (ent *Feed) Equal(other *Feed) bool {
 	return string(a1j) == string(a2j)
 }
 
-// SetID .
-func (ent *Feed) SetID(id int) {
-	ent.ID = id
-}
-
-// GetID .
-func (ent *Feed) GetID() int {
-	return ent.ID
-}
-
 // EntityID .
 func (ent *Feed) EntityID() string {
 	return strconv.Itoa(ent.ID)

dmfr/feed_fetch.go

@@ -4,6 +4,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
+// FeedFetch is a record of when feed data was fetched via a URL
 type FeedFetch struct {
 	FeedID        int
 	URLType       string

dmfr/feed_state.go

@@ -6,7 +6,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
-// FeedState stores the pointer to the active FeedVersion.
+// FeedState stores a pointer to the active FeedVersion and values that control feed fetch and permissions.
 type FeedState struct {
 	FeedID              int
 	FeedVersionID       tt.Int
@@ -23,16 +23,6 @@ func (ent *FeedState) EntityID() string {
 	return strconv.Itoa(ent.ID)
 }
 
-// SetID .
-func (ent *FeedState) SetID(id int) {
-	ent.ID = id
-}
-
-// GetID .
-func (ent *FeedState) GetID() int {
-	return ent.ID
-}
-
 // TableName .
 func (ent *FeedState) TableName() string {
 	return "feed_states"

dmfr/feed_version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
-// FeedVersion represents a single GTFS data source.
+// FeedVersion represents a single version of a GTFS data source.
 type FeedVersion struct {
 	FeedID               int
 	SHA1                 string
@@ -26,16 +26,6 @@ type FeedVersion struct {
 	tt.Timestamps
 }
 
-// SetID .
-func (ent *FeedVersion) SetID(id int) {
-	ent.ID = id
-}
-
-// GetID .
-func (ent *FeedVersion) GetID() int {
-	return ent.ID
-}
-
 // EntityID .
 func (ent *FeedVersion) EntityID() string {
 	return strconv.Itoa(ent.ID)

dmfr/feed_version_file_info.go

@@ -6,7 +6,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
-// FeedVersionFileInfo .
+// FeedVersionFileInfo describes the individual files in a static GTFS feed
 type FeedVersionFileInfo struct {
 	Name         string
 	Size         int64

dmfr/feed_version_import.go

@@ -6,7 +6,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
-// FeedVersionImport .
+// FeedVersionImport is a record of when a feed version was imported into the database.
 type FeedVersionImport struct {
 	ImportLog                 string
 	ExceptionLog              string
@@ -40,16 +40,6 @@ func NewFeedVersionImport() *FeedVersionImport {
 	return &fvi
 }
 
-// GetID returns the ID
-func (fvi *FeedVersionImport) GetID() int {
-	return fvi.ID
-}
-
-// SetID sets the ID.
-func (fvi *FeedVersionImport) SetID(v int) {
-	fvi.ID = v
-}
-
 // EntityID .
 func (fvi *FeedVersionImport) EntityID() string {
 	return strconv.Itoa(fvi.ID)

dmfr/feed_version_service_level.go

@@ -6,7 +6,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
-// FeedVersionServiceLevel .
+// FeedVersionServiceLevel is a cached summary of the number of seconds of scheduled service for a GTFS feed on a week-by-week basis.
 type FeedVersionServiceLevel struct {
 	StartDate tt.Date
 	EndDate   tt.Date

dmfr/feed_version_service_window.go

@@ -6,6 +6,7 @@ import (
 	"github.com/interline-io/transitland-lib/tt"
 )
 
+// FeedVersionServiceWindow is a cached summary of the overall start and end dates for a feed version, sourced from feed_info.txt and calendar.txt.
 type FeedVersionServiceWindow struct {
 	FeedStartDate        tt.Date
 	FeedEndDate          tt.Date

dmfr/feed_version_tables.go

@@ -1,4 +1,3 @@
-// Package dmfr provides tool and utilities for working with DMFR files and database representations of feeds and feed versions.
 package dmfr
 
 type FeedVersionTables struct {

dmfr/operator.go

@@ -43,16 +43,6 @@ func (Operator) TableName() string {
 	return "current_operators"
 }
 
-// SetID .
-func (ent *Operator) SetID(id int) {
-	ent.ID = id
-}
-
-// GetID .
-func (ent *Operator) GetID() int {
-	return ent.ID
-}
-
 ////////////
 
 type OperatorAssociatedFeed struct {

ext/builders/route_geometry_builder.go

@@ -7,6 +7,7 @@ import (
 	"github.com/interline-io/log"
 	"github.com/interline-io/transitland-lib/copier"
 	"github.com/interline-io/transitland-lib/gtfs"
+	"github.com/interline-io/transitland-lib/service"
 	"github.com/interline-io/transitland-lib/tlxy"
 	"github.com/interline-io/transitland-lib/tt"
 	"github.com/twpayne/go-geom"
@@ -61,7 +62,7 @@ func NewRouteGeometryBuilder() *RouteGeometryBuilder {
 // Counts the number of times a shape is used for each route,direction_id
 func (pp *RouteGeometryBuilder) AfterWrite(eid string, ent tt.Entity, emap *tt.EntityMap) error {
 	switch v := ent.(type) {
-	case *gtfs.Shape:
+	case *service.ShapeLine:
 		pts := make([]tlxy.Point, v.Geometry.Val.NumCoords())
 		for i, c := range v.Geometry.Val.Coords() {
 			pts[i] = tlxy.Point{Lon: c[0], Lat: c[1]}

filters/prefix_filter.go

@@ -90,6 +90,10 @@ func (filter *PrefixFilter) Filter(ent tt.Entity, emap *tt.EntityMap) error {
 		if prefix, ok := filter.getprefix(v.FeedVersionID); ok {
 			v.ShapeID.Set(fmt.Sprintf("%s%s", prefix, v.ShapeID.Val))
 		}
+	case *service.ShapeLine:
+		if prefix, ok := filter.getprefix(v.FeedVersionID); ok {
+			v.ShapeID.Set(fmt.Sprintf("%s%s", prefix, v.ShapeID.Val))
+		}
 	case *gtfs.FareAttribute:
 		if prefix, ok := filter.getprefix(v.FeedVersionID); ok {
 			v.FareID.Set(fmt.Sprintf("%s%s", prefix, v.FareID.Val))

gtfs/calendar_date.go

@@ -37,7 +37,7 @@ func (ent *CalendarDate) TableName() string {
 }
 
 // UpdateKeys updates Entity references.
-func (ent *CalendarDate) UpdateKeys(emap *EntityMap) error {
+func (ent *CalendarDate) UpdateKeys(emap *tt.EntityMap) error {
 	if serviceID, ok := emap.GetEntity(&Calendar{ServiceID: ent.ServiceID}); ok {
 		ent.ServiceID = serviceID
 	} else {

gtfs/gtfs.go

@@ -2,15 +2,38 @@ package gtfs
 
 import (
 	"strconv"
-
-	"github.com/interline-io/transitland-lib/tt"
 )
 
-type EntityMap = tt.EntityMap
-
 func entID(id int, gtfsid string) string {
 	if id > 0 {
 		return strconv.Itoa(id)
 	}
 	return gtfsid
 }
+
+type Reader interface {
+	Stops() chan Stop
+	StopTimes() chan StopTime
+	Agencies() chan Agency
+	Calendars() chan Calendar
+	CalendarDates() chan CalendarDate
+	FareAttributes() chan FareAttribute
+	FareRules() chan FareRule
+	FeedInfos() chan FeedInfo
+	Frequencies() chan Frequency
+	Routes() chan Route
+	Shapes() chan Shape
+	Transfers() chan Transfer
+	Pathways() chan Pathway
+	Levels() chan Level
+	Trips() chan Trip
+	Translations() chan Translation
+	Attributions() chan Attribution
+	Areas() chan Area
+	StopAreas() chan StopArea
+	FareLegRules() chan FareLegRule
+	FareTransferRules() chan FareTransferRule
+	FareProducts() chan FareProduct
+	RiderCategories() chan RiderCategory
+	FareMedia() chan FareMedia
+}

gtfs/shape.go

@@ -9,13 +9,11 @@ import (
 
 // Shape shapes.txt
 type Shape struct {
-	ShapeID           tt.String     `csv:",required"`
-	ShapePtLat        tt.Float      `db:"-" csv:",required" range:"-90,90"`
-	ShapePtLon        tt.Float      `db:"-" csv:",required" range:"-180,180"`
-	ShapePtSequence   tt.Int        `db:"-" csv:",required" range:"0,"`
-	ShapeDistTraveled tt.Float      `db:"-" range:"0,"`
-	Geometry          tt.LineString `db:"geometry" csv:"-"`
-	Generated         bool          `db:"generated" csv:"-"`
+	ShapeID           tt.String `csv:",required"`
+	ShapePtLat        tt.Float  `db:"-" csv:",required" range:"-90,90"`
+	ShapePtLon        tt.Float  `db:"-" csv:",required" range:"-180,180"`
+	ShapePtSequence   tt.Int    `db:"-" csv:",required" range:"0,"`
+	ShapeDistTraveled tt.Float  `db:"-" range:"0,"`
 	tt.BaseEntity
 }
 
@@ -34,11 +32,6 @@ func (ent *Shape) Filename() string {
 	return "shapes.txt"
 }
 
-// TableName gtfs_shapes
-func (ent *Shape) TableName() string {
-	return "gtfs_shapes"
-}
-
 // Errors for this Entity.
 func (ent *Shape) Errors() (errs []error) {
 	// Defer on moving this to reflect path for now