Ensure FeedVersionID is set on new shapes before filters (#381)

* Ensure FeedVersionID is set on new shapes before filters

* Allow empty strings to scan as null

* Minor tweaks to Scan and String

* Remove pathway custom GetString

* Add FeedVersionID to ShapeLine from Shapes

* Fix reversed commands

* Add ReadRowsIter method

cmd/transitland/main.go

@@ -52,8 +52,8 @@ func init() {
 		Hidden: true,
 	}
 	dmfrCommand.AddCommand(
-		tlcli.CobraHelper(&cmds.LintCommand{}, pc, "format"),
-		tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "lint"),
+		tlcli.CobraHelper(&cmds.LintCommand{}, pc, "lint"),
+		tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "format"),
 	)
 
 	genDocCommand := tlcli.CobraHelper(&tlcli.GenDocCommand{Command: rootCmd}, pc, "gendoc")
@@ -63,9 +63,9 @@ func init() {
 		tlcli.CobraHelper(&cmds.CopyCommand{}, pc, "copy"),
 		tlcli.CobraHelper(&cmds.ExtractCommand{}, pc, "extract"),
 		tlcli.CobraHelper(&cmds.FetchCommand{}, pc, "fetch"),
-		tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "dmfr-lint"),
+		tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "dmfr-format"),
 		tlcli.CobraHelper(&cmds.ImportCommand{}, pc, "import"),
-		tlcli.CobraHelper(&cmds.LintCommand{}, pc, "dmfr-format"),
+		tlcli.CobraHelper(&cmds.LintCommand{}, pc, "dmfr-lint"),
 		tlcli.CobraHelper(&cmds.MergeCommand{}, pc, "merge"),
 		tlcli.CobraHelper(&cmds.RebuildStatsCommand{}, pc, "rebuild-stats"),
 		tlcli.CobraHelper(&cmds.SyncCommand{}, pc, "sync"),

gtfs/pathway.go

@@ -1,9 +1,6 @@
 package gtfs
 
 import (
-	"errors"
-	"fmt"
-
 	"github.com/interline-io/transitland-lib/tt"
 )
 
@@ -43,47 +40,3 @@ func (ent *Pathway) Filename() string {
 func (ent *Pathway) TableName() string {
 	return "gtfs_pathways"
 }
-
-// GetString returns the string representation of an field.
-func (ent *Pathway) GetString(key string) (string, error) {
-	v := ""
-	switch key {
-	case "pathway_id":
-		v = ent.PathwayID.String()
-	case "from_stop_id":
-		v = ent.FromStopID.String()
-	case "to_stop_id":
-		v = ent.ToStopID.String()
-	case "pathway_mode":
-		v = ent.PathwayMode.String()
-	case "is_bidirectional":
-		v = ent.IsBidirectional.String()
-	case "length":
-		if ent.Length.Val > 0 {
-			v = fmt.Sprintf("%0.5f", ent.Length.Val)
-		}
-	case "traversal_time":
-		if ent.TraversalTime.Val > 0 {
-			v = ent.TraversalTime.String()
-		}
-	case "stair_count":
-		if ent.StairCount.Val != 0 && ent.StairCount.Val != -1 {
-			v = ent.StairCount.String()
-		}
-	case "max_slope":
-		if ent.MaxSlope.Val != 0 {
-			v = fmt.Sprintf("%0.2f", ent.MaxSlope.Val)
-		}
-	case "min_width":
-		if ent.MinWidth.Val != 0 {
-			v = fmt.Sprintf("%0.2f", ent.MinWidth.Val)
-		}
-	case "signposted_as":
-		v = ent.SignpostedAs.String()
-	case "reversed_signposted_as":
-		v = ent.ReverseSignpostedAs.String()
-	default:
-		return v, errors.New("unknown key")
-	}
-	return v, nil
-}

service/shape_line.go

@@ -78,31 +78,15 @@ func NewShapeLineFromShapes(shapes []gtfs.Shape) ShapeLine {
 			ent.SetExtra("expect_error", v)
 		}
 	}
-	ent.ShapeID.Set(shapes[0].ShapeID.Val)
+	if len(shapes) > 0 {
+		ent.ID = shapes[0].ID
+		ent.FeedVersionID = shapes[0].FeedVersionID
+		ent.ShapeID.Set(shapes[0].ShapeID.Val)
+	}
 	ent.Geometry = tt.NewLineStringFromFlatCoords(coords)
 	return ent
 }
 
-// ValidateShapes returns errors for an array of shapes.
-func ValidateShapes(shapes []gtfs.Shape) []error {
-	errs := []error{}
-	last := -1
-	dist := -1.0
-	for _, shape := range shapes {
-		// Check for duplicate ID errors
-		if shape.ShapePtSequence.Int() == last {
-			errs = append(errs, causes.NewSequenceError("shape_pt_sequence", tt.TryCsv(last)))
-		}
-		last = shape.ShapePtSequence.Int()
-		if shape.ShapeDistTraveled.Val < dist {
-			errs = append(errs, causes.NewSequenceError("shape_dist_traveled", tt.TryCsv(shape.ShapeDistTraveled)))
-		} else if shape.ShapeDistTraveled.Val > 0 {
-			dist = shape.ShapeDistTraveled.Val
-		}
-	}
-	return errs
-}
-
 func FlattenShape(ent ShapeLine) []gtfs.Shape {
 	coords := ent.Geometry.FlatCoords()
 	shapes := []gtfs.Shape{}
@@ -115,6 +99,8 @@ func FlattenShape(ent ShapeLine) []gtfs.Shape {
 			ShapePtLat:        tt.NewFloat(coords[i+1]),
 			ShapeDistTraveled: tt.NewFloat(coords[i+2]),
 		}
+		s.ID = ent.ID
+		s.FeedVersionID = ent.FeedVersionID
 		totaldist += coords[i+2]
 		shapes = append(shapes, s)
 	}
@@ -132,3 +118,23 @@ func FlattenShape(ent ShapeLine) []gtfs.Shape {
 	}
 	return shapes
 }
+
+// ValidateShapes returns errors for an array of shapes.
+func ValidateShapes(shapes []gtfs.Shape) []error {
+	errs := []error{}
+	last := -1
+	dist := -1.0
+	for _, shape := range shapes {
+		// Check for duplicate ID errors
+		if shape.ShapePtSequence.Int() == last {
+			errs = append(errs, causes.NewSequenceError("shape_pt_sequence", tt.TryCsv(last)))
+		}
+		last = shape.ShapePtSequence.Int()
+		if shape.ShapeDistTraveled.Val < dist {
+			errs = append(errs, causes.NewSequenceError("shape_dist_traveled", tt.TryCsv(shape.ShapeDistTraveled)))
+		} else if shape.ShapeDistTraveled.Val > 0 {
+			dist = shape.ShapeDistTraveled.Val
+		}
+	}
+	return errs
+}

tlcsv/row.go

@@ -3,6 +3,7 @@ package tlcsv
 import (
 	"encoding/csv"
 	"io"
+	"iter"
 	"strings"
 
 	"github.com/dimchansky/utfbom"
@@ -27,6 +28,8 @@ func (row *Row) Get(k string) (string, bool) {
 	return "", false
 }
 
+type csvOptFn func(*csv.Reader)
+
 // ReadRows iterates through csv rows with callback.
 func ReadRows(in io.Reader, cb func(Row)) error {
 	// Handle byte-order-marks.
@@ -39,6 +42,11 @@ func ReadRows(in io.Reader, cb func(Row)) error {
 	r.ReuseRecord = true
 	// Allow unescaped quotes
 	r.LazyQuotes = true
+	// Go
+	return readRows(r, cb)
+}
+
+func readRows(r *csv.Reader, cb func(Row)) error {
 	// Go for it.
 	firstRow, err := r.Read()
 	if err != nil {
@@ -81,3 +89,67 @@ func ReadRows(in io.Reader, cb func(Row)) error {
 	}
 	return nil
 }
+
+func ReadRowsIter(in io.Reader, optFns ...csvOptFn) iter.Seq2[Row, error] {
+	// Handle byte-order-marks.
+	r := csv.NewReader(utfbom.SkipOnly(in))
+	// Allow variable columns - very common in GTFS
+	r.FieldsPerRecord = -1
+	// Trimming is done elsewhere
+	r.TrimLeadingSpace = false
+	// Reuse record
+	r.ReuseRecord = true
+	// Allow unescaped quotes
+	r.LazyQuotes = true
+	// Add additional options
+	for _, optFn := range optFns {
+		optFn(r)
+	}
+	return func(yield func(Row, error) bool) {
+		// Go for it.
+		firstRow, err := r.Read()
+		if err != nil {
+			yield(Row{}, err)
+			return
+		}
+		// Copy header, since we will reuse the backing array
+		header := []string{}
+		for _, v := range firstRow {
+			header = append(header, strings.TrimSpace(v))
+		}
+		// Map the header to row index
+		hindex := map[string]int{}
+		for k, i := range header {
+			hindex[i] = k
+		}
+		for {
+			row, err := r.Read()
+			if err == nil {
+				// ok
+			} else if err == io.EOF {
+				break
+			} else if _, ok := err.(*csv.ParseError); ok {
+				// Parse error: clear row, add error to row
+				row = []string{}
+			} else {
+				// Serious error: break and return with error
+				yield(Row{}, err)
+				return
+			}
+			// Remove whitespace
+			for i := 0; i < len(row); i++ {
+				v := row[i]
+				// This is dumb but saves substantial time.
+				if len(v) > 0 && (v[0] == ' ' || v[len(v)-1] == ' ' || v[0] == '\t' || v[len(v)-1] == '\t') {
+					row[i] = strings.TrimSpace(v)
+				}
+			}
+			// Pass parse errors to row
+			line, _ := r.FieldPos(0)
+			cbrow := Row{Row: row, Line: line, Header: header, Hindex: hindex, Err: err}
+			if !yield(cbrow, nil) {
+				return
+			}
+		}
+	}
+}

tt/option.go

@@ -51,16 +51,19 @@ func (r Option[T]) String() string {
 		return ""
 	}
 	out := ""
-	if err := convertAssign(&out, r.Val); err != nil {
+	if _, err := convertAssign(&out, r.Val); err != nil {
 		b, _ := r.MarshalJSON()
-		return string(b)
+		out = string(b)
 	}
 	return out
 }
 
 func (r *Option[T]) Scan(src interface{}) error {
-	err := convertAssign(&r.Val, src)
-	r.Valid = (src != nil && err == nil)
+	r.Valid = false
+	ok, err := convertAssign(&r.Val, src)
+	if ok && err == nil {
+		r.Valid = true
+	}
 	return err
 }
 

tt/option_convert.go

@@ -139,10 +139,14 @@ func TryCsv(val any) string {
 	return a
 }
 
-func convertAssign(dest any, src any) error {
+func convertAssign(dest any, src any) (bool, error) {
 	if src == nil {
-		return nil
+		return false, nil
 	}
+	if s, ok := src.(string); ok && s == "" {
+		return false, nil
+	}
+	ok := true
 	var err error
 	switch d := dest.(type) {
 	case *string:
@@ -263,7 +267,7 @@ func convertAssign(dest any, src any) error {
 			err = cannotConvert(dest, src)
 		}
 	}
-	return err
+	return ok, err
 }
 
 func cannotConvert(dest any, src any) error {