diff --git a/cmd/transitland/main.go b/cmd/transitland/main.go index 1f219aaa..a5b8db8e 100644 --- a/cmd/transitland/main.go +++ b/cmd/transitland/main.go @@ -52,8 +52,8 @@ func init() { Hidden: true, } dmfrCommand.AddCommand( - tlcli.CobraHelper(&cmds.LintCommand{}, pc, "format"), - tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "lint"), + tlcli.CobraHelper(&cmds.LintCommand{}, pc, "lint"), + tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "format"), ) genDocCommand := tlcli.CobraHelper(&tlcli.GenDocCommand{Command: rootCmd}, pc, "gendoc") @@ -63,9 +63,9 @@ func init() { tlcli.CobraHelper(&cmds.CopyCommand{}, pc, "copy"), tlcli.CobraHelper(&cmds.ExtractCommand{}, pc, "extract"), tlcli.CobraHelper(&cmds.FetchCommand{}, pc, "fetch"), - tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "dmfr-lint"), + tlcli.CobraHelper(&cmds.FormatCommand{}, pc, "dmfr-format"), tlcli.CobraHelper(&cmds.ImportCommand{}, pc, "import"), - tlcli.CobraHelper(&cmds.LintCommand{}, pc, "dmfr-format"), + tlcli.CobraHelper(&cmds.LintCommand{}, pc, "dmfr-lint"), tlcli.CobraHelper(&cmds.MergeCommand{}, pc, "merge"), tlcli.CobraHelper(&cmds.RebuildStatsCommand{}, pc, "rebuild-stats"), tlcli.CobraHelper(&cmds.SyncCommand{}, pc, "sync"), diff --git a/gtfs/pathway.go b/gtfs/pathway.go index b5bad407..4295bd6f 100644 --- a/gtfs/pathway.go +++ b/gtfs/pathway.go @@ -1,9 +1,6 @@ package gtfs import ( - "errors" - "fmt" - "github.com/interline-io/transitland-lib/tt" ) @@ -43,47 +40,3 @@ func (ent *Pathway) Filename() string { func (ent *Pathway) TableName() string { return "gtfs_pathways" } - -// GetString returns the string representation of an field. -func (ent *Pathway) GetString(key string) (string, error) { - v := "" - switch key { - case "pathway_id": - v = ent.PathwayID.String() - case "from_stop_id": - v = ent.FromStopID.String() - case "to_stop_id": - v = ent.ToStopID.String() - case "pathway_mode": - v = ent.PathwayMode.String() - case "is_bidirectional": - v = ent.IsBidirectional.String() - case "length": - if ent.Length.Val > 0 { - v = fmt.Sprintf("%0.5f", ent.Length.Val) - } - case "traversal_time": - if ent.TraversalTime.Val > 0 { - v = ent.TraversalTime.String() - } - case "stair_count": - if ent.StairCount.Val != 0 && ent.StairCount.Val != -1 { - v = ent.StairCount.String() - } - case "max_slope": - if ent.MaxSlope.Val != 0 { - v = fmt.Sprintf("%0.2f", ent.MaxSlope.Val) - } - case "min_width": - if ent.MinWidth.Val != 0 { - v = fmt.Sprintf("%0.2f", ent.MinWidth.Val) - } - case "signposted_as": - v = ent.SignpostedAs.String() - case "reversed_signposted_as": - v = ent.ReverseSignpostedAs.String() - default: - return v, errors.New("unknown key") - } - return v, nil -} diff --git a/service/shape_line.go b/service/shape_line.go index 0c308956..9615e672 100644 --- a/service/shape_line.go +++ b/service/shape_line.go @@ -78,31 +78,15 @@ func NewShapeLineFromShapes(shapes []gtfs.Shape) ShapeLine { ent.SetExtra("expect_error", v) } } - ent.ShapeID.Set(shapes[0].ShapeID.Val) + if len(shapes) > 0 { + ent.ID = shapes[0].ID + ent.FeedVersionID = shapes[0].FeedVersionID + ent.ShapeID.Set(shapes[0].ShapeID.Val) + } ent.Geometry = tt.NewLineStringFromFlatCoords(coords) return ent } -// ValidateShapes returns errors for an array of shapes. -func ValidateShapes(shapes []gtfs.Shape) []error { - errs := []error{} - last := -1 - dist := -1.0 - for _, shape := range shapes { - // Check for duplicate ID errors - if shape.ShapePtSequence.Int() == last { - errs = append(errs, causes.NewSequenceError("shape_pt_sequence", tt.TryCsv(last))) - } - last = shape.ShapePtSequence.Int() - if shape.ShapeDistTraveled.Val < dist { - errs = append(errs, causes.NewSequenceError("shape_dist_traveled", tt.TryCsv(shape.ShapeDistTraveled))) - } else if shape.ShapeDistTraveled.Val > 0 { - dist = shape.ShapeDistTraveled.Val - } - } - return errs -} - func FlattenShape(ent ShapeLine) []gtfs.Shape { coords := ent.Geometry.FlatCoords() shapes := []gtfs.Shape{} @@ -115,6 +99,8 @@ func FlattenShape(ent ShapeLine) []gtfs.Shape { ShapePtLat: tt.NewFloat(coords[i+1]), ShapeDistTraveled: tt.NewFloat(coords[i+2]), } + s.ID = ent.ID + s.FeedVersionID = ent.FeedVersionID totaldist += coords[i+2] shapes = append(shapes, s) } @@ -132,3 +118,23 @@ func FlattenShape(ent ShapeLine) []gtfs.Shape { } return shapes } + +// ValidateShapes returns errors for an array of shapes. +func ValidateShapes(shapes []gtfs.Shape) []error { + errs := []error{} + last := -1 + dist := -1.0 + for _, shape := range shapes { + // Check for duplicate ID errors + if shape.ShapePtSequence.Int() == last { + errs = append(errs, causes.NewSequenceError("shape_pt_sequence", tt.TryCsv(last))) + } + last = shape.ShapePtSequence.Int() + if shape.ShapeDistTraveled.Val < dist { + errs = append(errs, causes.NewSequenceError("shape_dist_traveled", tt.TryCsv(shape.ShapeDistTraveled))) + } else if shape.ShapeDistTraveled.Val > 0 { + dist = shape.ShapeDistTraveled.Val + } + } + return errs +} diff --git a/tlcsv/row.go b/tlcsv/row.go index 12efd1ba..396fbd08 100644 --- a/tlcsv/row.go +++ b/tlcsv/row.go @@ -3,6 +3,7 @@ package tlcsv import ( "encoding/csv" "io" + "iter" "strings" "github.com/dimchansky/utfbom" @@ -27,6 +28,8 @@ func (row *Row) Get(k string) (string, bool) { return "", false } +type csvOptFn func(*csv.Reader) + // ReadRows iterates through csv rows with callback. func ReadRows(in io.Reader, cb func(Row)) error { // Handle byte-order-marks. @@ -39,6 +42,11 @@ func ReadRows(in io.Reader, cb func(Row)) error { r.ReuseRecord = true // Allow unescaped quotes r.LazyQuotes = true + // Go + return readRows(r, cb) +} + +func readRows(r *csv.Reader, cb func(Row)) error { // Go for it. firstRow, err := r.Read() if err != nil { @@ -81,3 +89,67 @@ func ReadRows(in io.Reader, cb func(Row)) error { } return nil } + +func ReadRowsIter(in io.Reader, optFns ...csvOptFn) iter.Seq2[Row, error] { + // Handle byte-order-marks. + r := csv.NewReader(utfbom.SkipOnly(in)) + // Allow variable columns - very common in GTFS + r.FieldsPerRecord = -1 + // Trimming is done elsewhere + r.TrimLeadingSpace = false + // Reuse record + r.ReuseRecord = true + // Allow unescaped quotes + r.LazyQuotes = true + // Add additional options + for _, optFn := range optFns { + optFn(r) + } + return func(yield func(Row, error) bool) { + // Go for it. + firstRow, err := r.Read() + if err != nil { + yield(Row{}, err) + return + } + // Copy header, since we will reuse the backing array + header := []string{} + for _, v := range firstRow { + header = append(header, strings.TrimSpace(v)) + } + // Map the header to row index + hindex := map[string]int{} + for k, i := range header { + hindex[i] = k + } + for { + row, err := r.Read() + if err == nil { + // ok + } else if err == io.EOF { + break + } else if _, ok := err.(*csv.ParseError); ok { + // Parse error: clear row, add error to row + row = []string{} + } else { + // Serious error: break and return with error + yield(Row{}, err) + return + } + // Remove whitespace + for i := 0; i < len(row); i++ { + v := row[i] + // This is dumb but saves substantial time. + if len(v) > 0 && (v[0] == ' ' || v[len(v)-1] == ' ' || v[0] == '\t' || v[len(v)-1] == '\t') { + row[i] = strings.TrimSpace(v) + } + } + // Pass parse errors to row + line, _ := r.FieldPos(0) + cbrow := Row{Row: row, Line: line, Header: header, Hindex: hindex, Err: err} + if !yield(cbrow, nil) { + return + } + } + } +} diff --git a/tt/option.go b/tt/option.go index bd66acdf..9ade7561 100644 --- a/tt/option.go +++ b/tt/option.go @@ -51,16 +51,19 @@ func (r Option[T]) String() string { return "" } out := "" - if err := convertAssign(&out, r.Val); err != nil { + if _, err := convertAssign(&out, r.Val); err != nil { b, _ := r.MarshalJSON() - return string(b) + out = string(b) } return out } func (r *Option[T]) Scan(src interface{}) error { - err := convertAssign(&r.Val, src) - r.Valid = (src != nil && err == nil) + r.Valid = false + ok, err := convertAssign(&r.Val, src) + if ok && err == nil { + r.Valid = true + } return err } diff --git a/tt/option_convert.go b/tt/option_convert.go index 3f3b9c87..c635f502 100644 --- a/tt/option_convert.go +++ b/tt/option_convert.go @@ -139,10 +139,14 @@ func TryCsv(val any) string { return a } -func convertAssign(dest any, src any) error { +func convertAssign(dest any, src any) (bool, error) { if src == nil { - return nil + return false, nil } + if s, ok := src.(string); ok && s == "" { + return false, nil + } + ok := true var err error switch d := dest.(type) { case *string: @@ -263,7 +267,7 @@ func convertAssign(dest any, src any) error { err = cannotConvert(dest, src) } } - return err + return ok, err } func cannotConvert(dest any, src any) error {