diff --git a/starlark/bench_test.go b/starlark/bench_test.go index 7f78a223..14952916 100644 --- a/starlark/bench_test.go +++ b/starlark/bench_test.go @@ -13,6 +13,7 @@ import ( "go.starlark.net/starlark" "go.starlark.net/starlarktest" + "go.starlark.net/syntax" ) func Benchmark(b *testing.B) { @@ -58,10 +59,11 @@ func Benchmark(b *testing.B) { } // BenchmarkProgram measures operations relevant to compiled programs. -// TODO(adonovan): use a bigger testdata program. func BenchmarkProgram(b *testing.B) { - // Measure time to read a source file (approx 600us but depends on hardware and file system). + // TODO(adonovan): use a bigger testdata program. filename := starlarktest.DataFile("starlark", "testdata/paths.star") + + // Measure time to read a source file (approx 600us but depends on hardware and file system). var src []byte b.Run("read", func(b *testing.B) { for i := 0; i < b.N; i++ { @@ -73,7 +75,26 @@ func BenchmarkProgram(b *testing.B) { } }) - // Measure time to turn a source filename into a compiled program (approx 450us). + // Measure time to scan (approx 170us). + b.Run("scan", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if err := syntax.ScanAndDiscard(filename, src, 0); err != nil { + b.Fatal(err) + } + } + }) + + // Measure time to parse (approx 300us). + b.Run("parse", func(b *testing.B) { + for i := 0; i < b.N; i++ { + if _, err := syntax.Parse(filename, src, 0); err != nil { + b.Fatal(err) + } + } + }) + + // Measure time to turn a source filename into a compiled program, + // that is, read + scan + parse + resolve + compile (approx 450us). var prog *starlark.Program b.Run("compile", func(b *testing.B) { for i := 0; i < b.N; i++ { diff --git a/syntax/parse.go b/syntax/parse.go index 0e4d284f..72828563 100644 --- a/syntax/parse.go +++ b/syntax/parse.go @@ -47,6 +47,23 @@ func Parse(filename string, src interface{}, mode Mode) (f *File, err error) { return f, nil } +// ScanAndDiscard tokenizes the input data and discards the tokens. +// Parameters are as for Parse. +// It exists only for internal benchmarking purposes. +func ScanAndDiscard(filename string, src interface{}, mode Mode) error { + in, err := newScanner(filename, src, mode&RetainComments != 0) + if err != nil { + return err + } + p := parser{in: in} + defer p.in.recover(&err) + p.nextToken() // read first lookahead token + for p.tok != EOF { + p.nextToken() + } + return nil +} + // ParseCompoundStmt parses a single compound statement: // a blank line, a def, for, while, or if statement, or a // semicolon-separated list of simple statements followed