From 4f1405d2eb9a0f4504692d936a590b753e947dea Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 01/11] rust: lints Unused use, repeated implicit locking. These don't really affect anything. --- main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/main.rs b/main.rs index 3acb970..617f90d 100644 --- a/main.rs +++ b/main.rs @@ -15,11 +15,11 @@ limitations under the License. */ use std::io::{Read, Write,stdin, stdout}; -use std::str; fn hit(needle: &[u8]) { - stdout().write(&needle).ok(); - stdout().write(&[b'\n']).ok(); + let mut out = stdout().lock(); + out.write_all(needle).unwrap(); + out.write_all(&[b'\n']).unwrap(); } fn scan_slice(inb: &[u8]) -> usize { From 9003c0e03816b2881663c4392568d76df0b8efde Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 02/11] rust: support named file as input --- main.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/main.rs b/main.rs index 617f90d..748b96f 100644 --- a/main.rs +++ b/main.rs @@ -62,5 +62,11 @@ fn sscan(mut input: impl Read) { } fn main() { - sscan(stdin().lock()) + let args: Vec = std::env::args().collect(); + if args.len() == 2 { + let file = std::fs::File::open(&args[1]).unwrap(); + sscan(file) + } else { + sscan(stdin().lock()) + } } From 46270c3d3cde25452b06383283f1130c144aa555 Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 03/11] rust: add basic skipping The switch from enumberate() to a regular loop is actually slower for the single-stepping case, but the performance gains from being able to skip large chunks of data with minimal comparisons more than makes up for it. This is the most basic optimization for this type of algorithm and this is the simplified version of it. If we're not in a match, check 20 characters ahead and if that also isn't a match, skip 20. If it is a match, then just continue on with checking each byte. --- README.md | 1 + main.rs | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dcf9ca2..a7bdd6e 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ implementations compare to each other. | ripgrep | 0m1.709s | 0m1.541s | 0m0.147s | | simple (Go) | 0m1.737s | 0m1.594s | 0m0.142s | | simple (Rust) | 0m1.461s | 0m1.325s | 0m0.131s | +| skip (Rust) | 0m0.231s | 0m0.105s | 0m0.124s | | simple (Node) | 0m6.458s | 0m6.043s | 0m0.627s | | custom (C) | **0m0.222s** | **0m0.079s** | **0m0.141s** | diff --git a/main.rs b/main.rs index 748b96f..2c8d4ec 100644 --- a/main.rs +++ b/main.rs @@ -25,15 +25,26 @@ fn hit(needle: &[u8]) { fn scan_slice(inb: &[u8]) -> usize { let mut count = 0; let len = inb.len(); - for (i, &b) in inb.into_iter().enumerate() { + let mut i = 0usize; + while i < len { + let b = inb[i]; + if count == 0 && i+20 < len { + let bs = inb[i+20]; + if !bs.is_ascii_digit() && !(b'a'..=b'f').contains(&bs) { + i += 20; + continue; + } + } if b.is_ascii_digit() || (b'a'..=b'f').contains(&b) { count += 1; + i += 1; continue } if count == 40 { hit(&inb[i-40..i]); } - count = 0 + count = 0; + i += 1; } if count == 40 { hit(&inb[len-40..]); From d70c7f466e546d4cb72dcd4722afdd3891f14e86 Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 04/11] go: add basic skipping Significant performance boost, as expected. --- README.md | 1 + main.go | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a7bdd6e..2e5f38f 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ implementations compare to each other. | grep | 0m18.034s | 0m15.713s | 0m2.257s | | ripgrep | 0m1.709s | 0m1.541s | 0m0.147s | | simple (Go) | 0m1.737s | 0m1.594s | 0m0.142s | +| skip (Go) | 0m0.338s | 0m0.187s | 0m0.152s | | simple (Rust) | 0m1.461s | 0m1.325s | 0m0.131s | | skip (Rust) | 0m0.231s | 0m0.105s | 0m0.124s | | simple (Node) | 0m6.458s | 0m6.043s | 0m0.627s | diff --git a/main.go b/main.go index ec6af40..873e380 100644 --- a/main.go +++ b/main.go @@ -28,7 +28,16 @@ const BUF = 64 * 4096 func searchWrite(buf []byte, out io.Writer) int { count := 0 - for i, b := range buf { + bl := len(buf) + for i := 0; i < bl; i++ { + b := buf[i] + if count == 0 && i+20 < bl { + bs := buf[i+20] + if !(bs >= '0' && bs <= '9') && !(bs >= 'a' && bs <= 'f') { + i += 20 + continue + } + } switch { case b >= '0' && b <= '9': fallthrough From 8b9647f00a0c88f3ac752e6d75fed0ae309ae5bb Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 05/11] node: add basic skipping The mighty skip comes to node, putting it nearly in the range of the compiled languages. Even though it isn't as fast as the compiled languages, this is still objectively fast. Unless the file is already cached in memory, you're well into the realm of being i/o bound instead of node being the bottleneck. --- README.md | 1 + main.js | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/README.md b/README.md index 2e5f38f..2a5e0ed 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ implementations compare to each other. | simple (Rust) | 0m1.461s | 0m1.325s | 0m0.131s | | skip (Rust) | 0m0.231s | 0m0.105s | 0m0.124s | | simple (Node) | 0m6.458s | 0m6.043s | 0m0.627s | +| skip (Node) | 0m1.368s | 0m1.062s | 0m0.686s | | custom (C) | **0m0.222s** | **0m0.079s** | **0m0.141s** | By comparing the times you can see that each implementation is more or less diff --git a/main.js b/main.js index 0cb7294..997909f 100644 --- a/main.js +++ b/main.js @@ -37,6 +37,10 @@ function is_hex(c) { function scan_slice(buf) { let count = 0; for (let i = 0; i< buf.length; i++) { + if (count === 0 && i+20 < buf.length && !is_hex(buf[i+20])) { + i += 20; + continue; + } if (is_hex(buf[i])) { count++; continue; From 54fabb2d1f6d65473a5427528669400931a8b0f5 Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 06/11] go: lints & cleanup Remove dead code, make 'go list' happy. --- go.mod | 3 +++ main.c | 1 + main.go | 31 ------------------------------- 3 files changed, 4 insertions(+), 31 deletions(-) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..1e03b32 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module main + +go 1.20 diff --git a/main.c b/main.c index baa67b9..85d90e4 100644 --- a/main.c +++ b/main.c @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +//go:build ignore #include #include #include diff --git a/main.go b/main.go index 873e380..4d5a232 100644 --- a/main.go +++ b/main.go @@ -16,12 +16,8 @@ limitations under the License. package main import ( - "fmt" "io" - "log" "os" - "runtime" - "runtime/pprof" ) const BUF = 64 * 4096 @@ -86,30 +82,3 @@ func scan(input io.Reader) { func main() { scan(os.Stdin) } - -func pmain() { - c, err := os.Create("sha1scan.cpu.prof") - if err != nil { - log.Fatal("could not create CPU profile: ", err) - } - defer c.Close() - if err := pprof.StartCPUProfile(c); err != nil { - log.Fatal("could not start CPU profile: ", err) - } - - scan(os.Stdin) - - pprof.StopCPUProfile() - runtime.GC() // get up-to-date statistics - - for _, p := range pprof.Profiles() { - m, err := os.Create(fmt.Sprintf("sha1scan.%s.prof", p.Name())) - if err != nil { - log.Fatalf("could not create %s profile: ", p.Name(), err) - } - defer m.Close() - if err := p.WriteTo(m, 1); err != nil { - log.Fatal("could not write %s profile: ", p.Name(), err) - } - } -} From 5ba15a36b0c8ebdf06241ff26046f8cb9442f727 Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 07/11] c: speeling --- main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.c b/main.c index 85d90e4..e703e2a 100644 --- a/main.c +++ b/main.c @@ -227,7 +227,7 @@ static const unsigned char * scan_hit_short(const unsigned char *buf, const unsi // Rather than checking them in linear order, we use statistics to determine // the optimal order to check for an early exit. // TODO: accept this ordering as input - // TOOD: extra credit, generate counts from first block + // TODO: extra credit, generate counts from first block const int checks[39] = { 5, 32, From fcefd8e02f2d33e9bef16a0451b988220059f4af Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 13:38:43 -0700 Subject: [PATCH 08/11] rust: cargo fmt --- main.rs | 114 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/main.rs b/main.rs index 2c8d4ec..e3c9408 100644 --- a/main.rs +++ b/main.rs @@ -14,70 +14,74 @@ See the License for the specific language governing permissions and limitations under the License. */ -use std::io::{Read, Write,stdin, stdout}; +use std::io::{stdin, stdout, Read, Write}; fn hit(needle: &[u8]) { - let mut out = stdout().lock(); - out.write_all(needle).unwrap(); - out.write_all(&[b'\n']).unwrap(); + let mut out = stdout().lock(); + out.write_all(needle).unwrap(); + out.write_all(&[b'\n']).unwrap(); } fn scan_slice(inb: &[u8]) -> usize { - let mut count = 0; - let len = inb.len(); - let mut i = 0usize; - while i < len { - let b = inb[i]; - if count == 0 && i+20 < len { - let bs = inb[i+20]; - if !bs.is_ascii_digit() && !(b'a'..=b'f').contains(&bs) { - i += 20; - continue; - } - } - if b.is_ascii_digit() || (b'a'..=b'f').contains(&b) { - count += 1; - i += 1; - continue - } - if count == 40 { - hit(&inb[i-40..i]); - } - count = 0; - i += 1; - } - if count == 40 { - hit(&inb[len-40..]); - count = 0 - } - if count > 40 { 41 } else { count } + let mut count = 0; + let len = inb.len(); + let mut i = 0usize; + while i < len { + let b = inb[i]; + if count == 0 && i + 20 < len { + let bs = inb[i + 20]; + if !bs.is_ascii_digit() && !(b'a'..=b'f').contains(&bs) { + i += 20; + continue; + } + } + if b.is_ascii_digit() || (b'a'..=b'f').contains(&b) { + count += 1; + i += 1; + continue; + } + if count == 40 { + hit(&inb[i - 40..i]); + } + count = 0; + i += 1; + } + if count == 40 { + hit(&inb[len - 40..]); + count = 0 + } + if count > 40 { + 41 + } else { + count + } } fn sscan(mut input: impl Read) { - let mut backbuf = vec![0u8; 64*4096]; - let bbuf = backbuf.as_mut_slice(); - // let mut bbuf = [0u8; 2*1024*1024]; - let mut off = 0; - let mut total_read = 0; - while let Ok(n) = input.read(&mut bbuf[off..]) { - total_read += n; - if n == 0 { - break - } - off = scan_slice(&bbuf[..n]); - for i in 0..off { - bbuf[i] = bbuf[n-off+i]; - } - } - eprintln!("Total bytes read: {}", total_read); + let mut backbuf = vec![0u8; 64 * 4096]; + let bbuf = backbuf.as_mut_slice(); + // let mut bbuf = [0u8; 2*1024*1024]; + let mut off = 0; + let mut total_read = 0; + while let Ok(n) = input.read(&mut bbuf[off..]) { + total_read += n; + if n == 0 { + break; + } + off = scan_slice(&bbuf[..n]); + for i in 0..off { + bbuf[i] = bbuf[n - off + i]; + } + } + eprintln!("Total bytes read: {}", total_read); } fn main() { - let args: Vec = std::env::args().collect(); - if args.len() == 2 { - let file = std::fs::File::open(&args[1]).unwrap(); - sscan(file) - } else { - sscan(stdin().lock()) - } + let args: Vec = std::env::args().collect(); + if args.len() == 2 { + let file = std::fs::File::open(&args[1]).unwrap(); + sscan(file) + } else { + sscan(stdin().lock()) + } } From e8ae2f4283a6c733df40a93c61481af3cefc224a Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 15:36:07 -0700 Subject: [PATCH 09/11] ci: skip grep tests on PR It's painfully slow compred to everything else now. --- .github/workflows/makefile.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 23aab78..2efb030 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -46,6 +46,7 @@ jobs: printf '### rs\n' >> "$GITHUB_STEP_SUMMARY" make rs.txt TIME_REPORT=$GITHUB_STEP_SUMMARY - name: make grep.txt + if: github.event_name != 'pull_request' run: | printf '### grep\n' >> "$GITHUB_STEP_SUMMARY" make grep.txt TIME_REPORT=$GITHUB_STEP_SUMMARY From de8594e75cbb7d2495db6860d04f1fb1f97eb434 Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 15:37:51 -0700 Subject: [PATCH 10/11] ci: quieter install --- .github/workflows/makefile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 2efb030..877fab2 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -20,8 +20,8 @@ jobs: run: make raw.tar - name: build & install run: | - sudo apt-get update - sudo apt-get install ripgrep + sudo apt-get update -q + sudo apt-get install -yq ripgrep make scan-c scan-go scan-rs { printf "## Versions\n" From 1298bf852b52f1753e49739f3e500b7cf5e8e9b4 Mon Sep 17 00:00:00 2001 From: Ryann Graham Date: Thu, 29 Jun 2023 16:09:25 -0700 Subject: [PATCH 11/11] ci: fancier build reporting --- .github/workflows/makefile.yml | 60 ++++++++++++++++------------------ Makefile | 15 ++++++--- times.awk | 39 ++++++++++++++++++++++ 3 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 times.awk diff --git a/.github/workflows/makefile.yml b/.github/workflows/makefile.yml index 877fab2..35bf177 100644 --- a/.github/workflows/makefile.yml +++ b/.github/workflows/makefile.yml @@ -23,38 +23,34 @@ jobs: sudo apt-get update -q sudo apt-get install -yq ripgrep make scan-c scan-go scan-rs - { - printf "## Versions\n" - grep -V - rg -V - node --version - rustc --version - go version - cc --version - } >> "$GITHUB_STEP_SUMMARY" - - name: make c.txt - run: | - printf '### c\n' >> "$GITHUB_STEP_SUMMARY" - make c.txt TIME_REPORT=$GITHUB_STEP_SUMMARY - - name: make go.txt - run: | - printf '### go\n' >> "$GITHUB_STEP_SUMMARY" - make go.txt TIME_REPORT=$GITHUB_STEP_SUMMARY - - name: make rs.txt - run: | - printf '### rs\n' >> "$GITHUB_STEP_SUMMARY" - make rs.txt TIME_REPORT=$GITHUB_STEP_SUMMARY - - name: make grep.txt + - run: make c.txt + - run: make go.txt + - run: make rs.txt + - run: make grep.txt if: github.event_name != 'pull_request' + - run: make ripgrep.txt + - run: make js.txt + - name: summarize run: | - printf '### grep\n' >> "$GITHUB_STEP_SUMMARY" - make grep.txt TIME_REPORT=$GITHUB_STEP_SUMMARY - - name: make ripgrep.txt - run: | - printf '### ripgrep\n' >> "$GITHUB_STEP_SUMMARY" - make ripgrep.txt TIME_REPORT=$GITHUB_STEP_SUMMARY - - name: make js.txt - run: | - printf '### js\n' >> "$GITHUB_STEP_SUMMARY" - make js.txt TIME_REPORT=$GITHUB_STEP_SUMMARY + make times.md + { + printf "## Best Times\n\n" + cat times.md + printf "\n## Versions\n" + printf "grep: " + grep -V | head -n 1 + printf "ripgrep: " + rg -V | head -n 1 + printf "node: " + node --version | head -n 1 + printf "rustc: " + rustc --version | head -n 1 + printf "go: " + go version | head -n 1 + printf "cc: " + cc --version | head -n 1 + printf "\n## Sample Data \n```" + ls -l raw.tar + printf "```\n" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/Makefile b/Makefile index 7fae1ca..bae8f8b 100644 --- a/Makefile +++ b/Makefile @@ -17,11 +17,7 @@ CFLAGS := -Werror -Wall FAST_CFLAGS := $(CFLAGS) -O3 -DNDEBUG DEBUG_CFLAGS := $(CFLAGS) -g SIMD_CFLAGS := $(CFLAGS) -DUSE_SIMD=1 -TIME_CMD := command time -p - -ifdef TIME_REPORT -TIME_CMD := $(TIME_CMD) -a -o $(TIME_REPORT) -endif +TIME_CMD = command time -p -a -o $@.times hexgrep scan-c-fast: main.c $(CC) $(FAST_CFLAGS) -o $@ $< @@ -73,6 +69,15 @@ scan-go: main.go scan-rs: main.rs rustc -O -o $@ $< +times.md: + { \ + printf '\n| | real | user | system |\n'; \ + printf '|-----|-------|-------|--------|\n'; \ + for t in *.txt.times; do \ + printf '| %3s | %5s | %5s | %6s |\n' $$(basename -s .txt.times $$t) $$(awk -f times.awk $$t); \ + done; \ + } > $@ + %.txt: scan-% $(SAMPLE) $(TIME_CMD) ./scan-$* < $(SAMPLE) > $@ $(TIME_CMD) ./scan-$* < $(SAMPLE) > $@ diff --git a/times.awk b/times.awk new file mode 100644 index 0000000..7c4bc38 --- /dev/null +++ b/times.awk @@ -0,0 +1,39 @@ +# Takes input with multiple run times (in POSIX time(1) format) and +# reduces them to a single line of the real, user and system times for +# fastest overall run. +# +# Example input: +# real 0.37 +# user 0.06 +# sys 0.15 +# real 0.18 +# user 0.05 +# sys 0.12 +# real 0.19 +# user 0.05 +# sys 0.13 +# +# Output: +# 0.18 0.05 0.12 + +BEGIN { + REAL_T = "" + USER_T = "" + SYS_T = "" +} +/real/ { + if (REAL_T == "" || REAL_T > $2) { + REAL_T = $2; + USER_T = ""; + SYS_T = ""; + } +} +/user/ { + if (USER_T == "") USER_T = $2 +} +/sys/ { + if (SYS_T == "") SYS_T = $2 +} +END { + print REAL_T, USER_T, SYS_T +}