initial commit

Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "ipwned-localdb"
+version = "0.9.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+reqwest = { version = "0.12.4", features = ["default-tls", "gzip", "http2", "macos-system-configuration"] }
+bytes = "1.6.0"
+tokio = { version = "1.38.0", features = ["sync", "rt", "macros", "signal", "time"] }
+futures = "0.3.30"
+tokio-rusqlite = "0.5.1"
+rusqlite = "0.31.0"
+nom = "7.1.3"
+faster-hex = "0.9.0"
+#qfilter = { path = "./qfilter", features = ["serde"] }
+qfilter = { version = "0.1.6", features = ["serde"] }
+serde = "1.0.203"
+serde_cbor = "0.11.2"
+parse_duration = "2.1.1"
+pretty-duration = "0.1.1"
+chrono = "0.4.38"
+argh = "0.1.12"
+indicatif = "0.17.8"
+indicatif-log-bridge = "0.2.2"
+log = "0.4.21"
+simplelog = { version = "0.12.2", features = ["termcolor"] }
+rocket = "0.5.1"

LICENSE

@@ -0,0 +1,25 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>
+

Readme.md

@@ -0,0 +1,111 @@
+# ipwned-localdb
+
+A program to download and efficiently store the hashed password lists offered by haveibeenpwned.com for local queries.
+
+## Features
+
+* multithreaded downloading of hash lists
+* storing hashes in a RSQF lookup table (comparable to bloom, cuckoo but more efficient) [1]
+* allows periodically updating lists without full filter rebuild
+* query interface is exposed through an HTTP service
+
+[1] see https://docs.rs/qfilter/latest/qfilter/
+
+## Notes
+
+With default settings the filter table will be a bit larger than 3gb. The HTTP server is serving this table from RAM,
+meaning the service will require at least that much RAM to run.
+
+As of June 13th 2024 there are 936_494_661 compromised passwords in the HIBP database. Downloading and building the
+filter table on my test system took about 20 minutes, network-limited at 30 MiB/s on average, with 500 parallel requests.
+
+## build
+
+    git clone https://github.com/OPSnet/ipwned-localdb
+    cd ipwned-localdb
+    cargo build --release
+
+Requires rust 1.65.0 or newer. You may need to install openssl and sqlite3 devel packages on your system. On debian
+this corresponds to `libssl-dev` and `libsqlite3-dev`.
+
+## run
+
+### create lookup table
+
+for creating or updating your local filter run
+
+    ./target/release/ipwned-builder
+
+settings can be adjusted, see `--help`, but the defaults should work for most people
+
+### serve lookup table
+
+    ./target/release/ipwned-server
+
+see `Rocket.toml.example` for adjusting the HTTP server settings. The `Rocket.toml` is expected in the current directory.
+
+## Usage
+
+### ipwned-builder
+
+    Usage: ipwned-builder [-d <base-path>] [-s <state-db-name>] [-f <filter-name>] [-a <max-age>] [-n <parallel>] [--start <start>] [--end <end>] [-c <max-count>] [-e <max-error-rate>] [-b <base-url>] [-r <max-retries>] [-l <log>]
+
+    Create or update a local lookup table for haveibeenpwned.com compromised passwords
+
+    Options:
+    -d, --base-path   base path to store filter and state db at. default: current
+                      directory
+    -s, --state-db-name
+                      file name of the state database file. default:
+                      ipwned_state.sqlite
+    -f, --filter-name file name of the lookup filter file. default:
+                      ipwned_qfilter.cbor
+    -a, --max-age     maximum age of a downloaded file before attempting an
+                      update. accepts a human-friendly string. default: 1 month
+    -n, --parallel    number of parallel download requests. default: 50
+    --start           update only ids starting from here. default: 0
+    --end             update only ids up to this id (inclusive). default: all
+                      (1048575)
+    -c, --max-count   maximum number of hashes to track in filter. If this number
+                      is exceeded a new filter must be built. This will influence
+                      the size of the filter. Only relevant when creating a new
+                      filter. default: 1_000_000_000
+    -e, --max-error-rate
+                      maximum error rate (false positives) for filter. This will
+                      influence the size of the filter. Only relevant when
+                      creating a new filter. default: 0.0000001
+    -b, --base-url    override base url for downloading hash lists. default:
+                      https://api.pwnedpasswords.com/range/
+    -r, --max-retries maximum number of retries when downloading a hash list in
+                      case of errors. default: 10
+    -l, --log         log level. allowed options: off error warn info debug trace.
+                      default: warn
+    --help            display usage information
+
+
+
+
+### ipwned-server
+
+    Usage: ipwned-server [-f <filter-path>]
+
+    run an HTTP server for querying a local haveibeenpwned.com password lookup table
+
+    Options:
+    -f, --filter-path file name of the lookup filter file. default:
+                      ./ipwned_qfilter.cbor
+    --help            display usage information
+
+
+## HTTP API
+
+POST requests are expected on `/` with the request body being the binary SHA1 hash (20 bytes) of the password to check.
+
+Response is encoded in the HTTP status code:
+
+    204 -> not found, good password
+    205 -> found, bad password
+
+for testing:
+
+    echo -n test | sha1sum | cut -c-40 | tr -d "\n" | xxd -r -p | curl -v http://127.0.0.1:7660/ --data-binary @-

Rocket.toml.example

@@ -0,0 +1,7 @@
+# see https://rocket.rs/guide/v0.5/configuration/#overview for available options
+[default]
+address = "127.0.0.1"
+port = 7660
+limits = { bytes = 20 }
+ip_header = false
+workers = 1