bump dependency versions and minor cleanup

.github/workflows/release.yml

@@ -0,0 +1,74 @@
+name: Release
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - '[0-9]+.[0-9]+.[0-9]+'
+permissions:
+  contents: write
+jobs:
+  build:
+    name: ${{ matrix.platform.os_name }} with rust ${{ matrix.toolchain }}
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - os_name: Linux-x86_64
+            os: ubuntu-latest
+            target: x86_64-unknown-linux-gnu
+            bin_postfix: linux-amd64
+          # fails, cannot find openssl
+          #- os_name: Linux-aarch64
+          #  os: ubuntu-latest
+          #  target: aarch64-unknown-linux-gnu
+          #  bin_postfix: linux-arm64
+          - os_name: Windows-x86_64
+            os: windows-latest
+            target: x86_64-pc-windows-msvc
+            bin_postfix: amd64.exe
+          - os_name: macOS-x86_64
+            os: macOS-latest
+            target: x86_64-apple-darwin
+            bin_postfix: darwin-amd64
+          - os_name: macOS-aarch64
+            os: macOS-latest
+            target: aarch64-apple-darwin
+            bin_postfix: darwin-arm64
+        toolchain:
+          - stable
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use bundled sqlite3 (windows)
+        shell: pwsh
+        run: (Get-Content Cargo.toml) -replace '^(rusqlite = )(".*")$', '$1{ version = $2, features = ["bundled"] }' | Set-Content Cargo.toml
+        if: matrix.platform.os_name == 'Windows-x86_64'
+      - name: Build binary
+        uses: houseabsolute/actions-rust-cross@v0
+        with:
+          command: "build"
+          target: ${{ matrix.platform.target }}
+          toolchain: ${{ matrix.toolchain }}
+          args: "--release"
+          strip: true
+      - name: Rename builder (linux and macos)
+        run: mv target/${{ matrix.platform.target }}/release/ipwned-builder target/${{ matrix.platform.target }}/release/ipwned-builder-${{ matrix.platform.bin_postfix }}
+        if: matrix.platform.os_name != 'Windows-x86_64'
+      - name: Rename builder (windows)
+        run: mv target/${{ matrix.platform.target }}/release/ipwned-builder.exe target/${{ matrix.platform.target }}/release/ipwned-builder-${{ matrix.platform.bin_postfix }}
+        if: matrix.platform.os_name == 'Windows-x86_64'
+      - name: Rename server (linux and macos)
+        run: mv target/${{ matrix.platform.target }}/release/ipwned-server target/${{ matrix.platform.target }}/release/ipwned-server-${{ matrix.platform.bin_postfix }}
+        if: matrix.platform.os_name != 'Windows-x86_64'
+      - name: Rename server (windows)
+        run: mv target/${{ matrix.platform.target }}/release/ipwned-server.exe target/${{ matrix.platform.target }}/release/ipwned-server-${{ matrix.platform.bin_postfix }}
+        if: matrix.platform.os_name == 'Windows-x86_64'
+      - name: Generate SHA-256
+        run: shasum -a 256 target/${{ matrix.platform.target }}/release/ipwned-builder-${{ matrix.platform.bin_postfix }} target/${{ matrix.platform.target }}/release/ipwned-server-${{ matrix.platform.bin_postfix }} > target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin_postfix }}.sha256
+      - name: Release binary and SHA-256 checksum to GitHub
+        uses: softprops/action-gh-release@v2
+        with:
+          files: |
+            target/${{ matrix.platform.target }}/release/ipwned-builder-${{ matrix.platform.bin_postfix }}
+            target/${{ matrix.platform.target }}/release/ipwned-server-${{ matrix.platform.bin_postfix }}
+            target/${{ matrix.platform.target }}/release/${{ matrix.platform.bin_postfix }}.sha256

Cargo.toml

@@ -1,29 +1,29 @@
 [package]
 name = "ipwned-localdb"
-version = "0.9.0"
-edition = "2021"
+version = "0.9.2"
+edition = "2024"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-reqwest = { version = "0.12.4", features = ["default-tls", "gzip", "http2", "macos-system-configuration"] }
-bytes = "1.6.0"
-tokio = { version = "1.38.0", features = ["sync", "rt", "macros", "signal", "time"] }
-futures = "0.3.30"
-tokio-rusqlite = "0.5.1"
-rusqlite = "0.31.0"
-nom = "7.1.3"
-faster-hex = "0.9.0"
+reqwest = { version = "0.12.24", features = ["default-tls", "gzip", "http2", "macos-system-configuration"] }
+bytes = "1.10.1"
+tokio = { version = "1.48.0", features = ["sync", "rt", "macros", "signal", "time"] }
+futures = "0.3.31"
+tokio-rusqlite = "0.6.0"
+rusqlite = "0.32.1"
+nom = "8.0.0"
+faster-hex = "0.10.0"
 #qfilter = { path = "./qfilter", features = ["serde"] }
-qfilter = { version = "0.1.6", features = ["serde"] }
-serde = "1.0.203"
-serde_cbor = "0.11.2"
+qfilter = { version = "0.2.5", features = ["serde"] }
+#serde = "1.0.228"
+ciborium = "0.2.2"
 parse_duration = "2.1.1"
 pretty-duration = "0.1.1"
-chrono = "0.4.38"
-argh = "0.1.12"
-indicatif = "0.17.8"
-indicatif-log-bridge = "0.2.2"
-log = "0.4.21"
+chrono = "0.4.42"
+argh = "0.1.13"
+indicatif = "0.18.0"
+indicatif-log-bridge = "0.2.3"
+log = "0.4.28"
 simplelog = { version = "0.12.2", features = ["termcolor"] }
-rocket = "0.5.1"
+rocket = "0.5.1"

Readme.md

@@ -25,8 +25,8 @@ filter table on my test system took about 20 minutes, network-limited at 30 MiB/
     cd ipwned-localdb
     cargo build --release
 
-Requires rust 1.65.0 or newer. You may need to install openssl and sqlite3 devel packages on your system. On debian
-this corresponds to `libssl-dev` and `libsqlite3-dev`.
+Requires rust 1.85.0 or newer. You may need to install openssl and sqlite3 devel packages and pkg-config on your system.
+On debian this corresponds to `libssl-dev` and `libsqlite3-dev`.
 
 ## run
 

Rocket.toml.example

@@ -2,6 +2,6 @@
 [default]
 address = "127.0.0.1"
 port = 7660
-limits = { bytes = 20 }
+limits = { bytes = 21 }
 ip_header = false
 workers = 1

src/bin/ipwned-builder.rs

@@ -11,15 +11,15 @@ mod statedb;
 
 use crate::downloader::download_retry;
 use crate::filter_builder::{FilterBuilder, FilterResult, HashList};
-use crate::misc::{DownloadStatus, MAX_COUNT};
+use crate::misc::{DownloadError, DownloadStatus, MAX_COUNT};
 use crate::statedb::{State, StateDatabase};
 use argh::FromArgs;
 use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, TimeDelta};
 use futures;
-use futures::{pin_mut, stream, StreamExt};
+use futures::{StreamExt, pin_mut, stream};
 use indicatif;
 use indicatif_log_bridge::LogWrapper;
-use log::{debug, error, info, LevelFilter};
+use log::{LevelFilter, debug, error, info, warn};
 use pretty_duration::pretty_duration;
 use reqwest::Client;
 use std::env::current_dir;
@@ -62,12 +62,12 @@ struct CliArgs {
     #[argh(option, default = "MAX_COUNT")]
     end: u32,
 
-    /// maximum number of hashes to track in filter. If this number is exceeded a new filter must be built. This will influence the size of the filter. Only relevant when creating a new filter. default: 1_000_000_000
-    #[argh(option, short = 'c', default = "1_000_000_000")]
+    /// maximum number of hashes to track in filter. If this number is exceeded a new filter must be built. This will influence the size of the filter. Only relevant when creating a new filter. default: 1_500_000_000
+    #[argh(option, short = 'c', default = "1_500_000_000")]
     max_count: u64,
 
-    /// maximum error rate (false positives) for filter. This will influence the size of the filter. Only relevant when creating a new filter. default: 0.0000001
-    #[argh(option, short = 'e', default = "0.0000001")]
+    /// maximum error rate (false positives) for filter. This will influence the size of the filter. Only relevant when creating a new filter. default: 0.000001
+    #[argh(option, short = 'e', default = "0.000001")]
     max_error_rate: f64,
 
     /// override base url for downloading hash lists. default: https://api.pwnedpasswords.com/range/
@@ -189,6 +189,10 @@ async fn main() -> ExitCode {
         loop {
             let mut update = false;
             tokio::select! {
+                _ = tokio::signal::ctrl_c() => {
+                    do_exit = true;
+                    break;
+                },
                 x = schedule_downloads.next() => {
                     if x.is_some() {
                         let result = x.unwrap();
@@ -207,10 +211,6 @@ async fn main() -> ExitCode {
                         },
                         _ => break,
                     }
-                },
-                _ = tokio::signal::ctrl_c() => {
-                    do_exit = true;
-                    break;
                 }
             }
             if update {
@@ -220,7 +220,7 @@ async fn main() -> ExitCode {
 
         if do_exit {
             // we received a ctrl+c, process all downloaded files and terminate
-            info!("received ctrl+c, waiting for workers to finish");
+            warn!("received ctrl+c, waiting for workers to finish");
             // signal our worker threads to exit
             if filter_builder.in_tx.send(None).await.is_ok() {
                 // if something went wrong we can just quit, otherwise wait for everything to finish
@@ -299,14 +299,6 @@ fn build_progress_meter(status: &Status) -> ProgressBars {
 }
 
 fn init_logger(level: LevelFilter, multibar: indicatif::MultiProgress) {
-    #[cfg(feature = "termcolor")]
-    let logger = simplelog::TermLogger::new(
-        level,
-        simplelog::Config::default(),
-        simplelog::TerminalMode::Stdout,
-        simplelog::ColorChoice::Auto,
-    );
-    #[cfg(not(feature = "termcolor"))]
     let logger = simplelog::SimpleLogger::new(level, simplelog::Config::default());
     LogWrapper::new(multibar.clone(), logger)
         .try_init()
@@ -329,15 +321,14 @@ async fn schedule_download(
         return Err(DownloadStatus::Skipped {});
     }
     let hash_prefix = format!("{:0>5X}", hash_list_id);
-    let res = download_retry(client, base_url, &hash_prefix, etag, max_retries).await;
-    if res.is_err() {
-        let err = res.err().unwrap();
-        if err.status_code.unwrap_or(0_u16) == 304_u16 {
-            return Err(DownloadStatus::NotOutdated {});
-        }
-        return Err(DownloadStatus::HTTPError(err));
-    }
-    let res = res.unwrap();
+    let res = download_retry(client, base_url, &hash_prefix, etag, max_retries)
+        .await
+        .map_err(|err: DownloadError| {
+            if err.status_code.unwrap_or(0_u16) == 304_u16 {
+                return DownloadStatus::NotOutdated {};
+            }
+            DownloadStatus::HTTPError(err)
+        })?;
     let data_len = res.data.len();
     if hash_list_chan
         .send(Some(HashList {
@@ -361,12 +352,9 @@ fn check_db_state(
 ) -> bool {
     let mut need_update = true;
     if state.is_ok() {
-        let state = state.unwrap();
-        if state.is_some() {
-            let state = state.unwrap();
-            let time = NaiveDateTime::parse_from_str(&state.last_update, "%Y-%m-%d %H:%M:%S");
-            if time.is_ok() {
-                let time = time.unwrap().and_utc().fixed_offset();
+        if let Some(state) = state.unwrap() {
+            if let Ok(time) = NaiveDateTime::parse_from_str(&state.last_update, "%Y-%m-%d %H:%M:%S") {
+                let time = time.and_utc().fixed_offset();
                 need_update = max_age > time;
             }
             if state.etag.is_some() {

src/bin/ipwned-server.rs

@@ -39,7 +39,7 @@ fn open_filter(file_name: PathBuf) -> qfilter::Filter {
         panic!("unable to open filter file: {:?}", filter_file.err());
     }
 
-    let filter_maybe = serde_cbor::from_reader(filter_file.unwrap());
+    let filter_maybe = ciborium::from_reader(filter_file.unwrap());
     if filter_maybe.is_err() {
         panic!("failed to read filter file: {:?}", filter_maybe.err());
     }

src/filter_builder.rs

@@ -1,8 +1,8 @@
 use crate::parse::parse_file;
 use bytes::Bytes;
+use ciborium;
 use log::{debug, error, info, trace, warn};
 use qfilter;
-use serde_cbor;
 use std::fs::File;
 use std::io::ErrorKind::NotFound;
 use std::path::PathBuf;
@@ -91,7 +91,7 @@ fn work_build(
     filter: &mut qfilter::Filter,
 ) {
     let mut changed = false;
-    loop {
+    'mainloop: loop {
         let mut added: u32 = 0;
         let parsed = match in_rx.blocking_recv() {
             Some(Some(x)) => x,
@@ -103,7 +103,7 @@ fn work_build(
                 Ok(false) => {}
                 Err(_) => {
                     error!("unable to add more items to filter");
-                    break;
+                    break 'mainloop;
                 }
             }
         }
@@ -141,7 +141,7 @@ fn work_build(
                     break 'write;
                 }
             };
-            match serde_cbor::to_writer(&mut writer, filter) {
+            match ciborium::into_writer(filter, &mut writer) {
                 Err(e) => {
                     error!("failed to write new filter file: {:?}", e);
                     break 'write;
@@ -184,10 +184,12 @@ impl FilterBuilder {
     fn open_filter_maybe(file_name: &PathBuf, max_entries: u64, max_er: f64) -> qfilter::Filter {
         let filter_file = File::open(file_name);
         match filter_file {
-            Err(ref e) if e.kind() == NotFound => qfilter::Filter::new(max_entries, max_er),
+            Err(ref e) if e.kind() == NotFound => {
+                qfilter::Filter::new(max_entries, max_er).expect("failed to initialize new filter")
+            }
             Err(e) => panic!("unable to open filter file: {:?}", e),
             Ok(ref fh) => {
-                let filter_maybe = serde_cbor::from_reader(fh);
+                let filter_maybe = ciborium::from_reader(fh);
                 if filter_maybe.is_err() {
                     panic!("failed to read filter file: {:?}", filter_maybe.err());
                 }

src/parse.rs

@@ -2,19 +2,18 @@ use bytes::{BufMut, Bytes};
 use faster_hex::hex_decode_unchecked;
 use nom::bytes::complete::{tag, take_while_m_n};
 use nom::character::complete::{digit1, line_ending};
-use nom::character::is_hex_digit;
 use nom::multi::separated_list0;
 use nom::sequence::separated_pair;
-use nom::IResult;
+use nom::{AsChar, IResult, Parser};
 use std::str::from_utf8_unchecked;
 
 fn parse_hash(s: &[u8]) -> IResult<&[u8], &[u8]> {
-    take_while_m_n(35, 35, is_hex_digit)(s)
+    take_while_m_n(35, 35, AsChar::is_hex_digit)(s)
 }
 
 fn parse_line(s: &[u8]) -> IResult<&[u8], &[u8]> {
     // discards count
-    let (rem, (hash, _)) = separated_pair(parse_hash, tag(":"), digit1)(s)?;
+    let (rem, (hash, _)) = separated_pair(parse_hash, tag(":"), digit1).parse(s)?;
     Ok((rem, hash))
 }
 
@@ -23,7 +22,7 @@ pub fn parse_file(prefix: u32, s: &[u8]) -> IResult<&[u8], Vec<Bytes>> {
     base_hash.put_u16((prefix >> 4) as u16);
     base_hash.put_u8((prefix as u8) << 4);
 
-    let (rem, hex_hashes) = separated_list0(line_ending, parse_line)(s)?;
+    let (rem, hex_hashes) = separated_list0(line_ending, parse_line).parse(s)?;
     let mut hashes: Vec<Bytes> = Vec::with_capacity(hex_hashes.len());
 
     for hex in hex_hashes {

src/statedb.rs

@@ -57,7 +57,7 @@ impl StateDatabase {
         self.conn
             .call(move |conn| Ok(conn.is_readonly(MAIN_DB)?))
             .await
-            .map_or(true, |x| x)
+            .unwrap_or(true)
     }
 
     async fn create(&self) -> Result<()> {