index: add some initial indexing scaffolding

This gives us some room to actually start writing out some index
operations.

.gitignore

@@ -20,3 +20,6 @@ ripgrep*_source.tar.bz2
 # Cargo timings
 cargo-timing-*.html
 cargo-timing.html
+
+# ripgrep index
+.ripgrep

Cargo.lock

@@ -418,6 +418,7 @@ dependencies = [
  "anyhow",
  "bstr",
  "grep",
+ "grep-index",
  "ignore",
  "lexopt",
  "log",

Cargo.toml

@@ -58,6 +58,7 @@ rust-version = "1.96"
 anyhow = "1.0.75"
 bstr = "1.7.0"
 grep = { version = "0.4.1", path = "crates/grep" }
+grep-index = { version = "0.0.1", path = "crates/index", optional = true }
 ignore = { version = "0.4.29", path = "crates/ignore" }
 lexopt = "0.3.0"
 log = "0.4.5"
@@ -77,7 +78,7 @@ walkdir = "2"
 pcre2 = ["grep/pcre2"]
 # This provides opt-in support for indexing. This is currently in active
 # development and may have very serious bugs. Use at your own risk.
-unstable-index = []
+unstable-index = ["dep:grep-index"]
 
 [profile.release]
 debug = 1

crates/core/flags/hiargs.rs

@@ -61,6 +61,7 @@ pub(crate) struct HiArgs {
     ignore_file_case_insensitive: bool,
     ignore_file: Vec<PathBuf>,
     include_zero: bool,
+    index: usize,
     invert_match: bool,
     is_terminal_stdout: bool,
     line_number: bool,
@@ -287,6 +288,7 @@ impl HiArgs {
             ignore_file: low.ignore_file,
             ignore_file_case_insensitive: low.ignore_file_case_insensitive,
             include_zero: low.include_zero,
+            index: low.index,
             invert_match: low.invert_match,
             is_terminal_stdout: state.is_terminal_stdout,
             line_number,
@@ -925,6 +927,31 @@ impl HiArgs {
         }
         Ok(builder)
     }
+
+    pub(crate) fn index(&self) -> usize {
+        self.index
+    }
+
+    #[cfg(feature = "unstable-index")]
+    pub(crate) fn index_write(&self) -> anyhow::Result<grep_index::Index> {
+        let index = grep_index::IndexDiscovery::new()
+            .create(true)
+            .discover()?
+            .ok_or_else(|| {
+                anyhow::anyhow!(
+                    "failed to find a place to create/update an index"
+                )
+            })?;
+        Ok(index)
+    }
+
+    #[cfg(feature = "unstable-index")]
+    pub(crate) fn index_read(&self) -> anyhow::Result<grep_index::Index> {
+        let index = grep_index::IndexDiscovery::new().discover()?.ok_or_else(
+            || anyhow::anyhow!("failed to find an index to read"),
+        )?;
+        Ok(index)
+    }
 }
 
 /// State that only needs to be computed once during argument parsing.

crates/core/index/disabled.rs

@@ -0,0 +1,9 @@
+use crate::flags::{HiArgs, SearchMode};
+
+pub(crate) fn write(_args: &HiArgs) -> anyhow::Result<()> {
+    anyhow::bail!("indexing not enabled in this build of ripgrep")
+}
+
+pub(crate) fn read(_args: &HiArgs, _mode: SearchMode) -> anyhow::Result<bool> {
+    anyhow::bail!("indexing not enabled in this build of ripgrep")
+}

crates/core/index/enabled.rs

@@ -0,0 +1,16 @@
+use crate::flags::{HiArgs, SearchMode};
+
+pub(crate) fn write(args: &HiArgs) -> anyhow::Result<()> {
+    let _index = args.index_write()?;
+    Ok(())
+}
+
+pub(crate) fn read(args: &HiArgs, mode: SearchMode) -> anyhow::Result<bool> {
+    let _index = args.index_read()?;
+    // Do an exhaustive search for now.
+    if args.threads() == 1 {
+        crate::search(args, mode)
+    } else {
+        crate::search_parallel(args, mode)
+    }
+}

crates/core/index/mod.rs

@@ -0,0 +1,8 @@
+pub(crate) use self::imp::*;
+
+#[cfg(not(feature = "unstable-index"))]
+#[path = "disabled.rs"]
+mod imp;
+#[cfg(feature = "unstable-index")]
+#[path = "enabled.rs"]
+mod imp;

crates/core/main.rs

@@ -85,9 +85,13 @@ fn run(result: crate::flags::ParseResult<HiArgs>) -> anyhow::Result<ExitCode> {
     };
     let matched = match args.mode() {
         Mode::Search(_) if !args.matches_possible() => false,
+        Mode::Search(mode) if args.index() > 0 => index::read(&args, mode)?,
         Mode::Search(mode) if args.threads() == 1 => search(&args, mode)?,
         Mode::Search(mode) => search_parallel(&args, mode)?,
-        Mode::Index(_) => anyhow::bail!("indexing not yet implemented"),
+        Mode::Index(_) => {
+            index::write(&args)?;
+            return Ok(ExitCode::from(0));
+        }
         Mode::Files if args.threads() == 1 => files(&args)?,
         Mode::Files => files_parallel(&args)?,
         Mode::Types => return types(&args),

crates/index/src/index.rs

@@ -0,0 +1,217 @@
+use std::{
+    ffi::{OsStr, OsString},
+    path::{Path, PathBuf},
+};
+
+use anyhow::Context;
+use redb::{Database, ReadOnlyDatabase};
+
+const INDEX_FILE_NAME: &str = "index.db";
+
+#[derive(Debug)]
+pub struct Index {
+    path: PathBuf,
+    db: Handle,
+}
+
+impl Index {
+    pub fn builder() -> IndexBuilder {
+        IndexBuilder::new()
+    }
+
+    pub fn exists(path: impl AsRef<Path>) -> bool {
+        let path = path.as_ref();
+        path.is_dir() && path.join(INDEX_FILE_NAME).is_file()
+    }
+
+    pub fn path(&self) -> &Path {
+        &self.path
+    }
+}
+
+#[derive(Default)]
+pub struct IndexBuilder {
+    create: bool,
+    write: bool,
+}
+
+impl IndexBuilder {
+    pub fn new() -> IndexBuilder {
+        IndexBuilder::default()
+    }
+
+    pub fn build(&self, path: impl AsRef<Path>) -> anyhow::Result<Index> {
+        let path = path.as_ref();
+        if self.create {
+            self.build_create(path)
+        } else {
+            self.build_open(path)
+        }
+    }
+
+    fn build_create(&self, path: &Path) -> anyhow::Result<Index> {
+        if let Err(err) = std::fs::create_dir(path)
+            && err.kind() != std::io::ErrorKind::AlreadyExists
+        {
+            return Err(err).with_context(|| {
+                format!("{}: failed to create directory", path.display())
+            });
+        }
+        let path = path.join(INDEX_FILE_NAME);
+        // TODO: Retry here if the database is already open.
+        let db = Database::create(&path).with_context(|| {
+            format!("{}: failed to create database", path.display())
+        })?;
+        let index = Index { path, db: Handle::ReadWrite(db) };
+        Ok(index)
+    }
+
+    fn build_open(&self, path: &Path) -> anyhow::Result<Index> {
+        let path = path.join(INDEX_FILE_NAME);
+        // TODO: Retry here if the database is already open.
+        let db = if self.write {
+            Handle::ReadWrite(Database::open(&path).with_context(|| {
+                format!(
+                    "{}: failed to open read-write database",
+                    path.display()
+                )
+            })?)
+        } else {
+            Handle::ReadOnly(ReadOnlyDatabase::open(&path).with_context(
+                || {
+                    format!(
+                        "{}: failed to open read-only database",
+                        path.display()
+                    )
+                },
+            )?)
+        };
+        let index = Index { path, db };
+        Ok(index)
+    }
+
+    pub fn create(&mut self, yes: bool) -> &mut IndexBuilder {
+        self.create = yes;
+        self
+    }
+
+    pub fn write(&mut self, yes: bool) -> &mut IndexBuilder {
+        self.write = yes;
+        self
+    }
+}
+
+pub struct IndexDiscovery {
+    builder: IndexBuilder,
+    cwd: Option<PathBuf>,
+    env_name: OsString,
+    dir_name: OsString,
+}
+
+impl IndexDiscovery {
+    pub fn new() -> IndexDiscovery {
+        IndexDiscovery {
+            builder: IndexBuilder::new(),
+            cwd: None,
+            env_name: "RIPGREP_INDEX_PATH".into(),
+            dir_name: ".ripgrep".into(),
+        }
+    }
+
+    pub fn discover(&self) -> anyhow::Result<Option<Index>> {
+        let cwd = match self.cwd.clone() {
+            Some(cwd) => cwd,
+            None => std::env::current_dir()
+                .context("failed to get current working directory")?,
+        };
+        anyhow::ensure!(
+            cwd.is_absolute(),
+            "current working directory `{}` is not absolute",
+            cwd.display()
+        );
+        if let Some(env_path) = std::env::var_os(&self.env_name) {
+            if !env_path.is_empty() {
+                return Ok(Some(self.builder.build(env_path)?));
+            }
+        }
+        for ancestor in cwd.ancestors() {
+            let path = ancestor.join(&self.dir_name);
+            // I guess this is subject to TOCTOU problems, but the underlying
+            // embedded database should handle synchronization problems for us.
+            if self.builder.create || Index::exists(&path) {
+                return Ok(Some(self.builder.build(&path)?));
+            }
+        }
+        Ok(None)
+    }
+
+    pub fn cwd(&mut self, cwd: impl AsRef<Path>) -> &mut IndexDiscovery {
+        self.cwd = Some(cwd.as_ref().into());
+        self
+    }
+
+    pub fn env_name(
+        &mut self,
+        name: impl AsRef<OsStr>,
+    ) -> &mut IndexDiscovery {
+        self.env_name = name.as_ref().into();
+        self
+    }
+
+    pub fn dir_name(
+        &mut self,
+        name: impl AsRef<OsStr>,
+    ) -> &mut IndexDiscovery {
+        self.dir_name = name.as_ref().into();
+        self
+    }
+
+    pub fn create(&mut self, yes: bool) -> &mut IndexDiscovery {
+        self.builder.create(yes);
+        self
+    }
+}
+
+enum Handle {
+    ReadWrite(Database),
+    ReadOnly(ReadOnlyDatabase),
+}
+
+impl Handle {
+    fn begin_read(
+        &self,
+    ) -> Result<redb::ReadTransaction, redb::TransactionError> {
+        use redb::ReadableDatabase;
+        match self {
+            Handle::ReadWrite(database) => database.begin_read(),
+            Handle::ReadOnly(database) => database.begin_read(),
+        }
+    }
+
+    fn read_write(&self) -> anyhow::Result<&Database> {
+        match self {
+            Handle::ReadWrite(database) => Ok(database),
+            Handle::ReadOnly(_) => {
+                anyhow::bail!("index was opened read-only")
+            }
+        }
+    }
+
+    fn read_write_mut(&mut self) -> anyhow::Result<&mut Database> {
+        match self {
+            Handle::ReadWrite(database) => Ok(database),
+            Handle::ReadOnly(_) => {
+                anyhow::bail!("index was opened read-only")
+            }
+        }
+    }
+}
+
+impl std::fmt::Debug for Handle {
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        match *self {
+            Handle::ReadWrite(_) => write!(f, "<redb read-write database>"),
+            Handle::ReadOnly(_) => write!(f, "<redb read-only database>"),
+        }
+    }
+}

crates/index/src/lib.rs

@@ -1 +1,6 @@
+#![allow(warnings)]
+
+pub use index::{Index, IndexBuilder, IndexDiscovery};
+
+mod index;
 pub mod literal;

tests/index/basic.rs

@@ -0,0 +1,6 @@
+use crate::util::{Dir, TestCommand};
+
+rgtest!(read_no_index, |dir: Dir, _cmd: TestCommand| {
+    dir.create("test", "foobar");
+    dir.command().arg("-X").arg("foobar").assert_err();
+});

tests/index/disallowed.rs

@@ -2,6 +2,8 @@ use crate::util::{Dir, TestCommand};
 
 rgtest!(search_mode, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command()
         .arg("-X")
         .arg("--files-without-match")
@@ -11,11 +13,15 @@ rgtest!(search_mode, |dir: Dir, _cmd: TestCommand| {
 
 rgtest!(binary, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--text").arg("foobar").assert_err();
 });
 
 rgtest!(encoding, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("-Eutf16").arg("foobar").assert_err();
     dir.command().arg("-X").arg("-Enone").arg("foobar").assert_err();
 
@@ -27,6 +33,8 @@ rgtest!(encoding, |dir: Dir, _cmd: TestCommand| {
 
 rgtest!(engine, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--engine=pcre2").arg("foobar").assert_err();
 
     eqnice!(
@@ -41,27 +49,37 @@ rgtest!(engine, |dir: Dir, _cmd: TestCommand| {
 
 rgtest!(follow, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--follow").arg("foobar").assert_err();
 });
 
 rgtest!(glob, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--glob=test").arg("foobar").assert_err();
 });
 
 rgtest!(hidden, |dir: Dir, _cmd: TestCommand| {
     dir.create(".test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--hidden").arg("foobar").assert_err();
 });
 
 rgtest!(iglob, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--iglob=test").arg("foobar").assert_err();
 });
 
 rgtest!(ignore_file, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
     dir.create("ignore-file", "");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command()
         .arg("-X")
         .arg("--ignore-file=ignore-file")
@@ -71,6 +89,8 @@ rgtest!(ignore_file, |dir: Dir, _cmd: TestCommand| {
 
 rgtest!(no_ignore, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--no-ignore").arg("foobar").assert_err();
     dir.command().arg("-X").arg("--no-ignore-dot").arg("foobar").assert_err();
     dir.command()
@@ -98,11 +118,15 @@ rgtest!(no_ignore, |dir: Dir, _cmd: TestCommand| {
 
 rgtest!(no_require_git, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--no-require-git").arg("foobar").assert_err();
 });
 
 rgtest!(one_file_system, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command()
         .arg("-X")
         .arg("--one-file-system")
@@ -113,16 +137,22 @@ rgtest!(one_file_system, |dir: Dir, _cmd: TestCommand| {
 #[cfg(unix)] // in order to use `cat`
 rgtest!(pre, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--pre").arg("cat").arg("foobar").assert_err();
 });
 
 rgtest!(search_zip, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("--search-zip").arg("foobar").assert_err();
 });
 
 rgtest!(unrestricted, |dir: Dir, _cmd: TestCommand| {
     dir.create("test", "foobar");
+    dir.command().arg("--x-crud").assert_exit_code(0);
+
     dir.command().arg("-X").arg("-u").arg("foobar").assert_err();
     dir.command().arg("-X").arg("-uu").arg("foobar").assert_err();
     dir.command().arg("-X").arg("-uuu").arg("foobar").assert_err();

tests/index/mod.rs

@@ -1,3 +1,5 @@
+// Some basic UX interactions with an index.
+mod basic;
 // At present, many features of ripgrep are not allowed when using indexing.
 // Ideally some or many of these restrictions will be lifted over time.
 mod disallowed;