diff --git a/Cargo.lock b/Cargo.lock index 90fc585d..530c0c96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4319,6 +4319,7 @@ name = "ide-host" version = "0.1.38" dependencies = [ "base64 0.22.1", + "blake3", "criterion", "dirs 6.0.0", "dunce", diff --git a/crates/ide-host/Cargo.toml b/crates/ide-host/Cargo.toml index a8473b6b..ada0bcd6 100644 --- a/crates/ide-host/Cargo.toml +++ b/crates/ide-host/Cargo.toml @@ -17,6 +17,7 @@ ide-vfs.workspace = true regex-syntax.workspace = true ide-workspace.workspace = true ide-language.workspace = true +blake3.workspace = true ide-index.workspace = true ide-syntax.workspace = true ide-search.workspace = true diff --git a/crates/ide-host/src/plugins/api.rs b/crates/ide-host/src/plugins/api.rs index 0fa9fea5..62e43dde 100644 --- a/crates/ide-host/src/plugins/api.rs +++ b/crates/ide-host/src/plugins/api.rs @@ -396,11 +396,62 @@ impl Shared { } } - pub fn storage_file(&self, plugin: &str) -> PathBuf { - self.data_dir.join(plugin).join("storage.json") + /// The plugin's stored values: one file per key, so a write only + /// rewrites its own value. + pub fn storage_dir(&self, plugin: &str) -> PathBuf { + self.data_dir.join(plugin).join("storage") } } +fn storage_io(e: std::io::Error) -> HostError { + HostError::Failed(format!("storage: {e}")) +} + +/// A key's file name: its hash, since keys may hold any character. +fn entry_name(key: &str) -> String { + blake3::hash(key.as_bytes()).to_hex().to_string() +} + +/// A stored entry: the key's length (2 bytes, little-endian), the key, then the value. +fn decode_entry(bytes: &[u8]) -> Option<(&str, &str)> { + let len = usize::from(u16::from_le_bytes(bytes.get(..2)?.try_into().ok()?)); + let key = std::str::from_utf8(bytes.get(2..2 + len)?).ok()?; + let value = std::str::from_utf8(&bytes[2 + len..]).ok()?; + Some((key, value)) +} + +fn write_entry(dir: &Path, key: &str, value: &str) -> Result<(), HostError> { + std::fs::create_dir_all(dir).map_err(storage_io)?; + let file = dir.join(entry_name(key)); + let len = u16::try_from(key.len()).expect("keys are at most MAX_KEY bytes"); + let mut bytes = Vec::with_capacity(2 + key.len() + value.len()); + bytes.extend_from_slice(&len.to_le_bytes()); + bytes.extend_from_slice(key.as_bytes()); + bytes.extend_from_slice(value.as_bytes()); + let tmp = file.with_extension("tmp"); + std::fs::write(&tmp, bytes).and_then(|()| std::fs::rename(&tmp, &file)).map_err(storage_io) +} + +/// The stored entries' files and the bytes of their keys and values. +fn entries(dir: &Path) -> Result, HostError> { + let list = match std::fs::read_dir(dir) { + Ok(list) => list, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new().into_iter()), + Err(e) => return Err(storage_io(e)), + }; + let mut out = Vec::new(); + for entry in list { + let entry = entry.map_err(storage_io)?; + let path = entry.path(); + if path.extension().is_some() { + continue; // a write that didn't finish + } + let size = entry.metadata().map_err(storage_io)?.len(); + out.push((path, usize::try_from(size).unwrap_or(usize::MAX).saturating_sub(2))); + } + Ok(out.into_iter()) +} + pub(super) struct PluginApi { pub pkg: Package, pub shared: Arc, @@ -411,21 +462,24 @@ impl PluginApi { self.pkg.id() } - fn load_storage(&self) -> Result, HostError> { - match std::fs::read(self.shared.storage_file(self.id())) { - Ok(bytes) => serde_json::from_slice(&bytes).map_err(|e| HostError::Failed(format!("storage: {e}"))), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(BTreeMap::new()), - Err(e) => Err(HostError::Failed(format!("storage: {e}"))), + /// The storage directory, after moving a `storage.json` written by an + /// older version into it. + fn storage(&self) -> Result { + let dir = self.shared.storage_dir(self.id()); + let legacy = self.shared.data_dir.join(self.id()).join("storage.json"); + match std::fs::read(&legacy) { + Ok(bytes) => { + let map: BTreeMap = + serde_json::from_slice(&bytes).map_err(|e| HostError::Failed(format!("storage: {e}")))?; + for (key, value) in &map { + write_entry(&dir, key, value)?; + } + std::fs::remove_file(&legacy).map_err(storage_io)?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(storage_io(e)), } - } - - fn save_storage(&self, map: &BTreeMap) -> Result<(), HostError> { - let file = self.shared.storage_file(self.id()); - let bytes = serde_json::to_vec(map).map_err(|e| HostError::Failed(e.to_string()))?; - let io = |e: std::io::Error| HostError::Failed(format!("storage: {e}")); - std::fs::create_dir_all(file.parent().expect("has parent")).map_err(io)?; - let tmp = file.with_extension("tmp"); - std::fs::write(&tmp, bytes).and_then(|()| std::fs::rename(&tmp, &file)).map_err(io) + Ok(dir) } /// The program approved for the (already checked) `process:` grant, @@ -839,33 +893,47 @@ impl HostApi for PluginApi { } fn storage_get(&self, key: &str) -> Result, HostError> { - Ok(self.load_storage()?.get(key).cloned()) + let file = self.storage()?.join(entry_name(key)); + match std::fs::read(&file) { + Ok(bytes) => Ok(decode_entry(&bytes).filter(|(k, _)| *k == key).map(|(_, v)| v.to_owned())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(storage_io(e)), + } } fn storage_set(&self, key: &str, value: Option) -> Result<(), HostError> { if key.is_empty() || key.len() > MAX_KEY { return Err(HostError::Invalid(format!("keys are 1–{MAX_KEY} bytes"))); } - let mut map = self.load_storage()?; - let changed = match value { - Some(v) => map.insert(key.to_owned(), v.clone()).as_ref() != Some(&v), - None => map.remove(key).is_some(), + let dir = self.storage()?; + let file = dir.join(entry_name(key)); + let Some(value) = value else { + return match std::fs::remove_file(&file) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(storage_io(e)), + _ => Ok(()), + }; }; - if !changed { - return Ok(()); - } - let size: usize = map.iter().map(|(k, v)| k.len() + v.len()).sum(); - if size > MAX_STORAGE { + let others: usize = entries(&dir)?.filter(|(name, _)| *name != file).map(|(_, size)| size).sum(); + if others + key.len() + value.len() > MAX_STORAGE { return Err(HostError::Invalid(format!( "storage is limited to {} MiB (UTF-8 keys and values)", MAX_STORAGE >> 20 ))); } - self.save_storage(&map) + write_entry(&dir, key, &value) } fn storage_keys(&self, prefix: &str) -> Result, HostError> { - Ok(self.load_storage()?.into_keys().filter(|k| k.starts_with(prefix)).collect()) + let dir = self.storage()?; + let mut keys = Vec::new(); + for (file, _) in entries(&dir)? { + let bytes = std::fs::read(&file).map_err(storage_io)?; + if let Some((key, _)) = decode_entry(&bytes).filter(|(k, _)| k.starts_with(prefix)) { + keys.push(key.to_owned()); + } + } + keys.sort(); + Ok(keys) } fn vcs_status(&self, root: &Path) -> Result, HostError> { @@ -1295,6 +1363,23 @@ mod tests { PluginApi { pkg: Package::new(manifest, dir.to_path_buf()), shared } } + #[test] + fn storage_moves_an_older_storage_json_to_one_file_per_key() { + let tmp = tempfile::tempdir().unwrap(); + let legacy = tmp.path().join(".data/acme.lint/storage.json"); + std::fs::create_dir_all(legacy.parent().unwrap()).unwrap(); + std::fs::write(&legacy, r#"{"a":"1","b/c":"two"}"#).unwrap(); + let api = api(tmp.path(), Arc::default()); + assert_eq!(api.storage_keys("").unwrap(), ["a", "b/c"]); + assert_eq!(api.storage_get("b/c").unwrap().as_deref(), Some("two")); + assert!(!legacy.exists()); + api.storage_set("a", Some("3".into())).unwrap(); + api.storage_set("b/c", None).unwrap(); + assert_eq!(api.storage_keys("").unwrap(), ["a"]); + assert_eq!(api.storage_get("a").unwrap().as_deref(), Some("3")); + assert_eq!(api.storage_get("b/c").unwrap(), None); + } + #[test] fn storage_keeps_multimegabyte_indexes_after_host_recreation() { let tmp = tempfile::tempdir().unwrap(); diff --git a/crates/ide-host/tests/plugins.rs b/crates/ide-host/tests/plugins.rs index e31c5284..9198ac65 100644 --- a/crates/ide-host/tests/plugins.rs +++ b/crates/ide-host/tests/plugins.rs @@ -1531,8 +1531,8 @@ fn official_key_promoter() { assert_eq!(tips.len(), 1, "{tips:?}"); assert_eq!(tips[0].0.as_deref(), Some("main"), "shown in the window the actions came from"); assert!(tips[0].1.contains("Ctrl+D") && tips[0].2.contains("3 times"), "{tips:?}"); - let stored = std::fs::read_to_string(f.plugins_dir.join(".data/openidea.key-promoter/storage.json")).unwrap(); - assert!(stored.contains("\"count:code.duplicate\":\"3\""), "{stored}"); + let stored = stored(&f, "openidea.key-promoter"); + assert!(stored.contains(&("count:code.duplicate".into(), "3".into())), "{stored:?}"); // Uninstalling removes its data. f.host.uninstall("openidea.key-promoter").unwrap(); @@ -1554,13 +1554,29 @@ fn storage_keeps_multimegabyte_indexes_after_idle_reclaim_and_removes_them_on_un assert_eq!(f.run("test.storage", "load", None).unwrap().as_deref(), Some("4194304")); assert_eq!(f.host.info_of("test.storage").unwrap().stats.activations, 2); let storage_dir = f.plugins_dir.join(".data/test.storage"); - assert!(storage_dir.join("storage.json").is_file()); + assert_eq!(std::fs::read_dir(storage_dir.join("storage")).unwrap().count(), 1, "one file per key"); f.host.uninstall("test.storage").unwrap(); assert!(!storage_dir.exists()); f.install(&pkg); assert!(f.run("test.storage", "load", None).unwrap_err().contains("missing index")); } +/// A plugin's stored entries (key, value), read from its files. +fn stored(f: &Fixture, plugin: &str) -> Vec<(String, String)> { + let dir = f.plugins_dir.join(".data").join(plugin).join("storage"); + let mut entries: Vec<_> = std::fs::read_dir(dir) + .unwrap() + .map(|e| { + let bytes = std::fs::read(e.unwrap().path()).unwrap(); + let len = usize::from(u16::from_le_bytes([bytes[0], bytes[1]])); + let key = String::from_utf8(bytes[2..2 + len].to_vec()).unwrap(); + (key, String::from_utf8(bytes[2 + len..].to_vec()).unwrap()) + }) + .collect(); + entries.sort(); + entries +} + fn git(dir: &Path, args: &[&str]) { let out = std::process::Command::new("git") .args(["-c", "user.name=t", "-c", "user.email=t@t", "-c", "init.defaultBranch=main"]) @@ -1625,8 +1641,8 @@ fn official_git_toolbox_fetches_on_schedule_after_idle_reclaim() { ) ); assert!(events.iter().any(|(_, e)| matches!(e, PluginEvent::TaskEnded { .. }))); - let stored = std::fs::read_to_string(f.plugins_dir.join(".data/openidea.git-toolbox/storage.json")).unwrap(); - assert!(stored.contains("last-fetch:"), "{stored}"); + let stored = stored(&f, "openidea.git-toolbox"); + assert!(stored.iter().any(|(key, _)| key.starts_with("last-fetch:")), "{stored:?}"); } /// Restricted Mode: git does not run in an untrusted project (no fetch).