mirror of
https://github.com/l0ng-ai/openidea.git
synced 2026-11-11 10:41:28 -10:00
Compare commits
61
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cbcd1818a7 | ||
|
|
c73d858895 | ||
|
|
6e5486586c | ||
|
|
93642b3741 | ||
|
|
b06f2e27ea | ||
|
|
86f69b7fca | ||
|
|
fefa135f5a | ||
|
|
8fe2d35d62 | ||
|
|
21046c3047 | ||
|
|
017561a20b | ||
|
|
d3a68b2754 | ||
|
|
e98a560523 | ||
|
|
6983e8ec2f | ||
|
|
dc355602a9 | ||
|
|
924fa9259d | ||
|
|
24deeb0dbb | ||
|
|
0ce7f6e430 | ||
|
|
84fd0d0a60 | ||
|
|
96c99e0396 | ||
|
|
fd4838fc36 | ||
|
|
4873905884 | ||
|
|
6b37788971 | ||
|
|
44bf648bee | ||
|
|
a0c3249368 | ||
|
|
757412cb73 | ||
|
|
a497178b3e | ||
|
|
8b179992e8 | ||
|
|
85541b3663 | ||
|
|
0418a2ce01 | ||
|
|
c8639b38ba | ||
|
|
f04658612f | ||
|
|
d5cb379cae | ||
|
|
e3a818b607 | ||
|
|
0b1872937a | ||
|
|
f11cd361a7 | ||
|
|
7034fe1640 | ||
|
|
fc264d3f5c | ||
|
|
8e90341b3a | ||
|
|
3644956755 | ||
|
|
90e7e3162e | ||
|
|
833ae4c3c4 | ||
|
|
584baa3f71 | ||
|
|
98010e6513 | ||
|
|
a3640f655f | ||
|
|
8aaa7daf83 | ||
|
|
6a9a970df4 | ||
|
|
a309b7e6fa | ||
|
|
6971936f78 | ||
|
|
9d781a2f83 | ||
|
|
3260e5e224 | ||
|
|
ed575a00d9 | ||
|
|
2b41e61ce9 | ||
|
|
908740b3f2 | ||
|
|
e702e4da7b | ||
|
|
91023de207 | ||
|
|
02ee2880fe | ||
|
|
5a316e81d6 | ||
|
|
7c2ca4d061 | ||
|
|
8d09479a65 | ||
|
|
8fafde10b4 | ||
|
|
b1cd917a96 |
No files matched your search
@@ -0,0 +1,2 @@
|
||||
custom:
|
||||
- https://github.com/l0ng-ai/openidea#sponsor
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 70 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 97 KiB |
Generated
+67
-46
@@ -4145,7 +4145,7 @@ checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||
|
||||
[[package]]
|
||||
name = "ide-agent"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dunce",
|
||||
"encoding_rs",
|
||||
@@ -4169,7 +4169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-analysis"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ast-grep-core",
|
||||
"ide-core",
|
||||
@@ -4183,7 +4183,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-android"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"serde",
|
||||
@@ -4194,7 +4194,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-build"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"chardetng",
|
||||
@@ -4212,7 +4212,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-cli"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dirs 6.0.0",
|
||||
"serde",
|
||||
@@ -4226,7 +4226,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-clipboard"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"gtk",
|
||||
"objc2 0.6.5",
|
||||
@@ -4236,7 +4236,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-core"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dunce",
|
||||
"encoding_rs",
|
||||
@@ -4253,7 +4253,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-coverage"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"ide-run",
|
||||
@@ -4267,10 +4267,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-dap"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"ide-core",
|
||||
"ide-heap",
|
||||
"ide-jdwp",
|
||||
"ide-jsonrpc",
|
||||
"ide-jvm-attach",
|
||||
@@ -4292,7 +4293,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-db"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"bb8",
|
||||
"blake3",
|
||||
@@ -4322,7 +4323,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-docker"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"serde",
|
||||
@@ -4335,7 +4336,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-extension"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"blake3",
|
||||
@@ -4356,7 +4357,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-fonts"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"objc2-core-foundation",
|
||||
"objc2-core-text",
|
||||
@@ -4368,7 +4369,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-github"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"ide-vcs",
|
||||
@@ -4379,9 +4380,27 @@ dependencies = [
|
||||
"which",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ide-heap"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"ide-core",
|
||||
"libc",
|
||||
"memmap2",
|
||||
"proptest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"specta",
|
||||
"tempfile",
|
||||
"thiserror 2.0.21",
|
||||
"tracing",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ide-host"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"blake3",
|
||||
@@ -4401,6 +4420,7 @@ dependencies = [
|
||||
"ide-db",
|
||||
"ide-docker",
|
||||
"ide-extension",
|
||||
"ide-heap",
|
||||
"ide-http",
|
||||
"ide-index",
|
||||
"ide-jsonrpc",
|
||||
@@ -4449,7 +4469,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-http"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"ide-vfs",
|
||||
@@ -4468,7 +4488,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-index"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"criterion",
|
||||
@@ -4490,7 +4510,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-jdwp"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-jvm-inspect",
|
||||
"proptest",
|
||||
@@ -4502,7 +4522,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-jsonrpc"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"encoding_rs",
|
||||
"futures",
|
||||
@@ -4519,7 +4539,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-jvm-attach"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"tempfile",
|
||||
@@ -4528,7 +4548,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-jvm-inspect"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"serde",
|
||||
@@ -4539,7 +4559,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-language"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"criterion",
|
||||
@@ -4595,7 +4615,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-lsp"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dunce",
|
||||
"futures",
|
||||
@@ -4622,7 +4642,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-mcp"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"dunce",
|
||||
@@ -4641,7 +4661,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-notebook"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"ide-core",
|
||||
@@ -4664,7 +4684,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-plugin"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"dunce",
|
||||
@@ -4687,9 +4707,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-profiler"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"ide-jvm-attach",
|
||||
"ide-run",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -4700,7 +4721,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-remote"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dunce",
|
||||
"globset",
|
||||
@@ -4719,7 +4740,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-run"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dunce",
|
||||
"encoding_rs",
|
||||
@@ -4743,7 +4764,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-scaffold"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"ide-tools",
|
||||
@@ -4759,7 +4780,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-sdk"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dunce",
|
||||
"ide-core",
|
||||
@@ -4779,7 +4800,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-search"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"encoding_rs",
|
||||
@@ -4800,7 +4821,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-spell"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"parking_lot",
|
||||
@@ -4813,7 +4834,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-ssh"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dirs 6.0.0",
|
||||
"flate2",
|
||||
@@ -4834,7 +4855,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-syntax"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"ide-language",
|
||||
@@ -4849,7 +4870,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-terminal"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"ide-core",
|
||||
"libc",
|
||||
@@ -4862,7 +4883,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-text"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"ide-core",
|
||||
@@ -4875,7 +4896,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-tools"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"hex",
|
||||
@@ -4896,7 +4917,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-vcs"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"dunce",
|
||||
@@ -4917,7 +4938,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-vfs"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"chardetng",
|
||||
"dunce",
|
||||
@@ -4941,11 +4962,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-web"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"dirs 6.0.0",
|
||||
"ide-core",
|
||||
"open",
|
||||
"parking_lot",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
"specta",
|
||||
@@ -4957,7 +4978,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "ide-workspace"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"blake3",
|
||||
"dunce",
|
||||
@@ -6535,7 +6556,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "openidea"
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"blake3",
|
||||
@@ -7776,9 +7797,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustix"
|
||||
version = "1.1.5"
|
||||
version = "1.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
|
||||
checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"errno",
|
||||
|
||||
+3
-1
@@ -5,7 +5,7 @@ members = ["crates/*", "src-tauri"]
|
||||
exclude = ["sdk", "plugins"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.41"
|
||||
version = "0.1.42"
|
||||
edition = "2024"
|
||||
rust-version = "1.96"
|
||||
license = "Apache-2.0"
|
||||
@@ -51,6 +51,7 @@ ide-cli = { path = "crates/ide-cli" }
|
||||
ide-jvm-attach = { path = "crates/ide-jvm-attach" }
|
||||
ide-jvm-inspect = { path = "crates/ide-jvm-inspect" }
|
||||
ide-jdwp = { path = "crates/ide-jdwp" }
|
||||
ide-heap = { path = "crates/ide-heap" }
|
||||
|
||||
# serialization & IPC typing
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
@@ -86,6 +87,7 @@ trash = "5"
|
||||
blake3 = "1"
|
||||
dunce = "1"
|
||||
dirs = "6"
|
||||
memmap2 = "0.9"
|
||||
zip = { version = "9.2.3", default-features = false, features = ["deflate"] }
|
||||
|
||||
# search, index, language
|
||||
|
||||
@@ -19,6 +19,8 @@
|
||||
<a href="#plugins">Plugins</a>
|
||||
·
|
||||
<a href="#community">Community</a>
|
||||
·
|
||||
<a href="#sponsor">Sponsor</a>
|
||||
</p>
|
||||
|
||||
https://github.com/user-attachments/assets/234dd54c-1255-4b3a-b35c-533d78d737c0
|
||||
@@ -68,6 +70,17 @@ the [official plugins](plugins/official). The plugin API is still a preview.
|
||||
|
||||
- Contributing: see [CONTRIBUTING.md](CONTRIBUTING.md) and the [contributor docs](https://openidea.sh/en/contribute/setup).
|
||||
|
||||
## Sponsor
|
||||
|
||||
If OpenIDEA saves you time, you can support its development with Alipay or WeChat Pay.
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td align="center"><img src=".github/assets/sponsor-alipay.png" alt="Alipay QR code" width="200"><br>Alipay</td>
|
||||
<td align="center"><img src=".github/assets/sponsor-wechat.png" alt="WeChat Pay QR code" width="200"><br>WeChat Pay</td>
|
||||
</tr>
|
||||
</table>
|
||||
|
||||
## License
|
||||
|
||||
[Apache 2.0](LICENSE). Bundled third-party assets are listed in [`THIRD_PARTY_LICENSES.md`](THIRD_PARTY_LICENSES.md).
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
//! A short-lived ACP session to learn models, effort levels and other config options
|
||||
//! before the user's first task (Cursor and other ACP agents do not expose these in
|
||||
//! `initialize` alone).
|
||||
|
||||
use super::{Conn, Dialect};
|
||||
use crate::config;
|
||||
use crate::model::AgentConfigOption;
|
||||
use crate::provider::ProviderSpec;
|
||||
use crate::{Shared, client_info};
|
||||
use parking_lot::Mutex;
|
||||
use serde_json::{Value, json};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
const DISCOVER_TIMEOUT: Duration = if cfg!(test) { Duration::from_secs(3) } else { Duration::from_secs(120) };
|
||||
|
||||
/// Opens `session/new`, reads `configOptions`, and records per-model `thought_level` choices.
|
||||
pub(crate) async fn run(shared: &Arc<Shared>, spec: &ProviderSpec) -> Result<(), String> {
|
||||
let root = shared.paths.project_root.clone();
|
||||
let launch = spec.clone();
|
||||
let launch = tokio::task::spawn_blocking({
|
||||
let root = root.clone();
|
||||
move || launch.resolve_for_launch(&root)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let cwd = dunce::canonicalize(&root).unwrap_or(root);
|
||||
let stderr: Arc<Mutex<Vec<String>>> = Arc::default();
|
||||
let lines = stderr.clone();
|
||||
let on_stderr: Box<dyn Fn(String) + Send + Sync> = Box::new(move |l| lines.lock().push(l));
|
||||
let client = Dialect::wrap(Arc::new(ide_jsonrpc::NoopHandler));
|
||||
let peer = (shared.connector)(&launch, &cwd, client.clone(), on_stderr)
|
||||
.map_err(|e| format!("Couldn't start {} (`{}`): {e}", launch.name, launch.command))?;
|
||||
let conn = Conn::new(peer.clone(), client);
|
||||
let work = async {
|
||||
conn.initialize(&client_info()).await.map_err(|e| e.to_string())?;
|
||||
let cwd_s = cwd.to_string_lossy().into_owned();
|
||||
// No project MCP during discovery: Cursor can spend a long time spawning servers.
|
||||
let new: Value = conn
|
||||
.request("session/new", json!({ "cwd": cwd_s, "mcpServers": [] }), None)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
let mut options = config::options_of(&new);
|
||||
config::add_client_options(&launch.id, &mut options);
|
||||
let model_option = options.iter().find(|o| o.category.as_deref() == Some("model"));
|
||||
let effort = levels_for_model_from_slice(&options);
|
||||
let levels: Vec<(String, Vec<AgentConfigOption>)> = match (model_option, effort) {
|
||||
(Some(model), Some(effort)) if !model.values.is_empty() => {
|
||||
model.values.iter().map(|v| (v.value.clone(), effort.clone())).collect()
|
||||
}
|
||||
(Some(model), Some(effort)) => model.current.clone().map(|m| vec![(m, effort)]).unwrap_or_default(),
|
||||
_ => Vec::new(),
|
||||
};
|
||||
let option_ids: Vec<_> = options.iter().map(|o| o.id.as_str()).collect();
|
||||
tracing::info!(
|
||||
"agent {}: settings discovery got {} config option(s) {:?}, {} model effort list(s)",
|
||||
launch.id,
|
||||
options.len(),
|
||||
option_ids,
|
||||
levels.len()
|
||||
);
|
||||
shared.remember_discovered(&launch.id, &options, &[], &levels);
|
||||
Ok::<(), String>(())
|
||||
};
|
||||
let result = tokio::time::timeout(DISCOVER_TIMEOUT, work).await;
|
||||
peer.kill();
|
||||
match result {
|
||||
Ok(Ok(())) => Ok(()),
|
||||
Ok(Err(e)) => Err(match stderr.lock().last() {
|
||||
Some(line) => format!("{e}: {line}"),
|
||||
None => e,
|
||||
}),
|
||||
Err(_) => {
|
||||
let tail = stderr.lock().join("\n");
|
||||
if tail.is_empty() {
|
||||
Err("the agent did not answer in time".into())
|
||||
} else {
|
||||
Err(format!("the agent did not answer in time:\n{tail}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_thought_level(o: &AgentConfigOption) -> bool {
|
||||
o.category.as_deref() == Some("thought_level")
|
||||
|| matches!(o.id.as_str(), "effort" | "reasoning_effort" | "thinking" | "reasoning")
|
||||
}
|
||||
|
||||
fn levels_for_model_from_slice(options: &[AgentConfigOption]) -> Option<Vec<AgentConfigOption>> {
|
||||
let levels: Vec<_> = options
|
||||
.iter()
|
||||
.filter(|o| is_thought_level(o))
|
||||
.cloned()
|
||||
.map(|mut o| {
|
||||
if o.category.as_deref() != Some("thought_level") {
|
||||
o.category = Some("thought_level".into());
|
||||
}
|
||||
o
|
||||
})
|
||||
.collect();
|
||||
(!levels.is_empty()).then_some(levels)
|
||||
}
|
||||
@@ -14,6 +14,7 @@
|
||||
//! - Agent → client ([`Dialect`]): `session/update`s are normalized ([`normalize`]) and
|
||||
//! permission requests get the v1 `toolCall`.
|
||||
|
||||
pub(crate) mod discover;
|
||||
mod normalize;
|
||||
|
||||
use ide_jsonrpc::{Handler, Peer, RpcError, codes};
|
||||
@@ -57,6 +58,9 @@ fn client_capabilities() -> Value {
|
||||
"terminal": false,
|
||||
// Subagents as sessions of their own (ACP subagents RFD; v2 baseline).
|
||||
"subagents": {},
|
||||
// Cursor exposes model parameters (e.g. Composer `fast`) as separate config options
|
||||
// only when the client advertises this flag (undocumented Cursor ACP extension).
|
||||
"_meta": { "parameterizedModelPicker": true },
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -93,9 +93,81 @@ fn the_negotiating_request_carries_both_versions() {
|
||||
assert!(p["capabilities"].is_object());
|
||||
assert_eq!(p["clientCapabilities"]["subagents"], json!({}));
|
||||
assert_eq!(p["clientCapabilities"]["terminal"], false);
|
||||
assert_eq!(
|
||||
p["clientCapabilities"]["_meta"]["parameterizedModelPicker"], true,
|
||||
"Cursor and similar agents need parameterized model options (e.g. fast on/off)"
|
||||
);
|
||||
// What we sent in the capture is what claude-agent-acp negotiated v2 and v1 from.
|
||||
let sent = &frames(TURN, "out")[0]["params"];
|
||||
assert_eq!(sent["clientCapabilities"], p["clientCapabilities"]);
|
||||
let sent = frames(TURN, "out")[0]["params"]["clientCapabilities"].clone();
|
||||
let mut expected = sent;
|
||||
expected["_meta"] = p["clientCapabilities"]["_meta"].clone();
|
||||
assert_eq!(expected, p["clientCapabilities"]);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discovery_reads_effort_and_fast_from_session_new_then_stops_the_agent() {
|
||||
#[derive(Default)]
|
||||
struct DiscoveryAgent {
|
||||
peer: std::sync::OnceLock<Peer>,
|
||||
calls: parking_lot::Mutex<Vec<(String, Value)>>,
|
||||
}
|
||||
impl Handler for DiscoveryAgent {
|
||||
fn on_notification(&self, _: &str, _: Value) {}
|
||||
fn on_request(&self, method: &str, params: Value) -> Result<Value, (i64, String)> {
|
||||
self.calls.lock().push((method.to_owned(), params));
|
||||
match method {
|
||||
"initialize" => Ok(json!({
|
||||
"protocolVersion": 1,
|
||||
"agentCapabilities": { "loadSession": true }
|
||||
})),
|
||||
"session/new" => Ok(json!({
|
||||
"sessionId": "discover",
|
||||
"configOptions": [
|
||||
{"id": "model", "category": "model", "type": "select", "currentValue": "m1",
|
||||
"options": [{"value": "m1", "name": "One"}, {"value": "m2", "name": "Two"}]},
|
||||
{"id": "effort", "type": "select", "currentValue": "medium",
|
||||
"options": [{"value": "low"}, {"value": "medium"}, {"value": "high"}]},
|
||||
{"id": "fast", "type": "select", "currentValue": "false",
|
||||
"options": [{"value": "false", "name": "Off"}, {"value": "true", "name": "On"}]}
|
||||
]
|
||||
})),
|
||||
_ => Ok(json!({})),
|
||||
}
|
||||
}
|
||||
}
|
||||
let agent = Arc::new(DiscoveryAgent::default());
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let agent_for_connector = agent.clone();
|
||||
let connector: crate::Connector = Arc::new(move |_, _, client, _| {
|
||||
let (ca, cb) = tokio::io::duplex(1 << 20);
|
||||
let (car, caw) = tokio::io::split(ca);
|
||||
let (cbr, cbw) = tokio::io::split(cb);
|
||||
let agent_peer = Peer::connect_framed(cbr, cbw, agent_for_connector.clone(), Framing::Lines);
|
||||
let _ = agent_for_connector.peer.set(agent_peer);
|
||||
Ok(Peer::connect_framed(car, caw, client, Framing::Lines))
|
||||
});
|
||||
let mgr = crate::AgentManager::with_connector(
|
||||
crate::AgentPaths {
|
||||
project_root: tmp.path().to_path_buf(),
|
||||
store_dir: tmp.path().join("store"),
|
||||
worktrees_dir: tmp.path().join("worktrees"),
|
||||
models_file: tmp.path().join("models.json"),
|
||||
custom_agents_dir: None,
|
||||
home_dir: None,
|
||||
},
|
||||
tokio::runtime::Handle::current(),
|
||||
connector,
|
||||
);
|
||||
mgr.commands("cursor", None).await.unwrap();
|
||||
let calls = agent.calls.lock();
|
||||
assert_eq!(calls.iter().map(|(m, _)| m.as_str()).collect::<Vec<_>>(), ["initialize", "session/new"]);
|
||||
assert_eq!(calls[1].1["mcpServers"], json!([]));
|
||||
let cursor = mgr.providers().into_iter().find(|p| p.id == "cursor").unwrap();
|
||||
let fast = cursor.config.iter().find(|o| o.id == "fast").expect("fast option");
|
||||
assert_eq!(fast.category.as_deref(), Some("model_config"));
|
||||
assert_eq!(fast.current.as_deref(), Some("false"));
|
||||
assert!(cursor.thought_levels.contains_key("m1") && cursor.thought_levels.contains_key("m2"));
|
||||
assert!(cursor.thought_levels["m1"].iter().any(|o| o.category.as_deref() == Some("thought_level")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -107,7 +107,15 @@ pub(crate) fn parse_options(options: &Value) -> Vec<AgentConfigOption> {
|
||||
Some(AgentConfigOption {
|
||||
name: str_of(o, "name").unwrap_or_else(|| id.clone()),
|
||||
description: str_of(o, "description"),
|
||||
category: str_of(o, "category").or_else(|| matches!(id.as_str(), "model" | "mode").then(|| id.clone())),
|
||||
category: str_of(o, "category").or_else(|| {
|
||||
matches!(id.as_str(), "model" | "mode")
|
||||
.then(|| id.clone())
|
||||
.or_else(|| {
|
||||
matches!(id.as_str(), "effort" | "reasoning_effort" | "thinking" | "reasoning")
|
||||
.then(|| "thought_level".into())
|
||||
})
|
||||
.or_else(|| matches!(id.as_str(), "fast").then(|| "model_config".into()))
|
||||
}),
|
||||
current: str_of(o, "currentValue"),
|
||||
values,
|
||||
via: AgentConfigVia::Option,
|
||||
@@ -333,6 +341,21 @@ mod tests {
|
||||
assert!(bypass_approvals(&url_mode));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effort_and_fast_categories_are_inferred_from_ids() {
|
||||
let c = options_of(&json!({
|
||||
"configOptions": [
|
||||
{"id": "effort", "type": "select", "currentValue": "high",
|
||||
"options": [{"value": "low"}, {"value": "high"}]},
|
||||
{"id": "fast", "type": "select", "currentValue": "false",
|
||||
"options": [{"value": "false", "name": "Off"}, {"value": "true", "name": "On"}]}
|
||||
]
|
||||
}));
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(c[0].category.as_deref(), Some("thought_level"));
|
||||
assert_eq!(c[1].category.as_deref(), Some("model_config"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn command_sources_in_updates_and_stored_tasks() {
|
||||
let update = json!({"availableCommands": [
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
use super::native::claude::history;
|
||||
use serde::Serialize;
|
||||
use specta::Type;
|
||||
use std::io::BufRead;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// A session on this machine that no task holds yet.
|
||||
@@ -62,11 +63,28 @@ fn message_text(e: &serde_json::Value) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Claude Code transcripts: `<config>/projects/*/<id>.jsonl`.
|
||||
pub fn claude_sessions(config_dir: &Path) -> Vec<ExternalSession> {
|
||||
/// Whether a session that ran in `cwd` belongs to the project at `root` (in it or below it).
|
||||
fn in_project(cwd: Option<&str>, root: &Path) -> bool {
|
||||
cwd.is_some_and(|cwd| Path::new(cwd).starts_with(root))
|
||||
}
|
||||
|
||||
/// Claude Code transcripts of the project at `root`: `<config>/projects/<escaped cwd>/<id>.jsonl`.
|
||||
/// Only the folders named after `root` or a folder below it are read, so other projects'
|
||||
/// transcripts (often gigabytes) are never opened.
|
||||
pub fn claude_sessions(config_dir: &Path, root: &Path) -> Vec<ExternalSession> {
|
||||
let mut out = Vec::new();
|
||||
let projects = config_dir.join("projects");
|
||||
let dirs: Vec<PathBuf> = std::fs::read_dir(&projects).into_iter().flatten().flatten().map(|d| d.path()).collect();
|
||||
let escaped = history::escape(root);
|
||||
let dirs: Vec<PathBuf> = std::fs::read_dir(&projects)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.flatten()
|
||||
.filter(|d| {
|
||||
let name = d.file_name().to_string_lossy().into_owned();
|
||||
name == escaped || name.starts_with(&format!("{escaped}-"))
|
||||
})
|
||||
.map(|d| d.path())
|
||||
.collect();
|
||||
for dir in dirs {
|
||||
let files: Vec<PathBuf> = std::fs::read_dir(&dir)
|
||||
.into_iter()
|
||||
@@ -93,6 +111,10 @@ pub fn claude_sessions(config_dir: &Path) -> Vec<ExternalSession> {
|
||||
.filter_map(|e| e.get("cwd").and_then(serde_json::Value::as_str))
|
||||
.next()
|
||||
.map(str::to_owned);
|
||||
// Escaping is lossy (`a/b-c` and `a/b/c` share a folder): the transcript decides.
|
||||
if !in_project(cwd.as_deref(), root) {
|
||||
continue;
|
||||
}
|
||||
out.push(ExternalSession {
|
||||
provider: "claude".to_owned(),
|
||||
session_id: id,
|
||||
@@ -138,12 +160,13 @@ fn codex_user_text(line: &serde_json::Value) -> Option<String> {
|
||||
|
||||
/// One rollout file's session, or `None` when it names no `session_meta`.
|
||||
fn codex_file(path: &Path) -> Option<ExternalSession> {
|
||||
let text = std::fs::read_to_string(path).ok()?;
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let mut id: Option<(String, Option<String>)> = None;
|
||||
let mut title: Option<String> = None;
|
||||
// `session_meta` is first, but cap the scan: giant rollouts must not stall listing.
|
||||
for line in text.lines().take(400) {
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else { continue };
|
||||
for line in std::io::BufReader::new(file).lines().take(400) {
|
||||
let Ok(line) = line else { break };
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(&line) else { continue };
|
||||
if id.is_none()
|
||||
&& let Some(meta) = codex_meta(&v)
|
||||
{
|
||||
@@ -183,11 +206,11 @@ fn rollout_files(dir: &Path, depth: usize, out: &mut Vec<PathBuf>) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Codex sessions: `<home>/.codex/sessions/**`.
|
||||
pub fn codex_sessions(codex_home: &Path) -> Vec<ExternalSession> {
|
||||
/// Codex sessions of the project at `root`: `<home>/.codex/sessions/**`.
|
||||
pub fn codex_sessions(codex_home: &Path, root: &Path) -> Vec<ExternalSession> {
|
||||
let mut files = Vec::new();
|
||||
rollout_files(&codex_home.join("sessions"), 4, &mut files);
|
||||
files.iter().filter_map(|p| codex_file(p)).collect()
|
||||
files.iter().filter_map(|p| codex_file(p)).filter(|s| in_project(s.cwd.as_deref(), root)).collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -222,7 +245,7 @@ mod tests {
|
||||
],
|
||||
);
|
||||
write(&tmp.path().join("projects/-work-app/empty.jsonl"), &[]);
|
||||
let sessions = claude_sessions(tmp.path());
|
||||
let sessions = claude_sessions(tmp.path(), Path::new("/work/app"));
|
||||
assert_eq!(sessions.len(), 1);
|
||||
let s = &sessions[0];
|
||||
assert_eq!((s.provider.as_str(), s.session_id.as_str()), ("claude", "abc123"));
|
||||
@@ -234,8 +257,8 @@ mod tests {
|
||||
#[test]
|
||||
fn claude_title_falls_back_to_the_first_prompt() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
write(&tmp.path().join("projects/-x/def456.jsonl"), &[claude_user("u1", "hello there")]);
|
||||
let sessions = claude_sessions(tmp.path());
|
||||
write(&tmp.path().join("projects/-work-app/def456.jsonl"), &[claude_user("u1", "hello there")]);
|
||||
let sessions = claude_sessions(tmp.path(), Path::new("/work/app"));
|
||||
assert_eq!(sessions.len(), 1);
|
||||
assert_eq!(sessions[0].title, "hello there");
|
||||
}
|
||||
@@ -253,7 +276,7 @@ mod tests {
|
||||
],
|
||||
);
|
||||
write(&tmp.path().join("sessions/notes.jsonl"), &[json!({"nope": true})]);
|
||||
let sessions = codex_sessions(tmp.path());
|
||||
let sessions = codex_sessions(tmp.path(), Path::new("/work"));
|
||||
assert_eq!(sessions.len(), 1);
|
||||
let s = &sessions[0];
|
||||
assert_eq!((s.provider.as_str(), s.session_id.as_str()), ("codex", "thread-1"));
|
||||
@@ -264,7 +287,39 @@ mod tests {
|
||||
#[test]
|
||||
fn missing_directories_list_nothing() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
assert!(claude_sessions(&tmp.path().join("nope")).is_empty());
|
||||
assert!(codex_sessions(&tmp.path().join("nope")).is_empty());
|
||||
assert!(claude_sessions(&tmp.path().join("nope"), Path::new("/work")).is_empty());
|
||||
assert!(codex_sessions(&tmp.path().join("nope"), Path::new("/work")).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_sessions_of_the_project_are_listed() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let user = |cwd: &str| {
|
||||
json!({"type": "user", "uuid": "u1", "parentUuid": null,
|
||||
"message": {"role": "user", "content": "hi"}, "cwd": cwd})
|
||||
};
|
||||
write(&tmp.path().join("projects/-work-app/in-root.jsonl"), &[user("/work/app")]);
|
||||
write(&tmp.path().join("projects/-work-app-web/below.jsonl"), &[user("/work/app/web")]);
|
||||
// `/work/app-old` escapes like a folder below `/work/app`; its transcript says otherwise.
|
||||
write(&tmp.path().join("projects/-work-app-old/sibling.jsonl"), &[user("/work/app-old")]);
|
||||
write(&tmp.path().join("projects/-other/elsewhere.jsonl"), &[user("/other")]);
|
||||
let mut ids: Vec<String> =
|
||||
claude_sessions(tmp.path(), Path::new("/work/app")).into_iter().map(|s| s.session_id).collect();
|
||||
ids.sort();
|
||||
assert_eq!(ids, ["below", "in-root"]);
|
||||
|
||||
let rollout = |name: &str, cwd: &str| {
|
||||
write(
|
||||
&tmp.path().join(format!("codex/sessions/2026/01/02/{name}.jsonl")),
|
||||
&[json!({"payload": {"type": "session_meta", "payload": {"id": name, "cwd": cwd}}})],
|
||||
)
|
||||
};
|
||||
rollout("mine", "/work/app/api");
|
||||
rollout("theirs", "/other");
|
||||
let codex: Vec<String> = codex_sessions(&tmp.path().join("codex"), Path::new("/work/app"))
|
||||
.into_iter()
|
||||
.map(|s| s.session_id)
|
||||
.collect();
|
||||
assert_eq!(codex, ["mine"]);
|
||||
}
|
||||
}
|
||||
+191
-27
@@ -16,6 +16,7 @@ mod model;
|
||||
mod native;
|
||||
mod provider;
|
||||
pub mod registry;
|
||||
pub mod security;
|
||||
mod skills;
|
||||
mod timeline;
|
||||
mod worktree;
|
||||
@@ -39,6 +40,9 @@ use std::time::Duration;
|
||||
/// Receives the events of one project's tasks.
|
||||
pub type AgentSink = Arc<dyn Fn(AgentEvent) + Send + Sync>;
|
||||
|
||||
/// Returns the active command execution security policy for the project.
|
||||
pub type SecurityHook = Arc<dyn Fn() -> ide_core::settings::AgentSecuritySettings + Send + Sync>;
|
||||
|
||||
/// Entry changes are batched for this long before they are sent.
|
||||
const FLUSH_DELAY: Duration = Duration::from_millis(40);
|
||||
/// Updates that arrive this late after `session/load` returns are still replayed history.
|
||||
@@ -102,6 +106,7 @@ struct Shared {
|
||||
runtime: tokio::runtime::Handle,
|
||||
connector: Connector,
|
||||
sink: RwLock<Option<AgentSink>>,
|
||||
security: RwLock<Option<SecurityHook>>,
|
||||
known: Mutex<HashMap<String, Known>>,
|
||||
providers: RwLock<Vec<provider::ProviderSpec>>,
|
||||
/// Last probe of each provider.
|
||||
@@ -177,9 +182,15 @@ impl Shared {
|
||||
let shared = self.clone();
|
||||
// Keep the lookup alive if a composer unmounts while it is waiting.
|
||||
self.runtime.spawn(async move {
|
||||
if let Ok(found) =
|
||||
native::claude::discover(spec.clone(), shared.paths.project_root.clone()).await
|
||||
{
|
||||
tracing::info!("agent {}: settings discovery started", spec.id);
|
||||
let discovered = if spec.driver == provider::Driver::Acp {
|
||||
acp::discover::run(&shared, &spec).await.map(|_| None)
|
||||
} else if spec.driver == provider::Driver::Claude {
|
||||
native::claude::discover(spec.clone(), shared.paths.project_root.clone()).await.map(Some)
|
||||
} else {
|
||||
return;
|
||||
};
|
||||
if let Ok(Some(found)) = discovered {
|
||||
let commands =
|
||||
config::commands_of(&json!({ "availableCommands": found.commands }), &spec.id);
|
||||
let options = config::options_of(&json!({ "configOptions": found.options }));
|
||||
@@ -198,6 +209,8 @@ impl Shared {
|
||||
shared.remember_discovered(&spec.id, &options, &commands, &levels);
|
||||
})
|
||||
.await;
|
||||
} else if let Err(e) = discovered {
|
||||
tracing::info!("agent {}: settings discovery failed: {e}", spec.id);
|
||||
}
|
||||
let _ = done.send(true);
|
||||
});
|
||||
@@ -220,12 +233,17 @@ impl Shared {
|
||||
fn knows_all_levels(&self, provider: &str) -> bool {
|
||||
let cache = self.known.lock();
|
||||
let Some(known) = cache.get(provider) else { return false };
|
||||
known
|
||||
let models: Vec<&str> = known
|
||||
.config
|
||||
.iter()
|
||||
.filter(|o| o.category.as_deref() == Some("model"))
|
||||
.flat_map(|o| &o.values)
|
||||
.all(|v| known.thought_levels.contains_key(&v.value))
|
||||
.flat_map(|o| o.values.iter().map(|v| v.value.as_str()))
|
||||
.collect();
|
||||
// No advertised models yet: nothing to attach effort levels to.
|
||||
if models.is_empty() {
|
||||
return false;
|
||||
}
|
||||
models.iter().all(|id| known.thought_levels.contains_key(*id))
|
||||
}
|
||||
|
||||
fn root_key(&self) -> String {
|
||||
@@ -434,6 +452,7 @@ impl AgentManager {
|
||||
runtime,
|
||||
connector,
|
||||
sink: RwLock::new(None),
|
||||
security: RwLock::new(None),
|
||||
known: Mutex::new(known),
|
||||
providers,
|
||||
probes: Mutex::new(HashMap::new()),
|
||||
@@ -477,6 +496,11 @@ impl AgentManager {
|
||||
tasks
|
||||
}
|
||||
|
||||
/// Sets the callback used to read the active command execution security policy.
|
||||
pub fn set_security_hook(&self, hook: SecurityHook) {
|
||||
*self.shared.security.write() = Some(hook);
|
||||
}
|
||||
|
||||
/// The agents (custom ones re-read), whether each is installed, and what they offered last time.
|
||||
pub fn providers(&self) -> Vec<AgentProvider> {
|
||||
let specs = provider::load(self.shared.paths.custom_agents_dir.as_deref());
|
||||
@@ -501,7 +525,9 @@ impl AgentManager {
|
||||
let new = specs.iter().find(|p| &p.id == id);
|
||||
matches!((old, new), (Some(a), Some(b)) if a.command == b.command && a.args == b.args && a.env == b.env)
|
||||
});
|
||||
*prev = specs;
|
||||
*prev = specs.clone();
|
||||
drop(probes);
|
||||
drop(prev);
|
||||
list
|
||||
}
|
||||
|
||||
@@ -522,6 +548,30 @@ impl AgentManager {
|
||||
provider::set_builtin_env(&dir, id, env)
|
||||
}
|
||||
|
||||
/// Saves a new custom ACP agent through its agent file.
|
||||
pub fn save_custom_agent(&self, input: CustomAgentInput) -> IdeResult<()> {
|
||||
let Some(dir) = self.shared.paths.custom_agents_dir.clone() else {
|
||||
return Err(IdeError::unavailable("there is no custom agents directory"));
|
||||
};
|
||||
provider::save_custom(&dir, &input)
|
||||
}
|
||||
|
||||
/// Removes a custom agent's file (registry installs go through `registry_remove`).
|
||||
pub fn remove_custom_agent(&self, id: &str) -> IdeResult<()> {
|
||||
let Some(dir) = self.shared.paths.custom_agents_dir.clone() else {
|
||||
return Err(IdeError::unavailable("there is no custom agents directory"));
|
||||
};
|
||||
provider::remove_custom(&dir, id)
|
||||
}
|
||||
|
||||
/// Imports a JetBrains `acp.json` file's `agent_servers` entries as custom agents.
|
||||
pub fn import_acp_agents(&self, text: &str) -> IdeResult<Vec<AcpImportOutcome>> {
|
||||
let Some(dir) = self.shared.paths.custom_agents_dir.clone() else {
|
||||
return Err(IdeError::unavailable("there is no custom agents directory"));
|
||||
};
|
||||
provider::import_acp(&dir, text)
|
||||
}
|
||||
|
||||
/// Re-reads configured/installed agents, drops cached probes, and notifies subscribers.
|
||||
pub fn reload_providers(&self) {
|
||||
let specs = provider::load(self.shared.paths.custom_agents_dir.as_deref());
|
||||
@@ -552,10 +602,16 @@ impl AgentManager {
|
||||
let mut offered = offered;
|
||||
if task.is_none()
|
||||
&& (offered.is_empty() || !self.shared.has_config(&provider) || !self.shared.knows_all_levels(&provider))
|
||||
&& let Some(spec) = self.shared.spec(&provider).filter(|s| s.driver == provider::Driver::Claude)
|
||||
&& let Some(spec) = self
|
||||
.shared
|
||||
.spec(&provider)
|
||||
.filter(|s| matches!(s.driver, provider::Driver::Claude | provider::Driver::Acp))
|
||||
{
|
||||
let claude = spec.driver == provider::Driver::Claude;
|
||||
self.shared.fetch_settings(spec).await;
|
||||
offered = self.shared.known_commands(&provider);
|
||||
if claude {
|
||||
offered = self.shared.known_commands(&provider);
|
||||
}
|
||||
}
|
||||
for &(name, description) in builtin_commands(&provider) {
|
||||
if !offered.iter().any(|c| c.name == name) {
|
||||
@@ -592,17 +648,19 @@ impl AgentManager {
|
||||
self.tasks.read().values().filter_map(|t| t.state.lock().session_id.clone()).collect()
|
||||
}
|
||||
|
||||
/// Sessions on this machine that no task holds yet, newest first (#312).
|
||||
/// Sessions of this project on this machine that no task holds yet, newest first. Reads
|
||||
/// transcripts from disk: call it off the UI and command threads.
|
||||
pub fn external_sessions(&self) -> Vec<ExternalSession> {
|
||||
let mut out = Vec::new();
|
||||
let root = self.shared.paths.project_root.clone();
|
||||
if let Some(spec) = self.shared.spec("claude")
|
||||
&& let Some(dir) = native::claude::cli::config_dir(&spec.env)
|
||||
{
|
||||
out.extend(external::claude_sessions(&dir));
|
||||
out.extend(external::claude_sessions(&dir, &root));
|
||||
}
|
||||
let home = self.shared.paths.home_dir.clone().or_else(home_dir);
|
||||
if let Some(home) = home {
|
||||
out.extend(external::codex_sessions(&home.join(".codex")));
|
||||
out.extend(external::codex_sessions(&home.join(".codex"), &root));
|
||||
}
|
||||
let known = self.imported_session_ids();
|
||||
out.retain(|s| !known.contains(&s.session_id));
|
||||
@@ -886,14 +944,38 @@ impl AgentManager {
|
||||
}
|
||||
o.clone()
|
||||
};
|
||||
let previous = target.current.clone();
|
||||
// Cursor's `session/set_config_option` (Fast, effort) can take seconds. Show the
|
||||
// choice first so the switch and slider move immediately.
|
||||
apply_config_choice(&task, option, &value);
|
||||
task.persist();
|
||||
task.emit_task();
|
||||
|
||||
let live = task.live.lock().clone().filter(|l| !l.peer.is_closed());
|
||||
let mut model_options =
|
||||
target.category.as_deref() != Some("model") || target.current.as_deref() == Some(value.as_str());
|
||||
target.category.as_deref() != Some("model") || previous.as_deref() == Some(value.as_str());
|
||||
if let Some(live) = live.filter(|_| option != config::CURSOR_APPROVAL) {
|
||||
let (method, mut params) = config::set_request(&target, &value);
|
||||
params["sessionId"] = json!(live.session);
|
||||
let reply: Value =
|
||||
live.peer.request(method, params, None).await.map_err(|e| IdeError::internal(task.describe(e)))?;
|
||||
let reply: Value = match live.peer.request(method, params, None).await {
|
||||
Ok(reply) => reply,
|
||||
Err(e) => {
|
||||
{
|
||||
let mut s = task.state.lock();
|
||||
s.wanted.retain(|c| c.id != option);
|
||||
if let Some(prev) = previous.clone() {
|
||||
s.wanted.push(AgentConfigChoice { id: option.to_owned(), value: prev });
|
||||
}
|
||||
if let Some(o) = s.task.config.iter_mut().find(|o| o.id == option) {
|
||||
o.current = previous.clone();
|
||||
}
|
||||
s.task.model = model_of(&s.task.config).or(s.task.model.take());
|
||||
}
|
||||
task.persist();
|
||||
task.emit_task();
|
||||
return Err(IdeError::internal(task.describe(e)));
|
||||
}
|
||||
};
|
||||
if let Some(options) = reply.get("configOptions") {
|
||||
config::merge_update(&mut task.state.lock().task.config, options);
|
||||
model_options = true;
|
||||
@@ -904,8 +986,6 @@ impl AgentManager {
|
||||
if !model_options {
|
||||
s.task.config.retain(|o| o.category.as_deref() != Some("thought_level"));
|
||||
}
|
||||
s.wanted.retain(|c| c.id != option);
|
||||
s.wanted.push(AgentConfigChoice { id: option.to_owned(), value: value.clone() });
|
||||
if let Some(o) = s.task.config.iter_mut().find(|o| o.id == option) {
|
||||
o.current = Some(value);
|
||||
}
|
||||
@@ -1232,15 +1312,6 @@ impl AgentManager {
|
||||
};
|
||||
// A copy of the task (a fork) may work in the same worktree.
|
||||
let shared = self.tasks.read().values().any(|t| t.id != id && t.state.lock().task.cwd.0 == cwd);
|
||||
if remove_worktree && !shared && isolation == Isolation::Worktree && cwd.exists() {
|
||||
// On Windows the agent's processes lock the folder (their working directory,
|
||||
// open files) until they have exited.
|
||||
if let Some(peer) = live {
|
||||
let _ = tokio::time::timeout(EXIT_WAIT, peer.exited()).await;
|
||||
}
|
||||
let worktrees = self.shared.paths.worktrees_dir.clone();
|
||||
blocking(move || worktree::remove(&root, &worktrees, &cwd)).await?;
|
||||
}
|
||||
self.tasks.write().remove(id);
|
||||
*task.deleted.lock() = true;
|
||||
let _ = std::fs::remove_file(self.shared.paths.store_dir.join(format!("{id}.json")));
|
||||
@@ -1250,7 +1321,17 @@ impl AgentManager {
|
||||
{
|
||||
let _ = std::fs::remove_dir_all(self.shared.paths.store_dir.join(saved));
|
||||
}
|
||||
// The list drops the task now; removing a large worktree's folder can take a while.
|
||||
self.shared.emit(AgentEvent::Removed { id: id.to_owned() });
|
||||
if remove_worktree && !shared && isolation == Isolation::Worktree && cwd.exists() {
|
||||
// On Windows the agent's processes lock the folder (their working directory,
|
||||
// open files) until they have exited.
|
||||
if let Some(peer) = live {
|
||||
let _ = tokio::time::timeout(EXIT_WAIT, peer.exited()).await;
|
||||
}
|
||||
let worktrees = self.shared.paths.worktrees_dir.clone();
|
||||
blocking(move || worktree::remove(&root, &worktrees, &cwd)).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1303,6 +1384,41 @@ impl AgentManager {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Accept: copies the task's changed files among `rel_paths` (all with `None`) from its
|
||||
/// worktree into the local checkout, uncommitted, for the user to review and commit.
|
||||
/// Files the checkout changed itself are left alone and reported. Only worktree tasks
|
||||
/// have anything to accept.
|
||||
pub async fn accept_changes(&self, id: &str, rel_paths: Option<Vec<String>>) -> IdeResult<AgentAcceptOutcome> {
|
||||
let task = self.task(id)?;
|
||||
if task.state.lock().busy {
|
||||
return Err(IdeError::invalid("the agent is working: stop it before accepting its changes"));
|
||||
}
|
||||
let (isolation, cwd, base, root, before) = {
|
||||
let s = task.state.lock();
|
||||
(
|
||||
s.task.isolation,
|
||||
s.task.cwd.0.clone(),
|
||||
s.task.base_commit.clone(),
|
||||
s.task.project_root.clone().map(|p| p.0).unwrap_or_else(|| self.shared.paths.project_root.clone()),
|
||||
(*s.before).clone(),
|
||||
)
|
||||
};
|
||||
if isolation != Isolation::Worktree {
|
||||
return Err(IdeError::invalid("the task works in the local checkout: there is nothing to accept"));
|
||||
}
|
||||
let Some(base) = base else {
|
||||
return Err(IdeError::invalid("the task has no base revision to accept from"));
|
||||
};
|
||||
blocking(move || {
|
||||
let wanted: Vec<AgentChange> = worktree::without_untouched(worktree::changes(&cwd, &base)?, &before)
|
||||
.into_iter()
|
||||
.filter(|c| rel_paths.as_ref().is_none_or(|p| p.contains(&c.rel_path)))
|
||||
.collect();
|
||||
worktree::accept(&root, &cwd, &base, &wanted)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Stops every agent (project closed).
|
||||
pub fn shutdown(&self) {
|
||||
for task in self.tasks.read().values() {
|
||||
@@ -2121,6 +2237,44 @@ impl Task {
|
||||
.iter()
|
||||
.map(|question| question.answer_key.clone().unwrap_or_else(|| question.question.clone()))
|
||||
.collect();
|
||||
if let Some(hook) = self.shared.security.read().clone()
|
||||
&& let Some(command) = security::extract_command(params)
|
||||
{
|
||||
match security::evaluate_command(&command, &hook()) {
|
||||
security::PolicyDecision::Blocked(pattern) => {
|
||||
let reject = options.iter().find(|o| o.kind.starts_with("reject")).cloned();
|
||||
let _ = self.push_entry(AgentEntry::Permission {
|
||||
id,
|
||||
title,
|
||||
options,
|
||||
questions: Vec::new(),
|
||||
chosen: Some(format!("Blocked by policy ({pattern})")),
|
||||
});
|
||||
return match reject {
|
||||
Some(option) => json!({ "outcome": { "outcome": "selected", "optionId": option.id } }),
|
||||
None => json!({ "outcome": { "outcome": "cancelled" } }),
|
||||
};
|
||||
}
|
||||
security::PolicyDecision::Allowed => {
|
||||
if let Some(allow) = options
|
||||
.iter()
|
||||
.find(|o| o.kind == "allow_once")
|
||||
.or_else(|| options.iter().find(|o| o.kind.starts_with("allow")))
|
||||
.cloned()
|
||||
{
|
||||
let _ = self.push_entry(AgentEntry::Permission {
|
||||
id,
|
||||
title,
|
||||
options,
|
||||
questions: Vec::new(),
|
||||
chosen: Some(format!("{} (whitelisted)", allow.name)),
|
||||
});
|
||||
return json!({ "outcome": { "outcome": "selected", "optionId": allow.id } });
|
||||
}
|
||||
}
|
||||
security::PolicyDecision::AskUser => {}
|
||||
}
|
||||
}
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
let entry =
|
||||
self.push_entry(AgentEntry::Permission { id, title, options: options.clone(), questions, chosen: None });
|
||||
@@ -2434,7 +2588,7 @@ fn with_details(message: String, data: Option<&Value>) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
fn client_info() -> Value {
|
||||
pub(crate) fn client_info() -> Value {
|
||||
json!({ "name": "OpenIDEA", "version": env!("CARGO_PKG_VERSION") })
|
||||
}
|
||||
|
||||
@@ -2493,6 +2647,16 @@ fn retry_of(update: &Value, now: f64) -> Option<AgentRetry> {
|
||||
})
|
||||
}
|
||||
|
||||
fn apply_config_choice(task: &Task, option: &str, value: &str) {
|
||||
let mut s = task.state.lock();
|
||||
s.wanted.retain(|c| c.id != option);
|
||||
s.wanted.push(AgentConfigChoice { id: option.to_owned(), value: value.to_owned() });
|
||||
if let Some(o) = s.task.config.iter_mut().find(|o| o.id == option) {
|
||||
o.current = Some(value.to_owned());
|
||||
}
|
||||
s.task.model = model_of(&s.task.config).or(s.task.model.take());
|
||||
}
|
||||
|
||||
fn model_of(config: &[AgentConfigOption]) -> Option<String> {
|
||||
config.iter().find(|o| o.category.as_deref() == Some("model")).and_then(|o| o.current.clone())
|
||||
}
|
||||
|
||||
@@ -602,3 +602,49 @@ pub struct McpServerInput {
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Payload for creating a custom ACP agent (always the ACP driver); the id
|
||||
/// becomes the `<config>/agents/<id>.toml` file name.
|
||||
#[derive(Clone, Debug, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase", deny_unknown_fields)]
|
||||
pub struct CustomAgentInput {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
pub command: String,
|
||||
#[serde(default)]
|
||||
pub args: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub env: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
/// What importing one entry of a JetBrains `acp.json` file did.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AcpImportOutcome {
|
||||
/// The `agent_servers` key the entry was imported from.
|
||||
pub name: String,
|
||||
/// The id its agent file would get (also when skipped).
|
||||
pub id: String,
|
||||
pub saved: bool,
|
||||
/// Why the entry was skipped; empty when it was saved.
|
||||
#[serde(default)]
|
||||
pub reason: String,
|
||||
}
|
||||
|
||||
/// What accepting a worktree task's changes copied into the local checkout.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AgentAcceptOutcome {
|
||||
/// Relative paths copied into the checkout (uncommitted), including ones already identical.
|
||||
pub accepted: Vec<String>,
|
||||
/// Files left alone because the checkout has its own edits to them.
|
||||
pub skipped: Vec<AgentAcceptSkipped>,
|
||||
}
|
||||
|
||||
/// A file accept left alone, with why.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct AgentAcceptSkipped {
|
||||
pub path: String,
|
||||
pub reason: String,
|
||||
}
|
||||
@@ -31,7 +31,7 @@ pub(super) fn transcript(config_dir: &Path, cwd: &Path, id: &str) -> PathBuf {
|
||||
}
|
||||
|
||||
/// The project directory name Claude Code uses for `cwd`.
|
||||
pub(super) fn escape(cwd: &Path) -> String {
|
||||
pub(crate) fn escape(cwd: &Path) -> String {
|
||||
cwd.to_string_lossy().chars().map(|c| if c.is_ascii_alphanumeric() { c } else { '-' }).collect()
|
||||
}
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
//! `<config>/agents/<id>.toml` (written by hand, or by installing one from the ACP
|
||||
//! Registry, see `registry`).
|
||||
|
||||
use crate::{AgentCommandReadOnlyReason, AgentProvider, AgentSource};
|
||||
use crate::{AcpImportOutcome, AgentCommandReadOnlyReason, AgentProvider, AgentSource, CustomAgentInput};
|
||||
use ide_core::{IdeError, IdeResult};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -278,6 +278,127 @@ fn update_builtin(custom_dir: &Path, id: &str, change: impl FnOnce(&mut CustomAg
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Saves a new custom agent; refuses built-in and ACP Registry ids, taken ids
|
||||
/// and an empty command. The id is also the agent file's name, so it must be
|
||||
/// a safe file stem.
|
||||
pub(crate) fn save_custom(custom_dir: &Path, input: &CustomAgentInput) -> IdeResult<()> {
|
||||
let id = validated_custom_id(&input.id)?;
|
||||
if builtin_spec(&id).is_some() {
|
||||
return Err(IdeError::invalid(format!("{id} is a built-in agent; change its command instead")));
|
||||
}
|
||||
if input.command.trim().is_empty() {
|
||||
return Err(IdeError::invalid("the agent command must not be empty"));
|
||||
}
|
||||
let file = custom_dir.join(format!("{id}.toml"));
|
||||
if file.try_exists()? {
|
||||
if CustomAgent::read(&file).is_ok_and(|a| a.registry.is_some()) {
|
||||
return Err(IdeError::invalid(format!(
|
||||
"{id} is installed from the ACP Registry; uninstall it there to replace it"
|
||||
)));
|
||||
}
|
||||
return Err(IdeError::invalid(format!("there is already a custom agent '{id}'")));
|
||||
}
|
||||
CustomAgent {
|
||||
name: Some(input.name.clone()),
|
||||
driver: Driver::Acp,
|
||||
command: input.command.clone(),
|
||||
args: input.args.clone(),
|
||||
env: input.env.clone(),
|
||||
registry: None,
|
||||
}
|
||||
.write(&file)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Removes a custom agent's file; refuses built-in, ACP Registry and missing ids.
|
||||
pub(crate) fn remove_custom(custom_dir: &Path, id: &str) -> IdeResult<()> {
|
||||
let id = validated_custom_id(id)?;
|
||||
let id = id.as_str();
|
||||
if builtin_spec(id).is_some() {
|
||||
return Err(IdeError::invalid(format!("{id} is a built-in agent and cannot be removed")));
|
||||
}
|
||||
let file = custom_dir.join(format!("{id}.toml"));
|
||||
let agent =
|
||||
CustomAgent::read(&file).map_err(|_| IdeError::not_found(format!("there is no custom agent '{id}'")))?;
|
||||
if agent.registry.is_some() {
|
||||
return Err(IdeError::invalid(format!("{id} is installed from the ACP Registry; uninstall it there")));
|
||||
}
|
||||
match std::fs::remove_file(&file) {
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(e.into()),
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Imports a JetBrains `acp.json` file's `agent_servers` entries as custom
|
||||
/// agents, one outcome per entry; entries that collide, lack a command or
|
||||
/// have no usable name are skipped with a reason.
|
||||
pub(crate) fn import_acp(custom_dir: &Path, text: &str) -> IdeResult<Vec<AcpImportOutcome>> {
|
||||
#[derive(Deserialize)]
|
||||
struct AcpJson {
|
||||
#[serde(default)]
|
||||
agent_servers: BTreeMap<String, AcpServer>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
struct AcpServer {
|
||||
#[serde(default)]
|
||||
command: String,
|
||||
#[serde(default)]
|
||||
args: Vec<String>,
|
||||
#[serde(default)]
|
||||
env: BTreeMap<String, String>,
|
||||
}
|
||||
let parsed: AcpJson =
|
||||
serde_json::from_str(text).map_err(|e| IdeError::invalid(format!("not an acp.json file: {e}")))?;
|
||||
let mut outcomes = Vec::with_capacity(parsed.agent_servers.len());
|
||||
for (name, server) in parsed.agent_servers {
|
||||
let id = slugify_id(&name);
|
||||
let mut outcome = AcpImportOutcome { name, id: id.clone(), saved: false, reason: String::new() };
|
||||
if id.is_empty() {
|
||||
outcome.reason = "the name has no usable characters for an id".to_owned();
|
||||
} else if builtin_spec(&id).is_some() {
|
||||
outcome.reason = format!("{id} is a built-in agent");
|
||||
} else if server.command.trim().is_empty() {
|
||||
outcome.reason = "the entry has no command".to_owned();
|
||||
} else if custom_dir.join(format!("{id}.toml")).try_exists()? {
|
||||
outcome.reason = format!("there is already a custom agent '{id}'");
|
||||
} else {
|
||||
CustomAgent {
|
||||
name: Some(outcome.name.clone()),
|
||||
driver: Driver::Acp,
|
||||
command: server.command,
|
||||
args: server.args,
|
||||
env: server.env,
|
||||
registry: None,
|
||||
}
|
||||
.write(&custom_dir.join(format!("{id}.toml")))?;
|
||||
outcome.saved = true;
|
||||
}
|
||||
outcomes.push(outcome);
|
||||
}
|
||||
Ok(outcomes)
|
||||
}
|
||||
|
||||
/// A custom agent id doubles as a file stem: letters, digits, `-` and `_` only.
|
||||
fn validated_custom_id(id: &str) -> IdeResult<String> {
|
||||
if !id.is_empty() && id.chars().all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') {
|
||||
return Ok(id.to_owned());
|
||||
}
|
||||
Err(IdeError::invalid(format!("'{id}' is not a usable agent id (letters, digits, '-' and '_' only)")))
|
||||
}
|
||||
|
||||
/// Turns an `acp.json` server name into a file-stem id (`"My Agent!"` → `"my-agent"`).
|
||||
fn slugify_id(name: &str) -> String {
|
||||
let mut id = String::with_capacity(name.len());
|
||||
for c in name.chars() {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
id.push(c.to_ascii_lowercase());
|
||||
} else if !id.ends_with('-') {
|
||||
id.push('-');
|
||||
}
|
||||
}
|
||||
id.trim_matches('-').to_owned()
|
||||
}
|
||||
|
||||
/// An agent file, `<config>/agents/<id>.toml`.
|
||||
#[derive(Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -753,4 +874,80 @@ mod tests {
|
||||
assert_eq!(spec.source, AgentSource::Registry { version: "0.10.0".into(), update: None });
|
||||
assert!(toml::from_str::<CustomAgent>("command = \"x\"\n[registry]\nid = \"x\"\n").is_err());
|
||||
}
|
||||
|
||||
fn custom_input(id: &str) -> CustomAgentInput {
|
||||
CustomAgentInput {
|
||||
id: id.into(),
|
||||
name: "Mine".into(),
|
||||
command: "/opt/mine".into(),
|
||||
args: vec!["acp".into()],
|
||||
env: BTreeMap::from([("K".to_owned(), "v".to_owned())]),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_agents_save_list_and_remove() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
save_custom(dir.path(), &custom_input("mine")).unwrap();
|
||||
let mine = load(Some(dir.path())).into_iter().find(|s| s.id == "mine").unwrap();
|
||||
assert_eq!((mine.name.as_str(), mine.command.as_str()), ("Mine", "/opt/mine"));
|
||||
assert_eq!(mine.env, vec![("K".to_owned(), "v".to_owned())]);
|
||||
assert_eq!(mine.source, AgentSource::Custom);
|
||||
remove_custom(dir.path(), "mine").unwrap();
|
||||
assert!(!load(Some(dir.path())).iter().any(|s| s.id == "mine"));
|
||||
remove_custom(dir.path(), "mine").unwrap_err();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_agents_refuse_builtin_registry_and_bad_ids() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
save_custom(dir.path(), &custom_input("claude")).unwrap_err();
|
||||
save_custom(dir.path(), &custom_input("../evil")).unwrap_err();
|
||||
save_custom(dir.path(), &custom_input("")).unwrap_err();
|
||||
let mut empty_command = custom_input("mine");
|
||||
empty_command.command = " ".into();
|
||||
save_custom(dir.path(), &empty_command).unwrap_err();
|
||||
save_custom(dir.path(), &custom_input("mine")).unwrap();
|
||||
save_custom(dir.path(), &custom_input("mine")).unwrap_err();
|
||||
remove_custom(dir.path(), "claude").unwrap_err();
|
||||
// An id that isn't a plain file stem never reaches outside the agents directory.
|
||||
let outside = dir.path().join("outside.toml");
|
||||
std::fs::write(&outside, "command = \"/bin/sh\"\n").unwrap();
|
||||
let agents = dir.path().join("agents");
|
||||
std::fs::create_dir(&agents).unwrap();
|
||||
remove_custom(&agents, "../outside").unwrap_err();
|
||||
assert!(outside.exists());
|
||||
std::fs::write(
|
||||
dir.path().join("amp-acp.toml"),
|
||||
"command = \"/registry/amp\"\n[registry]\nid = \"amp-acp\"\nversion = \"1.0\"\n",
|
||||
)
|
||||
.unwrap();
|
||||
save_custom(dir.path(), &custom_input("amp-acp")).unwrap_err();
|
||||
remove_custom(dir.path(), "amp-acp").unwrap_err();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn acp_json_imports_servers_and_skips_collisions() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
save_custom(dir.path(), &custom_input("taken")).unwrap();
|
||||
let outcomes = import_acp(
|
||||
dir.path(),
|
||||
r#"{"agent_servers": {
|
||||
"My Agent!": {"command": "/opt/agent", "args": ["--acp"], "env": {"K": "v"}},
|
||||
"taken": {"command": "/opt/taken"},
|
||||
"claude": {"command": "/opt/claude"},
|
||||
"!!!": {"command": "/opt/noname"},
|
||||
"silent": {"args": []}
|
||||
}}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let saved: Vec<&str> = outcomes.iter().filter(|o| o.saved).map(|o| o.id.as_str()).collect();
|
||||
assert_eq!(saved, ["my-agent"]);
|
||||
assert!(outcomes.iter().filter(|o| !o.saved).all(|o| !o.reason.is_empty()));
|
||||
let mine = load(Some(dir.path())).into_iter().find(|s| s.id == "my-agent").unwrap();
|
||||
assert_eq!((mine.name.as_str(), mine.command.as_str()), ("My Agent!", "/opt/agent"));
|
||||
assert_eq!(mine.source, AgentSource::Custom);
|
||||
import_acp(dir.path(), "not json").unwrap_err();
|
||||
assert!(import_acp(dir.path(), "{}").unwrap().is_empty());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,487 @@
|
||||
//! Command execution security policy (Whitelist / Blacklist) for coding agents.
|
||||
|
||||
use ide_core::settings::AgentSecuritySettings;
|
||||
use serde_json::Value;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum PolicyDecision {
|
||||
/// Command matched a blacklist pattern (`pattern`).
|
||||
Blocked(String),
|
||||
/// Command matched the whitelist and no blacklist pattern.
|
||||
Allowed,
|
||||
/// Neither matched; prompt the user as usual.
|
||||
AskUser,
|
||||
}
|
||||
|
||||
/// Extracts the shell command string from an ACP `session/request_permission` payload
|
||||
/// when the request is for executing a command.
|
||||
pub fn extract_command(params: &Value) -> Option<String> {
|
||||
let tool = params.get("toolCall").or_else(|| params.pointer("/subject/toolCall"))?;
|
||||
let raw_cmd = tool.pointer("/rawInput/command").or_else(|| tool.pointer("/rawInput/cmd"));
|
||||
if let Some(cmd) = raw_cmd.and_then(command_from_value) {
|
||||
return Some(cmd);
|
||||
}
|
||||
let kind = tool.get("kind").and_then(Value::as_str).unwrap_or_default();
|
||||
let name = tool.get("name").and_then(Value::as_str).unwrap_or_default();
|
||||
let title =
|
||||
tool.get("title").and_then(Value::as_str).or_else(|| params.get("title").and_then(Value::as_str))?.trim();
|
||||
if let Some(cmd) = title.strip_prefix("Run ").map(str::trim).filter(|s| !s.is_empty()) {
|
||||
return Some(cmd.to_owned());
|
||||
}
|
||||
if (kind == "execute" || matches!(name, "Bash" | "PowerShell" | "Execute" | "Run" | "terminal"))
|
||||
&& !title.is_empty()
|
||||
&& title != "Terminal"
|
||||
&& title != "Run a command"
|
||||
{
|
||||
return Some(title.to_owned());
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn command_from_value(val: &Value) -> Option<String> {
|
||||
match val {
|
||||
Value::String(s) => {
|
||||
let trimmed = s.trim();
|
||||
(!trimmed.is_empty()).then(|| trimmed.to_owned())
|
||||
}
|
||||
Value::Array(items) => {
|
||||
let words: Vec<&str> = items.iter().filter_map(Value::as_str).collect();
|
||||
if words.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let cmd = match words.as_slice() {
|
||||
[_, flag, script] if flag.starts_with('-') && flag.ends_with('c') => (*script).trim().to_owned(),
|
||||
_ => words.join(" ").trim().to_owned(),
|
||||
};
|
||||
(!cmd.is_empty()).then_some(cmd)
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Checks whether a single pattern matches `command` using prefix + glob rules:
|
||||
/// - If `pattern` contains `*` or `?`, matches using glob wildcards (where `*` matches
|
||||
/// any sequence of characters). A trailing ` *` in the pattern also matches the bare
|
||||
/// command without arguments (e.g. `git status *` matches `git status`).
|
||||
/// - Otherwise, matches when `command` equals `pattern` or starts with `pattern` followed
|
||||
/// by whitespace (prefix match at word boundary).
|
||||
pub fn matches_command(pattern: &str, command: &str) -> bool {
|
||||
let pat = pattern.trim();
|
||||
let cmd = command.trim();
|
||||
if pat.is_empty() || cmd.is_empty() {
|
||||
return false;
|
||||
}
|
||||
if pat.contains('*') || pat.contains('?') {
|
||||
if let Some(stem) = pat.strip_suffix(" *")
|
||||
&& cmd == stem.trim()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
wildcard_match(pat.as_bytes(), cmd.as_bytes())
|
||||
} else {
|
||||
cmd == pat || cmd.strip_prefix(pat).is_some_and(|rest| rest.starts_with(char::is_whitespace))
|
||||
}
|
||||
}
|
||||
|
||||
fn wildcard_match(pat: &[u8], text: &[u8]) -> bool {
|
||||
let (mut p, mut t) = (0, 0);
|
||||
let (mut star_p, mut star_t) = (None, 0);
|
||||
while t < text.len() {
|
||||
if p < pat.len() && (pat[p] == b'?' || pat[p] == text[t]) {
|
||||
p += 1;
|
||||
t += 1;
|
||||
} else if p < pat.len() && pat[p] == b'*' {
|
||||
star_p = Some(p);
|
||||
p += 1;
|
||||
star_t = t;
|
||||
} else if let Some(sp) = star_p {
|
||||
p = sp + 1;
|
||||
star_t += 1;
|
||||
t = star_t;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
while p < pat.len() && pat[p] == b'*' {
|
||||
p += 1;
|
||||
}
|
||||
p == pat.len()
|
||||
}
|
||||
|
||||
/// Matches an allowed command entry against a sub-command.
|
||||
/// An allow entry is a whole command or a prefix only up to a word boundary:
|
||||
/// e.g. `git status` matches `git status` and `git status -s`, but NOT `git statusx`.
|
||||
/// Any accidental trailing `*` in the allow entry is stripped so `git status*` behaves
|
||||
/// cleanly as a prefix at word boundaries instead of a glob wildcard.
|
||||
pub fn matches_allow_entry(entry: &str, command: &str) -> bool {
|
||||
let mut entry = entry.trim();
|
||||
if let Some(stripped) = entry.strip_suffix('*') {
|
||||
entry = stripped.trim_end();
|
||||
}
|
||||
let cmd = command.trim();
|
||||
if entry.is_empty() || cmd.is_empty() {
|
||||
return false;
|
||||
}
|
||||
cmd == entry || cmd.strip_prefix(entry).is_some_and(|rest| rest.starts_with(char::is_whitespace))
|
||||
}
|
||||
|
||||
/// Checks whether a word is an environment variable assignment (`IDENTIFIER=...`).
|
||||
fn is_env_assignment_word(word: &str) -> bool {
|
||||
let Some((name, _)) = word.split_once('=') else {
|
||||
return false;
|
||||
};
|
||||
if name.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let mut chars = name.chars();
|
||||
match chars.next() {
|
||||
Some(c) if c.is_ascii_alphabetic() || c == '_' => {}
|
||||
_ => return false,
|
||||
}
|
||||
chars.all(|c| c.is_ascii_alphanumeric() || c == '_')
|
||||
}
|
||||
|
||||
/// Returns true if the sub-command starts with an environment variable assignment,
|
||||
/// e.g. `FOO=bar cmd` or `PORT=3000`.
|
||||
fn has_leading_env_assignment(cmd: &str) -> bool {
|
||||
let trimmed = cmd.trim_start();
|
||||
let bytes = trimmed.as_bytes();
|
||||
let mut i = 0;
|
||||
let mut quote: Option<u8> = None;
|
||||
while i < bytes.len() {
|
||||
let b = bytes[i];
|
||||
if b == b'\\' && quote != Some(b'\'') && i + 1 < bytes.len() {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if let Some(q) = quote {
|
||||
if b == q {
|
||||
quote = None;
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if b == b'\'' || b == b'"' {
|
||||
quote = Some(b);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if b.is_ascii_whitespace() {
|
||||
break;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
let first_word = &trimmed[..i];
|
||||
is_env_assignment_word(first_word)
|
||||
}
|
||||
|
||||
/// Checks whether `command` contains shell constructs that must never be auto-approved:
|
||||
/// - Command substitution: `$(...)` or backticks
|
||||
/// - Redirection: `>`, `>>`, `<`
|
||||
/// - Here-docs: `<<`
|
||||
/// - Subshells: `(...)`
|
||||
/// - Leading environment assignments: `VAR=val cmd`
|
||||
pub fn has_unsafe_shell_syntax(command: &str) -> bool {
|
||||
let cmd = command.trim();
|
||||
if cmd.is_empty() {
|
||||
return false;
|
||||
}
|
||||
|
||||
if has_leading_env_assignment(cmd) {
|
||||
return true;
|
||||
}
|
||||
|
||||
let bytes = cmd.as_bytes();
|
||||
let mut i = 0;
|
||||
let mut quote: Option<u8> = None;
|
||||
|
||||
while i < bytes.len() {
|
||||
let b = bytes[i];
|
||||
|
||||
if b == b'\\' && quote != Some(b'\'') && i + 1 < bytes.len() {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(q) = quote {
|
||||
if b == q {
|
||||
quote = None;
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
// Inside double quotes: $(...) and `...` are evaluated by the shell.
|
||||
if q == b'"' {
|
||||
if b == b'`' {
|
||||
return true;
|
||||
}
|
||||
if b == b'$' && i + 1 < bytes.len() && bytes[i + 1] == b'(' {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
match b {
|
||||
b'\'' | b'"' => {
|
||||
quote = Some(b);
|
||||
i += 1;
|
||||
}
|
||||
b'`' => return true,
|
||||
b'$' if i + 1 < bytes.len() && bytes[i + 1] == b'(' => return true,
|
||||
b'<' | b'>' => return true,
|
||||
b'(' | b')' => return true,
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
|
||||
/// Splits a shell command into sub-commands separated by `;`, `&&`, `||`, `|`, or `\n`
|
||||
/// (outside single/double quotes) so compound commands cannot bypass policy checks.
|
||||
fn split_subcommands(command: &str) -> Vec<&str> {
|
||||
let bytes = command.as_bytes();
|
||||
let mut out = Vec::new();
|
||||
let mut start = 0;
|
||||
let mut i = 0;
|
||||
let mut quote: Option<u8> = None;
|
||||
|
||||
while i < bytes.len() {
|
||||
let b = bytes[i];
|
||||
if let Some(q) = quote {
|
||||
if b == b'\\' && q == b'"' && i + 1 < bytes.len() {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if b == q {
|
||||
quote = None;
|
||||
}
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
match b {
|
||||
b'\'' | b'"' => {
|
||||
quote = Some(b);
|
||||
i += 1;
|
||||
}
|
||||
b'\\' if i + 1 < bytes.len() => {
|
||||
i += 2;
|
||||
}
|
||||
b';' | b'\n' => {
|
||||
let seg = command[start..i].trim();
|
||||
if !seg.is_empty() {
|
||||
out.push(seg);
|
||||
}
|
||||
i += 1;
|
||||
start = i;
|
||||
}
|
||||
b'&' | b'|' => {
|
||||
let seg = command[start..i].trim();
|
||||
if !seg.is_empty() {
|
||||
out.push(seg);
|
||||
}
|
||||
if i + 1 < bytes.len() && bytes[i + 1] == b {
|
||||
i += 2;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
start = i;
|
||||
}
|
||||
_ => i += 1,
|
||||
}
|
||||
}
|
||||
let tail = command[start..].trim();
|
||||
if !tail.is_empty() {
|
||||
out.push(tail);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Evaluates `command` against `settings`:
|
||||
/// 1. Blacklist takes precedence: if the full command or any sub-command matches a
|
||||
/// `blocked_commands` entry, returns [`PolicyDecision::Blocked`].
|
||||
/// 2. Whitelist: allowed_commands cannot auto-approve when empty.
|
||||
/// 3. Every sub-command in a chain must be safe (no command substitution, redirection,
|
||||
/// subshell, or env assignments) and match at least one `allowed_commands` entry
|
||||
/// exactly or up to word boundary; otherwise falls back to [`PolicyDecision::AskUser`].
|
||||
pub fn evaluate_command(command: &str, settings: &AgentSecuritySettings) -> PolicyDecision {
|
||||
let cmd = command.trim();
|
||||
if cmd.is_empty() {
|
||||
return PolicyDecision::AskUser;
|
||||
}
|
||||
let subcommands = split_subcommands(cmd);
|
||||
|
||||
for pat in &settings.blocked_commands {
|
||||
let pat = pat.trim();
|
||||
if pat.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if matches_command(pat, cmd) || subcommands.iter().any(|sub| matches_command(pat, sub)) {
|
||||
return PolicyDecision::Blocked(pat.to_owned());
|
||||
}
|
||||
}
|
||||
|
||||
if settings.allowed_commands.is_empty() || subcommands.is_empty() {
|
||||
return PolicyDecision::AskUser;
|
||||
}
|
||||
|
||||
let all_allowed = subcommands.iter().all(|sub| {
|
||||
let s = sub.trim();
|
||||
if s.is_empty() || has_unsafe_shell_syntax(s) {
|
||||
return false;
|
||||
}
|
||||
settings.allowed_commands.iter().any(|entry| matches_allow_entry(entry, s))
|
||||
});
|
||||
|
||||
if all_allowed {
|
||||
return PolicyDecision::Allowed;
|
||||
}
|
||||
|
||||
PolicyDecision::AskUser
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[test]
|
||||
fn prefix_and_glob_matching() {
|
||||
assert!(matches_command("git status", "git status"));
|
||||
assert!(matches_command("git status", "git status -s"));
|
||||
assert!(!matches_command("git status", "git statusbar"));
|
||||
assert!(matches_command("git status*", "git status"));
|
||||
assert!(matches_command("git status*", "git status --short"));
|
||||
assert!(matches_command("npm test *", "npm test"));
|
||||
assert!(matches_command("npm test *", "npm test unit"));
|
||||
assert!(matches_command("rm -rf *", "rm -rf /tmp/foo"));
|
||||
assert!(matches_command("rm -rf *", "rm -rf /"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blacklist_takes_precedence_and_catches_chained_commands() {
|
||||
let settings = AgentSecuritySettings::default();
|
||||
assert_eq!(evaluate_command("rm -rf /tmp/build", &settings), PolicyDecision::Blocked("rm -rf *".into()));
|
||||
assert_eq!(evaluate_command("git status && rm -rf /", &settings), PolicyDecision::Blocked("rm -rf *".into()));
|
||||
assert_eq!(evaluate_command(":(){ :|:& };:", &settings), PolicyDecision::Blocked(":(){ :|:& };:".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allow_entry_matching_word_boundary() {
|
||||
assert!(matches_allow_entry("git status", "git status"));
|
||||
assert!(matches_allow_entry("git status", "git status -s"));
|
||||
assert!(!matches_allow_entry("git status", "git statusbar"));
|
||||
assert!(!matches_allow_entry("git status", "git statusx"));
|
||||
// Trailing asterisk in allow entry normalizes cleanly to word boundary prefix
|
||||
assert!(matches_allow_entry("git status*", "git status"));
|
||||
assert!(matches_allow_entry("git status*", "git status --short"));
|
||||
assert!(!matches_allow_entry("git status*", "git statusbar"));
|
||||
assert!(matches_allow_entry("ls", "ls"));
|
||||
assert!(matches_allow_entry("ls", "ls -la"));
|
||||
assert!(!matches_allow_entry("ls", "lsof"));
|
||||
assert!(matches_allow_entry("cargo check", "cargo check"));
|
||||
assert!(matches_allow_entry("cargo check", "cargo check --workspace"));
|
||||
assert!(!matches_allow_entry("cargo check", "cargo checkmate"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsafe_shell_syntax_detection() {
|
||||
// Command substitutions: $(...) and `...`
|
||||
assert!(has_unsafe_shell_syntax("echo $(whoami)"));
|
||||
assert!(has_unsafe_shell_syntax("echo \"user: $(whoami)\""));
|
||||
assert!(has_unsafe_shell_syntax("echo `whoami`"));
|
||||
assert!(has_unsafe_shell_syntax("echo \"`whoami`\""));
|
||||
|
||||
// Redirections: >, >>, <
|
||||
assert!(has_unsafe_shell_syntax("git status > /tmp/out"));
|
||||
assert!(has_unsafe_shell_syntax("git status >> /tmp/out"));
|
||||
assert!(has_unsafe_shell_syntax("cat < /tmp/in"));
|
||||
assert!(has_unsafe_shell_syntax("cat << 'EOF'"));
|
||||
|
||||
// Subshells: (...)
|
||||
assert!(has_unsafe_shell_syntax("(cd /tmp && ls)"));
|
||||
assert!(has_unsafe_shell_syntax("git status && (exit 1)"));
|
||||
|
||||
// Environment assignments: VAR=val ...
|
||||
assert!(has_unsafe_shell_syntax("FOO=bar git status"));
|
||||
assert!(has_unsafe_shell_syntax("PORT=3000 npm test"));
|
||||
assert!(has_unsafe_shell_syntax("_SECRET=123 cargo test"));
|
||||
assert!(has_unsafe_shell_syntax("A=\"b c\" git status"));
|
||||
|
||||
// Safe constructs with characters inside quotes or command flags
|
||||
assert!(!has_unsafe_shell_syntax("git commit -m \"fix <bug> and (issue)\""));
|
||||
assert!(!has_unsafe_shell_syntax("git commit -m 'fix <bug>'"));
|
||||
assert!(!has_unsafe_shell_syntax("cargo test --test=unit"));
|
||||
assert!(!has_unsafe_shell_syntax("git status -s"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_settings_has_no_auto_approval_and_blocks_dangerous_defaults() {
|
||||
let settings = AgentSecuritySettings::default();
|
||||
assert!(settings.allowed_commands.is_empty(), "allowed_commands must start empty");
|
||||
// Safe commands ask user by default
|
||||
assert_eq!(evaluate_command("git status", &settings), PolicyDecision::AskUser);
|
||||
assert_eq!(evaluate_command("ls", &settings), PolicyDecision::AskUser);
|
||||
assert_eq!(evaluate_command("cargo check", &settings), PolicyDecision::AskUser);
|
||||
// Blacklist patterns are still blocked by default
|
||||
assert_eq!(evaluate_command("rm -rf /", &settings), PolicyDecision::Blocked("rm -rf *".into()));
|
||||
assert_eq!(evaluate_command("rm -rf /tmp/test", &settings), PolicyDecision::Blocked("rm -rf *".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whitelist_allows_safe_single_and_chained_commands_only() {
|
||||
let settings = AgentSecuritySettings {
|
||||
allowed_commands: vec!["git status".into(), "git diff".into(), "cargo test".into()],
|
||||
blocked_commands: vec!["rm -rf *".into()],
|
||||
};
|
||||
assert_eq!(evaluate_command("git status -s", &settings), PolicyDecision::Allowed);
|
||||
assert_eq!(evaluate_command("git diff HEAD~1", &settings), PolicyDecision::Allowed);
|
||||
assert_eq!(evaluate_command("git status && cargo test", &settings), PolicyDecision::Allowed);
|
||||
|
||||
// Word boundary: git statusx must not match git status
|
||||
assert_eq!(evaluate_command("git statusx", &settings), PolicyDecision::AskUser);
|
||||
|
||||
// Unsafe syntax (redirection, command substitution, env assignment) falls back to AskUser
|
||||
assert_eq!(evaluate_command("git status > out.txt", &settings), PolicyDecision::AskUser);
|
||||
assert_eq!(evaluate_command("git status $(whoami)", &settings), PolicyDecision::AskUser);
|
||||
assert_eq!(evaluate_command("FOO=bar git status", &settings), PolicyDecision::AskUser);
|
||||
|
||||
// Chaining an unapproved command must still ask the user.
|
||||
assert_eq!(evaluate_command("git status && curl https://example.com", &settings), PolicyDecision::AskUser);
|
||||
assert_eq!(evaluate_command("cargo publish", &settings), PolicyDecision::AskUser);
|
||||
|
||||
// Blacklist still takes precedence over whitelist
|
||||
assert_eq!(evaluate_command("git status && rm -rf /", &settings), PolicyDecision::Blocked("rm -rf *".into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_command_from_v1_v2_and_codex_payloads() {
|
||||
let claude = json!({
|
||||
"toolCall": {
|
||||
"title": "git status",
|
||||
"kind": "execute",
|
||||
"rawInput": { "command": "git status --short" }
|
||||
}
|
||||
});
|
||||
assert_eq!(extract_command(&claude).as_deref(), Some("git status --short"));
|
||||
|
||||
let codex = json!({
|
||||
"toolCall": {
|
||||
"title": "Run cargo test",
|
||||
"kind": "execute",
|
||||
"rawInput": { "command": ["/bin/zsh", "-lc", "cargo test --workspace"] }
|
||||
}
|
||||
});
|
||||
assert_eq!(extract_command(&codex).as_deref(), Some("cargo test --workspace"));
|
||||
|
||||
let write = json!({
|
||||
"toolCall": {
|
||||
"title": "Write foo.txt",
|
||||
"kind": "edit",
|
||||
"rawInput": { "file_path": "foo.txt" }
|
||||
}
|
||||
});
|
||||
assert_eq!(extract_command(&write), None);
|
||||
}
|
||||
}
|
||||
@@ -43,6 +43,11 @@ struct FakeAgent {
|
||||
/// Hold the late mode reply until the prompt's permission request is already waiting.
|
||||
hold_late_mode: bool,
|
||||
empty_config: AtomicBool,
|
||||
/// Blocks `session/new` until discovery times out (see `acp::discover` test timeout).
|
||||
hold_session_new: AtomicBool,
|
||||
/// Sleeps this long at the start of `session/set_config_option`, after noting the time.
|
||||
delay_config: Duration,
|
||||
config_entered: Mutex<Option<std::time::Instant>>,
|
||||
/// Config options `session/new` reports after the model one (per-model efforts, say).
|
||||
extra_options: Mutex<Vec<Value>>,
|
||||
model_options: Mutex<HashMap<String, Vec<Value>>>,
|
||||
@@ -157,6 +162,9 @@ impl Handler for FakeAgent {
|
||||
)
|
||||
}
|
||||
"session/new" => {
|
||||
if self.hold_session_new.load(Ordering::Acquire) {
|
||||
std::thread::sleep(Duration::from_secs(5));
|
||||
}
|
||||
if self.empty_config.load(Ordering::Acquire) {
|
||||
return Ok(json!({"sessionId": "s1"}));
|
||||
}
|
||||
@@ -190,6 +198,10 @@ impl Handler for FakeAgent {
|
||||
}
|
||||
}
|
||||
"session/set_config_option" | "session/set_mode" => {
|
||||
if method == "session/set_config_option" && !self.delay_config.is_zero() {
|
||||
*self.config_entered.lock() = Some(std::time::Instant::now());
|
||||
std::thread::sleep(self.delay_config);
|
||||
}
|
||||
self.calls.lock().push(format!("{method} {}", params.get("value").or(params.get("modeId")).unwrap()));
|
||||
if params["configId"] == "model" && !self.model_options.lock().is_empty() {
|
||||
return Ok(json!({"configOptions": self.options_for(params["value"].as_str().unwrap())}));
|
||||
@@ -2631,6 +2643,92 @@ async fn session_settings_win_over_discovery_in_memory_and_on_disk() {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn provider_reload_does_not_start_acp_discovery() {
|
||||
let agent = Arc::new(FakeAgent::default());
|
||||
let mgr = manager(tempfile::tempdir().unwrap().path(), agent.clone());
|
||||
mgr.reload_providers();
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
assert!(agent.calls.lock().is_empty(), "reload must not spawn cursor-agent");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn composer_commands_discover_cursor_effort_and_fast() {
|
||||
let agent = Arc::new(FakeAgent {
|
||||
extra_options: Mutex::new(vec![
|
||||
json!({"id": "effort", "type": "select", "currentValue": "medium",
|
||||
"options": [{"value": "low"}, {"value": "medium"}, {"value": "high"}]}),
|
||||
json!({"id": "fast", "type": "select", "currentValue": "false",
|
||||
"options": [{"value": "false", "name": "Off"}, {"value": "true", "name": "On"}]}),
|
||||
]),
|
||||
..Default::default()
|
||||
});
|
||||
let mgr = manager(tempfile::tempdir().unwrap().path(), agent.clone());
|
||||
mgr.commands("cursor", None).await.unwrap();
|
||||
let calls = agent.calls.lock();
|
||||
assert!(calls.iter().any(|m| m.as_str() == "session/new"));
|
||||
assert_eq!(calls.iter().filter(|m| m.as_str() == "initialize").count(), 1);
|
||||
let cursor = mgr.providers().into_iter().find(|p| p.id == "cursor").unwrap();
|
||||
assert_eq!(cursor.config.iter().find(|o| o.id == "fast").unwrap().current.as_deref(), Some("false"));
|
||||
assert!(
|
||||
cursor
|
||||
.thought_levels
|
||||
.values()
|
||||
.next()
|
||||
.is_some_and(|levels| { levels.iter().any(|o| o.category.as_deref() == Some("thought_level")) })
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn config_choice_shows_before_the_agent_answers() {
|
||||
let agent = Arc::new(FakeAgent {
|
||||
extra_options: Mutex::new(vec![json!({
|
||||
"id": "fast", "category": "model_config", "type": "select", "currentValue": "true",
|
||||
"options": [{"value": "true", "name": "On"}, {"value": "false", "name": "Off"}]
|
||||
})]),
|
||||
delay_config: Duration::from_millis(300),
|
||||
..Default::default()
|
||||
});
|
||||
let mgr = manager(tempfile::tempdir().unwrap().path(), agent.clone());
|
||||
let shown: Arc<Mutex<Option<std::time::Instant>>> = Arc::default();
|
||||
let mark = shown.clone();
|
||||
mgr.subscribe(Arc::new(move |e| {
|
||||
if let AgentEvent::Task { task } = e
|
||||
&& task.config.iter().any(|o| o.id == "fast" && o.current.as_deref() == Some("false"))
|
||||
&& mark.lock().is_none()
|
||||
{
|
||||
*mark.lock() = Some(std::time::Instant::now());
|
||||
}
|
||||
}));
|
||||
let task = mgr
|
||||
.create(NewAgentTask {
|
||||
project_root: None,
|
||||
prompt: text("again"),
|
||||
provider: "claude".into(),
|
||||
config: vec![],
|
||||
isolation: Isolation::Local,
|
||||
base: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for(&mgr, &task.id, AgentStatus::Ready).await;
|
||||
mgr.set_config(&task.id, "fast", "false".into()).await.unwrap();
|
||||
let shown = (*shown.lock()).expect("the switch moved");
|
||||
let entered = (*agent.config_entered.lock()).expect("the agent received the change");
|
||||
assert!(shown < entered, "the composer updates before the agent answers");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discovery_times_out_when_session_new_hangs() {
|
||||
let agent = Arc::new(FakeAgent { hold_session_new: AtomicBool::new(true), ..Default::default() });
|
||||
let mgr = manager(tempfile::tempdir().unwrap().path(), agent.clone());
|
||||
let started = std::time::Instant::now();
|
||||
mgr.commands("cursor", None).await.unwrap();
|
||||
assert!(started.elapsed() < Duration::from_secs(10), "discovery should time out quickly in tests");
|
||||
let cursor = mgr.providers().into_iter().find(|p| p.id == "cursor").unwrap();
|
||||
assert!(!cursor.config.iter().any(|o| o.category.as_deref() == Some("model")));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discovered_effort_levels_fill_models_no_session_has_reported() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
@@ -2770,3 +2868,71 @@ async fn rolling_back_restores_the_start_of_the_task_and_keeps_the_users_own_edi
|
||||
mgr.delete(&task.id, false).await.unwrap();
|
||||
assert!(!dir.join("store").join(format!("{}.before", task.id)).exists());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn security_policy_auto_approves_whitelist_and_rejects_blacklist() {
|
||||
use ide_core::settings::AgentSecuritySettings;
|
||||
|
||||
// 1. Whitelist auto-approves "cargo test" without waiting for manual respond().
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let agent = Arc::new(FakeAgent::default());
|
||||
let mgr = manager(tmp.path(), agent);
|
||||
mgr.set_security_hook(Arc::new(|| AgentSecuritySettings {
|
||||
allowed_commands: vec!["cargo test".into()],
|
||||
blocked_commands: vec!["rm -rf *".into()],
|
||||
}));
|
||||
let task = mgr
|
||||
.create(NewAgentTask {
|
||||
project_root: None,
|
||||
prompt: text("run tests"),
|
||||
provider: "claude".into(),
|
||||
config: vec![],
|
||||
isolation: Isolation::Local,
|
||||
base: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for(&mgr, &task.id, AgentStatus::Ready).await;
|
||||
let entries = mgr.entries(&task.id, None).unwrap();
|
||||
assert!(
|
||||
entries.iter().any(|e| matches!(e, AgentEntry::Agent { text } if text.contains("chose yes"))),
|
||||
"{entries:?}"
|
||||
);
|
||||
assert!(
|
||||
entries.iter().any(|e| matches!(
|
||||
e,
|
||||
AgentEntry::Permission { chosen: Some(c), .. } if c.contains("whitelisted")
|
||||
)),
|
||||
"{entries:?}"
|
||||
);
|
||||
|
||||
// 2. Blacklist auto-rejects "cargo test" when matched by blocked_commands.
|
||||
mgr.set_security_hook(Arc::new(|| AgentSecuritySettings {
|
||||
allowed_commands: vec!["cargo test".into()],
|
||||
blocked_commands: vec!["cargo test*".into()],
|
||||
}));
|
||||
let blocked_task = mgr
|
||||
.create(NewAgentTask {
|
||||
project_root: None,
|
||||
prompt: text("run blocked"),
|
||||
provider: "claude".into(),
|
||||
config: vec![],
|
||||
isolation: Isolation::Local,
|
||||
base: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
wait_for(&mgr, &blocked_task.id, AgentStatus::Ready).await;
|
||||
let blocked_entries = mgr.entries(&blocked_task.id, None).unwrap();
|
||||
assert!(
|
||||
blocked_entries.iter().any(|e| matches!(e, AgentEntry::Agent { text } if text.contains("chose no"))),
|
||||
"{blocked_entries:?}"
|
||||
);
|
||||
assert!(
|
||||
blocked_entries.iter().any(|e| matches!(
|
||||
e,
|
||||
AgentEntry::Permission { chosen: Some(c), .. } if c.contains("Blocked by policy")
|
||||
)),
|
||||
"{blocked_entries:?}"
|
||||
);
|
||||
}
|
||||
@@ -14,8 +14,18 @@ pub(crate) struct Created {
|
||||
pub base_commit: String,
|
||||
}
|
||||
|
||||
/// The repository of `path`, free to execute: the host runs every agent command that
|
||||
/// reaches a repository only in a trusted project.
|
||||
fn discover(path: &Path) -> IdeResult<Option<Repo>> {
|
||||
let repo = Repo::discover(path)?;
|
||||
if let Some(repo) = &repo {
|
||||
repo.set_execution_allowed(true);
|
||||
}
|
||||
Ok(repo)
|
||||
}
|
||||
|
||||
fn repo_at(path: &Path) -> IdeResult<Repo> {
|
||||
Repo::discover(path)?.ok_or_else(|| IdeError::unavailable(format!("{} is not in a git repository", path.display())))
|
||||
discover(path)?.ok_or_else(|| IdeError::unavailable(format!("{} is not in a git repository", path.display())))
|
||||
}
|
||||
|
||||
/// The current branch (or commit when detached) and its commit.
|
||||
@@ -91,10 +101,11 @@ fn retrying(attempts: u32, pause: Duration, mut op: impl FnMut() -> std::io::Res
|
||||
/// Files under `cwd` that differ from `base` (untracked files included), with line counts.
|
||||
/// Outside a repository there is nothing to compare against: no changes.
|
||||
pub(crate) fn changes(cwd: &Path, base: &str) -> IdeResult<Vec<AgentChange>> {
|
||||
let Some(repo) = Repo::discover(cwd)? else { return Ok(Vec::new()) };
|
||||
let Some(repo) = discover(cwd)? else { return Ok(Vec::new()) };
|
||||
let root = repo.root().to_path_buf();
|
||||
let stats = repo.numstat_with_worktree(base)?;
|
||||
let stat = |rel: &str| stats.iter().find(|(p, ..)| p == rel).map_or((0, 0), |&(_, a, r)| (a, r));
|
||||
let stats: std::collections::HashMap<String, (u32, u32)> =
|
||||
repo.numstat_with_worktree(base)?.into_iter().map(|(p, a, r)| (p, (a, r))).collect();
|
||||
let stat = |rel: &str| stats.get(rel).copied().unwrap_or((0, 0));
|
||||
let mut out: Vec<AgentChange> = repo
|
||||
.diff_with_worktree(base)?
|
||||
.into_iter()
|
||||
@@ -103,8 +114,9 @@ pub(crate) fn changes(cwd: &Path, base: &str) -> IdeResult<Vec<AgentChange>> {
|
||||
AgentChange { path: IpcPath(root.join(&f.path)), rel_path: f.path, kind: f.kind, added, removed }
|
||||
})
|
||||
.collect();
|
||||
let tracked: std::collections::HashSet<String> = out.iter().map(|c| c.rel_path.clone()).collect();
|
||||
for s in repo.status()? {
|
||||
if s.unstaged == Some(ChangeKind::Untracked) && !out.iter().any(|c| c.rel_path == s.rel_path) {
|
||||
if s.unstaged == Some(ChangeKind::Untracked) && !tracked.contains(&s.rel_path) {
|
||||
let added = std::fs::read_to_string(&*s.path).map_or(0, |t| t.lines().count() as u32);
|
||||
out.push(AgentChange { path: s.path, rel_path: s.rel_path, kind: ChangeKind::Added, added, removed: 0 });
|
||||
}
|
||||
@@ -113,6 +125,50 @@ pub(crate) fn changes(cwd: &Path, base: &str) -> IdeResult<Vec<AgentChange>> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Copies the task's `changes` (against `base`) from the worktree `cwd` into the
|
||||
/// local checkout `root`, uncommitted: the user reviews and commits them with the
|
||||
/// VCS tools. Files the checkout changed itself are left alone and reported, so
|
||||
/// accepting never destroys the user's own edits. Returns accepted and skipped paths.
|
||||
pub(crate) fn accept(
|
||||
root: &Path,
|
||||
cwd: &Path,
|
||||
base: &str,
|
||||
changes: &[AgentChange],
|
||||
) -> IdeResult<crate::AgentAcceptOutcome> {
|
||||
let repo = repo_at(cwd)?;
|
||||
let mut accepted = Vec::with_capacity(changes.len());
|
||||
let mut skipped = Vec::new();
|
||||
for change in changes {
|
||||
let local = root.join(&change.rel_path);
|
||||
let base_bytes = repo.file_bytes_at(&change.path.0, Some(base))?;
|
||||
let current = std::fs::read(&change.path.0).ok();
|
||||
let local_bytes = std::fs::read(&local).ok();
|
||||
if local_bytes == current {
|
||||
accepted.push(change.rel_path.clone());
|
||||
} else if local_bytes == base_bytes {
|
||||
match current {
|
||||
Some(bytes) => {
|
||||
if let Some(parent) = local.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::write(&local, bytes)?;
|
||||
}
|
||||
None => match std::fs::remove_file(&local) {
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => return Err(e.into()),
|
||||
_ => {}
|
||||
},
|
||||
}
|
||||
accepted.push(change.rel_path.clone());
|
||||
} else {
|
||||
skipped.push(crate::AgentAcceptSkipped {
|
||||
path: change.rel_path.clone(),
|
||||
reason: "the checkout has changes of its own to it; accept it by hand".to_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(crate::AgentAcceptOutcome { accepted, skipped })
|
||||
}
|
||||
|
||||
/// Size and modification time of a file, `None` once it is gone.
|
||||
pub(crate) type Stamp = Option<(u64, u128)>;
|
||||
|
||||
@@ -124,7 +180,10 @@ pub(crate) fn stamp(path: &Path) -> Stamp {
|
||||
|
||||
/// The files already changed in `cwd` against `HEAD`, stamped so later edits to them show.
|
||||
pub(crate) fn snapshot(cwd: &Path) -> IdeResult<BTreeMap<String, Stamp>> {
|
||||
Ok(changes(cwd, "HEAD")?.into_iter().map(|c| (c.rel_path, stamp(&c.path))).collect())
|
||||
// Paths and stamps only: no diff and no line counts, which read every untracked file
|
||||
// and held up starting a task in a checkout with many of them.
|
||||
let Some(repo) = Repo::discover(cwd)? else { return Ok(BTreeMap::new()) };
|
||||
Ok(repo.status()?.into_iter().map(|s| (s.rel_path, stamp(&s.path))).collect())
|
||||
}
|
||||
|
||||
/// Drops the changes that were there before the task and haven't been touched since.
|
||||
@@ -204,7 +263,7 @@ pub(crate) fn restore(
|
||||
}
|
||||
|
||||
pub(crate) fn file_at(cwd: &Path, path: &Path, rev: &str) -> IdeResult<Option<String>> {
|
||||
let Some(repo) = Repo::discover(cwd)? else { return Ok(None) };
|
||||
let Some(repo) = discover(cwd)? else { return Ok(None) };
|
||||
let encoding = match ide_vfs::load_text(path) {
|
||||
Ok(loaded) => encoding_rs::Encoding::for_label(loaded.encoding.as_bytes()).unwrap_or(encoding_rs::UTF_8),
|
||||
Err(_) => ide_vfs::detect_encoding(&repo.file_bytes_at(path, Some(rev))?.unwrap_or_default(), None),
|
||||
@@ -391,4 +450,61 @@ mod tests {
|
||||
assert!(remove(&repo, &tmp.path().join("wt"), &elsewhere).is_err());
|
||||
assert!(elsewhere.join("keep.txt").exists());
|
||||
}
|
||||
|
||||
fn accept_repo() -> (tempfile::TempDir, PathBuf, Created) {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let repo = tmp.path().join("repo");
|
||||
std::fs::create_dir(&repo).unwrap();
|
||||
git(&repo, &["init", "-q", "-b", "main"]);
|
||||
std::fs::write(repo.join("a.txt"), "one\ntwo\n").unwrap();
|
||||
std::fs::write(repo.join("gone.txt"), "bye\n").unwrap();
|
||||
git(&repo, &["add", "."]);
|
||||
git(&repo, &["commit", "-q", "-m", "init"]);
|
||||
let created = create(&repo, &tmp.path().join("wt"), "Edit A", None).unwrap();
|
||||
(tmp, repo, created)
|
||||
}
|
||||
|
||||
fn listed(cwd: &Path, base: &str) -> Vec<AgentChange> {
|
||||
without_untouched(changes(cwd, base).unwrap(), &BTreeMap::new())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accept_copies_new_modified_and_deleted_files_into_the_checkout() {
|
||||
let (_tmp, repo, created) = accept_repo();
|
||||
std::fs::write(created.path.join("a.txt"), "one\nTWO\n").unwrap();
|
||||
std::fs::write(created.path.join("new.txt"), "x\n").unwrap();
|
||||
std::fs::remove_file(created.path.join("gone.txt")).unwrap();
|
||||
|
||||
let outcome =
|
||||
accept(&repo, &created.path, &created.base_commit, &listed(&created.path, &created.base_commit)).unwrap();
|
||||
assert!(outcome.skipped.is_empty());
|
||||
assert_eq!(outcome.accepted, ["a.txt", "gone.txt", "new.txt"]);
|
||||
assert_eq!(std::fs::read_to_string(repo.join("a.txt")).unwrap(), "one\nTWO\n");
|
||||
assert_eq!(std::fs::read_to_string(repo.join("new.txt")).unwrap(), "x\n");
|
||||
assert!(!repo.join("gone.txt").exists());
|
||||
|
||||
// Accepting again changes nothing: the checkout already matches.
|
||||
let again =
|
||||
accept(&repo, &created.path, &created.base_commit, &listed(&created.path, &created.base_commit)).unwrap();
|
||||
assert!(again.skipped.is_empty());
|
||||
assert_eq!(again.accepted, ["a.txt", "gone.txt", "new.txt"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accept_leaves_files_the_checkout_changed_itself() {
|
||||
let (_tmp, repo, created) = accept_repo();
|
||||
std::fs::write(created.path.join("a.txt"), "one\nTWO\n").unwrap();
|
||||
std::fs::write(created.path.join("gone.txt"), "bye\nbye\n").unwrap();
|
||||
std::fs::write(repo.join("a.txt"), "one\nMINE\n").unwrap();
|
||||
|
||||
let outcome =
|
||||
accept(&repo, &created.path, &created.base_commit, &listed(&created.path, &created.base_commit)).unwrap();
|
||||
assert_eq!(outcome.accepted, ["gone.txt"]);
|
||||
assert_eq!(outcome.skipped.len(), 1);
|
||||
assert_eq!(outcome.skipped[0].path, "a.txt");
|
||||
assert!(!outcome.skipped[0].reason.is_empty());
|
||||
// The user's own edit survives; the other file comes over.
|
||||
assert_eq!(std::fs::read_to_string(repo.join("a.txt")).unwrap(), "one\nMINE\n");
|
||||
assert_eq!(std::fs::read_to_string(repo.join("gone.txt")).unwrap(), "bye\nbye\n");
|
||||
}
|
||||
}
|
||||
@@ -265,6 +265,14 @@ pub fn menu_fingerprint(
|
||||
}
|
||||
|
||||
impl LaunchFingerprint {
|
||||
/// Folders (relative to the project) of the Maven projects whose POM is an input.
|
||||
pub(super) fn project_dirs(&self) -> impl Iterator<Item = &Path> {
|
||||
self.files
|
||||
.keys()
|
||||
.filter(|p| p.is_relative() && p.file_name().is_some_and(|n| n == "pom.xml"))
|
||||
.map(|pom| pom.parent().unwrap_or(Path::new("")))
|
||||
}
|
||||
|
||||
/// Relative paths whose content changed while `(length, mtime)` stayed the same
|
||||
/// compared to `previous` (timestamp-preserving external edits, coarse-granularity
|
||||
/// filesystems). Timestamp-incremental builders (Maven, the Kotlin plugin) would
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
//! Resources a Gradle build changes while copying them (filters, `expand`, renames, extra
|
||||
//! sources). The Java language server copies them as they are into Buildship's
|
||||
//! `bin/<source set>` folders, which launches run from; IntelliJ IDEA runs Gradle's copy.
|
||||
//! Before a launch, Gradle processes them and its output replaces those plain copies, so
|
||||
//! each resource is on the classpath once, as the build produces it.
|
||||
use super::{BuildPlan, BuildStep, BuildSystem, Format, run, subdir};
|
||||
use crate::BuildModel;
|
||||
use crate::gradle::{self, GradleModel, ReportedProject};
|
||||
use ide_core::{CancellationToken, IdeError, IdeResult, Severity};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// A source set whose resources Gradle processes, with the folder the launch reads them from.
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct Target {
|
||||
/// Task path from the root build (`:app:processResources`).
|
||||
task: String,
|
||||
/// Where the task writes the resources.
|
||||
output: PathBuf,
|
||||
/// The Java language server's output folder of the source set, on the launch classpath.
|
||||
jdtls: PathBuf,
|
||||
}
|
||||
|
||||
fn targets(reported: &[ReportedProject], class_paths: &[String]) -> Vec<Target> {
|
||||
// buildSrc is built before the build, never addressable as `:buildSrc:<task>`.
|
||||
let addressable = |p: &&ReportedProject| !p.build.as_deref().is_some_and(|b| b.ends_with(":buildSrc"));
|
||||
let sets: Vec<_> = reported
|
||||
.iter()
|
||||
.filter(addressable)
|
||||
.flat_map(|p| p.source_sets.iter().filter_map(move |s| Some((p, s, s.processed.as_ref()?))))
|
||||
.collect();
|
||||
if sets.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
// Canonical, as Gradle reports its folders: jdtls may spell them differently.
|
||||
let entries: Vec<PathBuf> = class_paths.iter().filter_map(|p| dunce::canonicalize(p).ok()).collect();
|
||||
sets.into_iter()
|
||||
.filter_map(|(p, set, resources)| {
|
||||
// Buildship's output folder of the source set.
|
||||
let jdtls = dunce::canonicalize(Path::new(&p.dir).join("bin").join(&set.name)).ok()?;
|
||||
let project = if p.path == ":" { "" } else { p.path.as_str() };
|
||||
entries.contains(&jdtls).then(|| Target {
|
||||
task: format!("{project}:{}", resources.task),
|
||||
output: PathBuf::from(&resources.dir),
|
||||
jdtls,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Before a JVM launch from the Java language server's output folders on `class_paths`:
|
||||
/// runs the Gradle tasks that process resources of those folders' source sets (see the
|
||||
/// module docs) and copies their output over the plain copies. `cache_dir` holds the
|
||||
/// linked builds' last syncs. Returns the tasks run, none for builds that copy their
|
||||
/// resources as they are.
|
||||
pub fn process_launch_resources(
|
||||
root: &Path,
|
||||
model: &BuildModel,
|
||||
cache_dir: &Path,
|
||||
class_paths: &[String],
|
||||
cancel: &CancellationToken,
|
||||
) -> IdeResult<Vec<String>> {
|
||||
// A composite also reports the builds it includes: the deepest build owning a folder runs its task.
|
||||
let mut builds: Vec<&GradleModel> = model.gradle.iter().collect();
|
||||
builds.sort_by_key(|g| std::cmp::Reverse(subdir(root, &g.dir).components().count()));
|
||||
let mut done: Vec<PathBuf> = Vec::new();
|
||||
let mut tasks = Vec::new();
|
||||
for g in builds {
|
||||
let reported = crate::sync::reported_gradle_projects(&gradle::cache_dir(cache_dir, &g.dir));
|
||||
let targets: Vec<Target> =
|
||||
targets(&reported, class_paths).into_iter().filter(|t| !done.contains(&t.jdtls)).collect();
|
||||
if targets.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let mut step = BuildStep::new(&g.command, &["--console=plain"], &subdir(root, &g.dir), Format::Gradle);
|
||||
step.args.extend(targets.iter().map(|t| t.task.clone()));
|
||||
step.env = g.env.clone();
|
||||
let plan = BuildPlan { system: BuildSystem::Gradle, title: "Process resources".into(), steps: vec![step] };
|
||||
let result = run(root, &plan, cancel, &|_| {})?;
|
||||
if !result.success {
|
||||
let errors: Vec<&str> = result
|
||||
.messages
|
||||
.iter()
|
||||
.filter(|m| m.severity == Severity::Error)
|
||||
.take(5)
|
||||
.map(|m| m.text.as_str())
|
||||
.collect();
|
||||
return Err(IdeError::internal(errors.join("\n")));
|
||||
}
|
||||
for t in targets {
|
||||
if cancel.is_cancelled() {
|
||||
return Err(IdeError::cancelled());
|
||||
}
|
||||
overlay(&t.output, &t.output, &t.jdtls)?;
|
||||
done.push(t.jdtls);
|
||||
tasks.push(t.task);
|
||||
}
|
||||
}
|
||||
Ok(tasks)
|
||||
}
|
||||
|
||||
/// Copies the files under `dir` to the same place under `to` (relative to `base`) where
|
||||
/// they differ, so unchanged resources are not rewritten on every launch.
|
||||
fn overlay(base: &Path, dir: &Path, to: &Path) -> IdeResult<()> {
|
||||
let entries = match fs::read_dir(dir) {
|
||||
// No resources: the task had nothing to copy.
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
entries => entries?,
|
||||
};
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
let kind = entry.file_type()?;
|
||||
if kind.is_dir() {
|
||||
overlay(base, &path, to)?;
|
||||
} else if kind.is_file() {
|
||||
let dest = to.join(path.strip_prefix(base).map_err(|e| IdeError::internal(e.to_string()))?);
|
||||
if !same_content(&path, &dest) {
|
||||
if let Some(parent) = dest.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
fs::copy(&path, &dest)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn same_content(a: &Path, b: &Path) -> bool {
|
||||
match (fs::metadata(a), fs::metadata(b)) {
|
||||
(Ok(m), Ok(n)) if m.len() == n.len() => matches!((fs::read(a), fs::read(b)), (Ok(x), Ok(y)) if x == y),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::gradle::{ProcessedResources, ReportedSourceSet};
|
||||
|
||||
fn write(path: &Path, text: &str) {
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
fn set(name: &str, processed: Option<&Path>) -> ReportedSourceSet {
|
||||
ReportedSourceSet {
|
||||
name: name.into(),
|
||||
test: name == "test",
|
||||
processed: processed.map(|dir| ProcessedResources {
|
||||
task: if name == "main" { "processResources".into() } else { "processTestResources".into() },
|
||||
dir: dir.to_string_lossy().into_owned(),
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn project(path: &str, dir: &Path, sets: Vec<ReportedSourceSet>) -> ReportedProject {
|
||||
ReportedProject {
|
||||
path: path.into(),
|
||||
dir: dir.to_string_lossy().into_owned(),
|
||||
source_sets: sets,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// Only processed source sets whose jdtls output is on the classpath: the launched
|
||||
/// module, upstream modules of the build, tests only for a test classpath; never buildSrc.
|
||||
#[test]
|
||||
fn targets_are_processed_sets_on_the_classpath() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(tmp.path()).unwrap();
|
||||
let (app, lib, plain, src) = (root.join("app"), root.join("lib"), root.join("plain"), root.join("buildSrc"));
|
||||
for dir in [&root, &app, &lib, &plain, &src] {
|
||||
for set in ["main", "test"] {
|
||||
fs::create_dir_all(dir.join("bin").join(set)).unwrap();
|
||||
}
|
||||
}
|
||||
let out = |dir: &Path, set: &str| dir.join("build/resources").join(set);
|
||||
let mut build_src = project(":buildSrc", &src, vec![set("main", Some(&out(&src, "main")))]);
|
||||
build_src.build = Some(":buildSrc".into());
|
||||
let reported = [
|
||||
project(":", &root, vec![set("main", Some(&out(&root, "main")))]),
|
||||
project(":app", &app, vec![set("main", Some(&out(&app, "main"))), set("test", Some(&out(&app, "test")))]),
|
||||
project(":lib", &lib, vec![set("main", Some(&out(&lib, "main")))]),
|
||||
project(":plain", &plain, vec![set("main", None)]),
|
||||
build_src,
|
||||
];
|
||||
let entry = |dir: &Path, set: &str| dir.join("bin").join(set).to_string_lossy().into_owned();
|
||||
let runtime = [
|
||||
entry(&app, "main"),
|
||||
// Spelled differently from Gradle's canonical folders.
|
||||
format!("{}/", entry(&lib, "main")),
|
||||
entry(&plain, "main"),
|
||||
entry(&src, "main"),
|
||||
"/m2/spring-core.jar".into(),
|
||||
];
|
||||
let tasks = |cp: &[String]| targets(&reported, cp).into_iter().map(|t| t.task).collect::<Vec<_>>();
|
||||
assert_eq!(tasks(&runtime), [":app:processResources", ":lib:processResources"]);
|
||||
let mut test = runtime.to_vec();
|
||||
test.insert(0, entry(&app, "test"));
|
||||
assert_eq!(tasks(&test), [":app:processResources", ":app:processTestResources", ":lib:processResources"]);
|
||||
assert_eq!(tasks(&[entry(&root, "main")]), [":processResources"], "the root project");
|
||||
let app_targets = targets(&reported, &runtime);
|
||||
let t = &app_targets[0];
|
||||
assert_eq!(
|
||||
(t.output.as_path(), t.jdtls.as_path()),
|
||||
(out(&app, "main").as_path(), app.join("bin/main").as_path())
|
||||
);
|
||||
|
||||
let unprocessed = [project(":app", &app, vec![set("main", None)])];
|
||||
assert!(targets(&unprocessed, &runtime).is_empty());
|
||||
}
|
||||
|
||||
/// The processed copy replaces jdtls's plain one (the reported `@version@` case), new
|
||||
/// files are added, identical ones are left alone and files Gradle doesn't produce stay.
|
||||
#[test]
|
||||
fn overlay_replaces_plain_copies_with_gradle_output() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let (output, jdtls) = (tmp.path().join("build/resources/main"), tmp.path().join("bin/main"));
|
||||
write(&output.join("application.yaml"), "spring:\n application:\n version: 0.0.1\n");
|
||||
write(&output.join("static/extra.txt"), "from an extra source");
|
||||
write(&output.join("migrations/V1__init.sql"), "create table t (id int);");
|
||||
write(&jdtls.join("application.yaml"), "spring:\n application:\n version: @version@\n");
|
||||
write(&jdtls.join("migrations/V1__init.sql"), "create table t (id int);");
|
||||
write(&jdtls.join("com/example/App.class"), "bytecode");
|
||||
let unchanged = fs::metadata(jdtls.join("migrations/V1__init.sql")).unwrap().modified().unwrap();
|
||||
std::thread::sleep(std::time::Duration::from_millis(20));
|
||||
|
||||
overlay(&output, &output, &jdtls).unwrap();
|
||||
let read = |rel: &str| fs::read_to_string(jdtls.join(rel)).unwrap();
|
||||
assert_eq!(read("application.yaml"), "spring:\n application:\n version: 0.0.1\n");
|
||||
assert_eq!(read("static/extra.txt"), "from an extra source");
|
||||
assert_eq!(read("com/example/App.class"), "bytecode");
|
||||
let modified = fs::metadata(jdtls.join("migrations/V1__init.sql")).unwrap().modified().unwrap();
|
||||
assert_eq!(modified, unchanged, "identical files are not rewritten");
|
||||
|
||||
overlay(&tmp.path().join("missing"), &tmp.path().join("missing"), &jdtls).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn builds_without_processed_resources_run_nothing() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
// Running this command would fail the call.
|
||||
let missing = tmp.path().join("missing-gradle").to_string_lossy().into_owned();
|
||||
let model =
|
||||
BuildModel { gradle: vec![GradleModel { command: missing, ..Default::default() }], ..Default::default() };
|
||||
let class_paths = [tmp.path().join("bin/main").to_string_lossy().into_owned()];
|
||||
let tasks =
|
||||
process_launch_resources(tmp.path(), &model, tmp.path(), &class_paths, &CancellationToken::new()).unwrap();
|
||||
assert!(tasks.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -4,11 +4,13 @@
|
||||
//! for the Build tool window, the Problems view and the editor.
|
||||
|
||||
mod fingerprint;
|
||||
mod launch_resources;
|
||||
mod parse;
|
||||
mod single;
|
||||
mod up_to_date;
|
||||
|
||||
pub use fingerprint::{LaunchFingerprint, launch_fingerprint, menu_fingerprint};
|
||||
pub use launch_resources::process_launch_resources;
|
||||
pub use single::{
|
||||
SingleFileTarget, compile_file_plan, find_javac, javac_step, single_file_classpath, single_file_target,
|
||||
};
|
||||
@@ -264,6 +266,11 @@ pub(super) fn maven_steps(
|
||||
BuildKind::Rebuild => &["clean", "test-compile"],
|
||||
BuildKind::Artifacts => &["package", "-DskipTests"],
|
||||
};
|
||||
// The IDE's compile builds are not coverage runs. JaCoCo's offline `instrument`
|
||||
// rewrites target/classes in `process-classes`; `jacoco.skip` prevents that in this
|
||||
// build. It does not restore classes instrumented by an earlier build: Rebuild's
|
||||
// `clean` removes those, while an incremental Build can still reuse them.
|
||||
let skip_jacoco = matches!(kind, BuildKind::Build | BuildKind::Rebuild);
|
||||
let tops: Vec<&maven::MavenProject> = m.projects.iter().filter(|p| !p.aggregated).collect();
|
||||
if tops.is_empty() {
|
||||
return Err(IdeError::not_found("no Maven project found"));
|
||||
@@ -276,6 +283,9 @@ pub(super) fn maven_steps(
|
||||
Format::Maven,
|
||||
);
|
||||
s.args.extend(p.options.iter().cloned());
|
||||
if skip_jacoco {
|
||||
s.args.push("-Djacoco.skip=true".into());
|
||||
}
|
||||
s.args.extend(extra.iter().cloned());
|
||||
s.args.extend(goals.iter().map(|g| (*g).to_owned()));
|
||||
s.env = p.env.clone();
|
||||
@@ -1039,6 +1049,22 @@ mod tests {
|
||||
std::fs::write(p, text).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn maven_compile_steps_skip_jacoco_but_artifacts_do_not() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let root = tmp.path().join("demo");
|
||||
write(&root, "pom.xml", "<project><artifactId>demo</artifactId></project>");
|
||||
let model = crate::load(&root, &tmp.path().join("cache"), None);
|
||||
for (kind, expected) in [(BuildKind::Build, true), (BuildKind::Rebuild, true), (BuildKind::Artifacts, false)] {
|
||||
let plan = plan(&root, &model, kind, None, None).unwrap();
|
||||
assert_eq!(
|
||||
plan.steps[0].args.iter().any(|arg| arg == "-Djacoco.skip=true"),
|
||||
expected,
|
||||
"{kind:?}: {plan:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plans_per_build_system() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
@@ -1055,7 +1081,10 @@ mod tests {
|
||||
assert_eq!(p.system, BuildSystem::Maven);
|
||||
assert_eq!(p.title, "demo");
|
||||
assert_eq!(p.steps.len(), 1);
|
||||
assert_eq!(p.steps[0].args, ["-B", "-Dstyle.color=never", maven::LOCAL_REPO_TRACKING_ARG, "test-compile"]);
|
||||
assert_eq!(
|
||||
p.steps[0].args,
|
||||
["-B", "-Dstyle.color=never", maven::LOCAL_REPO_TRACKING_ARG, "-Djacoco.skip=true", "test-compile"]
|
||||
);
|
||||
let p = plan(&root, &model, BuildKind::Rebuild, Some(&root.join("core/src/main/java/A.java")), None).unwrap();
|
||||
assert_eq!(
|
||||
p.steps[0].args,
|
||||
@@ -1063,6 +1092,7 @@ mod tests {
|
||||
"-B",
|
||||
"-Dstyle.color=never",
|
||||
maven::LOCAL_REPO_TRACKING_ARG,
|
||||
"-Djacoco.skip=true",
|
||||
"-pl",
|
||||
"core",
|
||||
"-am",
|
||||
@@ -1351,6 +1381,7 @@ mod tests {
|
||||
maven::LOCAL_REPO_TRACKING_ARG,
|
||||
"-s",
|
||||
"settings.xml",
|
||||
"-Djacoco.skip=true",
|
||||
"-pl",
|
||||
"app",
|
||||
"-am",
|
||||
@@ -1367,6 +1398,7 @@ mod tests {
|
||||
maven::LOCAL_REPO_TRACKING_ARG,
|
||||
"-s",
|
||||
"settings.xml",
|
||||
"-Djacoco.skip=true",
|
||||
"-pl",
|
||||
"app",
|
||||
"-am",
|
||||
|
||||
@@ -79,6 +79,14 @@ pub fn find_javac(env: &BTreeMap<String, String>) -> String {
|
||||
/// so `-am` stays cheap). Entries are platform-separated (`;` on Windows).
|
||||
pub fn single_file_classpath(root: &Path, model: &BuildModel, target: &SingleFileTarget) -> IdeResult<String> {
|
||||
let mut entries: Vec<String> = Vec::new();
|
||||
// The module's own classes first: the file almost always uses classes next to it.
|
||||
let own: &[&str] = if target.test { &["target/test-classes", "target/classes"] } else { &["target/classes"] };
|
||||
for out in own {
|
||||
let path = target.module.join(out);
|
||||
if path.is_dir() {
|
||||
entries.push(path.to_string_lossy().into_owned());
|
||||
}
|
||||
}
|
||||
if let Some(m) = &model.maven {
|
||||
// Upstream reactor modules first: their freshly built classes shadow
|
||||
// installed snapshots of the same artifacts.
|
||||
@@ -139,6 +147,16 @@ pub fn javac_step(target: &SingleFileTarget, file: &Path, classpath: &str, javac
|
||||
step.args.push(target.out_dir.to_string_lossy().into_owned());
|
||||
step.args.push("-cp".into());
|
||||
step.args.push(classpath.to_owned());
|
||||
// Classes of the module that were never built (or are older than their source) are read
|
||||
// from its sources; only `file` is written, so a test compile never puts main classes
|
||||
// into test-classes.
|
||||
let sources = source_roots(target);
|
||||
if !sources.is_empty() {
|
||||
let separator = if cfg!(windows) { ";" } else { ":" };
|
||||
step.args.push("-sourcepath".into());
|
||||
step.args.push(sources.iter().map(|p| p.to_string_lossy()).collect::<Vec<_>>().join(separator));
|
||||
step.args.push("-implicit:none".into());
|
||||
}
|
||||
if let Some(version) = target.java_version {
|
||||
step.args.push("--release".into());
|
||||
step.args.push(version.to_string());
|
||||
@@ -148,6 +166,27 @@ pub fn javac_step(target: &SingleFileTarget, file: &Path, classpath: &str, javac
|
||||
Ok(step)
|
||||
}
|
||||
|
||||
/// The module's Java source folders that exist, generated ones included; test sources see
|
||||
/// the main ones too.
|
||||
fn source_roots(target: &SingleFileTarget) -> Vec<PathBuf> {
|
||||
let scopes: &[(&str, &str)] = if target.test {
|
||||
&[("src/test/java", "target/generated-test-sources"), ("src/main/java", "target/generated-sources")]
|
||||
} else {
|
||||
&[("src/main/java", "target/generated-sources")]
|
||||
};
|
||||
let mut roots = Vec::new();
|
||||
for (src, generated) in scopes {
|
||||
roots.push(target.module.join(src));
|
||||
let generated = target.module.join(generated);
|
||||
let mut dirs: Vec<PathBuf> =
|
||||
fs::read_dir(&generated).into_iter().flatten().flatten().map(|d| d.path()).filter(|p| p.is_dir()).collect();
|
||||
dirs.sort();
|
||||
roots.extend(dirs);
|
||||
}
|
||||
roots.retain(|p| p.is_dir());
|
||||
roots
|
||||
}
|
||||
|
||||
/// Resolves the classpath and plans the `javac` invocation for one Java source.
|
||||
pub fn compile_file_plan(root: &Path, model: &BuildModel, file: &Path) -> IdeResult<BuildPlan> {
|
||||
let target = single_file_target(root, model, file)?;
|
||||
@@ -243,6 +282,11 @@ mod tests {
|
||||
assert!(args.contains("-d ") && args.contains("test-classes"), "{args}");
|
||||
assert!(args.contains("-cp /repo/a.jar"), "{args}");
|
||||
assert!(args.contains("MainTest.java"), "{args}");
|
||||
// Test sources resolve their own and the main sources, without writing them.
|
||||
let sourcepath = step.args.iter().position(|a| a == "-sourcepath").map(|i| step.args[i + 1].clone());
|
||||
let roots = sourcepath.expect("a -sourcepath");
|
||||
assert!(roots.contains("src/test/java") && roots.contains("src/main/java"), "{roots}");
|
||||
assert!(step.args.iter().any(|a| a == "-implicit:none"), "{args}");
|
||||
// No `JAVA_HOME`: plain `javac` from `PATH`.
|
||||
assert_eq!(find_javac(&BTreeMap::new()), exe);
|
||||
}
|
||||
@@ -275,6 +319,11 @@ mod tests {
|
||||
// Reactor class directories first, so fresh outputs shadow installed snapshots.
|
||||
let lib = root.join("lib/target/classes").to_string_lossy().into_owned();
|
||||
assert!(cp.starts_with(&lib), "{cp}");
|
||||
// The module's own classes come first once they exist.
|
||||
fs::create_dir_all(root.join("app/target/classes")).unwrap();
|
||||
let cp = single_file_classpath(root, &model, &target).unwrap();
|
||||
let own = root.join("app/target/classes").to_string_lossy().into_owned();
|
||||
assert!(cp.starts_with(&format!("{own}:{lib}")), "{cp}");
|
||||
assert!(cp.contains("/repo/a.jar"), "{cp}");
|
||||
assert!(cp.contains("/repo/b.jar"), "{cp}");
|
||||
assert!(cp.split(separator).count() >= 3, "{cp}");
|
||||
|
||||
@@ -14,6 +14,20 @@ use std::fs;
|
||||
#[derive(serde::Serialize, serde::Deserialize)]
|
||||
struct Record {
|
||||
fingerprint: LaunchFingerprint,
|
||||
/// Output folders of the built projects that existed after the build, relative to the
|
||||
/// project. Deleting one outside the IDE (`mvn clean` in a terminal) leaves every input
|
||||
/// unchanged, so a build is skipped only while they are all still there.
|
||||
#[serde(default)]
|
||||
outputs: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
/// The build's output folders that exist now, relative to `root`.
|
||||
fn existing_outputs(root: &Path, fingerprint: &LaunchFingerprint) -> Vec<PathBuf> {
|
||||
fingerprint
|
||||
.project_dirs()
|
||||
.flat_map(|dir| ["target/classes", "target/test-classes"].map(|out| dir.join(out)))
|
||||
.filter(|rel| root.join(rel).is_dir())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn record_path(cache: &Path, root: &Path, file: Option<&Path>) -> PathBuf {
|
||||
@@ -69,7 +83,7 @@ pub fn menu_up_to_date(
|
||||
restamp(root, &record.fingerprint.mtime_preserved_edits(¤t));
|
||||
return false;
|
||||
}
|
||||
true
|
||||
record.outputs.iter().all(|out| root.join(out).is_dir())
|
||||
}
|
||||
|
||||
/// Records a successful menu build as the baseline for future up-to-date
|
||||
@@ -91,7 +105,8 @@ pub fn record_menu_build(
|
||||
if path.parent().is_some_and(|p| fs::create_dir_all(p).is_err()) {
|
||||
return;
|
||||
}
|
||||
let bytes = match serde_json::to_vec(&Record { fingerprint: current }) {
|
||||
let outputs = existing_outputs(root, ¤t);
|
||||
let bytes = match serde_json::to_vec(&Record { fingerprint: current, outputs }) {
|
||||
Ok(bytes) => bytes,
|
||||
_ => return,
|
||||
};
|
||||
@@ -151,6 +166,25 @@ mod tests {
|
||||
assert!(menu_up_to_date(&cache, root, &model, None, &cancel));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deleted_outputs_invalidate_the_record() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dir.path();
|
||||
let model = fixture(root);
|
||||
let cache = root.join("cache");
|
||||
let cancel = CancellationToken::new();
|
||||
for out in ["app/target/classes/t", "lib/target/classes/t", "app/target/test-classes"] {
|
||||
fs::create_dir_all(root.join(out)).unwrap();
|
||||
}
|
||||
record_menu_build(&cache, root, &model, None, &cancel);
|
||||
assert!(menu_up_to_date(&cache, root, &model, None, &cancel));
|
||||
// `mvn clean` outside the IDE: no input changed, but the classes are gone.
|
||||
fs::remove_dir_all(root.join("lib/target")).unwrap();
|
||||
assert!(!menu_up_to_date(&cache, root, &model, None, &cancel));
|
||||
fs::create_dir_all(root.join("lib/target/classes")).unwrap();
|
||||
assert!(menu_up_to_date(&cache, root, &model, None, &cancel));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edit_invalidates_and_rerecord_restores_skip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -8,6 +8,24 @@ def canon = { File f -> f.canonicalPath }
|
||||
// the identity path (`:lib:core`, `:buildSrc`) tells them apart and addresses their tasks
|
||||
// from the root build. Internal API, hence the fallback.
|
||||
def identity = { Project p -> try { p.identityPath.toString() } catch (ignored) { p.path } }
|
||||
// The task copying a source set's resources and its output folder, when it changes them on the
|
||||
// way (filters, `expand`, renames, includes / excludes, extra sources): jdtls copies resources as
|
||||
// they are, so launches take them from Gradle. Internal API, hence the fallback.
|
||||
def processedResources = { Project p, s ->
|
||||
try {
|
||||
def task = p.tasks.findByName(s.processResourcesTaskName)
|
||||
if (task == null) return null
|
||||
def specs = 0
|
||||
def changes = task.mainSpec.sourcePaths.size() > 1
|
||||
task.rootSpec.buildRootResolver().walk { r ->
|
||||
specs++
|
||||
changes |= !r.allCopyActions.isEmpty() || !r.allIncludes.isEmpty() || !r.allExcludes.isEmpty() ||
|
||||
!r.allIncludeSpecs.isEmpty() || !r.allExcludeSpecs.isEmpty() || r.destPath.segments.length > 0
|
||||
}
|
||||
// The root and main specs exist for every task; more come from `from(…) { … }`.
|
||||
(changes || specs > 2) ? [task: task.name, dir: canon(task.destinationDir)] : null
|
||||
} catch (ignored) { null }
|
||||
}
|
||||
|
||||
def describe = { Project p ->
|
||||
def m = [path: identity(p), dir: canon(p.projectDir), buildDir: canon(p.layout.buildDirectory.get().asFile)]
|
||||
@@ -26,7 +44,10 @@ def describe = { Project p ->
|
||||
def test = s.name == 'test' || s.name == 'testFixtures' || suites.contains(s.name) ||
|
||||
s.name.endsWith('Test') || s.name.endsWith('Tests') ||
|
||||
(!src.isEmpty() && src.every { ideaTests.contains(it) })
|
||||
[name: s.name, test: test, src: src, res: res]
|
||||
def set = [name: s.name, test: test, src: src, res: res]
|
||||
def processed = processedResources(p, s)
|
||||
if (processed != null) set.processed = processed
|
||||
set
|
||||
}
|
||||
}
|
||||
try {
|
||||
|
||||
@@ -335,6 +335,19 @@ pub struct ReportedSourceSet {
|
||||
/// Absolute resource folders.
|
||||
#[serde(default)]
|
||||
pub res: Vec<String>,
|
||||
/// Set when Gradle changes the resources while copying them (filters, `expand`, renames,
|
||||
/// includes / excludes, extra sources), which the Java language server's copy does not.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub processed: Option<ProcessedResources>,
|
||||
}
|
||||
|
||||
/// The task that copies a source set's resources, and where it writes them.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ProcessedResources {
|
||||
/// Task name in its project (`processResources`).
|
||||
pub task: String,
|
||||
/// Absolute output folder.
|
||||
pub dir: String,
|
||||
}
|
||||
|
||||
const MODEL_LINE: &str = "@openidea-model\t";
|
||||
|
||||
@@ -536,6 +536,7 @@ mod tests {
|
||||
crate::maven::LOCAL_REPO_TRACKING_ARG,
|
||||
"-o",
|
||||
"-Peditor",
|
||||
"-Djacoco.skip=true",
|
||||
"test-compile"
|
||||
]
|
||||
);
|
||||
|
||||
@@ -73,6 +73,9 @@ struct GradleCache {
|
||||
/// The projects as Gradle evaluated them; empty in caches from before they were read.
|
||||
#[serde(default)]
|
||||
projects: Vec<ReportedProject>,
|
||||
/// The projects report how their resources are processed; older caches are stale.
|
||||
#[serde(default)]
|
||||
resources: bool,
|
||||
}
|
||||
|
||||
/// Init script printing every project, its source sets and Java version once configured.
|
||||
@@ -244,11 +247,12 @@ pub fn apply_gradle_cache(root: &Path, cache_dir: &Path, model: &mut GradleModel
|
||||
}
|
||||
}
|
||||
}
|
||||
// A successful sync from before projects were cached has not seen computed includes.
|
||||
let without_projects = cache.error.is_none() && cache.projects.is_empty();
|
||||
// A successful sync from before projects were cached has not seen computed includes,
|
||||
// nor, before resources were read, which launches need Gradle to process them.
|
||||
let outdated = cache.error.is_none() && (cache.projects.is_empty() || !cache.resources);
|
||||
model.synced_at = Some(cache.at_ms);
|
||||
model.sync_error = cache.error;
|
||||
model.stale = without_projects || cache.hash != hash_gradle_files(&gradle::build_files(root, model));
|
||||
model.stale = outdated || cache.hash != hash_gradle_files(&gradle::build_files(root, model));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- processes
|
||||
@@ -462,6 +466,7 @@ pub fn sync_gradle_model(
|
||||
error: (!out.success).then(|| error_summary(&out.text)),
|
||||
tasks: if out.success { gradle::parse_tasks(&out.text) } else { Vec::new() },
|
||||
projects,
|
||||
resources: true,
|
||||
};
|
||||
write_cache(&cache_dir, GRADLE_CACHE, &cache)
|
||||
}
|
||||
@@ -713,6 +718,7 @@ mod tests {
|
||||
test,
|
||||
src: src.iter().map(|p| core_src(p)).collect(),
|
||||
res: res.iter().map(|p| core_src(p)).collect(),
|
||||
processed: None,
|
||||
};
|
||||
// A settings script's own println comes first; task listing lines follow.
|
||||
let out = format!(
|
||||
@@ -737,6 +743,15 @@ mod tests {
|
||||
);
|
||||
let projects = parse_reported_projects(&out);
|
||||
assert_eq!(projects.len(), 3, "the malformed line is skipped");
|
||||
// As the init script prints a set whose resources Gradle filters or renames.
|
||||
let processed = parse_reported_projects(&format!(
|
||||
"@openidea-model\t{}",
|
||||
r#"{"path":":","dir":"/p","sourceSets":[{"name":"main","test":false,"src":[],"res":[],"processed":{"task":"processResources","dir":"/p/build/resources/main"}}]}"#
|
||||
));
|
||||
assert_eq!(
|
||||
processed[0].source_sets[0].processed,
|
||||
Some(gradle::ProcessedResources { task: "processResources".into(), dir: "/p/build/resources/main".into() })
|
||||
);
|
||||
let src = |p: &str| PathBuf::from(core_src(p));
|
||||
assert_eq!(
|
||||
gradle::reported_source_roots(&projects),
|
||||
@@ -802,12 +817,18 @@ mod tests {
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
resources: true,
|
||||
};
|
||||
write_cache(&cache_dir, GRADLE_CACHE, &cache).unwrap();
|
||||
apply_gradle_cache(&root, &cache_dir, &mut model);
|
||||
assert_eq!(model.projects[1].tasks.len(), 1);
|
||||
assert!(model.projects[0].tasks.iter().any(|t| t.name == "wrapper"), "root keeps its defaults");
|
||||
assert!(!model.stale);
|
||||
write_cache(&cache_dir, GRADLE_CACHE, &GradleCache { resources: false, ..cache.clone() }).unwrap();
|
||||
let mut m = gradle::load(&root).unwrap();
|
||||
apply_gradle_cache(&root, &cache_dir, &mut m);
|
||||
assert!(m.stale, "synced before resource processing was read");
|
||||
write_cache(&cache_dir, GRADLE_CACHE, &cache).unwrap();
|
||||
|
||||
let stale_after = |script: &str| {
|
||||
std::fs::write(root.join("app/build.gradle"), script).unwrap();
|
||||
@@ -841,6 +862,7 @@ mod tests {
|
||||
dir: root.to_string_lossy().into_owned(),
|
||||
..Default::default()
|
||||
}],
|
||||
resources: true,
|
||||
..Default::default()
|
||||
};
|
||||
write_cache(&cache_dir, GRADLE_CACHE, &cache).unwrap();
|
||||
|
||||
@@ -333,6 +333,23 @@ pub struct InlayHintsSettings {
|
||||
pub code_vision_usages: bool,
|
||||
/// Code vision: last author of a declaration (VCS blame) beside its usages.
|
||||
pub code_vision_author: bool,
|
||||
/// Code vision: "no usages" on declarations nothing uses (IDEA shows it; clicking searches everywhere).
|
||||
pub code_vision_no_usages: bool,
|
||||
/// Code vision: where the usages go, above the declaration or at the end of its line.
|
||||
pub code_vision_usages_position: CodeVisionPosition,
|
||||
/// Code vision: where the author goes, above the declaration or at the end of its line.
|
||||
pub code_vision_author_position: CodeVisionPosition,
|
||||
}
|
||||
|
||||
/// Editor › Inlay Hints › Code Vision position (IDEA's "Default position for metrics").
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum CodeVisionPosition {
|
||||
/// A row of its own above the declaration.
|
||||
#[default]
|
||||
Top,
|
||||
/// After the end of the declaration's first line.
|
||||
Right,
|
||||
}
|
||||
|
||||
impl Default for InlayHintsSettings {
|
||||
@@ -344,6 +361,9 @@ impl Default for InlayHintsSettings {
|
||||
disabled_languages: Vec::new(),
|
||||
code_vision_usages: true,
|
||||
code_vision_author: true,
|
||||
code_vision_no_usages: true,
|
||||
code_vision_usages_position: CodeVisionPosition::Top,
|
||||
code_vision_author_position: CodeVisionPosition::Top,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,6 +275,11 @@ pub enum DebugEvent {
|
||||
},
|
||||
/// The session's capabilities changed after it started: fetch them again.
|
||||
Capabilities,
|
||||
/// The adapter's `invalidated` event: what it shows for `areas` (`all`, `stacks`, `threads`,
|
||||
/// `variables`; empty means all) is stale and has to be fetched again.
|
||||
Invalidated {
|
||||
areas: Vec<String>,
|
||||
},
|
||||
JvmBackend {
|
||||
state: JvmBackendState,
|
||||
},
|
||||
@@ -330,4 +335,12 @@ mod tests {
|
||||
);
|
||||
assert_eq!(serde_json::to_value(DebugEvent::Capabilities).unwrap(), json!({"kind": "capabilities"}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalidated_event_json_shape() {
|
||||
assert_eq!(
|
||||
serde_json::to_value(DebugEvent::Invalidated { areas: vec!["variables".into()] }).unwrap(),
|
||||
json!({"kind": "invalidated", "areas": ["variables"]})
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,8 @@ pub struct Settings {
|
||||
pub agents: AgentSettings,
|
||||
/// Agents › MCP Server (global only).
|
||||
pub mcp_server: McpServerSettings,
|
||||
/// Tools › Agents: Command Execution Policy.
|
||||
pub agent_security: AgentSecuritySettings,
|
||||
/// Plugin settings (`[contributes.settings]`): plugin id → its values by key.
|
||||
#[specta(type = BTreeMap<String, BTreeMap<String, specta_typescript::Unknown>>)]
|
||||
pub plugins: BTreeMap<String, serde_json::Map<String, serde_json::Value>>,
|
||||
@@ -261,6 +263,9 @@ pub enum WhileRunning {
|
||||
pub struct McpServerSettings {
|
||||
/// Serve the tools on 127.0.0.1 while the IDE runs.
|
||||
pub enabled: bool,
|
||||
/// Run the tools that change the project or start processes without asking first
|
||||
/// (IDEA's brave mode). Read from the IDE's own settings only, never a project's.
|
||||
pub run_without_confirmation: bool,
|
||||
}
|
||||
|
||||
/// Absent profiles retain automatic discovery for existing projects.
|
||||
@@ -408,6 +413,7 @@ impl Default for Settings {
|
||||
cmake: CMakeSettings::default(),
|
||||
agents: AgentSettings::default(),
|
||||
mcp_server: McpServerSettings::default(),
|
||||
agent_security: AgentSecuritySettings::default(),
|
||||
plugins: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
@@ -580,12 +586,10 @@ pub struct AppearanceSettings {
|
||||
pub project_colors_in_toolbar: bool,
|
||||
pub dnd_with_alt_only: bool,
|
||||
pub menu_icons: bool,
|
||||
/// `hamburger` | `merged`: main menu presentation in the frameless title bar.
|
||||
/// `hamburger` | `merged` | `toolbar` (a separate row): main menu presentation in the frameless title bar.
|
||||
pub main_menu_style: String,
|
||||
/// A lone Alt press focuses or opens the main menu (IDEA `ide.windows.main.menu.focus.on.alt`).
|
||||
pub main_menu_alt: bool,
|
||||
/// Always show the main menu titles instead of the hamburger button (#1521).
|
||||
pub always_show_main_menu: bool,
|
||||
/// `subpixel` | `greyscale` | `none`
|
||||
pub ide_antialiasing: String,
|
||||
pub editor_antialiasing: String,
|
||||
@@ -663,7 +667,6 @@ impl Default for AppearanceSettings {
|
||||
menu_icons: true,
|
||||
main_menu_style: "hamburger".into(),
|
||||
main_menu_alt: true,
|
||||
always_show_main_menu: false,
|
||||
ide_antialiasing: "greyscale".into(),
|
||||
editor_antialiasing: "greyscale".into(),
|
||||
show_tool_window_bars: true,
|
||||
@@ -1579,11 +1582,6 @@ mod tests {
|
||||
let off: Settings = serde_json::from_str(r#"{"appearance":{"mainMenuAlt":false}}"#).unwrap();
|
||||
assert!(!off.appearance.main_menu_alt);
|
||||
assert_eq!(serde_json::to_value(off).unwrap()["appearance"]["mainMenuAlt"], false);
|
||||
let old: Settings = serde_json::from_str(r#"{"appearance":{"compactMode":true}}"#).unwrap();
|
||||
assert!(!old.appearance.always_show_main_menu);
|
||||
let shown: Settings = serde_json::from_str(r#"{"appearance":{"alwaysShowMainMenu":true}}"#).unwrap();
|
||||
assert!(shown.appearance.always_show_main_menu);
|
||||
assert_eq!(serde_json::to_value(shown).unwrap()["appearance"]["alwaysShowMainMenu"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -412,3 +412,30 @@ impl Default for ExternalTool {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tools › Agents: Command Execution Policy (whitelist and blacklist patterns).
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[serde(default, rename_all = "camelCase")]
|
||||
pub struct AgentSecuritySettings {
|
||||
/// Commands or patterns (`git status*`, `pwd`) automatically approved without prompting.
|
||||
pub allowed_commands: Vec<String>,
|
||||
/// Commands or patterns (`rm -rf *`, `mkfs*`) automatically rejected.
|
||||
pub blocked_commands: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for AgentSecuritySettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
allowed_commands: Vec::new(),
|
||||
blocked_commands: vec![
|
||||
"rm -rf *".into(),
|
||||
"rm -rf /".into(),
|
||||
"mkfs*".into(),
|
||||
"dd *".into(),
|
||||
":(){ :|:& };:".into(),
|
||||
"shutdown*".into(),
|
||||
"reboot*".into(),
|
||||
],
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ ide-jsonrpc.workspace = true
|
||||
ide-jvm-attach.workspace = true
|
||||
ide-jvm-inspect.workspace = true
|
||||
ide-jdwp.workspace = true
|
||||
ide-heap.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
specta.workspace = true
|
||||
|
||||
@@ -72,6 +72,13 @@ fn builtin(adapter: &str) -> Option<DebuggerConfig> {
|
||||
"stdio",
|
||||
"Install Kotlin Debug Adapter in Settings › Languages & Frameworks › Language Servers",
|
||||
),
|
||||
// Inside the language server: reached through `debugServer`, never started.
|
||||
"kotlin-lsp" => cfg(
|
||||
"kotlin-lsp",
|
||||
&[],
|
||||
"tcp",
|
||||
"Install JetBrains Kotlin LSP in Settings › Languages & Frameworks › Language Servers",
|
||||
),
|
||||
// Windows installs `python.exe`; `python3.exe` there is at most a Store alias.
|
||||
"debugpy" => cfg(
|
||||
if cfg!(windows) { "python" } else { "python3" },
|
||||
|
||||
@@ -342,4 +342,16 @@ mod tests {
|
||||
assert_eq!(error_message(&msg), "cannot find a.py");
|
||||
assert_eq!(error_message(&json!({"message": "nope"})), "nope");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_launch_response_keeps_delves_too_old_error_verbatim() {
|
||||
// The UI's repair offer keys off this exact wording (`missingAdapter`), so the
|
||||
// conversion to the IPC error must not fold it into a generic message.
|
||||
let message = "Failed to launch /p/api: Version of Delve is too old for Go version go1.27.0 \
|
||||
(maximum supported version 1.26, suppress this error with --check-go-version=false)";
|
||||
let e = ide_core::IdeError::from(DapError::Failed { command: "launch".into(), message: message.into() });
|
||||
assert_eq!(e.kind, ide_core::error::ErrorKind::InvalidInput);
|
||||
assert_eq!(e.message, format!("launch failed: {message}"));
|
||||
assert!(e.message.contains("Version of Delve is too old for Go version go1.27.0"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,293 @@
|
||||
//! A heap dump of a debugged JVM, written over its attach listener into a snapshot directory.
|
||||
|
||||
use super::snapshot::{AttachProbe, JdkProbe, attach_failure, gone, pick_jvm_pid};
|
||||
use super::{
|
||||
JvmMemoryState, JvmMemoryUnavailable, JvmTarget, histogram_pid, keep_found, lookup, resolve_jvm,
|
||||
safe_while_suspended, session_target,
|
||||
};
|
||||
use crate::session::Session;
|
||||
use ide_core::{CancellationToken, IdeError, IdeResult};
|
||||
use ide_jvm_attach::{AttachError, HeapDumpReply};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use specta::Type;
|
||||
use std::future::Future;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
const DUMP_FILE: &str = "dump.hprof";
|
||||
/// For a refusal whose reply holds no reason: empty, or only the line announcing the dump.
|
||||
const NO_REASON: &str = "the JVM wrote no heap dump and gave no reason";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]
|
||||
pub enum JvmHeapUnavailable {
|
||||
/// The reason a class table could not be read applies to the dump too.
|
||||
Memory {
|
||||
reason: JvmMemoryUnavailable,
|
||||
},
|
||||
/// Bytes.
|
||||
NotEnoughDiskSpace {
|
||||
needed: u64,
|
||||
available: u64,
|
||||
},
|
||||
DumpFailed {
|
||||
message: String,
|
||||
},
|
||||
DumpTimedOut {
|
||||
seconds: u32,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct JvmHeapDump {
|
||||
pub path: PathBuf,
|
||||
/// The class histogram taken just before the dump.
|
||||
pub histogram: Vec<ide_heap::HistogramEntry>,
|
||||
/// Raw `VM.flags`; empty when the JVM did not answer.
|
||||
pub vm_flags: String,
|
||||
/// 0 when unknown.
|
||||
pub java_major: u32,
|
||||
/// Empty when unknown.
|
||||
pub java_version: String,
|
||||
pub pid: u32,
|
||||
pub taken_while_suspended: bool,
|
||||
pub dump_ms: u32,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub enum JvmHeapOutcome {
|
||||
Ready(JvmHeapDump),
|
||||
/// The snapshot directory has been deleted.
|
||||
Unavailable(JvmHeapUnavailable),
|
||||
/// The snapshot directory has been deleted.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
pub(crate) async fn session_heap_dump(
|
||||
session: Arc<Session>,
|
||||
dir: PathBuf,
|
||||
cancel: CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
let launched = histogram_pid(&session);
|
||||
let capture = {
|
||||
let (dir, cancel) = (dir.clone(), cancel.clone());
|
||||
async move {
|
||||
let target = session_target(&session, launched);
|
||||
let probe = |suspended| -> Arc<dyn AttachProbe> { Arc::new(JdkProbe { suspended }) };
|
||||
let stopped = session.clone();
|
||||
let suspended = move || capture_suspended(&stopped);
|
||||
heap_dump_with(&session.jvm_memory, suspended, probe, target, dir, cancel).await
|
||||
}
|
||||
};
|
||||
detached(capture, dir, cancel).await
|
||||
}
|
||||
|
||||
/// The JVM can be suspended before the adapter reports the stop, so a session with a JDWP link
|
||||
/// also counts the link's view.
|
||||
pub(super) fn capture_suspended(session: &Session) -> bool {
|
||||
session.suspended.get()
|
||||
|| session
|
||||
.jdwp_link
|
||||
.lock()
|
||||
.as_ref()
|
||||
.is_some_and(|link| matches!(link.state(), ide_jdwp::LinkState::Suspended { .. }))
|
||||
}
|
||||
|
||||
/// Runs `capture` in a task of its own, so a caller that stops waiting cannot release the
|
||||
/// session's lock while the JVM is still writing the dump and let a class-table request run
|
||||
/// next to it. On `cancel` returns at once and deletes `dir`.
|
||||
pub(super) async fn detached(
|
||||
capture: impl Future<Output = IdeResult<JvmHeapOutcome>> + Send + 'static,
|
||||
dir: PathBuf,
|
||||
cancel: CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
let task = tokio::spawn(capture);
|
||||
tokio::select! {
|
||||
joined = task => joined.map_err(|e| IdeError::internal(format!("JVM heap dump failed: {e}")))?,
|
||||
() = cancel.cancelled() => {
|
||||
discard_later(dir).await;
|
||||
Ok(JvmHeapOutcome::Cancelled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Holds `state`'s lock throughout, as a class-table request does; `suspended` and then `probe`
|
||||
/// are read once it is held, and `suspended` again by [`take_heap_dump`]. Anything but `Ready`
|
||||
/// deletes `dir`.
|
||||
pub(super) async fn heap_dump_with(
|
||||
state: &tokio::sync::Mutex<JvmMemoryState>,
|
||||
suspended: impl Fn() -> bool + Send + Sync + 'static,
|
||||
probe: impl FnOnce(bool) -> Arc<dyn AttachProbe>,
|
||||
target: JvmTarget<'_>,
|
||||
dir: PathBuf,
|
||||
cancel: CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
// The capture's histogram never becomes the Memory view's baseline, so the state is unused.
|
||||
let _turn = state.lock().await;
|
||||
let outcome = capture(suspended, probe, target, dir.clone(), cancel).await;
|
||||
if !matches!(outcome, Ok(JvmHeapOutcome::Ready(_))) {
|
||||
discard_later(dir).await;
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
async fn capture(
|
||||
suspended_now: impl Fn() -> bool + Send + Sync + 'static,
|
||||
probe: impl FnOnce(bool) -> Arc<dyn AttachProbe>,
|
||||
target: JvmTarget<'_>,
|
||||
dir: PathBuf,
|
||||
cancel: CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
if cancel.is_cancelled() {
|
||||
return Ok(JvmHeapOutcome::Cancelled);
|
||||
}
|
||||
let suspended = suspended_now();
|
||||
let lookup = match lookup(target) {
|
||||
Ok(lookup) => lookup,
|
||||
Err(reason) => return Ok(memory(reason)),
|
||||
};
|
||||
let probe = probe(suspended);
|
||||
let (found, outcome) = tokio::task::spawn_blocking(move || match resolve_jvm(lookup, &*probe, suspended) {
|
||||
Ok((tree, found)) => (found, take_heap_dump(&*probe, &tree, suspended, &suspended_now, &dir, &cancel)),
|
||||
Err(reason) => (None, Ok(memory(reason))),
|
||||
})
|
||||
.await
|
||||
.map_err(|e| IdeError::internal(format!("JVM heap dump failed: {e}")))?;
|
||||
keep_found(target, found);
|
||||
outcome
|
||||
}
|
||||
|
||||
/// `tree` starts with the session's reported pid, followed by its descendants. `suspended` is
|
||||
/// the stop read when the capture began; `suspended_now` reads it again. Anything but `Ready`
|
||||
/// deletes `dir`.
|
||||
pub(crate) fn take_heap_dump(
|
||||
probe: &dyn AttachProbe,
|
||||
tree: &[i32],
|
||||
suspended: bool,
|
||||
suspended_now: &dyn Fn() -> bool,
|
||||
dir: &Path,
|
||||
cancel: &CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
let outcome = dump(probe, tree, suspended, suspended_now, dir, cancel);
|
||||
if !matches!(outcome, Ok(JvmHeapOutcome::Ready(_))) {
|
||||
discard(dir);
|
||||
}
|
||||
outcome
|
||||
}
|
||||
|
||||
fn dump(
|
||||
probe: &dyn AttachProbe,
|
||||
tree: &[i32],
|
||||
suspended: bool,
|
||||
suspended_now: &dyn Fn() -> bool,
|
||||
dir: &Path,
|
||||
cancel: &CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
use JvmHeapUnavailable as H;
|
||||
use JvmMemoryUnavailable as U;
|
||||
let Some(root) = tree.first().and_then(|&p| u32::try_from(p).ok()) else {
|
||||
return Ok(memory(U::NoJvmProcess));
|
||||
};
|
||||
let Some(jvm) = pick_jvm_pid(tree, |p| probe.has_listener(p)) else {
|
||||
return Ok(memory(gone(probe, root)));
|
||||
};
|
||||
|
||||
let version = probe.version(jvm);
|
||||
let java_major = version.as_ref().map_or(0, |v| v.major);
|
||||
let safe = safe_while_suspended(version.as_ref());
|
||||
let unsafe_stop = || Ok(memory(U::UnsafeWhileSuspended { java_major }));
|
||||
if suspended && !safe {
|
||||
return unsafe_stop();
|
||||
}
|
||||
// The program may stop during the histogram, and a stopped JDK below 17 hangs on the next
|
||||
// request until it resumes.
|
||||
let stopped_since = || !safe && suspended_now();
|
||||
|
||||
let text = match probe.histogram(jvm) {
|
||||
Ok(text) => text,
|
||||
Err(e) => return attach_failure(probe, jvm, e, "class histogram").map(memory),
|
||||
};
|
||||
let histogram: Vec<ide_heap::HistogramEntry> = ide_jvm_attach::parse_class_histogram(&text)
|
||||
.rows
|
||||
.into_iter()
|
||||
.map(|r| ide_heap::HistogramEntry { name: r.name, instances: r.instances, bytes: r.bytes })
|
||||
.collect();
|
||||
let (instances, bytes) =
|
||||
histogram.iter().fold((0u64, 0u64), |(i, b), e| (i.saturating_add(e.instances), b.saturating_add(e.bytes)));
|
||||
let needed = ide_heap::capture_need(bytes, instances);
|
||||
if let Some(available) = probe.free_bytes(dir)
|
||||
&& available < needed
|
||||
{
|
||||
return Ok(JvmHeapOutcome::Unavailable(H::NotEnoughDiskSpace { needed, available }));
|
||||
}
|
||||
|
||||
if stopped_since() {
|
||||
return unsafe_stop();
|
||||
}
|
||||
// Without flags the index assumes the default object layout.
|
||||
let vm_flags = probe.vm_flags(jvm).unwrap_or_else(|e| {
|
||||
tracing::debug!("VM.flags of JVM {jvm}: {e}");
|
||||
String::new()
|
||||
});
|
||||
if cancel.is_cancelled() {
|
||||
return Ok(JvmHeapOutcome::Cancelled);
|
||||
}
|
||||
if stopped_since() {
|
||||
return unsafe_stop();
|
||||
}
|
||||
|
||||
let path = dir.join(DUMP_FILE);
|
||||
let timeout = ide_jvm_attach::dump_timeout(bytes);
|
||||
let started = std::time::Instant::now();
|
||||
let reply = probe.heap_dump(jvm, &path, timeout);
|
||||
let dump_ms = u32::try_from(started.elapsed().as_millis()).unwrap_or(u32::MAX);
|
||||
if cancel.is_cancelled() {
|
||||
return Ok(JvmHeapOutcome::Cancelled);
|
||||
}
|
||||
let failed = |message: String| Ok(JvmHeapOutcome::Unavailable(H::DumpFailed { message }));
|
||||
match reply {
|
||||
Ok(HeapDumpReply::Created { .. }) => {
|
||||
if !std::fs::metadata(&path).is_ok_and(|m| m.is_file() && m.len() > 0) {
|
||||
return failed(format!("the JVM reported a heap dump, but {} is missing or empty", path.display()));
|
||||
}
|
||||
Ok(JvmHeapOutcome::Ready(JvmHeapDump {
|
||||
path,
|
||||
histogram,
|
||||
vm_flags,
|
||||
java_major,
|
||||
java_version: version.map(|v| v.text).unwrap_or_default(),
|
||||
pid: jvm,
|
||||
taken_while_suspended: suspended,
|
||||
dump_ms,
|
||||
}))
|
||||
}
|
||||
Ok(HeapDumpReply::Refused { message }) if message.is_empty() || message.starts_with("Dumping heap to") => {
|
||||
failed(NO_REASON.into())
|
||||
}
|
||||
Ok(HeapDumpReply::Refused { message }) => failed(message),
|
||||
Err(AttachError::Timeout) => {
|
||||
let seconds = u32::try_from(timeout.as_secs()).unwrap_or(u32::MAX);
|
||||
Ok(JvmHeapOutcome::Unavailable(H::DumpTimedOut { seconds }))
|
||||
}
|
||||
// A running JDK rejects an argument with a non-zero result code.
|
||||
Err(AttachError::Command { message, .. }) => failed(message),
|
||||
Err(e) => attach_failure(probe, jvm, e, "heap dump").map(memory),
|
||||
}
|
||||
}
|
||||
|
||||
fn memory(reason: JvmMemoryUnavailable) -> JvmHeapOutcome {
|
||||
JvmHeapOutcome::Unavailable(JvmHeapUnavailable::Memory { reason })
|
||||
}
|
||||
|
||||
fn discard(dir: &Path) {
|
||||
if let Err(e) = std::fs::remove_dir_all(dir)
|
||||
&& e.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
tracing::warn!("could not delete heap snapshot directory {}: {e}", dir.display());
|
||||
}
|
||||
}
|
||||
|
||||
/// A dump may be large, so it is deleted off the async threads.
|
||||
async fn discard_later(dir: PathBuf) {
|
||||
let _ = tokio::task::spawn_blocking(move || discard(&dir)).await;
|
||||
}
|
||||
@@ -2,6 +2,7 @@
|
||||
//! that is unavailable, counted by the debugger; and the JVM's objects through the debugger.
|
||||
|
||||
mod fallback;
|
||||
mod heap;
|
||||
mod inspect;
|
||||
mod scan;
|
||||
mod snapshot;
|
||||
@@ -20,6 +21,7 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicI64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
pub use heap::*;
|
||||
pub use inspect::{JvmFieldsResult, JvmInspectUnavailable, JvmInstancesResult, JvmObjectRef, JvmReferrersResult};
|
||||
pub(crate) use inspect::{fields, instances, referrers, set_inspector, watch_link};
|
||||
#[cfg(all(test, unix))]
|
||||
@@ -27,7 +29,7 @@ pub(crate) use scan::watch_for_attached_jvm;
|
||||
pub(crate) use scan::{JvmFinder, attach_target, find_attached_jvm, find_local_jvm, local_jdwp_port};
|
||||
use snapshot::{AttachProbe, JdkProbe, take_snapshot};
|
||||
|
||||
pub const JVM_MEMORY_ADAPTERS: &[&str] = &["java-debug", crate::kotlin::ADAPTER];
|
||||
pub const JVM_MEMORY_ADAPTERS: &[&str] = &["java-debug", crate::kotlin::ADAPTER, crate::kotlin_lsp::ADAPTER];
|
||||
pub const HISTOGRAM_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Type)]
|
||||
|
||||
@@ -6,14 +6,16 @@ use super::{
|
||||
safe_while_suspended,
|
||||
};
|
||||
use ide_core::IdeResult;
|
||||
use ide_jvm_attach::{AttachError, HistogramRow, JavaVersion};
|
||||
use ide_jvm_attach::{AttachError, HeapDumpReply, HistogramRow, JavaVersion};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Version probe for a JVM whose `release` file could not be read.
|
||||
const VERSION_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const FLAGS_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// The attach calls a snapshot makes, so tests can stand in for a JVM.
|
||||
/// The attach calls a snapshot or a heap dump makes, so tests can stand in for a JVM.
|
||||
pub(crate) trait AttachProbe: Send + Sync {
|
||||
fn has_listener(&self, pid: u32) -> bool;
|
||||
/// Asks only the JVMs whose pid `may_query` accepts.
|
||||
@@ -21,6 +23,10 @@ pub(crate) trait AttachProbe: Send + Sync {
|
||||
fn version(&self, pid: u32) -> Option<JavaVersion>;
|
||||
fn histogram(&self, pid: u32) -> Result<String, AttachError>;
|
||||
fn alive(&self, pid: u32) -> bool;
|
||||
fn vm_flags(&self, pid: u32) -> Result<String, AttachError>;
|
||||
fn heap_dump(&self, pid: u32, path: &Path, timeout: Duration) -> Result<HeapDumpReply, AttachError>;
|
||||
/// `None` where the free space is not known.
|
||||
fn free_bytes(&self, dir: &Path) -> Option<u64>;
|
||||
}
|
||||
|
||||
pub(super) struct JdkProbe {
|
||||
@@ -55,6 +61,18 @@ impl AttachProbe for JdkProbe {
|
||||
fn alive(&self, pid: u32) -> bool {
|
||||
alive(pid)
|
||||
}
|
||||
|
||||
fn vm_flags(&self, pid: u32) -> Result<String, AttachError> {
|
||||
ide_jvm_attach::vm_flags(pid, FLAGS_TIMEOUT)
|
||||
}
|
||||
|
||||
fn heap_dump(&self, pid: u32, path: &Path, timeout: Duration) -> Result<HeapDumpReply, AttachError> {
|
||||
ide_jvm_attach::heap_dump(pid, path, timeout)
|
||||
}
|
||||
|
||||
fn free_bytes(&self, dir: &Path) -> Option<u64> {
|
||||
ide_heap::free_bytes(dir)
|
||||
}
|
||||
}
|
||||
|
||||
fn alive(pid: u32) -> bool {
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
use super::fallback::{debugger_snapshot, falls_back};
|
||||
use super::heap::*;
|
||||
use super::scan::*;
|
||||
use super::snapshot::*;
|
||||
use super::test_support::{BareSession, FakeInspector, bare_session, bare_session_of};
|
||||
use super::*;
|
||||
use ide_jvm_attach::{AttachError, HistogramRow, JavaVersion};
|
||||
use ide_core::CancellationToken;
|
||||
use ide_jvm_attach::{AttachError, HeapDumpReply, HistogramRow, JavaVersion};
|
||||
use std::collections::VecDeque;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
|
||||
fn row(name: &str, instances: u64, bytes: u64) -> HistogramRow {
|
||||
@@ -25,8 +28,52 @@ struct FakeProbe {
|
||||
asked: parking_lot::Mutex<Vec<u32>>,
|
||||
/// Pids the scan sent a request to.
|
||||
queried: parking_lot::Mutex<Vec<u32>>,
|
||||
/// Attach commands sent, in order.
|
||||
/// Attach commands sent, in order: `histogram`, `vm_flags`, `heap_dump`.
|
||||
commands: parking_lot::Mutex<Vec<&'static str>>,
|
||||
dumps: parking_lot::Mutex<VecDeque<Result<HeapDumpReply, AttachError>>>,
|
||||
dump_paths: parking_lot::Mutex<Vec<PathBuf>>,
|
||||
/// Each heap dump waits for this gate to open.
|
||||
dump_gate: Option<Arc<Gate>>,
|
||||
/// A `Created` reply writes this many bytes to the dump path first.
|
||||
dump_bytes: usize,
|
||||
free: Option<u64>,
|
||||
/// Answering this command stops the program: runs the hook.
|
||||
stops_on: Option<(&'static str, StopHook)>,
|
||||
}
|
||||
|
||||
type StopHook = Box<dyn Fn() + Send + Sync>;
|
||||
|
||||
/// Holds a blocking call until the test opens it.
|
||||
#[derive(Default)]
|
||||
struct Gate {
|
||||
entered: tokio::sync::Notify,
|
||||
open: parking_lot::Mutex<bool>,
|
||||
opened: parking_lot::Condvar,
|
||||
}
|
||||
|
||||
impl Gate {
|
||||
fn pass(&self) {
|
||||
self.entered.notify_one();
|
||||
let mut open = self.open.lock();
|
||||
while !*open {
|
||||
self.opened.wait(&mut open);
|
||||
}
|
||||
}
|
||||
|
||||
fn release(&self) {
|
||||
*self.open.lock() = true;
|
||||
self.opened.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens the gate when dropped: a failed assertion must not leave a blocking thread waiting,
|
||||
/// since the runtime's shutdown waits for it.
|
||||
struct OpenOnDrop(Arc<Gate>);
|
||||
|
||||
impl Drop for OpenOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.release();
|
||||
}
|
||||
}
|
||||
|
||||
impl FakeProbe {
|
||||
@@ -44,11 +91,26 @@ impl FakeProbe {
|
||||
asked: parking_lot::Mutex::new(Vec::new()),
|
||||
queried: parking_lot::Mutex::new(Vec::new()),
|
||||
commands: parking_lot::Mutex::new(Vec::new()),
|
||||
dumps: parking_lot::Mutex::new(VecDeque::new()),
|
||||
dump_paths: parking_lot::Mutex::new(Vec::new()),
|
||||
dump_gate: None,
|
||||
dump_bytes: 64,
|
||||
free: None,
|
||||
stops_on: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn record(&self, command: &'static str) {
|
||||
self.commands.lock().push(command);
|
||||
if let Some((on, stop)) = &self.stops_on
|
||||
&& *on == command
|
||||
{
|
||||
stop();
|
||||
}
|
||||
}
|
||||
|
||||
fn enter(&self, command: &'static str) {
|
||||
self.commands.lock().push(command);
|
||||
self.record(command);
|
||||
let now = self.in_flight.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
self.max_in_flight.fetch_max(now, Ordering::SeqCst);
|
||||
}
|
||||
@@ -84,8 +146,31 @@ impl AttachProbe for FakeProbe {
|
||||
fn alive(&self, _pid: u32) -> bool {
|
||||
self.alive
|
||||
}
|
||||
fn vm_flags(&self, _pid: u32) -> Result<String, AttachError> {
|
||||
self.record("vm_flags");
|
||||
Ok(FLAGS.into())
|
||||
}
|
||||
fn heap_dump(&self, _pid: u32, path: &Path, _timeout: Duration) -> Result<HeapDumpReply, AttachError> {
|
||||
self.enter("heap_dump");
|
||||
self.dump_paths.lock().push(path.to_owned());
|
||||
if let Some(gate) = &self.dump_gate {
|
||||
gate.pass();
|
||||
}
|
||||
let reply = self.dumps.lock().pop_front().expect("unexpected heap dump request");
|
||||
if matches!(reply, Ok(HeapDumpReply::Created { .. })) && self.dump_bytes > 0 {
|
||||
// Fails like the JVM's write once the directory is gone.
|
||||
let _ = std::fs::write(path, vec![0; self.dump_bytes]);
|
||||
}
|
||||
self.leave();
|
||||
reply
|
||||
}
|
||||
fn free_bytes(&self, _dir: &Path) -> Option<u64> {
|
||||
self.free
|
||||
}
|
||||
}
|
||||
|
||||
const FLAGS: &str = "-XX:+UseCompressedOops -XX:+StartAttachListener";
|
||||
|
||||
const HISTOGRAM: &str = " num #instances #bytes class name (module)\n-------------------------------------------------------\n 1: 5 80 A\n 2: 1 16 B\nTotal 6 96\n";
|
||||
|
||||
fn unavailable(r: IdeResult<Result<(JvmMemorySnapshot, ClassCounts), JvmMemoryUnavailable>>) -> JvmMemoryUnavailable {
|
||||
@@ -691,12 +776,416 @@ fn debugger_rows_are_ordered_by_instances() {
|
||||
assert_eq!(names, ["A", "C", "B"]);
|
||||
}
|
||||
|
||||
/// A snapshot directory inside a temporary one, so the capture may delete it.
|
||||
fn snapshot_dir(name: &str) -> (tempfile::TempDir, PathBuf) {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let dir = root.path().join(name);
|
||||
std::fs::create_dir(&dir).unwrap();
|
||||
(root, dir)
|
||||
}
|
||||
|
||||
/// JVM 12 at Java `major`, answering [`HISTOGRAM`] once and `dumps` in turn.
|
||||
fn dumping(major: u32, dumps: Vec<Result<HeapDumpReply, AttachError>>) -> FakeProbe {
|
||||
let mut probe = FakeProbe::new(12, vec![Ok(HISTOGRAM.into())]);
|
||||
probe.version = Some(JavaVersion { major, text: format!("{major}.0.1") });
|
||||
probe.dumps = parking_lot::Mutex::new(dumps.into());
|
||||
probe
|
||||
}
|
||||
|
||||
fn created() -> Result<HeapDumpReply, AttachError> {
|
||||
Ok(HeapDumpReply::Created { bytes: Some(64) })
|
||||
}
|
||||
|
||||
fn memory(reason: JvmMemoryUnavailable) -> JvmHeapOutcome {
|
||||
JvmHeapOutcome::Unavailable(JvmHeapUnavailable::Memory { reason })
|
||||
}
|
||||
|
||||
fn dump_failed(message: &str) -> JvmHeapOutcome {
|
||||
JvmHeapOutcome::Unavailable(JvmHeapUnavailable::DumpFailed { message: message.into() })
|
||||
}
|
||||
|
||||
fn ready(outcome: IdeResult<JvmHeapOutcome>) -> JvmHeapDump {
|
||||
match outcome.unwrap() {
|
||||
JvmHeapOutcome::Ready(dump) => dump,
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn entry(name: &str, instances: u64, bytes: u64) -> ide_heap::HistogramEntry {
|
||||
ide_heap::HistogramEntry { name: name.into(), instances, bytes }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspended_jdk11_refuses_without_sending() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(11, vec![created()]);
|
||||
let outcome = take_heap_dump(&probe, &[12], true, &|| true, &dir, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(outcome, memory(JvmMemoryUnavailable::UnsafeWhileSuspended { java_major: 11 }));
|
||||
assert!(probe.commands.lock().is_empty());
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn suspended_jdk17_dumps() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(17, vec![created()]);
|
||||
let dump = ready(take_heap_dump(&probe, &[10, 12], true, &|| true, &dir, &CancellationToken::new()));
|
||||
assert_eq!(*probe.commands.lock(), ["histogram", "vm_flags", "heap_dump"]);
|
||||
assert_eq!(*probe.dump_paths.lock(), [dir.join("dump.hprof")]);
|
||||
let expected = JvmHeapDump {
|
||||
path: dir.join("dump.hprof"),
|
||||
histogram: vec![entry("A", 5, 80), entry("B", 1, 16)],
|
||||
vm_flags: FLAGS.into(),
|
||||
java_major: 17,
|
||||
java_version: "17.0.1".into(),
|
||||
pid: 12,
|
||||
taken_while_suspended: true,
|
||||
dump_ms: dump.dump_ms,
|
||||
};
|
||||
assert_eq!(dump, expected);
|
||||
assert!(dump.path.is_file());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_jdk8_dumps() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(8, vec![created()]);
|
||||
let dump = ready(take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()));
|
||||
assert_eq!((dump.java_major, dump.taken_while_suspended), (8, false));
|
||||
assert_eq!(*probe.commands.lock(), ["histogram", "vm_flags", "heap_dump"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn not_enough_disk_space_refuses_before_dump() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let mut probe = dumping(21, vec![created()]);
|
||||
let gib = 1u64 << 30;
|
||||
let histogram = format!(" 1: 10000000 {gib} [B\nTotal 10000000 {gib}\n");
|
||||
probe.histograms = parking_lot::Mutex::new(vec![Ok(histogram)].into());
|
||||
probe.free = Some(512 << 20);
|
||||
let outcome = take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).unwrap();
|
||||
let needed = gib + 1_060_000_000 + gib;
|
||||
assert_eq!(
|
||||
outcome,
|
||||
JvmHeapOutcome::Unavailable(JvmHeapUnavailable::NotEnoughDiskSpace { needed, available: 512 << 20 })
|
||||
);
|
||||
assert_eq!(*probe.commands.lock(), ["histogram"]);
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refused_dump_reports_message() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let message = "Unable to create /tmp/snap/dump.hprof: File exists";
|
||||
let probe = dumping(21, vec![Ok(HeapDumpReply::Refused { message: message.into() })]);
|
||||
let outcome = take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(outcome, dump_failed(message));
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refused_dump_without_a_reason_says_so() {
|
||||
for body in ["", "Dumping heap to /tmp/snap/dump.hprof ..."] {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(21, vec![Ok(HeapDumpReply::Refused { message: body.into() })]);
|
||||
let outcome = take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).unwrap();
|
||||
let JvmHeapOutcome::Unavailable(JvmHeapUnavailable::DumpFailed { message }) = outcome else {
|
||||
panic!("{body:?}: {outcome:?}")
|
||||
};
|
||||
assert!(!message.is_empty() && !message.starts_with("Dumping heap to"), "{body:?}: {message:?}");
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timed_out_dump_removes_snapshot_dir() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(21, vec![Err(AttachError::Timeout)]);
|
||||
let outcome = take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(outcome, JvmHeapOutcome::Unavailable(JvmHeapUnavailable::DumpTimedOut { seconds: 30 }));
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn capture_dir_with_space_is_quoted() {
|
||||
let (_root, dir) = snapshot_dir("a b");
|
||||
let probe = dumping(21, vec![created()]);
|
||||
ready(take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()));
|
||||
let path = probe.dump_paths.lock()[0].clone();
|
||||
assert!(path.to_str().unwrap().contains("a b"), "{path:?}");
|
||||
let command = ide_jvm_attach::heap_dump_command(&path).unwrap();
|
||||
assert_eq!(command, format!("GC.heap_dump -all \"{}\"", path.display()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn created_without_a_file_fails() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let mut probe = dumping(21, vec![created()]);
|
||||
probe.dump_bytes = 0;
|
||||
let outcome = take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).unwrap();
|
||||
assert!(matches!(outcome, JvmHeapOutcome::Unavailable(JvmHeapUnavailable::DumpFailed { .. })), "{outcome:?}");
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dump_errors_map_like_the_class_table() {
|
||||
let message = "java.lang.IllegalArgumentException: Unknown argument";
|
||||
let cases = [
|
||||
(Err(AttachError::Refused), false, memory(JvmMemoryUnavailable::ProgramExited)),
|
||||
(Err(AttachError::NoListener), true, memory(JvmMemoryUnavailable::ListenerNotRunning)),
|
||||
(Err(AttachError::Command { code: -1, message: message.into() }), true, dump_failed(message)),
|
||||
];
|
||||
for (reply, alive, expected) in cases {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let mut probe = dumping(21, vec![reply]);
|
||||
probe.alive = alive;
|
||||
assert_eq!(take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).unwrap(), expected);
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(21, vec![Err(AttachError::Protocol("bad".into()))]);
|
||||
assert!(take_heap_dump(&probe, &[12], false, &|| false, &dir, &CancellationToken::new()).is_err());
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
/// JDK `major`, running when the capture starts, stopped by the time it answers `command`.
|
||||
fn stopping_after(major: u32, command: &'static str) -> (FakeProbe, Arc<AtomicBool>) {
|
||||
let stopped = Arc::new(AtomicBool::new(false));
|
||||
let mut probe = dumping(major, vec![created()]);
|
||||
let flag = stopped.clone();
|
||||
probe.stops_on = Some((command, Box::new(move || flag.store(true, Ordering::SeqCst))));
|
||||
(probe, stopped)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jdk11_stopped_during_the_histogram_gets_no_further_request() {
|
||||
for (command, sent) in [("histogram", &["histogram"][..]), ("vm_flags", &["histogram", "vm_flags"][..])] {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let (probe, stopped) = stopping_after(11, command);
|
||||
let suspended_now = || stopped.load(Ordering::SeqCst);
|
||||
let outcome = take_heap_dump(&probe, &[12], false, &suspended_now, &dir, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(outcome, memory(JvmMemoryUnavailable::UnsafeWhileSuspended { java_major: 11 }), "{command}");
|
||||
assert_eq!(*probe.commands.lock(), sent, "{command}");
|
||||
assert!(!dir.exists(), "{command}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_stopped_during_the_histogram_is_read_again_before_the_dump() {
|
||||
let bare = bare_session("attach");
|
||||
let mut probe = dumping(11, vec![created()]);
|
||||
let stopping = bare.session.clone();
|
||||
probe.stops_on = Some(("histogram", Box::new(move || stopping.suspended.stopped(Some(1), true))));
|
||||
let fake = Arc::new(probe);
|
||||
let probe: Arc<dyn AttachProbe> = fake.clone();
|
||||
let pid = AtomicI64::new(12);
|
||||
let target = JvmTarget::Attached { pid: &pid, jdwp_port: None };
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let session = bare.session.clone();
|
||||
let suspended = move || capture_suspended(&session);
|
||||
let s = &bare.session;
|
||||
let outcome = heap_dump_with(&s.jvm_memory, suspended, |_| probe, target, dir.clone(), CancellationToken::new())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(outcome, memory(JvmMemoryUnavailable::UnsafeWhileSuspended { java_major: 11 }));
|
||||
assert_eq!(*fake.commands.lock(), ["histogram"]);
|
||||
assert!(!dir.exists());
|
||||
assert!(s.jvm_memory.try_lock().is_ok(), "the lock is released");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jdk17_stopped_during_the_histogram_still_dumps() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let (probe, stopped) = stopping_after(17, "histogram");
|
||||
let suspended_now = || stopped.load(Ordering::SeqCst);
|
||||
let dump = ready(take_heap_dump(&probe, &[12], false, &suspended_now, &dir, &CancellationToken::new()));
|
||||
assert!(!dump.taken_while_suspended, "the stop read when the capture began");
|
||||
assert_eq!(*probe.commands.lock(), ["histogram", "vm_flags", "heap_dump"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_capture_sends_no_dump() {
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let probe = dumping(21, vec![created()]);
|
||||
let cancel = CancellationToken::new();
|
||||
cancel.cancel();
|
||||
assert_eq!(take_heap_dump(&probe, &[12], false, &|| false, &dir, &cancel).unwrap(), JvmHeapOutcome::Cancelled);
|
||||
assert!(!probe.commands.lock().contains(&"heap_dump"));
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_capture_releases_lock() {
|
||||
let state = Arc::new(tokio::sync::Mutex::new(JvmMemoryState::default()));
|
||||
let gate = Arc::new(Gate::default());
|
||||
let opener = OpenOnDrop(gate.clone());
|
||||
let mut fake = dumping(21, vec![created()]);
|
||||
fake.histograms.lock().push_back(Ok(HISTOGRAM.into()));
|
||||
fake.dump_gate = Some(gate.clone());
|
||||
let fake = Arc::new(fake);
|
||||
let jvm_pid = Arc::new(AtomicI64::new(12));
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let cancel = CancellationToken::new();
|
||||
let capture = {
|
||||
let (state, fake, jvm_pid, dir, cancel) =
|
||||
(state.clone(), fake.clone(), jvm_pid.clone(), dir.clone(), cancel.clone());
|
||||
async move {
|
||||
let target = JvmTarget::Attached { pid: &jvm_pid, jdwp_port: None };
|
||||
heap_dump_with(&state, || false, |_| -> Arc<dyn AttachProbe> { fake }, target, dir, cancel).await
|
||||
}
|
||||
};
|
||||
let caller = tokio::spawn(detached(capture, dir.clone(), cancel.clone()));
|
||||
gate.entered.notified().await;
|
||||
cancel.cancel();
|
||||
assert_eq!(caller.await.unwrap().unwrap(), JvmHeapOutcome::Cancelled);
|
||||
assert_eq!(fake.in_flight.load(Ordering::SeqCst), 1, "the JVM is still writing the dump");
|
||||
assert!(!dir.exists());
|
||||
|
||||
let mut table = tokio::spawn({
|
||||
let (state, fake, jvm_pid) = (state.clone(), fake.clone(), jvm_pid.clone());
|
||||
async move {
|
||||
let target = JvmTarget::Attached { pid: &jvm_pid, jdwp_port: None };
|
||||
let probe: Arc<dyn AttachProbe> = fake;
|
||||
snapshot(&state, &Suspension::default(), |_| probe, target, None).await
|
||||
}
|
||||
});
|
||||
// While the gate is shut the dump holds the session's turn, so the class table cannot finish.
|
||||
let early = tokio::time::timeout(Duration::from_millis(100), &mut table).await;
|
||||
assert!(early.is_err(), "the class table ran beside the dump: {early:?}");
|
||||
drop(opener);
|
||||
let table = table.await.unwrap().unwrap();
|
||||
assert!(matches!(table, JvmMemoryResult::Ready { .. }), "{table:?}");
|
||||
assert_eq!(fake.max_in_flight.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(*fake.commands.lock(), ["histogram", "vm_flags", "heap_dump", "histogram"]);
|
||||
assert!(!dir.exists(), "the dump that finished after the cancel is deleted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capture_keeps_the_memory_baseline() {
|
||||
let state = tokio::sync::Mutex::new(JvmMemoryState::default());
|
||||
let running = Suspension::default();
|
||||
let pid = AtomicI64::new(12);
|
||||
let target = || JvmTarget::Attached { pid: &pid, jdwp_port: None };
|
||||
let table = |fake: FakeProbe| {
|
||||
let probe: Arc<dyn AttachProbe> = Arc::new(fake);
|
||||
snapshot(&state, &running, |_| probe, target(), None)
|
||||
};
|
||||
table(FakeProbe::new(12, vec![Ok(HISTOGRAM.into())])).await.unwrap();
|
||||
|
||||
let mut capture = dumping(21, vec![created()]);
|
||||
let grown = " 1: 50 800 A\nTotal 50 800\n";
|
||||
capture.histograms = parking_lot::Mutex::new(vec![Ok(grown.into())].into());
|
||||
let probe: Arc<dyn AttachProbe> = Arc::new(capture);
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
ready(heap_dump_with(&state, || false, |_| probe, target(), dir, CancellationToken::new()).await);
|
||||
|
||||
let JvmMemoryResult::Ready { snapshot } = table(FakeProbe::new(12, vec![Ok(HISTOGRAM.into())])).await.unwrap()
|
||||
else {
|
||||
panic!("no class table")
|
||||
};
|
||||
let a = snapshot.rows.iter().find(|r| r.name == "A").unwrap();
|
||||
assert_eq!((a.delta_instances, a.delta_bytes), (Some(0), Some(0)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_session_dumps_the_scanned_jvm() {
|
||||
let mut fake = dumping(21, vec![created()]);
|
||||
fake.listener = 4242;
|
||||
fake.jdwp = Some((5005, 4242));
|
||||
let fake = Arc::new(fake);
|
||||
let probe: Arc<dyn AttachProbe> = fake.clone();
|
||||
let state = tokio::sync::Mutex::new(JvmMemoryState::default());
|
||||
let pid = AtomicI64::new(0);
|
||||
let target = JvmTarget::Attached { pid: &pid, jdwp_port: Some(5005) };
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let dump = ready(heap_dump_with(&state, || false, |_| probe, target, dir, CancellationToken::new()).await);
|
||||
assert_eq!(dump.pid, 4242);
|
||||
assert_eq!(pid.load(Ordering::SeqCst), 4242);
|
||||
assert_eq!(fake.scans.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
/// A JDWP link whose fake JVM sent a suspending event, which the adapter would report next:
|
||||
/// (link, adapter end, JVM end).
|
||||
async fn suspended_link() -> (ide_jdwp::JdwpLink, tokio::net::TcpStream, tokio::net::TcpStream) {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
const HANDSHAKE: &[u8; 14] = b"JDWP-Handshake";
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let options = ide_jdwp::LinkOptions { hold_resumes: false };
|
||||
let (link, link_port) = ide_jdwp::JdwpLink::start("127.0.0.1".into(), port, options).await.unwrap();
|
||||
let mut adapter = tokio::net::TcpStream::connect(("127.0.0.1", link_port)).await.unwrap();
|
||||
adapter.write_all(HANDSHAKE).await.unwrap();
|
||||
let (mut jvm, _) = tokio::time::timeout(Duration::from_secs(2), listener.accept()).await.unwrap().unwrap();
|
||||
let mut buf = [0; 14];
|
||||
jvm.read_exact(&mut buf).await.unwrap();
|
||||
jvm.write_all(HANDSHAKE).await.unwrap();
|
||||
adapter.read_exact(&mut buf).await.unwrap();
|
||||
// Event.Composite (64/100): SUSPEND_ALL, one VM_START event, request 0, thread 1.
|
||||
let body = [2, 0, 0, 0, 1, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1];
|
||||
let mut packet = u32::try_from(11 + body.len()).unwrap().to_be_bytes().to_vec();
|
||||
packet.extend(1u32.to_be_bytes());
|
||||
packet.extend([0, 64, 100]);
|
||||
packet.extend(body);
|
||||
jvm.write_all(&packet).await.unwrap();
|
||||
let mut states = link.subscribe();
|
||||
let suspended = states.wait_for(|s| matches!(s, ide_jdwp::LinkState::Suspended { .. }));
|
||||
tokio::time::timeout(Duration::from_secs(2), suspended).await.unwrap().unwrap();
|
||||
(link, adapter, jvm)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn jdk11_is_refused_while_the_link_is_suspended_before_dap_stops() {
|
||||
let bare = bare_session_of(crate::kotlin::ADAPTER, "attach");
|
||||
let s = &bare.session;
|
||||
let (link, _adapter, _jvm) = suspended_link().await;
|
||||
*s.jdwp_link.lock() = Some(link);
|
||||
assert!(!s.suspended.get());
|
||||
|
||||
let fake = Arc::new(jdk11(12));
|
||||
let probe: Arc<dyn AttachProbe> = fake.clone();
|
||||
let probe = |suspended: bool| {
|
||||
assert!(suspended, "the probe must not ask a suspended JVM's version over attach");
|
||||
probe
|
||||
};
|
||||
let pid = AtomicI64::new(12);
|
||||
let target = JvmTarget::Attached { pid: &pid, jdwp_port: None };
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let session = bare.session.clone();
|
||||
let suspended = move || capture_suspended(&session);
|
||||
let outcome =
|
||||
heap_dump_with(&s.jvm_memory, suspended, probe, target, dir.clone(), CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(outcome, memory(JvmMemoryUnavailable::UnsafeWhileSuspended { java_major: 11 }));
|
||||
assert!(fake.commands.lock().is_empty());
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn capability_matches_memory() {
|
||||
for adapter in JVM_MEMORY_ADAPTERS.iter().copied().chain(["debugpy"]) {
|
||||
let bare = bare_session_of(adapter, "launch");
|
||||
let caps = crate::session_capabilities(&bare.session);
|
||||
let jvm = adapter != "debugpy";
|
||||
assert_eq!(caps.jvm_memory, jvm, "{adapter}");
|
||||
assert_eq!((caps.jvm_heap, caps.jvm_memory), (jvm, jvm), "{adapter}");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn session_without_a_jvm_reports_no_process() {
|
||||
let bare = bare_session("launch");
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let outcome = session_heap_dump(bare.session.clone(), dir.clone(), CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(outcome, memory(JvmMemoryUnavailable::NoJvmProcess));
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
#[tokio::test]
|
||||
async fn capture_is_unsupported_off_unix() {
|
||||
let bare = bare_session("launch");
|
||||
let (_root, dir) = snapshot_dir("snap");
|
||||
let outcome = session_heap_dump(bare.session.clone(), dir.clone(), CancellationToken::new()).await.unwrap();
|
||||
assert_eq!(outcome, memory(JvmMemoryUnavailable::UnsupportedPlatform));
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
@@ -9,11 +9,15 @@ use std::{
|
||||
|
||||
pub const ADAPTER: &str = "kotlin-debug-adapter";
|
||||
|
||||
/// The adapter attaches to a JVM the IDE starts, through the JDWP link. It resumes the VM while
|
||||
/// constructing its debuggee, before its breakpoint hooks exist, so the link holds resumes back
|
||||
/// until the attach is done.
|
||||
pub(crate) fn link_options() -> ide_jdwp::LinkOptions {
|
||||
ide_jdwp::LinkOptions { hold_resumes: true }
|
||||
/// fwcd's adapter or kotlin-lsp's: both debug a JVM the IDE starts, through the JDWP link.
|
||||
pub fn is_kotlin(adapter: &str) -> bool {
|
||||
adapter == ADAPTER || adapter == crate::kotlin_lsp::ADAPTER
|
||||
}
|
||||
|
||||
/// fwcd's adapter resumes the VM while constructing its debuggee, before its breakpoint hooks
|
||||
/// exist, so only its link holds resumes back until the attach is done.
|
||||
pub(crate) fn link_options(adapter: &str) -> ide_jdwp::LinkOptions {
|
||||
ide_jdwp::LinkOptions { hold_resumes: adapter == ADAPTER }
|
||||
}
|
||||
|
||||
pub struct SourceMirror {
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
//! JetBrains kotlin-lsp's debugger: a DAP server inside the language server, reached through
|
||||
//! the `debugServer` port it opens once the project is imported.
|
||||
//!
|
||||
//! Known limits, with no guard here: an evaluate that calls a function with a breakpoint, right
|
||||
//! after a step over a suspension point, loses every later stop at that breakpoint; while
|
||||
//! stepping, breakpoints hit by other coroutines are skipped.
|
||||
|
||||
use ide_core::{IdeError, IdeResult};
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
pub const ADAPTER: &str = "kotlin-lsp";
|
||||
/// The `adapterID` its debug server expects in `initialize`.
|
||||
pub const DAP_ADAPTER_ID: &str = "intellij_jvm";
|
||||
|
||||
/// The path of a `file:` URI, as kotlin-lsp reports some sources (percent-encoded); `None` for
|
||||
/// anything else, including a URI naming another host.
|
||||
pub(crate) fn source_path(path: &str) -> Option<PathBuf> {
|
||||
let rest = path.strip_prefix("file:")?;
|
||||
let path = match rest.strip_prefix("//") {
|
||||
Some(authority) => {
|
||||
let slash = authority.find('/')?;
|
||||
if !matches!(&authority[..slash], "" | "localhost") {
|
||||
return None;
|
||||
}
|
||||
&authority[slash..]
|
||||
}
|
||||
None => rest,
|
||||
};
|
||||
let decoded = percent_encoding::percent_decode_str(path).decode_utf8().ok()?;
|
||||
// `/C:/p/App.kt` on Windows.
|
||||
let decoded = match decoded.strip_prefix('/') {
|
||||
Some(drive) if cfg!(windows) && drive.as_bytes().get(1) == Some(&b':') => drive,
|
||||
_ => &decoded,
|
||||
};
|
||||
Some(PathBuf::from(decoded))
|
||||
}
|
||||
|
||||
pub(crate) fn frame_name(name: &str) -> &str {
|
||||
name.strip_prefix("at ").unwrap_or(name)
|
||||
}
|
||||
|
||||
/// What kotlin-lsp's DAP server does unlike other adapters.
|
||||
///
|
||||
/// It never answers a request that carries a thread id from an earlier stop (K1 G4), so such a
|
||||
/// request would wait out `REQUEST_TIMEOUT`: [`Self::check_thread`] refuses it first. Only thread
|
||||
/// ids were measured; a stale `frameId` or `variablesReference` may wait the same way. A step
|
||||
/// ends with `reason: "breakpoint"`; a real breakpoint hit during a step is then reported as a
|
||||
/// step too, since the stop carries neither `hitBreakpointIds` nor a description.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct Quirks {
|
||||
step_pending: AtomicBool,
|
||||
/// The stop (a `Suspension` ticket) and the thread ids seen in it.
|
||||
threads: Mutex<(u64, HashSet<i64>)>,
|
||||
}
|
||||
|
||||
impl Quirks {
|
||||
/// Before `next`, `stepIn`, `stepOut` or a forced step.
|
||||
pub fn step_sent(&self) {
|
||||
self.step_pending.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
/// Before `continue` or run to cursor, or after a step request failed.
|
||||
pub fn resume_sent(&self) {
|
||||
self.step_pending.store(false, Ordering::Release);
|
||||
}
|
||||
|
||||
/// The reason to report for a `stopped` event.
|
||||
pub fn stop_reason(&self, reason: String) -> String {
|
||||
let stepped = self.step_pending.swap(false, Ordering::AcqRel);
|
||||
if stepped && reason == "breakpoint" { "step".into() } else { reason }
|
||||
}
|
||||
|
||||
/// Thread ids seen in stop `stop`: a `stopped` event's, or a `threads` reply to a request sent
|
||||
/// during that stop. A reply from an older stop changes nothing.
|
||||
pub fn saw_threads(&self, stop: u64, ids: impl IntoIterator<Item = i64>) {
|
||||
let mut threads = self.threads.lock();
|
||||
if stop > threads.0 {
|
||||
*threads = (stop, ids.into_iter().collect());
|
||||
} else if stop == threads.0 {
|
||||
threads.1.extend(ids);
|
||||
}
|
||||
}
|
||||
|
||||
/// Refuses a thread id that is not one of current stop `stop`'s, once some are known.
|
||||
pub fn check_thread(&self, stop: u64, suspended: bool, id: i64) -> IdeResult<()> {
|
||||
let threads = self.threads.lock();
|
||||
if suspended && threads.0 == stop && !threads.1.is_empty() && !threads.1.contains(&id) {
|
||||
return Err(IdeError::invalid(format!("thread {id} belongs to an earlier stop")));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::Path;
|
||||
|
||||
#[test]
|
||||
fn file_uris_become_paths() {
|
||||
assert_eq!(source_path("file:///p/a%20b/App.kt").as_deref(), Some(Path::new("/p/a b/App.kt")));
|
||||
assert_eq!(source_path("file://localhost/p/x.kt").as_deref(), Some(Path::new("/p/x.kt")));
|
||||
assert_eq!(source_path("/p/x.kt"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frame_names_lose_the_at_prefix() {
|
||||
assert_eq!(frame_name("at com.x.AppKt.main(App.kt:13)"), "com.x.AppKt.main(App.kt:13)");
|
||||
assert_eq!(frame_name("main"), "main");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_pending_step_turns_a_breakpoint_into_a_step() {
|
||||
let quirks = Quirks::default();
|
||||
quirks.step_sent();
|
||||
assert_eq!(quirks.stop_reason("breakpoint".into()), "step");
|
||||
assert_eq!(quirks.stop_reason("breakpoint".into()), "breakpoint");
|
||||
quirks.step_sent();
|
||||
quirks.resume_sent();
|
||||
assert_eq!(quirks.stop_reason("breakpoint".into()), "breakpoint");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn threads_are_checked_against_the_current_stop() {
|
||||
let quirks = Quirks::default();
|
||||
quirks.saw_threads(3, [18]);
|
||||
assert!(quirks.check_thread(3, true, 18).is_ok());
|
||||
assert!(quirks.check_thread(3, true, 6).is_err());
|
||||
assert!(quirks.check_thread(4, true, 6).is_ok(), "nothing known about stop 4 yet");
|
||||
assert!(quirks.check_thread(3, false, 6).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_older_threads_reply_does_not_change_a_newer_stop() {
|
||||
let quirks = Quirks::default();
|
||||
quirks.saw_threads(4, [2]);
|
||||
quirks.saw_threads(3, [1]);
|
||||
assert!(quirks.check_thread(4, true, 1).is_err());
|
||||
quirks.saw_threads(4, [7]);
|
||||
assert!(quirks.check_thread(4, true, 2).is_ok() && quirks.check_thread(4, true, 7).is_ok());
|
||||
quirks.saw_threads(5, [9]);
|
||||
assert!(quirks.check_thread(5, true, 2).is_err());
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ pub mod connection;
|
||||
pub mod jdwp;
|
||||
mod jvm_memory;
|
||||
pub mod kotlin;
|
||||
pub mod kotlin_lsp;
|
||||
mod session;
|
||||
|
||||
use adapter::{AdapterLauncher, AdapterSpec, ProcessLauncher};
|
||||
@@ -133,6 +134,8 @@ pub struct DebugCapabilities {
|
||||
/// The session has a JVM inspector: instances, fields and referrers while suspended. Can
|
||||
/// change during the session ([`ide_core::event::DebugEvent::Capabilities`]).
|
||||
pub jvm_instances: bool,
|
||||
/// The session can be asked for a heap dump ([`DebugManager::jvm_heap_dump`]).
|
||||
pub jvm_heap: bool,
|
||||
}
|
||||
|
||||
/// Result of [`DebugManager::redefine_classes`].
|
||||
@@ -576,6 +579,7 @@ pub(crate) fn capabilities_from(caps: &Value) -> DebugCapabilities {
|
||||
hot_code_replace: false,
|
||||
jvm_memory: false,
|
||||
jvm_instances: false,
|
||||
jvm_heap: false,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -584,6 +588,7 @@ pub(crate) fn session_capabilities(session: &Session) -> DebugCapabilities {
|
||||
caps.hot_code_replace = HOT_CODE_REPLACE_ADAPTERS.contains(&session.adapter.as_str());
|
||||
caps.jvm_memory = JVM_MEMORY_ADAPTERS.contains(&session.adapter.as_str());
|
||||
caps.jvm_instances = session.jvm_inspector.lock().is_some();
|
||||
caps.jvm_heap = JVM_MEMORY_ADAPTERS.contains(&session.adapter.as_str());
|
||||
caps
|
||||
}
|
||||
|
||||
@@ -606,6 +611,16 @@ fn unverified(enabled: impl Iterator<Item = bool>) -> Vec<BreakpointState> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A step request's outcome; a failed one leaves no step pending, so the next stop keeps its reason.
|
||||
fn step_settled(session: &Session, sent: Result<Value, DapError>) -> Result<Value, DapError> {
|
||||
if sent.is_err()
|
||||
&& let Some(quirks) = session.kotlin_lsp_quirks()
|
||||
{
|
||||
quirks.resume_sent();
|
||||
}
|
||||
sent
|
||||
}
|
||||
|
||||
/// Delve compiles the debuggee to `__debug_bin` in the session's cwd (on Windows
|
||||
/// `__debug_bin.exe` plus `__debug_bin.exe<digits>` in DAP mode) and removes it on a
|
||||
/// graceful exit — a shutdown it doesn't always reach, so the files pile up. Sweep
|
||||
@@ -977,6 +992,7 @@ impl DebugManager {
|
||||
arguments["projectRoot"] = json!(mirror.root());
|
||||
*s.kotlin_mirror.lock() = Some(mirror);
|
||||
}
|
||||
let is_kotlin_lsp = s.adapter == kotlin_lsp::ADAPTER;
|
||||
let caps = s
|
||||
.conn
|
||||
.request_timeout(
|
||||
@@ -984,7 +1000,7 @@ impl DebugManager {
|
||||
json!({
|
||||
"clientID": "openidea",
|
||||
"clientName": "OpenIDEA",
|
||||
"adapterID": s.adapter,
|
||||
"adapterID": if is_kotlin_lsp { kotlin_lsp::DAP_ADAPTER_ID } else { s.adapter.as_str() },
|
||||
"locale": "en",
|
||||
"linesStartAt1": true,
|
||||
"columnsStartAt1": true,
|
||||
@@ -994,7 +1010,8 @@ impl DebugManager {
|
||||
"supportsRunInTerminalRequest": true,
|
||||
"supportsStartDebuggingRequest": true,
|
||||
"supportsProgressReporting": true,
|
||||
"supportsInvalidatedEvent": false,
|
||||
// kotlin-lsp sends `invalidated` either way.
|
||||
"supportsInvalidatedEvent": is_kotlin_lsp,
|
||||
"supportsMemoryReferences": false,
|
||||
"supportsArgsCanBeInterpretedByShell": false,
|
||||
}),
|
||||
@@ -1004,7 +1021,7 @@ impl DebugManager {
|
||||
*s.caps.lock() = if caps.is_object() { caps } else { json!({}) };
|
||||
|
||||
// The Kotlin adapters ignore the resolved classpath and project runtime on launch.
|
||||
let request = if s.adapter == kotlin::ADAPTER && s.request == "launch" {
|
||||
let request = if kotlin::is_kotlin(&s.adapter) && s.request == "launch" {
|
||||
session::start_kotlin_debuggee(s, &mut arguments).await?;
|
||||
"attach"
|
||||
} else {
|
||||
@@ -1028,7 +1045,7 @@ impl DebugManager {
|
||||
jvm_memory::find_attached_jvm(s, port, finder).await;
|
||||
}
|
||||
}
|
||||
if s.adapter == kotlin::ADAPTER {
|
||||
if kotlin::is_kotlin(&s.adapter) {
|
||||
let host = arguments
|
||||
.get("hostName")
|
||||
.and_then(Value::as_str)
|
||||
@@ -1040,7 +1057,7 @@ impl DebugManager {
|
||||
.and_then(|p| u16::try_from(p).ok())
|
||||
.filter(|p| *p != 0)
|
||||
.ok_or_else(|| IdeError::invalid("Invalid Kotlin JDWP port"))?;
|
||||
let (link, port) = ide_jdwp::JdwpLink::start(host, port, kotlin::link_options()).await?;
|
||||
let (link, port) = ide_jdwp::JdwpLink::start(host, port, kotlin::link_options(&s.adapter)).await?;
|
||||
*s.jvm_inspector.lock() = Some(Arc::new(ide_jdwp::LinkInspector::new(link.clone())));
|
||||
jvm_memory::watch_link(s, &link);
|
||||
*s.jdwp_link.lock() = Some(link);
|
||||
@@ -1377,6 +1394,8 @@ impl DebugManager {
|
||||
|
||||
pub async fn threads(&self, s: DebugSessionId) -> IdeResult<Vec<ThreadInfo>> {
|
||||
let session = self.session(s)?;
|
||||
// Before sending: a stop that overtakes the reply must not take the reply's ids as its own.
|
||||
let stop = session.suspended.ticket();
|
||||
let body = session.conn.request_timeout("threads", Value::Null, REQUEST_TIMEOUT).await?;
|
||||
let threads: Vec<ThreadInfo> = body
|
||||
.get("threads")
|
||||
@@ -1390,11 +1409,15 @@ impl DebugManager {
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
if let Some(quirks) = session.kotlin_lsp_quirks() {
|
||||
quirks.saw_threads(stop, threads.iter().map(|t| t.id));
|
||||
}
|
||||
Ok(threads)
|
||||
}
|
||||
|
||||
pub async fn stack_trace(&self, s: DebugSessionId, thread_id: i64) -> IdeResult<Vec<StackFrame>> {
|
||||
let session = self.session(s)?;
|
||||
session.check_thread(thread_id)?;
|
||||
let body = session
|
||||
.conn
|
||||
.request_timeout(
|
||||
@@ -1403,11 +1426,15 @@ impl DebugManager {
|
||||
REQUEST_TIMEOUT,
|
||||
)
|
||||
.await?;
|
||||
let kotlin_lsp = session.kotlin_lsp_quirks().is_some();
|
||||
let frame = |f: &Value| {
|
||||
let mut frame = frame_from(f);
|
||||
if let Some(client) = frame.path.as_deref().and_then(|p| session.client_path(&p.to_string_lossy())) {
|
||||
frame.path = Some(client.into());
|
||||
}
|
||||
if kotlin_lsp {
|
||||
frame.name = kotlin_lsp::frame_name(&frame.name).to_owned();
|
||||
}
|
||||
frame
|
||||
};
|
||||
Ok(body.get("stackFrames").and_then(Value::as_array).map(|f| f.iter().map(frame).collect()).unwrap_or_default())
|
||||
@@ -1519,6 +1546,10 @@ impl DebugManager {
|
||||
|
||||
pub async fn resume(&self, s: DebugSessionId, thread_id: i64) -> IdeResult<()> {
|
||||
let session = self.session(s)?;
|
||||
session.check_thread(thread_id)?;
|
||||
if let Some(quirks) = session.kotlin_lsp_quirks() {
|
||||
quirks.resume_sent();
|
||||
}
|
||||
let ticket = session.suspended.ticket();
|
||||
let body = session.conn.request_timeout("continue", json!({ "threadId": thread_id }), REQUEST_TIMEOUT).await?;
|
||||
// DAP: a response without the flag continued every thread.
|
||||
@@ -1563,8 +1594,13 @@ impl DebugManager {
|
||||
StepKind::Out => "stepOut",
|
||||
};
|
||||
let session = self.session(s)?;
|
||||
session.check_thread(thread_id)?;
|
||||
if let Some(quirks) = session.kotlin_lsp_quirks() {
|
||||
quirks.step_sent();
|
||||
}
|
||||
let ticket = session.suspended.ticket();
|
||||
session.conn.request_timeout(command, json!({ "threadId": thread_id }), REQUEST_TIMEOUT).await?;
|
||||
let sent = session.conn.request_timeout(command, json!({ "threadId": thread_id }), REQUEST_TIMEOUT).await;
|
||||
step_settled(&session, sent)?;
|
||||
// A step runs its thread; other threads stopped by their own events stay stopped.
|
||||
session.suspended.resumed(ticket, thread_id, false);
|
||||
session.emit(ide_core::event::DebugEvent::Continued { thread_id, all_threads_continued: true });
|
||||
@@ -1583,6 +1619,8 @@ impl DebugManager {
|
||||
force: bool,
|
||||
) -> IdeResult<()> {
|
||||
let session = self.session(s)?;
|
||||
// Before touching breakpoints: a refused `continue` would leave them replaced.
|
||||
session.check_thread(thread_id)?;
|
||||
if force {
|
||||
self.suspend_breakpoints(&session, Some(&path)).await?;
|
||||
}
|
||||
@@ -1657,6 +1695,7 @@ impl DebugManager {
|
||||
/// Force Step Over: `next` with every breakpoint suspended until the step ends.
|
||||
pub async fn force_step_over(&self, s: DebugSessionId, thread_id: i64) -> IdeResult<()> {
|
||||
let session = self.session(s)?;
|
||||
session.check_thread(thread_id)?;
|
||||
self.suspend_breakpoints(&session, None).await?;
|
||||
self.step(s, thread_id, StepKind::Over).await
|
||||
}
|
||||
@@ -1664,11 +1703,16 @@ impl DebugManager {
|
||||
/// Step into a specific call of the current line (Smart Step Into).
|
||||
pub async fn step_in_target(&self, s: DebugSessionId, thread_id: i64, target_id: i64) -> IdeResult<()> {
|
||||
let session = self.session(s)?;
|
||||
session.check_thread(thread_id)?;
|
||||
if let Some(quirks) = session.kotlin_lsp_quirks() {
|
||||
quirks.step_sent();
|
||||
}
|
||||
let ticket = session.suspended.ticket();
|
||||
session
|
||||
let sent = session
|
||||
.conn
|
||||
.request_timeout("stepIn", json!({ "threadId": thread_id, "targetId": target_id }), REQUEST_TIMEOUT)
|
||||
.await?;
|
||||
.await;
|
||||
step_settled(&session, sent)?;
|
||||
session.suspended.resumed(ticket, thread_id, false);
|
||||
session.emit(ide_core::event::DebugEvent::Continued { thread_id, all_threads_continued: true });
|
||||
Ok(())
|
||||
@@ -1714,6 +1758,20 @@ impl DebugManager {
|
||||
jvm_memory::session_classes(session).await
|
||||
}
|
||||
|
||||
/// Writes a heap dump of the session's JVM to `dir/dump.hprof`, one class-table or dump
|
||||
/// request per session at a time. Anything but `Ready` deletes `dir`, unless `s` is not a
|
||||
/// session. On `cancel` the result is `Cancelled` at once, while a dump the JVM is writing
|
||||
/// still holds the session's turn until the JVM answers.
|
||||
pub async fn jvm_heap_dump(
|
||||
&self,
|
||||
s: DebugSessionId,
|
||||
dir: PathBuf,
|
||||
cancel: ide_core::CancellationToken,
|
||||
) -> IdeResult<JvmHeapOutcome> {
|
||||
let session = self.session(s)?;
|
||||
jvm_memory::session_heap_dump(session, dir, cancel).await
|
||||
}
|
||||
|
||||
/// Gives the session the backend for [`Self::jvm_memory_instances`] and the debugger's class
|
||||
/// counts; the UI is told to reload the session's capabilities.
|
||||
pub fn set_jvm_inspector(
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
use crate::adapter::AdapterSpec;
|
||||
use crate::connection::{Connection, DapError, Incoming};
|
||||
use crate::kotlin_lsp;
|
||||
use crate::{Breakpoint, BreakpointState, DataBreakpoint, DebugManager, FunctionBreakpoint, SessionOwner};
|
||||
use ide_core::event::DebugEvent;
|
||||
#[cfg(unix)]
|
||||
@@ -103,6 +104,8 @@ pub(crate) struct Session {
|
||||
pub jvm_located: watch::Sender<bool>,
|
||||
pub kotlin_mirror: Mutex<Option<crate::kotlin::SourceMirror>>,
|
||||
pub jvm_memory: tokio::sync::Mutex<crate::JvmMemoryState>,
|
||||
/// Used only by a kotlin-lsp session: see [`Session::kotlin_lsp_quirks`].
|
||||
pub(crate) quirks: kotlin_lsp::Quirks,
|
||||
}
|
||||
|
||||
/// Which threads of the program are suspended: each from its `stopped` event until a
|
||||
@@ -252,6 +255,19 @@ impl Session {
|
||||
jvm_located: watch::channel(false).0,
|
||||
kotlin_mirror: Mutex::new(None),
|
||||
jvm_memory: tokio::sync::Mutex::default(),
|
||||
quirks: kotlin_lsp::Quirks::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn kotlin_lsp_quirks(&self) -> Option<&kotlin_lsp::Quirks> {
|
||||
(self.adapter == kotlin_lsp::ADAPTER).then_some(&self.quirks)
|
||||
}
|
||||
|
||||
/// Call before sending a request for thread `id`.
|
||||
pub fn check_thread(&self, id: i64) -> ide_core::IdeResult<()> {
|
||||
match self.kotlin_lsp_quirks() {
|
||||
Some(quirks) => quirks.check_thread(self.suspended.ticket(), self.suspended.get(), id),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -283,6 +299,11 @@ impl Session {
|
||||
|
||||
/// A path from the adapter as the IDE knows it (undoes [`Session::adapter_path`]).
|
||||
pub fn client_path(&self, path: &str) -> Option<PathBuf> {
|
||||
if self.adapter == kotlin_lsp::ADAPTER
|
||||
&& let Some(path) = kotlin_lsp::source_path(path)
|
||||
{
|
||||
return Some(path);
|
||||
}
|
||||
if let Some(real) = self.kotlin_mirror.lock().as_ref().and_then(|m| m.client_path(Path::new(path))) {
|
||||
return Some(real);
|
||||
}
|
||||
@@ -697,7 +718,11 @@ async fn on_event(mgr: &Arc<DebugManager>, s: &Arc<Session>, event: &str, mut bo
|
||||
"stopped" => {
|
||||
let all = body.get("allThreadsStopped").and_then(Value::as_bool).unwrap_or(false);
|
||||
s.suspended.stopped(int("threadId"), all);
|
||||
let reason = str_("reason").unwrap_or_else(|| "pause".into());
|
||||
let mut reason = str_("reason").unwrap_or_else(|| "pause".into());
|
||||
if let Some(quirks) = s.kotlin_lsp_quirks() {
|
||||
quirks.saw_threads(s.suspended.ticket(), int("threadId"));
|
||||
reason = quirks.stop_reason(reason);
|
||||
}
|
||||
mgr.restore_breakpoints(s).await;
|
||||
s.emit(DebugEvent::Stopped {
|
||||
thread_id: int("threadId").unwrap_or(0),
|
||||
@@ -799,6 +824,14 @@ async fn on_event(mgr: &Arc<DebugManager>, s: &Arc<Session>, event: &str, mut bo
|
||||
change_type: crate::hot_code_replace_type(body.get("changeType")),
|
||||
message: str_("message").unwrap_or_default(),
|
||||
}),
|
||||
// Sent only by adapters told `supportsInvalidatedEvent` (kotlin-lsp).
|
||||
"invalidated" => s.emit(DebugEvent::Invalidated {
|
||||
areas: body
|
||||
.get("areas")
|
||||
.and_then(Value::as_array)
|
||||
.map(|a| a.iter().filter_map(Value::as_str).map(str::to_owned).collect())
|
||||
.unwrap_or_default(),
|
||||
}),
|
||||
_ => tracing::trace!("dap: ignored event {event}"),
|
||||
}
|
||||
}
|
||||
|
||||
+288
-2
@@ -126,8 +126,22 @@ struct Script {
|
||||
continued_after_stop: bool,
|
||||
/// Close the connection when a `pause` request arrives (the adapter dies).
|
||||
hang_up_on_pause: bool,
|
||||
/// Close the connection at the first request after the first `stopped` event.
|
||||
hang_up_after_stop: bool,
|
||||
/// Answer `continue` with this body and keep the program running, instead of letting it exit.
|
||||
continue_reply: Option<Value>,
|
||||
/// Answer `continue` and stop at a breakpoint again, instead of letting the program exit.
|
||||
stop_after_continue: bool,
|
||||
/// The `reason` of the stop a `next` ends with (`step` when unset; kotlin-lsp says `breakpoint`).
|
||||
step_reason: Option<&'static str>,
|
||||
/// Refuse `next`, then stop at a breakpoint (one the program hit meanwhile).
|
||||
refuse_next: bool,
|
||||
/// Send `invalidated { areas: ["variables"] }` right after the first `stopped` event.
|
||||
invalidated_after_stop: bool,
|
||||
/// On `threads`: stop again on thread 2 first, then answer with thread 1 of the earlier stop.
|
||||
stop_during_threads: bool,
|
||||
/// Name of the first stack frame (`main` when unset).
|
||||
frame_name: Option<&'static str>,
|
||||
/// Answer `initialize` only after this long (a slow adapter keeps the session connecting).
|
||||
delay_initialize: Option<Duration>,
|
||||
initialized_delay: Duration,
|
||||
@@ -156,7 +170,14 @@ impl Default for Script {
|
||||
hold_set_breakpoints: 0,
|
||||
continued_after_stop: false,
|
||||
hang_up_on_pause: false,
|
||||
hang_up_after_stop: false,
|
||||
continue_reply: None,
|
||||
stop_after_continue: false,
|
||||
step_reason: None,
|
||||
refuse_next: false,
|
||||
invalidated_after_stop: false,
|
||||
stop_during_threads: false,
|
||||
frame_name: None,
|
||||
delay_initialize: None,
|
||||
initialized_delay: Duration::ZERO,
|
||||
launch_delay: Duration::ZERO,
|
||||
@@ -202,6 +223,7 @@ async fn fake_adapter<S: AsyncRead + AsyncWrite + Unpin + Send + 'static>(stream
|
||||
let mut seq = 1000;
|
||||
let mut pending_launch: Option<i64> = None;
|
||||
let mut held: Vec<Value> = Vec::new();
|
||||
let mut stopped = false;
|
||||
let mut running = false;
|
||||
macro_rules! send {
|
||||
($v:expr) => {{
|
||||
@@ -222,6 +244,9 @@ async fn fake_adapter<S: AsyncRead + AsyncWrite + Unpin + Send + 'static>(stream
|
||||
if msg["type"] == "response" {
|
||||
continue;
|
||||
}
|
||||
if script.hang_up_after_stop && stopped {
|
||||
break;
|
||||
}
|
||||
match msg["command"].as_str().unwrap_or_default() {
|
||||
"initialize" => {
|
||||
if let Some(delay) = script.delay_initialize {
|
||||
@@ -320,9 +345,18 @@ async fn fake_adapter<S: AsyncRead + AsyncWrite + Unpin + Send + 'static>(stream
|
||||
ok!(msg, Value::Null);
|
||||
send!(json!({"type": "event", "event": "stopped", "body": {"reason": "restart", "threadId": 1}}));
|
||||
}
|
||||
"next" if script.refuse_next => {
|
||||
send!(
|
||||
json!({"type": "response", "request_seq": msg["seq"], "command": "next", "success": false, "message": "refused"})
|
||||
);
|
||||
// Lets the client handle the refusal before the stop.
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
send!(json!({"type": "event", "event": "stopped", "body": {"reason": "breakpoint", "threadId": 1}}));
|
||||
}
|
||||
"next" => {
|
||||
ok!(msg, Value::Null);
|
||||
send!(json!({"type": "event", "event": "stopped", "body": {"reason": "step", "threadId": 1}}));
|
||||
let reason = script.step_reason.unwrap_or("step");
|
||||
send!(json!({"type": "event", "event": "stopped", "body": {"reason": reason, "threadId": 1}}));
|
||||
}
|
||||
"setVariable" => {
|
||||
ok!(msg, json!({"value": format!("{}!", msg["arguments"]["value"].as_str().unwrap()), "type": "int"}))
|
||||
@@ -358,6 +392,12 @@ async fn fake_adapter<S: AsyncRead + AsyncWrite + Unpin + Send + 'static>(stream
|
||||
send!(
|
||||
json!({"type": "event", "event": "stopped", "body": {"reason": "breakpoint", "threadId": 1, "allThreadsStopped": true}})
|
||||
);
|
||||
stopped = true;
|
||||
if script.invalidated_after_stop {
|
||||
send!(
|
||||
json!({"type": "event", "event": "invalidated", "body": {"areas": ["variables"], "threadId": 1}})
|
||||
);
|
||||
}
|
||||
if script.continued_after_stop {
|
||||
send!(json!({"type": "event", "event": "continued", "body": {"threadId": 1}}));
|
||||
}
|
||||
@@ -365,11 +405,19 @@ async fn fake_adapter<S: AsyncRead + AsyncWrite + Unpin + Send + 'static>(stream
|
||||
json!({"type": "event", "event": "breakpoint", "body": {"reason": "changed", "breakpoint": {"id": 1, "verified": false, "message": "moved", "line": 9, "source": {"path": script.source_path}}}})
|
||||
);
|
||||
}
|
||||
"threads" if script.stop_during_threads => {
|
||||
send!(
|
||||
json!({"type": "event", "event": "stopped", "body": {"reason": "breakpoint", "threadId": 2, "allThreadsStopped": true}})
|
||||
);
|
||||
// Lets the client handle the stop before it reads the reply.
|
||||
tokio::time::sleep(Duration::from_millis(50)).await;
|
||||
ok!(msg, json!({"threads": [{"id": 1, "name": "\"main\"@1"}]}));
|
||||
}
|
||||
"threads" => ok!(msg, json!({"threads": [{"id": 1, "name": "MainThread"}]})),
|
||||
"stackTrace" => ok!(
|
||||
msg,
|
||||
json!({"stackFrames": [
|
||||
{"id": 10, "name": "main", "line": 5, "column": 3, "source": {"path": script.source_path.as_deref().unwrap_or("/p/a.py")}},
|
||||
{"id": 10, "name": script.frame_name.unwrap_or("main"), "line": 5, "column": 3, "source": {"path": script.source_path.as_deref().unwrap_or("/p/a.py")}},
|
||||
{"id": 11, "name": "lib", "line": 1, "column": 1, "source": {"path": "/usr/lib/x.py", "presentationHint": "deemphasize"}},
|
||||
]})
|
||||
),
|
||||
@@ -393,6 +441,12 @@ async fn fake_adapter<S: AsyncRead + AsyncWrite + Unpin + Send + 'static>(stream
|
||||
]})
|
||||
),
|
||||
"continue" if script.continue_reply.is_some() => ok!(msg, script.continue_reply.clone().unwrap()),
|
||||
"continue" if script.stop_after_continue => {
|
||||
ok!(msg, json!({"allThreadsContinued": true}));
|
||||
send!(
|
||||
json!({"type": "event", "event": "stopped", "body": {"reason": "breakpoint", "threadId": 1, "allThreadsStopped": true}})
|
||||
);
|
||||
}
|
||||
"continue" => {
|
||||
ok!(msg, json!({"allThreadsContinued": true}));
|
||||
send!(json!({"type": "event", "event": "exited", "body": {"exitCode": 3}}));
|
||||
@@ -2155,11 +2209,13 @@ async fn attach_session_reports_capability() {
|
||||
let id = h.start(remote_attach(adapter, project.path())).await.unwrap();
|
||||
let caps = h.manager.capabilities(id).unwrap();
|
||||
assert!(caps.jvm_memory, "{adapter}");
|
||||
assert_eq!(caps.jvm_heap, caps.jvm_memory, "{adapter}");
|
||||
}
|
||||
|
||||
let id = h.start(config()).await.unwrap();
|
||||
let caps = h.manager.capabilities(id).unwrap();
|
||||
assert!(!caps.jvm_memory);
|
||||
assert_eq!(caps.jvm_heap, caps.jvm_memory);
|
||||
assert_eq!(
|
||||
h.manager.jvm_memory_classes(id).await.unwrap(),
|
||||
JvmMemoryResult::Unavailable { reason: JvmMemoryUnavailable::NoJvmProcess, can_count_when_paused: false }
|
||||
@@ -2399,6 +2455,13 @@ fn kotlin_attach(project: &std::path::Path, host: &str, port: u16) -> LaunchConf
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn link_options_hold_only_for_fwcd() {
|
||||
assert!(kotlin::link_options(kotlin::ADAPTER).hold_resumes);
|
||||
assert!(!kotlin::link_options(kotlin_lsp::ADAPTER).hold_resumes);
|
||||
assert!(!kotlin::link_options("x").hold_resumes);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kotlin_session_installs_link_inspector() {
|
||||
let h = harness(Script::default());
|
||||
@@ -2470,6 +2533,229 @@ async fn closing_the_session_drops_the_link() {
|
||||
assert!(session.jvm_inspector.lock().is_none());
|
||||
}
|
||||
|
||||
/// kotlin-lsp's debugger, reached through the language server's `debugServer` port (the fake ignores it).
|
||||
fn kotlin_lsp_attach(project: &std::path::Path) -> LaunchConfig {
|
||||
LaunchConfig {
|
||||
adapter: kotlin_lsp::ADAPTER.into(),
|
||||
request: "attach".into(),
|
||||
cwd: Some(project.to_owned().into()),
|
||||
extra_json: Some(
|
||||
json!({"debugServer": 4711, "hostName": "127.0.0.1", "port": 5005, "projectRoot": project}).to_string(),
|
||||
),
|
||||
..config()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kotlin_lsp_initializes_as_intellij_jvm() {
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
for (config, adapter_id, invalidated) in [
|
||||
(kotlin_lsp_attach(project.path()), "intellij_jvm", true),
|
||||
(kotlin_attach(project.path(), "127.0.0.1", 5005), "kotlin-debug-adapter", false),
|
||||
] {
|
||||
let h = harness(Script::default());
|
||||
let id = h.start(config).await.unwrap();
|
||||
let init = &h.request("initialize")["arguments"];
|
||||
assert_eq!(init["adapterID"], adapter_id);
|
||||
assert_eq!(init["supportsInvalidatedEvent"], invalidated, "{adapter_id}");
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kotlin_lsp_gets_real_paths_and_its_link_holds_nothing() {
|
||||
let h = harness(Script::default());
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let path = project.path().join("Main.kt");
|
||||
std::fs::write(&path, "package org.example\nfun main() {}\n").unwrap();
|
||||
h.manager.set_breakpoints(path.clone(), vec![bp(1)]).await.unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
assert_eq!(h.request("setBreakpoints")["arguments"]["source"]["path"], json!(path));
|
||||
let sent = &h.request("attach")["arguments"];
|
||||
assert_eq!(sent["projectRoot"], json!(project.path()));
|
||||
assert!(sent.get("debugServer").is_none());
|
||||
assert_eq!(sent["hostName"], "127.0.0.1");
|
||||
let link_port = sent["port"].as_u64().unwrap();
|
||||
assert!(link_port != 0 && link_port != 5005);
|
||||
let session = h.manager.session(id).unwrap();
|
||||
assert!(session.kotlin_mirror.lock().is_none());
|
||||
assert!(session.jdwp_link.lock().is_some() && session.jvm_inspector.lock().is_some());
|
||||
assert_eq!(session.attach_target.get(), Some(&("127.0.0.1".to_owned(), 5005)));
|
||||
assert!(!kotlin::link_options(&session.adapter).hold_resumes);
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kotlin_lsp_waits_for_the_configuration_done_reply() {
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
// fwcd never answers it (`kotlin_configures_without_configuration_done_response`).
|
||||
let h = harness(Script { ignore_configuration_done: true, ..Default::default() });
|
||||
let fwcd = h.start(kotlin_attach(project.path(), "127.0.0.1", 5005));
|
||||
let id = tokio::time::timeout(Duration::from_secs(2), fwcd).await.unwrap().unwrap();
|
||||
h.manager.stop(id).await.unwrap();
|
||||
|
||||
tokio::time::pause();
|
||||
let h = harness(Script { ignore_configuration_done: true, ..Default::default() });
|
||||
let started = tokio::time::Instant::now();
|
||||
let err = h.start(kotlin_lsp_attach(project.path())).await.unwrap_err();
|
||||
assert!(started.elapsed() >= REQUEST_TIMEOUT, "{:?}", started.elapsed());
|
||||
assert!(err.message.contains("configurationDone"), "{err:?}");
|
||||
assert!(h.manager.sessions(None).is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kotlin_lsp_sessions_have_the_memory_tab() {
|
||||
let h = harness(Script::default());
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
assert!(h.manager.capabilities(id).unwrap().jvm_memory);
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn losing_the_kotlin_lsp_connection_ends_the_session() {
|
||||
let h = harness(Script { hang_up_after_stop: true, ..Script::default() });
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
let session = h.manager.session(id).unwrap();
|
||||
assert!(session.jdwp_link.lock().is_some());
|
||||
let _ = session.conn.request_timeout("threads", Value::Null, Duration::from_secs(2)).await;
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Terminated)).await;
|
||||
assert!(h.manager.sessions(None).is_empty());
|
||||
assert!(session.jdwp_link.lock().is_none());
|
||||
assert!(session.jvm_inspector.lock().is_none());
|
||||
}
|
||||
|
||||
/// The reasons of the first `count` stops, once that many arrived.
|
||||
async fn stop_reasons(h: &Harness, count: usize) -> Vec<String> {
|
||||
for _ in 0..200 {
|
||||
let reasons: Vec<String> = h
|
||||
.events
|
||||
.lock()
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
IdeEvent::Debug { event: DebugEvent::Stopped { reason, .. }, .. } => Some(reason.clone()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
if reasons.len() >= count {
|
||||
return reasons;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
panic!("{count} stops not received; got {:?}", h.events.lock());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_step_stop_is_reported_as_a_step() {
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
for (config, reason) in [(kotlin_lsp_attach(project.path()), "step"), (config(), "breakpoint")] {
|
||||
let h = harness(Script { step_reason: Some("breakpoint"), ..Script::default() });
|
||||
let id = h.start(config).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
h.manager.step(id, 1, StepKind::Over).await.unwrap();
|
||||
assert_eq!(stop_reasons(&h, 2).await, ["breakpoint", reason]);
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_breakpoint_after_continue_stays_a_breakpoint() {
|
||||
let h = harness(Script { step_reason: Some("breakpoint"), stop_after_continue: true, ..Script::default() });
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
h.manager.step(id, 1, StepKind::Over).await.unwrap();
|
||||
stop_reasons(&h, 2).await;
|
||||
// The fake ends `stepIn` with no stop: the `continue` after it decides the next one.
|
||||
h.manager.step(id, 1, StepKind::Into).await.unwrap();
|
||||
h.manager.resume(id, 1).await.unwrap();
|
||||
assert_eq!(stop_reasons(&h, 3).await, ["breakpoint", "step", "breakpoint"]);
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_breakpoint_after_a_refused_step_stays_a_breakpoint() {
|
||||
let h = harness(Script { refuse_next: true, ..Script::default() });
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
assert!(h.manager.step(id, 1, StepKind::Over).await.is_err());
|
||||
assert_eq!(stop_reasons(&h, 2).await, ["breakpoint", "breakpoint"]);
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_thread_from_an_earlier_stop_is_refused() {
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
for (config, kotlin_lsp) in [(kotlin_lsp_attach(project.path()), true), (config(), false)] {
|
||||
let h = harness(Script::default());
|
||||
let id = h.start(config).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
assert_eq!(h.manager.threads(id).await.unwrap().len(), 1);
|
||||
let started = tokio::time::Instant::now();
|
||||
let stale = h.manager.stack_trace(id, 99).await;
|
||||
let asked =
|
||||
|command: &str| h.log.lock().iter().any(|m| m["command"] == command && m["arguments"]["threadId"] == 99);
|
||||
if kotlin_lsp {
|
||||
assert!(stale.is_err() && started.elapsed() < Duration::from_millis(100), "{stale:?}");
|
||||
assert!(h.manager.step(id, 99, StepKind::Over).await.is_err());
|
||||
assert!(h.manager.resume(id, 99).await.is_err());
|
||||
assert!(!asked("stackTrace") && !asked("next") && !asked("continue"));
|
||||
} else {
|
||||
assert!(stale.is_ok() && asked("stackTrace"), "{stale:?}");
|
||||
}
|
||||
assert!(h.manager.stack_trace(id, 1).await.is_ok());
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn a_threads_reply_from_before_a_stop_does_not_vouch_for_its_ids() {
|
||||
let h = harness(Script { stop_during_threads: true, ..Script::default() });
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
let threads = h.manager.threads(id).await.unwrap();
|
||||
// M3 maps kotlin-lsp's threads by these names.
|
||||
assert_eq!((threads[0].id, threads[0].name.as_str()), (1, "\"main\"@1"));
|
||||
stop_reasons(&h, 2).await;
|
||||
assert!(h.manager.stack_trace(id, 1).await.is_err());
|
||||
assert!(h.manager.stack_trace(id, 2).await.is_ok());
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn kotlin_lsp_frames_get_real_paths() {
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let at = "at com.x.AppKt.main(App.kt:13)";
|
||||
for (config, path, name) in [
|
||||
(kotlin_lsp_attach(project.path()), "/p/App.kt", "com.x.AppKt.main(App.kt:13)"),
|
||||
(java_config("launch"), "file:///p/App.kt", at),
|
||||
] {
|
||||
let script = Script { source_path: Some("file:///p/App.kt".into()), frame_name: Some(at), ..Script::default() };
|
||||
let h = harness(script);
|
||||
let id = h.start(config).await.unwrap();
|
||||
h.wait_event(|e| matches!(e, DebugEvent::Stopped { .. })).await;
|
||||
let frames = h.manager.stack_trace(id, 1).await.unwrap();
|
||||
assert_eq!(frames[0].path.as_ref().map(|p| p.0.clone()), Some(PathBuf::from(path)));
|
||||
assert_eq!(frames[0].name, name);
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn invalidated_events_reach_the_client() {
|
||||
let h = harness(Script { invalidated_after_stop: true, ..Script::default() });
|
||||
let project = tempfile::tempdir().unwrap();
|
||||
let id = h.start(kotlin_lsp_attach(project.path())).await.unwrap();
|
||||
match h.wait_event(|e| matches!(e, DebugEvent::Invalidated { .. })).await {
|
||||
DebugEvent::Invalidated { areas } => assert_eq!(areas, ["variables"]),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
h.manager.stop(id).await.unwrap();
|
||||
}
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn rdbg_drains_test_output_after_dap_closes_before_killing_the_process() {
|
||||
|
||||
@@ -136,24 +136,19 @@ pub struct DockerRegistry {
|
||||
pub username: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DockerSettings {
|
||||
#[serde(default = "default_connections")]
|
||||
#[serde(default)]
|
||||
pub connections: Vec<DockerConnection>,
|
||||
#[serde(default)]
|
||||
pub registries: Vec<DockerRegistry>,
|
||||
}
|
||||
|
||||
impl Default for DockerSettings {
|
||||
fn default() -> Self {
|
||||
Self { connections: default_connections(), registries: Vec::new() }
|
||||
}
|
||||
}
|
||||
|
||||
/// Before anything is configured the CLI's own daemon is shown, as before connections existed.
|
||||
fn default_connections() -> Vec<DockerConnection> {
|
||||
vec![DockerConnection { id: "default".into(), name: "Docker".into(), endpoint: DockerEndpoint::Default }]
|
||||
/// Whether `c` is the connection older versions injected when nothing was configured:
|
||||
/// Services lists only connections the user added, so this one is dropped on load.
|
||||
fn is_injected_default(c: &DockerConnection) -> bool {
|
||||
c.id == "default" && c.name == "Docker" && c.endpoint == DockerEndpoint::Default
|
||||
}
|
||||
|
||||
const SETTINGS_FILE: &str = "docker.json";
|
||||
@@ -161,10 +156,17 @@ const PASSWORDS_FILE: &str = "docker-passwords.json";
|
||||
|
||||
pub fn load_settings(data_dir: &Path) -> DockerSettings {
|
||||
match std::fs::read(data_dir.join(SETTINGS_FILE)) {
|
||||
Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_else(|e| {
|
||||
tracing::warn!("{SETTINGS_FILE}: {e}");
|
||||
DockerSettings::default()
|
||||
}),
|
||||
// A file written before the opt-in Services rework carries the injected default
|
||||
// (saved along with a registry or another connection); drop it unless edited.
|
||||
Ok(bytes) => serde_json::from_slice(&bytes)
|
||||
.map(|mut s: DockerSettings| {
|
||||
s.connections.retain(|c| !is_injected_default(c));
|
||||
s
|
||||
})
|
||||
.unwrap_or_else(|e| {
|
||||
tracing::warn!("{SETTINGS_FILE}: {e}");
|
||||
DockerSettings::default()
|
||||
}),
|
||||
Err(_) => DockerSettings::default(),
|
||||
}
|
||||
}
|
||||
@@ -373,7 +375,8 @@ mod tests {
|
||||
fn settings_round_trip_and_passwords() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let dir = tmp.path();
|
||||
assert_eq!(load_settings(dir).connections[0].endpoint, DockerEndpoint::Default);
|
||||
// Nothing configured: no connection is invented.
|
||||
assert!(load_settings(dir).connections.is_empty());
|
||||
let settings = DockerSettings {
|
||||
connections: vec![DockerConnection {
|
||||
id: "c1".into(),
|
||||
@@ -395,8 +398,32 @@ mod tests {
|
||||
assert!(auth_for(dir, "postgres").is_none());
|
||||
save_settings(dir, &DockerSettings { registries: Vec::new(), ..settings }).unwrap();
|
||||
assert_eq!(registry_password(dir, "r1"), None);
|
||||
// An empty list stays empty (the default connection is only for a missing file).
|
||||
// An empty list stays empty.
|
||||
save_settings(dir, &DockerSettings { connections: Vec::new(), registries: Vec::new() }).unwrap();
|
||||
assert!(load_settings(dir).connections.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_drops_the_injected_default_connection() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let dir = tmp.path();
|
||||
let injected =
|
||||
|| DockerConnection { id: "default".into(), name: "Docker".into(), endpoint: DockerEndpoint::Default };
|
||||
let registry =
|
||||
DockerRegistry { id: "r1".into(), name: "GHCR".into(), address: "ghcr.io".into(), username: "me".into() };
|
||||
// Saved untouched by an older version (e.g. along with a registry): dropped on load.
|
||||
save_settings(dir, &DockerSettings { connections: vec![injected()], registries: vec![registry.clone()] })
|
||||
.unwrap();
|
||||
let loaded = load_settings(dir);
|
||||
assert!(loaded.connections.is_empty());
|
||||
assert_eq!(loaded.registries, vec![registry]);
|
||||
// A connection the user renamed or re-pointed stays theirs.
|
||||
let renamed = DockerConnection { name: "Local engine".into(), ..injected() };
|
||||
save_settings(dir, &DockerSettings { connections: vec![renamed.clone()], registries: Vec::new() }).unwrap();
|
||||
assert_eq!(load_settings(dir).connections, vec![renamed]);
|
||||
let repointed =
|
||||
DockerConnection { endpoint: DockerEndpoint::Socket { path: "/var/run/docker.sock".into() }, ..injected() };
|
||||
save_settings(dir, &DockerSettings { connections: vec![repointed.clone()], registries: Vec::new() }).unwrap();
|
||||
assert_eq!(load_settings(dir).connections, vec![repointed]);
|
||||
}
|
||||
}
|
||||
@@ -71,7 +71,6 @@ impl Capability {
|
||||
match self {
|
||||
Self::ProjectRead | Self::EditorRead | Self::Storage | Self::Events | Self::VcsRead => Risk::Low,
|
||||
Self::EditorWrite
|
||||
| Self::ProjectWrite
|
||||
| Self::DatabaseSources
|
||||
| Self::VcsWrite
|
||||
| Self::Net(_)
|
||||
@@ -80,7 +79,14 @@ impl Capability {
|
||||
| Self::Secrets
|
||||
| Self::InputEvents
|
||||
| Self::Clipboard => Risk::Medium,
|
||||
Self::IdeCommands | Self::Ipc(_) | Self::Process(_) | Self::FsRead(_) | Self::FsWrite(_) => Risk::High,
|
||||
// A trusted project's own files name the programs the IDE starts (its settings, its
|
||||
// build scripts), so writing them is running code.
|
||||
Self::ProjectWrite
|
||||
| Self::IdeCommands
|
||||
| Self::Ipc(_)
|
||||
| Self::Process(_)
|
||||
| Self::FsRead(_)
|
||||
| Self::FsWrite(_) => Risk::High,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -402,6 +408,15 @@ mod tests {
|
||||
assert!(Capability::Storage.auto_migrates() && !Capability::EditorWrite.auto_migrates());
|
||||
}
|
||||
|
||||
/// A trusted project's files name programs the IDE starts by itself: the git and language
|
||||
/// server executables in `.openidea/settings.toml`, its build scripts. Writing them is
|
||||
/// running code, and the install dialog says so.
|
||||
#[test]
|
||||
fn writing_project_files_is_a_high_risk_grant() {
|
||||
assert_eq!(Capability::ProjectWrite.risk(), Risk::High);
|
||||
assert!(!Capability::ProjectWrite.auto_migrates());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn host_patterns() {
|
||||
assert!(host_matches("crates.io", "crates.io"));
|
||||
|
||||
@@ -352,6 +352,22 @@ pub struct IndexEntry {
|
||||
/// the IDE before showing it.
|
||||
#[serde(default)]
|
||||
pub icon: Option<PluginIcon>,
|
||||
/// Where to read more: the manifest's homepage, else the registry's source repository.
|
||||
#[serde(default)]
|
||||
pub homepage: Option<String>,
|
||||
/// This version's release notes (the manifest's `change-notes`).
|
||||
#[serde(default)]
|
||||
pub notes: Option<String>,
|
||||
/// Downloads of the plugin's packages, as counted by whoever serves them.
|
||||
#[serde(default)]
|
||||
pub downloads: Option<u64>,
|
||||
/// Picked by the marketplace's maintainers.
|
||||
#[serde(default)]
|
||||
pub featured: bool,
|
||||
/// Project-relative paths whose presence makes the plugin worth suggesting
|
||||
/// (`pom.xml`, `config/routes.rb`).
|
||||
#[serde(default)]
|
||||
pub suggest: Vec<String>,
|
||||
}
|
||||
|
||||
impl IndexEntry {
|
||||
@@ -389,10 +405,31 @@ impl Index {
|
||||
/// revocations, publisher keys (revoked ones) and `moved` — and lists no
|
||||
/// plugins ([`Index::is_newer`]): the IDE needs updating to install from it.
|
||||
pub fn parse(text: &str) -> Result<Self, String> {
|
||||
let (index, dropped) = Self::parse_dropping(text)?;
|
||||
if let Some(first) = dropped.first() {
|
||||
// One line per index, and a bounded one: whoever wrote the index chose the text.
|
||||
let id: String = first.id.chars().take(80).collect();
|
||||
tracing::warn!(
|
||||
"index: {} entries are not offered, their id or version is not one a manifest may carry (first: {id:?})",
|
||||
dropped.len()
|
||||
);
|
||||
}
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
/// [`Index::parse`], with the entries it does not offer: those whose id or
|
||||
/// version no manifest may carry, so no IDE could install them.
|
||||
pub fn parse_dropping(text: &str) -> Result<(Self, Vec<IndexEntry>), String> {
|
||||
let value: serde_json::Value = serde_json::from_str(text).map_err(|e| format!("index: {e}"))?;
|
||||
let format = value.get("format").and_then(serde_json::Value::as_u64).ok_or("index: `format` is missing")?;
|
||||
if format == u64::from(INDEX_FORMAT) {
|
||||
return serde_json::from_value(value).map_err(|e| format!("index: {e}"));
|
||||
let mut index: Index = serde_json::from_value(value).map_err(|e| format!("index: {e}"))?;
|
||||
// What no manifest may carry is not offered: the id and version also name files on disk.
|
||||
let (offered, dropped) = std::mem::take(&mut index.plugins)
|
||||
.into_iter()
|
||||
.partition(|e| crate::validate_plugin_id(&e.id).is_ok() && e.semver().is_some());
|
||||
index.plugins = offered;
|
||||
return Ok((index, dropped));
|
||||
}
|
||||
if format < u64::from(INDEX_FORMAT) {
|
||||
return Err(format!("index format {format} is not supported (this IDE reads {INDEX_FORMAT})"));
|
||||
@@ -401,13 +438,14 @@ impl Index {
|
||||
let items = value.get(field).and_then(serde_json::Value::as_array);
|
||||
items.into_iter().flatten().filter_map(|v| serde_json::from_value(v.clone()).ok()).collect()
|
||||
}
|
||||
Ok(Index {
|
||||
let index = Index {
|
||||
format: u32::try_from(format).unwrap_or(u32::MAX),
|
||||
publishers: each(&value, "publishers"),
|
||||
plugins: Vec::new(),
|
||||
revoked: each(&value, "revoked"),
|
||||
moved: value.get("moved").and_then(serde_json::Value::as_str).map(str::to_owned),
|
||||
})
|
||||
};
|
||||
Ok((index, Vec::new()))
|
||||
}
|
||||
|
||||
/// Written in a format newer than this IDE reads: it lists no plugins.
|
||||
@@ -685,9 +723,33 @@ mod tests {
|
||||
engine: "^1.0".into(),
|
||||
categories: Vec::new(),
|
||||
icon: None,
|
||||
homepage: None,
|
||||
notes: None,
|
||||
downloads: None,
|
||||
featured: false,
|
||||
suggest: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listing_details_are_optional() {
|
||||
let mut json = serde_json::to_value(entry("acme.a", "acme")).unwrap();
|
||||
for key in ["homepage", "notes", "downloads", "featured", "suggest"] {
|
||||
json.as_object_mut().unwrap().remove(key);
|
||||
}
|
||||
assert_eq!(serde_json::from_value::<IndexEntry>(json).unwrap(), entry("acme.a", "acme"));
|
||||
let full = IndexEntry {
|
||||
homepage: Some("https://example.com".into()),
|
||||
notes: Some("Fixes".into()),
|
||||
downloads: Some(42),
|
||||
featured: true,
|
||||
suggest: vec!["pom.xml".into()],
|
||||
..entry("acme.a", "acme")
|
||||
};
|
||||
let back: IndexEntry = serde_json::from_str(&serde_json::to_string(&full).unwrap()).unwrap();
|
||||
assert_eq!(back, full);
|
||||
}
|
||||
|
||||
fn version(id: &str, version: &str, engine: &str) -> IndexEntry {
|
||||
IndexEntry { version: version.into(), engine: engine.into(), ..entry(id, "acme") }
|
||||
}
|
||||
@@ -868,6 +930,51 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_index_offers_only_the_ids_and_versions_a_manifest_may_carry() {
|
||||
let invalid_ids = [
|
||||
"../x",
|
||||
"../../escape",
|
||||
"acme/../x",
|
||||
r"acme.\..\x",
|
||||
"acme./../../../escape",
|
||||
"/abs",
|
||||
r"C:\x",
|
||||
"ACME.x",
|
||||
"acme.",
|
||||
"acme..x",
|
||||
"acme.x y",
|
||||
"",
|
||||
];
|
||||
let invalid_versions =
|
||||
["../1.0.0", "1.0.0/../../../escape", r"1.0.0\..\x", "1.0.0-..", " 1.0.0", "1.0", "latest", ""];
|
||||
let valid = [("acme.x", "1.0.0"), ("acme.tools-2_b", "1.0.0-rc.1+b.2")];
|
||||
let mut listed: Vec<(&str, &str)> = valid.to_vec();
|
||||
listed.extend(invalid_ids.iter().map(|id| (*id, "1.0.0")));
|
||||
listed.extend(invalid_versions.iter().map(|version| ("acme.x", *version)));
|
||||
let plugins: Vec<serde_json::Value> = listed
|
||||
.iter()
|
||||
.map(|(id, version)| {
|
||||
serde_json::json!({
|
||||
"id": id, "name": "X", "version": version, "publisher": "acme",
|
||||
"download": "x.oiplugin", "digest": "d", "engine": "^1.0",
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let text = serde_json::json!({ "format": INDEX_FORMAT, "plugins": plugins }).to_string();
|
||||
let pairs = |entries: &[IndexEntry]| -> Vec<(String, String)> {
|
||||
entries.iter().map(|e| (e.id.clone(), e.version.clone())).collect()
|
||||
};
|
||||
let owned = |pairs: &[(&str, &str)]| -> Vec<(String, String)> {
|
||||
pairs.iter().map(|(id, version)| ((*id).to_owned(), (*version).to_owned())).collect()
|
||||
};
|
||||
|
||||
assert_eq!(pairs(&Index::parse(&text).unwrap().plugins), owned(&valid));
|
||||
let (index, dropped) = Index::parse_dropping(&text).unwrap();
|
||||
assert_eq!(pairs(&index.plugins), owned(&valid));
|
||||
assert_eq!(pairs(&dropped), owned(&listed[valid.len()..]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_index_still_revokes_and_moves() {
|
||||
let newer = format!(
|
||||
|
||||
@@ -102,6 +102,14 @@ fn parse<T>(r: serde_json::Result<T>) -> IdeResult<T> {
|
||||
r.map_err(|e| IdeError::new(ide_core::error::ErrorKind::Protocol, format!("unexpected gh output: {e}")))
|
||||
}
|
||||
|
||||
/// The repository of `root`, free to execute: the host runs the `gh_*` commands only in a
|
||||
/// trusted project.
|
||||
fn trusted_repo(root: &Path) -> IdeResult<Repo> {
|
||||
let repo = Repo::discover(root)?.ok_or_else(|| IdeError::invalid("not a git repository"))?;
|
||||
repo.set_execution_allowed(true);
|
||||
Ok(repo)
|
||||
}
|
||||
|
||||
/// Whether gh is installed and signed in, and which GitHub repository `root` maps to.
|
||||
pub fn availability(root: &Path) -> GhAvailability {
|
||||
if gh_executable().is_none() {
|
||||
@@ -233,7 +241,7 @@ pub fn create(
|
||||
return Err(IdeError::invalid("choose a base branch"));
|
||||
}
|
||||
if req.push {
|
||||
let repo = Repo::discover(root)?.ok_or_else(|| IdeError::invalid("not a git repository"))?;
|
||||
let repo = trusted_repo(root)?;
|
||||
repo.push(false, true, cancel, on_output)?;
|
||||
}
|
||||
let mut args = vec!["pr", "create", "--title", req.title.trim(), "--body", &req.body, "--base", req.base.trim()];
|
||||
@@ -354,7 +362,7 @@ pub fn prepare_diff(
|
||||
cancel: &CancellationToken,
|
||||
on_output: &dyn Fn(String),
|
||||
) -> IdeResult<PrDiffRevisions> {
|
||||
let repo = Repo::discover(root)?.ok_or_else(|| IdeError::invalid("not a git repository"))?;
|
||||
let repo = trusted_repo(root)?;
|
||||
if !repo.has_commit(head_oid) || !repo.has_commit(base_oid) {
|
||||
let remotes = repo.remotes()?;
|
||||
let remote = remote_for(&remotes, name_with_owner)
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
[package]
|
||||
name = "ide-heap"
|
||||
description = "HotSpot .hprof parser and heap-snapshot analysis for the debugger's JVM Memory view"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
ide-core.workspace = true
|
||||
memmap2.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
specta.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
libc = "0.2"
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { version = "0.5", default-features = false }
|
||||
proptest.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[[bench]]
|
||||
name = "heap"
|
||||
harness = false
|
||||
@@ -0,0 +1,100 @@
|
||||
//! `cargo bench -p ide-heap`: whole-dump analysis and the dominator tree at a million objects.
|
||||
|
||||
use criterion::{Criterion, black_box, criterion_group, criterion_main};
|
||||
use ide_core::CancellationToken;
|
||||
use ide_heap::testing::{MiniHeap, XorShift};
|
||||
use ide_heap::{AnalyzeOptions, BasicType, HeapError, HeapPhase, HeapRootKind, Store, analyze, dominators};
|
||||
|
||||
const CHAINS: u64 = 100_000;
|
||||
const CHAIN_LEN: u64 = 5;
|
||||
const NODES: u64 = CHAINS * CHAIN_LEN;
|
||||
/// Every object takes one slot: node `k` at slot `2k`, its payload at `2k + 1`.
|
||||
const BASE: u64 = 0x1_0000_0000;
|
||||
const SLOT: u64 = 0x20;
|
||||
|
||||
fn node_addr(k: u64) -> u64 {
|
||||
BASE + 2 * k * SLOT
|
||||
}
|
||||
|
||||
fn payload_addr(k: u64) -> u64 {
|
||||
node_addr(k) + SLOT
|
||||
}
|
||||
|
||||
/// 100k rooted chains of five `Node { next, payload }`, each node with its own `byte[16]`. Every
|
||||
/// seventh node points at the payload of the same position in the next chain instead, so that
|
||||
/// payload has two owners and its own one is garbage: 1M objects in all.
|
||||
fn million_objects() -> MiniHeap {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Node", Some("java.lang.Object"), &[("next", BasicType::Object), ("payload", BasicType::Object)]);
|
||||
for k in 0..NODES {
|
||||
let next = if (k + 1) % CHAIN_LEN == 0 { 0 } else { node_addr(k + 1) };
|
||||
let payload = if k % 7 == 0 { payload_addr((k + CHAIN_LEN) % NODES) } else { payload_addr(k) };
|
||||
heap.object(node_addr(k), "Node", &[("next", next), ("payload", payload)]);
|
||||
heap.prim(payload_addr(k), BasicType::Byte, 16);
|
||||
if k % CHAIN_LEN == 0 {
|
||||
heap.root(HeapRootKind::JniGlobal, node_addr(k));
|
||||
}
|
||||
}
|
||||
heap
|
||||
}
|
||||
|
||||
/// CSR of a random tree on `nodes` nodes rooted at 0 plus `extra` random edges, targets of each
|
||||
/// node in scrambled order.
|
||||
fn random_graph(nodes: u32, extra: u32, seed: u64) -> (Vec<u32>, Vec<u32>) {
|
||||
let mut rng = XorShift::new(seed);
|
||||
let mut edges: Vec<(u32, u32)> = (1..nodes).map(|v| (rng.below(v), v)).collect();
|
||||
edges.extend((0..extra).map(|_| (rng.below(nodes), rng.below(nodes))));
|
||||
for i in (1..edges.len()).rev() {
|
||||
edges.swap(i, rng.below(i as u32 + 1) as usize);
|
||||
}
|
||||
let mut start = vec![0u32; nodes as usize + 1];
|
||||
for &(from, _) in &edges {
|
||||
start[from as usize + 1] += 1;
|
||||
}
|
||||
for v in 0..nodes as usize {
|
||||
start[v + 1] += start[v];
|
||||
}
|
||||
let mut cursor = start.clone();
|
||||
let mut out = vec![0u32; edges.len()];
|
||||
for &(from, to) in &edges {
|
||||
out[cursor[from as usize] as usize] = to;
|
||||
cursor[from as usize] += 1;
|
||||
}
|
||||
(start, out)
|
||||
}
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bench(c: &mut Criterion) {
|
||||
let dir = tempfile::tempdir().expect("temp dir");
|
||||
let hprof = dir.path().join("dump.hprof");
|
||||
million_objects().write_to(&hprof);
|
||||
let (start, out) = random_graph(1_000_000, 500_000, 0x5EED_D0D0);
|
||||
let cancel = CancellationToken::new();
|
||||
let opts = AnalyzeOptions { in_memory: true, ..AnalyzeOptions::default() };
|
||||
let summary = analyze(&hprof, dir.path(), opts, &no_progress, &cancel).expect("analyze").summary;
|
||||
assert_eq!(summary.objects, 2 * NODES, "{summary:?}");
|
||||
assert_eq!(summary.unreachable_objects, NODES.div_ceil(7), "{summary:?}");
|
||||
|
||||
let mut g = c.benchmark_group("heap");
|
||||
g.sample_size(10);
|
||||
g.bench_function("index_1m_objects", |b| {
|
||||
b.iter(|| {
|
||||
let opts = AnalyzeOptions { in_memory: true, ..AnalyzeOptions::default() };
|
||||
let index = analyze(&hprof, dir.path(), opts, &no_progress, &cancel).expect("analyze");
|
||||
black_box(index.summary.objects)
|
||||
})
|
||||
});
|
||||
g.bench_function("dominators_1m_random", |b| {
|
||||
b.iter(|| {
|
||||
let d = dominators(&start, &out, 0, &Store::Ram, &no_progress, &cancel).expect("dominators");
|
||||
black_box(d.reachable)
|
||||
})
|
||||
});
|
||||
g.finish();
|
||||
}
|
||||
|
||||
criterion_group!(benches, bench);
|
||||
criterion_main!(benches);
|
||||
@@ -0,0 +1,202 @@
|
||||
//! Ties the passes together: graph, dominators, shallow and retained sizes, the class table and
|
||||
//! paths to GC roots, into a [`HeapIndex`].
|
||||
|
||||
mod classes;
|
||||
mod paths;
|
||||
mod retained;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::path::Path;
|
||||
use std::time::Instant;
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
use memmap2::Mmap;
|
||||
|
||||
use crate::column::Store;
|
||||
use crate::dominators::{Dominators, dominators};
|
||||
use crate::error::HeapError;
|
||||
use crate::files::INDEX_FILES;
|
||||
use crate::graph::{HeapGraph, build_graph};
|
||||
use crate::index::{HeapIndex, HeapSummary, write_summary};
|
||||
use crate::progress::{CHECK_EVERY, HeapPhase, Progress};
|
||||
use crate::shallow::{HistogramEntry, ShallowModel, VmLayout};
|
||||
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct AnalyzeOptions {
|
||||
/// The JVM's class histogram from the same stop as the dump.
|
||||
pub histogram: Option<Vec<HistogramEntry>>,
|
||||
/// `VM.flags` output of the JVM that wrote the dump.
|
||||
pub vm_flags: Option<String>,
|
||||
pub java_major: Option<u32>,
|
||||
/// Keep every array in RAM and write nothing to `index_dir`.
|
||||
pub in_memory: bool,
|
||||
}
|
||||
|
||||
/// Analyzes the dump at `hprof`, writing the index columns and `summary.json` into `index_dir`
|
||||
/// (which must exist and may hold the dump itself). `progress` gets every [`HeapPhase`] in order;
|
||||
/// an `Err` from it, a cancel or any other error removes every index file from `index_dir` and
|
||||
/// leaves the rest of the directory alone.
|
||||
pub fn analyze(
|
||||
hprof: &Path,
|
||||
index_dir: &Path,
|
||||
opts: AnalyzeOptions,
|
||||
progress: &dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<HeapIndex, HeapError> {
|
||||
let store = if opts.in_memory { Store::Ram } else { Store::Dir(index_dir.to_path_buf()) };
|
||||
let result = run(hprof, &store, opts, progress, cancel);
|
||||
// `run` has dropped every column by now: Windows cannot delete a mapped file.
|
||||
if result.is_err() && !matches!(store, Store::Ram) {
|
||||
remove_index_files(index_dir);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn run(
|
||||
hprof: &Path,
|
||||
store: &Store,
|
||||
opts: AnalyzeOptions,
|
||||
report: &dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<HeapIndex, HeapError> {
|
||||
let started = Instant::now();
|
||||
let progress = Progress::new(report, cancel);
|
||||
let file = File::open(hprof)?;
|
||||
// SAFETY: nothing writes the dump while it is analyzed or viewed: a capture finishes writing
|
||||
// it before analysis and deletes it only with the snapshot that holds this index.
|
||||
let dump = unsafe { Mmap::map(&file)? };
|
||||
let graph = build_graph(&dump, store, report, cancel)?;
|
||||
let Dominators { idom, preorder, .. } =
|
||||
dominators(&graph.out_start, &graph.out, graph.root, store, report, cancel)?;
|
||||
|
||||
let nodes = u64::from(graph.root) + 1;
|
||||
// Seven walks over the nodes, roughly: sizes, retained, child counts, child sorting, class
|
||||
// counts and the tree walk, which visits every node twice.
|
||||
let mut meter = Meter::start(&progress, HeapPhase::Retained, 7 * nodes)?;
|
||||
let layout = VmLayout::from_vm_flags(opts.vm_flags.as_deref().unwrap_or(""), opts.java_major);
|
||||
let model = ShallowModel::new(&graph, layout, opts.histogram.as_deref(), cancel)?;
|
||||
let (shallow8, retained) = retained::sizes(&graph, &model, &idom, &preorder, store, &mut meter)?;
|
||||
drop(preorder);
|
||||
let (kids_start, kids) = retained::children(&idom, &retained, graph.root, store, &mut meter)?;
|
||||
let tree = retained::Tree { idom, shallow8, retained, kids_start, kids };
|
||||
let (classes, totals) = classes::table(&graph, &tree, store, &mut meter)?;
|
||||
meter.finish()?;
|
||||
|
||||
let mut meter = Meter::start(&progress, HeapPhase::Paths, nodes + graph.out.len() as u64)?;
|
||||
let gc_parent = paths::gc_parents(&graph, store, &mut meter)?;
|
||||
meter.finish()?;
|
||||
|
||||
progress.report(HeapPhase::Saving, 0, 1)?;
|
||||
let HeapGraph {
|
||||
ids,
|
||||
types,
|
||||
offsets,
|
||||
roots,
|
||||
classes: class_info,
|
||||
class_slot,
|
||||
thread_objects,
|
||||
weak_edges_skipped,
|
||||
id_size,
|
||||
..
|
||||
} = graph;
|
||||
let summary = HeapSummary {
|
||||
objects: totals.reachable_objects + totals.unreachable_objects,
|
||||
reachable_objects: totals.reachable_objects,
|
||||
reachable_bytes: totals.reachable_bytes,
|
||||
unreachable_objects: totals.unreachable_objects,
|
||||
unreachable_bytes: totals.unreachable_bytes,
|
||||
classes: class_info.iter().filter(|c| c.id != 0).count() as u32,
|
||||
weak_edges_skipped,
|
||||
size_source: model.source,
|
||||
analysis_ms: u32::try_from(started.elapsed().as_millis()).unwrap_or(u32::MAX),
|
||||
};
|
||||
if let Store::Dir(dir) = store {
|
||||
write_summary(dir, &summary)?;
|
||||
}
|
||||
let mut root_kinds = HashMap::with_capacity(roots.len());
|
||||
for (object, kind, serial) in roots {
|
||||
root_kinds.entry(object).or_insert((kind, serial));
|
||||
}
|
||||
progress.report(HeapPhase::Saving, 1, 1)?;
|
||||
|
||||
let retained::Tree { idom, shallow8, retained, kids_start, kids } = tree;
|
||||
Ok(HeapIndex {
|
||||
hprof: dump,
|
||||
ids,
|
||||
types,
|
||||
offsets,
|
||||
shallow8,
|
||||
retained,
|
||||
idom,
|
||||
kids_start,
|
||||
kids,
|
||||
gc_parent,
|
||||
classes,
|
||||
class_info,
|
||||
class_slot,
|
||||
roots: root_kinds,
|
||||
thread_objects,
|
||||
id_size,
|
||||
summary,
|
||||
})
|
||||
}
|
||||
|
||||
/// Removes what an analysis may have written; a file that is not there is fine.
|
||||
fn remove_index_files(dir: &Path) {
|
||||
for name in INDEX_FILES {
|
||||
let path = dir.join(name);
|
||||
if let Err(err) = std::fs::remove_file(&path)
|
||||
&& err.kind() != std::io::ErrorKind::NotFound
|
||||
{
|
||||
tracing::warn!("could not remove {}: {err}", path.display());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Steps of one phase: reports progress every [`CHECK_EVERY`] steps, and each loop looks at the
|
||||
/// token before its first step.
|
||||
pub(super) struct Meter<'a> {
|
||||
progress: &'a Progress<'a>,
|
||||
phase: HeapPhase,
|
||||
/// An estimate; `done` is clamped to it.
|
||||
total: u64,
|
||||
done: u64,
|
||||
next_report: u64,
|
||||
}
|
||||
|
||||
impl<'a> Meter<'a> {
|
||||
fn start(progress: &'a Progress<'a>, phase: HeapPhase, total: u64) -> Result<Self, HeapError> {
|
||||
progress.report(phase, 0, total)?;
|
||||
Ok(Self { progress, phase, total, done: 0, next_report: CHECK_EVERY })
|
||||
}
|
||||
|
||||
pub(super) fn begin_loop(&self) -> Result<(), HeapError> {
|
||||
self.progress.check()
|
||||
}
|
||||
|
||||
pub(super) fn token(&self) -> &'a CancellationToken {
|
||||
self.progress.token()
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub(super) fn tick(&mut self) -> Result<(), HeapError> {
|
||||
self.advance(1)
|
||||
}
|
||||
|
||||
pub(super) fn advance(&mut self, steps: u64) -> Result<(), HeapError> {
|
||||
self.done += steps;
|
||||
if self.done < self.next_report {
|
||||
return Ok(());
|
||||
}
|
||||
self.next_report = self.done + CHECK_EVERY;
|
||||
self.progress.report(self.phase, self.done.min(self.total), self.total)
|
||||
}
|
||||
|
||||
fn finish(self) -> Result<(), HeapError> {
|
||||
self.progress.report(self.phase, self.total, self.total)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,117 @@
|
||||
//! The class table: counts and shallow sizes per type slot, retained sizes per type from a walk of
|
||||
//! the dominator tree, and the reachable / unreachable totals of the summary.
|
||||
|
||||
use super::Meter;
|
||||
use super::retained::Tree;
|
||||
use crate::column::{NONE, Store};
|
||||
use crate::error::HeapError;
|
||||
use crate::graph::{HeapGraph, PRIM_SLOTS};
|
||||
use crate::hprof::BasicType;
|
||||
use crate::index::ClassStat;
|
||||
use crate::names::display_name;
|
||||
|
||||
/// Instances and arrays only: class objects are left out.
|
||||
#[derive(Default)]
|
||||
pub(super) struct Totals {
|
||||
pub(super) reachable_objects: u64,
|
||||
pub(super) reachable_bytes: u64,
|
||||
pub(super) unreachable_objects: u64,
|
||||
pub(super) unreachable_bytes: u64,
|
||||
}
|
||||
|
||||
pub(super) fn table(
|
||||
graph: &HeapGraph,
|
||||
tree: &Tree,
|
||||
store: &Store,
|
||||
meter: &mut Meter,
|
||||
) -> Result<(Vec<ClassStat>, Totals), HeapError> {
|
||||
let slots = graph.classes.len() + PRIM_SLOTS as usize;
|
||||
let mut classes: Vec<ClassStat> = (0..slots as u32)
|
||||
.map(|slot| ClassStat { name: slot_name(graph, slot), count: 0, shallow: 0, retained: 0, unreachable_count: 0 })
|
||||
.collect();
|
||||
let mut totals = Totals::default();
|
||||
meter.begin_loop()?;
|
||||
for v in 0..graph.root as usize {
|
||||
meter.tick()?;
|
||||
let slot = graph.types[v];
|
||||
let bytes = u64::from(tree.shallow8[v]) * 8;
|
||||
let stat = &mut classes[slot as usize];
|
||||
let instance = slot != graph.class_slot;
|
||||
if tree.idom[v] == NONE {
|
||||
stat.unreachable_count += 1;
|
||||
if instance {
|
||||
totals.unreachable_objects += 1;
|
||||
totals.unreachable_bytes += bytes;
|
||||
}
|
||||
} else {
|
||||
stat.count += 1;
|
||||
stat.shallow += bytes;
|
||||
if instance {
|
||||
totals.reachable_objects += 1;
|
||||
totals.reachable_bytes += bytes;
|
||||
}
|
||||
}
|
||||
}
|
||||
retained_by_type(graph, tree, &mut classes, store, meter)?;
|
||||
Ok((classes, totals))
|
||||
}
|
||||
|
||||
/// Adds to each type the retained size of its instances that have no instance of the same type
|
||||
/// above them in the dominator tree. The walk keeps no stack: it goes up through `idom` and
|
||||
/// resumes a node's children from `cursor`, while `active[t]` counts the instances of type `t` on
|
||||
/// the path from the root.
|
||||
fn retained_by_type(
|
||||
graph: &HeapGraph,
|
||||
tree: &Tree,
|
||||
classes: &mut [ClassStat],
|
||||
store: &Store,
|
||||
meter: &mut Meter,
|
||||
) -> Result<(), HeapError> {
|
||||
let root = graph.root;
|
||||
let mut cursor = store.scratch::<u32>(root as usize + 1, 0)?;
|
||||
let mut active = vec![0u32; classes.len()];
|
||||
cursor[root as usize] = tree.kids_start[root as usize];
|
||||
let mut v = root;
|
||||
meter.begin_loop()?;
|
||||
loop {
|
||||
meter.tick()?;
|
||||
let next = cursor[v as usize];
|
||||
if next < tree.kids_start[v as usize + 1] {
|
||||
cursor[v as usize] = next + 1;
|
||||
let child = tree.kids[next as usize];
|
||||
let slot = graph.types[child as usize] as usize;
|
||||
if active[slot] == 0 {
|
||||
classes[slot].retained += tree.retained[child as usize];
|
||||
}
|
||||
active[slot] += 1;
|
||||
cursor[child as usize] = tree.kids_start[child as usize];
|
||||
v = child;
|
||||
} else if v == root {
|
||||
return Ok(());
|
||||
} else {
|
||||
active[graph.types[v as usize] as usize] -= 1;
|
||||
v = tree.idom[v as usize];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn slot_name(graph: &HeapGraph, slot: u32) -> String {
|
||||
match graph.classes.get(slot as usize) {
|
||||
Some(class) => display_name(&class.name),
|
||||
None => graph.prim_type(slot).map_or_else(String::new, |ty| prim_array_name(ty).to_owned()),
|
||||
}
|
||||
}
|
||||
|
||||
fn prim_array_name(ty: BasicType) -> &'static str {
|
||||
match ty {
|
||||
BasicType::Object => "java.lang.Object[]",
|
||||
BasicType::Boolean => "boolean[]",
|
||||
BasicType::Char => "char[]",
|
||||
BasicType::Float => "float[]",
|
||||
BasicType::Double => "double[]",
|
||||
BasicType::Byte => "byte[]",
|
||||
BasicType::Short => "short[]",
|
||||
BasicType::Int => "int[]",
|
||||
BasicType::Long => "long[]",
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
//! Parents on shortest paths from the GC roots, by breadth-first search over strong edges.
|
||||
|
||||
use super::Meter;
|
||||
use crate::column::{Column, NONE, Store};
|
||||
use crate::error::HeapError;
|
||||
use crate::files::GC_PARENT_FILE;
|
||||
use crate::graph::HeapGraph;
|
||||
|
||||
pub(super) fn gc_parents(graph: &HeapGraph, store: &Store, meter: &mut Meter) -> Result<Column<u32>, HeapError> {
|
||||
let root = graph.root;
|
||||
let nodes = root as usize + 1;
|
||||
let mut parent = store.column::<u32>(GC_PARENT_FILE, nodes, NONE)?;
|
||||
let mut queue = store.scratch::<u32>(nodes, 0)?;
|
||||
parent[root as usize] = root;
|
||||
queue[0] = root;
|
||||
let (mut head, mut tail) = (0, 1);
|
||||
meter.begin_loop()?;
|
||||
while head < tail {
|
||||
meter.tick()?;
|
||||
let v = queue[head];
|
||||
head += 1;
|
||||
for &w in graph.out_edges(v) {
|
||||
meter.tick()?;
|
||||
if parent[w as usize] == NONE {
|
||||
parent[w as usize] = v;
|
||||
queue[tail] = w;
|
||||
tail += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(parent)
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
//! Shallow and retained sizes, and the dominator tree as child lists sorted by retained size.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
|
||||
use super::Meter;
|
||||
use crate::column::{Column, NONE, Store};
|
||||
use crate::error::HeapError;
|
||||
use crate::files::{KIDS_FILE, KIDS_START_FILE, RETAINED_FILE, SHALLOW8_FILE};
|
||||
use crate::graph::{HeapGraph, accumulate};
|
||||
use crate::progress::CHECK_EVERY;
|
||||
use crate::shallow::ShallowModel;
|
||||
|
||||
/// The dominator tree with sizes; see [`crate::HeapIndex`] for each column.
|
||||
pub(super) struct Tree {
|
||||
pub(super) idom: Column<u32>,
|
||||
pub(super) shallow8: Column<u32>,
|
||||
pub(super) retained: Column<u64>,
|
||||
pub(super) kids_start: Column<u32>,
|
||||
pub(super) kids: Column<u32>,
|
||||
}
|
||||
|
||||
/// `(shallow8, retained)` of every node, the root included. `preorder` is the DFS preorder of the
|
||||
/// reachable nodes, from the root.
|
||||
pub(super) fn sizes(
|
||||
graph: &HeapGraph,
|
||||
model: &ShallowModel,
|
||||
idom: &[u32],
|
||||
preorder: &[u32],
|
||||
store: &Store,
|
||||
meter: &mut Meter,
|
||||
) -> Result<(Column<u32>, Column<u64>), HeapError> {
|
||||
let nodes = idom.len();
|
||||
let mut shallow8 = store.column::<u32>(SHALLOW8_FILE, nodes, 0)?;
|
||||
let mut retained = store.column::<u64>(RETAINED_FILE, nodes, 0)?;
|
||||
meter.begin_loop()?;
|
||||
for v in 0..graph.root {
|
||||
meter.tick()?;
|
||||
let units = u32::try_from(model.size_of(graph, v).div_ceil(8)).unwrap_or(u32::MAX);
|
||||
shallow8[v as usize] = units;
|
||||
if idom[v as usize] != NONE {
|
||||
retained[v as usize] = u64::from(units) * 8;
|
||||
}
|
||||
}
|
||||
// A dominator is a DFS ancestor, so in reverse preorder every node is complete before it is
|
||||
// added to its dominator.
|
||||
meter.begin_loop()?;
|
||||
for &v in preorder.iter().skip(1).rev() {
|
||||
meter.tick()?;
|
||||
let own = retained[v as usize];
|
||||
retained[idom[v as usize] as usize] += own;
|
||||
}
|
||||
Ok((shallow8, retained))
|
||||
}
|
||||
|
||||
/// `(kids_start, kids)`: the children of every reachable node in the dominator tree, as CSR.
|
||||
pub(super) fn children(
|
||||
idom: &[u32],
|
||||
retained: &[u64],
|
||||
root: u32,
|
||||
store: &Store,
|
||||
meter: &mut Meter,
|
||||
) -> Result<(Column<u32>, Column<u32>), HeapError> {
|
||||
let nodes = idom.len();
|
||||
// Counted into `starts[d + 1]`; filled through `starts[d]` as a cursor, which leaves it at d's
|
||||
// end until the column is shifted one slot.
|
||||
let mut starts = store.column::<u32>(KIDS_START_FILE, nodes + 1, 0)?;
|
||||
let mut longest = 0;
|
||||
meter.begin_loop()?;
|
||||
for (v, &d) in idom.iter().enumerate() {
|
||||
meter.tick()?;
|
||||
if d != NONE && v != root as usize {
|
||||
let count = &mut starts[d as usize + 1];
|
||||
*count += 1;
|
||||
longest = longest.max(*count);
|
||||
}
|
||||
}
|
||||
let total = accumulate(&mut starts, meter.token())?;
|
||||
let mut kids = store.column::<u32>(KIDS_FILE, total as usize, 0)?;
|
||||
meter.begin_loop()?;
|
||||
for (v, &d) in idom.iter().enumerate() {
|
||||
meter.tick()?;
|
||||
if d != NONE && v != root as usize {
|
||||
let cursor = &mut starts[d as usize];
|
||||
kids[*cursor as usize] = v as u32;
|
||||
*cursor += 1;
|
||||
}
|
||||
}
|
||||
starts.copy_within(0..nodes, 1);
|
||||
starts[0] = 0;
|
||||
sort_children(&mut kids, &starts, retained, longest as usize, CHECK_EVERY as usize, store, meter)?;
|
||||
Ok((starts, kids))
|
||||
}
|
||||
|
||||
/// Sorts each child list by retained size descending, then by node. A list longer than `run` is
|
||||
/// sorted in runs of `run` that are then merged, so no single step takes longer than one run.
|
||||
fn sort_children(
|
||||
kids: &mut [u32],
|
||||
starts: &[u32],
|
||||
retained: &[u64],
|
||||
longest: usize,
|
||||
run: usize,
|
||||
store: &Store,
|
||||
meter: &mut Meter,
|
||||
) -> Result<(), HeapError> {
|
||||
let key = |&k: &u32| (Reverse(retained[k as usize]), k);
|
||||
let mut buffer = if longest > run { store.scratch::<u32>(longest, 0)? } else { Column::ram(0, 0) };
|
||||
meter.begin_loop()?;
|
||||
for v in 0..starts.len() - 1 {
|
||||
meter.tick()?;
|
||||
let list = &mut kids[starts[v] as usize..starts[v + 1] as usize];
|
||||
if list.len() <= run {
|
||||
list.sort_unstable_by_key(key);
|
||||
meter.advance(list.len() as u64)?;
|
||||
continue;
|
||||
}
|
||||
for chunk in list.chunks_mut(run) {
|
||||
meter.begin_loop()?;
|
||||
chunk.sort_unstable_by_key(key);
|
||||
meter.advance(chunk.len() as u64)?;
|
||||
}
|
||||
let len = list.len();
|
||||
let (mut from, mut to) = (list, &mut buffer[..len]);
|
||||
let mut in_buffer = false;
|
||||
let mut width = run;
|
||||
while width < len {
|
||||
meter.begin_loop()?;
|
||||
for start in (0..len).step_by(2 * width) {
|
||||
let mid = (start + width).min(len);
|
||||
let end = (start + 2 * width).min(len);
|
||||
merge(&from[start..mid], &from[mid..end], &mut to[start..end], &key, meter)?;
|
||||
}
|
||||
std::mem::swap(&mut from, &mut to);
|
||||
in_buffer = !in_buffer;
|
||||
width *= 2;
|
||||
}
|
||||
if in_buffer {
|
||||
to.copy_from_slice(from);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn merge<K: Ord>(
|
||||
left: &[u32],
|
||||
right: &[u32],
|
||||
out: &mut [u32],
|
||||
key: &impl Fn(&u32) -> K,
|
||||
meter: &mut Meter,
|
||||
) -> Result<(), HeapError> {
|
||||
let (mut i, mut j) = (0, 0);
|
||||
for slot in out.iter_mut() {
|
||||
meter.tick()?;
|
||||
let take_left = j == right.len() || (i < left.len() && key(&left[i]) <= key(&right[j]));
|
||||
if take_left {
|
||||
*slot = left[i];
|
||||
i += 1;
|
||||
} else {
|
||||
*slot = right[j];
|
||||
j += 1;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use super::*;
|
||||
use crate::error::HeapError;
|
||||
use crate::progress::{HeapPhase, Progress};
|
||||
use crate::testing::XorShift;
|
||||
|
||||
fn sorted_with_run(lists: &[Vec<u32>], retained: &[u64], run: usize) -> Vec<u32> {
|
||||
let report = |_, _, _| Ok::<(), HeapError>(());
|
||||
let cancel = CancellationToken::new();
|
||||
let progress = Progress::new(&report, &cancel);
|
||||
let mut meter = Meter::start(&progress, HeapPhase::Retained, 1).unwrap();
|
||||
let mut starts = vec![0u32];
|
||||
for list in lists {
|
||||
starts.push(starts.last().unwrap() + list.len() as u32);
|
||||
}
|
||||
let mut kids: Vec<u32> = lists.concat();
|
||||
let longest = lists.iter().map(Vec::len).max().unwrap_or(0);
|
||||
sort_children(&mut kids, &starts, retained, longest, run, &Store::Ram, &mut meter).unwrap();
|
||||
kids
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_child_lists_are_merged_from_sorted_runs() {
|
||||
let mut rng = XorShift::new(7);
|
||||
let retained: Vec<u64> = (0..500).map(|_| u64::from(rng.below(40))).collect();
|
||||
let mut nodes: Vec<u32> = (0..500).collect();
|
||||
for i in (1..nodes.len()).rev() {
|
||||
nodes.swap(i, rng.below(i as u32 + 1) as usize);
|
||||
}
|
||||
// Lists below, at and above the run length, and one that ends on a partial run.
|
||||
let lists = vec![nodes[..3].to_vec(), nodes[3..7].to_vec(), nodes[7..400].to_vec(), nodes[400..].to_vec()];
|
||||
for run in [1, 4, 7, 64, 1000] {
|
||||
let mut expected = Vec::new();
|
||||
for list in &lists {
|
||||
let mut list = list.clone();
|
||||
list.sort_by_key(|&k| (Reverse(retained[k as usize]), k));
|
||||
expected.extend(list);
|
||||
}
|
||||
assert_eq!(sorted_with_run(&lists, &retained, run), expected, "run {run}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,265 @@
|
||||
use std::cell::Cell;
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use super::*;
|
||||
use crate::column::{Column, NONE};
|
||||
use crate::files::INDEX_FILES;
|
||||
use crate::hprof::{BasicType, HeapRootKind};
|
||||
use crate::index::{ClassStat, HeapSummary};
|
||||
use crate::shallow::SizeSource;
|
||||
use crate::testing::{MiniHeap, sample};
|
||||
|
||||
const DUMP: &str = "dump.hprof";
|
||||
const ROOT: Option<&str> = Some("java.lang.Object");
|
||||
const NODE_FIELDS: [(&str, BasicType); 2] = [("next", BasicType::Object), ("v", BasicType::Long)];
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn in_memory() -> AnalyzeOptions {
|
||||
AnalyzeOptions { in_memory: true, ..AnalyzeOptions::default() }
|
||||
}
|
||||
|
||||
/// A temp dir holding `heap` as `dump.hprof`, the way a capture lays out its snapshot dir.
|
||||
fn dump_dir(heap: &MiniHeap) -> (tempfile::TempDir, PathBuf) {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dump = dir.path().join(DUMP);
|
||||
heap.write_to(&dump);
|
||||
(dir, dump)
|
||||
}
|
||||
|
||||
fn analyze_with(heap: &MiniHeap, opts: AnalyzeOptions) -> HeapIndex {
|
||||
let (dir, dump) = dump_dir(heap);
|
||||
analyze(&dump, dir.path(), opts, &no_progress, &CancellationToken::new()).unwrap()
|
||||
}
|
||||
|
||||
fn file_names(dir: &Path) -> BTreeSet<String> {
|
||||
std::fs::read_dir(dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect()
|
||||
}
|
||||
|
||||
fn node(index: &HeapIndex, addr: u64) -> u32 {
|
||||
index.ids.rank(addr).unwrap_or_else(|| panic!("{addr:#x} is not in the dump"))
|
||||
}
|
||||
|
||||
fn shallow(index: &HeapIndex, addr: u64) -> u64 {
|
||||
u64::from(index.shallow8[node(index, addr) as usize]) * 8
|
||||
}
|
||||
|
||||
fn retained(index: &HeapIndex, addr: u64) -> u64 {
|
||||
index.retained[node(index, addr) as usize]
|
||||
}
|
||||
|
||||
fn idom(index: &HeapIndex, addr: u64) -> u32 {
|
||||
index.idom[node(index, addr) as usize]
|
||||
}
|
||||
|
||||
fn class<'a>(index: &'a HeapIndex, name: &str) -> &'a ClassStat {
|
||||
let mut rows = index.classes.iter().filter(|c| c.name == name);
|
||||
let row = rows.next().unwrap_or_else(|| panic!("no class {name}"));
|
||||
assert!(rows.next().is_none(), "two classes named {name}");
|
||||
row
|
||||
}
|
||||
|
||||
fn stat(c: &ClassStat) -> (u64, u64, u64, u64) {
|
||||
(c.count, c.shallow, c.retained, c.unreachable_count)
|
||||
}
|
||||
|
||||
fn kids_of(index: &HeapIndex, v: u32) -> &[u32] {
|
||||
&index.kids[index.kids_start[v as usize] as usize..index.kids_start[v as usize + 1] as usize]
|
||||
}
|
||||
|
||||
const H: u64 = 0x1000;
|
||||
const N1: u64 = 0x1100;
|
||||
const N2: u64 = 0x1200;
|
||||
const N3: u64 = 0x1300;
|
||||
const L: u64 = 0x1400;
|
||||
const R: u64 = 0x1500;
|
||||
const W: u64 = 0x1600;
|
||||
const N4: u64 = 0x1700;
|
||||
|
||||
#[test]
|
||||
fn sample_retained_sizes() {
|
||||
let index = analyze_with(&sample(), in_memory());
|
||||
let shallows: Vec<u64> = [H, N1, L, W, R].iter().map(|&a| shallow(&index, a)).collect();
|
||||
assert_eq!(shallows, [24, 24, 32, 24, 32]);
|
||||
let retained_sizes: Vec<u64> = [L, N3, N1, N2, H, W].iter().map(|&a| retained(&index, a)).collect();
|
||||
assert_eq!(retained_sizes, [32, 56, 24, 24, 72, 24]);
|
||||
let root = index.root();
|
||||
assert_eq!(
|
||||
[idom(&index, N1), idom(&index, N2), idom(&index, N3), idom(&index, L)],
|
||||
[node(&index, H), node(&index, H), root, node(&index, N3)]
|
||||
);
|
||||
assert_eq!((idom(&index, R), idom(&index, N4)), (NONE, NONE));
|
||||
assert_eq!((retained(&index, R), retained(&index, N4)), (0, 0));
|
||||
|
||||
// Children by retained size descending, ties by node.
|
||||
assert_eq!(kids_of(&index, node(&index, H)), [node(&index, N1), node(&index, N2)]);
|
||||
let top = kids_of(&index, root);
|
||||
assert_eq!(top[..2], [node(&index, H), node(&index, N3)]);
|
||||
assert!(top.windows(2).all(|p| (index.retained[p[0] as usize], p[1]) > (index.retained[p[1] as usize], p[0])));
|
||||
assert_eq!(index.retained[root as usize], top.iter().map(|&k| index.retained[k as usize]).sum::<u64>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_class_table() {
|
||||
let index = analyze_with(&sample(), in_memory());
|
||||
assert_eq!(stat(class(&index, "Node")), (3, 72, 104, 1));
|
||||
assert_eq!(stat(class(&index, "Holder")), (1, 24, 72, 0));
|
||||
assert_eq!(stat(class(&index, "long[]")), (1, 32, 32, 0));
|
||||
assert_eq!(stat(class(&index, "java.lang.ref.WeakReference")), (1, 24, 24, 0));
|
||||
let objects = class(&index, "java.lang.Object[]");
|
||||
assert_eq!((objects.count, objects.unreachable_count), (0, 1));
|
||||
// Indexed by type slot, primitive array slots included.
|
||||
assert_eq!(index.classes.len(), index.class_info.len() + crate::graph::PRIM_SLOTS as usize);
|
||||
let slot = index.types[node(&index, H) as usize] as usize;
|
||||
assert_eq!(index.classes[slot].name, "Holder");
|
||||
assert_eq!(index.class_info[slot].name, "Holder");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_summary() {
|
||||
let index = analyze_with(&sample(), in_memory());
|
||||
let s = &index.summary;
|
||||
assert_eq!(
|
||||
(s.reachable_objects, s.reachable_bytes, s.unreachable_objects, s.unreachable_bytes, s.weak_edges_skipped),
|
||||
(6, 152, 2, 56, 1)
|
||||
);
|
||||
assert_eq!(s.objects, 8);
|
||||
assert_eq!(s.classes, 7);
|
||||
assert_eq!(s.size_source, SizeSource::Layout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_gc_parents() {
|
||||
let index = analyze_with(&sample(), in_memory());
|
||||
let parent = |addr| index.gc_parent[node(&index, addr) as usize];
|
||||
let root = index.root();
|
||||
assert_eq!(index.gc_parent[root as usize], root);
|
||||
assert_eq!([parent(H), parent(N3), parent(W)], [root; 3]);
|
||||
assert_eq!([parent(N1), parent(L)], [node(&index, H), node(&index, N3)]);
|
||||
assert_eq!((parent(R), parent(N4)), (NONE, NONE));
|
||||
assert_eq!(index.roots[&node(&index, N3)], (HeapRootKind::JavaFrame, Some(1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cycle_is_retained_once() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Node", ROOT, &NODE_FIELDS)
|
||||
.object(0xA000, "Node", &[("next", 0xB000)])
|
||||
.object(0xB000, "Node", &[("next", 0xA000)])
|
||||
.root(HeapRootKind::JniGlobal, 0xA000);
|
||||
let index = analyze_with(&heap, in_memory());
|
||||
assert_eq!((retained(&index, 0xA000), retained(&index, 0xB000)), (48, 24));
|
||||
assert_eq!(class(&index, "Node").retained, 48);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_chain_retained_by_class() {
|
||||
// A small stack proves no pass recurses along the chain.
|
||||
std::thread::Builder::new()
|
||||
.stack_size(256 << 10)
|
||||
.spawn(|| {
|
||||
const NODES: u64 = 1_000_000;
|
||||
let addr = |i: u64| 0x10_0000 + i * 0x100;
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Node", ROOT, &NODE_FIELDS);
|
||||
for i in 0..NODES {
|
||||
let next = if i + 1 < NODES { addr(i + 1) } else { 0 };
|
||||
heap.object(addr(i), "Node", &[("next", next)]);
|
||||
}
|
||||
heap.root(HeapRootKind::JniGlobal, addr(0));
|
||||
let index = analyze_with(&heap, in_memory());
|
||||
assert_eq!(retained(&index, addr(0)), 24_000_000);
|
||||
assert_eq!(retained(&index, addr(NODES - 1)), 24);
|
||||
assert_eq!(class(&index, "Node").retained, 24_000_000);
|
||||
assert_eq!(index.summary.reachable_objects, NODES);
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_sizes_flow_into_retained() {
|
||||
let histogram = vec![HistogramEntry { name: "Node".to_owned(), instances: 3, bytes: 96 }];
|
||||
let index = analyze_with(&sample(), AnalyzeOptions { histogram: Some(histogram), ..in_memory() });
|
||||
assert_eq!(shallow(&index, N3), 32);
|
||||
assert_eq!(retained(&index, N3), 64);
|
||||
assert_eq!(index.summary.size_source, SizeSource::Jvm);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vm_flags_of_an_unknown_jdk_set_the_layout() {
|
||||
// Without compressed oops a Node is 12 + 8 + 8 = 28 → 32.
|
||||
let opts = AnalyzeOptions { vm_flags: Some("-XX:MaxHeapSize=42949672960".to_owned()), ..in_memory() };
|
||||
let index = analyze_with(&sample(), opts);
|
||||
assert_eq!(shallow(&index, N4), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_stops_analysis() {
|
||||
let (dir, dump) = dump_dir(&sample());
|
||||
let cancel = CancellationToken::new();
|
||||
let phases = Cell::new(Vec::new());
|
||||
let progress = |phase, _, _| {
|
||||
let mut seen = phases.take();
|
||||
seen.push(phase);
|
||||
phases.set(seen);
|
||||
if phase == HeapPhase::Dominators {
|
||||
cancel.cancel();
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = analyze(&dump, dir.path(), AnalyzeOptions::default(), &progress, &cancel);
|
||||
assert!(matches!(result, Err(HeapError::Cancelled)), "{:?}", result.err());
|
||||
assert_eq!(phases.take().last(), Some(&HeapPhase::Dominators), "stops at the first check");
|
||||
// The dump the capture analyzes in place stays; every file the analysis wrote is gone.
|
||||
assert_eq!(file_names(dir.path()), BTreeSet::from([DUMP.to_owned()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_error_stops_analysis_and_removes_its_files() {
|
||||
let (dir, dump) = dump_dir(&sample());
|
||||
// The last report, once every index file is written.
|
||||
let progress = |phase, done, _| match (phase, done) {
|
||||
(HeapPhase::Saving, 1) => Err(HeapError::NotEnoughDiskSpace { needed: 2, available: 1 }),
|
||||
_ => Ok(()),
|
||||
};
|
||||
let result = analyze(&dump, dir.path(), AnalyzeOptions::default(), &progress, &CancellationToken::new());
|
||||
assert!(matches!(result, Err(HeapError::NotEnoughDiskSpace { .. })), "{:?}", result.err());
|
||||
assert_eq!(file_names(dir.path()), BTreeSet::from([DUMP.to_owned()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_index_files() {
|
||||
let (dir, dump) = dump_dir(&sample());
|
||||
let index = analyze(&dump, dir.path(), AnalyzeOptions::default(), &no_progress, &CancellationToken::new()).unwrap();
|
||||
let names = file_names(dir.path());
|
||||
for name in [
|
||||
"types.u32",
|
||||
"shallow8.u32",
|
||||
"retained.u64",
|
||||
"idom.u32",
|
||||
"kids_start.u32",
|
||||
"kids.u32",
|
||||
"gc_parent.u32",
|
||||
"offsets.u64",
|
||||
"summary.json",
|
||||
] {
|
||||
assert!(names.contains(name), "{name} missing from {names:?}");
|
||||
}
|
||||
assert!(names.iter().any(|n| n.starts_with("ids.")), "{names:?}");
|
||||
// Nothing but the index files and the dump: scratch arrays are gone.
|
||||
let unexpected: Vec<_> = names.iter().filter(|n| *n != DUMP && !INDEX_FILES.contains(&n.as_str())).collect();
|
||||
assert!(unexpected.is_empty(), "{unexpected:?}");
|
||||
|
||||
let summary: HeapSummary =
|
||||
serde_json::from_slice(&std::fs::read(dir.path().join("summary.json")).unwrap()).unwrap();
|
||||
assert_eq!(summary, index.summary);
|
||||
drop(index);
|
||||
let retained = Column::<u64>::open(&dir.path().join("retained.u64")).unwrap();
|
||||
let in_ram = analyze_with(&sample(), in_memory());
|
||||
assert_eq!(&*retained, &*in_ram.retained);
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
//! Fixed-width arrays that live in RAM or in memory-mapped files, so the analysis arrays of a
|
||||
//! large heap do not count against the process's anonymous memory.
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::marker::PhantomData;
|
||||
use std::mem::size_of;
|
||||
use std::ops::{Deref, DerefMut};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use memmap2::{Mmap, MmapMut};
|
||||
|
||||
/// Index value meaning "no object" in every `u32` column.
|
||||
pub const NONE: u32 = u32::MAX;
|
||||
|
||||
/// # Safety
|
||||
/// Every bit pattern of `Self` is a valid value and `Self` has no padding, so a mapped file's
|
||||
/// bytes can be viewed as `[Self]` and a file can be written from one.
|
||||
pub unsafe trait Pod: Copy + 'static {}
|
||||
|
||||
unsafe impl Pod for u8 {}
|
||||
unsafe impl Pod for u32 {}
|
||||
unsafe impl Pod for u64 {}
|
||||
|
||||
enum Backing<T> {
|
||||
Ram(Vec<T>),
|
||||
Map(MmapMut),
|
||||
ReadOnly(Mmap),
|
||||
}
|
||||
|
||||
/// A fixed-length array of `T`. File contents use the machine's byte order: the arrays are a
|
||||
/// cache for this machine and are never moved to another one.
|
||||
pub struct Column<T: Pod> {
|
||||
backing: Backing<T>,
|
||||
/// Scratch file to delete on drop where it could not be unlinked while mapped.
|
||||
delete_on_drop: Option<PathBuf>,
|
||||
_marker: PhantomData<T>,
|
||||
}
|
||||
|
||||
fn byte_len<T: Pod>(len: usize) -> io::Result<usize> {
|
||||
len.checked_mul(size_of::<T>())
|
||||
.filter(|&bytes| bytes <= isize::MAX as usize)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "column is too long"))
|
||||
}
|
||||
|
||||
fn is_zero<T: Pod>(value: &T) -> bool {
|
||||
// SAFETY: `Pod` types have no padding, so all `size_of::<T>()` bytes are initialized.
|
||||
let bytes = unsafe { std::slice::from_raw_parts((value as *const T).cast::<u8>(), size_of::<T>()) };
|
||||
bytes.iter().all(|&b| b == 0)
|
||||
}
|
||||
|
||||
impl<T: Pod> Column<T> {
|
||||
fn new(backing: Backing<T>, delete_on_drop: Option<PathBuf>) -> Self {
|
||||
Self { backing, delete_on_drop, _marker: PhantomData }
|
||||
}
|
||||
|
||||
pub fn ram(len: usize, fill: T) -> Self {
|
||||
Self::new(Backing::Ram(vec![fill; len]), None)
|
||||
}
|
||||
|
||||
/// Creates (or truncates) `path` and maps it; the file stays after the column is dropped.
|
||||
pub fn file(path: &Path, len: usize, fill: T) -> io::Result<Self> {
|
||||
let bytes = byte_len::<T>(len)?;
|
||||
let file = OpenOptions::new().read(true).write(true).create(true).truncate(true).open(path)?;
|
||||
match Self::map_new(&file, len, bytes, fill) {
|
||||
Ok(column) => Ok(column),
|
||||
Err(err) => {
|
||||
drop(file);
|
||||
let _ = std::fs::remove_file(path);
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A temporary column in `dir` that never outlives the process. On unix the file is unlinked
|
||||
/// right after mapping, so even a crash leaves nothing behind; elsewhere (Windows cannot delete
|
||||
/// a mapped file) it is deleted on drop.
|
||||
pub fn scratch(dir: &Path, len: usize, fill: T) -> io::Result<Self> {
|
||||
let bytes = byte_len::<T>(len)?;
|
||||
if bytes == 0 {
|
||||
return Ok(Self::ram(0, fill));
|
||||
}
|
||||
let path = dir.join(format!("scratch-{}.tmp", uuid::Uuid::new_v4().simple()));
|
||||
let file = OpenOptions::new().read(true).write(true).create_new(true).open(&path)?;
|
||||
let mut column = match Self::map_new(&file, len, bytes, fill) {
|
||||
Ok(column) => column,
|
||||
Err(err) => {
|
||||
drop(file);
|
||||
let _ = std::fs::remove_file(&path);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
// If the unlink fails the drop-time delete is the fallback, so the file is never left behind.
|
||||
if !cfg!(unix) || std::fs::remove_file(&path).is_err() {
|
||||
column.delete_on_drop = Some(path);
|
||||
}
|
||||
Ok(column)
|
||||
}
|
||||
|
||||
/// Maps an existing file read-only; writing through the column panics.
|
||||
pub fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = File::open(path)?;
|
||||
let bytes = file.metadata()?.len();
|
||||
if bytes % size_of::<T>() as u64 != 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("{} is {bytes} bytes, not a multiple of {}", path.display(), size_of::<T>()),
|
||||
));
|
||||
}
|
||||
if bytes == 0 {
|
||||
return Ok(Self::ram(0, zero_value()));
|
||||
}
|
||||
// SAFETY: the file is private to the snapshot directory; nothing truncates it while mapped.
|
||||
let map = unsafe { Mmap::map(&file)? };
|
||||
Ok(Self::new(Backing::ReadOnly(map), None))
|
||||
}
|
||||
|
||||
fn map_new(file: &File, len: usize, bytes: usize, fill: T) -> io::Result<Self> {
|
||||
if bytes == 0 {
|
||||
return Ok(Self::ram(0, fill));
|
||||
}
|
||||
file.set_len(bytes as u64)?;
|
||||
// SAFETY: as in `open`; the file was just created by this process.
|
||||
let map = unsafe { MmapMut::map_mut(file)? };
|
||||
let mut column = Self::new(Backing::Map(map), None);
|
||||
// A fresh file reads as zeros, and writing a zero fill would fault in every page.
|
||||
if !is_zero(&fill) {
|
||||
debug_assert_eq!(column.len(), len);
|
||||
column.fill(fill);
|
||||
}
|
||||
Ok(column)
|
||||
}
|
||||
}
|
||||
|
||||
fn zero_value<T: Pod>() -> T {
|
||||
// SAFETY: every bit pattern of a `Pod` type is a valid value.
|
||||
unsafe { std::mem::zeroed() }
|
||||
}
|
||||
|
||||
impl<T: Pod> Deref for Column<T> {
|
||||
type Target = [T];
|
||||
|
||||
fn deref(&self) -> &[T] {
|
||||
match &self.backing {
|
||||
Backing::Ram(v) => v,
|
||||
// SAFETY: maps are page-aligned, `T` is `Pod` and the length is a multiple of its width.
|
||||
Backing::Map(m) => unsafe { std::slice::from_raw_parts(m.as_ptr().cast(), m.len() / size_of::<T>()) },
|
||||
Backing::ReadOnly(m) => unsafe { std::slice::from_raw_parts(m.as_ptr().cast(), m.len() / size_of::<T>()) },
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Pod> DerefMut for Column<T> {
|
||||
fn deref_mut(&mut self) -> &mut [T] {
|
||||
match &mut self.backing {
|
||||
Backing::Ram(v) => v,
|
||||
// SAFETY: as in `deref`; `&mut self` makes the access exclusive.
|
||||
Backing::Map(m) => unsafe {
|
||||
std::slice::from_raw_parts_mut(m.as_mut_ptr().cast(), m.len() / size_of::<T>())
|
||||
},
|
||||
Backing::ReadOnly(_) => panic!("column opened from a file is read-only"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Pod> Drop for Column<T> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(path) = self.delete_on_drop.take() {
|
||||
// Unmap before deleting: Windows refuses to delete a mapped file.
|
||||
self.backing = Backing::Ram(Vec::new());
|
||||
let _ = std::fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Where a computation puts its arrays: result columns are named files in `Dir`, temporary arrays
|
||||
/// are scratch columns in the same directory.
|
||||
pub enum Store {
|
||||
Ram,
|
||||
Dir(PathBuf),
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// `name` is the whole file name (`types.u32`); it is used as given.
|
||||
pub fn column<T: Pod>(&self, name: &str, len: usize, fill: T) -> io::Result<Column<T>> {
|
||||
match self {
|
||||
Store::Ram => Ok(Column::ram(len, fill)),
|
||||
Store::Dir(dir) => Column::file(&dir.join(name), len, fill),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scratch<T: Pod>(&self, len: usize, fill: T) -> io::Result<Column<T>> {
|
||||
match self {
|
||||
Store::Ram => Ok(Column::ram(len, fill)),
|
||||
Store::Dir(dir) => Column::scratch(dir, len, fill),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn dir_is_empty(dir: &Path) -> bool {
|
||||
std::fs::read_dir(dir).unwrap().next().is_none()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_column_round_trips() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("x.u32");
|
||||
{
|
||||
let mut col = Column::<u32>::file(&path, 4, 7).unwrap();
|
||||
assert_eq!(&*col, &[7, 7, 7, 7]);
|
||||
col.copy_from_slice(&[1, 2, 3, 4]);
|
||||
}
|
||||
let col = Column::<u32>::open(&path).unwrap();
|
||||
assert_eq!(&*col, &[1, 2, 3, 4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scratch_column_leaves_no_file() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
{
|
||||
let mut col = Column::<u64>::scratch(dir.path(), 1024, 9).unwrap();
|
||||
col[1023] = 5;
|
||||
assert_eq!((col[0], col[1023]), (9, 5));
|
||||
// Windows cannot delete a mapped file, so only unix unlinks while the map is alive.
|
||||
#[cfg(unix)]
|
||||
assert!(dir_is_empty(dir.path()));
|
||||
}
|
||||
assert!(dir_is_empty(dir.path()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zero_fill_and_non_zero_fill() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let zeros = Column::<u64>::file(&dir.path().join("z.u64"), 3000, 0).unwrap();
|
||||
assert!(zeros.iter().all(|&v| v == 0));
|
||||
let ones = Column::<u8>::file(&dir.path().join("o.u8"), 10_000, 0xAB).unwrap();
|
||||
assert!(ones.iter().all(|&v| v == 0xAB));
|
||||
let ram = Column::<u32>::ram(5, NONE);
|
||||
assert_eq!(&*ram, &[NONE; 5]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_columns_work_everywhere() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("e.u32");
|
||||
assert_eq!(Column::<u32>::ram(0, 1).len(), 0);
|
||||
assert_eq!(Column::<u32>::file(&path, 0, 1).unwrap().len(), 0);
|
||||
assert_eq!(Column::<u32>::open(&path).unwrap().len(), 0);
|
||||
assert_eq!(Column::<u32>::scratch(dir.path(), 0, 1).unwrap().len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn open_rejects_a_length_that_is_not_a_multiple_of_the_width() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("odd.u32");
|
||||
std::fs::write(&path, [0u8; 6]).unwrap();
|
||||
let err = Column::<u32>::open(&path).err().unwrap();
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_rejects_a_byte_length_that_overflows() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let err = Column::<u64>::file(&dir.path().join("big.u64"), usize::MAX, 0).err().unwrap();
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidInput);
|
||||
assert!(dir_is_empty(dir.path()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(expected = "read-only")]
|
||||
fn opened_columns_are_read_only() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let path = dir.path().join("r.u32");
|
||||
drop(Column::<u32>::file(&path, 2, 0).unwrap());
|
||||
let mut col = Column::<u32>::open(&path).unwrap();
|
||||
col[0] = 1;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_names_files_exactly_as_given() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let store = Store::Dir(dir.path().to_path_buf());
|
||||
let mut col = store.column::<u32>("types.u32", 3, 0).unwrap();
|
||||
col[2] = 8;
|
||||
drop(col);
|
||||
let names: Vec<_> =
|
||||
std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect();
|
||||
assert_eq!(names, ["types.u32"]);
|
||||
assert_eq!(&*Column::<u32>::open(&dir.path().join("types.u32")).unwrap(), &[0, 0, 8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn store_scratch_is_anonymous_and_ram_store_touches_no_disk() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let store = Store::Dir(dir.path().to_path_buf());
|
||||
drop(store.scratch::<u32>(16, 1).unwrap());
|
||||
assert!(dir_is_empty(dir.path()));
|
||||
let ram = Store::Ram.column::<u8>("ignored.u8", 4, 2).unwrap();
|
||||
assert_eq!(&*ram, &[2; 4]);
|
||||
assert_eq!(&*Store::Ram.scratch::<u8>(2, 3).unwrap(), &[3, 3]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
//! Class-by-class differences between two snapshots.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::index::HeapIndex;
|
||||
|
||||
/// One class name in two snapshots; a side without the name has zeros.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapClassDiff {
|
||||
pub name: String,
|
||||
pub base_count: u64,
|
||||
pub count: u64,
|
||||
pub base_shallow: u64,
|
||||
pub shallow: u64,
|
||||
pub base_retained: u64,
|
||||
pub retained: u64,
|
||||
/// Several classes have this name on one side or both, so the sides cannot be matched: each
|
||||
/// side shows the sum over its classes, and there is no difference to show.
|
||||
pub ambiguous: bool,
|
||||
}
|
||||
|
||||
/// Joined by class name, the only key two dumps share. By the change in retained size
|
||||
/// descending (none for an ambiguous name), then by name.
|
||||
pub fn compare(base: &HeapIndex, current: &HeapIndex) -> Vec<HeapClassDiff> {
|
||||
let mut by_name: HashMap<String, HeapClassDiff> = HashMap::new();
|
||||
for (index, is_base) in [(base, true), (current, false)] {
|
||||
for row in index.class_rows() {
|
||||
let diff = by_name.entry(row.name).or_insert_with_key(|name| HeapClassDiff {
|
||||
name: name.clone(),
|
||||
base_count: 0,
|
||||
count: 0,
|
||||
base_shallow: 0,
|
||||
shallow: 0,
|
||||
base_retained: 0,
|
||||
retained: 0,
|
||||
ambiguous: false,
|
||||
});
|
||||
let (count, shallow, retained) = if is_base {
|
||||
(&mut diff.base_count, &mut diff.base_shallow, &mut diff.base_retained)
|
||||
} else {
|
||||
(&mut diff.count, &mut diff.shallow, &mut diff.retained)
|
||||
};
|
||||
*count = count.saturating_add(row.count);
|
||||
*shallow = shallow.saturating_add(row.shallow);
|
||||
*retained = retained.saturating_add(row.retained);
|
||||
diff.ambiguous |= row.ambiguous;
|
||||
}
|
||||
}
|
||||
let change = |d: &HeapClassDiff| if d.ambiguous { 0 } else { d.retained.abs_diff(d.base_retained) };
|
||||
let mut diffs: Vec<HeapClassDiff> = by_name.into_values().collect();
|
||||
diffs.sort_by(|a, b| change(b).cmp(&change(a)).then_with(|| a.name.cmp(&b.name)));
|
||||
diffs
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,78 @@
|
||||
use super::*;
|
||||
use crate::hprof::HeapRootKind;
|
||||
use crate::test_index::analyzed;
|
||||
use crate::testing::{MiniHeap, sample, sample_with_two_more_nodes};
|
||||
|
||||
fn diff<'a>(diffs: &'a [HeapClassDiff], name: &str) -> &'a HeapClassDiff {
|
||||
let mut rows = diffs.iter().filter(|d| d.name == name);
|
||||
let row = rows.next().unwrap_or_else(|| panic!("no diff for {name}"));
|
||||
assert!(rows.next().is_none(), "two diffs for {name}");
|
||||
row
|
||||
}
|
||||
|
||||
fn sort_key(d: &HeapClassDiff) -> u64 {
|
||||
if d.ambiguous { 0 } else { d.retained.abs_diff(d.base_retained) }
|
||||
}
|
||||
|
||||
fn assert_sorted(diffs: &[HeapClassDiff]) {
|
||||
assert!(diffs.windows(2).all(|p| (sort_key(&p[1]), &p[0].name) < (sort_key(&p[0]), &p[1].name)), "{diffs:#?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compare_counts_by_class() {
|
||||
let base = analyzed(&sample());
|
||||
let mut current = sample_with_two_more_nodes();
|
||||
current
|
||||
.class("Extra", Some("java.lang.Object"), &[])
|
||||
.object(0x4000, "Extra", &[])
|
||||
.root(HeapRootKind::JniGlobal, 0x4000);
|
||||
let diffs = compare(&base, &analyzed(¤t));
|
||||
let node = diff(&diffs, "Node");
|
||||
assert_eq!((node.base_count, node.count), (3, 5));
|
||||
assert_eq!(node.shallow - node.base_shallow, 48);
|
||||
assert_eq!((node.base_retained, node.retained), (104, 152));
|
||||
assert!(!node.ambiguous);
|
||||
let extra = diff(&diffs, "Extra");
|
||||
assert_eq!((extra.base_count, extra.base_shallow, extra.base_retained, extra.count), (0, 0, 0, 1));
|
||||
assert_eq!(diffs[0].name, "Node");
|
||||
let holder = diff(&diffs, "Holder");
|
||||
assert_eq!((holder.base_count, holder.count, holder.base_retained, holder.retained), (1, 1, 72, 72));
|
||||
assert_sorted(&diffs);
|
||||
|
||||
// A class only in the base snapshot.
|
||||
let gone = compare(&analyzed(¤t), &base);
|
||||
let extra = diff(&gone, "Extra");
|
||||
assert_eq!((extra.base_count, extra.count, extra.shallow, extra.retained), (1, 0, 0, 0));
|
||||
assert_sorted(&gone);
|
||||
}
|
||||
|
||||
fn two_foos(extra: bool) -> MiniHeap {
|
||||
let mut heap = sample();
|
||||
heap.class("Foo", Some("java.lang.Object"), &[])
|
||||
.object(0x3000, "Foo", &[])
|
||||
.class("Foo", Some("java.lang.Object"), &[])
|
||||
.object(0x3100, "Foo", &[]);
|
||||
if extra {
|
||||
heap.object(0x3200, "Foo", &[]);
|
||||
}
|
||||
for addr in [0x3000, 0x3100, 0x3200] {
|
||||
heap.root(HeapRootKind::JniGlobal, addr);
|
||||
}
|
||||
heap
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compare_gives_no_diff_for_a_name_of_several_classes() {
|
||||
let diffs = compare(&analyzed(&two_foos(false)), &analyzed(&two_foos(true)));
|
||||
let foo = diff(&diffs, "Foo");
|
||||
assert!(foo.ambiguous);
|
||||
assert_eq!((foo.base_count, foo.count), (2, 3));
|
||||
assert!(diffs.iter().filter(|d| d.name != "Foo").all(|d| !d.ambiguous));
|
||||
assert_sorted(&diffs);
|
||||
|
||||
// Ambiguous on one side only is still ambiguous.
|
||||
let diffs = compare(&analyzed(&sample()), &analyzed(&two_foos(false)));
|
||||
let foo = diff(&diffs, "Foo");
|
||||
assert!(foo.ambiguous);
|
||||
assert_eq!((foo.base_count, foo.count), (0, 2));
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
//! Disk-space estimates and the per-snapshot directories under the app's cache.
|
||||
//!
|
||||
//! A snapshot directory holds the index columns and, for a capture, the dump itself. Its `owner`
|
||||
//! file names the host process that created it, so a host that starts later can remove the
|
||||
//! directories of hosts that died without cleaning up.
|
||||
|
||||
use std::fs;
|
||||
use std::io::{self, Read};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
/// Space to leave free on top of an estimate.
|
||||
pub const DISK_MARGIN: u64 = 1 << 30;
|
||||
|
||||
/// hprof ≈ `bytes + 16 × instances`; index columns and scratch arrays ≈ `90 × instances`.
|
||||
const CAPTURE_BYTES_PER_INSTANCE: u64 = 106;
|
||||
const INDEX_BYTES_PER_OBJECT: u64 = 90;
|
||||
|
||||
const OWNER_FILE: &str = "owner";
|
||||
const TMP_SUFFIX: &str = ".tmp";
|
||||
/// A directory still being created by a live host has no `owner` for a moment; one that stays
|
||||
/// unfinished for longer than this was abandoned.
|
||||
const TMP_GRACE: Duration = Duration::from_secs(60 * 60);
|
||||
|
||||
/// Bytes of free space that a dump of `heap_bytes` bytes holding `instances` instances needs,
|
||||
/// including its index.
|
||||
pub fn capture_need(heap_bytes: u64, instances: u64) -> u64 {
|
||||
heap_bytes.saturating_add(CAPTURE_BYTES_PER_INSTANCE.saturating_mul(instances)).saturating_add(DISK_MARGIN)
|
||||
}
|
||||
|
||||
/// Bytes of free space that indexing a dump of `objects` objects needs.
|
||||
pub fn index_need(objects: u64) -> u64 {
|
||||
INDEX_BYTES_PER_OBJECT.saturating_mul(objects).saturating_add(DISK_MARGIN)
|
||||
}
|
||||
|
||||
/// Bytes available to this process on the volume holding `dir`; `None` where it is not checked
|
||||
/// (not unix) or cannot be read, so callers go ahead rather than refuse.
|
||||
pub fn free_bytes(dir: &Path) -> Option<u64> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
|
||||
let path = std::ffi::CString::new(dir.as_os_str().as_bytes()).ok()?;
|
||||
// SAFETY: `path` is NUL-terminated and `stat` is only read after `statvfs` filled it in.
|
||||
let stat = unsafe {
|
||||
let mut stat = std::mem::MaybeUninit::<libc::statvfs>::zeroed();
|
||||
if libc::statvfs(path.as_ptr(), stat.as_mut_ptr()) != 0 {
|
||||
return None;
|
||||
}
|
||||
stat.assume_init()
|
||||
};
|
||||
// The field widths differ between unix targets.
|
||||
#[allow(clippy::unnecessary_cast)]
|
||||
Some((stat.f_bavail as u64).saturating_mul(stat.f_frsize as u64))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = dir;
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
/// `<local_data_dir>/cache/heap-snapshots`.
|
||||
pub fn snapshots_root(local_data_dir: &Path) -> PathBuf {
|
||||
local_data_dir.join("cache").join("heap-snapshots")
|
||||
}
|
||||
|
||||
/// Creates `<root>/<uuid>/` holding an `owner` file with this process's pid. The directory is
|
||||
/// built as `<uuid>.tmp` and renamed, so another host scanning `root` never sees it without its
|
||||
/// owner and removes it as abandoned.
|
||||
pub fn create_snapshot_dir(root: &Path) -> io::Result<PathBuf> {
|
||||
fs::create_dir_all(root)?;
|
||||
let name = uuid::Uuid::new_v4().to_string();
|
||||
let building = root.join(format!("{name}{TMP_SUFFIX}"));
|
||||
let dir = root.join(&name);
|
||||
create_private_dir(&building)?;
|
||||
let published =
|
||||
fs::write(building.join(OWNER_FILE), std::process::id().to_string()).and_then(|()| fs::rename(&building, &dir));
|
||||
if let Err(err) = published {
|
||||
let _ = fs::remove_dir_all(&building);
|
||||
return Err(err);
|
||||
}
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
/// A dump holds whatever was in the JVM's memory, passwords and tokens included, so on unix only
|
||||
/// the owner may enter the directory. Windows keeps the ACL inherited from the user's data dir.
|
||||
fn create_private_dir(path: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::DirBuilderExt;
|
||||
fs::DirBuilder::new().mode(0o700).create(path)
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
fs::DirBuilder::new().create(path)
|
||||
}
|
||||
}
|
||||
|
||||
/// Removes the directories under `root` whose owner file is missing, unreadable or names a pid
|
||||
/// for which `alive` is false, and unfinished `*.tmp` ones older than an hour. Only directories
|
||||
/// directly under `root` are touched; a symlink there is left alone.
|
||||
pub fn remove_stale(root: &Path, alive: impl Fn(u32) -> bool) {
|
||||
remove_stale_at(root, alive, SystemTime::now());
|
||||
}
|
||||
|
||||
fn remove_stale_at(root: &Path, alive: impl Fn(u32) -> bool, now: SystemTime) {
|
||||
let Ok(entries) = fs::read_dir(root) else { return };
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
// `symlink_metadata` does not follow: a link out of `root` is not a directory here.
|
||||
let Ok(meta) = fs::symlink_metadata(&path) else { continue };
|
||||
if !meta.file_type().is_dir() {
|
||||
continue;
|
||||
}
|
||||
let stale = if path.extension().is_some_and(|ext| ext == "tmp") {
|
||||
meta.modified()
|
||||
.ok()
|
||||
.and_then(|modified| now.duration_since(modified).ok())
|
||||
.is_some_and(|age| age >= TMP_GRACE)
|
||||
} else {
|
||||
read_owner(&path).is_none_or(|pid| !alive(pid))
|
||||
};
|
||||
if stale && let Err(err) = fs::remove_dir_all(&path) {
|
||||
tracing::warn!(dir = %path.display(), %err, "could not remove a stale heap snapshot directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_owner(dir: &Path) -> Option<u32> {
|
||||
let mut text = String::new();
|
||||
fs::File::open(dir.join(OWNER_FILE)).ok()?.take(32).read_to_string(&mut text).ok()?;
|
||||
text.trim().parse().ok()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,137 @@
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn names(dir: &Path) -> Vec<String> {
|
||||
let mut names: Vec<_> =
|
||||
std::fs::read_dir(dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
fn dir_with_owner(root: &Path, name: &str, owner: Option<&str>) {
|
||||
let dir = root.join(name);
|
||||
std::fs::create_dir(&dir).unwrap();
|
||||
std::fs::write(dir.join("dump.hprof"), b"x").unwrap();
|
||||
if let Some(owner) = owner {
|
||||
std::fs::write(dir.join(OWNER_FILE), owner).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimates() {
|
||||
assert_eq!(capture_need(664_018_520, 20_087_506), 664_018_520 + 106 * 20_087_506 + (1 << 30));
|
||||
assert_eq!(index_need(10), 900 + (1 << 30));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn estimates_saturate_instead_of_wrapping() {
|
||||
assert_eq!(capture_need(u64::MAX, u64::MAX), u64::MAX);
|
||||
assert_eq!(index_need(u64::MAX), u64::MAX);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn not_enough_space_for_estimate() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let free = free_bytes(dir.path()).expect("statvfs works on a temp dir");
|
||||
assert!(free > 0);
|
||||
assert!(capture_need(u64::MAX / 4, 0) > free);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn free_bytes_of_a_missing_dir_is_unknown() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
assert_eq!(free_bytes(&dir.path().join("missing")), None);
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
#[test]
|
||||
fn free_space_is_not_checked_off_unix() {
|
||||
assert_eq!(free_bytes(Path::new(".")), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshots_root_is_under_the_cache() {
|
||||
let local = Path::new("local");
|
||||
assert_eq!(snapshots_root(local), local.join("cache").join("heap-snapshots"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_dir_records_its_owner_and_leaves_no_tmp() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let missing_parent = root.path().join("cache").join("heap-snapshots");
|
||||
let dir = create_snapshot_dir(&missing_parent).unwrap();
|
||||
assert_eq!(dir.parent(), Some(missing_parent.as_path()));
|
||||
assert!(uuid::Uuid::parse_str(dir.file_name().unwrap().to_str().unwrap()).is_ok());
|
||||
assert_eq!(std::fs::read_to_string(dir.join(OWNER_FILE)).unwrap(), std::process::id().to_string());
|
||||
assert_eq!(names(&missing_parent), [dir.file_name().unwrap().to_str().unwrap()]);
|
||||
assert_ne!(create_snapshot_dir(&missing_parent).unwrap(), dir);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn snapshot_dir_is_private_to_the_owner() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let dir = create_snapshot_dir(root.path()).unwrap();
|
||||
assert_eq!(std::fs::metadata(&dir).unwrap().permissions().mode() & 0o777, 0o700);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_dirs_removed_on_start() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
dir_with_owner(root.path(), "a", Some("999999"));
|
||||
dir_with_owner(root.path(), "b", Some(&std::process::id().to_string()));
|
||||
dir_with_owner(root.path(), "c", None);
|
||||
remove_stale(root.path(), |pid| pid == std::process::id());
|
||||
assert_eq!(names(root.path()), ["b"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_that_is_not_a_pid_is_stale() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
dir_with_owner(root.path(), "empty", Some(""));
|
||||
dir_with_owner(root.path(), "word", Some("not-a-pid"));
|
||||
dir_with_owner(root.path(), "negative", Some("-1"));
|
||||
dir_with_owner(root.path(), "padded", Some(&format!(" {}\n", std::process::id())));
|
||||
remove_stale(root.path(), |pid| pid == std::process::id());
|
||||
assert_eq!(names(root.path()), ["padded"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_scan_tolerates_a_missing_root_and_leaves_loose_files_alone() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
remove_stale(&root.path().join("missing"), |_| false);
|
||||
std::fs::write(root.path().join("notes.txt"), b"keep").unwrap();
|
||||
remove_stale(root.path(), |_| false);
|
||||
assert_eq!(names(root.path()), ["notes.txt"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unfinished_dir_is_left_alone_for_an_hour() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
// Another host may be between `create_dir` and the rename of its own snapshot dir.
|
||||
dir_with_owner(root.path(), "young.tmp", None);
|
||||
let now = SystemTime::now();
|
||||
remove_stale_at(root.path(), |_| false, now);
|
||||
assert_eq!(names(root.path()), ["young.tmp"]);
|
||||
remove_stale_at(root.path(), |_| false, now + TMP_GRACE - Duration::from_secs(60));
|
||||
assert_eq!(names(root.path()), ["young.tmp"]);
|
||||
remove_stale_at(root.path(), |_| false, now + TMP_GRACE + Duration::from_secs(60));
|
||||
assert!(names(root.path()).is_empty());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn stale_scan_never_follows_a_symlink_out_of_the_root() {
|
||||
let outside = tempfile::tempdir().unwrap();
|
||||
std::fs::write(outside.path().join("precious"), b"x").unwrap();
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::os::unix::fs::symlink(outside.path(), root.path().join("link")).unwrap();
|
||||
remove_stale(root.path(), |_| false);
|
||||
remove_stale_at(root.path(), |_| false, SystemTime::now() + 2 * TMP_GRACE);
|
||||
assert_eq!(names(outside.path()), ["precious"]);
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
//! Dominator tree of the object graph, by Lengauer–Tarjan with path compression and no recursion:
|
||||
//! the DFS "stack" is the chain of DFS parents, and `eval` reverses the links of the path it
|
||||
//! compresses instead of keeping a stack.
|
||||
//!
|
||||
//! Between the DFS and the final translation every array is indexed by DFS number, so the root is
|
||||
//! 0 and a vertex's DFS ancestors all have smaller numbers.
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use crate::column::{Column, NONE, Store};
|
||||
use crate::error::HeapError;
|
||||
use crate::files::IDOM_FILE;
|
||||
use crate::graph::accumulate;
|
||||
use crate::progress::{CHECK_EVERY, HeapPhase, Progress};
|
||||
|
||||
const PHASE: HeapPhase = HeapPhase::Dominators;
|
||||
|
||||
pub struct Dominators {
|
||||
/// Immediate dominator of each node, saved as `idom.u32` in the store; [`NONE`] for
|
||||
/// unreachable nodes, `root` for `root`.
|
||||
pub idom: Column<u32>,
|
||||
/// Nodes in DFS preorder from `root`, `reachable` entries.
|
||||
pub preorder: Column<u32>,
|
||||
pub reachable: u32,
|
||||
}
|
||||
|
||||
/// The dominators of the graph whose node `v` has targets `out[out_start[v]..out_start[v + 1]]`,
|
||||
/// from `root` (which must be a node). `progress` gets `Dominators` from `(0, total)` to
|
||||
/// `(total, total)`; an `Err` from it stops the computation with that error.
|
||||
pub fn dominators(
|
||||
out_start: &[u32],
|
||||
out: &[u32],
|
||||
root: u32,
|
||||
store: &Store,
|
||||
progress: &dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<Dominators, HeapError> {
|
||||
compute(out_start, out, root, store, progress, cancel).map(|(dominators, _)| dominators)
|
||||
}
|
||||
|
||||
/// [`dominators`], plus the number of steps of work it took.
|
||||
fn compute(
|
||||
out_start: &[u32],
|
||||
out: &[u32],
|
||||
root: u32,
|
||||
store: &Store,
|
||||
progress: &dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<(Dominators, u64), HeapError> {
|
||||
let nodes = out_start.len().saturating_sub(1);
|
||||
if nodes >= NONE as usize {
|
||||
return Err(HeapError::TooLarge);
|
||||
}
|
||||
assert!((root as usize) < nodes, "root {root} is not one of the {nodes} nodes");
|
||||
// Three walks of at most `nodes` vertices each: DFS, semidominators, immediate dominators.
|
||||
let span = nodes as u64;
|
||||
let mut work = Work::new(Progress::new(progress, cancel), 3 * span);
|
||||
work.report(0)?;
|
||||
|
||||
// DFS: `pre` maps nodes to DFS numbers and `vertex` back; `cursor[d]` is the next edge of
|
||||
// DFS vertex d to follow.
|
||||
let mut pre = store.scratch::<u32>(nodes, NONE)?;
|
||||
let mut vertex = store.scratch::<u32>(nodes, 0)?;
|
||||
let mut parent = store.scratch::<u32>(nodes, 0)?;
|
||||
let mut cursor = store.scratch::<u32>(nodes, 0)?;
|
||||
pre[root as usize] = 0;
|
||||
vertex[0] = root;
|
||||
cursor[0] = out_start[root as usize];
|
||||
let (mut reachable, mut d) = (1u32, 0u32);
|
||||
loop {
|
||||
work.tick()?;
|
||||
let edge = cursor[d as usize];
|
||||
if edge < out_start[vertex[d as usize] as usize + 1] {
|
||||
cursor[d as usize] = edge + 1;
|
||||
let w = out[edge as usize] as usize;
|
||||
if pre[w] == NONE {
|
||||
let next = reachable as usize;
|
||||
pre[w] = reachable;
|
||||
vertex[next] = w as u32;
|
||||
parent[next] = d;
|
||||
cursor[next] = out_start[w];
|
||||
d = reachable;
|
||||
reachable += 1;
|
||||
work.done = u64::from(reachable);
|
||||
}
|
||||
} else if d == 0 {
|
||||
break;
|
||||
} else {
|
||||
d = parent[d as usize];
|
||||
}
|
||||
}
|
||||
let r = reachable as usize;
|
||||
work.report(span)?;
|
||||
|
||||
// Predecessors by DFS number, as CSR: counted into `starts[w + 1]`, then filled through
|
||||
// `starts[w]` as a cursor, which leaves it at w's end until the column is shifted one slot.
|
||||
let mut semi = cursor;
|
||||
let mut label = store.scratch::<u32>(r, 0)?;
|
||||
let mut starts = store.scratch::<u32>(r + 1, 0)?;
|
||||
for d in 0..r {
|
||||
work.tick()?;
|
||||
semi[d] = d as u32;
|
||||
label[d] = d as u32;
|
||||
for &w in targets(out_start, out, vertex[d]) {
|
||||
work.tick()?;
|
||||
starts[pre[w as usize] as usize + 1] += 1;
|
||||
}
|
||||
}
|
||||
let edges = accumulate(&mut starts, cancel)?;
|
||||
let mut preds = store.scratch::<u32>(edges as usize, 0)?;
|
||||
for d in 0..r {
|
||||
work.tick()?;
|
||||
for &w in targets(out_start, out, vertex[d]) {
|
||||
work.tick()?;
|
||||
let slot = &mut starts[pre[w as usize] as usize];
|
||||
preds[*slot as usize] = d as u32;
|
||||
*slot += 1;
|
||||
}
|
||||
}
|
||||
starts.copy_within(0..r, 1);
|
||||
starts[0] = 0;
|
||||
drop(pre);
|
||||
work.report(span)?;
|
||||
|
||||
// Semidominators in reverse preorder, linking each finished vertex to its parent. Vertex w
|
||||
// waits in its semidominator's bucket until a child of that semidominator is linked; then
|
||||
// `eval` gives w's idom, or a vertex whose idom is also w's, fixed up in preorder below.
|
||||
// `idom[v]` is the next vertex in v's bucket until v leaves the bucket.
|
||||
let mut ancestor = store.scratch::<u32>(r, NONE)?;
|
||||
let mut bucket = store.scratch::<u32>(r, NONE)?;
|
||||
let mut idom = store.scratch::<u32>(r, NONE)?;
|
||||
for w in (1..r).rev() {
|
||||
work.done = span + (r - w) as u64;
|
||||
work.tick()?;
|
||||
let mut s = semi[w];
|
||||
for &v in &preds[starts[w] as usize..starts[w + 1] as usize] {
|
||||
work.tick()?;
|
||||
let u = eval(v, &mut ancestor, &mut label, &semi, &mut work)?;
|
||||
s = s.min(semi[u as usize]);
|
||||
}
|
||||
semi[w] = s;
|
||||
idom[w] = bucket[s as usize];
|
||||
bucket[s as usize] = w as u32;
|
||||
let p = parent[w];
|
||||
ancestor[w] = p;
|
||||
let mut v = std::mem::replace(&mut bucket[p as usize], NONE);
|
||||
while v != NONE {
|
||||
work.tick()?;
|
||||
let next = idom[v as usize];
|
||||
let u = eval(v, &mut ancestor, &mut label, &semi, &mut work)?;
|
||||
idom[v as usize] = if semi[u as usize] < semi[v as usize] { u } else { p };
|
||||
v = next;
|
||||
}
|
||||
}
|
||||
drop((label, starts, preds, ancestor, bucket, parent));
|
||||
work.report(2 * span)?;
|
||||
|
||||
// Immediate dominators, translated back to nodes; in preorder, so a deferred vertex's
|
||||
// stand-in already has its final idom.
|
||||
let mut node_idom = store.column::<u32>(IDOM_FILE, nodes, NONE)?;
|
||||
let mut preorder = store.scratch::<u32>(r, 0)?;
|
||||
idom[0] = 0;
|
||||
node_idom[root as usize] = root;
|
||||
preorder[0] = root;
|
||||
for w in 1..r {
|
||||
work.done = 2 * span + w as u64;
|
||||
work.tick()?;
|
||||
if idom[w] != semi[w] {
|
||||
idom[w] = idom[idom[w] as usize];
|
||||
}
|
||||
node_idom[vertex[w] as usize] = vertex[idom[w] as usize];
|
||||
preorder[w] = vertex[w];
|
||||
}
|
||||
work.report(3 * span)?;
|
||||
Ok((Dominators { idom: node_idom, preorder, reachable }, work.steps))
|
||||
}
|
||||
|
||||
fn targets<'a>(out_start: &[u32], out: &'a [u32], v: u32) -> &'a [u32] {
|
||||
let v = v as usize;
|
||||
&out[out_start[v] as usize..out_start[v + 1] as usize]
|
||||
}
|
||||
|
||||
/// Counts steps of work and reports progress, which also looks at the cancellation token, at
|
||||
/// every phase boundary and after every [`CHECK_EVERY`] steps, whichever loop they are in.
|
||||
struct Work<'a> {
|
||||
progress: Progress<'a>,
|
||||
total: u64,
|
||||
/// Progress to report at the next check; the loops keep it current.
|
||||
done: u64,
|
||||
steps: u64,
|
||||
next_report: u64,
|
||||
}
|
||||
|
||||
impl<'a> Work<'a> {
|
||||
fn new(progress: Progress<'a>, total: u64) -> Self {
|
||||
Self { progress, total, done: 0, steps: 0, next_report: CHECK_EVERY }
|
||||
}
|
||||
|
||||
fn report(&mut self, done: u64) -> Result<(), HeapError> {
|
||||
self.done = done;
|
||||
self.next_report = self.steps + CHECK_EVERY;
|
||||
self.progress.report(PHASE, done, self.total)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn tick(&mut self) -> Result<(), HeapError> {
|
||||
self.steps += 1;
|
||||
if self.steps < self.next_report { Ok(()) } else { self.report(self.done) }
|
||||
}
|
||||
}
|
||||
|
||||
/// The vertex of smallest semidominator on the forest path from `v` up to, not including, its
|
||||
/// tree's root (`v` itself when it is a root), compressing that path so the root becomes the
|
||||
/// ancestor of every vertex on it. The walk up reverses the `ancestor` links so the walk down can
|
||||
/// fold labels nearest-root first, as the recursive compress does, without a stack.
|
||||
fn eval(v: u32, ancestor: &mut [u32], label: &mut [u32], semi: &[u32], work: &mut Work) -> Result<u32, HeapError> {
|
||||
let at = |x: u32| x as usize;
|
||||
if ancestor[at(v)] == NONE {
|
||||
return Ok(v);
|
||||
}
|
||||
let (mut x, mut below) = (v, NONE);
|
||||
while ancestor[at(ancestor[at(x)])] != NONE {
|
||||
work.tick()?;
|
||||
let up = ancestor[at(x)];
|
||||
ancestor[at(x)] = below;
|
||||
below = x;
|
||||
x = up;
|
||||
}
|
||||
// `x` is the root's child, already compressed; `below` is the vertex under it.
|
||||
let root = ancestor[at(x)];
|
||||
let mut best = label[at(x)];
|
||||
while below != NONE {
|
||||
work.tick()?;
|
||||
let y = at(below);
|
||||
if semi[at(best)] < semi[at(label[y])] {
|
||||
label[y] = best;
|
||||
} else {
|
||||
best = label[y];
|
||||
}
|
||||
below = ancestor[y];
|
||||
ancestor[y] = root;
|
||||
}
|
||||
Ok(label[at(v)])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,348 @@
|
||||
use std::cell::RefCell;
|
||||
use std::time::Instant;
|
||||
|
||||
use proptest::collection::vec;
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::NONE;
|
||||
use crate::graph::build_graph;
|
||||
use crate::testing::{XorShift, sample};
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// CSR of `edges` over `n` nodes, targets in edge order.
|
||||
fn csr(n: usize, edges: &[(u32, u32)]) -> (Vec<u32>, Vec<u32>) {
|
||||
let mut start = vec![0u32; n + 1];
|
||||
for &(from, _) in edges {
|
||||
start[from as usize + 1] += 1;
|
||||
}
|
||||
for v in 0..n {
|
||||
start[v + 1] += start[v];
|
||||
}
|
||||
let mut cursor = start.clone();
|
||||
let mut out = vec![0u32; edges.len()];
|
||||
for &(from, to) in edges {
|
||||
out[cursor[from as usize] as usize] = to;
|
||||
cursor[from as usize] += 1;
|
||||
}
|
||||
(start, out)
|
||||
}
|
||||
|
||||
fn run(n: usize, edges: &[(u32, u32)], root: u32) -> Dominators {
|
||||
let (start, out) = csr(n, edges);
|
||||
dominators(&start, &out, root, &Store::Ram, &no_progress, &CancellationToken::new()).unwrap()
|
||||
}
|
||||
|
||||
fn preds(n: usize, edges: &[(u32, u32)]) -> Vec<Vec<u32>> {
|
||||
let mut preds = vec![Vec::new(); n];
|
||||
for &(from, to) in edges {
|
||||
preds[to as usize].push(from);
|
||||
}
|
||||
preds
|
||||
}
|
||||
|
||||
/// Immediate dominators from iterated `Dom` sets (one `u64` bitset per node, `n <= 64`).
|
||||
fn dataflow_idom(n: usize, edges: &[(u32, u32)], root: u32) -> Vec<u32> {
|
||||
let (start, out) = csr(n, edges);
|
||||
let mut reachable = vec![false; n];
|
||||
let mut todo = vec![root];
|
||||
reachable[root as usize] = true;
|
||||
while let Some(v) = todo.pop() {
|
||||
for &w in &out[start[v as usize] as usize..start[v as usize + 1] as usize] {
|
||||
if !std::mem::replace(&mut reachable[w as usize], true) {
|
||||
todo.push(w);
|
||||
}
|
||||
}
|
||||
}
|
||||
let preds = preds(n, edges);
|
||||
let mut dom = vec![u64::MAX; n];
|
||||
dom[root as usize] = 1 << root;
|
||||
let mut changed = true;
|
||||
while changed {
|
||||
changed = false;
|
||||
for v in (0..n).filter(|&v| reachable[v] && v != root as usize) {
|
||||
let meet =
|
||||
preds[v].iter().filter(|&&p| reachable[p as usize]).fold(u64::MAX, |acc, &p| acc & dom[p as usize]);
|
||||
let next = meet | 1 << v;
|
||||
if next != dom[v] {
|
||||
dom[v] = next;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
(0..n)
|
||||
.map(|v| {
|
||||
if !reachable[v] {
|
||||
NONE
|
||||
} else if v == root as usize {
|
||||
root
|
||||
} else {
|
||||
// The strict dominator dominated by all the others has the most dominators itself.
|
||||
let strict = dom[v] & !(1 << v);
|
||||
(0..n as u32).filter(|&d| strict & 1 << d != 0).max_by_key(|&d| dom[d as usize].count_ones()).unwrap()
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Cooper–Harvey–Kennedy iteration over reverse postorder from an explicit-stack DFS.
|
||||
fn chk_idom(n: usize, edges: &[(u32, u32)], root: u32) -> Vec<u32> {
|
||||
let (start, out) = csr(n, edges);
|
||||
let mut post_number = vec![NONE; n];
|
||||
let mut postorder = Vec::new();
|
||||
let mut seen = vec![false; n];
|
||||
let mut stack = vec![(root, start[root as usize])];
|
||||
seen[root as usize] = true;
|
||||
while let Some(&mut (v, ref mut next)) = stack.last_mut() {
|
||||
if *next < start[v as usize + 1] {
|
||||
let w = out[*next as usize];
|
||||
*next += 1;
|
||||
if !std::mem::replace(&mut seen[w as usize], true) {
|
||||
stack.push((w, start[w as usize]));
|
||||
}
|
||||
} else {
|
||||
post_number[v as usize] = postorder.len() as u32;
|
||||
postorder.push(v);
|
||||
stack.pop();
|
||||
}
|
||||
}
|
||||
let preds = preds(n, edges);
|
||||
let mut idom = vec![NONE; n];
|
||||
idom[root as usize] = root;
|
||||
let intersect = |idom: &[u32], mut a: u32, mut b: u32| {
|
||||
while a != b {
|
||||
while post_number[a as usize] < post_number[b as usize] {
|
||||
a = idom[a as usize];
|
||||
}
|
||||
while post_number[b as usize] < post_number[a as usize] {
|
||||
b = idom[b as usize];
|
||||
}
|
||||
}
|
||||
a
|
||||
};
|
||||
let mut changed = true;
|
||||
while changed {
|
||||
changed = false;
|
||||
for &v in postorder.iter().rev().filter(|&&v| v != root) {
|
||||
let mut new = NONE;
|
||||
for &p in &preds[v as usize] {
|
||||
if idom[p as usize] != NONE {
|
||||
new = if new == NONE { p } else { intersect(&idom, p, new) };
|
||||
}
|
||||
}
|
||||
if new != idom[v as usize] {
|
||||
idom[v as usize] = new;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
idom
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn diamond() {
|
||||
let d = run(5, &[(0, 1), (0, 2), (1, 3), (2, 3), (3, 4)], 0);
|
||||
assert_eq!(&*d.idom, &[0, 0, 0, 0, 3]);
|
||||
assert_eq!(d.reachable, 5);
|
||||
assert_eq!(&*d.preorder, &[0, 1, 3, 4, 2]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unreachable_is_none() {
|
||||
let d = run(6, &[(0, 1), (0, 2), (1, 3), (2, 3), (3, 4), (5, 3)], 0);
|
||||
assert_eq!(&*d.idom, &[0, 0, 0, 0, 3, NONE]);
|
||||
assert_eq!(d.reachable, 5);
|
||||
assert_eq!(d.preorder.len(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cycle_back_to_root() {
|
||||
let d = run(3, &[(0, 1), (1, 2), (2, 1), (2, 0)], 0);
|
||||
assert_eq!(&*d.idom, &[0, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_need_not_be_node_zero() {
|
||||
let d = run(4, &[(3, 1), (1, 0), (3, 2), (2, 0), (0, 3)], 3);
|
||||
assert_eq!(&*d.idom, &[3, 3, 3, 3]);
|
||||
assert_eq!(d.preorder[0], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sample_heap_dominators() {
|
||||
let bytes = sample().bytes();
|
||||
let g = build_graph(&bytes, &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
let d = dominators(&g.out_start, &g.out, g.root, &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
let node = |id: u64| g.ids.rank(id).unwrap();
|
||||
let idom_of = |id: u64| d.idom[node(id) as usize];
|
||||
assert_eq!(idom_of(0x1100), node(0x1000), "N1 → H");
|
||||
assert_eq!(idom_of(0x1200), node(0x1000), "N2 → H");
|
||||
assert_eq!(idom_of(0x1300), g.root, "N3 → root");
|
||||
assert_eq!(idom_of(0x1400), node(0x1300), "L → N3");
|
||||
assert_eq!(idom_of(0x1500), NONE, "R is unreachable");
|
||||
assert_eq!(idom_of(0x1700), NONE, "N4 only through the weak referent");
|
||||
assert_eq!(d.idom[g.root as usize], g.root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn idom_is_written_to_the_store() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let (start, out) = csr(3, &[(0, 1), (1, 2)]);
|
||||
let store = Store::Dir(dir.path().to_path_buf());
|
||||
let d = dominators(&start, &out, 0, &store, &no_progress, &CancellationToken::new()).unwrap();
|
||||
drop(d);
|
||||
assert_eq!(&*Column::<u32>::open(&dir.path().join("idom.u32")).unwrap(), &[0, 0, 1]);
|
||||
let names: Vec<_> = std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name()).collect();
|
||||
assert_eq!(names, ["idom.u32"], "scratch columns leave no files");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reports_dominators_progress_and_stops_on_cancel() {
|
||||
let (start, out) = csr(3, &[(0, 1), (1, 2)]);
|
||||
let calls = RefCell::new(Vec::new());
|
||||
let record = |phase, done, total| {
|
||||
calls.borrow_mut().push((phase, done, total));
|
||||
Ok(())
|
||||
};
|
||||
dominators(&start, &out, 0, &Store::Ram, &record, &CancellationToken::new()).unwrap();
|
||||
let calls = calls.into_inner();
|
||||
let total = calls[0].2;
|
||||
assert_eq!(calls.first(), Some(&(HeapPhase::Dominators, 0, total)));
|
||||
assert_eq!(calls.last(), Some(&(HeapPhase::Dominators, total, total)));
|
||||
assert!(calls.windows(2).all(|w| w[0].1 <= w[1].1), "progress never goes back: {calls:?}");
|
||||
|
||||
let cancel = CancellationToken::new();
|
||||
let cancel_on_first = |_, _, _| {
|
||||
cancel.cancel();
|
||||
Ok(())
|
||||
};
|
||||
let result = dominators(&start, &out, 0, &Store::Ram, &cancel_on_first, &cancel);
|
||||
assert!(matches!(result, Err(HeapError::Cancelled)));
|
||||
|
||||
let refuse = |_, _, _| Err(HeapError::TooLarge);
|
||||
let result = dominators(&start, &out, 0, &Store::Ram, &refuse, &CancellationToken::new());
|
||||
assert!(matches!(result, Err(HeapError::TooLarge)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_during_idom_phase() {
|
||||
// Long enough that the last phase alone passes a check interval.
|
||||
const NODES: u32 = (CHECK_EVERY + CHECK_EVERY / 2) as u32;
|
||||
let edges: Vec<_> = (1..NODES).map(|v| (v - 1, v)).collect();
|
||||
let (start, out) = csr(NODES as usize, &edges);
|
||||
let span = u64::from(NODES);
|
||||
let cancel = CancellationToken::new();
|
||||
let calls = RefCell::new(Vec::new());
|
||||
let cancel_past_semidominators = |_, done, _| {
|
||||
calls.borrow_mut().push(done);
|
||||
if done > 2 * span {
|
||||
cancel.cancel();
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let result = dominators(&start, &out, 0, &Store::Ram, &cancel_past_semidominators, &cancel);
|
||||
assert!(matches!(result, Err(HeapError::Cancelled)));
|
||||
let calls = calls.into_inner();
|
||||
assert!(calls.contains(&(2 * span)), "the semidominator phase finished: {calls:?}");
|
||||
let last = *calls.last().unwrap();
|
||||
assert!(last > 2 * span && last < 3 * span, "stopped inside the idom phase: {calls:?}");
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn matches_dataflow_on_random_graphs(n in 2usize..64, edges in vec((0u32..64, 0u32..64), 0..192)) {
|
||||
let edges: Vec<_> = edges.into_iter().map(|(a, b)| (a % n as u32, b % n as u32)).collect();
|
||||
let d = run(n, &edges, 0);
|
||||
prop_assert_eq!(d.idom.to_vec(), dataflow_idom(n, &edges, 0));
|
||||
prop_assert_eq!(d.preorder.len(), d.reachable as usize);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_chk_on_a_large_random_graph() {
|
||||
const NODES: u32 = 200_000;
|
||||
const EXTRA_EDGES: u32 = 100_000;
|
||||
let mut rng = XorShift::new(0x5EED_D0D0);
|
||||
let mut edges: Vec<_> = (1..NODES).map(|v| (rng.below(v), v)).collect();
|
||||
edges.extend((0..EXTRA_EDGES).map(|_| (rng.below(NODES), rng.below(NODES))));
|
||||
// Visit edges in a scrambled order so the DFS tree differs from the generating tree.
|
||||
for i in (1..edges.len()).rev() {
|
||||
edges.swap(i, rng.below(i as u32 + 1) as usize);
|
||||
}
|
||||
let started = Instant::now();
|
||||
let d = run(NODES as usize, &edges, 0);
|
||||
let elapsed = started.elapsed();
|
||||
let expected = chk_idom(NODES as usize, &edges, 0);
|
||||
let mismatches = d.idom.iter().zip(&expected).filter(|(a, b)| a != b).count();
|
||||
assert_eq!(mismatches, 0);
|
||||
assert_eq!(d.reachable, NODES);
|
||||
eprintln!("dominators of {NODES} random nodes: {elapsed:?}");
|
||||
}
|
||||
|
||||
/// A doubly linked list whose items an array list also holds: super-root 0 → list 1 → node 0,
|
||||
/// 0 → array list 2 → element array 3 → every item; node j → item j, next, prev.
|
||||
fn shared_list_and_array(k: u32) -> (usize, Vec<(u32, u32)>) {
|
||||
let node = |j: u32| 4 + 2 * j;
|
||||
let item = |j: u32| 5 + 2 * j;
|
||||
let mut edges = vec![(0, 1), (0, 2), (2, 3), (1, node(0))];
|
||||
for j in 0..k {
|
||||
edges.push((node(j), item(j)));
|
||||
if j + 1 < k {
|
||||
edges.push((node(j), node(j + 1)));
|
||||
}
|
||||
if j > 0 {
|
||||
edges.push((node(j), node(j - 1)));
|
||||
}
|
||||
edges.push((3, item(j)));
|
||||
}
|
||||
((4 + 2 * k) as usize, edges)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shared_list_and_array_is_near_linear() {
|
||||
let (n, edges) = shared_list_and_array(29);
|
||||
assert_eq!(run(n, &edges, 0).idom.to_vec(), dataflow_idom(n, &edges, 0));
|
||||
let (n, edges) = shared_list_and_array(3_000);
|
||||
assert_eq!(run(n, &edges, 0).idom.to_vec(), chk_idom(n, &edges, 0));
|
||||
|
||||
const K: u32 = 200_000;
|
||||
let (n, edges) = shared_list_and_array(K);
|
||||
let (start, out) = csr(n, &edges);
|
||||
let started = Instant::now();
|
||||
let (d, steps) = compute(&start, &out, 0, &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
let elapsed = started.elapsed();
|
||||
let item = |j: u32| (5 + 2 * j) as usize;
|
||||
assert!((0..K).all(|j| d.idom[item(j)] == 0), "every item is held by both collections");
|
||||
let m = edges.len() as u64;
|
||||
let bound = (n as u64 + m) * u64::from(n.ilog2());
|
||||
eprintln!("shared list k={K}: {steps} steps (n={n}, m={m}, bound {bound}), {elapsed:?}");
|
||||
assert!(steps <= bound, "{steps} steps for n={n}, m={m}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_chain_has_linear_dominators() {
|
||||
// A small stack proves the algorithm does not recurse along the chain.
|
||||
std::thread::Builder::new()
|
||||
.stack_size(256 << 10)
|
||||
.spawn(|| {
|
||||
const NODES: u32 = 1_000_000;
|
||||
let mut edges: Vec<_> = (1..NODES).map(|v| (v - 1, v)).collect();
|
||||
edges.push((0, NODES - 1));
|
||||
// Makes `eval` compress the whole chain at once.
|
||||
edges.push((NODES - 1, 1));
|
||||
let started = Instant::now();
|
||||
let d = run(NODES as usize, &edges, 0);
|
||||
eprintln!("dominators of a {NODES}-node chain: {:?}", started.elapsed());
|
||||
assert_eq!(d.reachable, NODES);
|
||||
assert_eq!(d.idom[0], 0);
|
||||
assert_eq!(d.idom[NODES as usize - 1], 0);
|
||||
let wrong = (1..NODES - 1).filter(|&v| d.idom[v as usize] != v - 1).count();
|
||||
assert_eq!(wrong, 0);
|
||||
})
|
||||
.unwrap()
|
||||
.join()
|
||||
.unwrap();
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/// Offsets (`at`) are byte positions in the `.hprof` file.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum HeapError {
|
||||
#[error("the dump ends early at byte {at}")]
|
||||
Truncated { at: u64 },
|
||||
#[error("not a heap dump (no JAVA PROFILE header)")]
|
||||
NotHprof,
|
||||
#[error("unsupported identifier size {0}")]
|
||||
UnsupportedIdSize(u32),
|
||||
#[error("unknown heap sub-record tag {tag:#04x} at byte {at}")]
|
||||
UnknownSubRecord { tag: u8, at: u64 },
|
||||
#[error("unknown value type {ty} at byte {at}")]
|
||||
UnknownType { ty: u8, at: u64 },
|
||||
#[error("class {id:#x} used at byte {at} is not in the dump")]
|
||||
MissingClass { id: u64, at: u64 },
|
||||
#[error("the heap has too many objects to analyze")]
|
||||
TooLarge,
|
||||
#[error("the dump's class and field names are too long; it is not a dump a JVM wrote")]
|
||||
NamesTooLarge,
|
||||
#[error("not enough disk space: {needed} bytes needed, {available} free")]
|
||||
NotEnoughDiskSpace { needed: u64, available: u64 },
|
||||
#[error("cancelled")]
|
||||
Cancelled,
|
||||
#[error("{0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
//! Names of the files an analysis writes into its index directory. They live in one list so a
|
||||
//! failed analysis removes exactly these and never the dump or anything else sharing the directory.
|
||||
|
||||
pub(crate) const IDS_BITS_FILE: &str = "ids.bits.u64";
|
||||
pub(crate) const IDS_WRANK_FILE: &str = "ids.wrank.u32";
|
||||
pub(crate) const IDS_TABLE_FILE: &str = "ids.table.u32";
|
||||
pub(crate) const IDS_CHUNKS_FILE: &str = "ids.chunks.u32";
|
||||
pub(crate) const IDS_SORTED_FILE: &str = "ids.sorted.u64";
|
||||
pub(crate) const TYPES_FILE: &str = "types.u32";
|
||||
pub(crate) const OFFSETS_FILE: &str = "offsets.u64";
|
||||
pub(crate) const IDOM_FILE: &str = "idom.u32";
|
||||
pub(crate) const SHALLOW8_FILE: &str = "shallow8.u32";
|
||||
pub(crate) const RETAINED_FILE: &str = "retained.u64";
|
||||
pub(crate) const KIDS_START_FILE: &str = "kids_start.u32";
|
||||
pub(crate) const KIDS_FILE: &str = "kids.u32";
|
||||
pub(crate) const GC_PARENT_FILE: &str = "gc_parent.u32";
|
||||
pub(crate) const SUMMARY_FILE: &str = "summary.json";
|
||||
|
||||
pub(crate) const INDEX_FILES: &[&str] = &[
|
||||
IDS_BITS_FILE,
|
||||
IDS_WRANK_FILE,
|
||||
IDS_TABLE_FILE,
|
||||
IDS_CHUNKS_FILE,
|
||||
IDS_SORTED_FILE,
|
||||
TYPES_FILE,
|
||||
OFFSETS_FILE,
|
||||
IDOM_FILE,
|
||||
SHALLOW8_FILE,
|
||||
RETAINED_FILE,
|
||||
KIDS_START_FILE,
|
||||
KIDS_FILE,
|
||||
GC_PARENT_FILE,
|
||||
SUMMARY_FILE,
|
||||
];
|
||||
@@ -0,0 +1,274 @@
|
||||
//! The object graph of a heap dump in CSR form: one node per object, plus a super-root.
|
||||
|
||||
mod classes;
|
||||
mod link;
|
||||
|
||||
use std::collections::hash_map::Entry;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use self::classes::{ClassTable, IdHash};
|
||||
use self::link::Linker;
|
||||
use crate::column::{Column, Store};
|
||||
use crate::error::HeapError;
|
||||
use crate::files::{OFFSETS_FILE, TYPES_FILE};
|
||||
use crate::hprof::{BasicType, HeapRecord, HeapRootKind, Record, parse_header, scan, scan_segment};
|
||||
use crate::ids::{IdMap, IdMapBuilder};
|
||||
use crate::progress::{CHECK_EVERY, HeapPhase, Progress};
|
||||
|
||||
/// Number of primitive array type slots, one per non-object [`BasicType`].
|
||||
pub const PRIM_SLOTS: u32 = 8;
|
||||
|
||||
pub struct ClassInfo {
|
||||
pub id: u64,
|
||||
/// The hprof name with `/` turned into `.` (`java.lang.String`, `[Ljava.lang.Object;`).
|
||||
pub name: String,
|
||||
/// Index in [`HeapGraph::classes`]. Chains always end: a super cycle in the dump is cut where it closes.
|
||||
pub super_index: Option<u32>,
|
||||
/// This class's own instance fields, in dump order.
|
||||
pub fields: Vec<(String, BasicType)>,
|
||||
/// `(name, object id)` of reference statics; a null static has object id 0.
|
||||
pub statics: Vec<(String, u64)>,
|
||||
pub static_bytes: u64,
|
||||
/// Index in `fields` of `java.lang.ref.Reference.referent`, on that class only.
|
||||
pub weak_field: Option<usize>,
|
||||
}
|
||||
|
||||
pub struct HeapGraph {
|
||||
pub ids: IdMap,
|
||||
/// Type slot of each object: class index for instances and object arrays,
|
||||
/// [`Self::class_slot`] for class objects, [`Self::prim_slot`] for primitive arrays.
|
||||
pub types: Column<u32>,
|
||||
/// Array length, 0 for instances and classes.
|
||||
pub lens: Column<u32>,
|
||||
/// File offset of each object's record, for [`crate::record_at`].
|
||||
pub offsets: Column<u64>,
|
||||
/// `n + 2` entries: node `v`'s targets are `out[out_start[v]..out_start[v + 1]]`.
|
||||
pub out_start: Column<u32>,
|
||||
pub out: Column<u32>,
|
||||
/// The super-root, node `n`, with an edge to every rooted object.
|
||||
pub root: u32,
|
||||
/// `(object, kind, thread serial)` of every root record whose object is in the dump.
|
||||
pub roots: Vec<(u32, HeapRootKind, Option<u32>)>,
|
||||
/// Class dumps in dump order, then a synthetic `java.lang.Class` when the dump has none.
|
||||
pub classes: Vec<ClassInfo>,
|
||||
/// Type slot of class objects (`java.lang.Class`).
|
||||
pub class_slot: u32,
|
||||
/// Thread serial → thread object, from `ROOT THREAD OBJECT` records.
|
||||
pub thread_objects: HashMap<u32, u32>,
|
||||
/// `Reference.referent` edges left out of the graph.
|
||||
pub weak_edges_skipped: u64,
|
||||
pub id_size: u8,
|
||||
pub timestamp_ms: u64,
|
||||
}
|
||||
|
||||
impl HeapGraph {
|
||||
pub fn prim_slot(&self, ty: BasicType) -> u32 {
|
||||
self.classes.len() as u32 + (ty as u32 - BasicType::Boolean as u32)
|
||||
}
|
||||
|
||||
/// The element type of primitive array slot `slot`, or `None` for a class slot.
|
||||
pub fn prim_type(&self, slot: u32) -> Option<BasicType> {
|
||||
let offset = slot.checked_sub(self.classes.len() as u32).filter(|&o| o < PRIM_SLOTS)?;
|
||||
BasicType::from_u8(offset as u8 + BasicType::Boolean as u8)
|
||||
}
|
||||
|
||||
pub fn out_edges(&self, v: u32) -> &[u32] {
|
||||
let v = v as usize;
|
||||
&self.out[self.out_start[v] as usize..self.out_start[v + 1] as usize]
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the graph of the dump in `buf`. Strong edges only: `Reference.referent` is counted in
|
||||
/// [`HeapGraph::weak_edges_skipped`] instead. `progress` gets `Reading` by bytes, then
|
||||
/// `Linking` by objects, starting with `(Linking, 0, objects)` once the object count is known;
|
||||
/// an `Err` from it stops the build with that error.
|
||||
pub fn build_graph(
|
||||
buf: &[u8],
|
||||
store: &Store,
|
||||
progress: &dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<HeapGraph, HeapError> {
|
||||
let progress = Progress::new(progress, cancel);
|
||||
let header = parse_header(buf)?;
|
||||
let id_size = header.id_size;
|
||||
let len = buf.len() as u64;
|
||||
|
||||
// Pass 1: names, classes, roots and every object id.
|
||||
progress.report(HeapPhase::Reading, 0, len)?;
|
||||
let mut names: IdHash<&[u8]> = IdHash::default();
|
||||
let mut class_names: IdHash<u64> = IdHash::default();
|
||||
let mut dumps = Vec::new();
|
||||
let mut class_by_id: IdHash<u32> = IdHash::default();
|
||||
let mut raw_roots = Vec::new();
|
||||
let mut segments = Vec::new();
|
||||
let mut builder = IdMapBuilder::new();
|
||||
let (mut step, mut pos) = (0, header.body_start as u64);
|
||||
scan(buf, &header, &mut |record| {
|
||||
progress.step(step, HeapPhase::Reading, pos, len)?;
|
||||
step += 1;
|
||||
match record {
|
||||
Record::Utf8 { id, text } => {
|
||||
names.insert(id, text);
|
||||
}
|
||||
Record::LoadClass { class_id, name_id } => {
|
||||
class_names.insert(class_id, name_id);
|
||||
}
|
||||
Record::Segment(range) => {
|
||||
pos = range.start as u64;
|
||||
segments.push(range);
|
||||
}
|
||||
Record::Heap { at, record } => {
|
||||
pos = at as u64;
|
||||
match record {
|
||||
HeapRecord::Root { kind, id, thread_serial } => raw_roots.push((id, kind, thread_serial)),
|
||||
HeapRecord::Class(dump) => {
|
||||
builder.insert(dump.id);
|
||||
if let Entry::Vacant(slot) = class_by_id.entry(dump.id) {
|
||||
slot.insert(dumps.len() as u32);
|
||||
dumps.push(dump);
|
||||
}
|
||||
}
|
||||
HeapRecord::Instance { id, .. }
|
||||
| HeapRecord::ObjectArray { id, .. }
|
||||
| HeapRecord::PrimitiveArray { id, .. } => builder.insert(id),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
progress.report(HeapPhase::Reading, len, len)?;
|
||||
|
||||
let ids = builder.build(store, cancel)?;
|
||||
let table = classes::resolve(dumps, class_by_id, &names, &class_names, id_size, buf.len())?;
|
||||
drop((names, class_names));
|
||||
let n = ids.len();
|
||||
let objects = u64::from(n);
|
||||
progress.report(HeapPhase::Linking, 0, objects)?;
|
||||
|
||||
let mut roots = Vec::new();
|
||||
let mut root_targets = Vec::new();
|
||||
let mut thread_objects = HashMap::new();
|
||||
let (mut seen_records, mut seen_targets) = (HashSet::new(), HashSet::new());
|
||||
for (i, &(id, kind, serial)) in raw_roots.iter().enumerate() {
|
||||
progress.step(i as u64, HeapPhase::Linking, 0, objects)?;
|
||||
let Some(object) = ids.rank(id).filter(|_| id != 0) else { continue };
|
||||
if seen_records.insert((object, kind as u8, serial)) {
|
||||
roots.push((object, kind, serial));
|
||||
}
|
||||
if seen_targets.insert(object) {
|
||||
root_targets.push(object);
|
||||
}
|
||||
if let (HeapRootKind::ThreadObject, Some(serial)) = (kind, serial) {
|
||||
thread_objects.entry(serial).or_insert(object);
|
||||
}
|
||||
}
|
||||
drop((raw_roots, seen_records, seen_targets));
|
||||
|
||||
let linker = Linker::new(&ids, &table, id_size);
|
||||
let nodes = n as usize;
|
||||
let mut types = store.column::<u32>(TYPES_FILE, nodes, 0)?;
|
||||
let mut lens = store.scratch::<u32>(nodes, 0)?;
|
||||
let mut offsets = store.column::<u64>(OFFSETS_FILE, nodes, 0)?;
|
||||
let mut out_start = store.scratch::<u32>(nodes + 2, 0)?;
|
||||
// Both passes visit every object record once; report them as one walk over the objects.
|
||||
let mut visited = 0u64;
|
||||
let tick = |visited: &mut u64| {
|
||||
let result = progress.step(*visited, HeapPhase::Linking, (*visited / 2).min(objects), objects);
|
||||
*visited += 1;
|
||||
result
|
||||
};
|
||||
|
||||
// Pass 2: node columns and out-degrees, counted into `out_start[v + 1]`.
|
||||
let mut weak_edges_skipped = 0;
|
||||
for segment in &segments {
|
||||
scan_segment(buf, segment.clone(), id_size, &mut |at, record| {
|
||||
let Some(node) = linker.node(at, &record)? else { return Ok(()) };
|
||||
tick(&mut visited)?;
|
||||
let v = node.index as usize;
|
||||
types[v] = node.ty;
|
||||
lens[v] = node.len;
|
||||
offsets[v] = at as u64;
|
||||
let mut degree = 0u64;
|
||||
linker.edges(&record, &mut |_, weak| {
|
||||
if weak {
|
||||
weak_edges_skipped += 1;
|
||||
} else {
|
||||
degree += 1;
|
||||
}
|
||||
});
|
||||
let count = &mut out_start[v + 1];
|
||||
*count = u32::try_from(u64::from(*count) + degree).map_err(|_| HeapError::TooLarge)?;
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
out_start[nodes + 1] = u32::try_from(root_targets.len()).map_err(|_| HeapError::TooLarge)?;
|
||||
let edges = accumulate(&mut out_start, cancel)?;
|
||||
|
||||
// Pass 3: `out_start[v]` is node v's write cursor; once filled it holds v's end, so the
|
||||
// column is shifted one slot right to become the starts again.
|
||||
let mut out = store.scratch::<u32>(edges as usize, 0)?;
|
||||
for segment in &segments {
|
||||
scan_segment(buf, segment.clone(), id_size, &mut |at, record| {
|
||||
let Some(node) = linker.node(at, &record)? else { return Ok(()) };
|
||||
tick(&mut visited)?;
|
||||
let v = node.index as usize;
|
||||
let mut cursor = out_start[v] as usize;
|
||||
linker.edges(&record, &mut |target, weak| {
|
||||
if !weak && let Some(slot) = out.get_mut(cursor) {
|
||||
*slot = target;
|
||||
cursor += 1;
|
||||
}
|
||||
});
|
||||
out_start[v] = cursor as u32;
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
let first_root_edge = out_start[nodes] as usize;
|
||||
for (slot, &target) in out[first_root_edge..].iter_mut().zip(&root_targets) {
|
||||
*slot = target;
|
||||
}
|
||||
out_start[nodes] = edges;
|
||||
out_start.copy_within(0..nodes + 1, 1);
|
||||
out_start[0] = 0;
|
||||
progress.report(HeapPhase::Linking, objects, objects)?;
|
||||
|
||||
drop(linker);
|
||||
let ClassTable { classes, class_slot, .. } = table;
|
||||
Ok(HeapGraph {
|
||||
ids,
|
||||
types,
|
||||
lens,
|
||||
offsets,
|
||||
out_start,
|
||||
out,
|
||||
root: n,
|
||||
roots,
|
||||
classes,
|
||||
class_slot,
|
||||
thread_objects,
|
||||
weak_edges_skipped,
|
||||
id_size,
|
||||
timestamp_ms: header.timestamp_ms,
|
||||
})
|
||||
}
|
||||
|
||||
/// Turns per-node counts into running totals in place (`counts[k]` becomes the sum of
|
||||
/// `counts[..=k]`) and returns the last one; past `u32::MAX` the graph is `TooLarge`.
|
||||
pub(crate) fn accumulate(counts: &mut [u32], cancel: &CancellationToken) -> Result<u32, HeapError> {
|
||||
let mut sum = 0u32;
|
||||
for chunk in counts.chunks_mut(CHECK_EVERY as usize) {
|
||||
if cancel.is_cancelled() {
|
||||
return Err(HeapError::Cancelled);
|
||||
}
|
||||
for count in chunk {
|
||||
sum = sum.checked_add(*count).ok_or(HeapError::TooLarge)?;
|
||||
*count = sum;
|
||||
}
|
||||
}
|
||||
Ok(sum)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,188 @@
|
||||
//! Class dumps resolved into [`ClassInfo`]s, plus where the references sit in instance data.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::hash::BuildHasherDefault;
|
||||
|
||||
use super::{ClassInfo, PRIM_SLOTS};
|
||||
use crate::column::NONE;
|
||||
use crate::error::HeapError;
|
||||
use crate::hprof::{BasicType, ClassDump};
|
||||
use crate::ids::ChunkHasher;
|
||||
|
||||
/// Keyed by hprof id (an address).
|
||||
pub(super) type IdHash<V> = HashMap<u64, V, BuildHasherDefault<ChunkHasher>>;
|
||||
|
||||
const REFERENCE: &str = "java.lang.ref.Reference";
|
||||
const JAVA_LANG_CLASS: &str = "java.lang.Class";
|
||||
/// Not computed yet, in [`ClassTable::fielded`].
|
||||
const UNSET: u32 = NONE - 1;
|
||||
|
||||
pub(super) struct ClassTable {
|
||||
pub(super) classes: Vec<ClassInfo>,
|
||||
/// Class id → class index.
|
||||
pub(super) by_id: IdHash<u32>,
|
||||
pub(super) class_slot: u32,
|
||||
/// Per class: `(offset, is the referent)` of each reference field in the class's own part of
|
||||
/// the instance data.
|
||||
pub(super) refs: Vec<Vec<(usize, bool)>>,
|
||||
/// Per class: bytes of its own part of the instance data.
|
||||
pub(super) own_bytes: Vec<usize>,
|
||||
/// Per class: the nearest class up its chain (itself included) whose own part is not empty, or
|
||||
/// `NONE`. Walking these instead of every super keeps a walk bounded by the data it reads.
|
||||
pub(super) fielded: Vec<u32>,
|
||||
}
|
||||
|
||||
/// The longest name a JVM gives a class or field (a constant pool UTF8 has a `u2` length).
|
||||
const MAX_NAME_BYTES: usize = 65_535;
|
||||
/// Name bytes a dump may decode beyond 4 per byte of the file.
|
||||
const NAME_BUDGET_FLOOR: usize = 64 << 20;
|
||||
|
||||
/// Names decoded from the UTF8 records within a budget: every field of every class may name the
|
||||
/// same record, so a small crafted file could otherwise ask for terabytes and abort the IDE.
|
||||
struct Names<'m, 'b> {
|
||||
records: &'m IdHash<&'b [u8]>,
|
||||
left: usize,
|
||||
}
|
||||
|
||||
impl Names<'_, '_> {
|
||||
/// Empty for an id with no record.
|
||||
fn text(&mut self, id: u64) -> Result<String, HeapError> {
|
||||
let Some(bytes) = self.records.get(&id) else { return Ok(String::new()) };
|
||||
if bytes.len() > MAX_NAME_BYTES || bytes.len() > self.left {
|
||||
return Err(HeapError::NamesTooLarge);
|
||||
}
|
||||
self.left -= bytes.len();
|
||||
Ok(String::from_utf8_lossy(bytes).into_owned())
|
||||
}
|
||||
}
|
||||
|
||||
/// `dumps` holds one dump per class id, and `by_id` its position; `names` are the UTF8 records
|
||||
/// and `class_names` the LOAD CLASS class id → name id pairs. `file_len` bounds the name bytes.
|
||||
pub(super) fn resolve(
|
||||
dumps: Vec<ClassDump>,
|
||||
by_id: IdHash<u32>,
|
||||
names: &IdHash<&[u8]>,
|
||||
class_names: &IdHash<u64>,
|
||||
id_size: u8,
|
||||
file_len: usize,
|
||||
) -> Result<ClassTable, HeapError> {
|
||||
// Room for the synthetic java.lang.Class and the primitive slots in a `u32` type id.
|
||||
if dumps.len() > (u32::MAX - PRIM_SLOTS - 1) as usize {
|
||||
return Err(HeapError::TooLarge);
|
||||
}
|
||||
let mut names = Names { records: names, left: file_len.saturating_mul(4).saturating_add(NAME_BUDGET_FLOOR) };
|
||||
let mut supers: Vec<Option<u32>> =
|
||||
dumps.iter().map(|d| if d.super_id == 0 { None } else { by_id.get(&d.super_id).copied() }).collect();
|
||||
cut_super_cycles(&mut supers);
|
||||
|
||||
let mut classes = Vec::with_capacity(dumps.len() + 1);
|
||||
let mut refs = Vec::with_capacity(dumps.len() + 1);
|
||||
let mut own_bytes = Vec::with_capacity(dumps.len() + 1);
|
||||
for (dump, super_index) in dumps.into_iter().zip(supers) {
|
||||
let name = match class_names.get(&dump.id) {
|
||||
Some(&name_id) if names.records.contains_key(&name_id) => names.text(name_id)?.replace('/', "."),
|
||||
_ => format!("{:#x}", dump.id),
|
||||
};
|
||||
let fields =
|
||||
dump.fields.iter().map(|&(n, ty)| Ok((names.text(n)?, ty))).collect::<Result<Vec<_>, HeapError>>()?;
|
||||
let weak_field = if name == REFERENCE {
|
||||
fields.iter().position(|(n, ty)| n == "referent" && *ty == BasicType::Object)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut offset = 0;
|
||||
let mut class_refs = Vec::new();
|
||||
for (k, &(_, ty)) in fields.iter().enumerate() {
|
||||
if ty == BasicType::Object {
|
||||
class_refs.push((offset, weak_field == Some(k)));
|
||||
}
|
||||
offset += ty.hprof_size(id_size);
|
||||
}
|
||||
refs.push(class_refs);
|
||||
own_bytes.push(offset);
|
||||
classes.push(ClassInfo {
|
||||
id: dump.id,
|
||||
name,
|
||||
super_index,
|
||||
fields,
|
||||
statics: dump
|
||||
.statics
|
||||
.iter()
|
||||
.map(|&(n, object)| Ok((names.text(n)?, object)))
|
||||
.collect::<Result<_, HeapError>>()?,
|
||||
static_bytes: dump.static_bytes,
|
||||
weak_field,
|
||||
});
|
||||
}
|
||||
|
||||
let class_slot = match classes.iter().position(|c| c.name == JAVA_LANG_CLASS) {
|
||||
Some(slot) => slot as u32,
|
||||
None => {
|
||||
classes.push(ClassInfo {
|
||||
id: 0,
|
||||
name: JAVA_LANG_CLASS.to_owned(),
|
||||
super_index: None,
|
||||
fields: Vec::new(),
|
||||
statics: Vec::new(),
|
||||
static_bytes: 0,
|
||||
weak_field: None,
|
||||
});
|
||||
refs.push(Vec::new());
|
||||
own_bytes.push(0);
|
||||
classes.len() as u32 - 1
|
||||
}
|
||||
};
|
||||
let fielded = nearest_fielded(&classes, &own_bytes);
|
||||
Ok(ClassTable { classes, by_id, class_slot, refs, own_bytes, fielded })
|
||||
}
|
||||
|
||||
/// A dump may claim `A extends B extends A`; the edge that closes a cycle is dropped so every
|
||||
/// walk up the chain ends. Iterative, so a long chain cannot overflow the stack.
|
||||
fn cut_super_cycles(supers: &mut [Option<u32>]) {
|
||||
const NEW: u8 = 0;
|
||||
const ON_PATH: u8 = 1;
|
||||
const DONE: u8 = 2;
|
||||
let mut state = vec![NEW; supers.len()];
|
||||
let mut path = Vec::new();
|
||||
for start in 0..supers.len() {
|
||||
let mut c = start;
|
||||
while state[c] == NEW {
|
||||
state[c] = ON_PATH;
|
||||
path.push(c);
|
||||
match supers[c] {
|
||||
Some(s) if state[s as usize] == ON_PATH => {
|
||||
supers[c] = None;
|
||||
break;
|
||||
}
|
||||
Some(s) => c = s as usize,
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
for p in path.drain(..) {
|
||||
state[p] = DONE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn nearest_fielded(classes: &[ClassInfo], own_bytes: &[usize]) -> Vec<u32> {
|
||||
let mut fielded: Vec<u32> =
|
||||
own_bytes.iter().enumerate().map(|(c, &b)| if b > 0 { c as u32 } else { UNSET }).collect();
|
||||
let mut path = Vec::new();
|
||||
for start in 0..classes.len() {
|
||||
let mut c = start;
|
||||
let found = loop {
|
||||
if fielded[c] != UNSET {
|
||||
break fielded[c];
|
||||
}
|
||||
path.push(c);
|
||||
match classes[c].super_index {
|
||||
Some(s) => c = s as usize,
|
||||
None => break NONE,
|
||||
}
|
||||
};
|
||||
for p in path.drain(..) {
|
||||
fielded[p] = found;
|
||||
}
|
||||
}
|
||||
fielded
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
//! Turns one heap sub-record into its node (index, type, length) and its outgoing edges. Both
|
||||
//! linking passes go through here, so the degrees counted in the first match the edges written
|
||||
//! in the second.
|
||||
|
||||
use super::classes::ClassTable;
|
||||
use crate::column::NONE;
|
||||
use crate::error::HeapError;
|
||||
use crate::hprof::{BasicType, HeapRecord};
|
||||
use crate::ids::IdMap;
|
||||
|
||||
pub(super) struct Node {
|
||||
pub(super) index: u32,
|
||||
pub(super) ty: u32,
|
||||
pub(super) len: u32,
|
||||
}
|
||||
|
||||
pub(super) struct Linker<'a> {
|
||||
ids: &'a IdMap,
|
||||
table: &'a ClassTable,
|
||||
/// Per class: the node of its class object, `NONE` for the synthetic `java.lang.Class`.
|
||||
class_objects: Vec<u32>,
|
||||
id_size: u8,
|
||||
}
|
||||
|
||||
impl<'a> Linker<'a> {
|
||||
pub(super) fn new(ids: &'a IdMap, table: &'a ClassTable, id_size: u8) -> Self {
|
||||
let class_objects =
|
||||
table.classes.iter().map(|c| if c.id == 0 { NONE } else { ids.rank(c.id).unwrap_or(NONE) }).collect();
|
||||
Self { ids, table, class_objects, id_size }
|
||||
}
|
||||
|
||||
/// `None` for root records. `at` is the record's offset, for the error of an instance or array
|
||||
/// whose class is not in the dump.
|
||||
pub(super) fn node(&self, at: usize, record: &HeapRecord<'_>) -> Result<Option<Node>, HeapError> {
|
||||
let class_index = |class_id: u64| {
|
||||
self.table.by_id.get(&class_id).copied().ok_or(HeapError::MissingClass { id: class_id, at: at as u64 })
|
||||
};
|
||||
let (id, ty, len) = match record {
|
||||
HeapRecord::Root { .. } => return Ok(None),
|
||||
HeapRecord::Class(c) => (c.id, self.table.class_slot, 0),
|
||||
HeapRecord::Instance { id, class_id, .. } => (*id, class_index(*class_id)?, 0),
|
||||
HeapRecord::ObjectArray { id, class_id, len, .. } => (*id, class_index(*class_id)?, *len),
|
||||
HeapRecord::PrimitiveArray { id, ty, len, .. } => {
|
||||
let prim_base = self.table.classes.len() as u32;
|
||||
(*id, prim_base + (*ty as u32 - BasicType::Boolean as u32), *len)
|
||||
}
|
||||
};
|
||||
Ok(self.ids.rank(id).map(|index| Node { index, ty, len }))
|
||||
}
|
||||
|
||||
/// Calls `edge(target, is_referent)` for every reference to an object in the dump, in field
|
||||
/// order; `is_referent` marks `Reference.referent`, which callers leave out of the graph.
|
||||
pub(super) fn edges(&self, record: &HeapRecord<'_>, edge: &mut impl FnMut(u32, bool)) {
|
||||
let mut to = |id: u64, weak: bool| {
|
||||
if let Some(target) = self.target(id) {
|
||||
edge(target, weak);
|
||||
}
|
||||
};
|
||||
match record {
|
||||
HeapRecord::Root { .. } | HeapRecord::PrimitiveArray { .. } => {}
|
||||
HeapRecord::Class(c) => {
|
||||
to(c.super_id, false);
|
||||
to(c.loader_id, false);
|
||||
for &(_, object) in &c.statics {
|
||||
to(object, false);
|
||||
}
|
||||
}
|
||||
HeapRecord::ObjectArray { data, .. } => {
|
||||
for item in data.chunks_exact(usize::from(self.id_size)) {
|
||||
to(read_id(item, 0, self.id_size).unwrap_or(0), false);
|
||||
}
|
||||
}
|
||||
HeapRecord::Instance { class_id, data, .. } => {
|
||||
let Some(&class) = self.table.by_id.get(class_id) else { return };
|
||||
let mut base = 0;
|
||||
let mut next = self.table.fielded[class as usize];
|
||||
'chain: while next != NONE && base < data.len() {
|
||||
let c = next as usize;
|
||||
for &(offset, weak) in &self.table.refs[c] {
|
||||
let Some(id) = read_id(data, base + offset, self.id_size) else { break 'chain };
|
||||
to(id, weak);
|
||||
}
|
||||
base += self.table.own_bytes[c];
|
||||
next = self.table.classes[c].super_index.map_or(NONE, |s| self.table.fielded[s as usize]);
|
||||
}
|
||||
let object = self.class_objects[class as usize];
|
||||
if object != NONE {
|
||||
edge(object, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Id 0 is null; ids not in the dump have no node.
|
||||
fn target(&self, id: u64) -> Option<u32> {
|
||||
if id == 0 { None } else { self.ids.rank(id) }
|
||||
}
|
||||
}
|
||||
|
||||
/// The big-endian id at `at`, or `None` when `data` ends first.
|
||||
fn read_id(data: &[u8], at: usize, id_size: u8) -> Option<u64> {
|
||||
let bytes = data.get(at..at.checked_add(usize::from(id_size))?)?;
|
||||
Some(match *bytes {
|
||||
[a, b, c, d] => u64::from(u32::from_be_bytes([a, b, c, d])),
|
||||
_ => u64::from_be_bytes(bytes.try_into().ok()?),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,479 @@
|
||||
use std::cell::RefCell;
|
||||
|
||||
use proptest::prelude::*;
|
||||
|
||||
use super::*;
|
||||
use crate::hprof::{HeapRecord, record_at};
|
||||
use crate::testing::{HprofWriter, MiniHeap, sample};
|
||||
|
||||
const H: u64 = 0x1000;
|
||||
const N1: u64 = 0x1100;
|
||||
const N2: u64 = 0x1200;
|
||||
const N3: u64 = 0x1300;
|
||||
const L: u64 = 0x1400;
|
||||
const R: u64 = 0x1500;
|
||||
const W: u64 = 0x1600;
|
||||
const N4: u64 = 0x1700;
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn build(bytes: &[u8]) -> Result<HeapGraph, HeapError> {
|
||||
build_graph(bytes, &Store::Ram, &no_progress, &CancellationToken::new())
|
||||
}
|
||||
|
||||
fn index(g: &HeapGraph, id: u64) -> u32 {
|
||||
g.ids.rank(id).unwrap_or_else(|| panic!("{id:#x} is not in the graph"))
|
||||
}
|
||||
|
||||
fn class(g: &HeapGraph, name: &str) -> (u32, u64) {
|
||||
let slot = g.classes.iter().position(|c| c.name == name).unwrap_or_else(|| panic!("no class {name}"));
|
||||
(slot as u32, g.classes[slot].id)
|
||||
}
|
||||
|
||||
/// Targets of `id`'s edges, as ids, in edge order.
|
||||
fn out_of(g: &HeapGraph, id: u64) -> Vec<u64> {
|
||||
g.out_edges(index(g, id)).iter().map(|&t| g.ids.id_at(t)).collect()
|
||||
}
|
||||
|
||||
fn root_targets(g: &HeapGraph) -> Vec<u64> {
|
||||
g.out_edges(g.root).iter().map(|&t| g.ids.id_at(t)).collect()
|
||||
}
|
||||
|
||||
fn assert_csr(g: &HeapGraph) {
|
||||
let n = g.ids.len();
|
||||
assert_eq!(g.root, n);
|
||||
assert_eq!(g.out_start.len(), n as usize + 2);
|
||||
assert_eq!(g.out_start[0], 0);
|
||||
assert!(g.out_start.windows(2).all(|w| w[0] <= w[1]), "out_start is not monotone");
|
||||
assert_eq!(g.out_start[n as usize + 1] as usize, g.out.len());
|
||||
assert!(g.out.iter().all(|&t| t < n), "edge to a node outside the objects");
|
||||
assert_eq!((g.types.len(), g.lens.len(), g.offsets.len()), (n as usize, n as usize, n as usize));
|
||||
let slots = g.classes.len() as u32 + PRIM_SLOTS;
|
||||
assert!(g.types.iter().all(|&t| t < slots), "type outside the slots");
|
||||
for c in &g.classes {
|
||||
let mut steps = 0;
|
||||
let mut cursor = c.super_index;
|
||||
while let Some(s) = cursor {
|
||||
steps += 1;
|
||||
assert!(steps <= g.classes.len(), "super chain of {} does not end", c.name);
|
||||
cursor = g.classes[s as usize].super_index;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edges_follow_fields_and_skip_weak_referents() {
|
||||
let g = build(&sample().bytes()).unwrap();
|
||||
assert_csr(&g);
|
||||
let (_, holder) = class(&g, "Holder");
|
||||
let (_, node) = class(&g, "Node");
|
||||
let (_, weak) = class(&g, "java.lang.ref.WeakReference");
|
||||
assert_eq!(out_of(&g, H), [N1, N2, holder]);
|
||||
assert_eq!(out_of(&g, N1), [N3, node]);
|
||||
assert_eq!(out_of(&g, N3), [L, node]);
|
||||
assert_eq!(out_of(&g, W), [weak]);
|
||||
assert_eq!(out_of(&g, N4), [node]);
|
||||
assert_eq!(out_of(&g, L), Vec::<u64>::new());
|
||||
assert_eq!(g.weak_edges_skipped, 1);
|
||||
let (reference, _) = class(&g, "java.lang.ref.Reference");
|
||||
assert_eq!(g.classes[reference as usize].weak_field, Some(0));
|
||||
assert!(g.classes.iter().filter(|c| c.name != "java.lang.ref.Reference").all(|c| c.weak_field.is_none()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_arrays_skip_null_and_unknown() {
|
||||
let mut heap = sample();
|
||||
heap.array_of(0x1800, "[Ljava.lang.Object;", &[N2, 0, 0x9990, N1, 0xDEAD_BEEF]);
|
||||
let g = build(&heap.bytes()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(out_of(&g, R), [N1]);
|
||||
assert_eq!(out_of(&g, 0x1800), [N2, N1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn super_root_points_at_each_root_once() {
|
||||
let mut heap = sample();
|
||||
heap.root(HeapRootKind::JniGlobal, H).root(HeapRootKind::MonitorUsed, H).root(HeapRootKind::JniLocal, 0x9990);
|
||||
let g = build(&heap.bytes()).unwrap();
|
||||
assert_csr(&g);
|
||||
let targets = root_targets(&g);
|
||||
assert_eq!(targets.iter().filter(|&&t| t == H).count(), 1);
|
||||
assert!(targets.contains(&N3) && targets.contains(&W));
|
||||
assert!(!targets.contains(&N1) && !targets.contains(&R) && !targets.contains(&N4));
|
||||
let classes = g.classes.iter().filter(|c| c.id != 0).count();
|
||||
assert_eq!(targets.len(), 3 + classes, "H, N3, W and every sticky class");
|
||||
let mut sorted = targets.clone();
|
||||
sorted.sort_unstable();
|
||||
sorted.dedup();
|
||||
assert_eq!(sorted.len(), targets.len());
|
||||
|
||||
let h = index(&g, H);
|
||||
let of_h: Vec<_> = g.roots.iter().filter(|r| r.0 == h).map(|r| (r.1, r.2)).collect();
|
||||
assert_eq!(of_h, [(HeapRootKind::JniGlobal, None), (HeapRootKind::MonitorUsed, None)]);
|
||||
assert!(g.roots.contains(&(index(&g, N3), HeapRootKind::JavaFrame, Some(1))));
|
||||
assert!(g.roots.iter().all(|r| r.0 < g.root));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_dump_edges_cover_statics() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "java/lang/Object")
|
||||
.load_class(0x100, 1)
|
||||
.class(0x100, 0, &[], &[])
|
||||
.string(2, "Config")
|
||||
.string(3, "INSTANCE")
|
||||
.string(4, "EMPTY")
|
||||
.string(5, "LOST")
|
||||
.load_class(0x200, 2)
|
||||
.class_with_loader(0x200, 0x100, 0x1200, &[], &[(3, N1), (4, 0), (5, 0x9990)])
|
||||
.instance(N1, 0x100, &[])
|
||||
.instance(0x1200, 0x100, &[]);
|
||||
let g = build(&w.finish()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(out_of(&g, 0x200), [0x100, 0x1200, N1]);
|
||||
assert_eq!(out_of(&g, 0x100), Vec::<u64>::new());
|
||||
let (config, _) = class(&g, "Config");
|
||||
let info = &g.classes[config as usize];
|
||||
let statics: Vec<_> = info.statics.iter().map(|(n, id)| (n.as_str(), *id)).collect();
|
||||
assert_eq!(statics, [("INSTANCE", N1), ("EMPTY", 0), ("LOST", 0x9990)]);
|
||||
assert_eq!(info.super_index, Some(class(&g, "java.lang.Object").0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_class_is_an_error() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "java/lang/Object").load_class(0x100, 1).class(0x100, 0, &[], &[]).instance(N1, 0x999, &[]);
|
||||
let bytes = w.finish();
|
||||
let at = instance_offset(&bytes, N1);
|
||||
let err = build(&bytes).err().unwrap();
|
||||
assert!(matches!(err, HeapError::MissingClass { id: 0x999, at: a } if a == at as u64), "{err:?}");
|
||||
|
||||
let mut w = HprofWriter::new();
|
||||
w.obj_array(0x10, 0x999, &[]);
|
||||
assert!(matches!(build(&w.finish()), Err(HeapError::MissingClass { id: 0x999, .. })));
|
||||
}
|
||||
|
||||
fn instance_offset(bytes: &[u8], id: u64) -> usize {
|
||||
let header = crate::hprof::parse_header(bytes).unwrap();
|
||||
let mut found = None;
|
||||
crate::hprof::scan(bytes, &header, &mut |r| {
|
||||
if let crate::hprof::Record::Heap { at, record: HeapRecord::Instance { id: i, .. } } = r
|
||||
&& i == id
|
||||
{
|
||||
found = Some(at);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
found.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_during_linking_returns_cancelled() {
|
||||
let bytes = sample().bytes();
|
||||
let cancelled = CancellationToken::new();
|
||||
cancelled.cancel();
|
||||
let result = build_graph(&bytes, &Store::Ram, &no_progress, &cancelled);
|
||||
assert!(matches!(result, Err(HeapError::Cancelled)));
|
||||
|
||||
let cancel = CancellationToken::new();
|
||||
let phases = RefCell::new(Vec::new());
|
||||
let result = build_graph(
|
||||
&bytes,
|
||||
&Store::Ram,
|
||||
&|phase, done, total| {
|
||||
phases.borrow_mut().push((phase, done, total));
|
||||
if phase == HeapPhase::Linking {
|
||||
cancel.cancel();
|
||||
}
|
||||
Ok(())
|
||||
},
|
||||
&cancel,
|
||||
);
|
||||
assert!(matches!(result, Err(HeapError::Cancelled)));
|
||||
let phases = phases.into_inner();
|
||||
assert_eq!(phases.first(), Some(&(HeapPhase::Reading, 0, bytes.len() as u64)));
|
||||
assert_eq!(phases.last(), Some(&(HeapPhase::Linking, 0, 15)), "stops at the first Linking report");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_error_stops_the_build() {
|
||||
let bytes = sample().bytes();
|
||||
let fail = |phase, _, total| {
|
||||
if phase == HeapPhase::Linking {
|
||||
Err(HeapError::NotEnoughDiskSpace { needed: total * 90, available: 0 })
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
};
|
||||
let result = build_graph(&bytes, &Store::Ram, &fail, &CancellationToken::new());
|
||||
assert!(matches!(result, Err(HeapError::NotEnoughDiskSpace { needed: 1350, available: 0 })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn progress_reads_bytes_then_links_objects() {
|
||||
let bytes = sample().bytes();
|
||||
let phases = RefCell::new(Vec::new());
|
||||
let g = build_graph(
|
||||
&bytes,
|
||||
&Store::Ram,
|
||||
&|phase, done, total| {
|
||||
phases.borrow_mut().push((phase, done, total));
|
||||
Ok(())
|
||||
},
|
||||
&CancellationToken::new(),
|
||||
)
|
||||
.unwrap();
|
||||
let phases = phases.into_inner();
|
||||
let len = bytes.len() as u64;
|
||||
let objects = u64::from(g.root);
|
||||
assert_eq!(objects, 15, "7 classes and 8 objects");
|
||||
assert_eq!(phases.first(), Some(&(HeapPhase::Reading, 0, len)));
|
||||
assert!(phases.contains(&(HeapPhase::Reading, len, len)));
|
||||
let linking: Vec<_> = phases.iter().filter(|p| p.0 == HeapPhase::Linking).collect();
|
||||
assert_eq!(linking.first(), Some(&&(HeapPhase::Linking, 0, objects)));
|
||||
assert_eq!(linking.iter().filter(|p| ***p == (HeapPhase::Linking, 0, objects)).count(), 1, "{linking:?}");
|
||||
assert_eq!(linking.last(), Some(&&(HeapPhase::Linking, objects, objects)));
|
||||
assert!(phases.iter().all(|&(_, done, total)| done <= total));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn types_lengths_and_offsets() {
|
||||
let bytes = sample().bytes();
|
||||
let g = build(&bytes).unwrap();
|
||||
let (holder, holder_id) = class(&g, "Holder");
|
||||
let (node, _) = class(&g, "Node");
|
||||
let (array, _) = class(&g, "[Ljava.lang.Object;");
|
||||
let (java_lang_class, _) = class(&g, "java.lang.Class");
|
||||
assert_eq!(g.class_slot, java_lang_class);
|
||||
let ty = |id| g.types[index(&g, id) as usize];
|
||||
let len = |id| g.lens[index(&g, id) as usize];
|
||||
assert_eq!([ty(H), ty(N1), ty(R), ty(holder_id)], [holder, node, array, java_lang_class]);
|
||||
assert_eq!(ty(L), g.prim_slot(BasicType::Long));
|
||||
assert_eq!(g.prim_type(ty(L)), Some(BasicType::Long));
|
||||
assert_eq!(g.prim_type(holder), None);
|
||||
assert_eq!(g.prim_type(g.classes.len() as u32 + PRIM_SLOTS), None);
|
||||
assert_eq!(g.prim_slot(BasicType::Boolean), g.classes.len() as u32);
|
||||
assert_eq!(g.prim_slot(BasicType::Long), g.classes.len() as u32 + PRIM_SLOTS - 1);
|
||||
assert_eq!([len(H), len(L), len(R), len(holder_id)], [0, 2, 3, 0]);
|
||||
for i in 0..g.root {
|
||||
let id = match record_at(&bytes, g.offsets[i as usize] as usize, g.id_size).unwrap() {
|
||||
HeapRecord::Class(c) => c.id,
|
||||
HeapRecord::Instance { id, .. } | HeapRecord::ObjectArray { id, .. } => id,
|
||||
HeapRecord::PrimitiveArray { id, .. } => id,
|
||||
other => panic!("offset of object {i} points at {other:?}"),
|
||||
};
|
||||
assert_eq!(id, g.ids.id_at(i));
|
||||
}
|
||||
assert_eq!((g.id_size, g.timestamp_ms), (8, 1_700_000_000_000));
|
||||
let holder_info = &g.classes[holder as usize];
|
||||
let fields: Vec<_> = holder_info.fields.iter().map(|(n, t)| (n.as_str(), *t)).collect();
|
||||
assert_eq!(fields, [("a", BasicType::Object), ("b", BasicType::Object)]);
|
||||
assert_eq!(holder_info.super_index, Some(class(&g, "java.lang.Object").0));
|
||||
assert_eq!(g.classes[class(&g, "java.lang.Object").0 as usize].super_index, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dump_without_java_lang_class_gets_a_synthetic_slot() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "java/lang/Object").load_class(0x100, 1).class(0x100, 0, &[], &[]).instance(N1, 0x100, &[]);
|
||||
let g = build(&w.finish()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(g.classes.len(), 2);
|
||||
let synthetic = &g.classes[g.class_slot as usize];
|
||||
assert_eq!((synthetic.id, synthetic.name.as_str(), synthetic.super_index), (0, "java.lang.Class", None));
|
||||
assert_eq!(g.types[index(&g, 0x100) as usize], g.class_slot);
|
||||
assert_eq!(g.types[index(&g, N1) as usize], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classes_without_a_name_record_get_their_address() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.class(0x100, 0, &[(9, BasicType::Object)], &[])
|
||||
.instance(N1, 0x100, &N2.to_be_bytes())
|
||||
.instance(N2, 0x100, &[0; 8]);
|
||||
let g = build(&w.finish()).unwrap();
|
||||
assert_eq!(g.classes[0].name, "0x100");
|
||||
assert_eq!(g.classes[0].fields, [(String::new(), BasicType::Object)]);
|
||||
assert_eq!(out_of(&g, N1), [N2, 0x100]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_objects_map_serials_to_threads() {
|
||||
let mut heap = sample();
|
||||
heap.class("java.lang.Thread", Some("java.lang.Object"), &[])
|
||||
.object(0x2000, "java.lang.Thread", &[])
|
||||
.object(0x2100, "java.lang.Thread", &[])
|
||||
.root_in_thread(HeapRootKind::ThreadObject, 0x2000, 1)
|
||||
.root_in_thread(HeapRootKind::ThreadObject, 0x2100, 7)
|
||||
.root_in_thread(HeapRootKind::ThreadObject, 0x9990, 9);
|
||||
let g = build(&heap.bytes()).unwrap();
|
||||
let expected: HashMap<u32, u32> = [(1, index(&g, 0x2000)), (7, index(&g, 0x2100))].into();
|
||||
assert_eq!(g.thread_objects, expected);
|
||||
assert!(g.roots.contains(&(index(&g, 0x2100), HeapRootKind::ThreadObject, Some(7))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inherited_fields_follow_the_class_first_layout() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("A", Some("java.lang.Object"), &[("a", BasicType::Object), ("n", BasicType::Int)])
|
||||
.class("Empty", Some("A"), &[])
|
||||
.class("B", Some("Empty"), &[("flag", BasicType::Byte), ("b", BasicType::Object)])
|
||||
.object(0x10, "B", &[("a", 0x20), ("b", 0x30), ("flag", 1)])
|
||||
.object(0x20, "A", &[])
|
||||
.object(0x30, "A", &[]);
|
||||
let g = build(&heap.bytes()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(out_of(&g, 0x10), [0x30, 0x20, class(&g, "B").1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn super_class_cycles_are_cut() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "A")
|
||||
.string(2, "B")
|
||||
.string(3, "S")
|
||||
.string(4, "f")
|
||||
.load_class(0x100, 1)
|
||||
.load_class(0x200, 2)
|
||||
.load_class(0x300, 3)
|
||||
.class(0x100, 0x200, &[(4, BasicType::Object)], &[])
|
||||
.class(0x200, 0x100, &[(4, BasicType::Object)], &[])
|
||||
.class(0x300, 0x300, &[], &[])
|
||||
.instance(0x10, 0x100, &[[0; 7].as_slice(), &[0x20], &[0; 7], &[0x30]].concat())
|
||||
.instance(0x20, 0x300, &[])
|
||||
.instance(0x30, 0x300, &[0; 64]);
|
||||
let g = build(&w.finish()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(out_of(&g, 0x10), [0x20, 0x30, 0x100]);
|
||||
assert_eq!(out_of(&g, 0x30), [0x300]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_instance_data_stops_at_the_last_whole_field() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "P")
|
||||
.string(2, "x")
|
||||
.load_class(0x100, 1)
|
||||
.class(0x100, 0, &[(2, BasicType::Object), (2, BasicType::Object)], &[])
|
||||
.instance(0x10, 0x100, &[[0; 7].as_slice(), &[0x20], &[0; 5]].concat())
|
||||
.instance(0x20, 0x100, &[]);
|
||||
let g = build(&w.finish()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(out_of(&g, 0x10), [0x20, 0x100]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn id_size_4_dump_links() {
|
||||
let mut w = HprofWriter::with_id_size(4);
|
||||
w.string(1, "Node")
|
||||
.string(2, "next")
|
||||
.load_class(0x100, 1)
|
||||
.class(0x100, 0, &[(2, BasicType::Object)], &[(2, 0x20)])
|
||||
.instance(0x10, 0x100, &0x20u32.to_be_bytes())
|
||||
.instance(0x20, 0x100, &[0; 4])
|
||||
.obj_array(0x30, 0x100, &[0x10, 0, 0x20])
|
||||
.root(HeapRootKind::JniGlobal, 0x30);
|
||||
let g = build(&w.finish()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!(g.id_size, 4);
|
||||
assert_eq!(out_of(&g, 0x10), [0x20, 0x100]);
|
||||
assert_eq!(out_of(&g, 0x30), [0x10, 0x20]);
|
||||
assert_eq!(out_of(&g, 0x100), [0x20]);
|
||||
assert_eq!(root_targets(&g), [0x30]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dir_store_keeps_types_and_offsets_only() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
// The id map's own files (`ids.*`) depend on its layout; MiniHeap's class ids are unaligned.
|
||||
let names = || {
|
||||
let mut names: Vec<_> = std::fs::read_dir(dir.path())
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||||
.filter(|name| !name.starts_with("ids."))
|
||||
.collect();
|
||||
names.sort();
|
||||
names
|
||||
};
|
||||
let g =
|
||||
build_graph(&sample().bytes(), &Store::Dir(dir.path().to_path_buf()), &no_progress, &CancellationToken::new())
|
||||
.unwrap();
|
||||
assert_eq!(out_of(&g, H).len(), 3);
|
||||
let kept = ["offsets.u64", "types.u32"];
|
||||
assert!(dir.path().join("ids.sorted.u64").exists());
|
||||
#[cfg(unix)]
|
||||
assert_eq!(names(), kept);
|
||||
drop(g);
|
||||
assert_eq!(names(), kept);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_dump_has_only_the_super_root() {
|
||||
let g = build(&HprofWriter::new().finish()).unwrap();
|
||||
assert_csr(&g);
|
||||
assert_eq!((g.root, g.out.len(), g.classes.len()), (0, 0, 1));
|
||||
assert_eq!(g.class_slot, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn edge_count_past_u32_is_too_large() {
|
||||
let cancel = CancellationToken::new();
|
||||
let mut counts = [0, 2, 0, 3, 1];
|
||||
assert_eq!(accumulate(&mut counts, &cancel).unwrap(), 6);
|
||||
assert_eq!(counts, [0, 2, 2, 5, 6]);
|
||||
let mut full = [0, u32::MAX - 1, 1];
|
||||
assert_eq!(accumulate(&mut full, &cancel).unwrap(), u32::MAX);
|
||||
let mut over = [0, u32::MAX, 1];
|
||||
assert!(matches!(accumulate(&mut over, &cancel), Err(HeapError::TooLarge)));
|
||||
cancel.cancel();
|
||||
assert!(matches!(accumulate(&mut [0, 1], &cancel), Err(HeapError::Cancelled)));
|
||||
}
|
||||
|
||||
/// A class with `fields` instance fields, every one named by the same `name_len`-byte UTF8 record.
|
||||
fn shared_long_field_names(name_len: usize, fields: usize) -> Vec<u8> {
|
||||
let mut w = HprofWriter::new();
|
||||
let long = "f".repeat(name_len);
|
||||
w.string(1, "java/lang/Object").load_class(0x100, 1).class(0x100, 0, &[], &[]);
|
||||
w.string(2, "Wide").string(3, &long).load_class(0x200, 2);
|
||||
w.class(0x200, 0x100, &vec![(3, BasicType::Int); fields], &[]);
|
||||
w.finish()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_that_would_decode_far_larger_than_the_file_are_refused() {
|
||||
// 128 MiB of names from a file of about 70 KB.
|
||||
let bytes = shared_long_field_names(32 << 10, 4000);
|
||||
assert!(bytes.len() < 100 << 10, "{}", bytes.len());
|
||||
let err = build(&bytes).err().expect("refused");
|
||||
assert!(matches!(err, HeapError::NamesTooLarge), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_longer_than_the_jvm_allows_is_refused() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, &"C".repeat(70_000)).load_class(0x100, 1).class(0x100, 0, &[], &[]);
|
||||
assert!(matches!(build(&w.finish()), Err(HeapError::NamesTooLarge)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_sized_names_shared_by_many_fields_are_fine() {
|
||||
let g = build(&shared_long_field_names(40, 4000)).unwrap();
|
||||
assert_eq!(g.classes[class(&g, "Wide").0 as usize].fields.len(), 4000);
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn build_graph_never_panics_on_mutated_dumps(flips in proptest::collection::vec((0usize..4096, any::<u8>()), 1..12)) {
|
||||
let mut bytes = sample().bytes();
|
||||
for (pos, value) in flips {
|
||||
let at = pos % bytes.len();
|
||||
bytes[at] = value;
|
||||
}
|
||||
if let Ok(g) = build(&bytes) {
|
||||
assert_csr(&g);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,945 @@
|
||||
//! Reader for HotSpot `.hprof` heap dumps (format 1.0.1 / 1.0.2, 4- or 8-byte ids).
|
||||
|
||||
use std::ops::Range;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::HeapError;
|
||||
use crate::reader::Reader;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum BasicType {
|
||||
Object = 2,
|
||||
Boolean = 4,
|
||||
Char = 5,
|
||||
Float = 6,
|
||||
Double = 7,
|
||||
Byte = 8,
|
||||
Short = 9,
|
||||
Int = 10,
|
||||
Long = 11,
|
||||
}
|
||||
|
||||
impl BasicType {
|
||||
pub fn from_u8(v: u8) -> Option<Self> {
|
||||
Some(match v {
|
||||
2 => Self::Object,
|
||||
4 => Self::Boolean,
|
||||
5 => Self::Char,
|
||||
6 => Self::Float,
|
||||
7 => Self::Double,
|
||||
8 => Self::Byte,
|
||||
9 => Self::Short,
|
||||
10 => Self::Int,
|
||||
11 => Self::Long,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Bytes one value takes in the dump; `id_size` is the dump's identifier size.
|
||||
pub fn hprof_size(self, id_size: u8) -> usize {
|
||||
match self {
|
||||
Self::Object => usize::from(id_size),
|
||||
Self::Boolean | Self::Byte => 1,
|
||||
Self::Char | Self::Short => 2,
|
||||
Self::Float | Self::Int => 4,
|
||||
Self::Double | Self::Long => 8,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes one value takes in the JVM heap; `ref_size` is 4 with compressed oops, else 8.
|
||||
pub fn heap_size(self, ref_size: u8) -> u64 {
|
||||
match self {
|
||||
Self::Object => u64::from(ref_size),
|
||||
other => other.hprof_size(ref_size) as u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// JVM class name of an array of this type (`[I`); `Object` arrays have no single name, so
|
||||
/// they report the most general one.
|
||||
pub fn array_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Object => "[Ljava.lang.Object;",
|
||||
Self::Boolean => "[Z",
|
||||
Self::Char => "[C",
|
||||
Self::Float => "[F",
|
||||
Self::Double => "[D",
|
||||
Self::Byte => "[B",
|
||||
Self::Short => "[S",
|
||||
Self::Int => "[I",
|
||||
Self::Long => "[J",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum HeapRootKind {
|
||||
Unknown,
|
||||
JniGlobal,
|
||||
JniLocal,
|
||||
JavaFrame,
|
||||
NativeStack,
|
||||
StickyClass,
|
||||
ThreadBlock,
|
||||
MonitorUsed,
|
||||
ThreadObject,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct HprofHeader {
|
||||
pub version: String,
|
||||
pub id_size: u8,
|
||||
pub timestamp_ms: u64,
|
||||
/// Offset of the first record.
|
||||
pub body_start: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ClassDump {
|
||||
pub id: u64,
|
||||
pub super_id: u64,
|
||||
pub loader_id: u64,
|
||||
/// `(name id, object id)` of reference statics; a null static has object id 0.
|
||||
pub statics: Vec<(u64, u64)>,
|
||||
/// Heap bytes taken by the primitive statics.
|
||||
pub static_bytes: u64,
|
||||
/// Instance fields in dump order, `(name id, type)`.
|
||||
pub fields: Vec<(u64, BasicType)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum HeapRecord<'a> {
|
||||
/// `thread_serial` is present for the kinds that carry one.
|
||||
Root {
|
||||
kind: HeapRootKind,
|
||||
id: u64,
|
||||
thread_serial: Option<u32>,
|
||||
},
|
||||
Class(ClassDump),
|
||||
/// `data` is the field values, class first, then each superclass.
|
||||
Instance {
|
||||
id: u64,
|
||||
class_id: u64,
|
||||
data: &'a [u8],
|
||||
},
|
||||
/// `data` is `len` ids.
|
||||
ObjectArray {
|
||||
id: u64,
|
||||
class_id: u64,
|
||||
len: u32,
|
||||
data: &'a [u8],
|
||||
},
|
||||
/// `data` is `len` big-endian elements.
|
||||
PrimitiveArray {
|
||||
id: u64,
|
||||
ty: BasicType,
|
||||
len: u32,
|
||||
data: &'a [u8],
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum Record<'a> {
|
||||
Utf8 {
|
||||
id: u64,
|
||||
text: &'a [u8],
|
||||
},
|
||||
LoadClass {
|
||||
class_id: u64,
|
||||
name_id: u64,
|
||||
},
|
||||
/// Byte range of a heap-dump segment's sub-records; its contents follow as `Heap` records.
|
||||
Segment(Range<usize>),
|
||||
/// `at` is the offset of the sub-record's tag, for [`record_at`].
|
||||
Heap {
|
||||
at: usize,
|
||||
record: HeapRecord<'a>,
|
||||
},
|
||||
}
|
||||
|
||||
const MAGIC_1_0_2: &[u8] = b"JAVA PROFILE 1.0.2\0";
|
||||
const MAGIC_1_0_1: &[u8] = b"JAVA PROFILE 1.0.1\0";
|
||||
|
||||
const TAG_UTF8: u8 = 0x01;
|
||||
const TAG_LOAD_CLASS: u8 = 0x02;
|
||||
const TAG_HEAP_DUMP: u8 = 0x0C;
|
||||
const TAG_HEAP_DUMP_SEGMENT: u8 = 0x1C;
|
||||
const TAG_HEAP_DUMP_END: u8 = 0x2C;
|
||||
/// Tag (u1), timestamp (u4) and body length (u4).
|
||||
const RECORD_HEADER_LEN: usize = 9;
|
||||
|
||||
fn check_id_size(id_size: u8) -> Result<(), HeapError> {
|
||||
match id_size {
|
||||
4 | 8 => Ok(()),
|
||||
other => Err(HeapError::UnsupportedIdSize(u32::from(other))),
|
||||
}
|
||||
}
|
||||
|
||||
fn truncated(buf: &[u8]) -> HeapError {
|
||||
HeapError::Truncated { at: buf.len() as u64 }
|
||||
}
|
||||
|
||||
pub fn parse_header(buf: &[u8]) -> Result<HprofHeader, HeapError> {
|
||||
let magic =
|
||||
[MAGIC_1_0_2, MAGIC_1_0_1].into_iter().find(|magic| buf.starts_with(magic)).ok_or(HeapError::NotHprof)?;
|
||||
let mut r = Reader::new(buf, magic.len(), 8);
|
||||
let id_size = r.u4()?;
|
||||
let id_size = u8::try_from(id_size).map_err(|_| HeapError::UnsupportedIdSize(id_size))?;
|
||||
check_id_size(id_size)?;
|
||||
let timestamp_ms = r.u8()?;
|
||||
let version = String::from_utf8_lossy(&magic[..magic.len() - 1]).into_owned();
|
||||
Ok(HprofHeader { version, id_size, timestamp_ms, body_start: r.pos() })
|
||||
}
|
||||
|
||||
/// Walks every top-level record and, for heap-dump segments, the sub-records inside them.
|
||||
/// A record whose length runs past the end of `buf` is `Truncated`, and so are segments that no
|
||||
/// HEAP DUMP END follows: HotSpot writes whole segments, so a dump it did not finish ends after one.
|
||||
pub fn scan<'a>(
|
||||
buf: &'a [u8],
|
||||
header: &HprofHeader,
|
||||
visit: &mut dyn FnMut(Record<'a>) -> Result<(), HeapError>,
|
||||
) -> Result<(), HeapError> {
|
||||
let id_size = header.id_size;
|
||||
check_id_size(id_size)?;
|
||||
if header.body_start > buf.len() {
|
||||
return Err(truncated(buf));
|
||||
}
|
||||
let mut pos = header.body_start;
|
||||
let mut unended_segments = false;
|
||||
while pos < buf.len() {
|
||||
let mut head = Reader::new(buf, pos, id_size);
|
||||
let tag = head.u1()?;
|
||||
head.skip(4)?; // timestamp
|
||||
let len = head.u4()? as usize;
|
||||
let start = pos + RECORD_HEADER_LEN;
|
||||
let end = start.checked_add(len).filter(|&end| end <= buf.len()).ok_or_else(|| truncated(buf))?;
|
||||
let body = &buf[..end];
|
||||
let mut r = Reader::new(body, start, id_size);
|
||||
match tag {
|
||||
TAG_UTF8 => {
|
||||
let id = r.id()?;
|
||||
let text = r.bytes(end - r.pos())?;
|
||||
visit(Record::Utf8 { id, text })?;
|
||||
}
|
||||
TAG_LOAD_CLASS => {
|
||||
r.skip(4)?; // class serial
|
||||
let class_id = r.id()?;
|
||||
r.skip(4)?; // stack trace serial
|
||||
let name_id = r.id()?;
|
||||
visit(Record::LoadClass { class_id, name_id })?;
|
||||
}
|
||||
TAG_HEAP_DUMP | TAG_HEAP_DUMP_SEGMENT => {
|
||||
// A single HEAP DUMP (JDK 8, under 2 GB) has no end record.
|
||||
unended_segments |= tag == TAG_HEAP_DUMP_SEGMENT;
|
||||
visit(Record::Segment(start..end))?;
|
||||
walk_segment(buf, start..end, id_size, &mut |at, record| visit(Record::Heap { at, record }))?;
|
||||
}
|
||||
TAG_HEAP_DUMP_END => unended_segments = false,
|
||||
_ => {}
|
||||
}
|
||||
pos = end;
|
||||
}
|
||||
if unended_segments {
|
||||
return Err(truncated(buf));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Visits the sub-records of one segment (the range from [`Record::Segment`]) with their offsets.
|
||||
/// The last sub-record must end exactly at the segment's end.
|
||||
pub fn scan_segment<'a>(
|
||||
buf: &'a [u8],
|
||||
segment: Range<usize>,
|
||||
id_size: u8,
|
||||
visit: &mut dyn FnMut(usize, HeapRecord<'a>) -> Result<(), HeapError>,
|
||||
) -> Result<(), HeapError> {
|
||||
check_id_size(id_size)?;
|
||||
walk_segment(buf, segment, id_size, visit)
|
||||
}
|
||||
|
||||
fn walk_segment<'a>(
|
||||
buf: &'a [u8],
|
||||
segment: Range<usize>,
|
||||
id_size: u8,
|
||||
visit: &mut dyn FnMut(usize, HeapRecord<'a>) -> Result<(), HeapError>,
|
||||
) -> Result<(), HeapError> {
|
||||
if segment.start > segment.end || segment.end > buf.len() {
|
||||
return Err(truncated(buf));
|
||||
}
|
||||
let mut r = Reader::new(&buf[..segment.end], segment.start, id_size);
|
||||
while r.pos() < segment.end {
|
||||
let at = r.pos();
|
||||
let record = read_sub_record(&mut r)?;
|
||||
visit(at, record)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reads the sub-record whose tag is at `at`, as given by [`Record::Heap`].
|
||||
pub fn record_at(buf: &[u8], at: usize, id_size: u8) -> Result<HeapRecord<'_>, HeapError> {
|
||||
check_id_size(id_size)?;
|
||||
read_sub_record(&mut Reader::new(buf, at, id_size))
|
||||
}
|
||||
|
||||
fn read_sub_record<'a>(r: &mut Reader<'a>) -> Result<HeapRecord<'a>, HeapError> {
|
||||
let at = r.pos();
|
||||
let tag = r.u1()?;
|
||||
// (kind, carries a thread serial, bytes after it)
|
||||
let root = match tag {
|
||||
0xFF => Some((HeapRootKind::Unknown, false, 0)),
|
||||
0x01 => Some((HeapRootKind::JniGlobal, false, usize::from(r.id_size()))),
|
||||
0x02 => Some((HeapRootKind::JniLocal, true, 4)),
|
||||
0x03 => Some((HeapRootKind::JavaFrame, true, 4)),
|
||||
0x04 => Some((HeapRootKind::NativeStack, true, 0)),
|
||||
0x05 => Some((HeapRootKind::StickyClass, false, 0)),
|
||||
0x06 => Some((HeapRootKind::ThreadBlock, true, 0)),
|
||||
0x07 => Some((HeapRootKind::MonitorUsed, false, 0)),
|
||||
0x08 => Some((HeapRootKind::ThreadObject, true, 4)),
|
||||
_ => None,
|
||||
};
|
||||
if let Some((kind, has_serial, trailing)) = root {
|
||||
let id = r.id()?;
|
||||
let thread_serial = if has_serial { Some(r.u4()?) } else { None };
|
||||
r.skip(trailing)?;
|
||||
return Ok(HeapRecord::Root { kind, id, thread_serial });
|
||||
}
|
||||
match tag {
|
||||
0x20 => read_class_dump(r).map(HeapRecord::Class),
|
||||
0x21 => {
|
||||
let id = r.id()?;
|
||||
r.skip(4)?; // stack trace serial
|
||||
let class_id = r.id()?;
|
||||
let len = r.u4()? as usize;
|
||||
Ok(HeapRecord::Instance { id, class_id, data: r.bytes(len)? })
|
||||
}
|
||||
0x22 => {
|
||||
let id = r.id()?;
|
||||
r.skip(4)?;
|
||||
let len = r.u4()?;
|
||||
let class_id = r.id()?;
|
||||
let size = (len as usize).checked_mul(usize::from(r.id_size())).ok_or_else(|| truncated(r.buf()))?;
|
||||
Ok(HeapRecord::ObjectArray { id, class_id, len, data: r.bytes(size)? })
|
||||
}
|
||||
0x23 => {
|
||||
let id = r.id()?;
|
||||
r.skip(4)?;
|
||||
let len = r.u4()?;
|
||||
let ty = read_type(r)?;
|
||||
if ty == BasicType::Object {
|
||||
return Err(HeapError::UnknownType { ty: ty as u8, at: (r.pos() - 1) as u64 });
|
||||
}
|
||||
let size = (len as usize).checked_mul(ty.hprof_size(r.id_size())).ok_or_else(|| truncated(r.buf()))?;
|
||||
Ok(HeapRecord::PrimitiveArray { id, ty, len, data: r.bytes(size)? })
|
||||
}
|
||||
tag => Err(HeapError::UnknownSubRecord { tag, at: at as u64 }),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_type(r: &mut Reader<'_>) -> Result<BasicType, HeapError> {
|
||||
let at = r.pos();
|
||||
let ty = r.u1()?;
|
||||
BasicType::from_u8(ty).ok_or(HeapError::UnknownType { ty, at: at as u64 })
|
||||
}
|
||||
|
||||
fn read_class_dump(r: &mut Reader<'_>) -> Result<ClassDump, HeapError> {
|
||||
let id_size = r.id_size();
|
||||
let id = r.id()?;
|
||||
r.skip(4)?; // stack trace serial
|
||||
let super_id = r.id()?;
|
||||
let loader_id = r.id()?;
|
||||
r.skip(4 * usize::from(id_size))?; // signers, protection domain, 2 reserved
|
||||
r.skip(4)?; // instance size
|
||||
|
||||
for _ in 0..r.u2()? {
|
||||
r.skip(2)?; // constant pool index
|
||||
let ty = read_type(r)?;
|
||||
r.skip(ty.hprof_size(id_size))?;
|
||||
}
|
||||
|
||||
let mut statics = Vec::new();
|
||||
let mut static_bytes = 0;
|
||||
for _ in 0..r.u2()? {
|
||||
let name = r.id()?;
|
||||
let ty = read_type(r)?;
|
||||
if ty == BasicType::Object {
|
||||
statics.push((name, r.id()?));
|
||||
} else {
|
||||
r.skip(ty.hprof_size(id_size))?;
|
||||
static_bytes += ty.heap_size(id_size);
|
||||
}
|
||||
}
|
||||
|
||||
let mut fields = Vec::new();
|
||||
for _ in 0..r.u2()? {
|
||||
let name = r.id()?;
|
||||
fields.push((name, read_type(r)?));
|
||||
}
|
||||
Ok(ClassDump { id, super_id, loader_id, statics, static_bytes, fields })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::testing::{HprofWriter, MiniHeap};
|
||||
use proptest::prelude::*;
|
||||
|
||||
fn collect(bytes: &[u8]) -> Result<Vec<Record<'_>>, HeapError> {
|
||||
let header = parse_header(bytes)?;
|
||||
let mut out = Vec::new();
|
||||
scan(bytes, &header, &mut |r| {
|
||||
out.push(r);
|
||||
Ok(())
|
||||
})?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn segment_of(bytes: &[u8]) -> Range<usize> {
|
||||
collect(bytes)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find_map(|r| if let Record::Segment(s) = r { Some(s) } else { None })
|
||||
.expect("a segment")
|
||||
}
|
||||
|
||||
fn all_kinds(w: &mut HprofWriter) {
|
||||
w.string(1, "Node")
|
||||
.string(2, "next")
|
||||
.load_class(0x100, 1)
|
||||
.class(0x100, 0, &[(2, BasicType::Object), (3, BasicType::Long)], &[(2, 0x10)])
|
||||
.instance(0x10, 0x100, &[0u8; 16])
|
||||
.obj_array(0x20, 0x100, &[0x10, 0])
|
||||
.prim_array(0x30, BasicType::Int, 3)
|
||||
.root(HeapRootKind::JavaFrame, 0x10)
|
||||
.root(HeapRootKind::StickyClass, 0x100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_header() {
|
||||
let header = parse_header(&MiniHeap::new().bytes()).unwrap();
|
||||
assert_eq!((header.version.as_str(), header.id_size, header.body_start), ("JAVA PROFILE 1.0.2", 8, 31));
|
||||
|
||||
let mut old = b"JAVA PROFILE 1.0.1\0".to_vec();
|
||||
old.extend_from_slice(&[0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 2]);
|
||||
let header = parse_header(&old).unwrap();
|
||||
assert_eq!(
|
||||
(header.version.as_str(), header.id_size, header.timestamp_ms),
|
||||
("JAVA PROFILE 1.0.1", 4, (1 << 32) + 2)
|
||||
);
|
||||
|
||||
let bad_size = b"JAVA PROFILE 1.0.2\0\0\0\0\x03\0\0\0\0\0\0\0\0";
|
||||
assert!(matches!(parse_header(bad_size), Err(HeapError::UnsupportedIdSize(3))));
|
||||
assert!(matches!(parse_header(b"GIF89a"), Err(HeapError::NotHprof)));
|
||||
assert!(matches!(parse_header(b""), Err(HeapError::NotHprof)));
|
||||
assert!(matches!(parse_header(b"JAVA PROFILE 1.0.3\0\0\0\0\x08"), Err(HeapError::NotHprof)));
|
||||
assert!(matches!(parse_header(b"JAVA PROFILE 1.0.2\0\0\0"), Err(HeapError::Truncated { at: 21 })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scans_every_record_kind() {
|
||||
let mut w = HprofWriter::new();
|
||||
all_kinds(&mut w);
|
||||
let bytes = w.finish();
|
||||
let records = collect(&bytes).unwrap();
|
||||
|
||||
let (mut utf8, mut loads, mut segments) = (0, 0, 0);
|
||||
let (mut classes, mut instances, mut obj_arrays, mut prim_arrays, mut roots) = (0, 0, 0, 0, 0);
|
||||
for record in &records {
|
||||
match record {
|
||||
Record::Utf8 { .. } => utf8 += 1,
|
||||
Record::LoadClass { class_id, name_id } => {
|
||||
loads += 1;
|
||||
assert_eq!((*class_id, *name_id), (0x100, 1));
|
||||
}
|
||||
Record::Segment(_) => segments += 1,
|
||||
Record::Heap { record, .. } => match record {
|
||||
HeapRecord::Class(c) => {
|
||||
classes += 1;
|
||||
assert_eq!((c.id, c.super_id, c.loader_id), (0x100, 0, 0));
|
||||
assert_eq!(c.statics, [(2, 0x10)]);
|
||||
assert_eq!(c.fields, [(2, BasicType::Object), (3, BasicType::Long)]);
|
||||
}
|
||||
HeapRecord::Instance { id, class_id, data } => {
|
||||
instances += 1;
|
||||
assert_eq!((*id, *class_id, data.len()), (0x10, 0x100, 16));
|
||||
}
|
||||
HeapRecord::ObjectArray { id, class_id, len, data } => {
|
||||
obj_arrays += 1;
|
||||
assert_eq!((*id, *class_id, *len, data.len()), (0x20, 0x100, 2, 16));
|
||||
}
|
||||
HeapRecord::PrimitiveArray { id, ty, len, data } => {
|
||||
prim_arrays += 1;
|
||||
assert_eq!((*id, *ty, *len, data.len()), (0x30, BasicType::Int, 3, 12));
|
||||
}
|
||||
HeapRecord::Root { kind, id, thread_serial } => {
|
||||
roots += 1;
|
||||
match kind {
|
||||
HeapRootKind::JavaFrame => assert_eq!((*id, *thread_serial), (0x10, Some(1))),
|
||||
HeapRootKind::StickyClass => assert_eq!((*id, *thread_serial), (0x100, None)),
|
||||
other => panic!("unexpected root {other:?}"),
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
assert_eq!((utf8, loads, segments), (2, 1, 1));
|
||||
assert_eq!((classes, instances, obj_arrays, prim_arrays, roots), (1, 1, 1, 1, 2));
|
||||
assert!(matches!(&records[3], Record::Segment(s) if s.end <= bytes.len()), "segment precedes its contents");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn root_kinds_carry_the_thread_serial_they_have() {
|
||||
let kinds = [
|
||||
(HeapRootKind::Unknown, false),
|
||||
(HeapRootKind::JniGlobal, false),
|
||||
(HeapRootKind::JniLocal, true),
|
||||
(HeapRootKind::JavaFrame, true),
|
||||
(HeapRootKind::NativeStack, true),
|
||||
(HeapRootKind::StickyClass, false),
|
||||
(HeapRootKind::ThreadBlock, true),
|
||||
(HeapRootKind::MonitorUsed, false),
|
||||
(HeapRootKind::ThreadObject, true),
|
||||
];
|
||||
let mut w = HprofWriter::new();
|
||||
for (i, (kind, _)) in kinds.iter().enumerate() {
|
||||
w.root_in_thread(*kind, 0x40 + i as u64, 7);
|
||||
w.root(*kind, 0x80 + i as u64);
|
||||
}
|
||||
let bytes = w.finish();
|
||||
let mut seen = Vec::new();
|
||||
for r in collect(&bytes).unwrap() {
|
||||
if let Record::Heap { record: HeapRecord::Root { kind, id, thread_serial }, .. } = r {
|
||||
seen.push((kind, id, thread_serial));
|
||||
}
|
||||
}
|
||||
assert_eq!(seen.len(), kinds.len() * 2);
|
||||
for (i, (kind, carries)) in kinds.iter().enumerate() {
|
||||
let explicit = seen[2 * i];
|
||||
let default = seen[2 * i + 1];
|
||||
assert_eq!(explicit, (*kind, 0x40 + i as u64, carries.then_some(7)));
|
||||
assert_eq!(default, (*kind, 0x80 + i as u64, carries.then_some(1)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn primitive_arrays_expose_their_contents() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.prim_bytes(0x1, BasicType::Char, &[0, b'h', 0, 0xE9]).prim_bytes(0x2, BasicType::Byte, b"v42");
|
||||
let bytes = w.finish();
|
||||
let arrays: Vec<_> = collect(&bytes)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter_map(|r| match r {
|
||||
Record::Heap { record: HeapRecord::PrimitiveArray { ty, len, data, .. }, .. } => {
|
||||
Some((ty, len, data.to_vec()))
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(arrays, [(BasicType::Char, 2, vec![0, b'h', 0, 0xE9]), (BasicType::Byte, 3, b"v42".to_vec())]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mini_heap_builds_strings_and_defines_object_and_class() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.string(0x10, "v42", true).string(0x20, "hé", false);
|
||||
let bytes = heap.bytes();
|
||||
let records = collect(&bytes).unwrap();
|
||||
let names: Vec<_> = records
|
||||
.iter()
|
||||
.filter_map(|r| {
|
||||
if let Record::Utf8 { text, .. } = r { Some(String::from_utf8_lossy(text).into_owned()) } else { None }
|
||||
})
|
||||
.collect();
|
||||
assert!(names.iter().any(|n| n == "java/lang/Object") && names.iter().any(|n| n == "java/lang/Class"));
|
||||
let arrays: Vec<_> = records
|
||||
.iter()
|
||||
.filter_map(|r| match r {
|
||||
Record::Heap { record: HeapRecord::PrimitiveArray { data, .. }, .. } => Some(data.to_vec()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(arrays, [b"v42".to_vec(), vec![b'h', 0, 0xE9, 0]]);
|
||||
let coders: Vec<_> = records
|
||||
.iter()
|
||||
.filter_map(|r| match r {
|
||||
Record::Heap { record: HeapRecord::Instance { id, data, .. }, .. } => Some((*id, data[8])),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(coders, [(0x10, 0), (0x20, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mini_heap_lays_out_fields_class_first_then_super() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("A", Some("java.lang.Object"), &[("a", BasicType::Int)])
|
||||
.class("B", Some("A"), &[("b", BasicType::Object), ("c", BasicType::Byte)])
|
||||
.object(0x10, "B", &[("a", 7), ("b", 0x99), ("c", 0x1FF)]);
|
||||
let bytes = heap.bytes();
|
||||
let data = collect(&bytes)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find_map(|r| match r {
|
||||
Record::Heap { record: HeapRecord::Instance { data, .. }, .. } => Some(data.to_vec()),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(data, [0, 0, 0, 0, 0, 0, 0, 0x99, 0xFF, 0, 0, 0, 7]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_at_reads_back_an_instance() {
|
||||
let mut w = HprofWriter::new();
|
||||
all_kinds(&mut w);
|
||||
let bytes = w.finish();
|
||||
let at = collect(&bytes)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.find_map(|r| match r {
|
||||
Record::Heap { at, record: HeapRecord::Instance { .. } } => Some(at),
|
||||
_ => None,
|
||||
})
|
||||
.unwrap();
|
||||
match record_at(&bytes, at, 8).unwrap() {
|
||||
HeapRecord::Instance { id, class_id, data } => assert_eq!((id, class_id, data.len()), (0x10, 0x100, 16)),
|
||||
other => panic!("expected an instance, got {other:?}"),
|
||||
}
|
||||
assert!(matches!(record_at(&bytes, bytes.len(), 8), Err(HeapError::Truncated { .. })));
|
||||
assert!(matches!(record_at(&bytes, at, 3), Err(HeapError::UnsupportedIdSize(3))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_segment_visits_the_same_records_scan_does() {
|
||||
let mut w = HprofWriter::new();
|
||||
all_kinds(&mut w);
|
||||
let bytes = w.finish();
|
||||
let from_scan: Vec<_> = collect(&bytes)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter_map(|r| if let Record::Heap { at, record } = r { Some((at, record)) } else { None })
|
||||
.collect();
|
||||
let mut from_segment = Vec::new();
|
||||
scan_segment(&bytes, segment_of(&bytes), 8, &mut |at, record| {
|
||||
from_segment.push((at, record));
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(from_scan.len(), 6);
|
||||
assert_eq!(from_scan, from_segment);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn visitor_errors_stop_the_scan() {
|
||||
let bytes = MiniHeap::new().bytes();
|
||||
let header = parse_header(&bytes).unwrap();
|
||||
let mut calls = 0;
|
||||
let result = scan(&bytes, &header, &mut |_| {
|
||||
calls += 1;
|
||||
Err(HeapError::Cancelled)
|
||||
});
|
||||
assert!(matches!(result, Err(HeapError::Cancelled)));
|
||||
assert_eq!(calls, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_heap_dump_tags_are_segments() {
|
||||
let bytes = MiniHeap::new().bytes();
|
||||
let segment = segment_of(&bytes);
|
||||
let mut old_style = bytes.clone();
|
||||
old_style[segment.start - 9] = 0x0C;
|
||||
assert_eq!(segment_of(&old_style), segment);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn id_size_4_dump_parses() {
|
||||
let mut w = HprofWriter::with_id_size(4);
|
||||
w.string(1, "Node")
|
||||
.load_class(0x100, 1)
|
||||
.class(0x100, 0, &[(2, BasicType::Object), (3, BasicType::Long)], &[(2, 0x10)])
|
||||
.instance(0x10, 0x100, &[0u8; 12])
|
||||
.obj_array(0x20, 0x100, &[0x10, 0, 0xFFFF_FFFF])
|
||||
.prim_array(0x30, BasicType::Short, 5)
|
||||
.root(HeapRootKind::ThreadObject, 0x10);
|
||||
let bytes = w.finish();
|
||||
let header = parse_header(&bytes).unwrap();
|
||||
assert_eq!((header.id_size, header.body_start), (4, 31));
|
||||
let records: Vec<_> = collect(&bytes)
|
||||
.unwrap()
|
||||
.into_iter()
|
||||
.filter_map(|r| if let Record::Heap { record, .. } = r { Some(record) } else { None })
|
||||
.collect();
|
||||
assert_eq!(records.len(), 5);
|
||||
match &records[0] {
|
||||
HeapRecord::Class(c) => {
|
||||
assert_eq!((c.id, c.statics.as_slice(), c.fields.len()), (0x100, &[(2, 0x10)][..], 2));
|
||||
}
|
||||
other => panic!("expected the class, got {other:?}"),
|
||||
}
|
||||
assert!(matches!(&records[1], HeapRecord::Instance { id: 0x10, class_id: 0x100, data } if data.len() == 12));
|
||||
assert!(
|
||||
matches!(&records[2], HeapRecord::ObjectArray { len: 3, data, .. } if data.len() == 12 && data[8..] == [0xFF; 4])
|
||||
);
|
||||
assert!(
|
||||
matches!(&records[3], HeapRecord::PrimitiveArray { ty: BasicType::Short, len: 5, data, .. } if data.len() == 10)
|
||||
);
|
||||
assert!(matches!(
|
||||
&records[4],
|
||||
HeapRecord::Root { kind: HeapRootKind::ThreadObject, id: 0x10, thread_serial: Some(1) }
|
||||
));
|
||||
}
|
||||
|
||||
fn push_id(out: &mut Vec<u8>, v: u64) {
|
||||
out.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
|
||||
/// A class dump written by hand: loader, a constant-pool entry, and statics of each shape.
|
||||
fn class_dump_bytes(last_field_type: u8) -> Vec<u8> {
|
||||
let mut b = vec![0x20];
|
||||
push_id(&mut b, 0x500);
|
||||
b.extend_from_slice(&0u32.to_be_bytes());
|
||||
for id in [0x400, 0x77, 0, 0, 0, 0] {
|
||||
push_id(&mut b, id); // super, loader, signers, domain, 2 reserved
|
||||
}
|
||||
b.extend_from_slice(&24u32.to_be_bytes());
|
||||
b.extend_from_slice(&[0, 1, 0, 7, 10, 0, 0, 0, 99]); // constant pool: one int
|
||||
b.extend_from_slice(&[0, 3]);
|
||||
push_id(&mut b, 0xA);
|
||||
b.push(2);
|
||||
push_id(&mut b, 0xBEEF); // object static
|
||||
push_id(&mut b, 0xB);
|
||||
b.push(10);
|
||||
b.extend_from_slice(&5u32.to_be_bytes()); // int static
|
||||
push_id(&mut b, 0xC);
|
||||
b.push(11);
|
||||
b.extend_from_slice(&6u64.to_be_bytes()); // long static
|
||||
b.extend_from_slice(&[0, 2]);
|
||||
push_id(&mut b, 0xD);
|
||||
b.push(2);
|
||||
push_id(&mut b, 0xE);
|
||||
b.push(last_field_type);
|
||||
b
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_dump_reads_loader_statics_and_skips_the_constant_pool() {
|
||||
let bytes = class_dump_bytes(11);
|
||||
let HeapRecord::Class(c) = record_at(&bytes, 0, 8).unwrap() else { panic!("expected a class") };
|
||||
assert_eq!((c.id, c.super_id, c.loader_id), (0x500, 0x400, 0x77));
|
||||
assert_eq!(c.statics, [(0xA, 0xBEEF)]);
|
||||
assert_eq!(c.static_bytes, 12);
|
||||
assert_eq!(c.fields, [(0xD, BasicType::Object), (0xE, BasicType::Long)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_value_type_reports_its_offset() {
|
||||
let bytes = class_dump_bytes(0x63);
|
||||
let err = record_at(&bytes, 0, 8).unwrap_err();
|
||||
assert!(matches!(err, HeapError::UnknownType { ty: 0x63, at } if at == bytes.len() as u64 - 1), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_dump_reports_offset() {
|
||||
let mut bytes = MiniHeap::new().bytes();
|
||||
let body_start = parse_header(&bytes).unwrap().body_start as u64;
|
||||
bytes.truncate(bytes.len() - 5);
|
||||
let err = collect(&bytes).unwrap_err();
|
||||
assert!(matches!(err, HeapError::Truncated { at } if at > body_start && at == bytes.len() as u64), "{err:?}");
|
||||
assert!(err.to_string().contains(&bytes.len().to_string()));
|
||||
}
|
||||
|
||||
/// HotSpot flushes whole segments, so a JVM killed while dumping leaves a file that ends
|
||||
/// right after one, without HEAP DUMP END.
|
||||
#[test]
|
||||
fn a_dump_the_jvm_did_not_finish_ends_early() {
|
||||
let mut bytes = MiniHeap::new().bytes();
|
||||
bytes.truncate(bytes.len() - RECORD_HEADER_LEN);
|
||||
let err = collect(&bytes).unwrap_err();
|
||||
assert!(matches!(err, HeapError::Truncated { at } if at == bytes.len() as u64), "{err:?}");
|
||||
assert!(err.to_string().starts_with("the dump ends early"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_prefix_is_ok_or_truncated_at_its_length() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("N", Some("java.lang.Object"), &[("next", BasicType::Object)])
|
||||
.object(0x10, "N", &[("next", 0x20)])
|
||||
.string(0x20, "x", true)
|
||||
.root(HeapRootKind::ThreadObject, 0x10);
|
||||
let bytes = heap.bytes();
|
||||
let header = parse_header(&bytes).unwrap();
|
||||
let segment = segment_of(&bytes);
|
||||
for cut in header.body_start..bytes.len() {
|
||||
let prefix = &bytes[..cut];
|
||||
match scan(prefix, &header, &mut |_| Ok(())) {
|
||||
// Only a prefix without heap data is a complete (empty) dump.
|
||||
Ok(()) => assert!(cut < segment.start, "cut at {cut} has heap data but no end"),
|
||||
Err(HeapError::Truncated { at }) => assert_eq!(at, cut as u64, "cut at {cut}"),
|
||||
Err(other) => panic!("cut at {cut}: {other:?}"),
|
||||
}
|
||||
}
|
||||
let past_end = HprofHeader { body_start: bytes.len() + 1, ..header };
|
||||
assert!(matches!(scan(&bytes, &past_end, &mut |_| Ok(())), Err(HeapError::Truncated { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sub_record_overrunning_its_segment_is_truncated() {
|
||||
let mut bytes = MiniHeap::new().bytes();
|
||||
let segment = segment_of(&bytes);
|
||||
let len_at = segment.start - 4;
|
||||
let shrunk = (segment.len() as u32 - 3).to_be_bytes();
|
||||
bytes[len_at..segment.start].copy_from_slice(&shrunk);
|
||||
let err = collect(&bytes).unwrap_err();
|
||||
assert!(matches!(err, HeapError::Truncated { at } if at == segment.end as u64 - 3), "{err:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_sub_record_is_an_error() {
|
||||
let mut bytes = MiniHeap::new().bytes();
|
||||
let segment = segment_of(&bytes);
|
||||
bytes[segment.start] = 0x99;
|
||||
let err = collect(&bytes).unwrap_err();
|
||||
assert!(matches!(err, HeapError::UnknownSubRecord { tag: 0x99, at } if at == segment.start as u64), "{err:?}");
|
||||
assert!(err.to_string().contains("0x99"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn array_lengths_larger_than_the_dump_do_not_allocate_or_panic() {
|
||||
let mut b = vec![0x23];
|
||||
push_id(&mut b, 1);
|
||||
b.extend_from_slice(&0u32.to_be_bytes());
|
||||
b.extend_from_slice(&u32::MAX.to_be_bytes());
|
||||
b.push(11);
|
||||
assert!(matches!(record_at(&b, 0, 8), Err(HeapError::Truncated { .. })));
|
||||
b[0] = 0x22;
|
||||
assert!(matches!(record_at(&b, 0, 8), Err(HeapError::Truncated { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn primitive_array_of_objects_is_an_unknown_type() {
|
||||
let mut b = vec![0x23];
|
||||
push_id(&mut b, 1);
|
||||
b.extend_from_slice(&[0, 0, 0, 0, 0, 0, 0, 0]);
|
||||
b.push(2);
|
||||
assert!(matches!(record_at(&b, 0, 8), Err(HeapError::UnknownType { ty: 2, .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn basic_type_sizes_and_names() {
|
||||
assert_eq!(BasicType::from_u8(10), Some(BasicType::Int));
|
||||
assert_eq!([0, 1, 3, 12, 255].map(BasicType::from_u8), [None; 5]);
|
||||
assert_eq!((BasicType::Object.hprof_size(4), BasicType::Object.hprof_size(8)), (4, 8));
|
||||
assert_eq!((BasicType::Object.heap_size(4), BasicType::Long.heap_size(4)), (4, 8));
|
||||
assert_eq!((BasicType::Char.hprof_size(8), BasicType::Boolean.heap_size(8)), (2, 1));
|
||||
assert_eq!(
|
||||
(BasicType::Int.array_name(), BasicType::Boolean.array_name(), BasicType::Long.array_name()),
|
||||
("[I", "[Z", "[J")
|
||||
);
|
||||
}
|
||||
|
||||
fn sample() -> Vec<u8> {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Node", Some("java.lang.Object"), &[("next", BasicType::Object), ("v", BasicType::Long)])
|
||||
.class("[LNode;", Some("java.lang.Object"), &[])
|
||||
.object(0x10, "Node", &[("next", 0x20), ("v", 5)])
|
||||
.object(0x20, "Node", &[])
|
||||
.array_of(0x30, "[LNode;", &[0x10, 0x20, 0])
|
||||
.prim(0x40, BasicType::Int, 3)
|
||||
.string(0x50, "hello", true)
|
||||
.root(HeapRootKind::ThreadObject, 0x10)
|
||||
.root(HeapRootKind::JniGlobal, 0x30);
|
||||
heap.bytes()
|
||||
}
|
||||
|
||||
fn with_header(id_size: u8, body: &[u8]) -> Vec<u8> {
|
||||
let mut bytes = MAGIC_1_0_2.to_vec();
|
||||
bytes.extend_from_slice(&u32::from(id_size).to_be_bytes());
|
||||
bytes.extend_from_slice(&1_700_000_000_000u64.to_be_bytes());
|
||||
bytes.extend_from_slice(body);
|
||||
bytes
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#[test]
|
||||
fn parser_never_panics(bytes in proptest::collection::vec(any::<u8>(), 0..4096)) {
|
||||
let _ = parse_header(&bytes).and_then(|h| scan(&bytes, &h, &mut |_| Ok(())));
|
||||
}
|
||||
|
||||
/// Random bytes alone fail the magic check, so this puts a valid header in front to make
|
||||
/// `scan` and `scan_segment` run on arbitrary record streams.
|
||||
#[test]
|
||||
fn random_bytes_after_a_valid_header_never_panic(
|
||||
id_size in prop_oneof![Just(4u8), Just(8u8)],
|
||||
body in proptest::collection::vec(any::<u8>(), 0..4096),
|
||||
) {
|
||||
let bytes = with_header(id_size, &body);
|
||||
let header = parse_header(&bytes).unwrap();
|
||||
prop_assert_eq!(header.id_size, id_size);
|
||||
let _ = scan(&bytes, &header, &mut |_| Ok(()));
|
||||
let _ = scan_segment(&bytes, header.body_start..bytes.len(), id_size, &mut |_, _| Ok(()));
|
||||
for at in header.body_start..bytes.len() {
|
||||
let _ = record_at(&bytes, at, id_size);
|
||||
}
|
||||
}
|
||||
|
||||
/// Same, but every record has a correct length so the random bytes reach the heap-dump
|
||||
/// segments and the sub-record parsers instead of stopping at the first bad length.
|
||||
#[test]
|
||||
fn random_records_after_a_valid_header_never_panic(
|
||||
id_size in prop_oneof![Just(4u8), Just(8u8)],
|
||||
records in proptest::collection::vec(
|
||||
(
|
||||
prop_oneof![Just(0x01u8), Just(0x02), Just(0x0C), Just(0x1C), Just(0x2C), any::<u8>()],
|
||||
prop_oneof![
|
||||
Just(0x20u8), Just(0x21), Just(0x22), Just(0x23), Just(0x01), Just(0x02), Just(0x03),
|
||||
Just(0x04), Just(0x05), Just(0x06), Just(0x07), Just(0x08), Just(0xFF), any::<u8>()
|
||||
],
|
||||
proptest::collection::vec(any::<u8>(), 0..96),
|
||||
),
|
||||
0..12,
|
||||
),
|
||||
) {
|
||||
let mut body = Vec::new();
|
||||
for (tag, first, rest) in records {
|
||||
let mut record_body = vec![first];
|
||||
record_body.extend_from_slice(&rest);
|
||||
body.push(tag);
|
||||
body.extend_from_slice(&0u32.to_be_bytes());
|
||||
body.extend_from_slice(&(record_body.len() as u32).to_be_bytes());
|
||||
body.extend_from_slice(&record_body);
|
||||
}
|
||||
let bytes = with_header(id_size, &body);
|
||||
let header = parse_header(&bytes).unwrap();
|
||||
let mut segments = Vec::new();
|
||||
let _ = scan(&bytes, &header, &mut |r| {
|
||||
if let Record::Segment(s) = r {
|
||||
segments.push(s);
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
for s in segments {
|
||||
let _ = scan_segment(&bytes, s, id_size, &mut |_, _| Ok(()));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutated_dump_never_panics(flips in proptest::collection::vec((0usize..2048, any::<u8>()), 1..16)) {
|
||||
let pristine = sample();
|
||||
let header = parse_header(&pristine).unwrap();
|
||||
let segment = segment_of(&pristine);
|
||||
let mut bytes = pristine.clone();
|
||||
for (pos, value) in flips {
|
||||
let at = pos % bytes.len();
|
||||
bytes[at] = value;
|
||||
}
|
||||
let _ = scan(&bytes, &header, &mut |_| Ok(()));
|
||||
for id_size in [4, 8] {
|
||||
let _ = scan_segment(&bytes, segment.clone(), id_size, &mut |_, _| Ok(()));
|
||||
}
|
||||
for at in 0..bytes.len() {
|
||||
let _ = record_at(&bytes, at, 8);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
//! Maps hprof object ids (addresses) to dense `u32` indices in ascending address order.
|
||||
//!
|
||||
//! Object ids are addresses and objects are at least 8 bytes apart, so one bit per 8-byte slot
|
||||
//! ranks ids at about a byte per object. Addresses are grouped in 1 MiB chunks of
|
||||
//! [`WORDS_PER_CHUNK`] words; only chunks holding an id get words, and a dense table maps a chunk
|
||||
//! to its first word. The rank of an id is the number of set bits before its word plus the set
|
||||
//! bits below it in the word. Ids that are not multiples of 8, or that span too wide an address
|
||||
//! range for the table, use a sorted array and binary search instead.
|
||||
|
||||
mod builder;
|
||||
|
||||
pub(crate) use builder::ChunkHasher;
|
||||
pub use builder::IdMapBuilder;
|
||||
|
||||
use crate::column::{Column, NONE};
|
||||
|
||||
const SLOT_SHIFT: u32 = 3;
|
||||
const CHUNK_SLOT_BITS: u32 = 17;
|
||||
const WORD_BITS: u64 = 64;
|
||||
const WORDS_PER_CHUNK: usize = (1 << CHUNK_SLOT_BITS) / WORD_BITS as usize;
|
||||
|
||||
// The bit table is matched in hot paths; boxing the big variant would only add
|
||||
// an indirection to save stack space `IdMap` never holds by value in bulk.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
enum Repr {
|
||||
Bits {
|
||||
/// Chunk number of the first table entry.
|
||||
min_chunk: u64,
|
||||
words: Column<u64>,
|
||||
/// Set bits before each word.
|
||||
wrank: Column<u32>,
|
||||
/// Per chunk from `min_chunk` up: index of its first word, or `NONE` when it has no id.
|
||||
table: Column<u32>,
|
||||
/// Per chunk that has words, in word order: its offset from `min_chunk`.
|
||||
chunk_offsets: Column<u32>,
|
||||
len: u32,
|
||||
},
|
||||
Sorted(Column<u64>),
|
||||
}
|
||||
|
||||
/// Ids ranked in ascending address order: the index of an object is its position in the order.
|
||||
pub struct IdMap {
|
||||
repr: Repr,
|
||||
}
|
||||
|
||||
impl IdMap {
|
||||
pub fn len(&self) -> u32 {
|
||||
match &self.repr {
|
||||
Repr::Bits { len, .. } => *len,
|
||||
Repr::Sorted(ids) => ids.len() as u32,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
|
||||
/// The index of `id`, or `None` when it is not in the map.
|
||||
pub fn rank(&self, id: u64) -> Option<u32> {
|
||||
match &self.repr {
|
||||
Repr::Sorted(ids) => ids.binary_search(&id).ok().map(|r| r as u32),
|
||||
Repr::Bits { min_chunk, words, wrank, table, .. } => {
|
||||
if id & 7 != 0 {
|
||||
return None;
|
||||
}
|
||||
let slot = id >> SLOT_SHIFT;
|
||||
let offset = (slot >> CHUNK_SLOT_BITS).checked_sub(*min_chunk)?;
|
||||
let first = *table.get(usize::try_from(offset).ok()?)?;
|
||||
if first == NONE {
|
||||
return None;
|
||||
}
|
||||
let w = first as usize + (slot >> 6) as usize % WORDS_PER_CHUNK;
|
||||
let bit = slot & 63;
|
||||
let word = words[w];
|
||||
if word >> bit & 1 == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(wrank[w] + (word & ((1u64 << bit) - 1)).count_ones())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The id with index `rank`. Panics when `rank >= len()`.
|
||||
pub fn id_at(&self, rank: u32) -> u64 {
|
||||
match &self.repr {
|
||||
Repr::Sorted(ids) => ids[rank as usize],
|
||||
Repr::Bits { words, wrank, len, .. } => {
|
||||
assert!(rank < *len, "rank {rank} out of range for {len} ids");
|
||||
let w = wrank.partition_point(|&before| before <= rank) - 1;
|
||||
let mut word = words[w];
|
||||
for _ in 0..rank - wrank[w] {
|
||||
word &= word - 1;
|
||||
}
|
||||
(self.word_first_slot(w) + u64::from(word.trailing_zeros())) << SLOT_SHIFT
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// All ids in ascending order.
|
||||
pub fn ids(&self) -> impl Iterator<Item = u64> + '_ {
|
||||
Ids { map: self, next: 0, slot_base: 0, rest: 0 }
|
||||
}
|
||||
|
||||
/// Bytes of all the columns, in RAM or on disk.
|
||||
pub fn column_bytes(&self) -> u64 {
|
||||
match &self.repr {
|
||||
Repr::Sorted(ids) => std::mem::size_of_val(&**ids) as u64,
|
||||
Repr::Bits { words, wrank, table, chunk_offsets, .. } => {
|
||||
(std::mem::size_of_val(&**words)
|
||||
+ std::mem::size_of_val(&**wrank)
|
||||
+ std::mem::size_of_val(&**table)
|
||||
+ std::mem::size_of_val(&**chunk_offsets)) as u64
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Slot number of bit 0 of word `w` (bitmap layout only).
|
||||
fn word_first_slot(&self, w: usize) -> u64 {
|
||||
let Repr::Bits { min_chunk, chunk_offsets, .. } = &self.repr else {
|
||||
unreachable!("sorted layout has no words")
|
||||
};
|
||||
let chunk = min_chunk + u64::from(chunk_offsets[w / WORDS_PER_CHUNK]);
|
||||
(chunk << CHUNK_SLOT_BITS) + (w % WORDS_PER_CHUNK) as u64 * WORD_BITS
|
||||
}
|
||||
}
|
||||
|
||||
struct Ids<'a> {
|
||||
map: &'a IdMap,
|
||||
/// Next word (bitmap layout) or next id (sorted layout).
|
||||
next: usize,
|
||||
slot_base: u64,
|
||||
/// Bits of the current word not yielded yet.
|
||||
rest: u64,
|
||||
}
|
||||
|
||||
impl Iterator for Ids<'_> {
|
||||
type Item = u64;
|
||||
|
||||
fn next(&mut self) -> Option<u64> {
|
||||
match &self.map.repr {
|
||||
Repr::Sorted(ids) => {
|
||||
let id = *ids.get(self.next)?;
|
||||
self.next += 1;
|
||||
Some(id)
|
||||
}
|
||||
Repr::Bits { words, .. } => loop {
|
||||
if self.rest != 0 {
|
||||
let bit = u64::from(self.rest.trailing_zeros());
|
||||
self.rest &= self.rest - 1;
|
||||
return Some((self.slot_base + bit) << SLOT_SHIFT);
|
||||
}
|
||||
self.rest = *words.get(self.next)?;
|
||||
self.slot_base = self.map.word_first_slot(self.next);
|
||||
self.next += 1;
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use super::*;
|
||||
use crate::column::Store;
|
||||
|
||||
/// Deterministic generator so failures replay; the crate has no `rand`.
|
||||
struct XorShift(u64);
|
||||
|
||||
impl XorShift {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 ^= self.0 << 13;
|
||||
self.0 ^= self.0 >> 7;
|
||||
self.0 ^= self.0 << 17;
|
||||
self.0
|
||||
}
|
||||
}
|
||||
|
||||
fn build(ids: &[u64], store: &Store) -> IdMap {
|
||||
let mut builder = IdMapBuilder::new();
|
||||
for &id in ids {
|
||||
builder.insert(id);
|
||||
}
|
||||
builder.build(store, &CancellationToken::new()).unwrap()
|
||||
}
|
||||
|
||||
fn assert_matches_model(map: &IdMap, model: &[u64]) {
|
||||
assert_eq!(map.len() as usize, model.len());
|
||||
assert_eq!(map.ids().collect::<Vec<_>>(), model);
|
||||
for (rank, &id) in model.iter().enumerate() {
|
||||
assert_eq!(map.rank(id), Some(rank as u32), "rank of {id:#x}");
|
||||
assert_eq!(map.id_at(rank as u32), id, "id at {rank}");
|
||||
for near in [id.wrapping_sub(8), id.wrapping_add(8), id.wrapping_add(1), id.wrapping_add(4)] {
|
||||
if model.binary_search(&near).is_err() {
|
||||
assert_eq!(map.rank(near), None, "{near:#x} was never inserted");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rank_follows_address_order() {
|
||||
let map = build(&[0x2000, 0x1000, 0x1010, 0x7_0000_0000], &Store::Ram);
|
||||
assert_eq!(map.len(), 4);
|
||||
assert_eq!(
|
||||
[0x1000, 0x1010, 0x2000, 0x7_0000_0000].map(|id| map.rank(id)),
|
||||
[Some(0), Some(1), Some(2), Some(3)]
|
||||
);
|
||||
assert_eq!(map.rank(0x1008), None);
|
||||
assert_eq!(map.rank(0), None);
|
||||
assert_eq!(map.id_at(2), 0x2000);
|
||||
assert_eq!(map.ids().collect::<Vec<_>>(), [0x1000, 0x1010, 0x2000, 0x7_0000_0000]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unaligned_ids_use_sorted_fallback() {
|
||||
let map = build(&[0x2000, 0x1004, 0x1000, 0x1010, 0x7_0000_0000], &Store::Ram);
|
||||
assert_eq!(
|
||||
[0x1000, 0x1004, 0x1010, 0x2000, 0x7_0000_0000].map(|id| map.rank(id)),
|
||||
[Some(0), Some(1), Some(2), Some(3), Some(4)]
|
||||
);
|
||||
assert_eq!(map.rank(0x1008), None);
|
||||
assert_eq!(map.id_at(1), 0x1004);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_wide_span_uses_fallback() {
|
||||
let map = build(&[0x1000 + (1 << 50), 0x1000], &Store::Ram);
|
||||
assert_eq!(map.rank(0x1000), Some(0));
|
||||
assert_eq!(map.rank(0x1000 + (1 << 50)), Some(1));
|
||||
assert_eq!(map.rank(0x1008), None);
|
||||
assert!(map.column_bytes() < 64, "a two-id map must not allocate a chunk table");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ten_million_ids_cost_about_a_byte_each() {
|
||||
let mut builder = IdMapBuilder::new();
|
||||
for i in 0..10_000_000u64 {
|
||||
builder.insert(0x7_0000_0000 + i * 32);
|
||||
}
|
||||
let map = builder.build(&Store::Ram, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(map.len(), 10_000_000);
|
||||
assert!(map.column_bytes() < 16_000_000, "{} bytes", map.column_bytes());
|
||||
assert_eq!(map.rank(0x7_0000_0000 + 9_999_999 * 32), Some(9_999_999));
|
||||
assert_eq!(map.id_at(5_000_000), 0x7_0000_0000 + 5_000_000 * 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_map_and_duplicates() {
|
||||
let map = build(&[], &Store::Ram);
|
||||
assert!(map.is_empty());
|
||||
assert_eq!(map.rank(0x1000), None);
|
||||
assert_eq!(map.ids().count(), 0);
|
||||
let map = build(&[0x1000, 0x1000, 0x1008, 0x1000], &Store::Ram);
|
||||
assert_eq!(map.len(), 2);
|
||||
assert_eq!(map.rank(0x1008), Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_boundaries_and_gaps_between_chunks() {
|
||||
let ids = [0, 0xF_FFF8, 0x10_0000, 0x10_0008, 0x50_0000, 0x1_0000_0000];
|
||||
let mut model = ids.to_vec();
|
||||
model.sort_unstable();
|
||||
let map = build(&ids, &Store::Ram);
|
||||
assert_matches_model(&map, &model);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_outside_the_populated_chunks_are_absent() {
|
||||
let ids = [0x30_0000, 0x30_0008, 0x70_0000];
|
||||
let map = build(&ids, &Store::Ram);
|
||||
assert_matches_model(&map, &ids);
|
||||
// Chunk 5 lies between the lowest and highest chunk but holds nothing.
|
||||
assert_eq!(map.rank(0x50_0000), None);
|
||||
// Chunk 1 is below the lowest populated chunk.
|
||||
assert_eq!(map.rank(0x10_0000), None);
|
||||
// Chunk 9 is past the last table entry.
|
||||
assert_eq!(map.rank(0x900_0000), None);
|
||||
assert_eq!(map.rank(u64::MAX - 7), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn highest_address_round_trips() {
|
||||
let map = build(&[0xFFFF_FFFF_FFFF_FFF8], &Store::Ram);
|
||||
assert_eq!(map.rank(0xFFFF_FFFF_FFFF_FFF8), Some(0));
|
||||
assert_eq!(map.id_at(0), 0xFFFF_FFFF_FFFF_FFF8);
|
||||
let map = build(&[0, 0xFFFF_FFFF_FFFF_FFF8], &Store::Ram);
|
||||
assert_matches_model(&map, &[0, 0xFFFF_FFFF_FFFF_FFF8]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_ids_match_a_sorted_model_in_ram_and_on_disk() {
|
||||
for seed in 1..=8u64 {
|
||||
let mut rng = XorShift(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1);
|
||||
let aligned = seed % 4 != 0;
|
||||
let wide = seed % 3 == 0;
|
||||
let ids: Vec<u64> = (0..3000)
|
||||
.map(|_| {
|
||||
let r = rng.next();
|
||||
let base = if wide { (r >> 20) << 12 } else { 0x6_0000_0000 + (r % (6 << 20)) };
|
||||
if aligned { base & !7 } else { base }
|
||||
})
|
||||
.collect();
|
||||
let mut model = ids.clone();
|
||||
model.sort_unstable();
|
||||
model.dedup();
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
for store in [Store::Ram, Store::Dir(dir.path().to_path_buf())] {
|
||||
assert_matches_model(&build(&ids, &store), &model);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dir_store_keeps_the_index_columns_under_stable_names() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let map = build(&[0x1000, 0x2000], &Store::Dir(dir.path().to_path_buf()));
|
||||
let mut names: Vec<_> =
|
||||
std::fs::read_dir(dir.path()).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect();
|
||||
names.sort();
|
||||
assert_eq!(names, ["ids.bits.u64", "ids.chunks.u32", "ids.table.u32", "ids.wrank.u32"]);
|
||||
assert_eq!(map.rank(0x2000), Some(1));
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
build(&[0x1004], &Store::Dir(dir.path().to_path_buf()));
|
||||
assert!(dir.path().join("ids.sorted.u64").exists());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,409 @@
|
||||
//! Collects object ids in any order and lays them out as an [`IdMap`].
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BinaryHeap, HashMap};
|
||||
use std::hash::{BuildHasherDefault, Hasher};
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use super::{CHUNK_SLOT_BITS, IdMap, Repr, SLOT_SHIFT, WORD_BITS, WORDS_PER_CHUNK};
|
||||
use crate::column::{NONE, Store};
|
||||
use crate::error::HeapError;
|
||||
use crate::files::{IDS_BITS_FILE, IDS_CHUNKS_FILE, IDS_SORTED_FILE, IDS_TABLE_FILE, IDS_WRANK_FILE};
|
||||
|
||||
/// Widest chunk range the dense table covers (256 MiB of table at most).
|
||||
const MAX_CHUNK_SPAN: u64 = 1 << 26;
|
||||
/// Chunks whose first word index still fits in the table's `u32`.
|
||||
const MAX_CHUNKS: usize = (NONE as usize - 1) / WORDS_PER_CHUNK;
|
||||
/// Largest object count: `NONE` and the one after it stay free as sentinels for the columns above.
|
||||
const MAX_IDS: u64 = NONE as u64 - 2;
|
||||
|
||||
const CHUNK_BYTES: u64 = (WORDS_PER_CHUNK * size_of::<u64>()) as u64;
|
||||
/// The bitmaps may take this much before any id is inserted, then this many bytes per distinct id.
|
||||
/// A heap that fits in 4 GiB of address space never reaches the base allowance whatever the
|
||||
/// insertion order; sparse input (one id per chunk) is moved to plain `u64` ids, 8 bytes each,
|
||||
/// after at most `BITMAP_BASE_BYTES + BITMAP_BYTES_PER_ID * ids` of bitmaps, so builder memory is
|
||||
/// bounded by the number of ids rather than by the address range they spread over.
|
||||
const BITMAP_BASE_BYTES: u64 = 64 << 20;
|
||||
const BITMAP_BYTES_PER_ID: u64 = 32;
|
||||
|
||||
/// Longest stretch of work between two looks at the cancellation token.
|
||||
const CHECK_EVERY: usize = 1 << 20;
|
||||
|
||||
/// Chunk numbers are already well spread by a multiplicative hash; SipHash would dominate the
|
||||
/// per-insert cost on a 100M-object heap. The product keeps the low zero bits of its input, so
|
||||
/// the high half is folded down for the bucket index. The graph builder keys class and string
|
||||
/// ids (addresses) with it for the same reason.
|
||||
#[derive(Default)]
|
||||
pub(crate) struct ChunkHasher(u64);
|
||||
|
||||
impl Hasher for ChunkHasher {
|
||||
fn finish(&self) -> u64 {
|
||||
self.0 ^ (self.0 >> 32)
|
||||
}
|
||||
|
||||
fn write(&mut self, bytes: &[u8]) {
|
||||
for &b in bytes {
|
||||
self.write_u64(self.0.rotate_left(8) ^ u64::from(b));
|
||||
}
|
||||
}
|
||||
|
||||
fn write_u64(&mut self, n: u64) {
|
||||
self.0 = n.wrapping_mul(0x9E37_79B9_7F4A_7C15);
|
||||
}
|
||||
}
|
||||
|
||||
struct BitChunks {
|
||||
/// Chunk number → position in `words`.
|
||||
index: HashMap<u64, usize, BuildHasherDefault<ChunkHasher>>,
|
||||
words: Vec<Box<[u64]>>,
|
||||
min: u64,
|
||||
max: u64,
|
||||
/// The chunk of the previous insert; ids usually arrive in runs.
|
||||
last: Option<(u64, usize)>,
|
||||
distinct: u64,
|
||||
}
|
||||
|
||||
impl BitChunks {
|
||||
fn new() -> Self {
|
||||
Self { index: HashMap::default(), words: Vec::new(), min: u64::MAX, max: 0, last: None, distinct: 0 }
|
||||
}
|
||||
|
||||
/// `false` when the id's chunk does not fit the dense table or the bitmap budget; the id is
|
||||
/// then not recorded and the caller moves to plain ids.
|
||||
fn insert(&mut self, id: u64) -> bool {
|
||||
let slot = id >> SLOT_SHIFT;
|
||||
let chunk = slot >> CHUNK_SLOT_BITS;
|
||||
let at = match self.last {
|
||||
Some((c, at)) if c == chunk => at,
|
||||
_ => match self.index.get(&chunk) {
|
||||
Some(&at) => at,
|
||||
None => {
|
||||
let (min, max) = (self.min.min(chunk), self.max.max(chunk));
|
||||
let bitmap_bytes = (self.words.len() as u64 + 1) * CHUNK_BYTES;
|
||||
if self.words.len() >= MAX_CHUNKS
|
||||
|| max - min >= MAX_CHUNK_SPAN
|
||||
|| bitmap_bytes > BITMAP_BASE_BYTES + BITMAP_BYTES_PER_ID * self.distinct
|
||||
{
|
||||
return false;
|
||||
}
|
||||
(self.min, self.max) = (min, max);
|
||||
self.words.push(vec![0u64; WORDS_PER_CHUNK].into_boxed_slice());
|
||||
self.index.insert(chunk, self.words.len() - 1);
|
||||
self.words.len() - 1
|
||||
}
|
||||
},
|
||||
};
|
||||
self.last = Some((chunk, at));
|
||||
let word = &mut self.words[at][(slot >> 6) as usize % WORDS_PER_CHUNK];
|
||||
let bit = 1u64 << (slot & 63);
|
||||
if *word & bit == 0 {
|
||||
*word |= bit;
|
||||
self.distinct += 1;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// `(chunk number, position in words)` in ascending address order.
|
||||
fn sorted_chunks(&self) -> Vec<(u64, usize)> {
|
||||
let mut chunks: Vec<_> = self.index.iter().map(|(&chunk, &at)| (chunk, at)).collect();
|
||||
chunks.sort_unstable();
|
||||
chunks
|
||||
}
|
||||
|
||||
/// The recorded ids, ascending.
|
||||
fn ids(&self) -> BitIds<'_> {
|
||||
BitIds { chunks: self.sorted_chunks(), words: &self.words, chunk: 0, word: 0, base: 0, rest: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
struct BitIds<'a> {
|
||||
chunks: Vec<(u64, usize)>,
|
||||
words: &'a [Box<[u64]>],
|
||||
chunk: usize,
|
||||
word: usize,
|
||||
/// Slot number of bit 0 of the word `rest` came from.
|
||||
base: u64,
|
||||
rest: u64,
|
||||
}
|
||||
|
||||
impl Iterator for BitIds<'_> {
|
||||
type Item = u64;
|
||||
|
||||
fn next(&mut self) -> Option<u64> {
|
||||
loop {
|
||||
if self.rest != 0 {
|
||||
let bit = u64::from(self.rest.trailing_zeros());
|
||||
self.rest &= self.rest - 1;
|
||||
return Some((self.base + bit) << SLOT_SHIFT);
|
||||
}
|
||||
let &(chunk, at) = self.chunks.get(self.chunk)?;
|
||||
self.rest = self.words[at][self.word];
|
||||
self.base = (chunk << CHUNK_SLOT_BITS) + self.word as u64 * WORD_BITS;
|
||||
self.word += 1;
|
||||
if self.word == WORDS_PER_CHUNK {
|
||||
self.word = 0;
|
||||
self.chunk += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct IdMapBuilder {
|
||||
bits: BitChunks,
|
||||
/// Set once an id cannot go into `bits`; every later id goes here, and `bits` keeps what it
|
||||
/// had. Unsorted, with duplicates.
|
||||
plain: Option<Vec<u64>>,
|
||||
max_ids: u64,
|
||||
}
|
||||
|
||||
impl Default for IdMapBuilder {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl IdMapBuilder {
|
||||
pub fn new() -> Self {
|
||||
Self::with_limit(MAX_IDS)
|
||||
}
|
||||
|
||||
/// `max_ids` is a parameter so tests can reach the too-large path without billions of inserts.
|
||||
fn with_limit(max_ids: u64) -> Self {
|
||||
Self { bits: BitChunks::new(), plain: None, max_ids }
|
||||
}
|
||||
|
||||
/// Inserting an id twice is the same as once.
|
||||
pub fn insert(&mut self, id: u64) {
|
||||
match &mut self.plain {
|
||||
Some(plain) => plain.push(id),
|
||||
None => {
|
||||
if id & 7 != 0 || !self.bits.insert(id) {
|
||||
self.plain = Some(vec![id]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fails with [`HeapError::TooLarge`] when there are more than `u32::MAX - 2` distinct ids,
|
||||
/// and with [`HeapError::Cancelled`] when `cancel` fires (checked at least every `1 << 20`
|
||||
/// steps).
|
||||
pub fn build(self, store: &Store, cancel: &CancellationToken) -> Result<IdMap, HeapError> {
|
||||
match self.plain {
|
||||
None if self.bits.words.is_empty() => sorted_map(store, &[]),
|
||||
None => bit_map(&self.bits, store, self.max_ids, cancel),
|
||||
Some(plain) => merged_map(&self.bits, plain, store, self.max_ids, cancel),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn allocated_bytes(&self) -> u64 {
|
||||
self.bits.words.len() as u64 * CHUNK_BYTES
|
||||
+ self.plain.as_ref().map_or(0, |p| (p.capacity() * size_of::<u64>()) as u64)
|
||||
}
|
||||
}
|
||||
|
||||
fn check(cancel: &CancellationToken) -> Result<(), HeapError> {
|
||||
if cancel.is_cancelled() { Err(HeapError::Cancelled) } else { Ok(()) }
|
||||
}
|
||||
|
||||
fn sorted_map(store: &Store, ids: &[u64]) -> Result<IdMap, HeapError> {
|
||||
let mut column = store.column(IDS_SORTED_FILE, ids.len(), 0u64)?;
|
||||
column.copy_from_slice(ids);
|
||||
Ok(IdMap { repr: Repr::Sorted(column) })
|
||||
}
|
||||
|
||||
fn bit_map(bits: &BitChunks, store: &Store, max_ids: u64, cancel: &CancellationToken) -> Result<IdMap, HeapError> {
|
||||
if bits.distinct > max_ids {
|
||||
return Err(HeapError::TooLarge);
|
||||
}
|
||||
let chunks = bits.sorted_chunks();
|
||||
let span = (bits.max - bits.min + 1) as usize;
|
||||
let mut words = store.column(IDS_BITS_FILE, chunks.len() * WORDS_PER_CHUNK, 0u64)?;
|
||||
let mut wrank = store.column(IDS_WRANK_FILE, chunks.len() * WORDS_PER_CHUNK, 0u32)?;
|
||||
let mut table = store.column(IDS_TABLE_FILE, span, NONE)?;
|
||||
let mut chunk_offsets = store.column(IDS_CHUNKS_FILE, chunks.len(), 0u32)?;
|
||||
let mut before = 0u32;
|
||||
for (ordinal, &(chunk, at)) in chunks.iter().enumerate() {
|
||||
check(cancel)?;
|
||||
let first = ordinal * WORDS_PER_CHUNK;
|
||||
let offset = (chunk - bits.min) as usize;
|
||||
table[offset] = first as u32;
|
||||
chunk_offsets[ordinal] = offset as u32;
|
||||
for (w, &word) in bits.words[at].iter().enumerate() {
|
||||
words[first + w] = word;
|
||||
wrank[first + w] = before;
|
||||
before += word.count_ones();
|
||||
}
|
||||
}
|
||||
Ok(IdMap { repr: Repr::Bits { min_chunk: bits.min, words, wrank, table, chunk_offsets, len: before } })
|
||||
}
|
||||
|
||||
enum Source<'a> {
|
||||
Bits(BitIds<'a>),
|
||||
Run(std::iter::Copied<std::slice::Iter<'a, u64>>),
|
||||
}
|
||||
|
||||
impl Iterator for Source<'_> {
|
||||
type Item = u64;
|
||||
|
||||
fn next(&mut self) -> Option<u64> {
|
||||
match self {
|
||||
Source::Bits(it) => it.next(),
|
||||
Source::Run(it) => it.next(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sorts `plain` in runs and merges the runs with the ids already in `bits` (itself a sorted
|
||||
/// source), dropping duplicates. A single `sort_unstable` of a very large vector could not be
|
||||
/// interrupted.
|
||||
fn merged_map(
|
||||
bits: &BitChunks,
|
||||
mut plain: Vec<u64>,
|
||||
store: &Store,
|
||||
max_ids: u64,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<IdMap, HeapError> {
|
||||
for run in plain.chunks_mut(CHECK_EVERY) {
|
||||
check(cancel)?;
|
||||
run.sort_unstable();
|
||||
}
|
||||
let mut sources = vec![Source::Bits(bits.ids())];
|
||||
sources.extend(plain.chunks(CHECK_EVERY).map(|run| Source::Run(run.iter().copied())));
|
||||
let mut heap = BinaryHeap::with_capacity(sources.len());
|
||||
for (i, source) in sources.iter_mut().enumerate() {
|
||||
if let Some(id) = source.next() {
|
||||
heap.push(Reverse((id, i)));
|
||||
}
|
||||
}
|
||||
|
||||
let upper_bound = bits.distinct as usize + plain.len();
|
||||
let mut merged = store.scratch::<u64>(upper_bound, 0)?;
|
||||
let mut count = 0usize;
|
||||
let mut steps = 0usize;
|
||||
while let Some(Reverse((id, i))) = heap.pop() {
|
||||
steps += 1;
|
||||
if steps.is_multiple_of(CHECK_EVERY) {
|
||||
check(cancel)?;
|
||||
}
|
||||
if let Some(next) = sources[i].next() {
|
||||
heap.push(Reverse((next, i)));
|
||||
}
|
||||
if count > 0 && merged[count - 1] == id {
|
||||
continue;
|
||||
}
|
||||
if count as u64 >= max_ids {
|
||||
return Err(HeapError::TooLarge);
|
||||
}
|
||||
merged[count] = id;
|
||||
count += 1;
|
||||
}
|
||||
|
||||
let mut column = store.column(IDS_SORTED_FILE, count, 0u64)?;
|
||||
for (to, from) in column.chunks_mut(CHECK_EVERY).zip(merged[..count].chunks(CHECK_EVERY)) {
|
||||
check(cancel)?;
|
||||
to.copy_from_slice(from);
|
||||
}
|
||||
Ok(IdMap { repr: Repr::Sorted(column) })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn insert_all(builder: &mut IdMapBuilder, ids: &[u64]) {
|
||||
for &id in ids {
|
||||
builder.insert(id);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn too_many_ids_is_too_large() {
|
||||
// Dense layout, then the merged layout (one unaligned id); the duplicate does not count.
|
||||
for unaligned in [false, true] {
|
||||
let filled = |limit| {
|
||||
let mut builder = IdMapBuilder::with_limit(limit);
|
||||
insert_all(&mut builder, &[0x1000, 0x1008, 0x1010, 0x1010]);
|
||||
if unaligned {
|
||||
builder.insert(0x1_0004);
|
||||
}
|
||||
builder
|
||||
};
|
||||
let distinct = if unaligned { 4 } else { 3 };
|
||||
let map = filled(distinct).build(&Store::Ram, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(u64::from(map.len()), distinct);
|
||||
let err = filled(distinct - 1).build(&Store::Ram, &CancellationToken::new()).err().unwrap();
|
||||
assert!(matches!(err, HeapError::TooLarge), "{err}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_build_returns_cancelled() {
|
||||
let cancelled = CancellationToken::new();
|
||||
cancelled.cancel();
|
||||
let mut dense = IdMapBuilder::new();
|
||||
insert_all(&mut dense, &[0x1000, 0x1008, 0x20_0000]);
|
||||
assert!(matches!(dense.build(&Store::Ram, &cancelled), Err(HeapError::Cancelled)));
|
||||
|
||||
let mut merged = IdMapBuilder::new();
|
||||
insert_all(&mut merged, &[0x1000, 0x1004, 0x20_0000]);
|
||||
assert!(matches!(merged.build(&Store::Ram, &cancelled), Err(HeapError::Cancelled)));
|
||||
|
||||
let mut ok = IdMapBuilder::new();
|
||||
insert_all(&mut ok, &[0x1000, 0x1004]);
|
||||
assert_eq!(ok.build(&Store::Ram, &CancellationToken::new()).unwrap().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sparse_ids_do_not_amplify_builder_memory() {
|
||||
const IDS: u64 = 2_000_000;
|
||||
let mut builder = IdMapBuilder::new();
|
||||
let mut peak = 0;
|
||||
for i in 0..IDS {
|
||||
builder.insert(0x1_0000_0000 + (i << 20));
|
||||
if i % 1024 == 0 {
|
||||
peak = peak.max(builder.allocated_bytes());
|
||||
}
|
||||
}
|
||||
peak = peak.max(builder.allocated_bytes());
|
||||
// Bitmaps up to the budget, plus the plain ids' vector, which may have doubled its capacity.
|
||||
let budget = BITMAP_BASE_BYTES + (BITMAP_BYTES_PER_ID + 2 * size_of::<u64>() as u64) * IDS;
|
||||
assert!(peak <= budget, "peak {peak} bytes, budget {budget}");
|
||||
// One bitmap per id would have taken 16 KiB each.
|
||||
assert!(peak < IDS * CHUNK_BYTES / 100, "peak {peak} bytes");
|
||||
assert!(builder.plain.is_some(), "sparse input must have left the bitmaps");
|
||||
|
||||
let map = builder.build(&Store::Ram, &CancellationToken::new()).unwrap();
|
||||
assert_eq!(map.len() as u64, IDS);
|
||||
assert_eq!(map.rank(0x1_0000_0000 + (1_234_567 << 20)), Some(1_234_567));
|
||||
assert_eq!(map.rank(0x1_0000_0000 + (1_234_567 << 20) + 8), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dense_input_in_any_order_stays_in_bitmaps() {
|
||||
// 4 GiB of address space is the base allowance; random order must not trip the budget.
|
||||
let mut state = 0x2545_F491_4F6C_DD1Du64;
|
||||
let mut builder = IdMapBuilder::new();
|
||||
for _ in 0..300_000 {
|
||||
state ^= state << 13;
|
||||
state ^= state >> 7;
|
||||
state ^= state << 17;
|
||||
builder.insert((0x7_0000_0000 + (state % (4 << 30))) & !7);
|
||||
}
|
||||
assert!(builder.plain.is_none());
|
||||
assert!(builder.allocated_bytes() <= BITMAP_BASE_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_hasher_spreads_chunks_that_share_low_zero_bits() {
|
||||
let buckets: std::collections::HashSet<u64> = (0..1024u64)
|
||||
.map(|i| {
|
||||
let mut hasher = ChunkHasher::default();
|
||||
hasher.write_u64(i << 12);
|
||||
hasher.finish() & 1023
|
||||
})
|
||||
.collect();
|
||||
assert!(buckets.len() > 500, "{} buckets for 1024 keys", buckets.len());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
//! The result of analyzing a heap dump: per-object columns, the class table and the summary.
|
||||
|
||||
mod fields;
|
||||
mod path;
|
||||
mod queries;
|
||||
mod rows;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use memmap2::Mmap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::column::Column;
|
||||
use crate::error::HeapError;
|
||||
use crate::files::SUMMARY_FILE;
|
||||
use crate::graph::ClassInfo;
|
||||
use crate::hprof::HeapRootKind;
|
||||
use crate::ids::IdMap;
|
||||
use crate::shallow::SizeSource;
|
||||
|
||||
pub(crate) use self::fields::{id_value, instance_fields};
|
||||
pub use self::rows::{
|
||||
HeapClassRow, HeapEdge, HeapObjectPage, HeapObjectRef, HeapObjectRow, HeapPathStep, HeapRootPath, MAX_PAGE,
|
||||
};
|
||||
|
||||
/// Counts over instances and arrays; class objects (class dumps) are left out.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapSummary {
|
||||
/// `reachable_objects + unreachable_objects`.
|
||||
pub objects: u64,
|
||||
pub reachable_objects: u64,
|
||||
pub reachable_bytes: u64,
|
||||
pub unreachable_objects: u64,
|
||||
pub unreachable_bytes: u64,
|
||||
/// Class dumps in the dump.
|
||||
pub classes: u32,
|
||||
pub weak_edges_skipped: u64,
|
||||
pub size_source: SizeSource,
|
||||
pub analysis_ms: u32,
|
||||
}
|
||||
|
||||
/// One row of the class table, for one type slot.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ClassStat {
|
||||
/// [`crate::display_name`] of the class (`long[]` for a primitive array slot).
|
||||
pub name: String,
|
||||
/// Reachable instances.
|
||||
pub count: u64,
|
||||
/// Shallow bytes of the reachable instances.
|
||||
pub shallow: u64,
|
||||
/// Bytes the reachable instances retain together: the sum of the retained sizes of those
|
||||
/// with no instance of the same type above them in the dominator tree.
|
||||
pub retained: u64,
|
||||
pub unreachable_count: u64,
|
||||
}
|
||||
|
||||
/// Columns are indexed by node: objects `0..root`, then the super-root `root`. Columns of the
|
||||
/// objects only (`types`, `offsets`) have no entry for the root.
|
||||
pub struct HeapIndex {
|
||||
/// The dump, for [`crate::record_at`] at `offsets[v]`.
|
||||
pub hprof: Mmap,
|
||||
pub ids: IdMap,
|
||||
/// Type slot of each object, as in [`crate::HeapGraph::types`].
|
||||
pub types: Column<u32>,
|
||||
pub offsets: Column<u64>,
|
||||
/// Shallow size in units of 8 bytes, rounded up.
|
||||
pub shallow8: Column<u32>,
|
||||
/// Retained bytes; 0 for unreachable objects, every reachable byte for the root.
|
||||
pub retained: Column<u64>,
|
||||
/// Immediate dominator; [`crate::NONE`] for unreachable objects, `root` for the root.
|
||||
pub idom: Column<u32>,
|
||||
/// Children of node `v` in the dominator tree are `kids[kids_start[v]..kids_start[v + 1]]`,
|
||||
/// by retained size descending, then by node.
|
||||
pub kids_start: Column<u32>,
|
||||
pub kids: Column<u32>,
|
||||
/// Parent on a shortest path from the root (BFS over strong edges); [`crate::NONE`] for
|
||||
/// unreachable objects, `root` for the root.
|
||||
pub gc_parent: Column<u32>,
|
||||
/// By type slot: class slots, then the primitive array slots.
|
||||
pub classes: Vec<ClassStat>,
|
||||
/// By class slot, as in [`crate::HeapGraph::classes`].
|
||||
pub class_info: Vec<ClassInfo>,
|
||||
/// Type slot of class objects.
|
||||
pub class_slot: u32,
|
||||
/// Object → kind and thread serial of its first root record.
|
||||
pub roots: HashMap<u32, (HeapRootKind, Option<u32>)>,
|
||||
/// Thread serial → thread object.
|
||||
pub thread_objects: HashMap<u32, u32>,
|
||||
pub id_size: u8,
|
||||
pub summary: HeapSummary,
|
||||
}
|
||||
|
||||
impl HeapIndex {
|
||||
/// The super-root, whose children in both trees are the rooted objects.
|
||||
pub fn root(&self) -> u32 {
|
||||
self.ids.len()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn write_summary(dir: &Path, summary: &HeapSummary) -> Result<(), HeapError> {
|
||||
let json = serde_json::to_vec_pretty(summary).map_err(std::io::Error::from)?;
|
||||
std::fs::write(dir.join(SUMMARY_FILE), json)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,41 @@
|
||||
//! Named field values of an instance record.
|
||||
|
||||
use super::HeapIndex;
|
||||
use crate::hprof::BasicType;
|
||||
|
||||
pub(crate) struct FieldValue<'a> {
|
||||
pub(crate) name: &'a str,
|
||||
pub(crate) ty: BasicType,
|
||||
/// Big-endian, as in the dump.
|
||||
pub(crate) bytes: &'a [u8],
|
||||
/// `java.lang.ref.Reference.referent`, which the graph leaves out.
|
||||
pub(crate) weak: bool,
|
||||
}
|
||||
|
||||
/// The fields of an instance of class slot `class` whose record holds `data`: the class's own,
|
||||
/// then each superclass's, as HotSpot lays them out. Stops where `data` ends.
|
||||
pub(crate) fn instance_fields<'a>(index: &'a HeapIndex, class: u32, data: &'a [u8]) -> Vec<FieldValue<'a>> {
|
||||
let mut out = Vec::new();
|
||||
let mut base = 0usize;
|
||||
let mut cursor = Some(class);
|
||||
// Super chains end (their cycles are cut), so this is only a bound on corrupt input.
|
||||
for _ in 0..index.class_info.len() {
|
||||
let Some(info) = cursor.and_then(|c| index.class_info.get(c as usize)) else { break };
|
||||
for (k, (name, ty)) in info.fields.iter().enumerate() {
|
||||
let size = ty.hprof_size(index.id_size);
|
||||
let Some(bytes) = base.checked_add(size).and_then(|end| data.get(base..end)) else { return out };
|
||||
out.push(FieldValue { name, ty: *ty, bytes, weak: info.weak_field == Some(k) });
|
||||
base += size;
|
||||
}
|
||||
cursor = info.super_index;
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// A 4- or 8-byte big-endian id.
|
||||
pub(crate) fn id_value(bytes: &[u8]) -> Option<u64> {
|
||||
match *bytes {
|
||||
[a, b, c, d] => Some(u64::from(u32::from_be_bytes([a, b, c, d]))),
|
||||
_ => Some(u64::from_be_bytes(bytes.try_into().ok()?)),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
//! Paths to GC roots, along the BFS tree of `gc_parent`.
|
||||
|
||||
use super::{HeapEdge, HeapIndex, HeapObjectRef, HeapPathStep, HeapRootPath, id_value, instance_fields};
|
||||
use crate::column::NONE;
|
||||
use crate::hprof::{BasicType, HeapRecord, HeapRootKind};
|
||||
use crate::strings::{char_array_text, string_preview};
|
||||
|
||||
const THREAD_NAME_CHARS: usize = 100;
|
||||
|
||||
/// Steps a long path keeps at each end: one deep in a long linked list would otherwise send
|
||||
/// every node over IPC.
|
||||
const PATH_END_STEPS: u32 = 500;
|
||||
|
||||
impl HeapIndex {
|
||||
/// From `object` up to its GC root, each step with how the next one holds it; past 1000
|
||||
/// steps, only the 500 at each end. `None` for an unreachable object or a ref that names no
|
||||
/// object.
|
||||
pub fn path_to_root(&self, object: HeapObjectRef) -> Option<HeapRootPath> {
|
||||
let root = self.root();
|
||||
let mut v = object.0;
|
||||
if v >= root || self.gc_parent[v as usize] == NONE {
|
||||
return None;
|
||||
}
|
||||
let len = self.path_len(v)?;
|
||||
let end = PATH_END_STEPS;
|
||||
let cut = len > 2 * end;
|
||||
let mut steps = Vec::with_capacity(len.min(2 * end) as usize);
|
||||
for i in 0..len {
|
||||
let parent = self.gc_parent[v as usize];
|
||||
if !cut || i < end || i >= len - end {
|
||||
let via = if parent == root { self.root_edge(v) } else { self.edge(parent, v) };
|
||||
steps.push(HeapPathStep { object: self.object_row(HeapObjectRef(v))?, via });
|
||||
}
|
||||
v = parent;
|
||||
}
|
||||
let omitted = if cut { len - 2 * end } else { 0 };
|
||||
Some(HeapRootPath { steps, omitted, omitted_after: if cut { end } else { 0 } })
|
||||
}
|
||||
|
||||
/// Steps from `v` to the root; `None` when `gc_parent` leads nowhere.
|
||||
fn path_len(&self, mut v: u32) -> Option<u32> {
|
||||
let root = self.root();
|
||||
// A BFS tree has no cycles; the bound only stops a corrupt column from looping.
|
||||
for len in 1..=root {
|
||||
let parent = self.gc_parent[v as usize];
|
||||
if parent == root {
|
||||
return Some(len);
|
||||
}
|
||||
if parent > root {
|
||||
return None;
|
||||
}
|
||||
v = parent;
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn root_edge(&self, v: u32) -> HeapEdge {
|
||||
let (kind, serial) = self.roots.get(&v).copied().unwrap_or((HeapRootKind::Unknown, None));
|
||||
let thread = serial.and_then(|s| self.thread_objects.get(&s)).and_then(|&t| self.thread_name(t));
|
||||
HeapEdge::Root { root: kind, thread }
|
||||
}
|
||||
|
||||
/// `Thread.name`: a String, or a `char[]` before JDK 9.
|
||||
fn thread_name(&self, thread: u32) -> Option<String> {
|
||||
let Some(HeapRecord::Instance { data, .. }) = self.record(thread) else { return None };
|
||||
let fields = instance_fields(self, *self.types.get(thread as usize)?, data);
|
||||
let name = fields.iter().find(|f| f.name == "name" && f.ty == BasicType::Object)?;
|
||||
let target = self.ids.rank(id_value(name.bytes)?)?;
|
||||
string_preview(self, target, THREAD_NAME_CHARS).or_else(|| char_array_text(self, target, THREAD_NAME_CHARS))
|
||||
}
|
||||
|
||||
/// How `parent` refers to `child`, checked in the order the graph's edges were made. A strong
|
||||
/// edge from `parent` always exists; the empty field name only covers a dump that changed.
|
||||
fn edge(&self, parent: u32, child: u32) -> HeapEdge {
|
||||
self.find_edge(parent, self.ids.id_at(child)).unwrap_or(HeapEdge::Field { name: String::new() })
|
||||
}
|
||||
|
||||
fn find_edge(&self, parent: u32, child_id: u64) -> Option<HeapEdge> {
|
||||
match self.record(parent)? {
|
||||
HeapRecord::Instance { data, .. } => {
|
||||
let class = self.types[parent as usize];
|
||||
let field = instance_fields(self, class, data)
|
||||
.into_iter()
|
||||
.find(|f| !f.weak && f.ty == BasicType::Object && id_value(f.bytes) == Some(child_id));
|
||||
match field {
|
||||
Some(f) => Some(HeapEdge::Field { name: f.name.to_owned() }),
|
||||
None => (self.class_info.get(class as usize)?.id == child_id).then_some(HeapEdge::ClassOf),
|
||||
}
|
||||
}
|
||||
HeapRecord::ObjectArray { data, .. } => {
|
||||
let index =
|
||||
data.chunks_exact(usize::from(self.id_size)).position(|item| id_value(item) == Some(child_id))?;
|
||||
Some(HeapEdge::Element { index: u32::try_from(index).ok()? })
|
||||
}
|
||||
HeapRecord::Class(dump) if dump.super_id == child_id => Some(HeapEdge::SuperClass),
|
||||
HeapRecord::Class(dump) if dump.loader_id == child_id => Some(HeapEdge::ClassLoader),
|
||||
HeapRecord::Class(_) => {
|
||||
let info = self.class_info.get(self.class_of_object(parent)?)?;
|
||||
let (name, _) = info.statics.iter().find(|(_, object)| *object == child_id)?;
|
||||
Some(HeapEdge::Static { name: name.clone() })
|
||||
}
|
||||
HeapRecord::Root { .. } | HeapRecord::PrimitiveArray { .. } => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
//! Class table rows, pages of the dominator tree and single object rows.
|
||||
|
||||
use std::cmp::Reverse;
|
||||
use std::collections::{BinaryHeap, HashMap};
|
||||
|
||||
use super::{HeapClassRow, HeapIndex, HeapObjectPage, HeapObjectRef, HeapObjectRow, HeapSummary, MAX_PAGE};
|
||||
use crate::column::NONE;
|
||||
use crate::hprof::HeapRecord;
|
||||
use crate::strings::string_preview;
|
||||
|
||||
const PREVIEW_CHARS: usize = 100;
|
||||
|
||||
impl HeapIndex {
|
||||
pub fn summary(&self) -> &HeapSummary {
|
||||
&self.summary
|
||||
}
|
||||
|
||||
/// Types that have objects, by retained size descending, then by name.
|
||||
pub fn class_rows(&self) -> Vec<HeapClassRow> {
|
||||
let class_slots = &self.classes[..self.class_info.len().min(self.classes.len())];
|
||||
let mut names: HashMap<&str, u32> = HashMap::with_capacity(class_slots.len());
|
||||
for class in class_slots {
|
||||
*names.entry(class.name.as_str()).or_default() += 1;
|
||||
}
|
||||
let mut rows: Vec<HeapClassRow> = self
|
||||
.classes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, c)| c.count > 0 || c.unreachable_count > 0)
|
||||
.map(|(slot, c)| HeapClassRow {
|
||||
class_index: slot as u32,
|
||||
name: c.name.clone(),
|
||||
count: c.count,
|
||||
shallow: c.shallow,
|
||||
retained: c.retained,
|
||||
unreachable_count: c.unreachable_count,
|
||||
// Primitive array slots have no class dump and so no second loader.
|
||||
ambiguous: slot < class_slots.len() && names[c.name.as_str()] > 1,
|
||||
})
|
||||
.collect();
|
||||
rows.sort_by(|a, b| b.retained.cmp(&a.retained).then_with(|| a.name.cmp(&b.name)));
|
||||
rows
|
||||
}
|
||||
|
||||
/// A page of objects by retained size descending, then by index:
|
||||
///
|
||||
/// - no `parent`, no `class_index`: the children of the super-root, class objects left out;
|
||||
/// - `parent`: its children in the dominator tree, of type `class_index` if given;
|
||||
/// - `class_index` only: every reachable object of that type.
|
||||
///
|
||||
/// `limit` is clamped to [`MAX_PAGE`]. A `parent` or `class_index` that names nothing gives an
|
||||
/// empty page.
|
||||
pub fn dominated(
|
||||
&self,
|
||||
parent: Option<HeapObjectRef>,
|
||||
class_index: Option<u32>,
|
||||
offset: u32,
|
||||
limit: u32,
|
||||
) -> HeapObjectPage {
|
||||
let limit = limit.min(MAX_PAGE);
|
||||
let root = self.root();
|
||||
if class_index.is_some_and(|c| c as usize >= self.classes.len()) {
|
||||
return HeapObjectPage { rows: Vec::new(), total: 0 };
|
||||
}
|
||||
match (parent, class_index) {
|
||||
(Some(HeapObjectRef(p)), _) if p >= root => HeapObjectPage { rows: Vec::new(), total: 0 },
|
||||
(Some(HeapObjectRef(p)), class) => self.page(
|
||||
self.children(p).filter(|&v| class.is_none_or(|c| self.types[v as usize] == c)),
|
||||
offset,
|
||||
limit,
|
||||
),
|
||||
(None, None) => {
|
||||
self.page(self.children(root).filter(|&v| self.types[v as usize] != self.class_slot), offset, limit)
|
||||
}
|
||||
(None, Some(class)) => self.largest_of(class, offset, limit),
|
||||
}
|
||||
}
|
||||
|
||||
/// `None` for a ref that names no object, the super-root included.
|
||||
pub fn object_row(&self, object: HeapObjectRef) -> Option<HeapObjectRow> {
|
||||
let v = object.0;
|
||||
if v >= self.root() {
|
||||
return None;
|
||||
}
|
||||
let i = v as usize;
|
||||
let slot = self.types[i];
|
||||
let preview = if slot == self.class_slot {
|
||||
self.class_of_object(v).and_then(|c| self.classes.get(c)).map(|c| c.name.clone())
|
||||
} else {
|
||||
string_preview(self, v, PREVIEW_CHARS)
|
||||
};
|
||||
Some(HeapObjectRow {
|
||||
object,
|
||||
class_name: self.classes.get(slot as usize).map(|c| c.name.clone()).unwrap_or_default(),
|
||||
address: format!("{:#x}", self.ids.id_at(v)),
|
||||
shallow: u64::from(self.shallow8[i]) * 8,
|
||||
retained: self.retained[i],
|
||||
has_children: self.kids_start[i + 1] > self.kids_start[i],
|
||||
preview,
|
||||
})
|
||||
}
|
||||
|
||||
/// The record of object `v`, or `None` when it cannot be read.
|
||||
pub(crate) fn record(&self, v: u32) -> Option<HeapRecord<'_>> {
|
||||
let at = usize::try_from(*self.offsets.get(v as usize)?).ok()?;
|
||||
crate::hprof::record_at(&self.hprof, at, self.id_size).ok()
|
||||
}
|
||||
|
||||
/// The class slot a class object stands for.
|
||||
pub(crate) fn class_of_object(&self, v: u32) -> Option<usize> {
|
||||
if v >= self.root() {
|
||||
return None;
|
||||
}
|
||||
let id = self.ids.id_at(v);
|
||||
self.class_info.iter().position(|c| c.id == id && c.id != 0)
|
||||
}
|
||||
|
||||
/// Children of node `v` in the dominator tree, already in page order.
|
||||
fn children(&self, v: u32) -> impl Iterator<Item = u32> + '_ {
|
||||
let v = v as usize;
|
||||
self.kids[self.kids_start[v] as usize..self.kids_start[v + 1] as usize].iter().copied()
|
||||
}
|
||||
|
||||
fn page(&self, objects: impl Iterator<Item = u32>, offset: u32, limit: u32) -> HeapObjectPage {
|
||||
let mut rows = Vec::new();
|
||||
let mut total = 0u32;
|
||||
for v in objects {
|
||||
if total >= offset && total - offset < limit {
|
||||
rows.extend(self.object_row(HeapObjectRef(v)));
|
||||
}
|
||||
total = total.saturating_add(1);
|
||||
}
|
||||
HeapObjectPage { rows, total }
|
||||
}
|
||||
|
||||
/// Scans every object: the top `offset + limit` stay in a heap whose top is the smallest.
|
||||
fn largest_of(&self, class: u32, offset: u32, limit: u32) -> HeapObjectPage {
|
||||
let count = self.classes[class as usize].count;
|
||||
let total = u32::try_from(count).unwrap_or(u32::MAX);
|
||||
let keep = match offset.checked_add(limit) {
|
||||
Some(keep) if offset < total && limit > 0 => keep.min(total) as usize,
|
||||
_ => return HeapObjectPage { rows: Vec::new(), total },
|
||||
};
|
||||
let mut top: BinaryHeap<Reverse<(u64, Reverse<u32>)>> = BinaryHeap::with_capacity(keep + 1);
|
||||
for v in 0..self.root() {
|
||||
let i = v as usize;
|
||||
if self.types[i] != class || self.idom[i] == NONE {
|
||||
continue;
|
||||
}
|
||||
top.push(Reverse((self.retained[i], Reverse(v))));
|
||||
if top.len() > keep {
|
||||
top.pop();
|
||||
}
|
||||
}
|
||||
// Ascending `Reverse` order is descending retained, then ascending index.
|
||||
let rows = top
|
||||
.into_sorted_vec()
|
||||
.into_iter()
|
||||
.skip(offset as usize)
|
||||
.filter_map(|Reverse((_, Reverse(v)))| self.object_row(HeapObjectRef(v)))
|
||||
.collect();
|
||||
HeapObjectPage { rows, total }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//! What the index queries return over IPC.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::hprof::HeapRootKind;
|
||||
|
||||
/// Most rows one page of [`crate::HeapIndex::dominated`] returns.
|
||||
pub const MAX_PAGE: u32 = 500;
|
||||
|
||||
/// An object of one snapshot: its index in the [`crate::HeapIndex`] columns. Any value may
|
||||
/// arrive over IPC; queries answer an index that names no object with nothing.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(transparent)]
|
||||
pub struct HeapObjectRef(pub u32);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapClassRow {
|
||||
/// Type slot, as `classIndex` in [`crate::HeapIndex::dominated`].
|
||||
pub class_index: u32,
|
||||
pub name: String,
|
||||
pub count: u64,
|
||||
pub shallow: u64,
|
||||
pub retained: u64,
|
||||
pub unreachable_count: u64,
|
||||
/// Another class has the same name (several class loaders): nothing joined by name, such as a
|
||||
/// second snapshot, can tell them apart.
|
||||
pub ambiguous: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapObjectRow {
|
||||
pub object: HeapObjectRef,
|
||||
pub class_name: String,
|
||||
/// The object's hprof id in hex, `0x1300`.
|
||||
pub address: String,
|
||||
pub shallow: u64,
|
||||
pub retained: u64,
|
||||
/// Has children in the dominator tree.
|
||||
pub has_children: bool,
|
||||
/// The text of a `java.lang.String`, or the name of the class a class object stands for.
|
||||
pub preview: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapObjectPage {
|
||||
pub rows: Vec<HeapObjectRow>,
|
||||
/// Rows of every page together.
|
||||
pub total: u32,
|
||||
}
|
||||
|
||||
/// How an object is held by the next step of a path to a GC root.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]
|
||||
pub enum HeapEdge {
|
||||
/// An instance field; the name is empty when the dump does not show which one.
|
||||
Field {
|
||||
name: String,
|
||||
},
|
||||
Element {
|
||||
index: u32,
|
||||
},
|
||||
Static {
|
||||
name: String,
|
||||
},
|
||||
/// From an instance to its class.
|
||||
ClassOf,
|
||||
SuperClass,
|
||||
ClassLoader,
|
||||
/// `thread` is the name of the thread whose root it is, when the dump has it.
|
||||
Root {
|
||||
root: HeapRootKind,
|
||||
thread: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapPathStep {
|
||||
pub object: HeapObjectRow,
|
||||
/// How the next step holds this object; on the last step, the root.
|
||||
pub via: HeapEdge,
|
||||
}
|
||||
|
||||
/// A path from an object up to its GC root. A long one keeps only the steps nearest each end.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapRootPath {
|
||||
pub steps: Vec<HeapPathStep>,
|
||||
/// Steps left out between `steps[omitted_after - 1]` and `steps[omitted_after]`; 0 for none.
|
||||
pub omitted: u32,
|
||||
pub omitted_after: u32,
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
use super::*;
|
||||
use crate::hprof::BasicType;
|
||||
use crate::test_index::{analyzed, analyzed_bytes, obj, slot};
|
||||
use crate::testing::{HprofWriter, MiniHeap, sample, sample_with_string};
|
||||
|
||||
const H: u64 = 0x1000;
|
||||
const N1: u64 = 0x1100;
|
||||
const N3: u64 = 0x1300;
|
||||
const L: u64 = 0x1400;
|
||||
const W: u64 = 0x1600;
|
||||
const N4: u64 = 0x1700;
|
||||
|
||||
fn addresses(page: &HeapObjectPage) -> Vec<&str> {
|
||||
page.rows.iter().map(|r| r.address.as_str()).collect()
|
||||
}
|
||||
|
||||
fn retained(page: &HeapObjectPage) -> Vec<u64> {
|
||||
page.rows.iter().map(|r| r.retained).collect()
|
||||
}
|
||||
|
||||
fn path(index: &HeapIndex, addr: u64) -> Option<Vec<(String, HeapEdge)>> {
|
||||
let path = index.path_to_root(obj(index, addr))?;
|
||||
assert_eq!((path.omitted, path.omitted_after), (0, 0));
|
||||
Some(path.steps.into_iter().map(|s| (s.object.address, s.via)).collect())
|
||||
}
|
||||
|
||||
fn field(name: &str) -> HeapEdge {
|
||||
HeapEdge::Field { name: name.to_owned() }
|
||||
}
|
||||
|
||||
fn root(root: HeapRootKind, thread: Option<&str>) -> HeapEdge {
|
||||
HeapEdge::Root { root, thread: thread.map(str::to_owned) }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn top_level_dominators_sorted_by_retained() {
|
||||
let index = analyzed(&sample());
|
||||
let page = index.dominated(None, None, 0, 10);
|
||||
assert_eq!(addresses(&page), ["0x1000", "0x1300", "0x1600"]);
|
||||
assert_eq!(retained(&page), [72, 56, 24]);
|
||||
// The class objects, rooted as sticky classes, are left out.
|
||||
assert_eq!(page.total, 3);
|
||||
assert_eq!(addresses(&index.dominated(None, None, 1, 1)), ["0x1300"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn children_of_holder() {
|
||||
let index = analyzed(&sample());
|
||||
let page = index.dominated(Some(obj(&index, H)), None, 0, 10);
|
||||
assert_eq!(addresses(&page), ["0x1100", "0x1200"]);
|
||||
assert_eq!(retained(&page), [24, 24]);
|
||||
assert_eq!(page.total, 2);
|
||||
assert!(page.rows.iter().all(|r| !r.has_children && r.class_name == "Node"));
|
||||
let n3 = index.object_row(obj(&index, N3)).unwrap();
|
||||
assert!(n3.has_children);
|
||||
assert_eq!(index.dominated(Some(n3.object), None, 0, 10).rows[0].class_name, "long[]");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn largest_of_class() {
|
||||
let index = analyzed(&sample());
|
||||
let node = slot(&index, "Node");
|
||||
let page = index.dominated(None, Some(node), 0, 2);
|
||||
assert_eq!(addresses(&page), ["0x1300", "0x1100"]);
|
||||
assert_eq!(retained(&page), [56, 24]);
|
||||
assert_eq!(page.total, 3);
|
||||
assert_eq!(addresses(&index.dominated(None, Some(node), 2, 2)), ["0x1200"]);
|
||||
assert!(index.dominated(None, Some(node), 3, 2).rows.is_empty());
|
||||
// Within a parent's children.
|
||||
assert_eq!(addresses(&index.dominated(Some(obj(&index, H)), Some(node), 1, 5)), ["0x1200"]);
|
||||
assert_eq!(index.dominated(Some(obj(&index, H)), Some(slot(&index, "Holder")), 0, 5).total, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_row_describes_object() {
|
||||
let index = analyzed(&sample());
|
||||
let row = index.object_row(obj(&index, N3)).unwrap();
|
||||
assert_eq!(
|
||||
row,
|
||||
HeapObjectRow {
|
||||
object: obj(&index, N3),
|
||||
class_name: "Node".to_owned(),
|
||||
address: "0x1300".to_owned(),
|
||||
shallow: 24,
|
||||
retained: 56,
|
||||
has_children: true,
|
||||
preview: None,
|
||||
}
|
||||
);
|
||||
let unreachable = index.object_row(obj(&index, N4)).unwrap();
|
||||
assert_eq!((unreachable.retained, unreachable.has_children), (0, false));
|
||||
|
||||
let holder = &index.class_info[slot(&index, "Holder") as usize];
|
||||
let class_object = obj(&index, holder.id);
|
||||
let row = index.object_row(class_object).unwrap();
|
||||
assert_eq!((row.class_name.as_str(), row.preview.as_deref()), ("java.lang.Class", Some("Holder")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refs_that_name_no_object_answer_nothing() {
|
||||
let index = analyzed(&sample());
|
||||
let node = slot(&index, "Node");
|
||||
for bad in [index.root(), index.root() + 1, u32::MAX] {
|
||||
let bad = HeapObjectRef(bad);
|
||||
assert_eq!(index.object_row(bad), None);
|
||||
assert_eq!(index.path_to_root(bad), None);
|
||||
assert_eq!(index.dominated(Some(bad), None, 0, 10), HeapObjectPage { rows: Vec::new(), total: 0 });
|
||||
assert_eq!(index.dominated(Some(bad), Some(node), 0, 10).total, 0);
|
||||
}
|
||||
for bad in [index.classes.len() as u32, u32::MAX] {
|
||||
assert_eq!(index.dominated(None, Some(bad), 0, 10).total, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_to_root_names_fields_and_root() {
|
||||
let index = analyzed(&sample());
|
||||
assert_eq!(
|
||||
path(&index, L),
|
||||
Some(vec![("0x1400".to_owned(), field("next")), ("0x1300".to_owned(), root(HeapRootKind::JavaFrame, None))])
|
||||
);
|
||||
assert_eq!(
|
||||
path(&index, N1),
|
||||
Some(vec![("0x1100".to_owned(), field("a")), ("0x1000".to_owned(), root(HeapRootKind::JniGlobal, None))])
|
||||
);
|
||||
assert_eq!(path(&index, W), Some(vec![("0x1600".to_owned(), root(HeapRootKind::JniGlobal, None))]));
|
||||
assert_eq!(path(&index, N4), None);
|
||||
}
|
||||
|
||||
/// A rooted linked list of `len` Nodes from `0x100000`, each held by the next one's `next`.
|
||||
fn chain(len: u64) -> HeapIndex {
|
||||
let mut heap = sample();
|
||||
for i in 0..len {
|
||||
let next = if i == 0 { 0 } else { 0x10_0000 + (i - 1) * 0x20 };
|
||||
heap.object(0x10_0000 + i * 0x20, "Node", &[("next", next)]);
|
||||
}
|
||||
heap.root(HeapRootKind::JniGlobal, 0x10_0000 + (len - 1) * 0x20);
|
||||
analyzed(&heap)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_of_an_object_deep_in_a_linked_list_keeps_both_ends() {
|
||||
let index = chain(3000);
|
||||
let path = index.path_to_root(obj(&index, 0x10_0000)).unwrap();
|
||||
assert_eq!((path.steps.len(), path.omitted, path.omitted_after), (1000, 2000, 500));
|
||||
let address = |i: usize| path.steps[i].object.address.clone();
|
||||
assert_eq!([address(0), address(499)], ["0x100000", "0x103e60"]);
|
||||
// The step after the gap is the one 2000 nodes further up.
|
||||
assert_eq!([address(500), address(999)], ["0x113880", "0x1176e0"]);
|
||||
assert_eq!(path.steps[999].via, root(HeapRootKind::JniGlobal, None));
|
||||
assert!(path.steps[..999].iter().all(|s| s.via == field("next")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_of_exactly_a_thousand_steps_is_whole() {
|
||||
let index = chain(1000);
|
||||
let path = index.path_to_root(obj(&index, 0x10_0000)).unwrap();
|
||||
assert_eq!((path.steps.len(), path.omitted, path.omitted_after), (1000, 0, 0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_skips_the_weak_referent() {
|
||||
let mut heap = sample();
|
||||
heap.object(0x1c00, "java.lang.ref.WeakReference", &[("referent", 0x1d00), ("queue", 0x1d00)])
|
||||
.object(0x1d00, "Node", &[])
|
||||
.root(HeapRootKind::JniGlobal, 0x1c00);
|
||||
let index = analyzed(&heap);
|
||||
assert_eq!(path(&index, 0x1d00).unwrap()[0].1, field("queue"));
|
||||
}
|
||||
|
||||
/// Kid extends Base, loaded by loader `0x300`; Base's static `cache` holds X; K is a rooted Kid;
|
||||
/// `0x700` is a rooted `Object[] { null, Y }`. No class is rooted on its own.
|
||||
fn class_edges_heap() -> Vec<u8> {
|
||||
let mut w = HprofWriter::new();
|
||||
let names = [(1, "Base"), (2, "Kid"), (3, "Ldr"), (4, "[Ljava/lang/Object;"), (5, "cache")];
|
||||
for (id, name) in names {
|
||||
w.string(id, name);
|
||||
}
|
||||
for (class, name) in [(0x100, 1), (0x200, 2), (0x400, 3), (0x800, 4)] {
|
||||
w.load_class(class, name);
|
||||
}
|
||||
w.class(0x100, 0, &[], &[(5, 0x500)])
|
||||
.class_with_loader(0x200, 0x100, 0x300, &[], &[])
|
||||
.class(0x400, 0, &[], &[])
|
||||
.class(0x800, 0, &[], &[])
|
||||
.instance(0x300, 0x400, &[])
|
||||
.instance(0x500, 0x400, &[])
|
||||
.instance(0x600, 0x200, &[])
|
||||
.obj_array(0x700, 0x800, &[0, 0x900])
|
||||
.instance(0x900, 0x400, &[])
|
||||
.root(HeapRootKind::JniGlobal, 0x600)
|
||||
.root(HeapRootKind::JniGlobal, 0x700);
|
||||
w.finish()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn path_names_statics_supers_loaders_classes_and_elements() {
|
||||
let index = analyzed_bytes(&class_edges_heap());
|
||||
let k = ("0x600".to_owned(), root(HeapRootKind::JniGlobal, None));
|
||||
assert_eq!(
|
||||
path(&index, 0x500),
|
||||
Some(vec![
|
||||
("0x500".to_owned(), HeapEdge::Static { name: "cache".to_owned() }),
|
||||
("0x100".to_owned(), HeapEdge::SuperClass),
|
||||
("0x200".to_owned(), HeapEdge::ClassOf),
|
||||
k.clone(),
|
||||
])
|
||||
);
|
||||
assert_eq!(
|
||||
path(&index, 0x300),
|
||||
Some(vec![("0x300".to_owned(), HeapEdge::ClassLoader), ("0x200".to_owned(), HeapEdge::ClassOf), k])
|
||||
);
|
||||
assert_eq!(
|
||||
path(&index, 0x900),
|
||||
Some(vec![
|
||||
("0x900".to_owned(), HeapEdge::Element { index: 1 }),
|
||||
("0x700".to_owned(), root(HeapRootKind::JniGlobal, None))
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
fn with_main_thread(mut heap: MiniHeap, name: impl FnOnce(&mut MiniHeap, u64)) -> MiniHeap {
|
||||
heap.class("java.lang.Thread", Some("java.lang.Object"), &[("name", BasicType::Object)]);
|
||||
name(&mut heap, 0x2000);
|
||||
heap.object(0x2100, "java.lang.Thread", &[("name", 0x2000)]).root_in_thread(HeapRootKind::ThreadObject, 0x2100, 1);
|
||||
heap
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thread_name_on_java_frame_root() {
|
||||
let heap = with_main_thread(sample_with_string(), |h, at| {
|
||||
h.string(at, "main", true);
|
||||
});
|
||||
let index = analyzed(&heap);
|
||||
let steps = path(&index, L).unwrap();
|
||||
assert_eq!(steps.last().unwrap().1, root(HeapRootKind::JavaFrame, Some("main")));
|
||||
// A JNI global carries no thread.
|
||||
assert_eq!(path(&index, H).unwrap()[0].1, root(HeapRootKind::JniGlobal, None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jdk8_thread_name_is_a_char_array() {
|
||||
let heap = with_main_thread(sample(), |h, at| {
|
||||
let utf16: Vec<u8> = "main".encode_utf16().flat_map(u16::to_be_bytes).collect();
|
||||
h.prim_bytes(at, BasicType::Char, &utf16);
|
||||
});
|
||||
let index = analyzed(&heap);
|
||||
assert_eq!(path(&index, N3).unwrap()[0].1, root(HeapRootKind::JavaFrame, Some("main")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_limit_is_clamped() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Node", Some("java.lang.Object"), &[("next", BasicType::Object)]);
|
||||
for k in 0..600u64 {
|
||||
let addr = 0x10_0000 + k * 0x100;
|
||||
heap.object(addr, "Node", &[]).root(HeapRootKind::JniGlobal, addr);
|
||||
}
|
||||
let index = analyzed(&heap);
|
||||
let node = slot(&index, "Node");
|
||||
let page = index.dominated(None, Some(node), 0, 10_000);
|
||||
assert_eq!((page.rows.len(), page.total), (MAX_PAGE as usize, 600));
|
||||
let top = index.dominated(None, None, 0, 10_000);
|
||||
assert_eq!((top.rows.len(), top.total), (MAX_PAGE as usize, 600));
|
||||
assert_eq!(index.dominated(None, None, 550, 10_000).rows.len(), 50);
|
||||
for (offset, limit) in [(u32::MAX, u32::MAX), (u32::MAX - 10, 100), (599, u32::MAX)] {
|
||||
let page = index.dominated(None, Some(node), offset, limit);
|
||||
assert_eq!((page.rows.len(), page.total), (usize::from(offset == 599), 600), "{offset} {limit}");
|
||||
assert!(index.dominated(None, None, offset, limit).rows.len() <= 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_rows_skip_empty_slots() {
|
||||
let index = analyzed(&sample());
|
||||
let rows = index.class_rows();
|
||||
let node = rows.iter().find(|r| r.name == "Node").unwrap();
|
||||
assert_eq!(
|
||||
(node.class_index, node.count, node.shallow, node.retained, node.unreachable_count, node.ambiguous),
|
||||
(slot(&index, "Node"), 3, 72, 104, 1, false)
|
||||
);
|
||||
assert!(rows.iter().all(|r| r.count + r.unreachable_count > 0));
|
||||
assert!(!rows.iter().any(|r| r.name == "boolean[]"));
|
||||
assert!(rows.windows(2).all(|p| p[0].retained >= p[1].retained));
|
||||
assert_eq!(index.summary().objects, 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_rows_mark_names_of_several_classes() {
|
||||
let mut heap = sample();
|
||||
// `object` takes the class defined last under a name, so each Foo gets one instance.
|
||||
heap.class("Foo", Some("java.lang.Object"), &[])
|
||||
.object(0x3000, "Foo", &[])
|
||||
.class("Foo", Some("java.lang.Object"), &[])
|
||||
.object(0x3100, "Foo", &[])
|
||||
.object(0x3200, "Foo", &[]);
|
||||
let index = analyzed(&heap);
|
||||
let rows = index.class_rows();
|
||||
let foos: Vec<_> = rows.iter().filter(|r| r.name == "Foo").collect();
|
||||
assert_eq!(foos.len(), 2);
|
||||
assert!(foos.iter().all(|r| r.ambiguous));
|
||||
let mut counts: Vec<u64> = foos.iter().map(|r| r.unreachable_count).collect();
|
||||
counts.sort_unstable();
|
||||
assert_eq!(counts, [1, 2]);
|
||||
assert!(rows.iter().filter(|r| r.name != "Foo").all(|r| !r.ambiguous));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipc_shapes() {
|
||||
assert_eq!(json(&HeapObjectRef(7)), "7");
|
||||
assert_eq!(json(&HeapEdge::ClassOf), r#"{"kind":"classOf"}"#);
|
||||
assert_eq!(json(&HeapEdge::Element { index: 2 }), r#"{"kind":"element","index":2}"#);
|
||||
assert_eq!(
|
||||
json(&root(HeapRootKind::JavaFrame, Some("main"))),
|
||||
r#"{"kind":"root","root":"javaFrame","thread":"main"}"#
|
||||
);
|
||||
let row = HeapClassRow {
|
||||
class_index: 1,
|
||||
name: "A".to_owned(),
|
||||
count: 2,
|
||||
shallow: 3,
|
||||
retained: 4,
|
||||
unreachable_count: 5,
|
||||
ambiguous: false,
|
||||
};
|
||||
assert_eq!(
|
||||
json(&row),
|
||||
r#"{"classIndex":1,"name":"A","count":2,"shallow":3,"retained":4,"unreachableCount":5,"ambiguous":false}"#
|
||||
);
|
||||
let back: HeapObjectRef = serde_json::from_str("4294967295").unwrap();
|
||||
assert_eq!(back, HeapObjectRef(u32::MAX));
|
||||
}
|
||||
|
||||
fn json(value: &impl serde::Serialize) -> String {
|
||||
serde_json::to_string(value).unwrap()
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
//! HotSpot `.hprof` heap-dump reading and analysis for the debugger's JVM Memory view.
|
||||
|
||||
mod analysis;
|
||||
mod column;
|
||||
mod compare;
|
||||
mod disk;
|
||||
mod dominators;
|
||||
mod error;
|
||||
mod files;
|
||||
mod graph;
|
||||
mod hprof;
|
||||
mod ids;
|
||||
mod index;
|
||||
mod names;
|
||||
mod progress;
|
||||
mod reader;
|
||||
mod registry;
|
||||
mod shallow;
|
||||
mod strings;
|
||||
#[cfg(test)]
|
||||
mod test_index;
|
||||
/// Fixture builders shared by this crate's tests and by dependants' tests and benches; not API.
|
||||
#[doc(hidden)]
|
||||
pub mod testing;
|
||||
|
||||
pub use analysis::{AnalyzeOptions, analyze};
|
||||
pub use column::{Column, NONE, Pod, Store};
|
||||
pub use compare::{HeapClassDiff, compare};
|
||||
pub use disk::{DISK_MARGIN, capture_need, create_snapshot_dir, free_bytes, index_need, remove_stale, snapshots_root};
|
||||
pub use dominators::{Dominators, dominators};
|
||||
pub use error::HeapError;
|
||||
pub use graph::{ClassInfo, HeapGraph, PRIM_SLOTS, build_graph};
|
||||
pub use hprof::{
|
||||
BasicType, ClassDump, HeapRecord, HeapRootKind, HprofHeader, Record, parse_header, record_at, scan, scan_segment,
|
||||
};
|
||||
pub use ids::{IdMap, IdMapBuilder};
|
||||
pub use index::{
|
||||
ClassStat, HeapClassRow, HeapEdge, HeapIndex, HeapObjectPage, HeapObjectRef, HeapObjectRow, HeapPathStep,
|
||||
HeapRootPath, HeapSummary, MAX_PAGE,
|
||||
};
|
||||
pub use names::display_name;
|
||||
pub use progress::HeapPhase;
|
||||
pub use registry::{HeapOrigin, HeapRegistry, HeapSnapshotId, HeapSnapshotInfo, OpenSnapshot};
|
||||
pub use shallow::{HistogramEntry, ShallowModel, SizeSource, VmLayout};
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Class names in the form the JVM's class histogram and the UI use.
|
||||
|
||||
/// The Java source form of an hprof class name (`java/util/HashMap$Node`, `[Ljava/lang/String;`),
|
||||
/// as `ide_jvm_attach::display_class_name` writes it, plus `+0x` → `/0x` for hidden classes,
|
||||
/// which the histogram spells with a slash. Names that already use dots pass through unchanged.
|
||||
pub fn display_name(hprof_name: &str) -> String {
|
||||
let dotted = hprof_name.replace('/', ".");
|
||||
let dims = dotted.bytes().take_while(|&b| b == b'[').count();
|
||||
let shown = if dims == 0 { dotted } else { array_name(&dotted, dims) };
|
||||
// After the slash replacement, or the new slash would become a dot.
|
||||
shown.replace("+0x", "/0x")
|
||||
}
|
||||
|
||||
/// `[[I` → `int[][]`; a descriptor that is not one stays as it is.
|
||||
fn array_name(dotted: &str, dims: usize) -> String {
|
||||
let element = &dotted[dims..];
|
||||
let base = match element {
|
||||
"Z" => "boolean",
|
||||
"B" => "byte",
|
||||
"C" => "char",
|
||||
"S" => "short",
|
||||
"I" => "int",
|
||||
"J" => "long",
|
||||
"F" => "float",
|
||||
"D" => "double",
|
||||
_ => match element.strip_prefix('L').and_then(|e| e.strip_suffix(';')) {
|
||||
Some(class) => class,
|
||||
None => return dotted.to_owned(),
|
||||
},
|
||||
};
|
||||
format!("{base}{}", "[]".repeat(dims))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn display_names() {
|
||||
for (hprof, shown) in [
|
||||
("[Z", "boolean[]"),
|
||||
("[J", "long[]"),
|
||||
("[[D", "double[][]"),
|
||||
("[Ljava/lang/String;", "java.lang.String[]"),
|
||||
("java/util/HashMap$Node", "java.util.HashMap$Node"),
|
||||
("Gen$$Lambda+0x0000007800000a00", "Gen$$Lambda/0x0000007800000a00"),
|
||||
] {
|
||||
assert_eq!(display_name(hprof), shown, "{hprof}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dotted_names_and_hidden_class_arrays() {
|
||||
for (hprof, shown) in [
|
||||
("java.lang.String", "java.lang.String"),
|
||||
("[Ljava.lang.Object;", "java.lang.Object[]"),
|
||||
("[[Lpkg.Gen$$Lambda+0x0000007800000a00;", "pkg.Gen$$Lambda/0x0000007800000a00[][]"),
|
||||
("[Ljava.lang.String", "[Ljava.lang.String"),
|
||||
("[Q", "[Q"),
|
||||
("0x7f00", "0x7f00"),
|
||||
("", ""),
|
||||
] {
|
||||
assert_eq!(display_name(hprof), shown, "{hprof}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
//! Phases of a heap analysis and the progress/cancellation checks every long pass makes.
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::error::HeapError;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum HeapPhase {
|
||||
Reading,
|
||||
Linking,
|
||||
Dominators,
|
||||
Retained,
|
||||
Paths,
|
||||
Saving,
|
||||
}
|
||||
|
||||
/// Longest stretch of work between two progress reports (and looks at the cancellation token).
|
||||
pub(crate) const CHECK_EVERY: u64 = 1 << 20;
|
||||
|
||||
/// The caller's progress callback plus the token, looked at right after every report so a callback
|
||||
/// that cancels stops the work before the next step.
|
||||
pub(crate) struct Progress<'a> {
|
||||
report: &'a dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &'a CancellationToken,
|
||||
}
|
||||
|
||||
impl<'a> Progress<'a> {
|
||||
pub(crate) fn new(
|
||||
report: &'a dyn Fn(HeapPhase, u64, u64) -> Result<(), HeapError>,
|
||||
cancel: &'a CancellationToken,
|
||||
) -> Self {
|
||||
Self { report, cancel }
|
||||
}
|
||||
|
||||
pub(crate) fn report(&self, phase: HeapPhase, done: u64, total: u64) -> Result<(), HeapError> {
|
||||
(self.report)(phase, done, total)?;
|
||||
self.check()
|
||||
}
|
||||
|
||||
pub(crate) fn token(&self) -> &'a CancellationToken {
|
||||
self.cancel
|
||||
}
|
||||
|
||||
/// Looks at the token without reporting.
|
||||
pub(crate) fn check(&self) -> Result<(), HeapError> {
|
||||
if self.cancel.is_cancelled() { Err(HeapError::Cancelled) } else { Ok(()) }
|
||||
}
|
||||
|
||||
/// Looks at the token at step 0 and reports every [`CHECK_EVERY`] steps after it. Step 0 does
|
||||
/// not report: a loop starts right after its phase's own report.
|
||||
pub(crate) fn step(&self, step: u64, phase: HeapPhase, done: u64, total: u64) -> Result<(), HeapError> {
|
||||
if step == 0 {
|
||||
self.check()
|
||||
} else if step.is_multiple_of(CHECK_EVERY) {
|
||||
self.report(phase, done, total)
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::cell::RefCell;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn steps_check_at_zero_and_report_every_check_interval() {
|
||||
let calls = RefCell::new(Vec::new());
|
||||
let report = |phase, done, total| {
|
||||
calls.borrow_mut().push((phase, done, total));
|
||||
Ok(())
|
||||
};
|
||||
let cancel = CancellationToken::new();
|
||||
let progress = Progress::new(&report, &cancel);
|
||||
for step in [0, 1, CHECK_EVERY - 1, CHECK_EVERY, CHECK_EVERY + 1, 3 * CHECK_EVERY] {
|
||||
progress.step(step, HeapPhase::Linking, step, 9).unwrap();
|
||||
}
|
||||
assert_eq!(*calls.borrow(), [(HeapPhase::Linking, CHECK_EVERY, 9), (HeapPhase::Linking, 3 * CHECK_EVERY, 9)]);
|
||||
|
||||
cancel.cancel();
|
||||
assert!(matches!(progress.step(0, HeapPhase::Linking, 0, 9), Err(HeapError::Cancelled)));
|
||||
assert!(matches!(progress.report(HeapPhase::Saving, 1, 1), Err(HeapError::Cancelled)));
|
||||
assert_eq!(calls.borrow().last(), Some(&(HeapPhase::Saving, 1, 1)), "reports before it looks at the token");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn report_errors_pass_through() {
|
||||
let report = |_, _, _| Err(HeapError::NotEnoughDiskSpace { needed: 2, available: 1 });
|
||||
let cancel = CancellationToken::new();
|
||||
let progress = Progress::new(&report, &cancel);
|
||||
assert!(matches!(progress.report(HeapPhase::Linking, 0, 1), Err(HeapError::NotEnoughDiskSpace { .. })));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn phases_serialize_in_camel_case() {
|
||||
assert_eq!(serde_json::to_string(&HeapPhase::Dominators).unwrap(), "\"dominators\"");
|
||||
assert_eq!(serde_json::from_str::<HeapPhase>("\"saving\"").unwrap(), HeapPhase::Saving);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
use crate::error::HeapError;
|
||||
|
||||
/// Big-endian cursor over a dump. Every read is bounds-checked; running off `buf` yields
|
||||
/// `Truncated` at `buf.len()`, so callers bound a record by slicing `buf` to its end.
|
||||
pub(crate) struct Reader<'a> {
|
||||
buf: &'a [u8],
|
||||
pos: usize,
|
||||
id_size: u8,
|
||||
}
|
||||
|
||||
impl<'a> Reader<'a> {
|
||||
pub(crate) fn new(buf: &'a [u8], pos: usize, id_size: u8) -> Self {
|
||||
Self { buf, pos, id_size }
|
||||
}
|
||||
|
||||
pub(crate) fn pos(&self) -> usize {
|
||||
self.pos
|
||||
}
|
||||
|
||||
pub(crate) fn id_size(&self) -> u8 {
|
||||
self.id_size
|
||||
}
|
||||
|
||||
pub(crate) fn buf(&self) -> &'a [u8] {
|
||||
self.buf
|
||||
}
|
||||
|
||||
pub(crate) fn bytes(&mut self, n: usize) -> Result<&'a [u8], HeapError> {
|
||||
let end = self.pos.checked_add(n).filter(|&end| end <= self.buf.len());
|
||||
let Some(end) = end else {
|
||||
return Err(HeapError::Truncated { at: self.buf.len() as u64 });
|
||||
};
|
||||
let slice = &self.buf[self.pos..end];
|
||||
self.pos = end;
|
||||
Ok(slice)
|
||||
}
|
||||
|
||||
pub(crate) fn skip(&mut self, n: usize) -> Result<(), HeapError> {
|
||||
self.bytes(n).map(|_| ())
|
||||
}
|
||||
|
||||
fn array<const N: usize>(&mut self) -> Result<[u8; N], HeapError> {
|
||||
let mut out = [0u8; N];
|
||||
out.copy_from_slice(self.bytes(N)?);
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
pub(crate) fn u1(&mut self) -> Result<u8, HeapError> {
|
||||
Ok(self.array::<1>()?[0])
|
||||
}
|
||||
|
||||
pub(crate) fn u2(&mut self) -> Result<u16, HeapError> {
|
||||
Ok(u16::from_be_bytes(self.array()?))
|
||||
}
|
||||
|
||||
pub(crate) fn u4(&mut self) -> Result<u32, HeapError> {
|
||||
Ok(u32::from_be_bytes(self.array()?))
|
||||
}
|
||||
|
||||
pub(crate) fn u8(&mut self) -> Result<u64, HeapError> {
|
||||
Ok(u64::from_be_bytes(self.array()?))
|
||||
}
|
||||
|
||||
/// `id_size` is 4 or 8; the public entry points reject anything else before building a reader.
|
||||
pub(crate) fn id(&mut self) -> Result<u64, HeapError> {
|
||||
if self.id_size == 4 { self.u4().map(u64::from) } else { self.u8() }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn reads_big_endian_and_ids_of_both_sizes() {
|
||||
let buf = [1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5];
|
||||
let mut r = Reader::new(&buf, 0, 8);
|
||||
assert_eq!((r.u1().unwrap(), r.u2().unwrap(), r.u4().unwrap(), r.u8().unwrap()), (1, 2, 3, 4));
|
||||
assert_eq!(r.pos(), 15);
|
||||
assert_eq!(Reader::new(&buf[15..], 0, 4).id().unwrap(), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn running_off_the_end_reports_the_buffer_end() {
|
||||
let buf = [0u8; 6];
|
||||
let mut r = Reader::new(&buf, 4, 8);
|
||||
assert!(matches!(r.u4(), Err(HeapError::Truncated { at: 6 })));
|
||||
assert_eq!(r.pos(), 4, "a failed read does not advance");
|
||||
assert!(matches!(r.skip(usize::MAX), Err(HeapError::Truncated { at: 6 })));
|
||||
assert!(matches!(Reader::new(&buf, 99, 8).u1(), Err(HeapError::Truncated { at: 6 })));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
//! The heap snapshots that are open in this process.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
|
||||
|
||||
use ide_core::process::process_alive;
|
||||
use ide_core::{DebugSessionId, IpcPath};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::disk::remove_stale;
|
||||
use crate::index::{HeapIndex, HeapSummary};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(transparent)]
|
||||
pub struct HeapSnapshotId(pub u32);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(tag = "kind", rename_all = "camelCase", rename_all_fields = "camelCase")]
|
||||
pub enum HeapOrigin {
|
||||
/// Dumped from a live debug session.
|
||||
Captured { session: DebugSessionId, pid: u32, java_version: String, taken_while_suspended: bool, dump_ms: u32 },
|
||||
/// An `.hprof` the user opened; it is never deleted.
|
||||
File { path: IpcPath },
|
||||
}
|
||||
|
||||
impl HeapOrigin {
|
||||
/// Text identifying the snapshot in the UI: the process id of a capture, the file name of a
|
||||
/// file. The capture time is `created_ms`, which the UI formats in local time.
|
||||
pub fn label(&self) -> String {
|
||||
match self {
|
||||
HeapOrigin::Captured { pid, .. } => pid.to_string(),
|
||||
HeapOrigin::File { path } => path.file_name().unwrap_or(path.as_os_str()).to_string_lossy().into_owned(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct HeapSnapshotInfo {
|
||||
pub id: HeapSnapshotId,
|
||||
pub label: String,
|
||||
pub origin: HeapOrigin,
|
||||
pub summary: HeapSummary,
|
||||
/// Unix time in milliseconds when the snapshot was opened.
|
||||
pub created_ms: u64,
|
||||
}
|
||||
|
||||
/// An analyzed snapshot together with the directory that holds its files. Dropping it unmaps the
|
||||
/// index and then removes the directory, dump included for a capture; a dump opened from a file
|
||||
/// is outside the directory and stays.
|
||||
pub struct OpenSnapshot {
|
||||
pub info: HeapSnapshotInfo,
|
||||
/// `None` only while dropping, so the mappings are gone before the directory is deleted
|
||||
/// (Windows cannot delete a mapped file).
|
||||
index: Option<HeapIndex>,
|
||||
/// Holds the index files and the dump of a capture.
|
||||
pub dir: PathBuf,
|
||||
pub dump: PathBuf,
|
||||
/// Set by [`HeapRegistry::insert`] once `dir` is known to be one of the registry's own.
|
||||
remove_dir: bool,
|
||||
}
|
||||
|
||||
impl OpenSnapshot {
|
||||
/// The directory is not removed on drop until the snapshot is in a [`HeapRegistry`] whose
|
||||
/// root holds it.
|
||||
pub fn new(info: HeapSnapshotInfo, index: HeapIndex, dir: PathBuf, dump: PathBuf) -> Self {
|
||||
Self { info, index: Some(index), dir, dump, remove_dir: false }
|
||||
}
|
||||
|
||||
pub fn index(&self) -> &HeapIndex {
|
||||
self.index.as_ref().expect("the index is only taken while dropping")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OpenSnapshot {
|
||||
fn drop(&mut self) {
|
||||
drop(self.index.take());
|
||||
if !self.remove_dir {
|
||||
return;
|
||||
}
|
||||
// The user's own dump that happens to sit inside `dir` (an opened file is analyzed from a
|
||||
// private copy, which goes with the directory).
|
||||
if matches!(&self.info.origin, HeapOrigin::File { path } if **path == self.dump)
|
||||
&& self.dump.starts_with(&self.dir)
|
||||
{
|
||||
return;
|
||||
}
|
||||
// A link is not followed: only a real directory is deleted.
|
||||
if std::fs::symlink_metadata(&self.dir).is_ok_and(|meta| meta.file_type().is_dir())
|
||||
&& let Err(err) = std::fs::remove_dir_all(&self.dir)
|
||||
{
|
||||
tracing::warn!(dir = %self.dir.display(), %err, "could not remove a heap snapshot directory");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct State {
|
||||
next_id: u32,
|
||||
open: BTreeMap<u32, Arc<OpenSnapshot>>,
|
||||
}
|
||||
|
||||
/// The open snapshots, shared between threads. Ids are never reused within a process.
|
||||
pub struct HeapRegistry {
|
||||
root: PathBuf,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
impl HeapRegistry {
|
||||
/// Registry for snapshot directories under `root`; first removes the directories there that
|
||||
/// belong to processes that no longer exist.
|
||||
pub fn new(root: PathBuf) -> Self {
|
||||
remove_stale(&root, process_alive);
|
||||
Self { root, state: Mutex::new(State { next_id: 1, open: BTreeMap::new() }) }
|
||||
}
|
||||
|
||||
pub fn root(&self) -> &Path {
|
||||
&self.root
|
||||
}
|
||||
|
||||
fn state(&self) -> MutexGuard<'_, State> {
|
||||
self.state.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// Registers the snapshot that `make` builds for a fresh id. `make` runs outside the lock.
|
||||
pub fn insert(&self, make: impl FnOnce(HeapSnapshotId) -> OpenSnapshot) -> HeapSnapshotInfo {
|
||||
let id = {
|
||||
let mut state = self.state();
|
||||
let id = state.next_id;
|
||||
state.next_id = id.saturating_add(1);
|
||||
HeapSnapshotId(id)
|
||||
};
|
||||
let mut snapshot = make(id);
|
||||
snapshot.info.id = id;
|
||||
snapshot.remove_dir = snapshot.dir.file_name().is_some() && snapshot.dir.parent() == Some(self.root.as_path());
|
||||
if !snapshot.remove_dir {
|
||||
tracing::warn!(dir = %snapshot.dir.display(), "heap snapshot directory is outside the registry root; keeping it");
|
||||
}
|
||||
let info = snapshot.info.clone();
|
||||
self.state().open.insert(id.0, Arc::new(snapshot));
|
||||
info
|
||||
}
|
||||
|
||||
pub fn get(&self, id: HeapSnapshotId) -> Option<Arc<OpenSnapshot>> {
|
||||
self.state().open.get(&id.0).cloned()
|
||||
}
|
||||
|
||||
/// Forgets the snapshot; its directory goes once the last query using it lets go. `false` for
|
||||
/// an id that is not open.
|
||||
pub fn close(&self, id: HeapSnapshotId) -> bool {
|
||||
// Bound first so the lock is released before the snapshot may be dropped: that deletes files.
|
||||
let removed = self.state().open.remove(&id.0);
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
/// In the order the snapshots were opened.
|
||||
pub fn list(&self) -> Vec<HeapSnapshotInfo> {
|
||||
self.state().open.values().map(|snapshot| snapshot.info.clone()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,265 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use ide_core::{CancellationToken, DebugSessionId};
|
||||
|
||||
use super::*;
|
||||
use crate::analysis::{AnalyzeOptions, analyze};
|
||||
use crate::disk::create_snapshot_dir;
|
||||
use crate::error::HeapError;
|
||||
use crate::progress::HeapPhase;
|
||||
use crate::testing::sample;
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn captured() -> HeapOrigin {
|
||||
HeapOrigin::Captured {
|
||||
session: DebugSessionId(7),
|
||||
pid: 4242,
|
||||
java_version: "21.0.2".into(),
|
||||
taken_while_suspended: true,
|
||||
dump_ms: 35,
|
||||
}
|
||||
}
|
||||
|
||||
fn external(path: &Path) -> HeapOrigin {
|
||||
HeapOrigin::File { path: IpcPath(path.to_path_buf()) }
|
||||
}
|
||||
|
||||
/// A snapshot whose index files are mapped from `dir`, like a real one: `dump` is analyzed in
|
||||
/// place and `dir` receives the columns.
|
||||
fn open_in(dir: &Path, dump: &Path, origin: HeapOrigin) -> impl FnOnce(HeapSnapshotId) -> OpenSnapshot {
|
||||
let dir = dir.to_path_buf();
|
||||
let dump = dump.to_path_buf();
|
||||
move |id| {
|
||||
std::fs::write(&dump, sample().bytes()).unwrap();
|
||||
let index = analyze(&dump, &dir, AnalyzeOptions::default(), &no_progress, &CancellationToken::new()).unwrap();
|
||||
let info = HeapSnapshotInfo {
|
||||
id,
|
||||
label: origin.label(),
|
||||
origin,
|
||||
summary: index.summary.clone(),
|
||||
created_ms: 1_700_000_000_000,
|
||||
};
|
||||
OpenSnapshot::new(info, index, dir, dump)
|
||||
}
|
||||
}
|
||||
|
||||
/// Inserts a captured snapshot with its own directory under the registry's root.
|
||||
fn insert_captured(registry: &HeapRegistry) -> (HeapSnapshotInfo, PathBuf) {
|
||||
let dir = create_snapshot_dir(registry.root()).unwrap();
|
||||
let dump = dir.join("dump.hprof");
|
||||
let info = registry.insert(open_in(&dir, &dump, captured()));
|
||||
(info, dir)
|
||||
}
|
||||
|
||||
fn names(dir: &Path) -> Vec<String> {
|
||||
let mut names: Vec<_> =
|
||||
std::fs::read_dir(dir).unwrap().map(|e| e.unwrap().file_name().into_string().unwrap()).collect();
|
||||
names.sort();
|
||||
names
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn closing_snapshot_removes_dir() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let (info, dir) = insert_captured(®istry);
|
||||
assert!(dir.join("dump.hprof").exists() && dir.join("summary.json").exists());
|
||||
assert!(registry.close(info.id));
|
||||
assert!(!dir.exists());
|
||||
assert!(!registry.close(info.id));
|
||||
assert!(registry.get(info.id).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn external_dump_is_never_deleted() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let outside = tempfile::tempdir().unwrap();
|
||||
let dump = outside.path().join("app.hprof");
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let dir = create_snapshot_dir(registry.root()).unwrap();
|
||||
let info = registry.insert(open_in(&dir, &dump, external(&dump)));
|
||||
assert_eq!(info.label, "app.hprof");
|
||||
assert!(dir.join("summary.json").exists());
|
||||
assert!(registry.close(info.id));
|
||||
assert!(!dir.exists());
|
||||
assert!(dump.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dump_inside_the_dir_of_a_file_snapshot_keeps_the_dir() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let dir = create_snapshot_dir(registry.root()).unwrap();
|
||||
let dump = dir.join("opened.hprof");
|
||||
let info = registry.insert(open_in(&dir, &dump, external(&dump)));
|
||||
assert!(registry.close(info.id));
|
||||
assert!(dump.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dir_outside_the_root_is_never_deleted() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let elsewhere = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let dump = elsewhere.path().join("dump.hprof");
|
||||
let info = registry.insert(open_in(elsewhere.path(), &dump, captured()));
|
||||
assert!(registry.close(info.id));
|
||||
assert!(dump.exists());
|
||||
// Neither the root itself nor a path that climbs out of it counts as a snapshot dir.
|
||||
let sneaky = root.path().join("..").join(elsewhere.path().file_name().unwrap());
|
||||
let info = registry.insert(open_in(&sneaky, &dump, captured()));
|
||||
assert!(registry.close(info.id));
|
||||
assert!(dump.exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_symlinked_dir_is_not_followed() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let outside = tempfile::tempdir().unwrap();
|
||||
let dump = outside.path().join("dump.hprof");
|
||||
let link = root.path().join("link");
|
||||
std::os::unix::fs::symlink(outside.path(), &link).unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let info = registry.insert(open_in(&link, &dump, captured()));
|
||||
assert!(registry.close(info.id));
|
||||
assert!(dump.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ids_are_never_reused() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let (first, _) = insert_captured(®istry);
|
||||
assert!(registry.close(first.id));
|
||||
let (second, _) = insert_captured(®istry);
|
||||
let (third, _) = insert_captured(®istry);
|
||||
assert!(first.id.0 < second.id.0 && second.id.0 < third.id.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_registry_decides_the_id_in_the_info() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let dir = create_snapshot_dir(registry.root()).unwrap();
|
||||
let dump = dir.join("dump.hprof");
|
||||
let make = open_in(&dir, &dump, captured());
|
||||
let info = registry.insert(|id| {
|
||||
let mut snapshot = make(id);
|
||||
snapshot.info.id = HeapSnapshotId(u32::MAX);
|
||||
snapshot
|
||||
});
|
||||
assert_ne!(info.id, HeapSnapshotId(u32::MAX));
|
||||
assert_eq!(registry.get(info.id).unwrap().info.id, info.id);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn list_is_in_opening_order_and_get_shares_the_snapshot() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let (a, _) = insert_captured(®istry);
|
||||
let (b, _) = insert_captured(®istry);
|
||||
let (c, _) = insert_captured(®istry);
|
||||
assert!(registry.close(b.id));
|
||||
assert_eq!(registry.list(), [a.clone(), c.clone()]);
|
||||
let one = registry.get(a.id).unwrap();
|
||||
let two = registry.get(a.id).unwrap();
|
||||
assert!(Arc::ptr_eq(&one, &two));
|
||||
assert_eq!(one.info, a);
|
||||
assert_eq!(one.index().summary, a.summary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_snapshot_in_use_keeps_its_dir_until_the_last_holder_lets_go() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let (info, dir) = insert_captured(®istry);
|
||||
let held = registry.get(info.id).unwrap();
|
||||
assert!(registry.close(info.id));
|
||||
assert!(registry.get(info.id).is_none() && registry.list().is_empty());
|
||||
assert!(dir.exists());
|
||||
drop(held);
|
||||
assert!(!dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_the_registry_removes_the_open_snapshots() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
let (_, dir) = insert_captured(®istry);
|
||||
drop(registry);
|
||||
assert!(!dir.exists());
|
||||
assert!(names(root.path()).is_empty());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn new_clears_dirs_whose_owner_died() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let mut child = std::process::Command::new("true").spawn().unwrap();
|
||||
let dead = child.id();
|
||||
child.wait().unwrap();
|
||||
let dead_dir = root.path().join("dead");
|
||||
std::fs::create_dir(&dead_dir).unwrap();
|
||||
std::fs::write(dead_dir.join("owner"), dead.to_string()).unwrap();
|
||||
let live_dir = create_snapshot_dir(root.path()).unwrap();
|
||||
let registry = HeapRegistry::new(root.path().to_path_buf());
|
||||
assert!(!dead_dir.exists());
|
||||
assert!(live_dir.exists());
|
||||
assert_eq!(registry.root(), root.path());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_inserts_get_distinct_ids() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let registry = Arc::new(HeapRegistry::new(root.path().to_path_buf()));
|
||||
let handles: Vec<_> = (0..6)
|
||||
.map(|_| {
|
||||
let registry = registry.clone();
|
||||
std::thread::spawn(move || (0..3).map(|_| insert_captured(®istry).0.id).collect::<Vec<_>>())
|
||||
})
|
||||
.collect();
|
||||
let mut ids: Vec<_> = handles.into_iter().flat_map(|h| h.join().unwrap()).collect();
|
||||
ids.sort_by_key(|id| id.0);
|
||||
ids.dedup();
|
||||
assert_eq!(ids.len(), 18);
|
||||
assert_eq!(registry.list().len(), 18);
|
||||
for id in ids {
|
||||
assert!(registry.close(id));
|
||||
}
|
||||
assert!(names(root.path()).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn labels_name_the_pid_or_the_file() {
|
||||
assert_eq!(captured().label(), "4242");
|
||||
assert_eq!(external(Path::new("/tmp/dumps/app.hprof")).label(), "app.hprof");
|
||||
assert_eq!(external(Path::new("/")).label(), "/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn origin_and_info_serialize_as_the_ui_expects() {
|
||||
let value = serde_json::to_value(captured()).unwrap();
|
||||
assert_eq!(
|
||||
value,
|
||||
serde_json::json!({
|
||||
"kind": "captured", "session": 7, "pid": 4242, "javaVersion": "21.0.2",
|
||||
"takenWhileSuspended": true, "dumpMs": 35
|
||||
})
|
||||
);
|
||||
assert_eq!(serde_json::to_value(external(Path::new("/a/b.hprof"))).unwrap()["path"], "/a/b.hprof");
|
||||
assert_eq!(serde_json::to_value(HeapSnapshotId(3)).unwrap(), serde_json::json!(3));
|
||||
let back: HeapOrigin = serde_json::from_value(value).unwrap();
|
||||
assert_eq!(back, captured());
|
||||
}
|
||||
|
||||
// `HeapRegistry::get` hands out `Arc<OpenSnapshot>` to query threads.
|
||||
const _: fn() = || {
|
||||
fn check<T: Send + Sync>() {}
|
||||
check::<HeapIndex>();
|
||||
check::<OpenSnapshot>();
|
||||
check::<HeapRegistry>();
|
||||
};
|
||||
@@ -0,0 +1,265 @@
|
||||
//! Shallow sizes: the bytes each object takes in the JVM heap, from the object layout rules, the
|
||||
//! JVM's own class histogram, or the distances between object addresses in the dump.
|
||||
|
||||
mod layout;
|
||||
mod mirrors;
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub use self::layout::VmLayout;
|
||||
use self::mirrors::{Mirrors, injected_by_table};
|
||||
use crate::error::HeapError;
|
||||
use crate::graph::{ClassInfo, HeapGraph};
|
||||
use crate::hprof::BasicType;
|
||||
use crate::names::display_name;
|
||||
use crate::progress::CHECK_EVERY;
|
||||
|
||||
/// How far past the layout size an address gap may be and still count as an object size.
|
||||
const GAP_SLACK: u64 = 64;
|
||||
/// Share of the instance types that can have a gap (two or more instances), in tenths, that must
|
||||
/// take their size from address gaps for the sizes to be reported as coming from them.
|
||||
const GAP_TRUST_TENTHS: usize = 9;
|
||||
|
||||
/// Where the sizes of instances came from.
|
||||
///
|
||||
/// Class objects (mirrors) are approximate in every case: HotSpot leaves the fields it injects
|
||||
/// into `java.lang.Class` out of the dump, so their layout size is short by a fixed amount per
|
||||
/// mirror. With a histogram that amount is fitted to its `java.lang.Class` row; otherwise (or when
|
||||
/// the fit is rejected) it comes from a table measured per JDK and reference size, within a few
|
||||
/// bytes per mirror, or is 32 when the JDK is unknown.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, specta::Type)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum SizeSource {
|
||||
/// The JVM's class histogram from the capture.
|
||||
Jvm,
|
||||
/// The smallest distance between neighbouring objects of a type, in a dump the IDE did not
|
||||
/// capture.
|
||||
AddressGaps,
|
||||
Layout,
|
||||
}
|
||||
|
||||
/// A row of the JVM's class histogram.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct HistogramEntry {
|
||||
/// Display form: `java.lang.String`, `int[]`, `Gen$$Lambda/0x…`.
|
||||
pub name: String,
|
||||
pub instances: u64,
|
||||
pub bytes: u64,
|
||||
}
|
||||
|
||||
enum TypeSize {
|
||||
Fixed(u64),
|
||||
ObjectArray,
|
||||
/// Class objects are sized one by one; this is for an instance of `java.lang.Class` that is
|
||||
/// not a class object.
|
||||
ClassObject(u64),
|
||||
}
|
||||
|
||||
/// Shallow size of every object of a [`HeapGraph`]. Instances of a type share one size; arrays
|
||||
/// follow the layout; class objects add their statics to the size of `java.lang.Class`.
|
||||
pub struct ShallowModel {
|
||||
pub source: SizeSource,
|
||||
layout: VmLayout,
|
||||
/// Per type slot of a class; primitive array slots follow and have no entry.
|
||||
types: Vec<TypeSize>,
|
||||
/// `(node, size)` of each class object, by node.
|
||||
class_objects: Vec<(u32, u64)>,
|
||||
/// Per class slot: another class has the same display name.
|
||||
ambiguous: Vec<bool>,
|
||||
}
|
||||
|
||||
impl ShallowModel {
|
||||
/// `histogram` is the JVM's class histogram of the same stop as the dump, when the IDE captured
|
||||
/// it. Without it, sizes come from address gaps where they agree with the layout.
|
||||
///
|
||||
/// A name shared by several classes (several class loaders) never takes a histogram size,
|
||||
/// since the name is the only link between the dump and the histogram; see
|
||||
/// [`Self::is_ambiguous`]. `java.lang.Class` never does either, because mirror sizes vary with
|
||||
/// their statics; the histogram only tells how many bytes HotSpot adds to each mirror.
|
||||
pub fn new(
|
||||
graph: &HeapGraph,
|
||||
layout: VmLayout,
|
||||
histogram: Option<&[HistogramEntry]>,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<Self, HeapError> {
|
||||
if cancel.is_cancelled() {
|
||||
return Err(HeapError::Cancelled);
|
||||
}
|
||||
let classes = &graph.classes;
|
||||
let field_bytes = field_bytes(classes, layout);
|
||||
let by_layout: Vec<u64> = field_bytes.iter().map(|&bytes| layout.instance_size(bytes)).collect();
|
||||
let names: Vec<String> = classes.iter().map(|c| display_name(&c.name)).collect();
|
||||
let ambiguous = ambiguous_names(&names);
|
||||
|
||||
let class_slot = graph.class_slot as usize;
|
||||
let mirrors = Mirrors::collect(graph, by_layout[class_slot], layout);
|
||||
let mut fitted = None;
|
||||
if let Some(rows) = histogram
|
||||
&& !ambiguous[class_slot]
|
||||
{
|
||||
let mut class_rows = rows.iter().filter(|row| row.name == names[class_slot]);
|
||||
if let (Some(row), None) = (class_rows.next(), class_rows.next()) {
|
||||
fitted = mirrors.fit_injected(graph, layout, by_layout[class_slot], row, cancel)?;
|
||||
}
|
||||
}
|
||||
let injected = fitted.unwrap_or_else(|| injected_by_table(layout));
|
||||
|
||||
let mut types: Vec<TypeSize> = classes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(slot, class)| {
|
||||
if class.name.starts_with('[') {
|
||||
TypeSize::ObjectArray
|
||||
} else if slot == class_slot {
|
||||
TypeSize::ClassObject(layout.align(by_layout[slot] + injected))
|
||||
} else {
|
||||
TypeSize::Fixed(by_layout[slot])
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
let source = match histogram {
|
||||
Some(rows) => {
|
||||
apply_histogram(&mut types, &names, &ambiguous, rows);
|
||||
SizeSource::Jvm
|
||||
}
|
||||
None => apply_address_gaps(&mut types, graph, &by_layout, cancel)?,
|
||||
};
|
||||
Ok(Self { source, layout, types, class_objects: mirrors.sizes(layout, injected), ambiguous })
|
||||
}
|
||||
|
||||
/// Shallow size of `object` (a node of the graph); 0 for the super-root.
|
||||
pub fn size_of(&self, graph: &HeapGraph, object: u32) -> u64 {
|
||||
let Some(&slot) = graph.types.get(object as usize) else { return 0 };
|
||||
match self.types.get(slot as usize) {
|
||||
Some(&TypeSize::Fixed(size)) => size,
|
||||
Some(TypeSize::ObjectArray) => self.array_size(BasicType::Object, graph.lens[object as usize]),
|
||||
Some(&TypeSize::ClassObject(fallback)) => self
|
||||
.class_objects
|
||||
.binary_search_by_key(&object, |&(node, _)| node)
|
||||
.map_or(fallback, |at| self.class_objects[at].1),
|
||||
None => match graph.prim_type(slot) {
|
||||
Some(ty) => self.array_size(ty, graph.lens[object as usize]),
|
||||
None => 0,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether another class has the same display name as the class in `slot`. Sizes of such
|
||||
/// classes come from the layout, and anything joined to them by name (histogram rows, a
|
||||
/// second snapshot) cannot tell them apart.
|
||||
pub fn is_ambiguous(&self, slot: u32) -> bool {
|
||||
self.ambiguous.get(slot as usize).copied().unwrap_or(false)
|
||||
}
|
||||
|
||||
fn array_size(&self, ty: BasicType, len: u32) -> u64 {
|
||||
self.layout.array_size(ty, u64::from(len))
|
||||
}
|
||||
}
|
||||
|
||||
/// Bytes of instance fields per class, its superclasses included. Super chains always end
|
||||
/// (`ClassInfo::super_index` has its cycles cut).
|
||||
fn field_bytes(classes: &[ClassInfo], layout: VmLayout) -> Vec<u64> {
|
||||
let mut total: Vec<Option<u64>> = vec![None; classes.len()];
|
||||
let mut path = Vec::new();
|
||||
for start in 0..classes.len() {
|
||||
let mut cursor = Some(start);
|
||||
let mut below = 0;
|
||||
while let Some(c) = cursor {
|
||||
if let Some(known) = total[c] {
|
||||
below = known;
|
||||
break;
|
||||
}
|
||||
path.push(c);
|
||||
cursor = classes[c].super_index.map(|s| s as usize);
|
||||
}
|
||||
while let Some(c) = path.pop() {
|
||||
let own: u64 = classes[c].fields.iter().map(|(_, ty)| ty.heap_size(layout.ref_size)).sum();
|
||||
below += own;
|
||||
total[c] = Some(below);
|
||||
}
|
||||
}
|
||||
total.into_iter().map(Option::unwrap_or_default).collect()
|
||||
}
|
||||
|
||||
fn ambiguous_names(names: &[String]) -> Vec<bool> {
|
||||
let mut counts: HashMap<&str, u32> = HashMap::with_capacity(names.len());
|
||||
for name in names {
|
||||
*counts.entry(name).or_default() += 1;
|
||||
}
|
||||
names.iter().map(|name| counts[name.as_str()] > 1).collect()
|
||||
}
|
||||
|
||||
/// `bytes / instances` for each plain class with a histogram row of its name. A name that is
|
||||
/// ambiguous in the dump or repeated in the histogram, or a row that is not a whole number of
|
||||
/// objects, keeps its layout size.
|
||||
fn apply_histogram(types: &mut [TypeSize], names: &[String], ambiguous: &[bool], rows: &[HistogramEntry]) {
|
||||
let mut per_object: HashMap<&str, Option<u64>> = HashMap::with_capacity(rows.len());
|
||||
for row in rows {
|
||||
let size = row.bytes.checked_div(row.instances).filter(|&s| s > 0 && row.bytes % row.instances == 0);
|
||||
per_object.entry(&row.name).and_modify(|seen| *seen = None).or_insert(size);
|
||||
}
|
||||
for (slot, ty) in types.iter_mut().enumerate() {
|
||||
if let TypeSize::Fixed(size) = ty
|
||||
&& !ambiguous[slot]
|
||||
&& let Some(&Some(from_histogram)) = per_object.get(names[slot].as_str())
|
||||
{
|
||||
*size = from_histogram;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Takes the smallest distance from an instance to the next object in address order as the size
|
||||
/// of its type, for types with two or more instances where that distance is no smaller than the
|
||||
/// layout size and at most [`GAP_SLACK`] bigger. The result says [`SizeSource::AddressGaps`] when
|
||||
/// nine in ten of the types with two or more instances did; types with one instance have no gap
|
||||
/// to measure, so they neither count for nor against.
|
||||
fn apply_address_gaps(
|
||||
types: &mut [TypeSize],
|
||||
graph: &HeapGraph,
|
||||
by_layout: &[u64],
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<SizeSource, HeapError> {
|
||||
let slots = types.len();
|
||||
let mut instances = vec![0u32; slots];
|
||||
let mut smallest_gap = vec![u64::MAX; slots];
|
||||
let mut ids = graph.ids.ids();
|
||||
let mut current = ids.next();
|
||||
let mut object = 0usize;
|
||||
while let Some(id) = current {
|
||||
if (object as u64).is_multiple_of(CHECK_EVERY) && cancel.is_cancelled() {
|
||||
return Err(HeapError::Cancelled);
|
||||
}
|
||||
let next = ids.next();
|
||||
let slot = graph.types[object] as usize;
|
||||
if matches!(types.get(slot), Some(TypeSize::Fixed(_))) {
|
||||
instances[slot] = instances[slot].saturating_add(1);
|
||||
if let Some(next) = next.filter(|&n| n > id) {
|
||||
smallest_gap[slot] = smallest_gap[slot].min(next - id);
|
||||
}
|
||||
}
|
||||
current = next;
|
||||
object += 1;
|
||||
}
|
||||
|
||||
let (mut measurable, mut by_gap) = (0usize, 0usize);
|
||||
for (slot, ty) in types.iter_mut().enumerate() {
|
||||
let TypeSize::Fixed(size) = ty else { continue };
|
||||
if instances[slot] < 2 {
|
||||
continue;
|
||||
}
|
||||
measurable += 1;
|
||||
let gap = smallest_gap[slot];
|
||||
if (by_layout[slot]..=by_layout[slot] + GAP_SLACK).contains(&gap) {
|
||||
*size = gap;
|
||||
by_gap += 1;
|
||||
}
|
||||
}
|
||||
let trusted = measurable > 0 && by_gap * 10 >= measurable * GAP_TRUST_TENTHS;
|
||||
Ok(if trusted { SizeSource::AddressGaps } else { SizeSource::Layout })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,113 @@
|
||||
//! The object layout rules of the JVM that wrote a dump.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::hprof::BasicType;
|
||||
|
||||
/// Object layout of the JVM that wrote the dump.
|
||||
#[derive(Clone, Copy, Debug, PartialEq)]
|
||||
pub struct VmLayout {
|
||||
pub header: u8,
|
||||
/// Bytes of a reference: 4 with compressed oops.
|
||||
pub ref_size: u8,
|
||||
/// Offset of the length field of an array.
|
||||
pub length_offset: u8,
|
||||
/// JDK 21 and older put array elements at the next 8-byte boundary after the length, later
|
||||
/// JDKs at the next boundary of the element size.
|
||||
pub legacy_array_base: bool,
|
||||
pub alignment: u16,
|
||||
/// The JDK the flags came from; `None` when nothing is known about the JVM.
|
||||
pub java_major: Option<u32>,
|
||||
}
|
||||
|
||||
impl VmLayout {
|
||||
pub const DEFAULT: VmLayout = VmLayout {
|
||||
header: 12,
|
||||
ref_size: 4,
|
||||
length_offset: 12,
|
||||
legacy_array_base: false,
|
||||
alignment: 8,
|
||||
java_major: None,
|
||||
};
|
||||
|
||||
/// `flags` is the output of `VM.flags`, which lists the flags set on this run. Compressed oops
|
||||
/// are off by default and only switched on by ergonomics, so `-XX:+UseCompressedOops` is
|
||||
/// printed exactly when references take 4 bytes; a heap too big for them prints no such flag.
|
||||
/// Before JDK 15 compressed class pointers follow the same rule; with `java_major: None` (JDK
|
||||
/// unknown) they follow the JDK 15+ rule and the array base that of the newest JDKs. Flags
|
||||
/// without any `-XX:` option (an external dump) give [`Self::DEFAULT`] with the array base of
|
||||
/// `java_major` and an unknown JDK, since the reference size is a guess.
|
||||
/// An `ObjectAlignmentInBytes` that is not a power of two in `8..=256` is ignored.
|
||||
pub fn from_vm_flags(flags: &str, java_major: Option<u32>) -> Self {
|
||||
let legacy_array_base = java_major.map_or(Self::DEFAULT.legacy_array_base, |major| major <= 21);
|
||||
let flags = parse_flags(flags);
|
||||
if flags.is_empty() {
|
||||
return VmLayout { legacy_array_base, ..Self::DEFAULT };
|
||||
}
|
||||
let on = |name: &str| flags.get(name) == Some(&"true");
|
||||
let compressed_classes = if java_major.is_none_or(|major| major >= 15) {
|
||||
flags.get("UseCompressedClassPointers") != Some(&"false")
|
||||
} else {
|
||||
on("UseCompressedClassPointers")
|
||||
};
|
||||
let (header, length_offset) = if on("UseCompactObjectHeaders") {
|
||||
(8, 8)
|
||||
} else if compressed_classes {
|
||||
(12, 12)
|
||||
} else {
|
||||
(16, 16)
|
||||
};
|
||||
let alignment = flags
|
||||
.get("ObjectAlignmentInBytes")
|
||||
.and_then(|v| v.parse::<u16>().ok())
|
||||
.filter(|a| (8..=256).contains(a) && a.is_power_of_two())
|
||||
.unwrap_or(Self::DEFAULT.alignment);
|
||||
VmLayout {
|
||||
header,
|
||||
ref_size: if on("UseCompressedOops") { 4 } else { 8 },
|
||||
length_offset,
|
||||
legacy_array_base,
|
||||
alignment,
|
||||
java_major,
|
||||
}
|
||||
}
|
||||
|
||||
/// Size of an instance whose fields (its class and every superclass) take `field_bytes`.
|
||||
pub fn instance_size(&self, field_bytes: u64) -> u64 {
|
||||
let smallest = if self.header == 8 { 8 } else { 16 };
|
||||
self.align(u64::from(self.header) + field_bytes).max(smallest)
|
||||
}
|
||||
|
||||
pub fn array_size(&self, ty: BasicType, len: u64) -> u64 {
|
||||
let element = ty.heap_size(self.ref_size);
|
||||
let base_alignment = if self.legacy_array_base { 8 } else { element };
|
||||
let base = align_to(u64::from(self.length_offset) + 4, base_alignment);
|
||||
self.align(base + element * len)
|
||||
}
|
||||
|
||||
/// `value` rounded up to the object alignment.
|
||||
pub(super) fn align(&self, value: u64) -> u64 {
|
||||
align_to(value, u64::from(self.alignment))
|
||||
}
|
||||
}
|
||||
|
||||
fn align_to(value: u64, to: u64) -> u64 {
|
||||
value.div_ceil(to.max(1)) * to.max(1)
|
||||
}
|
||||
|
||||
/// `-XX:+Name` / `-XX:-Name` / `-XX:Name=value` tokens by name; booleans become `"true"` / `"false"`.
|
||||
/// A flag given twice keeps its last value, as in the JVM.
|
||||
fn parse_flags(flags: &str) -> HashMap<&str, &str> {
|
||||
let mut parsed = HashMap::new();
|
||||
for token in flags.split_whitespace() {
|
||||
let Some(body) = token.strip_prefix("-XX:") else { continue };
|
||||
if let Some(name) = body.strip_prefix('+') {
|
||||
parsed.insert(name, "true");
|
||||
} else if let Some(name) = body.strip_prefix('-') {
|
||||
parsed.insert(name, "false");
|
||||
} else if let Some((name, value)) = body.split_once('=') {
|
||||
parsed.insert(name, value);
|
||||
}
|
||||
}
|
||||
parsed
|
||||
}
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Sizes of class objects (mirrors): the `java.lang.Class` instance plus the class's statics, plus
|
||||
//! the fields HotSpot injects into every mirror and leaves out of the class dump.
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use super::{HistogramEntry, VmLayout};
|
||||
use crate::error::HeapError;
|
||||
use crate::graph::HeapGraph;
|
||||
use crate::progress::CHECK_EVERY;
|
||||
|
||||
/// Largest injected byte count (per mirror, a multiple of 4) the histogram fit tries.
|
||||
const MAX_INJECTED: u64 = 128;
|
||||
/// How far, in percent of the histogram's total, the best fit may be off before the row is taken
|
||||
/// for something else than the mirrors of this dump.
|
||||
const MAX_FIT_ERROR_PERCENT: u128 = 5;
|
||||
|
||||
/// Bytes HotSpot injects into every mirror (`klass`, `array_klass`, `oop_size`,
|
||||
/// `static_oop_field_count`, and oops such as `protection_domain`) without listing them in the class
|
||||
/// dump, as measured by comparing the dump with `GC.class_histogram` on JDK 8, 11, 21, 23 and 25,
|
||||
/// with and without compressed oops. Each X is the best fit in steps of 4; the real value is
|
||||
/// within a few bytes (statics padding). `None` for a reference size the table does not cover.
|
||||
pub(super) fn injected_mirror_bytes(java_major: u32, ref_size: u8) -> Option<u64> {
|
||||
match (ref_size, java_major) {
|
||||
// 8, 11, 21 and 23 measure 36; 25 is the first JDK that shows 28.
|
||||
(4, ..=24) => Some(36),
|
||||
(4, 25..) => Some(28),
|
||||
// 8, 11 and 21 measure 44; 23 is the first to show 40 and 25 the first to show 32.
|
||||
(8, ..=22) => Some(44),
|
||||
(8, 23..=24) => Some(40),
|
||||
(8, 25..) => Some(32),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Used when the JVM is unknown (a dump without `VM.flags`): the midpoint of the measured values
|
||||
/// with 4-byte references, 36 and 28.
|
||||
const UNKNOWN_JVM_INJECTED: u64 = 32;
|
||||
|
||||
/// The injected bytes per mirror the layout implies, without looking at a histogram.
|
||||
pub(super) fn injected_by_table(layout: VmLayout) -> u64 {
|
||||
layout.java_major.and_then(|major| injected_mirror_bytes(major, layout.ref_size)).unwrap_or(UNKNOWN_JVM_INJECTED)
|
||||
}
|
||||
|
||||
pub(super) struct Mirrors {
|
||||
/// `(node, size before alignment)` of every class dump that is an object of the graph, by node.
|
||||
dumped: Vec<(u32, u64)>,
|
||||
}
|
||||
|
||||
impl Mirrors {
|
||||
/// `instance` is the layout size of `java.lang.Class`, the base of every mirror.
|
||||
pub(super) fn collect(graph: &HeapGraph, instance: u64, layout: VmLayout) -> Self {
|
||||
let mut dumped: Vec<(u32, u64)> = graph
|
||||
.classes
|
||||
.iter()
|
||||
.filter(|class| class.id != 0)
|
||||
.filter_map(|class| {
|
||||
let node = graph.ids.rank(class.id)?;
|
||||
let references = class.statics.len() as u64 * u64::from(layout.ref_size);
|
||||
Some((node, instance + class.static_bytes + references))
|
||||
})
|
||||
.collect();
|
||||
dumped.sort_unstable_by_key(|&(node, _)| node);
|
||||
Self { dumped }
|
||||
}
|
||||
|
||||
/// `(node, size)` of each class object when HotSpot adds `injected` bytes to every mirror.
|
||||
pub(super) fn sizes(&self, layout: VmLayout, injected: u64) -> Vec<(u32, u64)> {
|
||||
self.dumped.iter().map(|&(node, bytes)| (node, layout.align(bytes + injected))).collect()
|
||||
}
|
||||
|
||||
/// The injected bytes per mirror (a multiple of 4, at most 128) that make the mirrors of this
|
||||
/// dump add up to `class_row`'s share of the histogram, or `None` when no value gets within 5%:
|
||||
/// a histogram of other mirrors than these (a different JVM stop) must not skew the sizes.
|
||||
///
|
||||
/// The fit compares averages, since the histogram also counts mirrors the dump has no class
|
||||
/// dump for (primitive types, classes only archived by CDS). Every mirror still gets the same
|
||||
/// extra bytes, never the average size, because statics make mirror sizes differ.
|
||||
pub(super) fn fit_injected(
|
||||
&self,
|
||||
graph: &HeapGraph,
|
||||
layout: VmLayout,
|
||||
instance: u64,
|
||||
class_row: &HistogramEntry,
|
||||
cancel: &CancellationToken,
|
||||
) -> Result<Option<u64>, HeapError> {
|
||||
if class_row.instances == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
// Mirrors without a class dump (primitive types) have no statics.
|
||||
let class_typed = count_slot(graph, graph.class_slot, cancel)?;
|
||||
let bare = class_typed.saturating_sub(self.dumped.len() as u64);
|
||||
let mirrors = self.dumped.len() as u64 + bare;
|
||||
if mirrors == 0 {
|
||||
return Ok(None);
|
||||
}
|
||||
let target = u128::from(class_row.bytes) * u128::from(mirrors) / u128::from(class_row.instances);
|
||||
let total = |injected: u64| -> u128 {
|
||||
let dumped: u128 = self.dumped.iter().map(|&(_, bytes)| u128::from(layout.align(bytes + injected))).sum();
|
||||
dumped + u128::from(bare) * u128::from(layout.align(instance + injected))
|
||||
};
|
||||
let (best, error) = (0..=MAX_INJECTED)
|
||||
.step_by(4)
|
||||
.map(|injected| (injected, total(injected).abs_diff(target)))
|
||||
.min_by_key(|&(injected, error)| (error, injected))
|
||||
.unwrap_or((0, u128::MAX));
|
||||
Ok((error * 100 <= target * MAX_FIT_ERROR_PERCENT).then_some(best))
|
||||
}
|
||||
}
|
||||
|
||||
fn count_slot(graph: &HeapGraph, slot: u32, cancel: &CancellationToken) -> Result<u64, HeapError> {
|
||||
let mut count = 0;
|
||||
for chunk in graph.types.chunks(CHECK_EVERY as usize) {
|
||||
if cancel.is_cancelled() {
|
||||
return Err(HeapError::Cancelled);
|
||||
}
|
||||
count += chunk.iter().filter(|&&ty| ty == slot).count() as u64;
|
||||
}
|
||||
Ok(count)
|
||||
}
|
||||
@@ -0,0 +1,593 @@
|
||||
use super::*;
|
||||
use crate::column::Store;
|
||||
use crate::graph::build_graph;
|
||||
use crate::progress::HeapPhase;
|
||||
use crate::testing::{HprofWriter, MiniHeap, sample};
|
||||
|
||||
const ROOT: Option<&str> = Some("java.lang.Object");
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn graph_of(heap: &MiniHeap) -> HeapGraph {
|
||||
build_graph(&heap.bytes(), &Store::Ram, &no_progress, &CancellationToken::new()).unwrap()
|
||||
}
|
||||
|
||||
fn model(g: &HeapGraph, layout: VmLayout, histogram: Option<&[HistogramEntry]>) -> ShallowModel {
|
||||
ShallowModel::new(g, layout, histogram, &CancellationToken::new()).unwrap()
|
||||
}
|
||||
|
||||
fn size_at(g: &HeapGraph, m: &ShallowModel, addr: u64) -> u64 {
|
||||
m.size_of(g, g.ids.rank(addr).unwrap_or_else(|| panic!("{addr:#x} is not in the graph")))
|
||||
}
|
||||
|
||||
fn slot_at(g: &HeapGraph, addr: u64) -> u32 {
|
||||
g.types[g.ids.rank(addr).unwrap() as usize]
|
||||
}
|
||||
|
||||
fn entry(name: &str, instances: u64, bytes: u64) -> HistogramEntry {
|
||||
HistogramEntry { name: name.to_owned(), instances, bytes }
|
||||
}
|
||||
|
||||
fn flags(flags: &str, java_major: u32) -> VmLayout {
|
||||
VmLayout::from_vm_flags(flags, Some(java_major))
|
||||
}
|
||||
|
||||
const SHAPES: [&str; 9] = ["A", "B", "C", "D", "E", "F", "G", "H", "Node"];
|
||||
|
||||
/// One instance of each shape, 0x100 apart so no address gap qualifies.
|
||||
fn shapes() -> MiniHeap {
|
||||
use BasicType::{Byte, Int, Long, Object, Short};
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("A", ROOT, &[("f", Byte)])
|
||||
.class("B", ROOT, &[("f", Long)])
|
||||
.class("C", ROOT, &[("i", Int), ("l", Long), ("b", Byte)])
|
||||
.class("D", Some("C"), &[("s", Short), ("o", Object)])
|
||||
.class("E", ROOT, &[("a", Object), ("b", Object), ("c", Object)])
|
||||
.class("F", Some("A"), &[("g", Byte)])
|
||||
.class("G", ROOT, &[])
|
||||
.class("H", Some("B"), &[("i", Int)])
|
||||
.class("Node", ROOT, &[("l", Long), ("a", Object), ("b", Object)]);
|
||||
for (i, name) in SHAPES.iter().enumerate() {
|
||||
heap.object(0x1000 + i as u64 * 0x100, name, &[]);
|
||||
}
|
||||
heap
|
||||
}
|
||||
|
||||
fn shape_sizes(layout: VmLayout) -> Vec<u64> {
|
||||
let g = graph_of(&shapes());
|
||||
let m = model(&g, layout, None);
|
||||
(0..SHAPES.len() as u64).map(|i| size_at(&g, &m, 0x1000 + i * 0x100)).collect()
|
||||
}
|
||||
|
||||
// `jcmd <pid> VM.flags` of `-Xmx1g` and `-Xmx40g` runs (the flags that matter here).
|
||||
const J8_11_1G: &str = "-XX:MaxHeapSize=1073741824 -XX:+UseCompressedClassPointers -XX:+UseCompressedOops";
|
||||
const J8_11_40G: &str = "-XX:MaxHeapSize=42949672960";
|
||||
const J21_25_1G: &str = "-XX:MaxHeapSize=1073741824 -XX:SoftMaxHeapSize=1073741824 -XX:+UseCompressedOops";
|
||||
const J21_25_40G: &str = "-XX:MaxHeapSize=42949672960 -XX:SoftMaxHeapSize=42949672960";
|
||||
/// `-Xmx40g -XX:+UseCompressedOops`: the JVM warns and turns them off.
|
||||
const J21_40G_OFF: &str = "-XX:MaxHeapSize=42949672960 -XX:SoftMaxHeapSize=42949672960 -XX:-UseCompressedOops";
|
||||
const J25_COMPACT: &str = "-XX:MaxHeapSize=1073741824 -XX:+UseCompressedOops -XX:+UseCompactObjectHeaders";
|
||||
|
||||
#[test]
|
||||
fn layout_from_flags() {
|
||||
// The layout of a JVM of `major` with default flags otherwise.
|
||||
let jvm = |major: u32, layout: VmLayout| VmLayout { java_major: Some(major), ..layout };
|
||||
let legacy = VmLayout { legacy_array_base: true, ..VmLayout::DEFAULT };
|
||||
assert_eq!(flags(J8_11_1G, 8), jvm(8, legacy));
|
||||
assert_eq!(flags(J8_11_1G, 11), jvm(11, legacy));
|
||||
assert_eq!(flags(J21_25_1G, 21), jvm(21, legacy));
|
||||
assert_eq!(flags(J21_25_1G, 25), jvm(25, VmLayout::DEFAULT));
|
||||
assert_eq!(flags(J25_COMPACT, 25), jvm(25, VmLayout { header: 8, length_offset: 8, ..VmLayout::DEFAULT }));
|
||||
assert_eq!(flags(&format!("{J21_25_1G} -XX:+UseG1GC"), 21), jvm(21, legacy));
|
||||
assert_eq!(
|
||||
flags(&format!("{J21_25_1G} -XX:ObjectAlignmentInBytes=16"), 21),
|
||||
jvm(21, VmLayout { alignment: 16, ..legacy })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heaps_too_big_for_compressed_oops_print_no_flag() {
|
||||
let jvm = |major: u32, layout: VmLayout| VmLayout { java_major: Some(major), ..layout };
|
||||
let wide_legacy =
|
||||
VmLayout { header: 16, ref_size: 8, length_offset: 16, legacy_array_base: true, ..VmLayout::DEFAULT };
|
||||
assert_eq!(flags(J8_11_40G, 8), jvm(8, wide_legacy));
|
||||
assert_eq!(flags(J8_11_40G, 11), jvm(11, wide_legacy));
|
||||
// From JDK 15 compressed class pointers stay on without compressed oops.
|
||||
let wide = VmLayout { ref_size: 8, ..VmLayout::DEFAULT };
|
||||
assert_eq!(flags(J21_25_40G, 21), jvm(21, VmLayout { legacy_array_base: true, ..wide }));
|
||||
assert_eq!(flags(J21_25_40G, 25), jvm(25, wide));
|
||||
assert_eq!(flags(J21_40G_OFF, 21), flags(J21_25_40G, 21));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn flags_of_an_unknown_jdk_follow_the_jdk_15_class_pointer_rule() {
|
||||
let unknown = |f: &str| VmLayout::from_vm_flags(f, None);
|
||||
assert_eq!(unknown(J21_25_1G), VmLayout::DEFAULT);
|
||||
assert_eq!(unknown(J8_11_40G), VmLayout { ref_size: 8, ..VmLayout::DEFAULT });
|
||||
let no_ccp = unknown("-XX:+UseCompressedOops -XX:-UseCompressedClassPointers");
|
||||
assert_eq!((no_ccp.header, no_ccp.length_offset, no_ccp.java_major), (16, 16, None));
|
||||
assert_eq!(unknown(J25_COMPACT).header, 8);
|
||||
// The mirror table has no row for an unknown JDK.
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
assert_eq!(mirror_sizes(&g, &addrs, unknown(J21_25_1G), None), expected_mirrors(&MIRROR_STATICS, 32));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layout_flag_edge_cases() {
|
||||
let legacy = VmLayout { legacy_array_base: true, ..VmLayout::DEFAULT };
|
||||
// No VM.flags (an external dump): the default layout, of an unknown JVM.
|
||||
assert_eq!(flags("", 25), VmLayout::DEFAULT);
|
||||
assert_eq!(flags("", 21), legacy);
|
||||
assert_eq!(flags("-Xmx1g --add-opens=x -XX:Foo", 25), VmLayout::DEFAULT);
|
||||
// Before JDK 15 compressed class pointers are on only when the flags say so.
|
||||
assert_eq!(flags("-XX:+UseCompressedOops", 11).header, 16);
|
||||
assert_eq!(flags("-XX:+UseCompressedOops -XX:+UseCompressedClassPointers", 11).header, 12);
|
||||
assert_eq!(flags("-XX:+UseCompressedOops -XX:-UseCompressedClassPointers", 11).header, 16);
|
||||
assert_eq!(flags("-XX:+UseCompressedOops", 15).header, 12);
|
||||
let no_ccp = flags("-XX:+UseCompressedOops -XX:-UseCompressedClassPointers", 21);
|
||||
assert_eq!((no_ccp.header, no_ccp.length_offset, no_ccp.ref_size), (16, 16, 4));
|
||||
// Compact headers win over the class pointer flag; the last mention of a flag wins.
|
||||
assert_eq!(flags("-XX:-UseCompressedClassPointers -XX:+UseCompactObjectHeaders", 25).header, 8);
|
||||
assert_eq!(flags("-XX:-UseCompressedOops -XX:+UseCompressedOops", 21).ref_size, 4);
|
||||
assert_eq!(flags("-XX:+UseCompressedOops -XX:-UseCompressedOops", 21).ref_size, 8);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_alignment_accepts_powers_of_two_from_8_to_256() {
|
||||
for good in [8u16, 16, 32, 64, 128, 256] {
|
||||
assert_eq!(flags(&format!("-XX:ObjectAlignmentInBytes={good}"), 25).alignment, good);
|
||||
}
|
||||
for bad in ["0", "4", "7", "12", "255", "512", "abc", "", "999999999999"] {
|
||||
assert_eq!(flags(&format!("-XX:ObjectAlignmentInBytes={bad}"), 25).alignment, 8, "{bad:?}");
|
||||
}
|
||||
let wide = flags("-XX:+UseCompressedOops -XX:ObjectAlignmentInBytes=256", 21);
|
||||
assert_eq!(wide.instance_size(1), 256);
|
||||
assert_eq!(wide.instance_size(300), 512);
|
||||
assert_eq!(wide.array_size(BasicType::Byte, 1), 256);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn instance_sizes_match_the_jvm() {
|
||||
let at = |sizes: &[u64], name: &str| sizes[SHAPES.iter().position(|n| *n == name).unwrap()];
|
||||
|
||||
let coops = shape_sizes(flags(J21_25_1G, 21));
|
||||
for (name, size) in
|
||||
[("A", 16), ("B", 24), ("C", 32), ("D", 32), ("E", 24), ("F", 16), ("G", 16), ("H", 24), ("Node", 32)]
|
||||
{
|
||||
assert_eq!(at(&coops, name), size, "{name} with compressed oops");
|
||||
}
|
||||
let wide = shape_sizes(flags(J21_25_40G, 21));
|
||||
for (name, size) in [("D", 40), ("E", 40), ("Node", 40)] {
|
||||
assert_eq!(at(&wide, name), size, "{name} without compressed oops");
|
||||
}
|
||||
let compact = shape_sizes(flags(J25_COMPACT, 25));
|
||||
for (name, size) in [("G", 8), ("B", 16), ("C", 24), ("D", 32), ("Node", 24)] {
|
||||
assert_eq!(at(&compact, name), size, "{name} with compact headers");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn instance_size_bounds() {
|
||||
assert_eq!(VmLayout::DEFAULT.instance_size(0), 16);
|
||||
assert_eq!(flags("-XX:+UseCompactObjectHeaders", 25).instance_size(0), 8);
|
||||
assert_eq!(flags("-XX:ObjectAlignmentInBytes=16", 21).instance_size(1), 16);
|
||||
assert_eq!(flags("-XX:ObjectAlignmentInBytes=16", 21).instance_size(8), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn array_sizes() {
|
||||
let d = VmLayout::DEFAULT;
|
||||
assert_eq!(d.array_size(BasicType::Long, 2), 32);
|
||||
assert_eq!(d.array_size(BasicType::Byte, 5), 24);
|
||||
assert_eq!(d.array_size(BasicType::Object, 3), 32);
|
||||
let compact = flags("-XX:+UseCompactObjectHeaders", 25);
|
||||
assert_eq!(compact.array_size(BasicType::Byte, 5), 24);
|
||||
assert_eq!(compact.array_size(BasicType::Int, 1), 16);
|
||||
assert_eq!(compact.array_size(BasicType::Long, 1), 24);
|
||||
assert_eq!(flags("-XX:-UseCompressedClassPointers", 21).array_size(BasicType::Byte, 1), 32);
|
||||
assert_eq!(flags("-XX:-UseCompressedClassPointers", 25).array_size(BasicType::Byte, 1), 24);
|
||||
assert_eq!(flags("-XX:-UseCompressedClassPointers", 25).array_size(BasicType::Long, 1), 32);
|
||||
// Empty arrays and the largest length fit.
|
||||
assert_eq!(d.array_size(BasicType::Int, 0), 16);
|
||||
assert_eq!(d.array_size(BasicType::Long, u64::from(u32::MAX)), 16 + 8 * u64::from(u32::MAX));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn arrays_in_the_graph_follow_the_layout() {
|
||||
let g = graph_of(&sample());
|
||||
let m = model(&g, VmLayout::DEFAULT, None);
|
||||
assert_eq!(size_at(&g, &m, 0x1400), 32, "long[2]");
|
||||
assert_eq!(size_at(&g, &m, 0x1500), 32, "Object[3]");
|
||||
assert_eq!(size_at(&g, &m, 0x1100), 24, "Node");
|
||||
let wide = model(&g, flags("-XX:-UseCompressedOops", 21), None);
|
||||
assert_eq!(size_at(&g, &wide, 0x1500), 40, "Object[3] with 8-byte references");
|
||||
assert_eq!(m.size_of(&g, g.root), 0, "the super-root has no size");
|
||||
assert_eq!(m.source, SizeSource::Layout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_wins_for_instances() {
|
||||
use BasicType::Int;
|
||||
let fields: Vec<_> = (0..23).map(|i| (format!("f{i}"), Int)).collect();
|
||||
let fields: Vec<(&str, BasicType)> = fields.iter().map(|(n, t)| (n.as_str(), *t)).collect();
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("java.lang.Thread", ROOT, &fields)
|
||||
.class("Plain", ROOT, &[("f", Int)])
|
||||
.object(0x1000, "java.lang.Thread", &[])
|
||||
.object(0x1100, "Plain", &[]);
|
||||
let g = graph_of(&heap);
|
||||
|
||||
let without = model(&g, VmLayout::DEFAULT, None);
|
||||
assert_eq!(size_at(&g, &without, 0x1000), 104);
|
||||
|
||||
let histogram = [entry("java.lang.Thread", 13, 1560), entry("Unrelated", 2, 64)];
|
||||
let with = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
assert_eq!(with.source, SizeSource::Jvm);
|
||||
assert_eq!(size_at(&g, &with, 0x1000), 120);
|
||||
assert_eq!(size_at(&g, &with, 0x1100), 16, "no row: layout");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_rows_that_do_not_divide_or_are_empty_fall_back_to_layout() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Plain", ROOT, &[("f", BasicType::Int)]).object(0x1000, "Plain", &[]);
|
||||
let g = graph_of(&heap);
|
||||
for row in [entry("Plain", 3, 100), entry("Plain", 0, 0), entry("Plain", 0, 48), entry("Plain", 2, 0)] {
|
||||
let m = model(&g, VmLayout::DEFAULT, Some(std::slice::from_ref(&row)));
|
||||
assert_eq!(m.source, SizeSource::Jvm);
|
||||
assert_eq!(size_at(&g, &m, 0x1000), 16, "{row:?}");
|
||||
}
|
||||
// The same name twice in the histogram cannot be told apart either.
|
||||
let twice = [entry("Plain", 1, 48), entry("Plain", 1, 64)];
|
||||
assert_eq!(size_at(&g, &model(&g, VmLayout::DEFAULT, Some(&twice)), 0x1000), 16);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_matches_arrays_by_display_name_but_never_sizes_them() {
|
||||
let g = graph_of(&sample());
|
||||
let histogram = [entry("java.lang.Object[]", 1, 4096), entry("long[]", 1, 4096), entry("Node", 4, 160)];
|
||||
let m = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
assert_eq!(size_at(&g, &m, 0x1500), 32);
|
||||
assert_eq!(size_at(&g, &m, 0x1400), 32);
|
||||
assert_eq!(size_at(&g, &m, 0x1100), 40, "Node from the histogram");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_name_shared_by_several_classes_never_takes_a_histogram_size() {
|
||||
use BasicType::{Int, Long};
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("dup.Foo", ROOT, &[("a", Int)])
|
||||
.object(0x1000, "dup.Foo", &[])
|
||||
.class("dup.Foo", ROOT, &[("a", Long), ("b", Long)])
|
||||
.object(0x1100, "dup.Foo", &[])
|
||||
.class("Solo", ROOT, &[("a", Int)])
|
||||
.object(0x1200, "Solo", &[]);
|
||||
let g = graph_of(&heap);
|
||||
let histogram = [entry("dup.Foo", 2, 80), entry("Solo", 1, 48)];
|
||||
let m = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
|
||||
assert_eq!(size_at(&g, &m, 0x1000), 16, "first dup.Foo uses its layout");
|
||||
assert_eq!(size_at(&g, &m, 0x1100), 32, "second dup.Foo uses its layout");
|
||||
assert_eq!(size_at(&g, &m, 0x1200), 48, "Solo still takes its histogram size");
|
||||
let (first, second, solo) = (slot_at(&g, 0x1000), slot_at(&g, 0x1100), slot_at(&g, 0x1200));
|
||||
assert_ne!(first, second);
|
||||
assert!(m.is_ambiguous(first) && m.is_ambiguous(second));
|
||||
assert!(!m.is_ambiguous(solo));
|
||||
let ambiguous: Vec<_> = (0..g.classes.len() as u32).filter(|&s| m.is_ambiguous(s)).collect();
|
||||
assert_eq!(ambiguous.len(), 2, "no other class is flagged");
|
||||
assert!(!m.is_ambiguous(u32::MAX), "slots past the classes are never ambiguous");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ambiguity_covers_array_classes_and_ignores_the_source() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("[Lx.Y;", ROOT, &[]).class("[Lx/Y;", ROOT, &[]).class("Solo", ROOT, &[]);
|
||||
let g = graph_of(&heap);
|
||||
let m = model(&g, VmLayout::DEFAULT, None);
|
||||
let flagged: Vec<_> =
|
||||
g.classes.iter().enumerate().filter(|(i, _)| m.is_ambiguous(*i as u32)).map(|(_, c)| c.name.as_str()).collect();
|
||||
assert_eq!(flagged, ["[Lx.Y;", "[Lx.Y;"]);
|
||||
}
|
||||
|
||||
/// A dump with three classes: `java.lang.Class` (three reference fields), `Statics` (two reference
|
||||
/// statics, one null) and `Plain`; `at` is where each class object lives.
|
||||
fn class_dump() -> Vec<u8> {
|
||||
use BasicType::Object;
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "java/lang/Class").load_class(0x100, 1).class(
|
||||
0x100,
|
||||
0,
|
||||
&[(10, Object), (11, Object), (12, Object)],
|
||||
&[],
|
||||
);
|
||||
w.string(2, "Statics").load_class(0x200, 2).class(0x200, 0, &[], &[(13, 0x300), (13, 0)]);
|
||||
w.string(3, "Plain").load_class(0x300, 3).class(0x300, 0, &[], &[]);
|
||||
w.finish()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn class_objects_use_the_layout_even_when_the_histogram_names_them() {
|
||||
let g = build_graph(&class_dump(), &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
let histogram = [entry("java.lang.Class", 3, 3000)];
|
||||
let by_histogram = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
let by_layout = model(&g, VmLayout::DEFAULT, None);
|
||||
assert_eq!(by_histogram.source, SizeSource::Jvm);
|
||||
for m in [&by_histogram, &by_layout] {
|
||||
// java.lang.Class has 12 bytes of fields: 24; Statics adds two references; every mirror
|
||||
// gets the 32 bytes assumed for an unknown JVM (the histogram row does not fit).
|
||||
assert_eq!(size_at(&g, m, 0x100), 56, "java.lang.Class itself");
|
||||
assert_eq!(size_at(&g, m, 0x200), 64, "two reference statics");
|
||||
assert_eq!(size_at(&g, m, 0x300), 56, "no statics");
|
||||
}
|
||||
// 8-byte references: 40 for java.lang.Class, 16 for the statics, 44 injected (JDK 21), aligned.
|
||||
let wide = model(&g, flags("-XX:-UseCompressedOops", 21), None);
|
||||
assert_eq!(size_at(&g, &wide, 0x200), 104);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn primitive_statics_add_to_the_class_object() {
|
||||
let mut g = build_graph(&class_dump(), &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
let statics = g.classes.iter().position(|c| c.name == "Statics").unwrap();
|
||||
g.classes[statics].static_bytes = 12;
|
||||
let m = model(&g, VmLayout::DEFAULT, None);
|
||||
assert_eq!(size_at(&g, &m, 0x200), 80, "24 + 12 + 2 × 4 + 32 injected = 76, aligned");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dump_without_java_lang_class_sizes_class_objects_as_empty_mirrors() {
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "Statics").load_class(0x200, 1).class(0x200, 0, &[], &[(2, 0), (2, 0), (2, 0)]);
|
||||
let g = build_graph(&w.finish(), &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
let m = model(&g, VmLayout::DEFAULT, None);
|
||||
assert_eq!(size_at(&g, &m, 0x200), 64, "16 + 3 × 4 + 32 injected = 60, aligned");
|
||||
}
|
||||
|
||||
fn one_class_two_instances(stride: u64) -> (HeapGraph, ShallowModel) {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("A", ROOT, &[("f", BasicType::Byte)]).object(0x1000, "A", &[]).object(0x1000 + stride, "A", &[]);
|
||||
let g = graph_of(&heap);
|
||||
let m = model(&g, VmLayout::DEFAULT, None);
|
||||
(g, m)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn address_gaps_for_external_dumps() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("A", ROOT, &[("f", BasicType::Byte)])
|
||||
.class("Wide", ROOT, &[("f", BasicType::Long)])
|
||||
.class("Module", ROOT, &[("f", BasicType::Long)])
|
||||
.object(0x1000, "A", &[])
|
||||
.object(0x1018, "A", &[])
|
||||
.object(0x1030, "A", &[])
|
||||
.object(0x1048, "Module", &[])
|
||||
.object(0x2000, "Wide", &[])
|
||||
.object(0x3000, "Wide", &[]);
|
||||
let g = graph_of(&heap);
|
||||
let m = model(&g, VmLayout::DEFAULT, None);
|
||||
assert_eq!(size_at(&g, &m, 0x1000), 24, "layout says 16, the smallest gap between As is 24");
|
||||
assert_eq!(size_at(&g, &m, 0x1030), 24, "every A takes the type's size");
|
||||
assert_eq!(size_at(&g, &m, 0x1048), 24, "a single instance has no gap to trust: layout");
|
||||
assert_eq!(size_at(&g, &m, 0x2000), 24, "a gap beyond layout + 64 is not an object size: layout");
|
||||
assert_eq!(m.source, SizeSource::Layout, "one of three types agrees");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn address_gaps_are_taken_within_layout_and_layout_plus_64() {
|
||||
for (stride, expected) in [(8, 16), (16, 16), (24, 24), (80, 80), (88, 16)] {
|
||||
let (g, m) = one_class_two_instances(stride);
|
||||
assert_eq!(size_at(&g, &m, 0x1000), expected, "stride {stride}");
|
||||
assert_eq!(
|
||||
m.source,
|
||||
if expected == stride && stride >= 16 { SizeSource::AddressGaps } else { SizeSource::Layout }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// `used` types with two instances 16 bytes apart (the layout size), `far` types with two
|
||||
/// instances 0x1000 apart (a gap that is no object size) and `single` types with one instance.
|
||||
fn types_by_gap_use(used: usize, far: usize, single: usize) -> SizeSource {
|
||||
let mut heap = MiniHeap::new();
|
||||
for i in 0..used + far + single {
|
||||
let name = format!("T{i}");
|
||||
heap.class(&name, ROOT, &[("f", BasicType::Byte)]);
|
||||
let base = 0x10_0000 * (i as u64 + 1);
|
||||
heap.object(base, &name, &[]);
|
||||
if i < used {
|
||||
heap.object(base + 16, &name, &[]);
|
||||
} else if i < used + far {
|
||||
heap.object(base + 0x1000, &name, &[]);
|
||||
}
|
||||
}
|
||||
let g = graph_of(&heap);
|
||||
model(&g, VmLayout::DEFAULT, None).source
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn address_gap_source_needs_ninety_percent_of_the_types_that_have_a_gap() {
|
||||
assert_eq!(types_by_gap_use(10, 0, 0), SizeSource::AddressGaps);
|
||||
assert_eq!(types_by_gap_use(9, 1, 0), SizeSource::AddressGaps);
|
||||
assert_eq!(types_by_gap_use(8, 2, 0), SizeSource::Layout);
|
||||
// Types with one instance have no gap: they count neither for nor against.
|
||||
assert_eq!(types_by_gap_use(9, 1, 5), SizeSource::AddressGaps);
|
||||
assert_eq!(types_by_gap_use(8, 2, 5), SizeSource::Layout);
|
||||
assert_eq!(types_by_gap_use(10, 0, 400), SizeSource::AddressGaps);
|
||||
assert_eq!(types_by_gap_use(0, 0, 3), SizeSource::Layout, "no type has a gap");
|
||||
assert_eq!(types_by_gap_use(0, 2, 0), SizeSource::Layout);
|
||||
assert_eq!(types_by_gap_use(0, 0, 0), SizeSource::Layout, "nothing to size");
|
||||
}
|
||||
|
||||
/// A dump of `java.lang.Class` (three reference fields, so 24 bytes of layout) and one class per
|
||||
/// entry of `statics`: `(reference statics, primitive static bytes)`. Class `i` is at `0x1000 + 0x100 × i`.
|
||||
fn mirror_heap(statics: &[(usize, u64)]) -> (HeapGraph, Vec<u64>) {
|
||||
use BasicType::Object;
|
||||
let mut w = HprofWriter::new();
|
||||
w.string(1, "java/lang/Class").load_class(0x100, 1).class(
|
||||
0x100,
|
||||
0,
|
||||
&[(10, Object), (11, Object), (12, Object)],
|
||||
&[],
|
||||
);
|
||||
let mut addrs = Vec::new();
|
||||
for (i, &(refs, _)) in statics.iter().enumerate() {
|
||||
let (addr, name_id) = (0x1000 + 0x100 * i as u64, 100 + i as u64);
|
||||
w.string(name_id, &format!("C{i}")).load_class(addr, name_id).class(addr, 0, &[], &vec![(13, 0); refs]);
|
||||
addrs.push(addr);
|
||||
}
|
||||
let mut g = build_graph(&w.finish(), &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
for (i, &(_, bytes)) in statics.iter().enumerate() {
|
||||
let class = g.classes.iter().position(|c| c.name == format!("C{i}")).unwrap();
|
||||
g.classes[class].static_bytes = bytes;
|
||||
}
|
||||
(g, addrs)
|
||||
}
|
||||
|
||||
/// What HotSpot would make of those classes if it added `injected` bytes to every mirror.
|
||||
fn expected_mirrors(statics: &[(usize, u64)], injected: u64) -> Vec<u64> {
|
||||
statics.iter().map(|&(refs, bytes)| (24 + bytes + 4 * refs as u64 + injected).next_multiple_of(8)).collect()
|
||||
}
|
||||
|
||||
const MIRROR_STATICS: [(usize, u64); 8] = [(0, 0), (3, 0), (1, 12), (6, 8), (2, 0), (9, 24), (0, 4), (4, 1)];
|
||||
|
||||
#[test]
|
||||
fn class_objects_get_the_injected_bytes_the_histogram_implies() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
// java.lang.Class itself and the eight classes.
|
||||
let mut sizes = expected_mirrors(&MIRROR_STATICS, 36);
|
||||
let java_lang_class = 24 + 36;
|
||||
let total = sizes.iter().sum::<u64>() + (java_lang_class as u64).next_multiple_of(8);
|
||||
sizes.insert(0, (java_lang_class as u64).next_multiple_of(8));
|
||||
let histogram = [entry("java.lang.Class", 9, total)];
|
||||
let m = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
assert_eq!(m.source, SizeSource::Jvm);
|
||||
assert_eq!(size_at(&g, &m, 0x100), sizes[0]);
|
||||
for (i, &addr) in addrs.iter().enumerate() {
|
||||
assert_eq!(size_at(&g, &m, addr), sizes[i + 1], "class {i}");
|
||||
}
|
||||
let spread: Vec<_> = addrs.iter().map(|&a| size_at(&g, &m, a)).collect();
|
||||
assert!(spread.iter().any(|&s| s != spread[0]), "statics still make the mirrors differ: {spread:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn injected_bytes_are_found_for_every_multiple_of_four() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
for injected in (0..=128).step_by(4) {
|
||||
let total =
|
||||
expected_mirrors(&MIRROR_STATICS, injected).iter().sum::<u64>() + (24 + injected).next_multiple_of(8);
|
||||
let histogram = [entry("java.lang.Class", 9, total)];
|
||||
let m = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
let got: Vec<_> = addrs.iter().map(|&a| size_at(&g, &m, a)).collect();
|
||||
assert_eq!(got, expected_mirrors(&MIRROR_STATICS, injected), "injected {injected}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_fit_tolerates_mirrors_the_dump_has_no_class_for() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
// The histogram counts the same average mirror 90 times more often (CDS-archived classes).
|
||||
let total = expected_mirrors(&MIRROR_STATICS, 28).iter().sum::<u64>() + (24u64 + 28).next_multiple_of(8);
|
||||
let histogram = [entry("java.lang.Class", 9 * 91, total * 91)];
|
||||
let m = model(&g, VmLayout::DEFAULT, Some(&histogram));
|
||||
let got: Vec<_> = addrs.iter().map(|&a| size_at(&g, &m, a)).collect();
|
||||
assert_eq!(got, expected_mirrors(&MIRROR_STATICS, 28));
|
||||
}
|
||||
|
||||
fn mirror_sizes(g: &HeapGraph, addrs: &[u64], layout: VmLayout, histogram: Option<&[HistogramEntry]>) -> Vec<u64> {
|
||||
let m = model(g, layout, histogram);
|
||||
addrs.iter().map(|&a| size_at(g, &m, a)).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_histogram_row_that_cannot_describe_these_mirrors_falls_back_to_the_table() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
let rows = [
|
||||
entry("java.lang.Class", 9, 9 * 5000),
|
||||
entry("java.lang.Class", 9, 9),
|
||||
entry("java.lang.Class", 0, 0),
|
||||
entry("Something.Else", 9, 9 * 100),
|
||||
];
|
||||
for row in &rows {
|
||||
let one = std::slice::from_ref(row);
|
||||
// Unknown JVM: the documented 32; JDK 11 with 4-byte references: 36.
|
||||
assert_eq!(
|
||||
mirror_sizes(&g, &addrs, VmLayout::DEFAULT, Some(one)),
|
||||
expected_mirrors(&MIRROR_STATICS, 32),
|
||||
"{row:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
mirror_sizes(&g, &addrs, flags(J8_11_1G, 11), Some(one)),
|
||||
expected_mirrors(&MIRROR_STATICS, 36),
|
||||
"{row:?}"
|
||||
);
|
||||
}
|
||||
let twice = [entry("java.lang.Class", 9, 9 * 60), entry("java.lang.Class", 9, 9 * 60)];
|
||||
assert_eq!(mirror_sizes(&g, &addrs, flags(J8_11_1G, 11), Some(&twice)), expected_mirrors(&MIRROR_STATICS, 36));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_accepted_fit_beats_the_table() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
let total = expected_mirrors(&MIRROR_STATICS, 20).iter().sum::<u64>() + 48;
|
||||
let histogram = [entry("java.lang.Class", 9, total)];
|
||||
// JDK 11's table says 36; the histogram says 20.
|
||||
assert_eq!(mirror_sizes(&g, &addrs, flags(J8_11_1G, 11), Some(&histogram)), expected_mirrors(&MIRROR_STATICS, 20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn injected_mirror_bytes_table() {
|
||||
// (JDK, 4-byte references, 8-byte references): the measured cells, then the JDKs between.
|
||||
for (major, narrow, wide) in
|
||||
[(8, 36, 44), (11, 36, 44), (21, 36, 44), (22, 36, 44), (23, 36, 40), (24, 36, 40), (25, 28, 32), (26, 28, 32)]
|
||||
{
|
||||
assert_eq!(mirrors::injected_mirror_bytes(major, 4), Some(narrow), "JDK {major}, compressed oops");
|
||||
assert_eq!(mirrors::injected_mirror_bytes(major, 8), Some(wide), "JDK {major}, 8-byte references");
|
||||
}
|
||||
assert_eq!(mirrors::injected_mirror_bytes(21, 2), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_histogram_class_objects_get_the_injected_bytes_of_their_jvm() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
// The class dump of the fixture has 4-byte references; the layout under test decides the table
|
||||
// cell, so use layouts whose reference size matches it.
|
||||
for (layout, injected) in [
|
||||
(flags(J8_11_1G, 8), 36),
|
||||
(flags(J8_11_1G, 11), 36),
|
||||
(flags(J21_25_1G, 21), 36),
|
||||
(flags(J21_25_1G, 23), 36),
|
||||
(flags(J21_25_1G, 25), 28),
|
||||
(flags(J25_COMPACT, 25), 28),
|
||||
] {
|
||||
assert_eq!(mirror_sizes(&g, &addrs, layout, None), expected_mirrors(&MIRROR_STATICS, injected), "{layout:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eight_byte_references_use_their_own_cells() {
|
||||
let g = build_graph(&class_dump(), &Store::Ram, &no_progress, &CancellationToken::new()).unwrap();
|
||||
// java.lang.Class = 12 + 3 × 8 = 36 → 40; Plain has no statics: 40 + injected, aligned.
|
||||
for (major, injected) in [(8, 44), (21, 44), (23, 40), (25, 32)] {
|
||||
let m = model(&g, flags(J21_25_40G, major), None);
|
||||
assert_eq!(size_at(&g, &m, 0x300), (40u64 + injected).next_multiple_of(8), "JDK {major}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unknown_jvm_uses_the_midpoint_of_the_compressed_oop_cells() {
|
||||
let (g, addrs) = mirror_heap(&MIRROR_STATICS);
|
||||
assert_eq!(mirror_sizes(&g, &addrs, VmLayout::DEFAULT, None), expected_mirrors(&MIRROR_STATICS, 32));
|
||||
// Flags without any -XX: option say nothing about the JVM either.
|
||||
assert_eq!(mirror_sizes(&g, &addrs, flags("", 21), None), expected_mirrors(&MIRROR_STATICS, 32));
|
||||
assert_eq!(VmLayout::DEFAULT.java_major, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_token_stops_the_model() {
|
||||
let g = graph_of(&sample());
|
||||
let cancel = CancellationToken::new();
|
||||
cancel.cancel();
|
||||
assert!(matches!(ShallowModel::new(&g, VmLayout::DEFAULT, None, &cancel), Err(HeapError::Cancelled)));
|
||||
let histogram = [entry("Node", 1, 24)];
|
||||
assert!(matches!(ShallowModel::new(&g, VmLayout::DEFAULT, Some(&histogram), &cancel), Err(HeapError::Cancelled)));
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
//! Text of `java.lang.String` objects.
|
||||
|
||||
use crate::hprof::{BasicType, HeapRecord};
|
||||
use crate::index::{HeapIndex, id_value, instance_fields};
|
||||
|
||||
const STRING: &str = "java.lang.String";
|
||||
const LATIN1: u8 = 0;
|
||||
const UTF16: u8 = 1;
|
||||
|
||||
/// The text of String `object`, cut to `max_chars` chars plus `…`. `None` for anything that is
|
||||
/// not a String, or one whose `value` / `coder` the dump does not hold as expected.
|
||||
///
|
||||
/// JDK 9+ keeps a `byte[]` and a `coder`; UTF-16 bytes are in the JVM's native order, which
|
||||
/// hprof copies as is, read here as little-endian (x86-64, AArch64; a big-endian JVM's text would
|
||||
/// come out garbled). JDK 8 keeps a `char[]`, whose elements hprof writes big-endian.
|
||||
pub(crate) fn string_preview(index: &HeapIndex, object: u32, max_chars: usize) -> Option<String> {
|
||||
let slot = *index.types.get(object as usize)?;
|
||||
if index.class_info.get(slot as usize)?.name != STRING {
|
||||
return None;
|
||||
}
|
||||
let Some(HeapRecord::Instance { data, .. }) = index.record(object) else { return None };
|
||||
let fields = instance_fields(index, slot, data);
|
||||
let value = fields.iter().find(|f| f.name == "value" && f.ty == BasicType::Object)?;
|
||||
let array = index.ids.rank(id_value(value.bytes).filter(|&id| id != 0)?)?;
|
||||
let coder = fields.iter().find(|f| f.name == "coder" && f.ty == BasicType::Byte).and_then(|f| f.bytes.first());
|
||||
let Some(HeapRecord::PrimitiveArray { ty, data, .. }) = index.record(array) else { return None };
|
||||
match (ty, coder) {
|
||||
(BasicType::Byte, Some(&LATIN1)) => Some(cut(data.iter().map(|&b| char::from(b)), max_chars)),
|
||||
(BasicType::Byte, Some(&UTF16)) => utf16(data, u16::from_le_bytes, max_chars),
|
||||
(BasicType::Char, _) => utf16(data, u16::from_be_bytes, max_chars),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The text of `char[]` `object`, as [`string_preview`] cuts it.
|
||||
pub(crate) fn char_array_text(index: &HeapIndex, object: u32, max_chars: usize) -> Option<String> {
|
||||
match index.record(object)? {
|
||||
HeapRecord::PrimitiveArray { ty: BasicType::Char, data, .. } => utf16(data, u16::from_be_bytes, max_chars),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// An odd byte count is `None`; a lone surrogate becomes U+FFFD.
|
||||
fn utf16(bytes: &[u8], unit: fn([u8; 2]) -> u16, max_chars: usize) -> Option<String> {
|
||||
if !bytes.len().is_multiple_of(2) {
|
||||
return None;
|
||||
}
|
||||
let units = bytes.as_chunks::<2>().0.iter().map(|&pair| unit(pair));
|
||||
Some(cut(char::decode_utf16(units).map(|c| c.unwrap_or(char::REPLACEMENT_CHARACTER)), max_chars))
|
||||
}
|
||||
|
||||
/// Chars are whole code points, so a cut never splits a surrogate pair.
|
||||
fn cut(chars: impl Iterator<Item = char>, max_chars: usize) -> String {
|
||||
let mut out = String::new();
|
||||
for (i, c) in chars.enumerate() {
|
||||
if i == max_chars {
|
||||
out.push('…');
|
||||
break;
|
||||
}
|
||||
out.push(c);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,113 @@
|
||||
use super::*;
|
||||
use crate::hprof::{BasicType, HeapRootKind};
|
||||
use crate::test_index::{analyzed, obj};
|
||||
use crate::testing::{MiniHeap, sample_with_string};
|
||||
|
||||
const AT: u64 = 0x1000;
|
||||
const ARRAY: u64 = 0x2000;
|
||||
const JDK9: [(&str, BasicType); 2] = [("value", BasicType::Object), ("coder", BasicType::Byte)];
|
||||
|
||||
fn preview(heap: &MiniHeap, addr: u64) -> Option<String> {
|
||||
let index = analyzed(heap);
|
||||
string_preview(&index, obj(&index, addr).0, 100)
|
||||
}
|
||||
|
||||
fn text(text: &str, latin1: bool) -> Option<String> {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.string(AT, text, latin1).root(HeapRootKind::JniGlobal, AT);
|
||||
preview(&heap, AT)
|
||||
}
|
||||
|
||||
/// A `java.lang.String` class with `fields`, and one String at [`AT`] with `values`.
|
||||
fn custom(fields: &[(&str, BasicType)], values: &[(&str, u64)], array: impl FnOnce(&mut MiniHeap)) -> Option<String> {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("java.lang.String", Some("java.lang.Object"), fields);
|
||||
array(&mut heap);
|
||||
heap.object(AT, "java.lang.String", values);
|
||||
preview(&heap, AT)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_preview_latin1_and_utf16() {
|
||||
assert_eq!(preview(&sample_with_string(), 0x1800).as_deref(), Some("v42"));
|
||||
assert_eq!(text("hé", false).as_deref(), Some("hé"));
|
||||
assert_eq!(text("ÿ é", true).as_deref(), Some("ÿ é"));
|
||||
assert_eq!(text("", true).as_deref(), Some(""));
|
||||
|
||||
let long = "a".repeat(150);
|
||||
assert_eq!(text(&long, true), Some(format!("{}…", "a".repeat(100))));
|
||||
assert_eq!(text(&long, false), Some(format!("{}…", "a".repeat(100))));
|
||||
let exactly = "b".repeat(100);
|
||||
assert_eq!(text(&exactly, true), Some(exactly));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncation_keeps_surrogate_pairs_whole() {
|
||||
let s = format!("{}😀{}", "a".repeat(99), "b".repeat(50));
|
||||
assert_eq!(text(&s, false), Some(format!("{}😀…", "a".repeat(99))));
|
||||
// A lone surrogate becomes the replacement character.
|
||||
let lone: Vec<u8> = [0x68u16, 0xD800, 0x69].iter().flat_map(|u| u.to_le_bytes()).collect();
|
||||
let got = custom(&JDK9, &[("value", ARRAY), ("coder", 1)], |h| {
|
||||
h.prim_bytes(ARRAY, BasicType::Byte, &lone);
|
||||
});
|
||||
assert_eq!(got.as_deref(), Some("h\u{FFFD}i"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn jdk8_strings_hold_big_endian_chars() {
|
||||
let chars: Vec<u8> = "hé".encode_utf16().flat_map(u16::to_be_bytes).collect();
|
||||
let fields = [("value", BasicType::Object), ("hash", BasicType::Int)];
|
||||
let got = custom(&fields, &[("value", ARRAY)], |h| {
|
||||
h.prim_bytes(ARRAY, BasicType::Char, &chars);
|
||||
});
|
||||
assert_eq!(got.as_deref(), Some("hé"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_strings_have_no_preview() {
|
||||
let odd = custom(&JDK9, &[("value", ARRAY), ("coder", 1)], |h| {
|
||||
h.prim_bytes(ARRAY, BasicType::Byte, &[0x68, 0, 0x69]);
|
||||
});
|
||||
assert_eq!(odd, None);
|
||||
let bad_coder = custom(&JDK9, &[("value", ARRAY), ("coder", 2)], |h| {
|
||||
h.prim_bytes(ARRAY, BasicType::Byte, b"hi");
|
||||
});
|
||||
assert_eq!(bad_coder, None);
|
||||
let no_value = custom(&[("coder", BasicType::Byte)], &[], |_| {});
|
||||
assert_eq!(no_value, None);
|
||||
let no_coder = custom(&[("value", BasicType::Object)], &[("value", ARRAY)], |h| {
|
||||
h.prim_bytes(ARRAY, BasicType::Byte, b"hi");
|
||||
});
|
||||
assert_eq!(no_coder, None);
|
||||
let null_value = custom(&JDK9, &[], |_| {});
|
||||
assert_eq!(null_value, None);
|
||||
let int_array = custom(&JDK9, &[("value", ARRAY)], |h| {
|
||||
h.prim(ARRAY, BasicType::Int, 2);
|
||||
});
|
||||
assert_eq!(int_array, None);
|
||||
let not_an_array = custom(&JDK9, &[("value", ARRAY)], |h| {
|
||||
h.object(ARRAY, "java.lang.Object", &[]);
|
||||
});
|
||||
assert_eq!(not_an_array, None);
|
||||
// `value` stored as a long instead of a reference.
|
||||
let wrong_type = custom(&[("value", BasicType::Long), ("coder", BasicType::Byte)], &[("value", ARRAY)], |h| {
|
||||
h.prim_bytes(ARRAY, BasicType::Byte, b"hi");
|
||||
});
|
||||
assert_eq!(wrong_type, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_strings_have_a_preview() {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Text", Some("java.lang.Object"), &JDK9).prim_bytes(ARRAY, BasicType::Byte, b"hi").object(
|
||||
AT,
|
||||
"Text",
|
||||
&[("value", ARRAY)],
|
||||
);
|
||||
assert_eq!(preview(&heap, AT), None);
|
||||
assert_eq!(preview(&heap, ARRAY), None);
|
||||
let index = analyzed(&heap);
|
||||
for bad in [index.root(), u32::MAX] {
|
||||
assert_eq!(string_preview(&index, bad, 100), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
//! Analyzed fixtures for the query tests.
|
||||
|
||||
use ide_core::CancellationToken;
|
||||
|
||||
use crate::analysis::{AnalyzeOptions, analyze};
|
||||
use crate::error::HeapError;
|
||||
use crate::index::{HeapIndex, HeapObjectRef};
|
||||
use crate::progress::HeapPhase;
|
||||
use crate::testing::MiniHeap;
|
||||
|
||||
fn no_progress(_: HeapPhase, _: u64, _: u64) -> Result<(), HeapError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `heap` analyzed in memory, with layout sizes.
|
||||
pub(crate) fn analyzed(heap: &MiniHeap) -> HeapIndex {
|
||||
analyzed_bytes(&heap.bytes())
|
||||
}
|
||||
|
||||
pub(crate) fn analyzed_bytes(hprof: &[u8]) -> HeapIndex {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dump = dir.path().join("dump.hprof");
|
||||
std::fs::write(&dump, hprof).unwrap();
|
||||
let opts = AnalyzeOptions { in_memory: true, ..AnalyzeOptions::default() };
|
||||
analyze(&dump, dir.path(), opts, &no_progress, &CancellationToken::new()).unwrap()
|
||||
}
|
||||
|
||||
pub(crate) fn obj(index: &HeapIndex, addr: u64) -> HeapObjectRef {
|
||||
HeapObjectRef(index.ids.rank(addr).unwrap_or_else(|| panic!("{addr:#x} is not in the dump")))
|
||||
}
|
||||
|
||||
/// The slot of the class named `name`.
|
||||
pub(crate) fn slot(index: &HeapIndex, name: &str) -> u32 {
|
||||
index.classes.iter().position(|c| c.name == name).unwrap_or_else(|| panic!("no class {name}")) as u32
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
//! Builders for `.hprof` fixtures. They panic on misuse (unknown class name, text that does not
|
||||
//! fit the requested encoding): they are test helpers, not input handling.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
|
||||
use crate::hprof::{BasicType, HeapRootKind};
|
||||
|
||||
const TAG_UTF8: u8 = 0x01;
|
||||
const TAG_LOAD_CLASS: u8 = 0x02;
|
||||
const TAG_HEAP_SEGMENT: u8 = 0x1C;
|
||||
const TAG_HEAP_DUMP_END: u8 = 0x2C;
|
||||
const TIMESTAMP_MS: u64 = 1_700_000_000_000;
|
||||
|
||||
fn put_id(out: &mut Vec<u8>, id_size: u8, id: u64) {
|
||||
if id_size == 4 {
|
||||
out.extend_from_slice(&(id as u32).to_be_bytes());
|
||||
} else {
|
||||
out.extend_from_slice(&id.to_be_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes records one by one; heap sub-records are collected into a single heap-dump segment that
|
||||
/// `finish` places after the UTF8 / LOAD CLASS records, followed by HEAP DUMP END.
|
||||
pub struct HprofWriter {
|
||||
id_size: u8,
|
||||
top: Vec<u8>,
|
||||
heap: Vec<u8>,
|
||||
class_serial: u32,
|
||||
}
|
||||
|
||||
impl Default for HprofWriter {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl HprofWriter {
|
||||
pub fn new() -> Self {
|
||||
Self::with_id_size(8)
|
||||
}
|
||||
|
||||
/// `id_size` is 4 or 8; ids are truncated to the width.
|
||||
pub fn with_id_size(id_size: u8) -> Self {
|
||||
assert!(id_size == 4 || id_size == 8, "hprof ids are 4 or 8 bytes");
|
||||
let mut top = b"JAVA PROFILE 1.0.2\0".to_vec();
|
||||
top.extend_from_slice(&u32::from(id_size).to_be_bytes());
|
||||
top.extend_from_slice(&TIMESTAMP_MS.to_be_bytes());
|
||||
Self { id_size, top, heap: Vec::new(), class_serial: 0 }
|
||||
}
|
||||
|
||||
fn top_record(&mut self, tag: u8, body: &[u8]) {
|
||||
self.top.push(tag);
|
||||
self.top.extend_from_slice(&0u32.to_be_bytes());
|
||||
self.top.extend_from_slice(&(body.len() as u32).to_be_bytes());
|
||||
self.top.extend_from_slice(body);
|
||||
}
|
||||
|
||||
fn heap_id(&mut self, id: u64) {
|
||||
put_id(&mut self.heap, self.id_size, id);
|
||||
}
|
||||
|
||||
fn heap_u4(&mut self, v: u32) {
|
||||
self.heap.extend_from_slice(&v.to_be_bytes());
|
||||
}
|
||||
|
||||
pub fn string(&mut self, id: u64, s: &str) -> &mut Self {
|
||||
let mut body = Vec::new();
|
||||
put_id(&mut body, self.id_size, id);
|
||||
body.extend_from_slice(s.as_bytes());
|
||||
self.top_record(TAG_UTF8, &body);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn load_class(&mut self, class_id: u64, name_id: u64) -> &mut Self {
|
||||
self.class_serial += 1;
|
||||
let mut body = self.class_serial.to_be_bytes().to_vec();
|
||||
put_id(&mut body, self.id_size, class_id);
|
||||
body.extend_from_slice(&0u32.to_be_bytes());
|
||||
put_id(&mut body, self.id_size, name_id);
|
||||
self.top_record(TAG_LOAD_CLASS, &body);
|
||||
self
|
||||
}
|
||||
|
||||
/// `statics` are `(name id, object id)` reference statics.
|
||||
pub fn class(&mut self, id: u64, super_id: u64, fields: &[(u64, BasicType)], statics: &[(u64, u64)]) -> &mut Self {
|
||||
self.class_with_loader(id, super_id, 0, fields, statics)
|
||||
}
|
||||
|
||||
pub fn class_with_loader(
|
||||
&mut self,
|
||||
id: u64,
|
||||
super_id: u64,
|
||||
loader_id: u64,
|
||||
fields: &[(u64, BasicType)],
|
||||
statics: &[(u64, u64)],
|
||||
) -> &mut Self {
|
||||
self.heap.push(0x20);
|
||||
self.heap_id(id);
|
||||
self.heap_u4(0);
|
||||
self.heap_id(super_id);
|
||||
self.heap_id(loader_id);
|
||||
for _ in 0..4 {
|
||||
self.heap_id(0); // signers, protection domain, 2 reserved
|
||||
}
|
||||
let instance_size: usize = fields.iter().map(|(_, ty)| ty.hprof_size(self.id_size)).sum();
|
||||
self.heap_u4(instance_size as u32);
|
||||
self.heap.extend_from_slice(&0u16.to_be_bytes()); // constant pool
|
||||
self.heap.extend_from_slice(&(statics.len() as u16).to_be_bytes());
|
||||
for &(name, object) in statics {
|
||||
self.heap_id(name);
|
||||
self.heap.push(BasicType::Object as u8);
|
||||
self.heap_id(object);
|
||||
}
|
||||
self.heap.extend_from_slice(&(fields.len() as u16).to_be_bytes());
|
||||
for &(name, ty) in fields {
|
||||
self.heap_id(name);
|
||||
self.heap.push(ty as u8);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn instance(&mut self, id: u64, class_id: u64, data: &[u8]) -> &mut Self {
|
||||
self.heap.push(0x21);
|
||||
self.heap_id(id);
|
||||
self.heap_u4(0);
|
||||
self.heap_id(class_id);
|
||||
self.heap_u4(data.len() as u32);
|
||||
self.heap.extend_from_slice(data);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn obj_array(&mut self, id: u64, class_id: u64, items: &[u64]) -> &mut Self {
|
||||
self.heap.push(0x22);
|
||||
self.heap_id(id);
|
||||
self.heap_u4(0);
|
||||
self.heap_u4(items.len() as u32);
|
||||
self.heap_id(class_id);
|
||||
for &item in items {
|
||||
self.heap_id(item);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// An array of `len` zeroed elements.
|
||||
pub fn prim_array(&mut self, id: u64, ty: BasicType, len: u32) -> &mut Self {
|
||||
let bytes = vec![0u8; len as usize * ty.hprof_size(self.id_size)];
|
||||
self.prim_bytes(id, ty, &bytes)
|
||||
}
|
||||
|
||||
/// An array whose elements are `bytes` (big-endian, as stored in the dump).
|
||||
pub fn prim_bytes(&mut self, id: u64, ty: BasicType, bytes: &[u8]) -> &mut Self {
|
||||
let size = ty.hprof_size(self.id_size);
|
||||
assert!(ty != BasicType::Object && bytes.len().is_multiple_of(size), "{ty:?} array needs whole elements");
|
||||
self.heap.push(0x23);
|
||||
self.heap_id(id);
|
||||
self.heap_u4(0);
|
||||
self.heap_u4((bytes.len() / size) as u32);
|
||||
self.heap.push(ty as u8);
|
||||
self.heap.extend_from_slice(bytes);
|
||||
self
|
||||
}
|
||||
|
||||
/// Kinds that carry a thread serial get serial 1; see [`Self::root_in_thread`].
|
||||
pub fn root(&mut self, kind: HeapRootKind, id: u64) -> &mut Self {
|
||||
self.root_in_thread(kind, id, 1)
|
||||
}
|
||||
|
||||
/// `serial` is ignored for kinds that do not carry one.
|
||||
pub fn root_in_thread(&mut self, kind: HeapRootKind, id: u64, serial: u32) -> &mut Self {
|
||||
let (tag, has_serial, trailing) = match kind {
|
||||
HeapRootKind::Unknown => (0xFF, false, Trailing::None),
|
||||
HeapRootKind::JniGlobal => (0x01, false, Trailing::Id),
|
||||
HeapRootKind::JniLocal => (0x02, true, Trailing::U4),
|
||||
HeapRootKind::JavaFrame => (0x03, true, Trailing::U4),
|
||||
HeapRootKind::NativeStack => (0x04, true, Trailing::None),
|
||||
HeapRootKind::StickyClass => (0x05, false, Trailing::None),
|
||||
HeapRootKind::ThreadBlock => (0x06, true, Trailing::None),
|
||||
HeapRootKind::MonitorUsed => (0x07, false, Trailing::None),
|
||||
HeapRootKind::ThreadObject => (0x08, true, Trailing::U4),
|
||||
};
|
||||
self.heap.push(tag);
|
||||
self.heap_id(id);
|
||||
if has_serial {
|
||||
self.heap_u4(serial);
|
||||
}
|
||||
match trailing {
|
||||
Trailing::None => {}
|
||||
Trailing::U4 => self.heap_u4(0),
|
||||
Trailing::Id => self.heap_id(0),
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
pub fn finish(&self) -> Vec<u8> {
|
||||
let mut out = self.top.clone();
|
||||
out.push(TAG_HEAP_SEGMENT);
|
||||
out.extend_from_slice(&0u32.to_be_bytes());
|
||||
let heap_len = u32::try_from(self.heap.len()).expect("fixture heap segment exceeds 4 GiB");
|
||||
out.extend_from_slice(&heap_len.to_be_bytes());
|
||||
out.extend_from_slice(&self.heap);
|
||||
out.push(TAG_HEAP_DUMP_END);
|
||||
out.extend_from_slice(&0u32.to_be_bytes());
|
||||
out.extend_from_slice(&0u32.to_be_bytes());
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
enum Trailing {
|
||||
None,
|
||||
U4,
|
||||
Id,
|
||||
}
|
||||
|
||||
struct MiniClass {
|
||||
id: u64,
|
||||
super_name: Option<String>,
|
||||
fields: Vec<(String, BasicType)>,
|
||||
}
|
||||
|
||||
/// Ids for classes, names and string backing arrays come from here, above any address a test
|
||||
/// would pick by hand.
|
||||
const GENERATED_ID_BASE: u64 = 1 << 32;
|
||||
|
||||
/// Heap fixtures by class and field name on top of [`HprofWriter`] (id size 8).
|
||||
///
|
||||
/// Class names are written in the HotSpot form (`.` becomes `/`); look-ups use the name as given.
|
||||
pub struct MiniHeap {
|
||||
w: HprofWriter,
|
||||
next_id: u64,
|
||||
classes: HashMap<String, MiniClass>,
|
||||
}
|
||||
|
||||
impl Default for MiniHeap {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl MiniHeap {
|
||||
/// Starts with `java.lang.Object` and `java.lang.Class` already defined.
|
||||
pub fn new() -> Self {
|
||||
let mut heap = Self { w: HprofWriter::new(), next_id: GENERATED_ID_BASE, classes: HashMap::new() };
|
||||
heap.class("java.lang.Object", None, &[]);
|
||||
heap.class("java.lang.Class", Some("java.lang.Object"), &[]);
|
||||
heap
|
||||
}
|
||||
|
||||
fn fresh_id(&mut self) -> u64 {
|
||||
self.next_id += 1;
|
||||
self.next_id
|
||||
}
|
||||
|
||||
fn class_id(&self, name: &str) -> u64 {
|
||||
self.classes.get(name).unwrap_or_else(|| panic!("MiniHeap: class {name} is not defined")).id
|
||||
}
|
||||
|
||||
/// `super_name: None` writes super id 0.
|
||||
pub fn class(&mut self, name: &str, super_name: Option<&str>, fields: &[(&str, BasicType)]) -> &mut Self {
|
||||
let super_id = super_name.map_or(0, |sup| self.class_id(sup));
|
||||
let id = self.fresh_id();
|
||||
let name_id = self.fresh_id();
|
||||
let mut field_ids = Vec::with_capacity(fields.len());
|
||||
for &(field, ty) in fields {
|
||||
let field_id = self.fresh_id();
|
||||
self.w.string(field_id, field);
|
||||
field_ids.push((field_id, ty));
|
||||
}
|
||||
self.w.string(name_id, &name.replace('.', "/"));
|
||||
self.w.load_class(id, name_id);
|
||||
self.w.class(id, super_id, &field_ids, &[]);
|
||||
self.w.root(HeapRootKind::StickyClass, id);
|
||||
let fields = fields.iter().map(|&(field, ty)| (field.to_owned(), ty)).collect();
|
||||
self.classes.insert(name.to_owned(), MiniClass { id, super_name: super_name.map(str::to_owned), fields });
|
||||
self
|
||||
}
|
||||
|
||||
/// Fields are laid out class first, then superclass, as HotSpot does. A field missing from
|
||||
/// `values` is 0; narrower types keep the low bytes of the value.
|
||||
pub fn object(&mut self, addr: u64, class: &str, values: &[(&str, u64)]) -> &mut Self {
|
||||
let class_id = self.class_id(class);
|
||||
let mut data = Vec::new();
|
||||
let mut cursor = Some(class);
|
||||
while let Some(name) = cursor {
|
||||
let c = &self.classes[name];
|
||||
for (field, ty) in &c.fields {
|
||||
let value = values.iter().find(|(n, _)| *n == field.as_str()).map_or(0, |&(_, v)| v);
|
||||
let width = ty.hprof_size(8);
|
||||
data.extend_from_slice(&value.to_be_bytes()[8 - width..]);
|
||||
}
|
||||
cursor = c.super_name.as_deref();
|
||||
}
|
||||
self.w.instance(addr, class_id, &data);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn array_of(&mut self, addr: u64, class: &str, items: &[u64]) -> &mut Self {
|
||||
let class_id = self.class_id(class);
|
||||
self.w.obj_array(addr, class_id, items);
|
||||
self
|
||||
}
|
||||
|
||||
/// A primitive array of `len` zeroed elements.
|
||||
pub fn prim(&mut self, addr: u64, ty: BasicType, len: u32) -> &mut Self {
|
||||
self.w.prim_array(addr, ty, len);
|
||||
self
|
||||
}
|
||||
|
||||
/// A primitive array holding `bytes` (big-endian elements).
|
||||
pub fn prim_bytes(&mut self, addr: u64, ty: BasicType, bytes: &[u8]) -> &mut Self {
|
||||
self.w.prim_bytes(addr, ty, bytes);
|
||||
self
|
||||
}
|
||||
|
||||
/// A `java.lang.String` (JDK 9+ layout: `value` byte[] + `coder`) at `addr`. `latin1` stores
|
||||
/// one byte per char (coder 0, every char must be ≤ U+00FF); otherwise UTF-16LE (coder 1).
|
||||
pub fn string(&mut self, addr: u64, text: &str, latin1: bool) -> &mut Self {
|
||||
if !self.classes.contains_key("java.lang.String") {
|
||||
let fields = [("value", BasicType::Object), ("coder", BasicType::Byte)];
|
||||
self.class("java.lang.String", Some("java.lang.Object"), &fields);
|
||||
}
|
||||
let bytes: Vec<u8> = if latin1 {
|
||||
text.chars()
|
||||
.map(|c| u8::try_from(u32::from(c)).unwrap_or_else(|_| panic!("{c:?} is not Latin-1")))
|
||||
.collect()
|
||||
} else {
|
||||
text.encode_utf16().flat_map(u16::to_le_bytes).collect()
|
||||
};
|
||||
let array = self.fresh_id();
|
||||
self.w.prim_bytes(array, BasicType::Byte, &bytes);
|
||||
self.object(addr, "java.lang.String", &[("value", array), ("coder", u64::from(!latin1))])
|
||||
}
|
||||
|
||||
/// Kinds that carry a thread serial get serial 1.
|
||||
pub fn root(&mut self, kind: HeapRootKind, addr: u64) -> &mut Self {
|
||||
self.w.root(kind, addr);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn root_in_thread(&mut self, kind: HeapRootKind, addr: u64, serial: u32) -> &mut Self {
|
||||
self.w.root_in_thread(kind, addr, serial);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn bytes(&self) -> Vec<u8> {
|
||||
self.w.finish()
|
||||
}
|
||||
|
||||
pub fn write_to(&self, path: &Path) {
|
||||
std::fs::write(path, self.bytes()).unwrap_or_else(|e| panic!("MiniHeap: writing {}: {e}", path.display()));
|
||||
}
|
||||
}
|
||||
|
||||
/// The heap most analysis tests share:
|
||||
///
|
||||
/// - `Holder { a: Object, b: Object }`, `Node { next: Object, v: long }`,
|
||||
/// `java.lang.ref.Reference { referent: Object, queue: Object }`,
|
||||
/// `java.lang.ref.WeakReference extends Reference`, `[Ljava.lang.Object;`
|
||||
/// - H `0x1000` Holder { a: N1, b: N2 }; N1 `0x1100` Node { next: N3 }; N2 `0x1200` Node { next: N3 };
|
||||
/// N3 `0x1300` Node { next: L }; L `0x1400` long[2]; R `0x1500` Object[] { N1, null, null };
|
||||
/// W `0x1600` WeakReference { referent: N4 }; N4 `0x1700` Node {}
|
||||
/// - roots: JNI global H, Java frame N3 (thread serial 1), JNI global W
|
||||
///
|
||||
/// R and N4 are unreachable (N4 only through the weak referent).
|
||||
pub fn sample() -> MiniHeap {
|
||||
let mut heap = MiniHeap::new();
|
||||
heap.class("Holder", Some("java.lang.Object"), &[("a", BasicType::Object), ("b", BasicType::Object)])
|
||||
.class("Node", Some("java.lang.Object"), &[("next", BasicType::Object), ("v", BasicType::Long)])
|
||||
.class(
|
||||
"java.lang.ref.Reference",
|
||||
Some("java.lang.Object"),
|
||||
&[("referent", BasicType::Object), ("queue", BasicType::Object)],
|
||||
)
|
||||
.class("java.lang.ref.WeakReference", Some("java.lang.ref.Reference"), &[])
|
||||
.class("[Ljava.lang.Object;", Some("java.lang.Object"), &[])
|
||||
.object(0x1000, "Holder", &[("a", 0x1100), ("b", 0x1200)])
|
||||
.object(0x1100, "Node", &[("next", 0x1300)])
|
||||
.object(0x1200, "Node", &[("next", 0x1300)])
|
||||
.object(0x1300, "Node", &[("next", 0x1400)])
|
||||
.prim(0x1400, BasicType::Long, 2)
|
||||
.array_of(0x1500, "[Ljava.lang.Object;", &[0x1100, 0, 0])
|
||||
.object(0x1600, "java.lang.ref.WeakReference", &[("referent", 0x1700)])
|
||||
.object(0x1700, "Node", &[])
|
||||
.root(HeapRootKind::JniGlobal, 0x1000)
|
||||
.root(HeapRootKind::JavaFrame, 0x1300)
|
||||
.root(HeapRootKind::JniGlobal, 0x1600);
|
||||
heap
|
||||
}
|
||||
|
||||
/// [`sample`] plus S `0x1800` String `"v42"` (Latin-1) held as `a` of a second Holder `0x1900`,
|
||||
/// rooted as a JNI global. That Holder retains 72 bytes, as much as H.
|
||||
pub fn sample_with_string() -> MiniHeap {
|
||||
let mut heap = sample();
|
||||
heap.string(0x1800, "v42", true).object(0x1900, "Holder", &[("a", 0x1800)]).root(HeapRootKind::JniGlobal, 0x1900);
|
||||
heap
|
||||
}
|
||||
|
||||
/// [`sample`] plus two Nodes `0x1a00` and `0x1b00`, each rooted as a JNI global.
|
||||
pub fn sample_with_two_more_nodes() -> MiniHeap {
|
||||
let mut heap = sample();
|
||||
heap.object(0x1a00, "Node", &[])
|
||||
.object(0x1b00, "Node", &[])
|
||||
.root(HeapRootKind::JniGlobal, 0x1a00)
|
||||
.root(HeapRootKind::JniGlobal, 0x1b00);
|
||||
heap
|
||||
}
|
||||
|
||||
/// Seeded xorshift64 numbers for reproducible random test data (the workspace has no `rand`).
|
||||
pub struct XorShift(u64);
|
||||
|
||||
impl XorShift {
|
||||
pub fn new(seed: u64) -> Self {
|
||||
// Zero is xorshift's one fixed point.
|
||||
Self(seed.max(1))
|
||||
}
|
||||
|
||||
pub fn next_u64(&mut self) -> u64 {
|
||||
let mut x = self.0;
|
||||
x ^= x << 13;
|
||||
x ^= x >> 7;
|
||||
x ^= x << 17;
|
||||
self.0 = x;
|
||||
x
|
||||
}
|
||||
|
||||
/// A number in `0..bound`; `bound` must not be 0.
|
||||
pub fn below(&mut self, bound: u32) -> u32 {
|
||||
(self.next_u64() % u64::from(bound)) as u32
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,7 @@ ide-search.workspace = true
|
||||
ide-lsp.workspace = true
|
||||
ide-vcs.workspace = true
|
||||
ide-dap.workspace = true
|
||||
ide-heap.workspace = true
|
||||
ide-jvm-attach.workspace = true
|
||||
ide-build.workspace = true
|
||||
ide-analysis.workspace = true
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
mod templates;
|
||||
|
||||
use ide_extension::{Manifest, validate_plugin_id};
|
||||
use ide_host::plugins::{PluginHost, plugin_icon, read_market_url, resolve_market_href};
|
||||
use ide_host::plugins::{PluginHost, market_href_reachable_from, plugin_icon, read_market_url, resolve_market_href};
|
||||
use ide_plugin::{Limits, PluginEngine};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitCode};
|
||||
@@ -591,7 +591,7 @@ fn cmd_index(args: &[String]) -> Result<(), String> {
|
||||
let base = a.value("download-base").map(|b| b.trim_end_matches('/'));
|
||||
// Revocations and other publishers of an existing index are kept.
|
||||
let mut index = match std::fs::read_to_string(&out) {
|
||||
Ok(text) => current_format(Index::parse(&text)?, &out.to_string_lossy())?,
|
||||
Ok(text) => parse_index(&text, &out.to_string_lossy())?,
|
||||
Err(_) => Index { format: package::INDEX_FORMAT, ..Index::default() },
|
||||
};
|
||||
let publisher = match (a.value("publisher"), a.value("key")) {
|
||||
@@ -653,6 +653,11 @@ fn cmd_index(args: &[String]) -> Result<(), String> {
|
||||
engine: package::engine_requirement(&m.engine, built_for.as_ref()),
|
||||
categories: m.categories(),
|
||||
icon: plugin_icon(&ide_extension::Package::new(m.clone(), scratch.clone()))?,
|
||||
homepage: m.homepage.clone().filter(|h| !h.trim().is_empty()),
|
||||
notes: m.change_notes.clone().filter(|n| !n.trim().is_empty()),
|
||||
downloads: None,
|
||||
featured: false,
|
||||
suggest: Vec::new(),
|
||||
})
|
||||
})();
|
||||
let entry = entry.map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
@@ -717,14 +722,22 @@ fn register_key(
|
||||
Ok(publishers)
|
||||
}
|
||||
|
||||
/// `index`, unless a newer `openidea-plugin` wrote it (rewriting it would drop what this one can't read).
|
||||
fn current_format(index: Index, file: &str) -> Result<Index, String> {
|
||||
/// The index in `text` (read from `file`), unless a newer `openidea-plugin` wrote it (rewriting it
|
||||
/// would drop what this one can't read). Says which entries it does not keep: no IDE offers them.
|
||||
fn parse_index(text: &str, file: &str) -> Result<Index, String> {
|
||||
let (index, dropped) = Index::parse_dropping(text)?;
|
||||
if index.is_newer() {
|
||||
return Err(format!(
|
||||
"{file}: index format {} is newer than this tool writes; update openidea-plugin",
|
||||
index.format
|
||||
));
|
||||
}
|
||||
for e in &dropped {
|
||||
eprintln!(
|
||||
"warning: {file}: dropped {:?} {:?}: its id or version is not one a manifest may carry",
|
||||
e.id, e.version
|
||||
);
|
||||
}
|
||||
Ok(index)
|
||||
}
|
||||
|
||||
@@ -735,14 +748,15 @@ fn cmd_merge(args: &[String]) -> Result<(), String> {
|
||||
};
|
||||
let keep = a.value("keep").ok_or("--keep is required")?;
|
||||
let out = a.value("out").unwrap_or(base);
|
||||
let mut index =
|
||||
current_format(Index::parse(&std::fs::read_to_string(base).map_err(|e| format!("{base}: {e}"))?)?, base)?;
|
||||
let mut index = parse_index(&std::fs::read_to_string(base).map_err(|e| format!("{base}: {e}"))?, base)?;
|
||||
let bytes = read_market_url(other, 16 << 20)?;
|
||||
let theirs = Index::parse(&String::from_utf8(bytes).map_err(|_| format!("{other}: not UTF-8"))?)?;
|
||||
let mut theirs = current_format(theirs, other)?;
|
||||
let mut theirs = parse_index(&String::from_utf8(bytes).map_err(|_| format!("{other}: not UTF-8"))?, other)?;
|
||||
// Their downloads are relative to their index, not to ours.
|
||||
for e in &mut theirs.plugins {
|
||||
e.download = resolve_market_href(other, &e.download);
|
||||
if !market_href_reachable_from(other, &e.download) {
|
||||
return Err(format!("{other}: the download of {} {} is not on the network", e.id, e.version));
|
||||
}
|
||||
}
|
||||
let count = theirs.plugins.len();
|
||||
index.include(theirs, &[keep]).map_err(|e| format!("{other}: {e}"))?;
|
||||
@@ -942,6 +956,47 @@ mod tests {
|
||||
assert!(err.contains("unknown publisher"), "{err}");
|
||||
}
|
||||
|
||||
/// An index read over the network is not merged when it names a download on this machine:
|
||||
/// the merged index is a local one, which may.
|
||||
#[test]
|
||||
fn an_index_read_over_the_network_is_not_merged_with_local_downloads() {
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let base = tmp.path().join("index.json");
|
||||
write_index(&Index { format: package::INDEX_FORMAT, ..Index::default() }, &base).unwrap();
|
||||
let local = tmp.path().join("alice.dawn-1.0.0.oiplugin").to_string_lossy().into_owned();
|
||||
let theirs = serde_json::json!({
|
||||
"format": package::INDEX_FORMAT,
|
||||
"publishers": [{ "id": "alice", "name": "Alice", "keys": [] }],
|
||||
"plugins": [{
|
||||
"id": "alice.dawn", "name": "Dawn", "version": "1.0.0", "publisher": "alice",
|
||||
"download": local, "digest": "d", "engine": "^1.0",
|
||||
}],
|
||||
})
|
||||
.to_string();
|
||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let other = format!("http://127.0.0.1:{}/index.json", listener.local_addr().unwrap().port());
|
||||
std::thread::spawn(move || {
|
||||
for mut stream in listener.incoming().flatten() {
|
||||
let mut reader = BufReader::new(stream.try_clone().unwrap());
|
||||
let mut line = String::new();
|
||||
while reader.read_line(&mut line).unwrap_or(0) > 0 && line != "\r\n" {
|
||||
line.clear();
|
||||
}
|
||||
let _ = write!(
|
||||
stream,
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{theirs}",
|
||||
theirs.len()
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
let err = cmd_merge(&strings(&[base.to_str().unwrap(), &other, "--keep", "openidea"])).unwrap_err();
|
||||
assert!(err.contains("is not on the network"), "{err}");
|
||||
let merged = Index::parse(&std::fs::read_to_string(&base).unwrap()).unwrap();
|
||||
assert!(merged.plugins.is_empty(), "{:?}", merged.plugins);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_index_carries_plugin_icons() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -9,6 +9,14 @@ use std::path::{Path, PathBuf};
|
||||
/// configuration and roaming data keep following the OS.
|
||||
pub const LOCAL_DATA_ENV: &str = "OPENIDEA_LOCAL_DATA_DIR";
|
||||
|
||||
/// A custom local-data root chosen in Settings › System Settings › Caches,
|
||||
/// stored in the configuration directory. The environment variable still wins.
|
||||
pub const OVERRIDE_FILE: &str = "local-data-dir.txt";
|
||||
|
||||
/// A pending move of the local data to a newly chosen directory, carried out
|
||||
/// at the next start (before any project opens), like File › Invalidate Caches.
|
||||
const MIGRATION_FILE: &str = "local-data-migration.json";
|
||||
|
||||
/// `$OPENIDEA_LOCAL_DATA_DIR/<identifier>` when set, else the OS local data directory
|
||||
/// (Tauri's `app_local_data_dir`). The identifier keeps release and per-worktree
|
||||
/// debug builds apart under one root. A set but unusable path is an error, never a
|
||||
@@ -17,6 +25,203 @@ pub fn local_data_dir(identifier: &str) -> Result<PathBuf, String> {
|
||||
resolve_local(std::env::var_os(LOCAL_DATA_ENV), dirs::data_local_dir(), identifier)
|
||||
}
|
||||
|
||||
/// The directory to actually start with: the environment override, else a valid
|
||||
/// custom directory from Settings, else the OS default. A stored but unusable
|
||||
/// custom directory is an error, never a silent fallback.
|
||||
pub fn startup_local_data_dir(identifier: &str) -> Result<PathBuf, String> {
|
||||
startup_resolve(std::env::var_os(LOCAL_DATA_ENV), stored_override(identifier), dirs::data_local_dir(), identifier)
|
||||
}
|
||||
|
||||
fn startup_resolve(
|
||||
var: Option<OsString>,
|
||||
stored: Option<PathBuf>,
|
||||
os: Option<PathBuf>,
|
||||
identifier: &str,
|
||||
) -> Result<PathBuf, String> {
|
||||
if var.as_ref().is_some_and(|v| !v.is_empty()) {
|
||||
return resolve_local(var, os, identifier);
|
||||
}
|
||||
let Some(root) = stored else { return resolve_local(None, os, identifier) };
|
||||
if !root.is_absolute() {
|
||||
return Err(format!("custom local data directory must be an absolute path: {}", root.display()));
|
||||
}
|
||||
resolve_root(root, identifier, "custom local data directory")
|
||||
}
|
||||
|
||||
/// The file holding the custom local-data root, if the platform exposes its
|
||||
/// configuration directory. Both startup and Settings use this one path.
|
||||
pub fn override_path(identifier: &str) -> Option<PathBuf> {
|
||||
dirs::config_dir().map(|dir| dir.join(identifier).join(OVERRIDE_FILE))
|
||||
}
|
||||
|
||||
fn migration_path(identifier: &str) -> Option<PathBuf> {
|
||||
dirs::config_dir().map(|dir| dir.join(identifier).join(MIGRATION_FILE))
|
||||
}
|
||||
|
||||
/// The custom root stored by Settings, if any (validated when it is used).
|
||||
pub fn stored_override(identifier: &str) -> Option<PathBuf> {
|
||||
stored_override_in(&override_path(identifier)?)
|
||||
}
|
||||
|
||||
fn stored_override_in(path: &Path) -> Option<PathBuf> {
|
||||
let text = fs::read_to_string(path).ok()?;
|
||||
let trimmed = text.trim();
|
||||
(!trimmed.is_empty()).then(|| PathBuf::from(trimmed))
|
||||
}
|
||||
|
||||
/// The OS default local data directory (what clearing the custom directory
|
||||
/// returns to), ignoring any stored override. The environment still applies.
|
||||
pub fn default_local_data_dir(identifier: &str) -> Result<PathBuf, String> {
|
||||
local_data_dir(identifier)
|
||||
}
|
||||
|
||||
/// Stores (`Some`) or clears (`None`) the custom local-data root, returning the
|
||||
/// directory that takes effect after a restart. A pending custom migration to
|
||||
/// anywhere else is dropped: only the caller schedules the matching one.
|
||||
pub fn set_override(identifier: &str, root: Option<&Path>) -> Result<PathBuf, String> {
|
||||
let Some(path) = override_path(identifier) else {
|
||||
return Err("cannot find the configuration directory".into());
|
||||
};
|
||||
set_override_in(&path, &migration_path(identifier), root, identifier)
|
||||
}
|
||||
|
||||
fn set_override_in(
|
||||
path: &Path,
|
||||
migration: &Option<PathBuf>,
|
||||
root: Option<&Path>,
|
||||
identifier: &str,
|
||||
) -> Result<PathBuf, String> {
|
||||
if let Some(migration) = migration {
|
||||
let _ = fs::remove_file(migration);
|
||||
}
|
||||
let Some(root) = root else {
|
||||
if let Err(error) = fs::remove_file(path)
|
||||
&& error.kind() != io::ErrorKind::NotFound
|
||||
{
|
||||
return Err(format!("cannot clear the custom local data directory: {error}"));
|
||||
}
|
||||
return local_data_dir(identifier);
|
||||
};
|
||||
if !root.is_absolute() {
|
||||
return Err(format!("custom local data directory must be an absolute path: {}", root.display()));
|
||||
}
|
||||
let dir = resolve_root(root.to_path_buf(), identifier, "custom local data directory")?;
|
||||
if let Some(parent) = path.parent()
|
||||
&& let Err(error) = fs::create_dir_all(parent)
|
||||
{
|
||||
return Err(format!("cannot store the custom local data directory: {error}"));
|
||||
}
|
||||
if let Err(error) = fs::write(path, format!("{}\n", root.display())) {
|
||||
return Err(format!("cannot store the custom local data directory: {error}"));
|
||||
}
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
struct CustomMigration {
|
||||
from: PathBuf,
|
||||
to: PathBuf,
|
||||
}
|
||||
|
||||
/// Schedules a move of the local data from `from` to `to` at the next start.
|
||||
/// Returns false when there is nothing to move. The caller restarts the IDE.
|
||||
pub fn schedule_custom_migration(identifier: &str, from: &Path, to: &Path) -> Result<bool, String> {
|
||||
let Some(path) = migration_path(identifier) else {
|
||||
return Err("cannot find the configuration directory".into());
|
||||
};
|
||||
schedule_custom_migration_in(&path, from, to)
|
||||
}
|
||||
|
||||
fn schedule_custom_migration_in(path: &Path, from: &Path, to: &Path) -> Result<bool, String> {
|
||||
if from == to {
|
||||
return Ok(false);
|
||||
}
|
||||
if let Some(parent) = path.parent()
|
||||
&& let Err(error) = fs::create_dir_all(parent)
|
||||
{
|
||||
return Err(format!("cannot schedule the data move: {error}"));
|
||||
}
|
||||
let json = serde_json::to_vec(&CustomMigration { from: from.into(), to: to.into() })
|
||||
.map_err(|error| format!("cannot schedule the data move: {error}"))?;
|
||||
if let Err(error) = fs::write(path, json) {
|
||||
return Err(format!("cannot schedule the data move: {error}"));
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Carries out a scheduled custom migration aimed at `local` (call at start-up,
|
||||
/// before projects open). Entries that appear in the meantime are never
|
||||
/// overwritten; a partially moved tree is retried at the next start.
|
||||
pub fn apply_custom_migration(identifier: &str, local: &Path) {
|
||||
let Some(path) = migration_path(identifier) else { return };
|
||||
apply_custom_migration_in(&path, local);
|
||||
}
|
||||
|
||||
fn apply_custom_migration_in(path: &Path, local: &Path) {
|
||||
let Ok(bytes) = fs::read(path) else { return };
|
||||
let Ok(job) = serde_json::from_slice::<CustomMigration>(&bytes) else {
|
||||
let _ = fs::remove_file(path);
|
||||
return;
|
||||
};
|
||||
if job.to != local || job.from == local {
|
||||
let _ = fs::remove_file(path);
|
||||
return;
|
||||
}
|
||||
move_entries(&job.from, local, &custom_entries(&job.from));
|
||||
let leftover = custom_entries(&job.from).into_iter().any(|entry| job.from.join(entry).exists());
|
||||
if !leftover {
|
||||
let _ = fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
|
||||
/// Entries of a local-data tree that move to a newly chosen directory.
|
||||
/// Non-relocatable tool installs (virtual environments with absolute interpreter
|
||||
/// paths) stay behind and are reinstalled on demand, as in `migrate_entries`.
|
||||
fn custom_entries(root: &Path) -> Vec<PathBuf> {
|
||||
let mut entries = local_entries(root);
|
||||
for extra in ["logs", "agents/tools"] {
|
||||
let entry = PathBuf::from(extra);
|
||||
if !entries.contains(&entry) {
|
||||
entries.push(entry);
|
||||
}
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
/// Moves the local-data tree from `from` to `to`, entry by entry. Same-volume
|
||||
/// entries rename at once; cross-volume copies run in the background. Targets
|
||||
/// created in the meantime are never merged into or overwritten.
|
||||
fn move_entries(from: &Path, to: &Path, entries: &[PathBuf]) {
|
||||
if from == to {
|
||||
return;
|
||||
}
|
||||
let mut copies = Vec::new();
|
||||
for entry in entries {
|
||||
let source = from.join(entry);
|
||||
let target = to.join(entry);
|
||||
match try_rename(&source, &target) {
|
||||
Ok(true) => copies.push((source, target)),
|
||||
Ok(false) => {}
|
||||
Err(error) => {
|
||||
tracing::warn!(?source, ?target, %error, "data move failed; keeping old data, using new location")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !copies.is_empty() {
|
||||
let result = std::thread::Builder::new().name("openidea-data-migration".into()).spawn(move || {
|
||||
for (source, target) in copies {
|
||||
tracing::info!(?source, ?target, "copying data to the new location in background");
|
||||
if let Err(error) = copy_then_remove(&source, &target) {
|
||||
tracing::warn!(?source, ?target, %error, "data copy failed; keeping old data");
|
||||
}
|
||||
}
|
||||
});
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "cannot start data migration worker; keeping old data");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_local(var: Option<OsString>, os: Option<PathBuf>, identifier: &str) -> Result<PathBuf, String> {
|
||||
let Some(root) = var.filter(|v| !v.is_empty()).map(PathBuf::from) else {
|
||||
return os.map(|dir| dir.join(identifier)).ok_or_else(|| "cannot find the local data directory".into());
|
||||
@@ -25,10 +230,14 @@ fn resolve_local(var: Option<OsString>, os: Option<PathBuf>, identifier: &str) -
|
||||
if !root.is_absolute() {
|
||||
return Err(format!("{LOCAL_DATA_ENV} must be an absolute path: {}", root.display()));
|
||||
}
|
||||
resolve_root(root, identifier, LOCAL_DATA_ENV)
|
||||
}
|
||||
|
||||
fn resolve_root(root: PathBuf, identifier: &str, what: &str) -> Result<PathBuf, String> {
|
||||
let dir = root.join(identifier);
|
||||
fs::create_dir_all(&dir)
|
||||
.and_then(|()| tempfile::NamedTempFile::new_in(&dir).map(drop))
|
||||
.map_err(|error| format!("{LOCAL_DATA_ENV}: cannot write to {}: {error}", dir.display()))?;
|
||||
.map_err(|error| format!("{what}: cannot write to {}: {error}", dir.display()))?;
|
||||
Ok(dir)
|
||||
}
|
||||
|
||||
@@ -77,6 +286,11 @@ pub fn migrate(data: &Path, local: &Path) {
|
||||
}
|
||||
|
||||
fn migrate_entries(data: &Path, local: &Path) {
|
||||
move_entries(data, local, &local_entries(data));
|
||||
}
|
||||
|
||||
/// The rebuildable entries of the legacy roaming tree that move to Local.
|
||||
fn local_entries(data: &Path) -> Vec<PathBuf> {
|
||||
let mut entries: Vec<PathBuf> = [
|
||||
"language-server-runtimes",
|
||||
"index",
|
||||
@@ -118,31 +332,7 @@ fn migrate_entries(data: &Path, local: &Path) {
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => tracing::warn!(%error, "cannot enumerate legacy tools; keeping old data, using Local"),
|
||||
}
|
||||
let mut copies = Vec::new();
|
||||
for entry in entries {
|
||||
let source = data.join(&entry);
|
||||
let target = data_path(data, local, &entry);
|
||||
match try_rename(&source, &target) {
|
||||
Ok(true) => copies.push((source, target)),
|
||||
Ok(false) => {}
|
||||
Err(error) => {
|
||||
tracing::warn!(?source, ?target, %error, "data migration failed; keeping old data, using Local")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !copies.is_empty() {
|
||||
let result = std::thread::Builder::new().name("openidea-data-migration".into()).spawn(move || {
|
||||
for (source, target) in copies {
|
||||
tracing::info!(?source, ?target, "copying legacy data in background");
|
||||
if let Err(error) = copy_then_remove(&source, &target) {
|
||||
tracing::warn!(?source, ?target, %error, "data copy failed; keeping old data, using Local");
|
||||
}
|
||||
}
|
||||
});
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "cannot start data migration worker; keeping old data, using Local");
|
||||
}
|
||||
}
|
||||
entries
|
||||
}
|
||||
|
||||
/// Returns true when a cross-volume copy is needed.
|
||||
@@ -597,4 +787,108 @@ mod tests {
|
||||
.any(|e| e.file_name().to_string_lossy().starts_with(".migration-"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_override_round_trips_and_validates() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let file = tmp.path().join("local-data-dir.txt");
|
||||
let migration = tmp.path().join("local-data-migration.json");
|
||||
let id = "ai.test.openidea";
|
||||
assert_eq!(stored_override_in(&file), None);
|
||||
|
||||
let root = tmp.path().join("custom-root");
|
||||
let dir = set_override_in(&file, &Some(migration.clone()), Some(&root), id).unwrap();
|
||||
assert_eq!(dir, root.join(id));
|
||||
assert!(dir.is_dir());
|
||||
assert_eq!(stored_override_in(&file), Some(root.clone()));
|
||||
assert_eq!(startup_resolve(None, stored_override_in(&file), Some(tmp.path().join("os")), id).unwrap(), dir);
|
||||
|
||||
// A relative root is rejected before anything is written.
|
||||
let before = fs::read_to_string(&file).unwrap();
|
||||
let error = set_override_in(&file, &Some(migration.clone()), Some(Path::new("relative")), id).unwrap_err();
|
||||
assert!(error.contains("absolute"), "{error}");
|
||||
assert_eq!(fs::read_to_string(&file).unwrap(), before);
|
||||
|
||||
// An unwritable root (an existing file) is rejected too.
|
||||
let blocker = tmp.path().join("blocker");
|
||||
fs::write(&blocker, "").unwrap();
|
||||
let error = set_override_in(&file, &Some(migration.clone()), Some(&blocker), id).unwrap_err();
|
||||
assert!(error.contains("cannot write"), "{error}");
|
||||
|
||||
// Clearing removes the file and returns the default.
|
||||
let dir = set_override_in(&file, &Some(migration), None, id).unwrap();
|
||||
assert!(!file.exists());
|
||||
assert_eq!(dir, local_data_dir(id).unwrap());
|
||||
assert_eq!(stored_override_in(&file), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn startup_prefers_the_environment_then_the_override() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let os = tmp.path().join("os");
|
||||
let custom = tmp.path().join("custom");
|
||||
let env = tmp.path().join("env");
|
||||
let id = "ai.test.openidea";
|
||||
|
||||
let error = startup_resolve(None, Some(PathBuf::from("relative")), Some(os.clone()), id).unwrap_err();
|
||||
assert!(error.contains("absolute"), "{error}");
|
||||
|
||||
// The environment wins over a stored override.
|
||||
let dir = startup_resolve(Some(env.clone().into()), Some(custom.clone()), Some(os.clone()), id).unwrap();
|
||||
assert_eq!(dir, env.join(id));
|
||||
|
||||
// Without the environment the override wins over the OS default.
|
||||
let dir = startup_resolve(None, Some(custom.clone()), Some(os.clone()), id).unwrap();
|
||||
assert_eq!(dir, custom.join(id));
|
||||
|
||||
// And without either the OS default applies.
|
||||
let dir = startup_resolve(None, None, Some(os.clone()), id).unwrap();
|
||||
assert_eq!(dir, os.join(id));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_migration_moves_the_tree_and_clears_itself() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let marker = tmp.path().join("local-data-migration.json");
|
||||
let from = tmp.path().join("old-local");
|
||||
let to = tmp.path().join("new-local");
|
||||
fs::create_dir_all(from.join("index/p1")).unwrap();
|
||||
fs::write(from.join("index/p1/symbols"), "index").unwrap();
|
||||
fs::create_dir_all(from.join("cache/p1")).unwrap();
|
||||
fs::write(from.join("cache/p1/build"), "cache").unwrap();
|
||||
fs::create_dir_all(from.join("tools/demo/bin")).unwrap();
|
||||
fs::write(from.join("tools/demo/bin/demo.cmd"), "payload").unwrap();
|
||||
fs::write(from.join("history"), "roaming stays").unwrap();
|
||||
|
||||
// Nothing to move: no marker is written.
|
||||
assert!(!schedule_custom_migration_in(&marker, &from, &from).unwrap());
|
||||
assert!(!marker.exists());
|
||||
|
||||
assert!(schedule_custom_migration_in(&marker, &from, &to).unwrap());
|
||||
apply_custom_migration_in(&marker, &to);
|
||||
assert!(!marker.exists());
|
||||
assert_eq!(fs::read_to_string(to.join("index/p1/symbols")).unwrap(), "index");
|
||||
assert_eq!(fs::read_to_string(to.join("cache/p1/build")).unwrap(), "cache");
|
||||
assert!(to.join("tools/demo/bin/demo.cmd").exists());
|
||||
assert!(!from.join("index").exists());
|
||||
// Roaming content is not local data and stays behind.
|
||||
assert_eq!(fs::read_to_string(from.join("history")).unwrap(), "roaming stays");
|
||||
|
||||
// A migration aimed elsewhere is dropped without touching anything.
|
||||
fs::create_dir_all(from.join("lsp")).unwrap();
|
||||
assert!(schedule_custom_migration_in(&marker, &from, &tmp.path().join("elsewhere")).unwrap());
|
||||
apply_custom_migration_in(&marker, &to);
|
||||
assert!(!marker.exists());
|
||||
assert!(from.join("lsp").exists());
|
||||
assert!(!to.join("lsp").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn custom_entries_cover_logs_and_agent_tools() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let entries = custom_entries(tmp.path());
|
||||
for expected in ["logs", "agents/tools", "index", "cache", "lsp", "agents/registry.json"] {
|
||||
assert!(entries.contains(&PathBuf::from(expected)), "{entries:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -73,6 +73,11 @@ pub struct Host {
|
||||
pub processes: ProcessManager,
|
||||
/// Debug sessions (adapters run on this machine, next to the project).
|
||||
pub debugger: Arc<ide_dap::DebugManager>,
|
||||
/// Heap snapshots open in any window, captured from a debug session or opened from a file.
|
||||
pub heap: Arc<ide_heap::HeapRegistry>,
|
||||
/// The window that owns each open heap snapshot; its project's close closes them.
|
||||
pub(crate) heap_windows: Mutex<HashMap<ide_heap::HeapSnapshotId, String>>,
|
||||
pub(crate) heap_requests: ops::heap::HeapRequests,
|
||||
/// SSH connections of Settings › SSH Configurations.
|
||||
pub ssh: Arc<ide_ssh::SshManager>,
|
||||
/// SFTP deployment, remote file editing and runs on run targets.
|
||||
@@ -143,6 +148,9 @@ impl Host {
|
||||
terminals: TerminalManager::new(),
|
||||
processes: ProcessManager::new(),
|
||||
debugger: ide_dap::DebugManager::new(languages.clone()),
|
||||
heap: Arc::new(ide_heap::HeapRegistry::new(ide_heap::snapshots_root(&local_data_dir))),
|
||||
heap_windows: Mutex::default(),
|
||||
heap_requests: Default::default(),
|
||||
languages,
|
||||
syntax,
|
||||
config_dir,
|
||||
@@ -255,6 +263,8 @@ impl Host {
|
||||
for request in self.project_requests.write().remove(self.owner_window(window)).unwrap_or_default() {
|
||||
self.cancel_request(request);
|
||||
}
|
||||
// After the cancels: a capture or open still running then discards what it registers.
|
||||
ops::heap::close_window_snapshots(self, window);
|
||||
let project = self.projects.write().remove(window)?;
|
||||
if self.windows_for_root(&project.root).is_empty()
|
||||
&& let Some(plugins) = self.plugin_host()
|
||||
@@ -405,9 +415,12 @@ impl Host {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stops what this host started: notebook kernels, language servers, agents, runs and terminals.
|
||||
/// Stops what this host started: heap snapshots, notebook kernels, language servers, agents,
|
||||
/// runs and terminals.
|
||||
/// For a process that exits without dropping the host (the remote server).
|
||||
pub async fn shutdown(&self) {
|
||||
// A capture's dump may hold secrets from the debuggee's memory: delete it before exiting.
|
||||
ops::heap::shutdown(self).await;
|
||||
self.environment.shutdown();
|
||||
self.processes.kill_all();
|
||||
self.terminals.kill_all();
|
||||
|
||||
@@ -51,7 +51,12 @@ impl Call {
|
||||
pub async fn ask(&self, request: ide_core::event::McpUiRequest) -> IdeResult<()> {
|
||||
self.host.mcp_requests.ask(self, request).await
|
||||
}
|
||||
/// Asks the user before a tool changes the project or starts a process, unless brave mode
|
||||
/// is on — read from the IDE's own settings only, which a project's settings can't set.
|
||||
pub async fn confirm(&self, subject: &str, detail: Option<String>) -> IdeResult<()> {
|
||||
if self.host.global_settings.effective().mcp_server.run_without_confirmation {
|
||||
return self.project.require_trust();
|
||||
}
|
||||
self.ask(ide_core::event::McpUiRequest::Confirm {
|
||||
client: self.client.clone(),
|
||||
tool: self.tool.clone(),
|
||||
|
||||
+311
-28
@@ -45,26 +45,12 @@ fn innermost_repo<T>(repos: Vec<(PathBuf, T)>, path: &Path) -> Option<(PathBuf,
|
||||
// Read tools must never invoke repository-configured executables, including in Restricted Mode.
|
||||
fn read_command(git: &str, root: &Path) -> std::process::Command {
|
||||
let mut command = std::process::Command::new(git);
|
||||
ide_vcs::restrict(&mut command);
|
||||
command
|
||||
.current_dir(root)
|
||||
.args([
|
||||
"-c",
|
||||
"core.fsmonitor=false",
|
||||
"-c",
|
||||
"core.hooksPath=",
|
||||
"-c",
|
||||
"diff.external=",
|
||||
"-c",
|
||||
"core.attributesFile=",
|
||||
"-c",
|
||||
"submodule.recurse=false",
|
||||
"-c",
|
||||
"color.ui=false",
|
||||
])
|
||||
.env("GIT_NO_LAZY_FETCH", "1")
|
||||
.args(["-c", "color.ui=false"])
|
||||
.env("GIT_TERMINAL_PROMPT", "0")
|
||||
.env("GIT_OPTIONAL_LOCKS", "0")
|
||||
.env_remove("GIT_EXTERNAL_DIFF")
|
||||
.env_remove("GIT_DIR")
|
||||
.env_remove("GIT_WORK_TREE")
|
||||
.env_remove("GIT_INDEX_FILE");
|
||||
@@ -72,6 +58,32 @@ fn read_command(git: &str, root: &Path) -> std::process::Command {
|
||||
command
|
||||
}
|
||||
|
||||
/// `-c` arguments that empty the clean and process filters the repository's own config names:
|
||||
/// git runs them to compare a changed file with the index. Filters of the user's config stay
|
||||
/// (Git LFS), as they are not the repository's choice.
|
||||
fn filter_overrides(git: &str, root: &Path) -> IdeResult<Vec<String>> {
|
||||
let mut cmd = read_command(git, root);
|
||||
cmd.args(["config", "--null", "--list", "--show-scope"]);
|
||||
let out = cmd.output().map_err(|e| IdeError::unavailable(format!("cannot run git: {e}")))?;
|
||||
if !out.status.success() {
|
||||
return Err(IdeError::internal(String::from_utf8_lossy(&out.stderr).trim().to_owned()));
|
||||
}
|
||||
repository_filter_overrides(&out.stdout)
|
||||
}
|
||||
|
||||
/// [`filter_overrides`] out of `listing`, the output of `git config --null --list --show-scope`.
|
||||
fn repository_filter_overrides(listing: &[u8]) -> IdeResult<Vec<String>> {
|
||||
let mut fields = listing.split(|b| *b == 0);
|
||||
let mut own = Vec::new();
|
||||
while let (Some(scope), Some(record)) = (fields.next(), fields.next()) {
|
||||
if matches!(scope, b"local" | b"worktree") {
|
||||
own.extend_from_slice(record);
|
||||
own.push(0);
|
||||
}
|
||||
}
|
||||
ide_vcs::filter_overrides_from(&own)
|
||||
}
|
||||
|
||||
/// The branch HEAD is on (unborn too); `None` when HEAD is detached or git fails.
|
||||
fn read_branch(git: &str, root: &Path) -> Option<String> {
|
||||
let mut cmd = read_command(git, root);
|
||||
@@ -167,7 +179,10 @@ fn git_executable(call: &Call) -> String {
|
||||
/// `git status` of the repository at `root`, paths relative to it.
|
||||
fn read_status(git: &str, root: &Path, untracked: bool, ignored: bool) -> IdeResult<Vec<StatusEntry>> {
|
||||
let mut cmd = read_command(git, root);
|
||||
cmd.args(filter_overrides(git, root)?);
|
||||
cmd.args(["-c", "core.quotepath=false", "status", "--porcelain", "-z", "--find-renames"]);
|
||||
// git asks a submodule with changes for its own status, which runs under the submodule's config.
|
||||
cmd.arg("--ignore-submodules=dirty");
|
||||
// git refuses `--ignored=matching` with `--untracked-files=no`: untracked files are dropped below.
|
||||
cmd.arg(if untracked || ignored { "--untracked-files=all" } else { "--untracked-files=no" });
|
||||
if ignored {
|
||||
@@ -183,6 +198,39 @@ fn read_status(git: &str, root: &Path, untracked: bool, ignored: bool) -> IdeRes
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// `git diff` of the repository at `root` (of `file` alone, when given): the index against HEAD
|
||||
/// when `staged`, else disk against the index.
|
||||
fn read_diff(
|
||||
git: &str,
|
||||
root: &Path,
|
||||
file: Option<&Path>,
|
||||
staged: bool,
|
||||
cancel: &ide_core::CancellationToken,
|
||||
) -> IdeResult<String> {
|
||||
let mut cmd = read_command(git, root);
|
||||
cmd.args(filter_overrides(git, root)?);
|
||||
cmd.args(["--no-pager", "diff", "--no-ext-diff", "--no-textconv"]);
|
||||
// git asks a submodule with changes for its own status, and for its own diff when the
|
||||
// repository's `diff.submodule` says so: both run under the submodule's config.
|
||||
cmd.args(["--ignore-submodules=dirty", "--submodule=short"]);
|
||||
if staged {
|
||||
cmd.arg("--cached");
|
||||
}
|
||||
cmd.arg("--");
|
||||
if let Some(file) = file {
|
||||
cmd.arg(file.strip_prefix(root).map_err(|e| IdeError::invalid(e.to_string()))?);
|
||||
}
|
||||
ide_core::process::prepare(&mut cmd);
|
||||
if cancel.is_cancelled() {
|
||||
return Err(IdeError::cancelled());
|
||||
}
|
||||
let out = cmd.output()?;
|
||||
if !out.status.success() {
|
||||
return Err(IdeError::unavailable(String::from_utf8_lossy(&out.stderr).trim()));
|
||||
}
|
||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||
}
|
||||
|
||||
/// A git repository in the project, for paths outside the repositories IDEA knows: the
|
||||
/// closest folder with `.git` from `start` up to the project root.
|
||||
fn find_git_root(root: &Path, start: &Path) -> Option<PathBuf> {
|
||||
@@ -307,18 +355,7 @@ fn git_diff() -> Tool {
|
||||
let staged = call.flag("staged", false)?;
|
||||
let git = git_executable(&call);
|
||||
let cancel = call.cancel.clone();
|
||||
let text = call.host.blocking(move || {
|
||||
let mut cmd = read_command(&git, &root);
|
||||
cmd.args(["--no-pager", "diff", "--no-ext-diff", "--no-textconv"]);
|
||||
if staged { cmd.arg("--cached"); }
|
||||
cmd.arg("--");
|
||||
if let Some(file) = file { cmd.arg(file.strip_prefix(&root).map_err(|e| IdeError::invalid(e.to_string()))?); }
|
||||
ide_core::process::prepare(&mut cmd);
|
||||
if cancel.is_cancelled() { return Err(IdeError::cancelled()); }
|
||||
let out = cmd.output()?;
|
||||
if !out.status.success() { return Err(IdeError::unavailable(String::from_utf8_lossy(&out.stderr).trim())); }
|
||||
Ok(String::from_utf8_lossy(&out.stdout).into_owned())
|
||||
}).await?;
|
||||
let text = call.host.blocking(move || read_diff(&git, &root, file.as_deref(), staged, &cancel)).await?;
|
||||
Ok(Output::text(text))
|
||||
})
|
||||
.param(Param::optional("repositoryPathRelativeToProject", ParamType::String, "Select the containing repository"))
|
||||
@@ -342,6 +379,252 @@ mod tests {
|
||||
assert!(e[3].conflicted() && !e[3].staged() && !e[3].unstaged());
|
||||
}
|
||||
|
||||
/// Security audit lead 2 (variant): reading status or a diff must not run the repository's filters.
|
||||
#[test]
|
||||
fn status_and_diff_do_not_run_repository_filters() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(dir.path()).unwrap();
|
||||
let git = |args: &[&str]| {
|
||||
let out = std::process::Command::new("git").current_dir(&root).args(args).output().unwrap();
|
||||
assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr));
|
||||
};
|
||||
git(&["init", "-q"]);
|
||||
git(&["config", "user.email", "[email protected]"]);
|
||||
git(&["config", "user.name", "t"]);
|
||||
git(&["config", "commit.gpgsign", "false"]);
|
||||
std::fs::write(root.join(".gitattributes"), "*.txt filter=marker\n").unwrap();
|
||||
std::fs::write(root.join("a.txt"), "one\n").unwrap();
|
||||
git(&["add", "."]);
|
||||
git(&["commit", "-qm", "init"]);
|
||||
git(&["config", "filter.marker.clean", "echo x > executed-marker; cat"]);
|
||||
git(&["config", "filter.marker.process", "echo x > executed-marker; cat"]);
|
||||
git(&["config", "filter.marker.required", "true"]);
|
||||
let marker = root.join("executed-marker");
|
||||
let mut ran = Vec::new();
|
||||
// The size it has in the index, so git has to read the file to tell whether it changed.
|
||||
std::fs::write(root.join("a.txt"), "two\n").unwrap();
|
||||
let status = read_status("git", &root, false, false);
|
||||
if marker.exists() {
|
||||
ran.push("status");
|
||||
}
|
||||
let _ = std::fs::remove_file(&marker);
|
||||
std::fs::write(root.join("a.txt"), "one\ntwo\nthree\n").unwrap();
|
||||
let diff = read_diff("git", &root, None, false, &ide_core::CancellationToken::new());
|
||||
if marker.exists() {
|
||||
ran.push("diff");
|
||||
}
|
||||
// Control: the fixture's filter really runs when git is used directly.
|
||||
let _ = std::fs::remove_file(&marker);
|
||||
std::fs::write(root.join("a.txt"), "six\n").unwrap();
|
||||
let _ = std::process::Command::new("git").current_dir(&root).args(["status", "--porcelain"]).output();
|
||||
assert!(marker.exists(), "control: plain `git status` did not run the filter, fixture is not live");
|
||||
assert!(ran.is_empty(), "a read tool ran the repository's filter via: {ran:?}");
|
||||
assert_eq!(status.unwrap().len(), 1);
|
||||
let diff = diff.unwrap();
|
||||
assert!(diff.contains("+three"), "{diff}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_filters_of_the_repository_s_own_config_are_emptied() {
|
||||
let listing = b"system\0filter.lfs.clean\ngit-lfs clean -- %f\0\
|
||||
global\0filter.lfs.process\ngit-lfs filter-process\0\
|
||||
global\0filter.lfs.required\ntrue\0\
|
||||
local\0core.bare\nfalse\0\
|
||||
local\0filter.a.clean\nx\0\
|
||||
worktree\0filter.b.process\ny\0\
|
||||
local\0filter.a.required\ntrue\0\
|
||||
command\0filter.c.clean\nz\0";
|
||||
assert_eq!(
|
||||
repository_filter_overrides(listing).unwrap(),
|
||||
["-c", "filter.a.clean=", "-c", "filter.b.process=", "-c", "filter.a.required=false"]
|
||||
);
|
||||
}
|
||||
|
||||
fn git(dir: &Path, args: &[&str]) {
|
||||
let out = std::process::Command::new("git").current_dir(dir).args(args).output().unwrap();
|
||||
assert!(out.status.success(), "git {args:?}: {}", String::from_utf8_lossy(&out.stderr));
|
||||
}
|
||||
|
||||
/// A repository at `dir` with `a.txt` committed.
|
||||
fn repository(dir: &Path, attributes: &[u8]) {
|
||||
std::fs::create_dir_all(dir).unwrap();
|
||||
git(dir, &["init", "-q"]);
|
||||
git(dir, &["config", "user.email", "[email protected]"]);
|
||||
git(dir, &["config", "user.name", "t"]);
|
||||
git(dir, &["config", "commit.gpgsign", "false"]);
|
||||
std::fs::write(dir.join(".gitattributes"), attributes).unwrap();
|
||||
std::fs::write(dir.join("a.txt"), "one\n").unwrap();
|
||||
git(dir, &["add", "."]);
|
||||
git(dir, &["commit", "-qm", "init"]);
|
||||
}
|
||||
|
||||
/// A shell command that leaves `marker` behind and passes its input on.
|
||||
fn leaves(marker: &Path) -> String {
|
||||
format!("echo x > '{}'; cat", marker.to_string_lossy().replace('\\', "/"))
|
||||
}
|
||||
|
||||
/// Whether a program of the fixture left `marker` behind; removes it.
|
||||
fn left(marker: &Path) -> bool {
|
||||
let was = marker.exists();
|
||||
let _ = std::fs::remove_file(marker);
|
||||
was
|
||||
}
|
||||
|
||||
/// Reads the status and then the diff of `root` after rewriting `file`, then has plain git
|
||||
/// read the status as a control. Each write keeps the size the file has in the index, so git
|
||||
/// has to read it to tell whether it changed.
|
||||
fn read_both(root: &Path, file: &Path, marker: &Path) -> (IdeResult<Vec<StatusEntry>>, IdeResult<String>) {
|
||||
let mut ran = Vec::new();
|
||||
std::fs::write(file, "two\n").unwrap();
|
||||
let status = read_status("git", root, false, false);
|
||||
if left(marker) {
|
||||
ran.push("status");
|
||||
}
|
||||
std::fs::write(file, "six\n").unwrap();
|
||||
let diff = read_diff("git", root, None, false, &ide_core::CancellationToken::new());
|
||||
if left(marker) {
|
||||
ran.push("diff");
|
||||
}
|
||||
std::fs::write(file, "ten\n").unwrap();
|
||||
let _ = std::process::Command::new("git").current_dir(root).args(["status", "--porcelain"]).output();
|
||||
assert!(left(marker), "control: plain `git status` ran no program of the fixture, it is not live");
|
||||
assert!(ran.is_empty(), "a read tool ran a program of the repository via: {ran:?}");
|
||||
(status, diff)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn status_and_diff_do_not_run_filters_of_any_repository_config() {
|
||||
type Configure = fn(&Path, &str);
|
||||
let cases: [(&str, Configure); 4] = [
|
||||
("clean", |root, command| git(root, &["config", "filter.marker.clean", command])),
|
||||
("process", |root, command| git(root, &["config", "filter.marker.process", command])),
|
||||
("included file", |root, command| {
|
||||
let included = format!("[filter \"marker\"]\n\tclean = \"{command}\"\n");
|
||||
std::fs::write(root.join("extra.gitconfig"), included).unwrap();
|
||||
git(root, &["config", "include.path", "../extra.gitconfig"]);
|
||||
}),
|
||||
("worktree config", |root, command| {
|
||||
git(root, &["config", "extensions.worktreeConfig", "true"]);
|
||||
git(root, &["config", "--worktree", "filter.marker.clean", command]);
|
||||
}),
|
||||
];
|
||||
for (case, configure) in cases {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(dir.path()).unwrap().join("repo");
|
||||
let marker = root.with_file_name("marker");
|
||||
repository(&root, b"*.txt filter=marker\n");
|
||||
configure(&root, &leaves(&marker));
|
||||
let (status, diff) = read_both(&root, &root.join("a.txt"), &marker);
|
||||
assert_eq!(status.unwrap().len(), 1, "{case}");
|
||||
let diff = diff.unwrap();
|
||||
assert!(diff.contains("+six"), "{case}: {diff}");
|
||||
}
|
||||
}
|
||||
|
||||
/// git splits `-c name=value` at the first `=` and takes no name that is not UTF-8 on every
|
||||
/// platform, so such a filter cannot be emptied: the tools read nothing instead.
|
||||
#[test]
|
||||
fn status_and_diff_refuse_a_repository_filter_git_cannot_be_told_to_skip() {
|
||||
for name in [&b"x=y"[..], &[0xE9]] {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(dir.path()).unwrap().join("repo");
|
||||
let marker = root.with_file_name("marker");
|
||||
repository(&root, &[b"*.txt filter=", name, b"\n"].concat());
|
||||
let section =
|
||||
[b"[filter \"", name, format!("\"]\n\tclean = \"{}\"\n", leaves(&marker)).as_bytes()].concat();
|
||||
let config = root.join(".git").join("config");
|
||||
std::fs::write(&config, [std::fs::read(&config).unwrap(), section].concat()).unwrap();
|
||||
let (status, diff) = read_both(&root, &root.join("a.txt"), &marker);
|
||||
assert!(status.is_err() && diff.is_err(), "{name:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// git asks a submodule for its own status to tell whether it is dirty, under the
|
||||
/// submodule's config.
|
||||
#[test]
|
||||
fn status_and_diff_do_not_run_filters_of_a_submodule() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(dir.path()).unwrap().join("repo");
|
||||
let (sub, marker) = (root.join("sub"), root.with_file_name("marker"));
|
||||
repository(&root, b"");
|
||||
repository(&sub, b"*.txt filter=marker\n");
|
||||
git(&root, &["add", "sub"]);
|
||||
git(&root, &["commit", "-qm", "submodule"]);
|
||||
git(&sub, &["config", "filter.marker.clean", &leaves(&marker)]);
|
||||
git(&root, &["config", "diff.ignoreSubmodules", "none"]);
|
||||
git(&root, &["config", "submodule.sub.ignore", "none"]);
|
||||
let (status, diff) = read_both(&root, &sub.join("a.txt"), &marker);
|
||||
status.unwrap();
|
||||
diff.unwrap();
|
||||
}
|
||||
|
||||
/// `git diff` writes the index back when it finds a file whose stat changed and whose content
|
||||
/// did not, and git runs the hooks a config names for `post-index-change` on that write.
|
||||
#[test]
|
||||
fn status_and_diff_do_not_run_hooks_of_the_repository_s_config() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(dir.path()).unwrap().join("repo");
|
||||
let marker = root.with_file_name("marker");
|
||||
repository(&root, b"");
|
||||
git(&root, &["config", "hook.marker.event", "post-index-change"]);
|
||||
let command = format!("echo x > '{}'", marker.to_string_lossy().replace('\\', "/"));
|
||||
git(&root, &["config", "hook.marker.command", &command]);
|
||||
let listed = std::process::Command::new("git")
|
||||
.current_dir(&root)
|
||||
.args(["hook", "list", "post-index-change"])
|
||||
.output()
|
||||
.unwrap();
|
||||
if !String::from_utf8_lossy(&listed.stdout).contains("marker") {
|
||||
return; // This git has no hooks named by a config.
|
||||
}
|
||||
// The content it has in the index, with a time of its own that is not the index's.
|
||||
let touch = |seconds: u64| {
|
||||
let file = std::fs::File::options().write(true).open(root.join("a.txt")).unwrap();
|
||||
file.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(seconds)).unwrap();
|
||||
};
|
||||
let mut ran = Vec::new();
|
||||
touch(60);
|
||||
let status = read_status("git", &root, false, false);
|
||||
if left(&marker) {
|
||||
ran.push("status");
|
||||
}
|
||||
touch(120);
|
||||
let diff = read_diff("git", &root, None, false, &ide_core::CancellationToken::new());
|
||||
if left(&marker) {
|
||||
ran.push("diff");
|
||||
}
|
||||
touch(180);
|
||||
let _ = std::process::Command::new("git").current_dir(&root).args(["--no-pager", "diff"]).output();
|
||||
assert!(left(&marker), "control: plain `git diff` did not run the hook, fixture is not live");
|
||||
assert!(ran.is_empty(), "a read tool ran a hook of the repository's config via: {ran:?}");
|
||||
assert!(status.unwrap().is_empty());
|
||||
assert_eq!(diff.unwrap(), "");
|
||||
}
|
||||
|
||||
/// `diff.submodule=diff` has git run `git diff` in the submodule, under the submodule's config.
|
||||
#[test]
|
||||
fn diff_does_not_run_a_diff_driver_of_a_submodule() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let root = dunce::canonicalize(dir.path()).unwrap().join("repo");
|
||||
let (sub, marker) = (root.join("sub"), root.with_file_name("marker"));
|
||||
repository(&root, b"");
|
||||
repository(&sub, b"*.txt diff=marker\n");
|
||||
git(&root, &["add", "sub"]);
|
||||
git(&root, &["commit", "-qm", "submodule"]);
|
||||
std::fs::write(sub.join("a.txt"), "one\ntwo\n").unwrap();
|
||||
git(&sub, &["commit", "-qam", "second"]);
|
||||
git(&sub, &["config", "diff.marker.command", &leaves(&marker)]);
|
||||
git(&root, &["config", "diff.submodule", "diff"]);
|
||||
let diff = read_diff("git", &root, None, false, &ide_core::CancellationToken::new());
|
||||
let ran = left(&marker);
|
||||
let _ = std::process::Command::new("git").current_dir(&root).args(["--no-pager", "diff"]).output();
|
||||
assert!(left(&marker), "control: plain `git diff` did not run the diff driver, fixture is not live");
|
||||
assert!(!ran, "git_diff ran the submodule's diff driver");
|
||||
let diff = diff.unwrap();
|
||||
assert!(diff.contains("Subproject commit"), "{diff}");
|
||||
assert_eq!(read_status("git", &root, false, false).unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn paths_through_a_symlinked_root_compare_canonically() {
|
||||
|
||||
@@ -5,8 +5,9 @@ use crate::Host;
|
||||
use crate::rpc::Dispatcher;
|
||||
use ide_agent::registry::{AgentRegistry, RegistryPaths};
|
||||
use ide_agent::{
|
||||
AgentChange, AgentCommand, AgentEntry, AgentEvent, AgentProbe, AgentPrompt, AgentProvider, AgentQueued, AgentTask,
|
||||
ExternalSession, McpScope, McpServer, McpServerInput, NewAgentTask,
|
||||
AcpImportOutcome, AgentAcceptOutcome, AgentChange, AgentCommand, AgentEntry, AgentEvent, AgentProbe, AgentPrompt,
|
||||
AgentProvider, AgentQueued, AgentTask, CustomAgentInput, ExternalSession, McpScope, McpServer, McpServerInput,
|
||||
NewAgentTask,
|
||||
};
|
||||
use ide_core::error::ErrorKind;
|
||||
use ide_core::{IdeError, IdeEvent, IdeResult, IpcPath, RequestId, SharedSink, TaskId};
|
||||
@@ -44,6 +45,24 @@ pub fn set_env(
|
||||
host.project(window)?.agents.set_builtin_env(&id, env)
|
||||
}
|
||||
|
||||
/// Saves a new custom ACP agent through its agent file.
|
||||
pub fn save_custom_agent(host: &Host, window: &str, input: CustomAgentInput) -> IdeResult<()> {
|
||||
host.require_trust(window)?;
|
||||
host.project(window)?.agents.save_custom_agent(input)
|
||||
}
|
||||
|
||||
/// Removes a custom agent's file (registry installs go through `registry_remove`).
|
||||
pub fn remove_custom_agent(host: &Host, window: &str, id: String) -> IdeResult<()> {
|
||||
host.require_trust(window)?;
|
||||
host.project(window)?.agents.remove_custom_agent(&id)
|
||||
}
|
||||
|
||||
/// Imports a JetBrains `acp.json` file's `agent_servers` entries as custom agents.
|
||||
pub fn import_acp_agents(host: &Host, window: &str, text: String) -> IdeResult<Vec<AcpImportOutcome>> {
|
||||
host.require_trust(window)?;
|
||||
host.project(window)?.agents.import_acp_agents(&text)
|
||||
}
|
||||
|
||||
/// Where ACP Registry agents are saved (`<config>/agents`, like the user's own) and installed.
|
||||
pub(crate) fn registry_paths(host: &Host) -> RegistryPaths {
|
||||
RegistryPaths::new(&host.config_dir, &host.local_data_dir)
|
||||
@@ -181,15 +200,18 @@ pub fn entries(host: &Host, window: &str, task: String, subagent: Option<String>
|
||||
host.project(window)?.agents.entries(&task, subagent)
|
||||
}
|
||||
|
||||
/// Sessions on this machine that no task holds yet, newest first (#312).
|
||||
pub fn external_sessions(host: &Host, window: &str) -> IdeResult<Vec<ExternalSession>> {
|
||||
Ok(host.project(window)?.agents.external_sessions())
|
||||
/// Sessions of the project on this machine that no task holds yet, newest first. Reads
|
||||
/// transcripts from disk on the blocking pool.
|
||||
pub async fn external_sessions(host: &Host, window: &str) -> IdeResult<Vec<ExternalSession>> {
|
||||
let agents = host.project(window)?.agents.clone();
|
||||
tokio::task::spawn_blocking(move || agents.external_sessions()).await.map_err(join_err)
|
||||
}
|
||||
|
||||
/// Imports an external session as a task: its first turn loads and replays it (#312).
|
||||
pub fn import_session(host: &Host, window: &str, provider: String, session_id: String) -> IdeResult<AgentTask> {
|
||||
/// Imports an external session as a task: its first turn loads and replays it.
|
||||
pub async fn import_session(host: &Host, window: &str, provider: String, session_id: String) -> IdeResult<AgentTask> {
|
||||
host.require_trust(window)?;
|
||||
host.project(window)?.agents.import_session(&provider, &session_id)
|
||||
let agents = host.project(window)?.agents.clone();
|
||||
tokio::task::spawn_blocking(move || agents.import_session(&provider, &session_id)).await.map_err(join_err)?
|
||||
}
|
||||
|
||||
pub async fn create(host: &Host, window: &str, mut request: NewAgentTask) -> IdeResult<AgentTask> {
|
||||
@@ -294,6 +316,18 @@ pub async fn rollback_changes(host: &Host, window: &str, task: String, paths: Op
|
||||
host.project(window)?.agents.rollback_changes(&task, paths).await
|
||||
}
|
||||
|
||||
/// Accept: copies a worktree task's changed files among `paths` (all without) into the
|
||||
/// local checkout, uncommitted; files the checkout changed itself are left alone and reported.
|
||||
pub async fn accept_changes(
|
||||
host: &Host,
|
||||
window: &str,
|
||||
task: String,
|
||||
paths: Option<Vec<String>>,
|
||||
) -> IdeResult<AgentAcceptOutcome> {
|
||||
host.require_trust(window)?;
|
||||
host.project(window)?.agents.accept_changes(&task, paths).await
|
||||
}
|
||||
|
||||
type McpServerMap = std::collections::BTreeMap<String, ide_core::settings::AgentMcpServer>;
|
||||
|
||||
fn mcp_maps(host: &Host, window: &str) -> (McpServerMap, McpServerMap, Option<std::sync::Arc<crate::Project>>) {
|
||||
@@ -409,9 +443,9 @@ pub(crate) fn register(d: &mut Dispatcher) {
|
||||
d.add("agent_commands", |c| async move { commands(&c.host, &c.window, c.arg("provider")?, c.arg("task")?).await });
|
||||
d.add("agent_entries", |c| async move { entries(&c.host, &c.window, c.arg("task")?, c.arg("subagent")?) });
|
||||
d.add("agent_create", |c| async move { create(&c.host, &c.window, c.arg("request")?).await });
|
||||
d.add("agent_external_sessions", |c| async move { external_sessions(&c.host, &c.window) });
|
||||
d.add("agent_external_sessions", |c| async move { external_sessions(&c.host, &c.window).await });
|
||||
d.add("agent_import_session", |c| async move {
|
||||
import_session(&c.host, &c.window, c.arg("provider")?, c.arg("sessionId")?)
|
||||
import_session(&c.host, &c.window, c.arg("provider")?, c.arg("sessionId")?).await
|
||||
});
|
||||
d.add("agent_prompt", |c| async move {
|
||||
prompt(&c.host, &c.window, c.arg("task")?, c.arg("prompt")?, c.arg("now")?).await
|
||||
@@ -446,6 +480,9 @@ pub(crate) fn register(d: &mut Dispatcher) {
|
||||
d.add("agent_rollback_changes", |c| async move {
|
||||
rollback_changes(&c.host, &c.window, c.arg("task")?, c.arg("paths")?).await
|
||||
});
|
||||
d.add("agent_accept_changes", |c| async move {
|
||||
accept_changes(&c.host, &c.window, c.arg("task")?, c.arg("paths")?).await
|
||||
});
|
||||
}
|
||||
|
||||
fn validate_project_root(root: &std::path::Path, requested: Option<&std::path::Path>) -> IdeResult<std::path::PathBuf> {
|
||||
|
||||
@@ -17,9 +17,11 @@ pub async fn fs_read_range(_host: &Host, _window: &str, path: PathBuf, offset: u
|
||||
blocking(move || ide_vfs::read_range(&path, offset, len)).await
|
||||
}
|
||||
|
||||
/// An image file as a `data:` URL.
|
||||
pub async fn fs_read_image(_host: &Host, _window: &str, path: PathBuf) -> IdeResult<String> {
|
||||
blocking(move || ide_vfs::read_data_url(&path)).await
|
||||
/// An image file as a `data:` URL. A network path is read only from a share an open project is
|
||||
/// on: the path may come from Markdown, and opening a share signs in to its server.
|
||||
pub async fn fs_read_image(host: &Host, _window: &str, path: PathBuf) -> IdeResult<String> {
|
||||
let opened: Vec<PathBuf> = host.projects().iter().map(|project| project.root.clone()).collect();
|
||||
blocking(move || ide_vfs::read_data_url(&path, &opened)).await
|
||||
}
|
||||
|
||||
/// Commands of this module the remote server answers.
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
mod kotlin;
|
||||
#[cfg(test)]
|
||||
mod kotlin_tests;
|
||||
mod output;
|
||||
|
||||
use crate::Host;
|
||||
@@ -41,6 +44,7 @@ pub async fn debug_start(host: &Host, window: &str, config: LaunchConfig) -> Ide
|
||||
})
|
||||
.await?;
|
||||
}
|
||||
config = kotlin::choose_kotlin_debugger(host, window, config).await?;
|
||||
if config.adapter == ide_dap::kotlin::ADAPTER {
|
||||
let data_dir = host.local_data_dir.clone();
|
||||
let runtime_root = config.cwd.clone().map(PathBuf::from).unwrap_or_default();
|
||||
|
||||
@@ -0,0 +1,415 @@
|
||||
//! Kotlin debugging: kotlin-lsp's debugger when the project's kotlin-lsp can serve it, else
|
||||
//! kotlin-debug-adapter with a notice saying why.
|
||||
|
||||
use crate::Host;
|
||||
use ide_core::event::NotifyLevel;
|
||||
use ide_core::task::CancelHandle;
|
||||
use ide_core::{CancellationToken, IdeEvent, IdeResult, SharedSink, TaskId};
|
||||
use ide_dap::LaunchConfig;
|
||||
use ide_lsp::{KotlinDebugServer, KotlinDebugUnavailable};
|
||||
use parking_lot::Mutex;
|
||||
use std::collections::HashSet;
|
||||
use std::future::Future;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
pub(super) const KOTLIN_IMPORT_WAIT: Duration = Duration::from_secs(120);
|
||||
/// A ready kotlin-lsp answers well within this; only a longer wait shows a task.
|
||||
const TASK_DELAY: Duration = Duration::from_millis(500);
|
||||
|
||||
// Catalogued as written (`{detail}` replaced at runtime): the text catalog test counts these
|
||||
// literals as the use of its entries.
|
||||
const TASK_TITLE: &str = "Waiting for kotlin-lsp to import the project";
|
||||
const NOTICE_TITLE: &str = "Debugging Kotlin with kotlin-debug-adapter";
|
||||
const NOT_CONFIGURED: &str = "kotlin-lsp is not the Kotlin language server of this project.";
|
||||
const NOT_INSTALLED: &str = "kotlin-lsp is not installed.";
|
||||
const NOT_STARTED: &str = "kotlin-lsp did not start: {detail}";
|
||||
const STILL_IMPORTING: &str =
|
||||
"kotlin-lsp is still importing the project. Debug again once the import has finished to use its debugger.";
|
||||
const IMPORT_FAILED: &str = "kotlin-lsp could not import the project.";
|
||||
const FAILED: &str = "kotlin-lsp could not start its debugger: {detail}";
|
||||
|
||||
/// `config` for kotlin-lsp's debug server on `port`; the rest of the launch stays as it is.
|
||||
pub(super) fn use_kotlin_lsp(mut config: LaunchConfig, port: u16) -> IdeResult<LaunchConfig> {
|
||||
let mut extra = extra_object(&config)?;
|
||||
extra.insert("debugServer".into(), port.into());
|
||||
config.adapter = ide_dap::kotlin_lsp::ADAPTER.into();
|
||||
config.extra_json = Some(serde_json::Value::Object(extra).to_string());
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
/// The dedupe key of `reason` and the notice body (English, translated by the UI's catalog).
|
||||
pub(super) fn fallback_notice(reason: &KotlinDebugUnavailable) -> (&'static str, String) {
|
||||
match reason {
|
||||
KotlinDebugUnavailable::NotConfigured => ("notConfigured", NOT_CONFIGURED.into()),
|
||||
KotlinDebugUnavailable::NotInstalled => ("notInstalled", NOT_INSTALLED.into()),
|
||||
KotlinDebugUnavailable::NotStarted(detail) => ("notStarted", NOT_STARTED.replace("{detail}", detail)),
|
||||
KotlinDebugUnavailable::StillImporting => ("stillImporting", STILL_IMPORTING.into()),
|
||||
KotlinDebugUnavailable::ImportFailed => ("importFailed", IMPORT_FAILED.into()),
|
||||
KotlinDebugUnavailable::Failed(detail) => ("failed", FAILED.replace("{detail}", detail)),
|
||||
}
|
||||
}
|
||||
|
||||
/// A kotlin-debug-adapter launch becomes a kotlin-lsp one when the project's kotlin-lsp has
|
||||
/// imported the module and started its debug server; otherwise it stays on
|
||||
/// kotlin-debug-adapter and the window is told why (once per reason).
|
||||
pub(super) async fn choose_kotlin_debugger(host: &Host, window: &str, config: LaunchConfig) -> IdeResult<LaunchConfig> {
|
||||
if config.adapter == ide_dap::kotlin_lsp::ADAPTER {
|
||||
require_debug_server(&config)?;
|
||||
return Ok(config);
|
||||
}
|
||||
if config.adapter != ide_dap::kotlin::ADAPTER {
|
||||
return Ok(config);
|
||||
}
|
||||
let Ok(project) = host.project(window) else {
|
||||
tracing::info!(
|
||||
"debugging Kotlin with kotlin-debug-adapter: {}",
|
||||
fallback_notice(&KotlinDebugUnavailable::NotConfigured).1
|
||||
);
|
||||
return Ok(config);
|
||||
};
|
||||
let path = debug_path(&config).unwrap_or_else(|| project.root.clone());
|
||||
let lsp = project.lsp.clone();
|
||||
let chosen = wait_for_debug_server(&project.sink, |cancel| async move {
|
||||
lsp.kotlin_debug_server(&path, KOTLIN_IMPORT_WAIT, &cancel).await
|
||||
})
|
||||
.await;
|
||||
match chosen {
|
||||
Ok(server) => use_kotlin_lsp(config, server.port),
|
||||
Err(reason) => {
|
||||
report_fallback(&project.kotlin_debug_notices, &project.sink, &reason);
|
||||
Ok(config)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// kotlin-lsp is only reached through the port its language server opened; without one
|
||||
/// ide-dap would try to spawn a `kotlin-lsp` adapter that does not exist.
|
||||
fn require_debug_server(config: &LaunchConfig) -> IdeResult<()> {
|
||||
match extra_object(config)?.get("debugServer") {
|
||||
None | Some(serde_json::Value::Null) => Err(ide_core::IdeError::invalid(
|
||||
"A kotlin-lsp debug configuration needs \"debugServer\": the port of kotlin-lsp's debug server. \
|
||||
Use kotlin-debug-adapter to let the IDE start it.",
|
||||
)),
|
||||
Some(_) => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
fn extra_object(config: &LaunchConfig) -> IdeResult<serde_json::Map<String, serde_json::Value>> {
|
||||
match config.extra_json.as_deref().map(serde_json::from_str).transpose() {
|
||||
Ok(None) => Ok(serde_json::Map::new()),
|
||||
Ok(Some(serde_json::Value::Object(extra))) => Ok(extra),
|
||||
Ok(Some(other)) => Err(ide_core::IdeError::invalid(format!("launch arguments are not an object: {other}"))),
|
||||
Err(e) => Err(ide_core::IdeError::invalid(e.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
/// The module to debug: the launch's `projectRoot`, else its working directory.
|
||||
fn debug_path(config: &LaunchConfig) -> Option<PathBuf> {
|
||||
let extra: Option<serde_json::Value> = config.extra_json.as_deref().and_then(|e| serde_json::from_str(e).ok());
|
||||
extra
|
||||
.as_ref()
|
||||
.and_then(|e| e.get("projectRoot")?.as_str())
|
||||
.filter(|root| !root.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.or_else(|| config.cwd.as_deref().map(PathBuf::from))
|
||||
}
|
||||
|
||||
/// Runs `debug_server` with a token the user can cancel; a wait longer than [`TASK_DELAY`]
|
||||
/// shows as a cancellable task, which ends however the wait does (also when dropped).
|
||||
async fn wait_for_debug_server<F, Fut>(
|
||||
sink: &SharedSink,
|
||||
debug_server: F,
|
||||
) -> Result<KotlinDebugServer, KotlinDebugUnavailable>
|
||||
where
|
||||
F: FnOnce(CancellationToken) -> Fut,
|
||||
Fut: Future<Output = Result<KotlinDebugServer, KotlinDebugUnavailable>>,
|
||||
{
|
||||
let cancel = CancellationToken::new();
|
||||
let id = TaskId::next();
|
||||
let handle = ide_core::task::on_cancel(id, {
|
||||
let cancel = cancel.clone();
|
||||
move || cancel.cancel()
|
||||
});
|
||||
let wait = debug_server(cancel);
|
||||
tokio::pin!(wait);
|
||||
if let Ok(answer) = tokio::time::timeout(TASK_DELAY, &mut wait).await {
|
||||
return answer;
|
||||
}
|
||||
let _task = WaitTask::start(sink.clone(), id, handle);
|
||||
wait.await
|
||||
}
|
||||
|
||||
/// The visible import wait; dropping it ends the task.
|
||||
struct WaitTask {
|
||||
sink: SharedSink,
|
||||
id: TaskId,
|
||||
_cancel: CancelHandle,
|
||||
}
|
||||
|
||||
impl WaitTask {
|
||||
fn start(sink: SharedSink, id: TaskId, cancel: CancelHandle) -> Self {
|
||||
let task = Self { sink, id, _cancel: cancel };
|
||||
task.emit(false);
|
||||
task
|
||||
}
|
||||
|
||||
fn emit(&self, done: bool) {
|
||||
self.sink.emit(IdeEvent::Task {
|
||||
id: self.id,
|
||||
title: TASK_TITLE.into(),
|
||||
message: None,
|
||||
fraction: None,
|
||||
done,
|
||||
cancellable: !done,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WaitTask {
|
||||
fn drop(&mut self) {
|
||||
self.emit(true);
|
||||
}
|
||||
}
|
||||
|
||||
/// Logs every fallback; notifies the window once per reason.
|
||||
fn report_fallback(notices: &Mutex<HashSet<&'static str>>, sink: &SharedSink, reason: &KotlinDebugUnavailable) {
|
||||
let (key, body) = fallback_notice(reason);
|
||||
tracing::info!("debugging Kotlin with kotlin-debug-adapter: {body}");
|
||||
if notices.lock().insert(key) {
|
||||
sink.emit(IdeEvent::Notify { level: NotifyLevel::Info, title: NOTICE_TITLE.into(), body: Some(body) });
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn launch(adapter: &str, extra: serde_json::Value) -> LaunchConfig {
|
||||
LaunchConfig {
|
||||
name: "AppKt".into(),
|
||||
adapter: adapter.into(),
|
||||
request: "launch".into(),
|
||||
program: None,
|
||||
args: vec![],
|
||||
cwd: Some("/cwd".into()),
|
||||
env: Default::default(),
|
||||
extra_json: Some(extra.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
fn extra(config: &LaunchConfig) -> serde_json::Value {
|
||||
serde_json::from_str(config.extra_json.as_deref().unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn reasons() -> Vec<KotlinDebugUnavailable> {
|
||||
vec![
|
||||
KotlinDebugUnavailable::NotConfigured,
|
||||
KotlinDebugUnavailable::NotInstalled,
|
||||
KotlinDebugUnavailable::NotStarted("needs JDK 21".into()),
|
||||
KotlinDebugUnavailable::StillImporting,
|
||||
KotlinDebugUnavailable::ImportFailed,
|
||||
KotlinDebugUnavailable::Failed("boom".into()),
|
||||
]
|
||||
}
|
||||
|
||||
fn recording_sink() -> (SharedSink, Arc<Mutex<Vec<IdeEvent>>>) {
|
||||
let events = Arc::new(Mutex::new(Vec::new()));
|
||||
let recorded = events.clone();
|
||||
(Arc::new(move |event| recorded.lock().push(event)), events)
|
||||
}
|
||||
|
||||
/// `(done, cancellable)` of each task event, with the task id.
|
||||
fn tasks(events: &Mutex<Vec<IdeEvent>>) -> Vec<(ide_core::TaskId, String, bool, bool)> {
|
||||
events
|
||||
.lock()
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
IdeEvent::Task { id, title, done, cancellable, .. } => Some((*id, title.clone(), *done, *cancellable)),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn server() -> KotlinDebugServer {
|
||||
KotlinDebugServer { port: 4711, root: PathBuf::from("/p") }
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kotlin_lsp_takes_the_debug_server_port() {
|
||||
let config = launch(ide_dap::kotlin::ADAPTER, serde_json::json!({"mainClass": "a.AppKt", "projectRoot": "/p"}));
|
||||
let c = use_kotlin_lsp(config, 4711).unwrap();
|
||||
assert_eq!(c.adapter, ide_dap::kotlin_lsp::ADAPTER);
|
||||
let args = extra(&c);
|
||||
assert_eq!(args["debugServer"], 4711);
|
||||
assert_eq!(args["mainClass"], "a.AppKt");
|
||||
assert_eq!(args["projectRoot"], "/p");
|
||||
assert!(args.get("__adapterJavaHome").is_none());
|
||||
assert_eq!(
|
||||
(c.name.as_str(), c.request.as_str(), c.cwd.as_deref()),
|
||||
("AppKt", "launch", Some(Path::new("/cwd")))
|
||||
);
|
||||
let mut bare = launch(ide_dap::kotlin::ADAPTER, serde_json::json!({}));
|
||||
bare.extra_json = None;
|
||||
assert_eq!(extra(&use_kotlin_lsp(bare, 4711).unwrap())["debugServer"], 4711);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_fallback_reason_has_a_catalogued_text() {
|
||||
let catalogs: [(&str, serde_json::Value); 2] = [
|
||||
("en", serde_json::from_str(include_str!("../../../../../src/platform/locales/text/en.json")).unwrap()),
|
||||
("zh", serde_json::from_str(include_str!("../../../../../src/platform/locales/text/zh.json")).unwrap()),
|
||||
];
|
||||
let templates = [NOT_CONFIGURED, NOT_INSTALLED, NOT_STARTED, STILL_IMPORTING, IMPORT_FAILED, FAILED];
|
||||
for reason in reasons() {
|
||||
let (_, body) = fallback_notice(&reason);
|
||||
let detail = match &reason {
|
||||
KotlinDebugUnavailable::NotStarted(d) | KotlinDebugUnavailable::Failed(d) => d.as_str(),
|
||||
_ => "",
|
||||
};
|
||||
assert!(
|
||||
templates.iter().any(|t| t.replace("{detail}", detail) == body),
|
||||
"{reason:?}: {body:?} is not built from a catalogued template"
|
||||
);
|
||||
}
|
||||
for text in templates.iter().chain([&NOTICE_TITLE, &TASK_TITLE]) {
|
||||
for (locale, catalog) in &catalogs {
|
||||
let translated = catalog[*text].as_str().unwrap_or_else(|| panic!("{locale}.json lacks {text:?}"));
|
||||
if *locale == "zh" {
|
||||
assert_ne!(translated, *text, "zh.json leaves {text:?} untranslated");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fallback_reasons_have_distinct_keys() {
|
||||
let keys: HashSet<_> = reasons().iter().map(|r| fallback_notice(r).0).collect();
|
||||
assert_eq!(keys.len(), reasons().len());
|
||||
// The detail does not make another reason.
|
||||
assert_eq!(
|
||||
fallback_notice(&KotlinDebugUnavailable::Failed("a".into())).0,
|
||||
fallback_notice(&KotlinDebugUnavailable::Failed("b".into())).0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_kotlin_lsp_config_without_a_debug_server_is_refused() {
|
||||
let refused = require_debug_server(&launch(ide_dap::kotlin_lsp::ADAPTER, serde_json::json!({}))).unwrap_err();
|
||||
assert_eq!(refused.kind, ide_core::error::ErrorKind::InvalidInput);
|
||||
assert!(refused.message.contains("debugServer"), "{}", refused.message);
|
||||
let mut bare = launch(ide_dap::kotlin_lsp::ADAPTER, serde_json::json!({}));
|
||||
bare.extra_json = None;
|
||||
assert!(require_debug_server(&bare).is_err());
|
||||
assert!(
|
||||
require_debug_server(&launch(ide_dap::kotlin_lsp::ADAPTER, serde_json::json!({"debugServer": null})))
|
||||
.is_err()
|
||||
);
|
||||
for port in [serde_json::json!(4711), serde_json::json!("10.0.0.5:4711")] {
|
||||
require_debug_server(&launch(ide_dap::kotlin_lsp::ADAPTER, serde_json::json!({"debugServer": port})))
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_module_is_the_project_root_else_the_working_directory() {
|
||||
let with_root = launch(ide_dap::kotlin::ADAPTER, serde_json::json!({"projectRoot": "/p/app"}));
|
||||
assert_eq!(debug_path(&with_root), Some(PathBuf::from("/p/app")));
|
||||
assert_eq!(debug_path(&launch(ide_dap::kotlin::ADAPTER, serde_json::json!({}))), Some(PathBuf::from("/cwd")));
|
||||
let mut neither = launch(ide_dap::kotlin::ADAPTER, serde_json::json!({}));
|
||||
neither.cwd = None;
|
||||
assert_eq!(debug_path(&neither), None);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_quick_answer_shows_no_task() {
|
||||
let (sink, events) = recording_sink();
|
||||
assert_eq!(wait_for_debug_server(&sink, |_| async { Ok(server()) }).await, Ok(server()));
|
||||
assert!(events.lock().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_longer_wait_shows_a_task_that_ends_on_every_outcome() {
|
||||
for outcome in [Ok(server()), Err(KotlinDebugUnavailable::ImportFailed)] {
|
||||
let (sink, events) = recording_sink();
|
||||
let answer = outcome.clone();
|
||||
let result = wait_for_debug_server(&sink, |_| async move {
|
||||
tokio::time::sleep(Duration::from_secs(5)).await;
|
||||
answer
|
||||
})
|
||||
.await;
|
||||
assert_eq!(result, outcome);
|
||||
let tasks = tasks(&events);
|
||||
assert_eq!(tasks.len(), 2, "{tasks:?}");
|
||||
assert_eq!(tasks[0].0, tasks[1].0);
|
||||
assert_eq!(tasks[0].1, TASK_TITLE);
|
||||
assert_eq!((tasks[0].2, tasks[0].3), (false, true));
|
||||
assert_eq!((tasks[1].2, tasks[1].3), (true, false));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancelling_the_task_stops_the_wait() {
|
||||
let (sink, events) = recording_sink();
|
||||
let wait = tokio::spawn(async move {
|
||||
wait_for_debug_server(&sink, |cancel| async move {
|
||||
cancel.cancelled().await;
|
||||
Err(KotlinDebugUnavailable::StillImporting)
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
let id = tasks(&events)[0].0;
|
||||
assert!(ide_core::task::cancel(id));
|
||||
assert_eq!(wait.await.unwrap(), Err(KotlinDebugUnavailable::StillImporting));
|
||||
assert_eq!(tasks(&events).last().map(|t| t.2), Some(true));
|
||||
assert!(!ide_core::task::cancel(id), "an ended wait is no longer cancellable");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_dropped_wait_ends_its_task() {
|
||||
let (sink, events) = recording_sink();
|
||||
let wait = tokio::spawn(async move {
|
||||
wait_for_debug_server(&sink, |_| std::future::pending::<Result<KotlinDebugServer, _>>()).await
|
||||
});
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
wait.abort();
|
||||
let _ = wait.await;
|
||||
assert_eq!(tasks(&events).iter().map(|t| t.2).collect::<Vec<_>>(), [false, true]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_notice_shows_once_per_reason() {
|
||||
let (sink, events) = recording_sink();
|
||||
let notices = Mutex::new(HashSet::new());
|
||||
for reason in [
|
||||
KotlinDebugUnavailable::StillImporting,
|
||||
KotlinDebugUnavailable::StillImporting,
|
||||
KotlinDebugUnavailable::NotStarted("needs JDK 21".into()),
|
||||
KotlinDebugUnavailable::NotStarted("crashed".into()),
|
||||
] {
|
||||
report_fallback(¬ices, &sink, &reason);
|
||||
}
|
||||
let notes: Vec<_> = events
|
||||
.lock()
|
||||
.iter()
|
||||
.map(|e| match e {
|
||||
IdeEvent::Notify { level, title, body } => (*level, title.clone(), body.clone()),
|
||||
other => panic!("{other:?}"),
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
notes,
|
||||
[
|
||||
(NotifyLevel::Info, NOTICE_TITLE.to_string(), Some(STILL_IMPORTING.to_string())),
|
||||
(
|
||||
NotifyLevel::Info,
|
||||
NOTICE_TITLE.to_string(),
|
||||
Some("kotlin-lsp did not start: needs JDK 21".to_string())
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
}
|
||||
Loaded 100 of 406 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user