mirror of
https://github.com/atuinsh/atuin.git
synced 2026-10-11 22:29:51 +00:00
feat(secrets): add several new known secrets formats (atuinsh/atuin#4364)
Add these to - commands - outputs - tools Integrate history filters and credential redacting to captured tool calls, ensuring we do not leak private data by capturing ai activity. We can periodically review and improve this. Patterns are also user configurable ## Checks - [ ] I am happy for maintainers to push small adjustments to this PR, to speed up the review cycle - [ ] I have checked that there are no existing pull requests for the same thing
This commit is contained in:
Ellie Huxtable
committed by
Ellie Huxtable
Ellie Huxtable
parent
ec1fe496a9
commit
56fcb05a4f
@@ -24,26 +24,38 @@
|
||||
[Uu]ndecryptable
|
||||
[Xx]onsh
|
||||
[Zz]sh
|
||||
Artifactory
|
||||
Atlassian
|
||||
Atuin
|
||||
autostart
|
||||
autosuggest
|
||||
bindable
|
||||
Brevo
|
||||
Caddy
|
||||
ciphertext
|
||||
Clojars
|
||||
Databricks
|
||||
deserialization
|
||||
devcontainers
|
||||
dirs
|
||||
Dockerfiles
|
||||
dotdir
|
||||
Dynatrace
|
||||
Entra
|
||||
enum
|
||||
fullscreen
|
||||
fulltext
|
||||
fzf
|
||||
gcloud
|
||||
Ghostty
|
||||
gitleaks
|
||||
glibc
|
||||
Grafana
|
||||
Groq
|
||||
histdb
|
||||
IDE
|
||||
ignorespace
|
||||
Infracost
|
||||
LLM
|
||||
loopback
|
||||
matcher
|
||||
@@ -56,6 +68,7 @@ OAuth
|
||||
Ollama
|
||||
opencode
|
||||
Pacman
|
||||
Pinecone
|
||||
plaintext
|
||||
Podman
|
||||
Postgres
|
||||
@@ -65,10 +78,13 @@ reenabling
|
||||
replxx
|
||||
runbooks
|
||||
scrollbar
|
||||
Shopify
|
||||
SQLite
|
||||
subshells
|
||||
sudo
|
||||
Supabase
|
||||
systemd
|
||||
Tailscale
|
||||
Termux
|
||||
tmux
|
||||
toolchain
|
||||
|
||||
Generated
+1
@@ -495,6 +495,7 @@ dependencies = [
|
||||
"fs-err",
|
||||
"futures",
|
||||
"getrandom 0.4.3",
|
||||
"glob-match",
|
||||
"humantime",
|
||||
"itertools 0.15.0",
|
||||
"libsqlite3-sys",
|
||||
|
||||
@@ -38,7 +38,7 @@ pub struct AtuinOutputSearchToolCall {
|
||||
|
||||
impl AtuinOutputSearchToolCall {
|
||||
pub(crate) async fn execute(&self, db: &Sqlite, settings: &Settings) -> ToolOutcome {
|
||||
if matches!(settings.output, OutputCapture::Disabled) {
|
||||
if matches!(settings.output, OutputCapture::Disabled(_)) {
|
||||
return ToolOutcome::Error(
|
||||
"Output search is unavailable: output capture is disabled in the Atuin config \
|
||||
(the [output] section), so no command output has been recorded. History search \
|
||||
|
||||
@@ -41,6 +41,7 @@ mod kv;
|
||||
pub(crate) mod meta;
|
||||
pub mod output;
|
||||
mod scripts;
|
||||
pub mod security;
|
||||
pub mod shells;
|
||||
pub mod watcher;
|
||||
|
||||
@@ -49,6 +50,7 @@ pub use daemon::Daemon;
|
||||
pub use disk_usage_limit::{DiskUsageLimit, DiskUsageLimitParseError};
|
||||
use output::OutputCaptureConfig;
|
||||
pub use output::{CaptureLimits, CommandFilter, OutputCapture};
|
||||
pub use security::Security;
|
||||
pub use shells::Shells;
|
||||
|
||||
/// Default sync address for Atuin's hosted service, parsed once.
|
||||
@@ -1186,6 +1188,9 @@ pub struct Settings {
|
||||
|
||||
#[serde(default)]
|
||||
pub ai: Ai,
|
||||
|
||||
#[serde(default)]
|
||||
pub security: Security,
|
||||
}
|
||||
|
||||
impl Settings {
|
||||
@@ -2249,6 +2254,43 @@ mod tests {
|
||||
assert!(Settings::validate_str("[output]\ncommand_filter = [\"(\"]\n").is_err());
|
||||
}
|
||||
|
||||
/// AI session capture applies `command_filter` to the commands agents run, whether or not
|
||||
/// output capture is on.
|
||||
#[rstest]
|
||||
#[case::enabled("enabled = true\n")]
|
||||
#[case::disabled("enabled = false\n")]
|
||||
fn output_command_filter_is_kept_with_capture_off(#[case] enabled: &str) {
|
||||
let settings =
|
||||
parse_settings(&format!("[output]\n{enabled}command_filter = [\"^cat \"]\n"));
|
||||
assert!(settings.output.command_filter().is_match("cat .env"));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn security_settings_load() {
|
||||
let settings = parse_settings(concat!(
|
||||
"[security]\n",
|
||||
"redact_patterns = [\"ACME-[0-9]{6}\"]\n",
|
||||
"sensitive_files = [\"*.secret\"]\n",
|
||||
));
|
||||
assert_eq!(settings.security.redactor(true).redact("id ACME-123456"), "id ****");
|
||||
let files = settings.security.sensitive_files(&settings.key_path);
|
||||
assert!(files.matches("notes.secret", None));
|
||||
assert!(files.matches(&settings.key_path.to_string_lossy(), None));
|
||||
assert!(files.matches(".env", None), "the built-in files stay");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn security_settings_default_to_the_built_in_rules() {
|
||||
let settings = parse_settings("");
|
||||
assert!(settings.security.redact_patterns.is_empty());
|
||||
assert!(settings.security.sensitive_files(&settings.key_path).matches(".env", None));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn redact_patterns_reject_an_invalid_expression() {
|
||||
assert!(Settings::validate_str("[security]\nredact_patterns = [\"(\"]\n").is_err());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn sync_frequency_accepts_humantime_and_bare_seconds() {
|
||||
let secs = std::time::Duration::from_secs;
|
||||
|
||||
@@ -6,26 +6,41 @@ use serde::{Deserialize, Serialize};
|
||||
use super::DiskUsageLimit;
|
||||
|
||||
/// The `[output]` section of `config.toml`: capturing and storing command output.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize, Serialize)]
|
||||
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(from = "OutputCaptureConfig", into = "OutputCaptureConfig")]
|
||||
pub enum OutputCapture {
|
||||
/// `enabled = false`: nothing is captured and the other keys are ignored.
|
||||
#[default]
|
||||
Disabled,
|
||||
/// `enabled = false`: nothing is captured, and the other keys are ignored but for
|
||||
/// `command_filter`, which AI session capture also applies to the commands agents run.
|
||||
Disabled(CommandFilter),
|
||||
/// `enabled = true`, with the limits that govern what is kept.
|
||||
Enabled(CaptureLimits),
|
||||
}
|
||||
|
||||
impl Default for OutputCapture {
|
||||
fn default() -> Self {
|
||||
Self::Disabled(CommandFilter::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl OutputCapture {
|
||||
/// The limits in force, or `None` when capture is disabled.
|
||||
#[must_use]
|
||||
pub const fn limits(&self) -> Option<&CaptureLimits> {
|
||||
match self {
|
||||
Self::Disabled => None,
|
||||
Self::Disabled(_) => None,
|
||||
Self::Enabled(limits) => Some(limits),
|
||||
}
|
||||
}
|
||||
|
||||
/// Commands whose output is never stored, whether capture is enabled or not.
|
||||
#[must_use]
|
||||
pub const fn command_filter(&self) -> &CommandFilter {
|
||||
match self {
|
||||
Self::Disabled(filter) => filter,
|
||||
Self::Enabled(limits) => &limits.command_filter,
|
||||
}
|
||||
}
|
||||
|
||||
/// The limits in force, treating a disabled config as the enabled defaults.
|
||||
///
|
||||
/// Capture is always active in this crate; the disabled case is delegated to a
|
||||
@@ -114,7 +129,7 @@ impl Default for OutputCaptureConfig {
|
||||
impl From<OutputCaptureConfig> for OutputCapture {
|
||||
fn from(config: OutputCaptureConfig) -> Self {
|
||||
if !config.enabled {
|
||||
return Self::Disabled;
|
||||
return Self::Disabled(config.command_filter);
|
||||
}
|
||||
|
||||
Self::Enabled(CaptureLimits {
|
||||
@@ -129,7 +144,10 @@ impl From<OutputCaptureConfig> for OutputCapture {
|
||||
impl From<OutputCapture> for OutputCaptureConfig {
|
||||
fn from(capture: OutputCapture) -> Self {
|
||||
let (enabled, limits) = match capture {
|
||||
OutputCapture::Disabled => (false, CaptureLimits::default()),
|
||||
OutputCapture::Disabled(command_filter) => (false, CaptureLimits {
|
||||
command_filter,
|
||||
..CaptureLimits::default()
|
||||
}),
|
||||
OutputCapture::Enabled(limits) => (true, limits),
|
||||
};
|
||||
Self {
|
||||
@@ -150,7 +168,7 @@ mod tests {
|
||||
|
||||
#[rstest]
|
||||
fn effective_limits_uses_defaults_when_disabled() {
|
||||
assert_eq!(OutputCapture::Disabled.effective_limits(), CaptureLimits::default());
|
||||
assert_eq!(OutputCapture::default().effective_limits(), CaptureLimits::default());
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
//! The `[security]` section of `config.toml`: what Atuin keeps out of what it stores, on top of
|
||||
//! its built-in rules.
|
||||
|
||||
use atuin_common::secrets::Redactor;
|
||||
use atuin_common::secrets::files::SensitiveFiles;
|
||||
use regex::Regex;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize, Serialize)]
|
||||
pub struct Security {
|
||||
/// Regular expressions for credentials of your own, redacted wherever Atuin redacts: in
|
||||
/// captured command output and in AI sessions. A pattern's `secret` group is what goes, where
|
||||
/// it names one (`api-key: (?<secret>\w+)`), else all it matches.
|
||||
#[serde(with = "serde_regex", default, skip_serializing)]
|
||||
pub redact_patterns: Vec<Regex>,
|
||||
|
||||
/// Files of your own whose contents are credentials, on top of the built-in ones (`.env`,
|
||||
/// `~/.ssh/id_*`, `~/.aws/credentials`, ...): what an AI agent reads out of them or writes to
|
||||
/// them is not captured, nor the output of a command naming one. A glob without a `/` is
|
||||
/// matched against a file's name (`*.secret`); one with a `/` against its path
|
||||
/// (`~/work/creds/**`).
|
||||
#[serde(default)]
|
||||
pub sensitive_files: Vec<String>,
|
||||
}
|
||||
|
||||
impl Security {
|
||||
/// Redaction with [`Self::redact_patterns`], and the built-in patterns while `builtin`
|
||||
/// (`secrets_filter`).
|
||||
#[must_use]
|
||||
pub fn redactor(&self, builtin: bool) -> Redactor {
|
||||
Redactor::new(self.redact_patterns.clone(), builtin)
|
||||
}
|
||||
|
||||
/// The files whose contents are credentials: the built-in ones, `key_path` (Atuin's own
|
||||
/// encryption key) and [`Self::sensitive_files`].
|
||||
#[must_use]
|
||||
pub fn sensitive_files(&self, key_path: &std::path::Path) -> SensitiveFiles {
|
||||
SensitiveFiles::new(&self.sensitive_files, &[key_path.to_path_buf()], dirs::home_dir())
|
||||
}
|
||||
}
|
||||
@@ -48,6 +48,7 @@ dirs = { workspace = true }
|
||||
easy-cast = { workspace = true }
|
||||
enum_dispatch = { workspace = true }
|
||||
flume = { workspace = true }
|
||||
glob-match = { workspace = true }
|
||||
fs-err = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
getrandom = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
//! What a tool call touches, whichever harness made it: the command it runs, and the files it
|
||||
//! reads and writes. What capture keeps of a call can depend on it (a command history would
|
||||
//! ignore, a read of `.env`).
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::harnesstools::note::command_line;
|
||||
|
||||
/// What a tool call touches.
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Access {
|
||||
/// The command line a shell tool runs.
|
||||
pub command: Option<String>,
|
||||
/// Where it runs it, when the call says (Codex's `workdir`).
|
||||
pub workdir: Option<String>,
|
||||
/// Files the call reads or searches.
|
||||
pub reads: Vec<String>,
|
||||
/// Files the call writes, edits or patches.
|
||||
pub writes: Vec<String>,
|
||||
}
|
||||
|
||||
/// The input keys a file's path is given under.
|
||||
const PATH_KEYS: &[&str] = &["file_path", "filePath", "path", "notebook_path", "filename", "file"];
|
||||
const PATHS_KEYS: &[&str] = &["paths", "file_paths", "filePaths", "files"];
|
||||
|
||||
impl Access {
|
||||
/// What the call `name`, with `input`, touches. Tools are known by name, case aside and
|
||||
/// without an MCP server's prefix (`mcp__fs__write_file`): shells by the command they run, and
|
||||
/// a writer by the paths it's given; any other tool given a path reads it (a read, a search,
|
||||
/// a listing).
|
||||
#[must_use]
|
||||
pub fn of(name: &str, input: &Value) -> Self {
|
||||
let lower = name.to_ascii_lowercase();
|
||||
let tool = lower.rsplit("__").next().unwrap_or(&lower);
|
||||
let mut access = Self::default();
|
||||
if tool == "apply_patch" {
|
||||
let patch = match input {
|
||||
Value::String(text) => object(input)
|
||||
.and_then(|o| o.get("input").and_then(Value::as_str).map(str::to_owned))
|
||||
.or_else(|| Some(text.clone())),
|
||||
_ => object(input).and_then(|o| {
|
||||
["input", "patch"].iter().find_map(|k| o.get(*k)?.as_str().map(str::to_owned))
|
||||
}),
|
||||
};
|
||||
access.writes = patch.as_deref().map(patched).unwrap_or_default();
|
||||
return access;
|
||||
}
|
||||
let Some(input) = object(input) else {
|
||||
return access;
|
||||
};
|
||||
if matches!(
|
||||
tool,
|
||||
"bash"
|
||||
| "shell"
|
||||
| "shell_command"
|
||||
| "exec_command"
|
||||
| "local_shell"
|
||||
| "container.exec"
|
||||
| "run_terminal_cmd"
|
||||
| "execute_command"
|
||||
| "run_command"
|
||||
) {
|
||||
let command = match input.get("command").or_else(|| input.get("cmd")) {
|
||||
Some(Value::String(line)) => Some(line.clone()),
|
||||
Some(Value::Array(argv)) => Some(command_line(argv)),
|
||||
_ => None,
|
||||
};
|
||||
// Codex patches through its shell too (`["apply_patch", "*** Begin Patch ..."]`), and
|
||||
// any shell writes where it redirects to (`printf ... > .env`).
|
||||
access.writes = command
|
||||
.as_deref()
|
||||
.map(|command| patched(command).into_iter().chain(redirected(command)).collect())
|
||||
.unwrap_or_default();
|
||||
access.command = command;
|
||||
access.workdir = ["workdir", "working_directory", "cwd"]
|
||||
.iter()
|
||||
.find_map(|k| input.get(*k)?.as_str().map(str::to_owned));
|
||||
return access;
|
||||
}
|
||||
// opencode's patch tool names its files only in its patch.
|
||||
if let Some(patch) =
|
||||
["patchText", "patch_text", "patch"].iter().find_map(|k| input.get(*k)?.as_str())
|
||||
{
|
||||
access.writes = patched(patch);
|
||||
return access;
|
||||
}
|
||||
let paths: Vec<String> = PATH_KEYS
|
||||
.iter()
|
||||
.filter_map(|k| input.get(*k)?.as_str())
|
||||
.chain(
|
||||
PATHS_KEYS
|
||||
.iter()
|
||||
.filter_map(|k| input.get(*k)?.as_array())
|
||||
.flatten()
|
||||
.filter_map(Value::as_str),
|
||||
)
|
||||
.map(str::to_owned)
|
||||
.collect();
|
||||
let writes = matches!(
|
||||
tool,
|
||||
"write"
|
||||
| "edit"
|
||||
| "multiedit"
|
||||
| "notebookedit"
|
||||
| "write_file"
|
||||
| "edit_file"
|
||||
| "create_file"
|
||||
| "str_replace_editor"
|
||||
| "str_replace_based_edit_tool"
|
||||
| "patch"
|
||||
| "create"
|
||||
| "apply_diff"
|
||||
);
|
||||
if writes {
|
||||
access.writes = paths;
|
||||
} else {
|
||||
access.reads = paths;
|
||||
}
|
||||
access
|
||||
}
|
||||
}
|
||||
|
||||
/// A call's input as an object: Codex keeps a function call's arguments as the JSON text the
|
||||
/// model wrote.
|
||||
fn object(input: &Value) -> Option<Map<String, Value>> {
|
||||
match input {
|
||||
Value::Object(map) => Some(map.clone()),
|
||||
Value::String(text) => serde_json::from_str(text).ok(),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The files a shell `command` writes to by redirecting its output (`> .env`, `>> log`,
|
||||
/// `&> out`) or through `tee`.
|
||||
fn redirected(command: &str) -> Vec<String> {
|
||||
let words = shlex::split(command)
|
||||
.unwrap_or_else(|| command.split_whitespace().map(str::to_owned).collect());
|
||||
let mut files = Vec::new();
|
||||
let mut words = words.iter().map(String::as_str);
|
||||
let mut teeing = false;
|
||||
while let Some(word) = words.next() {
|
||||
if matches!(word, "|" | "||" | "&&" | ";" | "&") {
|
||||
teeing = false;
|
||||
continue;
|
||||
}
|
||||
if teeing && !word.starts_with('-') {
|
||||
files.push(word.to_owned());
|
||||
continue;
|
||||
}
|
||||
if word.rsplit('/').next() == Some("tee") {
|
||||
teeing = true;
|
||||
continue;
|
||||
}
|
||||
// `> file`, `>file`, `2>>file`, `&>file`, `echo x>file`: what follows the last `>`, else
|
||||
// the next word. Not a duplicated descriptor (`2>&1`).
|
||||
if let Some(at) = word.rfind('>') {
|
||||
let target = word[at + 1..].trim_start_matches('|');
|
||||
let target = if target.is_empty() {
|
||||
words.next().unwrap_or_default()
|
||||
} else {
|
||||
target
|
||||
};
|
||||
if !target.is_empty() && !target.starts_with('&') {
|
||||
files.push(target.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
files
|
||||
}
|
||||
|
||||
/// The files a patch (`*** Begin Patch` ... `*** End Patch`) names in its headers.
|
||||
fn patched(patch: &str) -> Vec<String> {
|
||||
patch
|
||||
.lines()
|
||||
.filter_map(|line| {
|
||||
let line = line.trim();
|
||||
["*** Update File: ", "*** Add File: ", "*** Delete File: ", "*** Move to: "]
|
||||
.iter()
|
||||
.find_map(|header| line.strip_prefix(header))
|
||||
})
|
||||
.map(|path| path.trim().to_owned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn shell(command: &str, workdir: Option<&str>) -> Access {
|
||||
Access {
|
||||
command: Some(command.to_owned()),
|
||||
workdir: workdir.map(str::to_owned),
|
||||
..Access::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn reads(paths: &[&str]) -> Access {
|
||||
Access {
|
||||
reads: paths.iter().map(|&p| p.to_owned()).collect(),
|
||||
..Access::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn writes(paths: &[&str]) -> Access {
|
||||
Access {
|
||||
writes: paths.iter().map(|&p| p.to_owned()).collect(),
|
||||
..Access::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::claude_bash("Bash", json!({"command": "cat .env"}), shell("cat .env", None))]
|
||||
#[case::codex_argv("shell", json!({"command": ["bash", "-lc", "ls"], "workdir": "/w"}), shell("ls", Some("/w")))]
|
||||
#[case::codex_exec("exec_command", json!(r#"{"cmd":"git status","workdir":"/w"}"#), shell("git status", Some("/w")))]
|
||||
#[case::claude_read("Read", json!({"file_path": "/w/.env"}), reads(&["/w/.env"]))]
|
||||
#[case::opencode_read("read", json!({"filePath": "/w/.env"}), reads(&["/w/.env"]))]
|
||||
#[case::grep("Grep", json!({"pattern": "x", "path": "/w"}), reads(&["/w"]))]
|
||||
#[case::claude_write("Write", json!({"file_path": "/w/.env", "content": "A=1"}), writes(&["/w/.env"]))]
|
||||
#[case::pi_edit("edit", json!({"path": "a.rs", "edits": []}), writes(&["a.rs"]))]
|
||||
#[case::mcp_write("mcp__fs__write_file", json!({"path": "a.txt", "content": ""}), writes(&["a.txt"]))]
|
||||
#[case::patch_freeform("apply_patch", json!("*** Begin Patch\n*** Update File: .env\n@@\n-A=1\n+A=2\n*** End Patch"), writes(&[".env"]))]
|
||||
#[case::patch_object("apply_patch", json!({"input": "*** Begin Patch\n*** Add File: k.pem\n*** End Patch"}), writes(&["k.pem"]))]
|
||||
#[case::patch_by_shell("shell", json!({"command": ["apply_patch", "*** Begin Patch\n*** Delete File: a.rs\n*** End Patch"]}), Access { writes: vec!["a.rs".into()], ..shell("apply_patch '*** Begin Patch\n*** Delete File: a.rs\n*** End Patch'", None) })]
|
||||
#[case::redirect("Bash", json!({"command": "printf '%s' 'pw' > .env"}), Access { writes: vec![".env".into()], ..shell("printf '%s' 'pw' > .env", None) })]
|
||||
#[case::redirect_attached("Bash", json!({"command": "echo A=1>>.env.local"}), Access { writes: vec![".env.local".into()], ..shell("echo A=1>>.env.local", None) })]
|
||||
#[case::heredoc("Bash", json!({"command": "cat > .env <<EOF\nA=1\nEOF"}), Access { writes: vec![".env".into()], ..shell("cat > .env <<EOF\nA=1\nEOF", None) })]
|
||||
#[case::tee("Bash", json!({"command": "echo A=1 | tee -a .env"}), Access { writes: vec![".env".into()], ..shell("echo A=1 | tee -a .env", None) })]
|
||||
#[case::descriptor("Bash", json!({"command": "make 2>&1"}), shell("make 2>&1", None))]
|
||||
#[case::opencode_patch("patch", json!({"patchText": "*** Begin Patch\n*** Update File: .env\n+A=1\n*** End Patch"}), writes(&[".env"]))]
|
||||
#[case::other("WebSearch", json!({"query": "rust"}), Access::default())]
|
||||
#[case::uncaptured("Read", Value::Null, Access::default())]
|
||||
fn what_a_call_touches(#[case] name: &str, #[case] input: Value, #[case] want: Access) {
|
||||
assert_eq!(Access::of(name, &input), want);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use std::path::PathBuf;
|
||||
|
||||
use enum_dispatch::enum_dispatch;
|
||||
|
||||
pub mod access;
|
||||
pub mod ccode;
|
||||
pub mod codex;
|
||||
pub mod continuation;
|
||||
|
||||
@@ -16,6 +16,12 @@
|
||||
//! - [`redact`] only replaces captured groups, so that same string comes back untouched -- there
|
||||
//! is nothing in it to take out.
|
||||
//!
|
||||
//! Patterns come in two kinds. The credential formats (`SECRET_PATTERNS`, and the services' in
|
||||
//! `vendors`) are what both see. The credentials recognised by where they sit (in `structural`: a
|
||||
//! URL's password, a private key's armour, a value assigned to `DB_PASSWORD`) are only redacted,
|
||||
//! so they never keep a command out of history; nor is a value of theirs that is plainly a
|
||||
//! stand-in (`${TOKEN}`, `<password>`). A [`Redactor`] also takes the user's own patterns.
|
||||
//!
|
||||
//! Both are best-effort. [`redact`] in particular is applied to rendered terminal output, where a
|
||||
//! credential can be broken up by SGR escape sequences, which will not match. (A soft wrap at the
|
||||
//! right margin is re-joined by the terminal emulator before capture, so a wrap alone does not
|
||||
@@ -24,12 +30,17 @@
|
||||
use std::borrow::Cow;
|
||||
use std::ops::Range;
|
||||
use std::sync::LazyLock;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use regex::{Regex, RegexSet};
|
||||
|
||||
#[cfg(test)]
|
||||
use self::tests::Test;
|
||||
|
||||
pub mod files;
|
||||
mod structural;
|
||||
mod vendors;
|
||||
|
||||
/// The string every credential [`redact`] locates is replaced with.
|
||||
pub const REDACTED: &str = "****";
|
||||
|
||||
@@ -391,42 +402,173 @@ static SECRET_PATTERNS: &[Pattern] = &[
|
||||
},
|
||||
];
|
||||
|
||||
/// The patterns [`contains_secret`] recognises: the credential formats.
|
||||
#[cfg(test)]
|
||||
fn history_patterns() -> impl Iterator<Item = &'static Pattern> {
|
||||
SECRET_PATTERNS.iter().chain(vendors::VENDOR_PATTERNS)
|
||||
}
|
||||
|
||||
/// The patterns [`redact`] takes out: the credential formats, then the structural ones, then
|
||||
/// those of a value assigned to a credential's name.
|
||||
#[cfg(test)]
|
||||
fn redaction_patterns() -> impl Iterator<Item = &'static Pattern> {
|
||||
history_patterns().chain(structural::STRUCTURAL_PATTERNS).chain(structural::NAMED_PATTERNS)
|
||||
}
|
||||
|
||||
static PREFILTER: LazyLock<RegexSet> = LazyLock::new(|| {
|
||||
RegexSet::new(SECRET_PATTERNS.iter().map(|pattern| pattern.prefilter.unwrap_or(pattern.regex)))
|
||||
.expect("failed to build secrets regex set")
|
||||
});
|
||||
|
||||
static REGEXES: LazyLock<Vec<Regex>> = LazyLock::new(|| {
|
||||
SECRET_PATTERNS
|
||||
.iter()
|
||||
.map(|pattern| {
|
||||
Regex::new(pattern.regex)
|
||||
.unwrap_or_else(|e| panic!("failed to compile regex for {}: {e}", pattern.name))
|
||||
/// Which values a pattern's matches are kept for, though it finds them.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Spare {
|
||||
/// None: a credential format is a credential wherever it is.
|
||||
Nothing,
|
||||
/// Stand-ins and references (see [`structural::is_placeholder`]).
|
||||
Placeholders,
|
||||
/// Those, and words (see [`structural::is_word`]): a name's value can be a type.
|
||||
Words,
|
||||
}
|
||||
|
||||
/// Patterns compiled to find what [`redact`] takes out.
|
||||
struct Table {
|
||||
/// Finds which of `regexes` can match, in one pass, where that is fast: for a handful of
|
||||
/// patterns. A set of many is a state machine big enough that a long hostile line overflows
|
||||
/// the regex engine's cache, and it falls back to an engine many times slower; each of those
|
||||
/// is run on its own instead, which, for the many that begin with a literal (`glpat-`), is a
|
||||
/// fast search for it.
|
||||
set: Option<RegexSet>,
|
||||
regexes: Vec<Regex>,
|
||||
spare: Vec<Spare>,
|
||||
}
|
||||
|
||||
impl Table {
|
||||
fn new<'p>(patterns: impl IntoIterator<Item = (&'p Pattern, Spare)>, set: bool) -> Self {
|
||||
let (patterns, spare): (Vec<_>, Vec<_>) = patterns.into_iter().unzip();
|
||||
let set = set.then(|| {
|
||||
RegexSet::new(patterns.iter().map(|pattern| pattern.prefilter.unwrap_or(pattern.regex)))
|
||||
.expect("failed to build the redaction regex set")
|
||||
});
|
||||
Self {
|
||||
set,
|
||||
regexes: patterns.iter().map(|pattern| compile(pattern)).collect(),
|
||||
spare,
|
||||
}
|
||||
}
|
||||
|
||||
/// Where in `s` this table's `i`th pattern finds something to take out.
|
||||
fn spans<'s>(&'s self, i: usize, s: &'s str) -> impl Iterator<Item = Range<usize>> + 's {
|
||||
self.regexes[i].captures_iter(s).filter_map(move |caps| {
|
||||
let secret = caps.name(SECRET_GROUP)?;
|
||||
let spared = match self.spare[i] {
|
||||
Spare::Nothing => false,
|
||||
Spare::Placeholders => structural::is_placeholder(secret.as_str()),
|
||||
Spare::Words => structural::is_word(secret.as_str()),
|
||||
};
|
||||
(!spared).then(|| secret.range())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn compile(pattern: &Pattern) -> Regex {
|
||||
Regex::new(pattern.regex)
|
||||
.unwrap_or_else(|e| panic!("failed to compile regex for {}: {e}", pattern.name))
|
||||
}
|
||||
|
||||
/// The vendor formats, each compiled on its own (see [`Table::set`]), for [`contains_secret`].
|
||||
static VENDOR_REGEXES: LazyLock<Vec<Regex>> =
|
||||
LazyLock::new(|| vendors::VENDOR_PATTERNS.iter().map(compile).collect());
|
||||
|
||||
/// The tables [`redact`] scans with: the original credential formats, in a set; the vendor formats
|
||||
/// and the structural patterns, each on its own.
|
||||
static REDACTION: LazyLock<[Table; 3]> = LazyLock::new(|| {
|
||||
[
|
||||
Table::new(SECRET_PATTERNS.iter().map(|pattern| (pattern, Spare::Nothing)), true),
|
||||
Table::new(vendors::VENDOR_PATTERNS.iter().map(|pattern| (pattern, Spare::Nothing)), false),
|
||||
Table::new(
|
||||
structural::STRUCTURAL_PATTERNS
|
||||
.iter()
|
||||
.map(|pattern| (pattern, Spare::Placeholders))
|
||||
.chain(structural::NAMED_PATTERNS.iter().map(|pattern| (pattern, Spare::Words))),
|
||||
false,
|
||||
),
|
||||
]
|
||||
});
|
||||
|
||||
static SGR: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\x1b\[[0-9;]*m").expect("sgr regex"));
|
||||
|
||||
macro_rules! executed {
|
||||
($subcommand:literal) => {
|
||||
/// `$command` run as a command (not merely mentioned, in an `echo` or a commit message): at the
|
||||
/// start of a line or after a separator, behind any `sudo`, `env`, `VAR=value` and the like, and
|
||||
/// by any path.
|
||||
macro_rules! ran {
|
||||
($command:expr) => {
|
||||
concat!(
|
||||
r"(?:^|[;&|(`{!\n])[ \t]*(?:(?:sudo|command|exec|time|env|nohup|then|do|else)\s+|\w+=\S*\s+)*(?:\S*/)?atuin\s+",
|
||||
$subcommand
|
||||
r"(?:^|[;&|(`{!\n])[ \t]*(?:(?:sudo|command|exec|time|env|nohup|then|do|else)\s+|\w+=\S*\s+)*(?:\S*/)?",
|
||||
$command
|
||||
)
|
||||
};
|
||||
}
|
||||
|
||||
/// The rest of a command's line, up to the next separator: where a subcommand follows the
|
||||
/// program's global flags (`aws --profile x configure get ...`).
|
||||
macro_rules! then {
|
||||
() => {
|
||||
r"\b[^\n;&|]*?\b"
|
||||
};
|
||||
}
|
||||
|
||||
macro_rules! executed {
|
||||
($subcommand:literal) => {
|
||||
ran!(concat!(r"atuin\s+", $subcommand))
|
||||
};
|
||||
}
|
||||
|
||||
/// Commands whose output may hold a credential: Atuin's own, and other tools' that print a token,
|
||||
/// a password or a key, or the whole environment.
|
||||
static OUTPUT_UNSAFE: LazyLock<RegexSet> = LazyLock::new(|| {
|
||||
RegexSet::new([
|
||||
executed!(r"key\b"),
|
||||
executed!(r"(?:account\s+)?login\b"),
|
||||
executed!(r"(?:account\s+)?register\b"),
|
||||
executed!(r"account\s+change-password\b"),
|
||||
ran!(r"gh\s+auth\s+token\b"),
|
||||
ran!(r"gh\s+auth\s+status\b[^\n;&|]*?\s(?:--show-token|-t)\b"),
|
||||
ran!(r"glab\s+auth\s+status\b[^\n;&|]*--show-token\b"),
|
||||
ran!(r"gcloud\s+auth\s+(?:application-default\s+)?print-(?:access|identity)-token\b"),
|
||||
ran!(concat!(r"aws", then!(), r"configure\s+(?:get|export-credentials)\b")),
|
||||
ran!(concat!(
|
||||
r"aws",
|
||||
then!(),
|
||||
r"sts\s+(?:get-session-token|assume-role\S*|get-federation-token)\b"
|
||||
)),
|
||||
ran!(concat!(r"aws", then!(), r"(?:ecr|ecr-public)\s+get-login-password\b")),
|
||||
ran!(concat!(r"aws", then!(), r"codeartifact\s+get-authorization-token\b")),
|
||||
ran!(r"az\s+account\s+get-access-token\b"),
|
||||
ran!(r"az\s+ad\s+sp\s+(?:create-for-rbac|credential\s+reset)\b"),
|
||||
ran!(concat!(r"kubectl", then!(), r"(?:get|describe)\s+secrets?\b")),
|
||||
ran!(concat!(r"kubectl", then!(), r"config\s+view\b[^\n;&|]*--raw\b")),
|
||||
ran!(concat!(r"kubectl", then!(), r"create\s+token\b")),
|
||||
ran!(r"vault\s+(?:read|login|print\s+token|token\s+create|kv\s+get)\b"),
|
||||
ran!(r"op\s+(?:read|inject|item\s+get|signin\b[^\n;&|]*--raw)\b"),
|
||||
ran!(r"pass\s+show\b"),
|
||||
ran!(r"gopass\s+(?:show|cat)\b"),
|
||||
ran!(r"security\s+find-(?:generic|internet)-password\b[^\n;&|]*\s-[a-zA-Z]*[wg]"),
|
||||
ran!(r"secret-tool\s+lookup\b"),
|
||||
ran!(r"heroku\s+auth:token\b"),
|
||||
ran!(r"doppler\s+secrets\b"),
|
||||
ran!(r"npm\s+token\s+create\b"),
|
||||
ran!(r"git\s+credential\s+fill\b"),
|
||||
ran!(r"terraform\s+output\b[^\n;&|]*\s-(?:json|raw)\b"),
|
||||
ran!(r"printenv\b"),
|
||||
// The environment, the shell's variables: alone, not running a command.
|
||||
ran!(r"(?:env|set|export\s+-p|declare\s+-[a-zA-Z]*[px])[ \t]*(?:\z|[;&|)\n])"),
|
||||
])
|
||||
.expect("failed to build output-unsafe set")
|
||||
});
|
||||
|
||||
/// Whether `command` runs one of the commands whose output may hold a credential (`atuin key`,
|
||||
/// `gh auth token`, `printenv`, ...), there or in a script it hands a shell.
|
||||
#[must_use]
|
||||
pub fn output_unsafe(command: &str) -> bool {
|
||||
let command = SGR.replace_all(command, "");
|
||||
@@ -462,7 +604,7 @@ fn shell_scripts(command: &str) -> Vec<String> {
|
||||
/// Whether `s` contains anything that looks like it involves a credential.
|
||||
#[must_use]
|
||||
pub fn contains_secret(s: &str) -> bool {
|
||||
PREFILTER.is_match(s)
|
||||
PREFILTER.is_match(s) || VENDOR_REGEXES.iter().any(|regex| regex.is_match(s))
|
||||
}
|
||||
|
||||
fn is_multiline_opener(value: &str) -> bool {
|
||||
@@ -474,21 +616,54 @@ fn is_multiline_opener(value: &str) -> bool {
|
||||
&& value[1..].chars().all(|c| c.is_alphanumeric() || c == '_'))
|
||||
}
|
||||
|
||||
/// A single pass over `s`, replacing every credential the patterns can locate with [`REDACTED`].
|
||||
fn redact_once(s: &str) -> Cow<'_, str> {
|
||||
let mut spans: Vec<Range<usize>> = PREFILTER
|
||||
.matches(s)
|
||||
.iter()
|
||||
.flat_map(|i| REGEXES[i].captures_iter(s))
|
||||
.filter_map(|caps| Some(caps.name(SECRET_GROUP)?.range()))
|
||||
// Text that is already the marker is not a change. This is what makes `redact`
|
||||
// idempotent and keeps "Borrowed iff nothing changed" exact.
|
||||
.filter(|span| &s[span.clone()] != REDACTED)
|
||||
.filter(|span| !is_multiline_opener(&s[span.clone()]))
|
||||
.collect();
|
||||
/// Redaction ran past its deadline (see [`Redactor::redact_within`]).
|
||||
struct TooSlow;
|
||||
|
||||
/// A single pass over `s`, replacing every credential `redactor`'s patterns can locate with
|
||||
/// [`REDACTED`]; [`TooSlow`] once `deadline` passes, checked between patterns.
|
||||
fn redact_once<'a>(
|
||||
s: &'a str,
|
||||
redactor: &Redactor,
|
||||
deadline: Option<Instant>,
|
||||
) -> Result<Cow<'a, str>, TooSlow> {
|
||||
let late = || deadline.is_some_and(|deadline| Instant::now() >= deadline);
|
||||
let mut spans: Vec<Range<usize>> = Vec::new();
|
||||
let tables: &[Table] = if redactor.builtin {
|
||||
&*REDACTION
|
||||
} else {
|
||||
&[]
|
||||
};
|
||||
for table in tables {
|
||||
let candidates: Vec<usize> = match &table.set {
|
||||
Some(set) => set.matches(s).into_iter().collect(),
|
||||
None => (0..table.regexes.len()).collect(),
|
||||
};
|
||||
for i in candidates {
|
||||
if late() {
|
||||
return Err(TooSlow);
|
||||
}
|
||||
// A YAML or shell block opener (`|`, `<<EOF`) is where a value starts, not the
|
||||
// value.
|
||||
spans.extend(table.spans(i, s).filter(|span| !is_multiline_opener(&s[span.clone()])));
|
||||
}
|
||||
}
|
||||
// The user's own: the `secret` group where one names it, else the whole match.
|
||||
for regex in &redactor.extra {
|
||||
if late() {
|
||||
return Err(TooSlow);
|
||||
}
|
||||
spans.extend(
|
||||
regex.captures_iter(s).filter_map(|caps| {
|
||||
caps.name(SECRET_GROUP).or_else(|| caps.get(0)).map(|m| m.range())
|
||||
}),
|
||||
);
|
||||
}
|
||||
// Text that is already the marker is not a change. This is what makes `redact` idempotent
|
||||
// and keeps "Borrowed iff nothing changed" exact.
|
||||
spans.retain(|span| !span.is_empty() && &s[span.clone()] != REDACTED);
|
||||
|
||||
if spans.is_empty() {
|
||||
return Cow::Borrowed(s);
|
||||
return Ok(Cow::Borrowed(s));
|
||||
}
|
||||
spans.sort_unstable_by_key(|span| span.start);
|
||||
|
||||
@@ -507,7 +682,7 @@ fn redact_once(s: &str) -> Cow<'_, str> {
|
||||
}
|
||||
out.push_str(&s[cursor..]);
|
||||
|
||||
Cow::Owned(out)
|
||||
Ok(Cow::Owned(out))
|
||||
}
|
||||
|
||||
/// Replace every credential [`redact`] can locate in `s` with [`REDACTED`].
|
||||
@@ -515,37 +690,104 @@ fn redact_once(s: &str) -> Cow<'_, str> {
|
||||
/// Returns [`Cow::Borrowed`] if and only if nothing was replaced, so
|
||||
/// `matches!(redact(s), Cow::Owned(_))` is an exact test for "something was taken out". Note that
|
||||
/// this is a stronger condition than [`contains_secret`]: everything replaced was recognised, but
|
||||
/// a pattern can also match text that holds no value to remove.
|
||||
/// a pattern can also match text that holds no value to remove, and the structural patterns
|
||||
/// replace what [`contains_secret`] doesn't recognise.
|
||||
///
|
||||
/// Best-effort; in particular it cannot see through SGR escape sequences.
|
||||
#[must_use]
|
||||
pub fn redact(s: &str) -> Cow<'_, str> {
|
||||
let mut out = match redact_once(s) {
|
||||
Cow::Borrowed(_) => return Cow::Borrowed(s),
|
||||
Cow::Owned(out) => out,
|
||||
};
|
||||
Redactor::default().redact(s)
|
||||
}
|
||||
|
||||
// Two patterns can carve one stretch into spans whose leftovers a second pass captures
|
||||
// differently (see `redaction_reaches_a_fixed_point_in_one_call`), so iterate until a pass
|
||||
// changes nothing. Each pass replaces at least one span that is not already the marker, so
|
||||
// this converges in a couple of iterations; the bound is a backstop, not a budget — a test
|
||||
// panics on it, a running sink only warns.
|
||||
for _ in 0..8 {
|
||||
let next = match redact_once(&out) {
|
||||
Cow::Borrowed(_) => None,
|
||||
Cow::Owned(next) => Some(next),
|
||||
};
|
||||
let Some(next) = next else {
|
||||
return Cow::Owned(out);
|
||||
};
|
||||
out = next;
|
||||
/// How long redacting one capture may take before what it redacts is better not kept (see
|
||||
/// [`Redactor::redact_within`]): typical output of a megabyte takes a few milliseconds. A debug
|
||||
/// build is many times slower, so tests on a busy machine would find ordinary text withheld; it
|
||||
/// waits longer.
|
||||
pub const REDACT_BUDGET: Duration = if cfg!(debug_assertions) {
|
||||
Duration::from_secs(5)
|
||||
} else {
|
||||
Duration::from_millis(250)
|
||||
};
|
||||
|
||||
/// [`redact`], taking out what the user's own patterns (`[security] redact_patterns`) match too:
|
||||
/// a pattern's `secret` group where it names one, else all it matches.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Redactor {
|
||||
extra: Vec<Regex>,
|
||||
/// Whether the built-in patterns apply (`secrets_filter`), or only `extra`.
|
||||
builtin: bool,
|
||||
}
|
||||
|
||||
impl Default for Redactor {
|
||||
fn default() -> Self {
|
||||
Self::new(Vec::new(), true)
|
||||
}
|
||||
#[allow(clippy::manual_assert)]
|
||||
if cfg!(test) {
|
||||
panic!("redact did not reach a fixed point in 8 passes on {s:?}");
|
||||
}
|
||||
|
||||
impl Redactor {
|
||||
#[must_use]
|
||||
pub const fn new(extra: Vec<Regex>, builtin: bool) -> Self {
|
||||
Self { extra, builtin }
|
||||
}
|
||||
|
||||
/// Whether this redacts anything at all.
|
||||
#[must_use]
|
||||
pub const fn is_active(&self) -> bool {
|
||||
self.builtin || !self.extra.is_empty()
|
||||
}
|
||||
|
||||
/// See [`redact`].
|
||||
#[must_use]
|
||||
pub fn redact<'a>(&self, s: &'a str) -> Cow<'a, str> {
|
||||
match self.redact_by(s, None) {
|
||||
Ok(redacted) => redacted,
|
||||
Err(TooSlow) => unreachable!("no deadline"),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`Self::redact`], or `None` once it has taken `budget`: text that can't be redacted
|
||||
/// quickly (a hostile line, a pathological pattern of the user's) is better not kept than
|
||||
/// kept slowly. Checked between patterns, so it can overrun by one pattern's scan.
|
||||
#[must_use]
|
||||
pub fn redact_within<'a>(&self, s: &'a str, budget: Duration) -> Option<Cow<'a, str>> {
|
||||
self.redact_by(s, Some(Instant::now() + budget)).ok()
|
||||
}
|
||||
|
||||
fn redact_by<'a>(
|
||||
&self,
|
||||
s: &'a str,
|
||||
deadline: Option<Instant>,
|
||||
) -> Result<Cow<'a, str>, TooSlow> {
|
||||
if !self.is_active() {
|
||||
return Ok(Cow::Borrowed(s));
|
||||
}
|
||||
let mut out = match redact_once(s, self, deadline)? {
|
||||
Cow::Borrowed(_) => return Ok(Cow::Borrowed(s)),
|
||||
Cow::Owned(out) => out,
|
||||
};
|
||||
|
||||
// Two patterns can carve one stretch into spans whose leftovers a second pass captures
|
||||
// differently (see `redaction_reaches_a_fixed_point_in_one_call`), so iterate until a
|
||||
// pass changes nothing. Each pass replaces at least one span that is not already the
|
||||
// marker, so this converges in a couple of iterations; the bound is a backstop, not a
|
||||
// budget — a test panics on it, a running sink only warns.
|
||||
for _ in 0..8 {
|
||||
let next = match redact_once(&out, self, deadline)? {
|
||||
Cow::Borrowed(_) => None,
|
||||
Cow::Owned(next) => Some(next),
|
||||
};
|
||||
let Some(next) = next else {
|
||||
return Ok(Cow::Owned(out));
|
||||
};
|
||||
out = next;
|
||||
}
|
||||
#[allow(clippy::manual_assert)]
|
||||
if cfg!(test) {
|
||||
panic!("redact did not reach a fixed point in 8 passes on {s:?}");
|
||||
}
|
||||
tracing::warn!("redact did not reach a fixed point in 8 passes; storing the last pass");
|
||||
Ok(Cow::Owned(out))
|
||||
}
|
||||
tracing::warn!("redact did not reach a fixed point in 8 passes; storing the last pass");
|
||||
Cow::Owned(out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -558,7 +800,8 @@ mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::{
|
||||
REDACTED, REGEXES, SECRET_GROUP, SECRET_PATTERNS, contains_secret, output_unsafe, redact,
|
||||
REDACTED, REDACTION, Redactor, SECRET_GROUP, contains_secret, history_patterns,
|
||||
output_unsafe, redact, redaction_patterns, vendors,
|
||||
};
|
||||
|
||||
pub(super) struct Test {
|
||||
@@ -570,8 +813,7 @@ mod tests {
|
||||
/// string must yield exactly one [`REDACTED`]. Derived from the table rather than repeated, so
|
||||
/// a new pattern joins the property tests for free.
|
||||
fn plantable() -> Vec<&'static str> {
|
||||
SECRET_PATTERNS
|
||||
.iter()
|
||||
redaction_patterns()
|
||||
.flat_map(|pattern| pattern.tests)
|
||||
.filter(|test| test.redacted == REDACTED)
|
||||
.map(|test| test.input)
|
||||
@@ -661,8 +903,13 @@ mod tests {
|
||||
"npm_[A-Za-z0-9]{36}",
|
||||
"pul-[0-9a-f]{40}",
|
||||
];
|
||||
static OLD: LazyLock<RegexSet> =
|
||||
LazyLock::new(|| RegexSet::new(OLD_PATTERNS).expect("frozen old patterns compile"));
|
||||
/// The old set, and the credential formats of particular services added since, each reviewed
|
||||
/// for what it keeps out of history (see `vendors`).
|
||||
static OLD: LazyLock<RegexSet> = LazyLock::new(|| {
|
||||
let vendors = vendors::VENDOR_PATTERNS.iter().map(|pattern| pattern.regex);
|
||||
RegexSet::new(OLD_PATTERNS.iter().copied().chain(vendors))
|
||||
.expect("frozen old patterns compile")
|
||||
});
|
||||
|
||||
/// Boundary strings where a rewrite is most likely to drift from the old set: bare mentions,
|
||||
/// odd whitespace inside the Azure wildcard, newlines inside `atuin\s+login`, near-misses.
|
||||
@@ -691,7 +938,8 @@ mod tests {
|
||||
/// is the credential and which is the variable name holding it.
|
||||
#[rstest]
|
||||
fn every_pattern_names_its_secret_group() {
|
||||
for (pattern, regex) in SECRET_PATTERNS.iter().zip(REGEXES.iter()) {
|
||||
let regexes = REDACTION.iter().flat_map(|table| &table.regexes);
|
||||
for (pattern, regex) in redaction_patterns().zip(regexes) {
|
||||
assert!(
|
||||
regex.capture_names().any(|name| name == Some(SECRET_GROUP)),
|
||||
"{} does not name a `{SECRET_GROUP}` capture group",
|
||||
@@ -702,7 +950,7 @@ mod tests {
|
||||
|
||||
#[rstest]
|
||||
fn every_pattern_is_recognised() {
|
||||
for pattern in SECRET_PATTERNS {
|
||||
for pattern in history_patterns() {
|
||||
for test in pattern.tests {
|
||||
assert!(
|
||||
contains_secret(test.input),
|
||||
@@ -718,7 +966,7 @@ mod tests {
|
||||
/// and embedded in surrounding text, since output is rarely just the credential.
|
||||
#[rstest]
|
||||
fn every_pattern_redacts_to_its_declared_value(#[values(false, true)] embed: bool) {
|
||||
for pattern in SECRET_PATTERNS {
|
||||
for pattern in redaction_patterns() {
|
||||
for test in pattern.tests {
|
||||
let wrap = |s: &str| {
|
||||
if embed {
|
||||
@@ -930,7 +1178,7 @@ mod tests {
|
||||
|
||||
#[rstest]
|
||||
fn each_prefilter_matches_exactly_where_its_pattern_does(s in credential_dense()) {
|
||||
for pattern in SECRET_PATTERNS {
|
||||
for pattern in history_patterns() {
|
||||
let Some(prefilter) = pattern.prefilter else { continue };
|
||||
prop_assert_eq!(
|
||||
regex::Regex::new(prefilter).unwrap().is_match(&s),
|
||||
@@ -953,7 +1201,10 @@ mod tests {
|
||||
// from a filter are capped far higher.
|
||||
#[rstest]
|
||||
fn text_with_nothing_recognisable_is_returned_as_is(
|
||||
s in credential_dense().prop_filter("contains a secret", |s| !contains_secret(s)),
|
||||
s in credential_dense().prop_filter("contains a secret", |s| {
|
||||
!contains_secret(s)
|
||||
&& !REDACTION.iter().flat_map(|table| &table.regexes).any(|r| r.is_match(s))
|
||||
}),
|
||||
) {
|
||||
prop_assert!(matches!(redact(&s), Cow::Borrowed(_)));
|
||||
}
|
||||
@@ -1000,7 +1251,7 @@ mod tests {
|
||||
("NAME=", "NAME="),
|
||||
("NAME= ", "NAME= "),
|
||||
("NAME=\nOTHER=value", "NAME=\nOTHER=value"),
|
||||
("NAME:\n password: hunter2", "NAME:\n password: hunter2"),
|
||||
("NAME:\n password: hunter2", "NAME:\n password: ****"),
|
||||
("echo $NAME", "echo $NAME"),
|
||||
("NAME is required", "NAME is required"),
|
||||
("set NAME, and OTHER", "set NAME, and OTHER"),
|
||||
@@ -1215,7 +1466,6 @@ mod tests {
|
||||
#[case::ghp_one_short(&format!("ghp_{}", "a".repeat(35)))]
|
||||
#[case::ghs_one_short(&format!("ghs_{}", "a".repeat(35)))]
|
||||
#[case::akia_one_short(&format!("AKIA{}", "B".repeat(15)))]
|
||||
#[case::lowercase_aws_name("aws_secret_access_key = wJalrXUtnFEMI")]
|
||||
#[case::pulumi_uppercase_hex(&format!("pul-{}", "ABCDEF0123".repeat(4)))]
|
||||
#[case::v1_unescaped_dot(&format!("v1x{}", "0".repeat(40)))]
|
||||
#[case::github_pat_no_leading_digit(&format!("github_pat_A{}_{}", "a".repeat(21), "b".repeat(59)))]
|
||||
@@ -1223,7 +1473,7 @@ mod tests {
|
||||
#[case::stripe_one_short(&format!("sk_live_{}", "a".repeat(23)))]
|
||||
#[case::slack_webhook_short_team(&format!("T1234567/B12345678/{}", "x".repeat(24)))]
|
||||
#[case::slack_webhook_wrong_team_prefix(&format!("U00000000/B00000000/{}", "x".repeat(24)))]
|
||||
#[case::slack_bot_short_first_group(&format!("xoxb-1234567890-12345678901-{}", "x".repeat(24)))]
|
||||
#[case::slack_bot_short_first_group(&format!("xoxb-123456789-12345678901-{}", "x".repeat(24)))]
|
||||
#[case::npm_one_short(&format!("npm_{}", "a".repeat(35)))]
|
||||
#[case::netlify_unknown_kind(&format!("nfx_{}", "a".repeat(36)))]
|
||||
#[case::stripe_test_one_short(&format!("sk_test_{}", "a".repeat(23)))]
|
||||
@@ -1232,12 +1482,24 @@ mod tests {
|
||||
assert!(matches!(redact(input), Cow::Borrowed(_)), "{input:?} should be left alone");
|
||||
}
|
||||
|
||||
/// Credentials the structural patterns redact, but no credential format recognises: these
|
||||
/// don't keep a command out of history. The lowercase AWS case pins that the history formats
|
||||
/// are case-sensitive.
|
||||
#[rstest]
|
||||
#[case::lowercase_aws_name("aws_secret_access_key = wJalrXUtnFEMI")]
|
||||
#[case::url_password("postgres://app:123456@db/app")]
|
||||
#[case::quoted_password(r#"{"password": "violetmeadow"}"#)]
|
||||
fn redacted_but_not_kept_out_of_history(#[case] input: &str) {
|
||||
assert!(!contains_secret(input), "{input:?} should not drop a command");
|
||||
assert!(matches!(redact(input), Cow::Owned(_)), "{input:?} should be redacted");
|
||||
}
|
||||
|
||||
/// The table-driven tests and the planted-credential properties only cover a pattern if it
|
||||
/// brings its own cases. Without this floor, `tests: &[]` silently removes a pattern from all
|
||||
/// of them and the suite stays green.
|
||||
#[rstest]
|
||||
fn every_pattern_has_a_case_that_actually_redacts() {
|
||||
for pattern in SECRET_PATTERNS {
|
||||
for pattern in redaction_patterns() {
|
||||
assert!(!pattern.tests.is_empty(), "{} has no test cases", pattern.name);
|
||||
assert!(
|
||||
pattern.tests.iter().any(|test| test.input != test.redacted),
|
||||
@@ -1288,10 +1550,84 @@ mod tests {
|
||||
#[case::in_a_loop("for i in 1; do atuin login -u me; done", true)]
|
||||
#[case::otherwise("if false; then :; else atuin key; fi", true)]
|
||||
#[case::absolute_path("/usr/local/bin/atuin key --base64", true)]
|
||||
fn commands_whose_output_holds_the_encryption_key(
|
||||
#[case] command: &str,
|
||||
#[case] unsafe_: bool,
|
||||
) {
|
||||
#[case::gh_token("gh auth token", true)]
|
||||
#[case::gh_status_token("gh auth status --show-token", true)]
|
||||
#[case::gh_status("gh auth status", false)]
|
||||
#[case::gcloud("gcloud auth print-access-token", true)]
|
||||
#[case::gcloud_adc("gcloud auth application-default print-access-token", true)]
|
||||
#[case::aws_configure("aws --profile prod configure get aws_secret_access_key", true)]
|
||||
#[case::aws_ecr("aws ecr get-login-password | docker login --password-stdin x", true)]
|
||||
#[case::aws_sts("aws sts assume-role --role-arn x --role-session-name y", true)]
|
||||
#[case::aws_other("aws s3 ls", false)]
|
||||
#[case::az("az account get-access-token", true)]
|
||||
#[case::kubectl_secret("kubectl -n prod get secret db -o yaml", true)]
|
||||
#[case::kubectl_pods("kubectl get pods", false)]
|
||||
#[case::kubectl_raw("kubectl config view --raw", true)]
|
||||
#[case::vault("vault kv get secret/db", true)]
|
||||
#[case::op("op read op://vault/item/password", true)]
|
||||
#[case::pass("pass show email/work", true)]
|
||||
#[case::keychain("security find-generic-password -s x -w", true)]
|
||||
#[case::keychain_no_password("security find-generic-password -s x", false)]
|
||||
#[case::printenv("printenv GITHUB_TOKEN", true)]
|
||||
#[case::env_alone("env", true)]
|
||||
#[case::env_piped("env | grep AWS", true)]
|
||||
#[case::env_prefix_runs("env FOO=1 cargo test", false)]
|
||||
#[case::set_alone("set", true)]
|
||||
#[case::set_option("set -euo pipefail", false)]
|
||||
#[case::export_p("export -p", true)]
|
||||
#[case::terraform_json("terraform output -json", true)]
|
||||
#[case::terraform("terraform output", false)]
|
||||
#[case::git_credential("git credential fill", true)]
|
||||
#[case::wrapped("bash -lc 'gh auth token'", true)]
|
||||
#[case::mentioned("echo gh auth token", false)]
|
||||
fn commands_whose_output_may_hold_a_credential(#[case] command: &str, #[case] unsafe_: bool) {
|
||||
assert_eq!(output_unsafe(command), unsafe_);
|
||||
}
|
||||
|
||||
/// The user's patterns redact their `secret` group, or all they match.
|
||||
#[rstest]
|
||||
#[case::whole(r"ACME-[0-9]{6}", "id ACME-123456 ok", "id **** ok")]
|
||||
#[case::group(r"pin: (?<secret>\d+)", "pin: 4242", "pin: ****")]
|
||||
#[case::with_builtin(
|
||||
r"ACME-[0-9]{6}",
|
||||
"ACME-123456 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",
|
||||
"**** ****"
|
||||
)]
|
||||
#[case::nothing(r"ACME-[0-9]{6}", "nothing here", "nothing here")]
|
||||
#[case::like_a_block_opener(r"!![A-Za-z0-9]+", "pw !!S3cret42", "pw ****")]
|
||||
fn the_users_patterns_redact_too(
|
||||
#[case] pattern: &str,
|
||||
#[case] input: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let redactor = Redactor::new(vec![regex::Regex::new(pattern).unwrap()], true);
|
||||
assert_eq!(redactor.redact(input), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn redaction_past_its_budget_gives_up() {
|
||||
let line = "atuin login ".repeat(87_381);
|
||||
assert!(Redactor::default().redact_within(&line, std::time::Duration::ZERO).is_none());
|
||||
let clean = "nothing to see";
|
||||
assert_eq!(
|
||||
Redactor::default().redact_within(clean, std::time::Duration::from_secs(5)),
|
||||
Some(Cow::Borrowed(clean))
|
||||
);
|
||||
}
|
||||
|
||||
/// What a structural pattern finds that is plainly not a credential stays.
|
||||
#[rstest]
|
||||
#[case::reference("export GITHUB_TOKEN=$(gh auth token)")]
|
||||
#[case::braced("DB_PASSWORD=${DB_PASSWORD}")]
|
||||
#[case::actions("GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}")]
|
||||
#[case::stand_in(r#"{"password": "changeme"}"#)]
|
||||
#[case::masked("postgres://app:****@db/app")]
|
||||
#[case::masked_x("API_KEY=xxxxxxxx")]
|
||||
#[case::typed_field(" password: Option<String>,")]
|
||||
#[case::usage(r#"{"max_tokens": 4096, "input_tokens": 123456789}"#)]
|
||||
#[case::count("tokens: 1500")]
|
||||
#[case::prose("Set the token in your settings.")]
|
||||
fn stand_ins_are_kept(#[case] input: &str) {
|
||||
assert_eq!(redact(input), input);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
//! Files whose contents are credentials (`.env`, `~/.ssh/id_ed25519`, `~/.aws/credentials`, an
|
||||
//! agent's own login), so that what reads them out, or writes to them, is not kept: a coding
|
||||
//! agent's file reads and writes, and a command naming one (`cat .env`).
|
||||
//!
|
||||
//! Redaction catches the credentials it knows the format or the name of; a file like these holds
|
||||
//! nothing else, in whatever format, so nothing read out of one is kept at all.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use glob_match::glob_match;
|
||||
|
||||
/// Files whose name alone says they hold credentials, wherever they are.
|
||||
const NAMES: &[&str] = &[
|
||||
".env",
|
||||
".env.*",
|
||||
"*.env",
|
||||
".envrc",
|
||||
"id_rsa",
|
||||
"id_dsa",
|
||||
"id_ecdsa",
|
||||
"id_ed25519",
|
||||
"id_ecdsa_sk",
|
||||
"id_ed25519_sk",
|
||||
"*.pem",
|
||||
"*.key",
|
||||
"*.p12",
|
||||
"*.pfx",
|
||||
"*.jks",
|
||||
"*.keystore",
|
||||
"*.tfstate",
|
||||
"*.tfstate.backup",
|
||||
"*.tfvars",
|
||||
".netrc",
|
||||
"_netrc",
|
||||
".pgpass",
|
||||
".git-credentials",
|
||||
".npmrc",
|
||||
".pypirc",
|
||||
".htpasswd",
|
||||
".vault-token",
|
||||
];
|
||||
|
||||
/// Of [`NAMES`], the ones that are a template or a public half, not a credential. The user's own
|
||||
/// names aren't excepted: one they name is one they mean.
|
||||
const NOT_NAMES: &[&str] = &["*.example", "*.sample", "*.template", "*.dist", "*.pub"];
|
||||
|
||||
/// Files holding credentials where a tool keeps them: cloud and package registry logins, and the
|
||||
/// coding agents' own.
|
||||
const PATHS: &[&str] = &[
|
||||
"~/.aws/credentials",
|
||||
"~/.aws/sso/cache/**",
|
||||
"~/.aws/cli/cache/**",
|
||||
"~/.config/gcloud/application_default_credentials.json",
|
||||
"~/.config/gcloud/credentials.db",
|
||||
"~/.config/gcloud/access_tokens.db",
|
||||
"~/.config/gcloud/legacy_credentials/**",
|
||||
"~/.azure/msal_token_cache.*",
|
||||
"~/.kube/config",
|
||||
"~/.docker/config.json",
|
||||
"~/.config/gh/hosts.yml",
|
||||
"~/.config/glab-cli/config.yml",
|
||||
"~/.config/hub",
|
||||
"~/.terraform.d/credentials.tfrc.json",
|
||||
"~/.cargo/credentials",
|
||||
"~/.cargo/credentials.toml",
|
||||
"~/.gem/credentials",
|
||||
"~/.gnupg/private-keys-v1.d/**",
|
||||
"~/.password-store/**",
|
||||
"~/.claude/.credentials.json",
|
||||
"~/.codex/auth.json",
|
||||
"~/.local/share/opencode/auth.json",
|
||||
"~/.pi/agent/auth.json",
|
||||
];
|
||||
|
||||
/// Which files hold credentials: the built-in ones, and the user's (`[security] sensitive_files`).
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SensitiveFiles {
|
||||
/// Globs matched against a file's name: the built-in ones, then the user's.
|
||||
names: Vec<String>,
|
||||
/// Globs matched against a file's whole path, `~` expanded.
|
||||
paths: Vec<String>,
|
||||
home: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl Default for SensitiveFiles {
|
||||
fn default() -> Self {
|
||||
Self::new(&[], &[], dirs::home_dir())
|
||||
}
|
||||
}
|
||||
|
||||
impl SensitiveFiles {
|
||||
/// The built-in files, `extra` exact paths (Atuin's own key), and the user's `patterns`: a
|
||||
/// glob without a `/` is matched against a file's name wherever it is (`*.secret`), one with
|
||||
/// a `/` against its path (`~/work/creds/**`; a relative one anywhere below, as
|
||||
/// `config/secrets.yml` matches `/srv/app/config/secrets.yml`).
|
||||
#[must_use]
|
||||
pub fn new(patterns: &[String], extra: &[PathBuf], home: Option<PathBuf>) -> Self {
|
||||
let mut names: Vec<String> = NAMES.iter().map(|&n| n.to_owned()).collect();
|
||||
let mut paths = Vec::new();
|
||||
let expand = |glob: &str| match (glob.strip_prefix("~/"), &home) {
|
||||
(Some(rest), Some(home)) => format!("{}/{rest}", slashed(home)),
|
||||
_ if absolute(glob) => glob.to_owned(),
|
||||
_ => format!("**/{glob}"),
|
||||
};
|
||||
for glob in PATHS.iter().copied().chain(patterns.iter().map(String::as_str)) {
|
||||
let glob = glob.trim();
|
||||
if glob.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if glob.contains('/') {
|
||||
paths.push(expand(glob));
|
||||
} else {
|
||||
names.push(glob.to_owned());
|
||||
}
|
||||
}
|
||||
paths.extend(extra.iter().map(|path| slashed(path)));
|
||||
Self { names, paths, home }
|
||||
}
|
||||
|
||||
/// Whether `path` (as a tool or a command gave it: relative to `cwd`, or from `~`) is one of
|
||||
/// these files.
|
||||
#[must_use]
|
||||
pub fn matches(&self, path: &str, cwd: Option<&Path>) -> bool {
|
||||
let path = path.trim();
|
||||
if path.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let full = self.resolve(path, cwd);
|
||||
let name = full.rsplit('/').next().unwrap_or(&full);
|
||||
if name.is_empty() {
|
||||
return false;
|
||||
}
|
||||
let (builtin, users) = self.names.split_at(NAMES.len());
|
||||
let named = (builtin.iter().any(|glob| glob_match(glob, name))
|
||||
&& !NOT_NAMES.iter().any(|glob| glob_match(glob, name)))
|
||||
|| users.iter().any(|glob| glob_match(glob, name));
|
||||
// `//x` is a network path on Windows, and `/x` on Linux: it is checked as both.
|
||||
let local = full.strip_prefix('/').filter(|rest| rest.starts_with('/'));
|
||||
named
|
||||
|| std::iter::once(full.as_str())
|
||||
.chain(local)
|
||||
.any(|full| self.paths.iter().any(|glob| glob_match(glob, full)))
|
||||
}
|
||||
|
||||
/// Whether `command` names one of these files (`cat .env`, `less < ~/.aws/credentials`,
|
||||
/// `curl -d @.env`), in itself or in a script it hands a shell.
|
||||
#[must_use]
|
||||
pub fn named_in_command(&self, command: &str, cwd: Option<&Path>) -> bool {
|
||||
let words = shlex::split(command)
|
||||
.unwrap_or_else(|| command.split_whitespace().map(str::to_owned).collect());
|
||||
let named = words
|
||||
.iter()
|
||||
.flat_map(|word| word.split(['|', ';', '&', '(', ')', '<', '>', '`']))
|
||||
.map(|word| word.trim_start_matches('@').trim_matches(['"', '\'']))
|
||||
.flat_map(|word| {
|
||||
// `--file=.env`, `env_file=.env`: the value.
|
||||
let value = word.split_once('=').map(|(_, value)| value);
|
||||
std::iter::once(word).chain(value)
|
||||
})
|
||||
.filter(|word| !word.starts_with('-'))
|
||||
.any(|word| self.matches(word, cwd));
|
||||
named
|
||||
|| super::shell_scripts(command).iter().any(|script| self.named_in_command(script, cwd))
|
||||
}
|
||||
|
||||
/// `path` from `~` or `cwd`, without its `.` and `..`, its parts joined by `/` whatever the
|
||||
/// platform (`C:/Users/me/.env`), as the globs are.
|
||||
fn resolve(&self, path: &str, cwd: Option<&Path>) -> String {
|
||||
let path = path.replace('\\', "/");
|
||||
let path = match (path.strip_prefix("~/"), &self.home) {
|
||||
(Some(rest), Some(home)) => format!("{}/{rest}", slashed(home)),
|
||||
_ => path,
|
||||
};
|
||||
let path = match cwd {
|
||||
Some(cwd) if !absolute(&path) => format!("{}/{path}", slashed(cwd)),
|
||||
_ => path,
|
||||
};
|
||||
let mut parts: Vec<&str> = Vec::new();
|
||||
for part in path.split('/') {
|
||||
match part {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
parts.pop();
|
||||
}
|
||||
part => parts.push(part),
|
||||
}
|
||||
}
|
||||
let joined = parts.join("/");
|
||||
// A network path (`//server/share`, `\\server\share`) keeps both its slashes.
|
||||
let root = if path.starts_with("//") {
|
||||
"//"
|
||||
} else if path.starts_with('/') {
|
||||
"/"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
format!("{root}{joined}")
|
||||
}
|
||||
}
|
||||
|
||||
/// `path` as a string with `/` between its parts, and none at its end.
|
||||
fn slashed(path: &Path) -> String {
|
||||
path.to_string_lossy().replace('\\', "/").trim_end_matches('/').to_owned()
|
||||
}
|
||||
|
||||
/// Whether `path` (with `/` between its parts) starts at a root: `/`, or a drive (`C:/`).
|
||||
fn absolute(path: &str) -> bool {
|
||||
let bytes = path.as_bytes();
|
||||
path.starts_with('/')
|
||||
|| (bytes.len() >= 3 && bytes[0].is_ascii_alphabetic() && &bytes[1..3] == b":/")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::{fixture, rstest};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[fixture]
|
||||
fn files() -> SensitiveFiles {
|
||||
SensitiveFiles::new(
|
||||
&[
|
||||
"*.secret".to_owned(),
|
||||
"~/work/creds/**".to_owned(),
|
||||
"config/master.key".to_owned(),
|
||||
"credentials.sample".to_owned(),
|
||||
"//server/share/creds.txt".to_owned(),
|
||||
],
|
||||
&[PathBuf::from("/home/me/.local/share/atuin/key")],
|
||||
Some(PathBuf::from("/home/me")),
|
||||
)
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::dotenv(".env", true)]
|
||||
#[case::dotenv_local("app/.env.local", true)]
|
||||
#[case::dotenv_example(".env.example", false)]
|
||||
#[case::named_env("deploy/prod.env", true)]
|
||||
#[case::ssh_key("~/.ssh/id_ed25519", true)]
|
||||
#[case::ssh_public_key("~/.ssh/id_ed25519.pub", false)]
|
||||
#[case::pem("/etc/ssl/private/server.pem", true)]
|
||||
#[case::aws("/home/me/.aws/credentials", true)]
|
||||
#[case::aws_by_tilde("~/.aws/credentials", true)]
|
||||
#[case::aws_config("~/.aws/config", false)]
|
||||
#[case::kube_from_home("../.kube/config", true)]
|
||||
#[case::claude_login("~/.claude/.credentials.json", true)]
|
||||
#[case::codex_login("/home/me/.codex/auth.json", true)]
|
||||
#[case::atuin_key("/home/me/.local/share/atuin/key", true)]
|
||||
#[case::source("src/main.rs", false)]
|
||||
#[case::readme("README.md", false)]
|
||||
#[case::users_name("notes.secret", true)]
|
||||
#[case::users_name_over_an_exception("credentials.sample", true)]
|
||||
#[case::users_path("/home/me/work/creds/prod/db.txt", true)]
|
||||
#[case::users_relative_path("/srv/app/config/master.key", true)]
|
||||
#[case::windows_home(r"C:\Users\me\.aws\credentials", false)]
|
||||
#[case::windows_dotenv(r"C:\work\app\.env", true)]
|
||||
#[case::windows_relative(r"app\.env.local", true)]
|
||||
#[case::network_share(r"\\server\share\creds.txt", true)]
|
||||
#[case::network_share_slashes("//server/share/creds.txt", true)]
|
||||
#[case::extra_slashes("///home/me/.local/share/atuin/key", true)]
|
||||
#[case::extra_slashes_from_home("//home/me/.aws/credentials", true)]
|
||||
#[case::empty("", false)]
|
||||
fn files_holding_credentials(
|
||||
files: SensitiveFiles,
|
||||
#[case] path: &str,
|
||||
#[case] sensitive: bool,
|
||||
) {
|
||||
assert_eq!(files.matches(path, Some(Path::new("/home/me/project"))), sensitive, "{path}");
|
||||
}
|
||||
|
||||
/// Windows paths, with either separator, from a Windows home.
|
||||
#[rstest]
|
||||
#[case::backslashes(r"C:\Users\me\.aws\credentials")]
|
||||
#[case::forward_slashes("C:/Users/me/.aws/credentials")]
|
||||
#[case::from_home("~/.aws/credentials")]
|
||||
#[case::relative(r"..\.aws\credentials")]
|
||||
fn windows_paths(#[case] path: &str) {
|
||||
let files = SensitiveFiles::new(&[], &[], Some(PathBuf::from(r"C:\Users\me")));
|
||||
assert!(files.matches(path, Some(Path::new(r"C:\Users\me\project"))), "{path}");
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::cat("cat .env", true)]
|
||||
#[case::redirect("psql < ~/.pgpass", true)]
|
||||
#[case::redirect_attached("wc -l <.env", true)]
|
||||
#[case::curl_data("curl -d @.env https://example.com", true)]
|
||||
#[case::flag_value("docker run --env-file=.env app", true)]
|
||||
#[case::piped("cat ~/.aws/credentials | head", true)]
|
||||
#[case::shell_script("bash -lc 'cat .env.production'", true)]
|
||||
#[case::example("cp .env.example .env.template", false)]
|
||||
#[case::other("cargo test", false)]
|
||||
#[case::mention_in_words("echo environment", false)]
|
||||
fn commands_naming_them(files: SensitiveFiles, #[case] command: &str, #[case] named: bool) {
|
||||
assert_eq!(files.named_in_command(command, Some(Path::new("/home/me/project"))), named);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
//! Credentials recognised by where they sit rather than by a format of their own: a private key's
|
||||
//! armour, a URL's password, an `Authorization` header, a value assigned to a name like
|
||||
//! `DB_PASSWORD` or `"api_key"`. Several follow
|
||||
//! [gitleaks](https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml) (MIT).
|
||||
//!
|
||||
//! These only redact: unlike the formats in the parent module and in `vendors`, they don't decide
|
||||
//! what history keeps (`secrets_filter`). A value that is plainly not a credential -- a placeholder
|
||||
//! (`${TOKEN}`, `<password>`, `changeme`), a type (`token: String`), a number -- is left as it is
|
||||
//! (see [`is_placeholder`] and [`is_word`]).
|
||||
|
||||
use super::Pattern;
|
||||
#[cfg(test)]
|
||||
use super::REDACTED;
|
||||
#[cfg(test)]
|
||||
use super::tests::Test;
|
||||
|
||||
/// The words a name ends with when it holds a credential, as an environment variable spells them.
|
||||
macro_rules! env_secret_name {
|
||||
() => {
|
||||
r"(?-u:\b)[A-Z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|_PWD|API_?KEY|APIKEY|PRIVATE_KEY|ACCESS_KEY|SECRET_KEY|CREDENTIALS?|AUTH)S?(?-u:\b)"
|
||||
};
|
||||
}
|
||||
|
||||
/// The same, as code, JSON and config files spell them (`apiKey`, `"client_secret"`,
|
||||
/// `db.password`). Not `auth` alone (`"auth": "required"`) nor `tokens` (`max_tokens`).
|
||||
macro_rules! code_secret_name {
|
||||
() => {
|
||||
r"(?i-u:[\w.-]*?(?:passw(?:or)?d|secret|api[_-]?key|private[_-]?key|access[_-]?key|(?:auth|access|refresh|bearer|id|session)[_-]?token|token))"
|
||||
};
|
||||
}
|
||||
|
||||
/// An assignment's separator: `=`, `:`, `:=`, `=>`, `?=`, with a closing quote or `]` before it.
|
||||
macro_rules! separator {
|
||||
() => {
|
||||
r#"["'\]]?[ \t]*(?::=|=>|\?=|[=:])[ \t]*"#
|
||||
};
|
||||
}
|
||||
|
||||
/// A quoted value of at least six characters, quotes included.
|
||||
macro_rules! quoted {
|
||||
() => {
|
||||
r#""(?:[^"\\\n]|\\.){6,}"|'[^'\n]{6,}'"#
|
||||
};
|
||||
}
|
||||
|
||||
pub(super) static STRUCTURAL_PATTERNS: &[Pattern] = &[
|
||||
// gitleaks private-key; the armour lines stay, so it still says what was there. A key cut off
|
||||
// before its end line (clipped output) loses the rest.
|
||||
Pattern {
|
||||
name: "Private key",
|
||||
regex: r"-----BEGIN[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----(?<secret>(?s:.+?))(?:-----END[ A-Z0-9_-]{0,100}PRIVATE KEY(?: BLOCK)?-----|\z)",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!(
|
||||
"-----BEGIN OPENSSH PRIVATE ",
|
||||
"KEY-----\nb3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQ\nAAAAAAAAAB\n-----END \
|
||||
OPENSSH PRIVATE KEY-----"
|
||||
),
|
||||
redacted: "-----BEGIN OPENSSH PRIVATE KEY-----****-----END OPENSSH PRIVATE \
|
||||
KEY-----",
|
||||
},
|
||||
// Escaped, in JSON.
|
||||
Test {
|
||||
input: concat!(
|
||||
"-----BEGIN RSA PRIVATE ",
|
||||
"KEY-----\\nMIIEowIBAAKCAQEA7\\n-----END RSA PRIVATE KEY-----"
|
||||
),
|
||||
redacted: "-----BEGIN RSA PRIVATE KEY-----****-----END RSA PRIVATE KEY-----",
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks jwt
|
||||
Pattern {
|
||||
name: "JSON web token",
|
||||
regex: r"(?-u:\b)(?<secret>eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[Test {
|
||||
input: concat!(
|
||||
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
|
||||
".eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"
|
||||
),
|
||||
redacted: REDACTED,
|
||||
}],
|
||||
},
|
||||
Pattern {
|
||||
name: "Bearer token",
|
||||
regex: r"(?i-u:\bbearer)[ \t]+(?<secret>[A-Za-z0-9._~+/-]{16,}=*)",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[Test {
|
||||
input: "Authorization: Bearer 8f2d0c1e9b7a4f6e8d2c",
|
||||
redacted: "Authorization: Bearer ****",
|
||||
}],
|
||||
},
|
||||
Pattern {
|
||||
name: "Authorization header",
|
||||
regex: r#"(?i-u:\bauthorization)["']?[ \t]*[:=][ \t]*["']?(?i-u:basic|token)[ \t]+(?<secret>[A-Za-z0-9._~+/-]{8,}=*)"#,
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[Test {
|
||||
input: "-H 'Authorization: Basic dXNlcjpodW50ZXIy'",
|
||||
redacted: "-H 'Authorization: Basic ****'",
|
||||
}],
|
||||
},
|
||||
// gitleaks curl-auth-user
|
||||
Pattern {
|
||||
name: "curl user password",
|
||||
regex: r#"(?-u:\bcurl\b)[^\n]*?[ \t](?:-u|--user)(?:=|[ \t]+)["']?[^:\s"']*:(?<secret>[^\s"']+)"#,
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[Test {
|
||||
input: "curl -u admin:hunter2 https://example.com",
|
||||
redacted: "curl -u admin:**** https://example.com",
|
||||
}],
|
||||
},
|
||||
// A URL's userinfo: postgres://user:pass@host, redis://:pass@host, https://u:p@h.
|
||||
Pattern {
|
||||
name: "URL password",
|
||||
regex: r#"(?-u:\b)(?i-u:[a-z][a-z0-9+.-]{1,31})://[^\s/?#@"'`<>:]*:(?<secret>[^\s/?#@"'`<>]+)@"#,
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: "DATABASE_URL=postgres://app:[email protected]:5432/app",
|
||||
redacted: "DATABASE_URL=postgres://app:****@db.internal:5432/app",
|
||||
},
|
||||
Test {
|
||||
input: "redis://:hunter22@localhost:6379/0",
|
||||
redacted: "redis://:****@localhost:6379/0",
|
||||
},
|
||||
Test {
|
||||
input: "postgres://app:123456@db/app",
|
||||
redacted: "postgres://app:****@db/app",
|
||||
},
|
||||
],
|
||||
},
|
||||
// A connection string's `password=` (keyword DSNs, JDBC and ODBC strings, query strings).
|
||||
Pattern {
|
||||
name: "Connection string password",
|
||||
regex: r#"(?:^|[?&;\s])(?i-u:password|pwd)=(?<secret>"[^"\n]*"|'[^'\n]*'|[^&;\s"']+)"#,
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: "host=db user=app password=hunter22 dbname=app",
|
||||
redacted: "host=db user=app password=**** dbname=app",
|
||||
},
|
||||
Test {
|
||||
input: "Server=db;User Id=sa;Password=Hunter2!;",
|
||||
redacted: "Server=db;User Id=sa;Password=****;",
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks twilio-api-key: only redacted, as `SK` and 32 hex digits is too plain a shape to
|
||||
// keep a command out of history.
|
||||
Pattern {
|
||||
name: "Twilio API key",
|
||||
regex: r"(?-u:\b)(?<secret>SK[0-9a-fA-F]{32})(?-u:\b)",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[Test {
|
||||
input: concat!("SK", "0f9e8d7c6b5a49382716051f2e3d4c5b"),
|
||||
redacted: REDACTED,
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
/// Values assigned to a name a credential is kept under. A value of one of these that is a word
|
||||
/// (see [`is_word`]) is kept too.
|
||||
pub(super) static NAMED_PATTERNS: &[Pattern] = &[
|
||||
Pattern {
|
||||
name: "Credential variable",
|
||||
regex: concat!(
|
||||
env_secret_name!(),
|
||||
separator!(),
|
||||
r#"(?<secret>"#,
|
||||
quoted!(),
|
||||
r#"|[^\s"'`,;)\]}{(<>]{6,})"#
|
||||
),
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: "export GITHUB_TOKEN=4f1c0e7d9b2a",
|
||||
redacted: "export GITHUB_TOKEN=****",
|
||||
},
|
||||
Test {
|
||||
input: "DB_PASSWORD: \"correct horse 9\"",
|
||||
redacted: "DB_PASSWORD: ****",
|
||||
},
|
||||
Test {
|
||||
input: "DB_PASSWORD=123456",
|
||||
redacted: "DB_PASSWORD=****",
|
||||
},
|
||||
Test {
|
||||
input: "ADMIN_PASSWORD=Violetmeadow",
|
||||
redacted: "ADMIN_PASSWORD=****",
|
||||
},
|
||||
// A reference, not a value.
|
||||
Test {
|
||||
input: "OPENAI_API_KEY=${OPENAI_API_KEY}",
|
||||
redacted: "OPENAI_API_KEY=${OPENAI_API_KEY}",
|
||||
},
|
||||
],
|
||||
},
|
||||
Pattern {
|
||||
name: "Credential key",
|
||||
regex: concat!(
|
||||
r#"(?:^|[^A-Za-z0-9_.-])["']?"#,
|
||||
code_secret_name!(),
|
||||
separator!(),
|
||||
r#"(?<secret>"#,
|
||||
quoted!(),
|
||||
")"
|
||||
),
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: r#"{"client_secret": "a8f3kq0z7x1m", "client_id": "app"}"#,
|
||||
redacted: r#"{"client_secret": ****, "client_id": "app"}"#,
|
||||
},
|
||||
Test {
|
||||
input: "const apiKey = 'live-9f8e7d6c';",
|
||||
redacted: "const apiKey = ****;",
|
||||
},
|
||||
Test {
|
||||
input: r#"{"password": "<password>"}"#,
|
||||
redacted: r#"{"password": "<password>"}"#,
|
||||
},
|
||||
// A word, quoted, is a literal.
|
||||
Test {
|
||||
input: r#"{"password": "violetmeadow"}"#,
|
||||
redacted: r#"{"password": ****}"#,
|
||||
},
|
||||
],
|
||||
},
|
||||
// YAML, `.env`, INI and `.properties` lines: an unquoted value to the end of the line.
|
||||
Pattern {
|
||||
name: "Credential setting",
|
||||
regex: concat!(
|
||||
r#"(?m)^[ \t]*(?:-[ \t]+)?["']?"#,
|
||||
code_secret_name!(),
|
||||
r#"["']?[ \t]*[:=][ \t]*(?<secret>[^\s"'#|>&*!{}\[\],=][^\n#="]*?)[ \t]*$"#
|
||||
),
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: "database:\n password: hunter2\n host: db",
|
||||
redacted: "database:\n password: ****\n host: db",
|
||||
},
|
||||
Test {
|
||||
input: "\nspring.datasource.password=s3cret!\n",
|
||||
redacted: "\nspring.datasource.password=****\n",
|
||||
},
|
||||
// A struct's field, not a setting.
|
||||
Test {
|
||||
input: " token: String,",
|
||||
redacted: " token: String,",
|
||||
},
|
||||
Test {
|
||||
input: "\npassword: violetmeadow\n",
|
||||
redacted: "\npassword: ****\n",
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
/// Values a structural pattern finds that are plainly not credentials: references to one
|
||||
/// (`$TOKEN`, `${{ secrets.X }}`, `os.environ[...]`), documentation's stand-ins (`<password>`,
|
||||
/// `changeme`, `xxxx`), the marker itself, and numbers.
|
||||
pub(super) fn is_placeholder(value: &str) -> bool {
|
||||
let value = value.trim().trim_matches(|c| c == '"' || c == '\'').trim();
|
||||
if value.is_empty() {
|
||||
return true;
|
||||
}
|
||||
let lower = value.to_ascii_lowercase();
|
||||
let reference = ["$", "%", "{{", "os.environ", "process.env", "env.", "env(", "env[", "getenv"];
|
||||
if reference.iter().any(|r| lower.starts_with(r)) {
|
||||
return true;
|
||||
}
|
||||
if value.len() > 2
|
||||
&& value.starts_with('<')
|
||||
&& value.ends_with('>')
|
||||
&& value[1..value.len() - 1].chars().all(|c| c.is_ascii_alphabetic() || "_- ".contains(c))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
let first = value.chars().next().unwrap_or(' ');
|
||||
if value.len() >= 3 && "*x.-•".contains(first) && value.chars().all(|c| c == first) {
|
||||
return true;
|
||||
}
|
||||
const STAND_INS: &[&str] = &[
|
||||
"changeme",
|
||||
"change_me",
|
||||
"change-me",
|
||||
"example",
|
||||
"password",
|
||||
"passwd",
|
||||
"secret",
|
||||
"token",
|
||||
"your_password",
|
||||
"your-password",
|
||||
"yourpassword",
|
||||
"your_token",
|
||||
"your_api_key",
|
||||
"redacted",
|
||||
"placeholder",
|
||||
"dummy",
|
||||
"undefined",
|
||||
"required",
|
||||
"optional",
|
||||
];
|
||||
if STAND_INS.contains(&lower.as_str()) {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// [`is_placeholder`], or, unquoted, a type: what a name's value in code is far more often than
|
||||
/// a credential (`token: String,`, `Option<String>`, `typing.Optional[str]`, `str`). A quoted
|
||||
/// value is a literal, and a credential whatever it looks like; so is a number or a word.
|
||||
pub(super) fn is_word(value: &str) -> bool {
|
||||
const TYPES: &[&str] = &[
|
||||
"str", "string", "bool", "boolean", "int", "integer", "number", "float", "any", "object",
|
||||
"bytes", "char", "none", "null", "nil", "true", "false",
|
||||
];
|
||||
let value = value.trim();
|
||||
if is_placeholder(value) {
|
||||
return true;
|
||||
}
|
||||
if value.starts_with(['"', '\'']) || value.chars().any(|c| c.is_ascii_digit()) {
|
||||
return false;
|
||||
}
|
||||
value.contains(['<', '>', '[', ']', '&', ':', '(', ')', '{', '}', ',', ';'])
|
||||
|| TYPES.contains(&value.to_ascii_lowercase().as_str())
|
||||
}
|
||||
@@ -0,0 +1,759 @@
|
||||
//! Credential formats of particular services, mostly from the rules of
|
||||
//! [gitleaks](https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml) (MIT), where
|
||||
//! a format has a prefix of its own (`sk-ant-`, `glpat-`, ...) and so can't be mistaken for
|
||||
//! anything else; and of a few services gitleaks has no rule for, from their own documentation.
|
||||
//!
|
||||
//! Like the patterns in the parent module, these decide what history keeps
|
||||
//! (`secrets_filter`), not only what is redacted, so a new one needs a prefix that real text
|
||||
//! doesn't begin with.
|
||||
//!
|
||||
//! Every test value is a made-up credential of the right shape, split with `concat!` so this
|
||||
//! file holds no contiguous token GitHub push protection would take for a real one.
|
||||
|
||||
use super::Pattern;
|
||||
#[cfg(test)]
|
||||
use super::REDACTED;
|
||||
#[cfg(test)]
|
||||
use super::tests::Test;
|
||||
|
||||
/// The credential formats of particular services.
|
||||
pub(super) static VENDOR_PATTERNS: &[Pattern] = &[
|
||||
// gitleaks anthropic-api-key, anthropic-admin-api-key
|
||||
Pattern {
|
||||
name: "Anthropic API key",
|
||||
regex: r"(?<secret>sk-ant-(?:api|admin|oat|ort)[0-9]{2}-[A-Za-z0-9_-]{60,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sk-ant-", "api03-3B51gfhrUARY3nyFt9x7rVcfHh9Twq8FEHlqILUEE2C8mhHWGW7N6_yW8wbV7XpfvqFCWf2Fpeqb61yImVhwzL0z02vZmAA"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("sk-ant-", "oat01-uawRmzL3Bp2AM5TUeuUQP3Rh4uUX9TheDuN1GRyqTWMWR83Zm6WgxpRpngWdhUHRwMF8H7qqlHMJODw6AIYFFt3jCCpAaX5"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks openai-api-key
|
||||
Pattern {
|
||||
name: "OpenAI API key",
|
||||
regex: r"(?<secret>sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{20,}T3BlbkFJ[A-Za-z0-9_-]{20,}|sk-(?:proj|svcacct|admin)-[A-Za-z0-9_-]{60,}|sk-[A-Za-z0-9]{20}T3BlbkFJ[A-Za-z0-9]{20})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sk-", "proj-RL1KFOza2IgZ8DTuINYnwUtShkSsOZPzXCWmXsgEXSwNDyHKL7jnpcW5AOT3BlbkFJsh6O1Mnq4VM5DGHXpqhgF8hJr9flWWlfuFLIPG0rMl1Ulks1EZTMWP9BLK"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("sk-", "ZwRP8JaUlieNfgx28h0pT3BlbkFJMAiLDq2nHaVoQELwrSP0"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("sk-", "svcacct-6MasLXI2iQhl2_3y4OkgXv4uZQE2a0Byr7aedknDnTxzj4_XaT2HdDpKYsufHCGz0sgx-Cfa23wL0IwH_L8M7sW0JipMXVsl_NUmWKeE8CUklw5ynKn2aMxB"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// OpenRouter's documented key format
|
||||
Pattern {
|
||||
name: "OpenRouter API key",
|
||||
regex: r"(?<secret>sk-or-v1-[a-f0-9]{64})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sk-or-", "v1-2d181fe8bf2d2a204aacfada671ef04c0b9b91da55e8e71c006ac73ebaad35da"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Groq's documented key format
|
||||
Pattern {
|
||||
name: "Groq API key",
|
||||
regex: r"(?<secret>gsk_[A-Za-z0-9]{52})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("gsk_", "qvRTVjhlBwDSiofDC0TnewFeFiWbvvx9fWSNHVUhrpUd7C2pIZhQ"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// xAI's documented key format
|
||||
Pattern {
|
||||
name: "xAI API key",
|
||||
regex: r"(?<secret>xai-[A-Za-z0-9]{60,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("xai-", "VIJsOWIWe4FOyg0UBbhuou8lzOBndtt5tWl8x6f6NebeHzmXApp6JhkgwdQvfx7cuxEE6XbIJHc3Vfbh"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks perplexity-api-key
|
||||
Pattern {
|
||||
name: "Perplexity API key",
|
||||
regex: r"(?<secret>pplx-[A-Za-z0-9]{48})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("pplx-", "2tXJBgxkuLnAtDECNI60kxK68sfoDuhO7QMFVYnF2guHxaiK"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Pinecone's documented key format
|
||||
Pattern {
|
||||
name: "Pinecone API key",
|
||||
regex: r"(?<secret>pcsk_[A-Za-z0-9_]{50,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("pcsk_", "tIQ34wPFMVh4hgTULe0GpEnbKwxGdSYD8sL7pq0mvHIk6Q42vZtYSOa3V5CiHFXf6aFshL"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// LangSmith's documented key format
|
||||
Pattern {
|
||||
name: "LangSmith API key",
|
||||
regex: r"(?<secret>lsv2_(?:pt|sk)_[a-f0-9]{32}_[a-f0-9]{10})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("lsv2_", "pt_09bd6fc3217b72dbe8d21d988d22280b_56c2370277"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks gcp-api-key
|
||||
Pattern {
|
||||
name: "Google API key",
|
||||
regex: r"(?<secret>AIza[0-9A-Za-z_-]{35})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("AIza", "NuuiKsCW-pCl-Pscx5XxesvgIbsImncYUwu"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks huggingface-access-token, huggingface-organization-api-token
|
||||
Pattern {
|
||||
name: "Hugging Face token",
|
||||
regex: r"(?<secret>hf_[A-Za-z]{34}|api_org_[A-Za-z]{34})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("hf_", "uJYJVQlvPpxxKpalfUflUjMGgdjvzTwmSP"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("api_org_", "wscVsqvMPUqYxTKawGdgjDmOpMtVFmGZay"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks digitalocean-pat, -access-token, -refresh-token
|
||||
Pattern {
|
||||
name: "DigitalOcean token",
|
||||
regex: r"(?<secret>do[opr]_v1_[a-f0-9]{64})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("dop_", "v1_aee3f3a11aa5abfd9d3c5d14dad70bfe8739620559acab3253643910438854ac"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks sendgrid-api-token
|
||||
Pattern {
|
||||
name: "SendGrid API key",
|
||||
regex: r"(?<secret>SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("SG.", "38XFZU7fu4Kjm0408GQvom.5W1D2Wqzz1z4sZPDfGhQv5K3ILdfaPK3PlYL43nMgZA"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks sendinblue-api-token
|
||||
Pattern {
|
||||
name: "Brevo (Sendinblue) API key",
|
||||
regex: r"(?<secret>xkeysib-[a-f0-9]{64}-[A-Za-z0-9]{16})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("xkeysib-", "c87bba7769192bbcc846de9a6bb39caeaf430d80d4d944bc255f247fc430d0b0-ZoPPrarpKsQOhmIR"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks linear-api-key
|
||||
Pattern {
|
||||
name: "Linear API key",
|
||||
regex: r"(?<secret>lin_api_[A-Za-z0-9]{40})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("lin_api_", "T0V46WSSxBDmOVXMJyYVXQ4HLW7vH5JPUtX2EQaJ"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks doppler-api-token
|
||||
Pattern {
|
||||
name: "Doppler token",
|
||||
regex: r"(?<secret>dp\.(?:pt|st|ct|sa|scim|audit)\.[A-Za-z0-9]{40,44})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("dp.", "pt.C2cBiLC56nPm3KcAip85V9fci5vhqEQJDynta5vohbz"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks shopify-*
|
||||
Pattern {
|
||||
name: "Shopify token",
|
||||
regex: r"(?<secret>shp(?:at|ca|pa|ss)_[a-fA-F0-9]{32})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("shpat_", "acffe3344d9e39f2dd113dc0cfc9f02b"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks databricks-api-token
|
||||
Pattern {
|
||||
name: "Databricks token",
|
||||
regex: r"(?<secret>dapi[a-f0-9]{32}(?:-[0-9])?)",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("dapi", "423b74fd91df3276823d0ed0a46b66f0"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks pypi-upload-token
|
||||
Pattern {
|
||||
name: "PyPI upload token",
|
||||
regex: r"(?<secret>pypi-AgEIcHlwaS5vcmc[A-Za-z0-9_-]{50,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("pypi-", "AgEIcHlwaS5vcmcRUbocYZ3exkTRPwyPRlF_c_Cv4NF33HD0ZyLl3f70HMfSeKiUKWOJWr2oqO04FXLiq8p6tenNC9xptJZ"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks rubygems-api-token
|
||||
Pattern {
|
||||
name: "RubyGems API key",
|
||||
regex: r"(?<secret>rubygems_[a-f0-9]{48})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("rubygems_", "53cc2b547eb3f5bb0feb81453ba7d0f60ff6fc5f440031f4"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks notion-api-token
|
||||
Pattern {
|
||||
name: "Notion token",
|
||||
regex: r"(?<secret>ntn_[0-9]{11}[A-Za-z0-9]{35})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("ntn_", "24638176846w4J3TE0MyJ2JlxJ3nRj31D4Bka5jnYseWpe"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Atlassian's documented token format
|
||||
Pattern {
|
||||
name: "Atlassian API token",
|
||||
regex: r"(?<secret>ATATT3[A-Za-z0-9_=-]{60,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("ATATT3", "cWlpL3ZfTLmsS09hGejxnkftMyDEHXjdQqIlvkFZFbCBCoGfI2ROyj4F5y368z3AUHZcwVQvh0DZtoWEvieHrzGCXERsSmbjSMsp6ysODh8hFb2AQiXdJwIrbWqfPAE4GjpFWEUepQPFRI68YS9QPBf9rS4ULZJcYsht68yDnp24Bw3ppcGmFHDn6q"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks age-secret-key
|
||||
Pattern {
|
||||
name: "age secret key",
|
||||
regex: r"(?<secret>AGE-SECRET-KEY-1[QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L]{58})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("AGE-SECRET-", "KEY-1X48AHA66E8U8T9MGSLEK920UUST906PXFMX9QREJGVR8RWL7AYQEH6YUSU"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks 1password-service-account-token
|
||||
Pattern {
|
||||
name: "1Password service account token",
|
||||
regex: r"(?<secret>ops_eyJ[A-Za-z0-9+/]{60,}={0,3})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("ops_", "eyJgiu5fD482QaQiKo/h1QsFGgyoFxPecQgdspp8TocdS4xcC/MEz/DO6+YbX5025V1369p5z2UArClniQoEGK4xqqdwcfjVy8sDs50bbpE1bDJOZetCaRsnSdKUakf1/ANvIRtN+Zj1zzZwagGX6zbqGHzLYjtVr72EZZsGyYbTp7pwHrowh131S/QlmA6TQ5jCFbj/U6uYyFYh0dIhm/kGxCb+v5QIyzOlkXbzQOwqBOuocnR1yMhjMq8fANfBeaAWC7m"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks slack-*
|
||||
Pattern {
|
||||
name: "Slack token",
|
||||
regex: r"(?<secret>xox[bpe](?:-[0-9]{10,13}){2,3}-[A-Za-z0-9-]{24,34}|xapp-[0-9]-[A-Z0-9]+-[0-9]+-[a-z0-9]+|xoxe(?:\.xox[bp])?-[0-9]-[A-Za-z0-9]{60,}|xox[ar]-(?:[0-9]-)?[0-9a-zA-Z]{8,48}|xox[os]-[0-9]+-[0-9]+-[0-9]+-[a-fA-F[0-9]]+)",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("xoxb-", "263704386311-9415197113976-LqMisGkclaYVDnzZMwo7pkSG"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("xapp-", "1-A1S7NN9WGQX-3013421030248-pk77qi9f1qa8ckx8dtwpgyrl0dzosqo205efln88s34kzakp50cwu60iokrebit1"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks gitlab-*
|
||||
Pattern {
|
||||
name: "GitLab token",
|
||||
regex: r"(?<secret>glpat-[A-Za-z0-9_-]{20,}(?:\.[0-9a-z]{9})?|glptt-[0-9a-f]{40}|glrt-[A-Za-z0-9_-]{20,}(?:\.[0-9a-z]{9})?|gl(?:dt|ft|soat|ffct)-[A-Za-z0-9_-]{20}|glcbt-[A-Za-z0-9]{1,5}_[A-Za-z0-9_-]{20}|glagent-[A-Za-z0-9_-]{50}|glimt-[A-Za-z0-9_-]{25}|gloas-[A-Za-z0-9_-]{64}|GR1348941[A-Za-z0-9_-]{20})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("glpat-", "MdwSCAqi5jtpCNvAIzwCTMAh8kN.2ozx6mveg"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("glrt-", "ffRGP9mc16jfJteSDZuI"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("gldt-", "m3EdPylFsIZuPcrRDFZP"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks grafana-*
|
||||
Pattern {
|
||||
name: "Grafana token",
|
||||
regex: r"(?<secret>glc_[A-Za-z0-9+/]{32,}={0,3}|glsa_[A-Za-z0-9]{32}_[A-Fa-f0-9]{8}|eyJrIjoi[A-Za-z0-9]{60,}={0,3})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("glsa_", "4BRM2cVKQLjDekxcvRsPHjJ2vuxpjp2A_5af7bd4d"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("glc_", "aThhALbknfhC1sUEi4mwbyHeIV8YofIVWAxePbK4"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks sentry-user-token, sentry-org-token
|
||||
Pattern {
|
||||
name: "Sentry token",
|
||||
regex: r"(?<secret>sntryu_[a-f0-9]{64}|sntrys_eyJ[A-Za-z0-9+/=_]{40,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sntryu_", "b1f9bccc834ac13719995035c84362fe34a9c31fcb0ad51af617f10d122fba04"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks vault-service-token, vault-batch-token
|
||||
Pattern {
|
||||
name: "HashiCorp Vault token",
|
||||
regex: r"(?<secret>hvs\.[A-Za-z0-9_-]{60,}|hvb\.[A-Za-z0-9_-]{60,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("hvs.", "WIkfExSAvat6V4v4uTj9lBXaEijBVOFMuiIrSRPjZrheQxPQsONnUJZ3OhA58j4wTTarlEyBhJAu7XJ4Z7QO0SYzLWXRN1d"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks hashicorp-tf-api-token
|
||||
Pattern {
|
||||
name: "Terraform Cloud token",
|
||||
regex: r"(?<secret>[A-Za-z0-9]{14}\.atlasv1\.[A-Za-z0-9_=-]{60,70})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("5WCNGuTOl83yen.atlas", "v1.CKGmlgD5pV7L9wzGrZQyNOkYuXZ8k8ikHCDKczrBIvx2Na0C7gq8fMPL9uDrm6X7rWC"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks planetscale-*
|
||||
Pattern {
|
||||
name: "PlanetScale token",
|
||||
regex: r"(?<secret>pscale_(?:tkn|oauth|pw)_[A-Za-z0-9_=.-]{32,64})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("pscale_", "tkn_xJIB1AugE2HZzQeHOVUC98zTRg4l8r202t10XM95XHN"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks postman-api-token
|
||||
Pattern {
|
||||
name: "Postman API key",
|
||||
regex: r"(?<secret>PMAK-[a-fA-F0-9]{24}-[a-fA-F0-9]{34})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("PMAK-", "c271de63804731dce9f2a47b-f83134351731197d48898f884675a046db"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks readme-api-token
|
||||
Pattern {
|
||||
name: "ReadMe API key",
|
||||
regex: r"(?<secret>rdme_[a-z0-9]{70})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("rdme_", "ycjauupatyhs4v1waxb84ma9979016dd723lfotlfiphqr9co1srn6ba6wfpned6zoggss"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks prefect-api-token
|
||||
Pattern {
|
||||
name: "Prefect API key",
|
||||
regex: r"(?<secret>pnu_[A-Za-z0-9]{36})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("pnu_", "npvXA1O9iEhNMQ67KzQKajjN1o1RkcaW2ZPs"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks heroku-api-key-v2
|
||||
Pattern {
|
||||
name: "Heroku API key",
|
||||
regex: r"(?<secret>HRKU-AA[A-Za-z0-9_-]{58})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("HRKU-", "AAKviFRDTmBX3f6QWeT86xU3krZb84YiRbrxuztn5oHkMRK8mHnALAfxz5qv"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Supabase's documented key formats
|
||||
Pattern {
|
||||
name: "Supabase secret key",
|
||||
regex: r"(?<secret>sb_secret_[A-Za-z0-9_-]{31,}|sbp_[a-f0-9]{40})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sb_secret_", "QvektgaafqYYtZt4qGBTNvWf2vywUYG"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
Test {
|
||||
input: concat!("sbp_", "85bfa375c1d71925ab85db25c4323f20e359c9e2"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks artifactory-api-key
|
||||
Pattern {
|
||||
name: "Artifactory API key",
|
||||
regex: r"(?<secret>AKCp[A-Za-z0-9]{69})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("AKCp", "Fq1TakzzfwYQ7wQwsmJHmnuporDV2eCUdQSCcnSZLwCWKgqCoOh8EO2NqhrrGQa6AU9RU"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks aws-amazon-bedrock-api-key-long-lived
|
||||
Pattern {
|
||||
name: "Amazon Bedrock API key",
|
||||
regex: r"(?<secret>ABSK[A-Za-z0-9+/]{60,}={0,2})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("ABSK", "i3lvj6iwYA6IbJtkDEfhBZBBvRYAE38Iz4CydIKvYHGCQKvFScri8DptYbci0eh2YwuN2PRD0MXPrhMekVz1DtcaApGjQJtuC2GPRFb5IgCTyOxqX02hi7pKrma3523aVp98"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks easypost-api-token
|
||||
Pattern {
|
||||
name: "EasyPost API key",
|
||||
regex: r"(?<secret>EZ[AT]K[A-Za-z0-9]{54})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("EZAK", "ExqMi0yBeQ9y1f296NVrgmEjW5FXB7ZshK5DoRiK6vuLXgmfZp4u68"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks flyio-access-token
|
||||
Pattern {
|
||||
name: "Fly.io token",
|
||||
regex: r"(?<secret>fo1_[A-Za-z0-9_-]{43}|fm[12][ar]?_[A-Za-z0-9+/]{60,}={0,3})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("fo1_", "h5vbX8FgYECcuHKMbibewyhO1KePlDUr7oPEHB6Ve6L"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks infracost-api-token
|
||||
Pattern {
|
||||
name: "Infracost API key",
|
||||
regex: r"(?<secret>ico-[A-Za-z0-9]{32})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("ico-", "4pj8YwmQ1P8hhcyacWj8fByFWR92QCYX"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks dynatrace-api-token
|
||||
Pattern {
|
||||
name: "Dynatrace token",
|
||||
regex: r"(?<secret>dt0c01\.[A-Za-z0-9]{24}\.[A-Za-z0-9]{64})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("dt0c01.", "LL9rlVWTsxAp1yEal4jrtIsW.eDrOmlhDRUDZjIxZg9KwZgUcivD8xRIlPbA1gYI7yEhaEF8g7RHigsm7wUc6RNtD"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks frameio-api-token
|
||||
Pattern {
|
||||
name: "Frame.io token",
|
||||
regex: r"(?<secret>fio-u-[A-Za-z0-9_=-]{64})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("fio-u-", "zPyHxsa89BTIdwYT9kq7mI9WEV6fjyqhysJJFDP8kN3Cf6xoqhuhlzksIMbkWtXv"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks duffel-api-token
|
||||
Pattern {
|
||||
name: "Duffel API key",
|
||||
regex: r"(?<secret>duffel_(?:test|live)_[A-Za-z0-9_=-]{43})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("duffel_", "test_8D9dLjodoJ1NSKfy6wrodULoPYTOIqqoLUehe7K8DXO"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks clojars-api-token
|
||||
Pattern {
|
||||
name: "Clojars token",
|
||||
regex: r"(?<secret>CLOJARS_[a-z0-9]{60})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("CLOJARS_", "37ciyudqslofatzgv3dcykdl4prrsiy6fq5t0oczhha2mkpie69rbmzylppj"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks square-access-token
|
||||
Pattern {
|
||||
name: "Square token",
|
||||
regex: r"(?<secret>sq0atp-[A-Za-z0-9_-]{22,60}|sq0csp-[A-Za-z0-9_-]{43})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sq0atp-", "XcUsJDJhcWy2liqUqEbqdp"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks stripe-access-token
|
||||
Pattern {
|
||||
name: "Stripe restricted key",
|
||||
regex: r"(?<secret>rk_(?:live|test)_[A-Za-z0-9]{24,}|sk_prod_[A-Za-z0-9]{24,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("rk_", "live_EU5L4p3ZlRgreygIOYFRE6ix"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Docker's documented token format
|
||||
Pattern {
|
||||
name: "Docker Hub token",
|
||||
regex: r"(?<secret>dckr_pat_[A-Za-z0-9_-]{27})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("dckr_", "pat_pZeI1IESD5xIT1OKqhirzKDPDsg"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Tailscale's documented key format
|
||||
Pattern {
|
||||
name: "Tailscale key",
|
||||
regex: r"(?<secret>tskey-(?:api|auth|client|scim|webhook)-[A-Za-z0-9]{8,}-[A-Za-z0-9]{20,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("tskey-", "auth-kf7KpW9W3d2CNTRL-fCipfMm6ToVUNO1homoxO4UXsSXKH5"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks cloudflare-origin-ca-key
|
||||
Pattern {
|
||||
name: "Cloudflare origin CA key",
|
||||
regex: r"(?<secret>v1\.0-[a-f0-9]{24}-[a-f0-9]{60,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("v1.0-", "186f84c3ac586e0a72bd8093-ec3ba54d51dae0be46a9db37552764784c26f73fedd30f83e8da9936c2da0b222ce27a42d6a38c2f19ea8585f07e2862c05cbe6fe3fa6c58437ad4dd7a711f6fa5d41e741ae0d9f331"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks azure-ad-client-secret
|
||||
Pattern {
|
||||
name: "Azure AD client secret",
|
||||
regex: r"(?<secret>[A-Za-z0-9_~.]{3}[0-9]Q~[A-Za-z0-9_~.-]{31,34})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("TN88Q~", "pSQEwdfDbLXP6zhxFZKL4hQHWvNVBRQqFX"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks telegram-bot-api-token
|
||||
Pattern {
|
||||
name: "Telegram bot token",
|
||||
regex: r"(?<secret>(?-u:\b)[0-9]{8,10}:AA[A-Za-z0-9_-]{33})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("1692872828:", "AAH59LIKCi8eGAOVdt4tn61OrleUzf5b6PV"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// Discord's documented webhook format
|
||||
Pattern {
|
||||
name: "Discord webhook",
|
||||
regex: r"(?<secret>https://(?:ptb\.|canary\.)?discord(?:app)?\.com/api/webhooks/[0-9]+/[A-Za-z0-9_-]{60,})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("https://discord.com/api/", "webhooks/634000504883015187/GytNCMs5aXiSXJkmtnVphQOkDBVOazMNTS8GUcJFqP1mamrDiMRnu4hwYxxJB7Sbaid9"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks microsoft-teams-webhook
|
||||
Pattern {
|
||||
name: "Microsoft Teams webhook",
|
||||
regex: r"(?<secret>https://[a-z0-9]+\.webhook\.office\.com/webhookb2/[a-z0-9@-]+/IncomingWebhook/[a-z0-9]{32}/[a-z0-9-]{36})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("https://acme.webhook.office.com/", "webhookb2/74d30252-5821-bd5d-ae48-6c7bdfb83014@b623fbbf-12dc-a9a9-3c55-87a71023b5f0/IncomingWebhook/eb872ac7ca5c59c26fb92de8af03a38e/1c4c417a-2d3f-781a-061f-12e543fbb8f8"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
// gitleaks openshift-user-token
|
||||
Pattern {
|
||||
name: "OpenShift token",
|
||||
regex: r"(?<secret>sha256~[A-Za-z0-9_-]{43})",
|
||||
prefilter: None,
|
||||
#[cfg(test)]
|
||||
tests: &[
|
||||
Test {
|
||||
input: concat!("sha256~", "kRwNMI6B5Af2FBhgrTw5Sdf4VmGYMxrXcy9HDEoGwpv"),
|
||||
redacted: REDACTED,
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
@@ -37,6 +37,7 @@ lasso = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
prost = { workspace = true }
|
||||
prost-types = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -234,11 +234,14 @@ impl GrpcService for Service {
|
||||
let request = request.into_inner();
|
||||
let id = request.history_id()?;
|
||||
let capture = request.capture()?.into();
|
||||
let output = self.daemon_handle.settings().await.output.clone();
|
||||
let settings = self.daemon_handle.settings().await;
|
||||
let output = settings.output.clone();
|
||||
let files = settings.security.sensitive_files(&settings.key_path);
|
||||
drop(settings);
|
||||
let journal = self.journal.clone();
|
||||
// Spawned so a client disconnect cannot drop the call half-way.
|
||||
tokio::spawn(
|
||||
async move { journal.register_command_output(id, capture, &output).await }
|
||||
async move { journal.register_command_output(id, capture, &output, &files).await }
|
||||
.instrument(tracing::Span::current()),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -89,6 +89,7 @@ use atuin_client::history::store::HistoryStore;
|
||||
use atuin_client::history::{CommandCapture, History, HistoryId};
|
||||
use atuin_client::packfile;
|
||||
use atuin_client::settings::{OutputCapture, Search};
|
||||
use atuin_common::secrets::files::SensitiveFiles;
|
||||
use atuin_common::sync::AsyncShardedMutex;
|
||||
use atuin_domain::caps::{CapClient, PackfileCap};
|
||||
use atuin_domain::record::{RecordId, RecordIdx, RecordSeriesKey, RecordTag};
|
||||
@@ -617,8 +618,9 @@ impl HistoryJournal {
|
||||
marks
|
||||
}
|
||||
|
||||
/// Store a command's captured output as `output` allows: nothing while capture is disabled, and
|
||||
/// nothing for a command its `command_filter` matches.
|
||||
/// Store a command's captured output as `output` allows: nothing while capture is disabled,
|
||||
/// nothing for a command its `command_filter` matches, and nothing for one that may print a
|
||||
/// credential (`atuin key`, `gh auth token`) or names one of `files` (`cat .env`).
|
||||
///
|
||||
/// If the output is received for an unknown command, this returns a
|
||||
/// [`RegisterOutputError::NotLive`].
|
||||
@@ -627,6 +629,7 @@ impl HistoryJournal {
|
||||
id: HistoryId,
|
||||
capture: CommandCapture,
|
||||
output: &OutputCapture,
|
||||
files: &SensitiveFiles,
|
||||
) -> Result<(), RegisterOutputError> {
|
||||
let _lifecycle = self.lifecycle_mutex.lock(&id).await;
|
||||
|
||||
@@ -637,17 +640,17 @@ impl HistoryJournal {
|
||||
// Resolve the command backing this id: an in-flight entry wins, otherwise the
|
||||
// (non-deleted) history row. `None` means the command is gone or already deleted.
|
||||
let command = if let Some(cmd) = self.active_cmds.get(&id) {
|
||||
Some(cmd.history.command.clone())
|
||||
Some((cmd.history.command.clone(), cmd.history.cwd.clone()))
|
||||
} else {
|
||||
self.history_db
|
||||
.load(id)
|
||||
.await
|
||||
.map_err(|e| RegisterOutputError::HistoryDbFailed(e.into()))?
|
||||
.filter(|h| h.deleted_at.is_none())
|
||||
.map(|h| h.command)
|
||||
.map(|h| (h.command, h.cwd))
|
||||
};
|
||||
|
||||
let Some(command) = command else {
|
||||
let Some((command, cwd)) = command else {
|
||||
return Err(RegisterOutputError::NotLive(id));
|
||||
};
|
||||
|
||||
@@ -660,6 +663,7 @@ impl HistoryJournal {
|
||||
// Never persist output for commands that may carry secrets, or that the user excluded.
|
||||
if atuin_common::secrets::output_unsafe(&command)
|
||||
|| limits.command_filter.is_match(&command)
|
||||
|| files.named_in_command(&command, Some(std::path::Path::new(&cwd)))
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
@@ -325,6 +325,7 @@ pub async fn boot(
|
||||
db.clone(),
|
||||
settings.ai.capture_sessions,
|
||||
session_capture::ToolCapture::from_setting(settings.ai.capture_tools),
|
||||
session_capture::CapturePolicy::from_settings(&settings),
|
||||
blocking_pool.clone(),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -3,11 +3,14 @@ mod engine;
|
||||
mod hooks;
|
||||
mod import;
|
||||
mod message_enricher;
|
||||
mod policy;
|
||||
mod recovery;
|
||||
mod tools;
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::borrow::Cow;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::num::NonZeroUsize;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -25,6 +28,8 @@ use engine::SessionCaptureEngine;
|
||||
use futures::{FutureExt, Stream, StreamExt};
|
||||
pub use import::ImportProgress;
|
||||
use import::SessionImporter;
|
||||
pub use policy::CapturePolicy;
|
||||
use policy::Verdict;
|
||||
use recovery::{Backoff, Coordinator, Msg};
|
||||
use tokio::sync::{Mutex, broadcast, mpsc, oneshot, watch};
|
||||
use tokio::task::JoinHandle;
|
||||
@@ -91,6 +96,8 @@ pub(crate) struct Sink {
|
||||
state: watch::Receiver<StoreState>,
|
||||
/// What [`sanitize`] keeps of tool calls, live and imported alike.
|
||||
tools: ToolCapture,
|
||||
/// What capture keeps beyond that: the user's filters, credential files, redaction.
|
||||
policy: Arc<CapturePolicy>,
|
||||
/// Calls seen running a command whose output may carry a credential, and not answered yet:
|
||||
/// their results' output is never kept (see [`Self::withhold_unsafe_output`]).
|
||||
unsafe_calls: Arc<parking_lot::Mutex<UnsafeCalls>>,
|
||||
@@ -118,6 +125,7 @@ impl Sink {
|
||||
pending_projection: Arc::default(),
|
||||
state,
|
||||
tools,
|
||||
policy: Arc::default(),
|
||||
unsafe_calls: Arc::default(),
|
||||
#[cfg(test)]
|
||||
hooks: None,
|
||||
@@ -164,7 +172,7 @@ impl Sink {
|
||||
let locks = self.lock_ready().await?;
|
||||
// Under the locks, the sidecar holds every local row (see `withhold_unsafe_output`).
|
||||
self.withhold_unsafe_output(&mut msg).await?;
|
||||
sanitize(&mut msg, self.tools);
|
||||
sanitize(&mut msg, self.tools, &self.policy);
|
||||
// Captured here, so on this host; a reproject reads the same from the record envelope.
|
||||
msg.host = Some(self.records.host_id());
|
||||
let sink = self.clone();
|
||||
@@ -182,27 +190,36 @@ impl Sink {
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop the output of a call running a command whose output may carry a credential
|
||||
/// (`atuin key`, `atuin login`, ...), which shell output capture never stores either: no
|
||||
/// secret pattern knows a sync key's words. A result usually comes in a later line than its
|
||||
/// call, so the call is remembered until it is answered: a session's lines come here in
|
||||
/// order, live and imported. One made before the daemon (re)started is found among the
|
||||
/// session's stored rows, the first time one of its results comes in, which capture's locks
|
||||
/// (held) make trustworthy. A stored call whose input wasn't captured (made while
|
||||
/// `ai.capture_tools` was off) can't be checked, so its output is withheld too.
|
||||
/// Apply [`CapturePolicy`] to `msg`'s tool calls (see [`Verdict`]): drop what a call that
|
||||
/// history would ignore was given, and what one writing to a credential file wrote, and the
|
||||
/// output of either, or of a call whose output may hold a credential (`atuin key`,
|
||||
/// `gh auth token`, a read of `.env`), as shell output capture never stores it either; and,
|
||||
/// in a directory `cwd_filter` matches, every call's.
|
||||
///
|
||||
/// A result usually comes in a later line than its call, so the call is remembered until it
|
||||
/// is answered: a session's lines come here in order, live and imported. One made before the
|
||||
/// daemon (re)started is found among the session's stored rows, the first time the session
|
||||
/// comes in, which capture's locks (held) make trustworthy; so is where the session runs. A stored call whose input
|
||||
/// wasn't captured (made while `ai.capture_tools` was off, or dropped) can't be checked, so
|
||||
/// its output is withheld too.
|
||||
async fn withhold_unsafe_output(&self, msg: &mut Message) -> Result<(), AppendError> {
|
||||
if self.tools == ToolCapture::Names {
|
||||
return Ok(());
|
||||
}
|
||||
let answers = msg.content.iter().any(|c| matches!(c, Content::ToolResult(_)));
|
||||
if answers && !self.unsafe_calls.lock().warmed.contains(&msg.session) {
|
||||
if !self.unsafe_calls.lock().warmed.contains(&msg.session) {
|
||||
let mut unanswered = HashSet::new();
|
||||
let mut cwd = None;
|
||||
let mut stored = std::pin::pin!(self.sidecar.messages(&msg.session));
|
||||
while let Some(stored) = stored.next().await {
|
||||
for block in stored?.content {
|
||||
let stored = stored?;
|
||||
cwd = stored.cwd.or(cwd);
|
||||
let ignored = cwd.as_deref().is_some_and(|cwd| self.policy.ignores_dir(cwd));
|
||||
for block in stored.content {
|
||||
match block {
|
||||
Content::ToolUse(call)
|
||||
if call.input.is_null() || tools::runs_output_unsafe(&call.input) =>
|
||||
if call.input.is_null()
|
||||
|| ignored
|
||||
|| self.policy.judge(&call, cwd.as_deref()) != Verdict::Keep =>
|
||||
{
|
||||
unanswered.insert(call.id);
|
||||
}
|
||||
@@ -216,17 +233,42 @@ impl Sink {
|
||||
let mut calls = self.unsafe_calls.lock();
|
||||
calls.warmed.insert(msg.session.clone());
|
||||
calls.pending.extend(unanswered.into_iter().map(|id| (msg.session.clone(), id)));
|
||||
if let Some(cwd) = cwd {
|
||||
calls.cwds.insert(msg.session.clone(), cwd);
|
||||
}
|
||||
}
|
||||
let mut calls = self.unsafe_calls.lock();
|
||||
// Most rows don't say where the session runs (Codex says it in its context rows, Pi in
|
||||
// its header): the last row that did does.
|
||||
let cwd = match &msg.cwd {
|
||||
Some(cwd) => {
|
||||
calls.cwds.insert(msg.session.clone(), cwd.clone());
|
||||
Some(cwd.clone())
|
||||
}
|
||||
None => calls.cwds.get(&msg.session).cloned(),
|
||||
};
|
||||
let ignored_dir = cwd.as_deref().is_some_and(|cwd| self.policy.ignores_dir(cwd));
|
||||
for block in &mut msg.content {
|
||||
match block {
|
||||
Content::ToolUse(call) if tools::runs_output_unsafe(&call.input) => {
|
||||
Content::ToolUse(call) => {
|
||||
let verdict = if ignored_dir {
|
||||
Verdict::Drop
|
||||
} else {
|
||||
self.policy.judge(call, cwd.as_deref())
|
||||
};
|
||||
match verdict {
|
||||
Verdict::Keep => continue,
|
||||
Verdict::WithholdOutput => {}
|
||||
Verdict::WithholdWritten => tools::withhold_written(&mut call.input),
|
||||
Verdict::Drop => call.input = serde_json::Value::Null,
|
||||
}
|
||||
calls.pending.insert((msg.session.clone(), call.id.clone()));
|
||||
}
|
||||
Content::ToolResult(result)
|
||||
if calls.pending.remove(&(msg.session.clone(), result.call.clone())) =>
|
||||
{
|
||||
result.output = serde_json::Value::Null;
|
||||
Content::ToolResult(result) => {
|
||||
let pending = calls.pending.remove(&(msg.session.clone(), result.call.clone()));
|
||||
if pending || ignored_dir {
|
||||
result.output = serde_json::Value::Null;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
@@ -362,25 +404,33 @@ struct UnsafeCalls {
|
||||
pending: HashSet<(HarnessSession, ToolCallId)>,
|
||||
/// Sessions whose stored rows were looked through for such calls since the daemon started.
|
||||
warmed: HashSet<HarnessSession>,
|
||||
/// Where each session runs, as its last row to say did.
|
||||
cwds: HashMap<HarnessSession, PathBuf>,
|
||||
}
|
||||
|
||||
/// Retain conversation text, and of tool calls what `tools` keeps: by default payload-free
|
||||
/// breadcrumbs, never execution payloads (null payloads, which keep the wire format without
|
||||
/// storing arguments or results). This only affects new captures; existing synced records are
|
||||
/// not rewritten.
|
||||
fn sanitize(msg: &mut Message, tools: ToolCapture) {
|
||||
/// storing arguments or results). Text is redacted as `policy` does, and text that takes too long
|
||||
/// to redact is not kept. This only affects new captures; existing synced records are not
|
||||
/// rewritten.
|
||||
fn sanitize(msg: &mut Message, tools: ToolCapture, policy: &CapturePolicy) {
|
||||
let redact = |text: &mut String| match policy.redact(text) {
|
||||
Some(Cow::Borrowed(_)) => {}
|
||||
Some(Cow::Owned(redacted)) => *text = redacted,
|
||||
None => tools::NOT_REDACTED.clone_into(text),
|
||||
};
|
||||
let conversation = matches!(msg.role, Role::User | Role::Assistant);
|
||||
msg.content.retain_mut(|block| match block {
|
||||
Content::Text(text) if conversation => {
|
||||
*text = atuin_common::secrets::redact(text).into_owned();
|
||||
redact(text);
|
||||
true
|
||||
}
|
||||
Content::ToolUse(tool) => {
|
||||
tools.apply(&mut tool.input);
|
||||
tools.apply(&mut tool.input, policy);
|
||||
true
|
||||
}
|
||||
Content::ToolResult(result) => {
|
||||
tools.apply(&mut result.output);
|
||||
tools.apply(&mut result.output, policy);
|
||||
true
|
||||
}
|
||||
Content::Reasoning(_) => {
|
||||
@@ -389,7 +439,7 @@ fn sanitize(msg: &mut Message, tools: ToolCapture) {
|
||||
}
|
||||
// Model-written summaries and failure reasons are conversation, whatever the role.
|
||||
Content::Summary(text) | Content::Error(text) => {
|
||||
*text = atuin_common::secrets::redact(text).into_owned();
|
||||
redact(text);
|
||||
true
|
||||
}
|
||||
Content::ReasoningSummary { .. } => true,
|
||||
@@ -398,7 +448,7 @@ fn sanitize(msg: &mut Message, tools: ToolCapture) {
|
||||
// Both titles a row carries: the session's, and the one its own line set.
|
||||
let change = msg.title_change.as_mut().and_then(|change| change.text.as_mut());
|
||||
for title in msg.session_title.iter_mut().chain(change) {
|
||||
*title = atuin_common::secrets::redact(title).into_owned();
|
||||
redact(title);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -532,6 +582,7 @@ impl AiHarnessSessionCapture {
|
||||
sidecar: AiSessionDatabase,
|
||||
capture: bool,
|
||||
tools: ToolCapture,
|
||||
policy: CapturePolicy,
|
||||
pool: BlockingPool,
|
||||
) -> Self {
|
||||
Self::open_with(
|
||||
@@ -539,6 +590,7 @@ impl AiHarnessSessionCapture {
|
||||
sidecar,
|
||||
capture,
|
||||
tools,
|
||||
policy,
|
||||
pool,
|
||||
#[cfg(test)]
|
||||
None,
|
||||
@@ -550,12 +602,13 @@ impl AiHarnessSessionCapture {
|
||||
sidecar: AiSessionDatabase,
|
||||
capture: bool,
|
||||
tools: ToolCapture,
|
||||
policy: CapturePolicy,
|
||||
pool: BlockingPool,
|
||||
#[cfg(test)] hooks: Option<Arc<dyn hooks::Hooks>>,
|
||||
) -> Self {
|
||||
let (state_tx, state) = watch::channel(StoreState::Recovering);
|
||||
#[cfg_attr(not(test), expect(unused_mut))]
|
||||
let mut sink = Sink::with_state(records, sidecar, tools, state.clone());
|
||||
sink.policy = Arc::new(policy);
|
||||
#[cfg(test)]
|
||||
{
|
||||
sink.hooks = hooks;
|
||||
@@ -963,7 +1016,7 @@ mod tests {
|
||||
Content::Other(serde_json::json!({"attachment": "PRIVATE_ATTACHMENT"})),
|
||||
];
|
||||
sink.append(msg.clone()).await.unwrap();
|
||||
sanitize(&mut msg, ToolCapture::Names);
|
||||
sanitize(&mut msg, ToolCapture::Names, &CapturePolicy::default());
|
||||
// Capture stamps the local host, and the session's id.
|
||||
msg.host = Some(sink.records.host_id());
|
||||
msg.atuin_id =
|
||||
@@ -1107,7 +1160,11 @@ mod tests {
|
||||
for block in stored.unwrap().content {
|
||||
match block {
|
||||
Content::ToolResult(r) => outputs.push((r.call.to_string(), r.output)),
|
||||
Content::ToolUse(u) => assert!(!u.input.is_null(), "the call is kept"),
|
||||
// `atuin login` is a command history doesn't keep (`secrets_filter`): nor
|
||||
// is what the call was given.
|
||||
Content::ToolUse(u) => {
|
||||
assert_eq!(u.input.is_null(), u.id.as_ref() == "login", "{u:?}");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
@@ -1161,6 +1218,104 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// What history and the credential files keep out, capture keeps out of a session: a read of
|
||||
/// `.env` loses its output, a write to it what it wrote, a command history ignores all but
|
||||
/// its name, and every call in a directory `cwd_filter` matches its input and output.
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn the_users_filters_and_credential_files_are_kept_out() {
|
||||
let mut sink = Sink::new(mem_store().await, AiSessionDatabase::in_memory().await.unwrap());
|
||||
sink.tools = ToolCapture::Payloads;
|
||||
sink.policy = Arc::new(policy::tests::policy());
|
||||
let call = |id: &str, name: &str, input: serde_json::Value| {
|
||||
Content::ToolUse(ToolUse {
|
||||
id: ToolCallId::from(id.to_owned()),
|
||||
name: name.to_owned(),
|
||||
input,
|
||||
})
|
||||
};
|
||||
let result = |id: &str| {
|
||||
Content::ToolResult(ToolResult {
|
||||
call: ToolCallId::from(id.to_owned()),
|
||||
output: serde_json::json!("abandon ability able about above absent"),
|
||||
error: false,
|
||||
})
|
||||
};
|
||||
let row = |n: u32, role, cwd: Option<&str>, content| {
|
||||
let mut msg = sample_message();
|
||||
msg.id = RecordId(atuin_common::utils::uuid_v7());
|
||||
msg.source_id = SourceId::from(format!("line-{n}"));
|
||||
msg.timestamp += time::Duration::seconds(n.into());
|
||||
msg.role = role;
|
||||
msg.cwd = cwd.map(Into::into);
|
||||
msg.content = content;
|
||||
msg
|
||||
};
|
||||
let ls = |id: &str| call(id, "Bash", serde_json::json!({"command": "ls"}));
|
||||
let rows = [
|
||||
row(1, Role::Assistant, Some("/w"), vec![
|
||||
call("read", "Read", serde_json::json!({"file_path": ".env"})),
|
||||
call("write", "Write", serde_json::json!({"file_path": ".env", "content": "A=1"})),
|
||||
call("psql", "Bash", serde_json::json!({"command": "psql -c 'select 1'"})),
|
||||
ls("ls"),
|
||||
]),
|
||||
row(2, Role::User, Some("/w"), vec![
|
||||
result("read"),
|
||||
result("write"),
|
||||
result("psql"),
|
||||
result("ls"),
|
||||
]),
|
||||
// Codex and Pi say where a session runs on a row of its own; its calls' rows don't.
|
||||
row(3, Role::User, Some("/secret/project"), vec![]),
|
||||
row(4, Role::Assistant, None, vec![ls("there")]),
|
||||
row(5, Role::User, None, vec![result("there")]),
|
||||
];
|
||||
for msg in &rows {
|
||||
sink.append(msg.clone()).await.unwrap();
|
||||
}
|
||||
// Still so after a restart.
|
||||
let mut restarted = Sink::with_state(
|
||||
sink.records.clone(),
|
||||
sink.sidecar.clone(),
|
||||
ToolCapture::Payloads,
|
||||
watch::channel(StoreState::Ready).1,
|
||||
);
|
||||
restarted.policy = sink.policy.clone();
|
||||
restarted.append(row(6, Role::Assistant, None, vec![ls("again")])).await.unwrap();
|
||||
restarted.append(row(7, Role::User, None, vec![result("again")])).await.unwrap();
|
||||
|
||||
let mut messages = Box::pin(sink.sidecar.messages(&rows[0].session));
|
||||
let (mut inputs, mut outputs) = (Vec::new(), Vec::new());
|
||||
while let Some(stored) = messages.next().await {
|
||||
for block in stored.unwrap().content {
|
||||
match block {
|
||||
Content::ToolUse(u) => inputs.push((u.id.to_string(), u.input)),
|
||||
Content::ToolResult(r) => {
|
||||
outputs.push((r.call.to_string(), !r.output.is_null()));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
let withheld = "[not captured: a file holding credentials]";
|
||||
assert_eq!(inputs, [
|
||||
("read".to_owned(), serde_json::json!({"file_path": ".env"})),
|
||||
("write".to_owned(), serde_json::json!({"file_path": ".env", "content": withheld})),
|
||||
("psql".to_owned(), serde_json::Value::Null),
|
||||
("ls".to_owned(), serde_json::json!({"command": "ls"})),
|
||||
("there".to_owned(), serde_json::Value::Null),
|
||||
("again".to_owned(), serde_json::Value::Null),
|
||||
]);
|
||||
assert_eq!(outputs, [
|
||||
("read".to_owned(), false),
|
||||
("write".to_owned(), false),
|
||||
("psql".to_owned(), false),
|
||||
("ls".to_owned(), true),
|
||||
("there".to_owned(), false),
|
||||
("again".to_owned(), false),
|
||||
]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(None, "Reasoned")]
|
||||
#[case(Some(185), "Reasoning · 185 tokens")]
|
||||
@@ -1305,6 +1460,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(!capture.ready().await);
|
||||
@@ -1321,6 +1477,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(capture.ready().await);
|
||||
@@ -1364,6 +1521,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
// Capture and import hold on this, so the dedup gate sees every persisted message.
|
||||
@@ -1401,6 +1559,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(capture.ready().await);
|
||||
@@ -1454,6 +1613,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(capture.ready().await);
|
||||
@@ -1492,6 +1652,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
capture.rebuild().await.unwrap();
|
||||
@@ -1577,6 +1738,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
Some(gate.clone() as Arc<dyn hooks::Hooks>),
|
||||
);
|
||||
@@ -1625,6 +1787,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
Some(gate.clone() as Arc<dyn hooks::Hooks>),
|
||||
);
|
||||
@@ -1668,6 +1831,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
Some(gate.clone() as Arc<dyn hooks::Hooks>),
|
||||
);
|
||||
@@ -1700,6 +1864,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(capture.ready().await);
|
||||
@@ -1744,6 +1909,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(capture.ready().await);
|
||||
@@ -1893,6 +2059,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
Some(hooks.clone() as Arc<dyn hooks::Hooks>),
|
||||
);
|
||||
@@ -1995,6 +2162,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
Some(hooks.clone() as Arc<dyn hooks::Hooks>),
|
||||
);
|
||||
@@ -2060,6 +2228,7 @@ mod tests {
|
||||
sidecar,
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
);
|
||||
assert!(!capture.ready().await);
|
||||
@@ -2083,6 +2252,7 @@ mod tests {
|
||||
sidecar.clone(),
|
||||
false,
|
||||
ToolCapture::Names,
|
||||
CapturePolicy::default(),
|
||||
BlockingPool::new(NonZeroUsize::MIN),
|
||||
)
|
||||
};
|
||||
@@ -2184,7 +2354,7 @@ mod tests {
|
||||
fn non_conversation_text_is_omitted(#[case] role: Role) {
|
||||
let mut msg = sample_message();
|
||||
msg.role = role;
|
||||
sanitize(&mut msg, ToolCapture::Names);
|
||||
sanitize(&mut msg, ToolCapture::Names, &CapturePolicy::default());
|
||||
assert!(msg.content.is_empty());
|
||||
}
|
||||
|
||||
@@ -2199,7 +2369,7 @@ mod tests {
|
||||
Content::Summary("earlier: AWS_SECRET_ACCESS_KEY=SUMMARYSECRET".to_owned()),
|
||||
Content::Error("overloaded".to_owned()),
|
||||
];
|
||||
sanitize(&mut msg, ToolCapture::Names);
|
||||
sanitize(&mut msg, ToolCapture::Names, &CapturePolicy::default());
|
||||
assert_eq!(msg.content, vec: the user's history
|
||||
//! and output filters, applied to the commands an agent runs as to the ones they run themselves;
|
||||
//! files whose contents are credentials; and redaction, with the user's own patterns.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
|
||||
use atuin_client::settings::Settings;
|
||||
use atuin_client::settings::output::CommandFilter;
|
||||
use atuin_common::harnesstools::access::Access;
|
||||
use atuin_common::harnesstools::session::ToolUse;
|
||||
use atuin_common::secrets::files::SensitiveFiles;
|
||||
use atuin_common::secrets::{self, REDACT_BUDGET, Redactor};
|
||||
use regex::RegexSet;
|
||||
|
||||
use super::tools;
|
||||
|
||||
/// What a call is kept with.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(super) enum Verdict {
|
||||
/// Its input and output, as [`ToolCapture`](super::ToolCapture) keeps them.
|
||||
Keep,
|
||||
/// Its input, but not its output: a command whose output may hold a credential (`atuin key`,
|
||||
/// `gh auth token`, one `[output] command_filter` matches), or a read of a credential file.
|
||||
WithholdOutput,
|
||||
/// Its input but for what it writes, and not its output: a write to a credential file.
|
||||
WithholdWritten,
|
||||
/// Neither: a command history would not keep (`history_filter`, `secrets_filter`), or one run
|
||||
/// in a directory `cwd_filter` matches.
|
||||
Drop,
|
||||
}
|
||||
|
||||
/// The settings capture applies to what it keeps.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct CapturePolicy {
|
||||
history_filter: RegexSet,
|
||||
cwd_filter: RegexSet,
|
||||
secrets_filter: bool,
|
||||
command_filter: CommandFilter,
|
||||
files: SensitiveFiles,
|
||||
redactor: Redactor,
|
||||
}
|
||||
|
||||
impl Default for CapturePolicy {
|
||||
/// The built-in rules alone.
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
history_filter: RegexSet::empty(),
|
||||
cwd_filter: RegexSet::empty(),
|
||||
secrets_filter: true,
|
||||
command_filter: CommandFilter::default(),
|
||||
files: SensitiveFiles::default(),
|
||||
redactor: Redactor::default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CapturePolicy {
|
||||
#[must_use]
|
||||
pub fn from_settings(settings: &Settings) -> Self {
|
||||
Self {
|
||||
history_filter: settings.history_filter.clone(),
|
||||
cwd_filter: settings.cwd_filter.clone(),
|
||||
secrets_filter: settings.secrets_filter,
|
||||
command_filter: settings.output.command_filter().clone(),
|
||||
files: settings.security.sensitive_files(&settings.key_path),
|
||||
redactor: settings.security.redactor(true),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether history ignores what runs in `cwd`: then none of a session's calls there are kept.
|
||||
pub(super) fn ignores_dir(&self, cwd: &Path) -> bool {
|
||||
self.cwd_filter.is_match(&cwd.to_string_lossy())
|
||||
}
|
||||
|
||||
/// What `call`, made in `cwd`, is kept with.
|
||||
pub(super) fn judge(&self, call: &ToolUse, cwd: Option<&Path>) -> Verdict {
|
||||
let access = Access::of(&call.name, &call.input);
|
||||
let dir = access.workdir.as_deref().map(Path::new).or(cwd);
|
||||
if let Some(command) = &access.command {
|
||||
let ignored = self.history_filter.is_match(command)
|
||||
|| (self.secrets_filter && secrets::contains_secret(command))
|
||||
|| access.workdir.as_deref().is_some_and(|dir| self.ignores_dir(Path::new(dir)));
|
||||
if ignored {
|
||||
return Verdict::Drop;
|
||||
}
|
||||
}
|
||||
if access.writes.iter().any(|path| self.files.matches(path, dir)) {
|
||||
return Verdict::WithholdWritten;
|
||||
}
|
||||
let unsafe_command = access.command.as_deref().is_some_and(|command| {
|
||||
secrets::output_unsafe(command)
|
||||
|| self.command_filter.is_match(command)
|
||||
|| self.files.named_in_command(command, dir)
|
||||
});
|
||||
if unsafe_command
|
||||
|| access.reads.iter().any(|path| self.files.matches(path, dir))
|
||||
|| tools::runs_output_unsafe(&call.input)
|
||||
{
|
||||
return Verdict::WithholdOutput;
|
||||
}
|
||||
Verdict::Keep
|
||||
}
|
||||
|
||||
/// `text` redacted, or `None` when that takes too long (see [`REDACT_BUDGET`]): then it is
|
||||
/// better not kept.
|
||||
pub(super) fn redact<'a>(&self, text: &'a str) -> Option<Cow<'a, str>> {
|
||||
self.redactor.redact_within(text, REDACT_BUDGET)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) mod tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
use atuin_common::harnesstools::session::ToolCallId;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[fixture]
|
||||
pub(crate) fn policy() -> CapturePolicy {
|
||||
CapturePolicy {
|
||||
history_filter: RegexSet::new(["^psql"]).unwrap(),
|
||||
cwd_filter: RegexSet::new(["^/secret"]).unwrap(),
|
||||
secrets_filter: true,
|
||||
command_filter: CommandFilter::from(RegexSet::new(["^cat notes"]).unwrap()),
|
||||
files: SensitiveFiles::new(
|
||||
&["*.secret".to_owned()],
|
||||
&[],
|
||||
Some(PathBuf::from("/home/me")),
|
||||
),
|
||||
redactor: Redactor::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn call(name: &str, input: Value) -> ToolUse {
|
||||
ToolUse {
|
||||
id: ToolCallId::from("c".to_owned()),
|
||||
name: name.to_owned(),
|
||||
input,
|
||||
}
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::history_filter("Bash", json!({"command": "psql -c 'select 1'"}), Verdict::Drop)]
|
||||
#[case::secrets_filter("Bash", json!({"command": "atuin login -u me -p hunter2"}), Verdict::Drop)]
|
||||
#[case::cwd_filter("shell", json!({"command": ["ls"], "workdir": "/secret/x"}), Verdict::Drop)]
|
||||
#[case::command_filter("Bash", json!({"command": "cat notes.md"}), Verdict::WithholdOutput)]
|
||||
#[case::credential_command("Bash", json!({"command": "gh auth token"}), Verdict::WithholdOutput)]
|
||||
#[case::atuin_key("exec_command", json!(r#"{"cmd":"atuin key"}"#), Verdict::WithholdOutput)]
|
||||
#[case::cat_dotenv("Bash", json!({"command": "cat .env"}), Verdict::WithholdOutput)]
|
||||
#[case::read_dotenv("Read", json!({"file_path": "/w/.env"}), Verdict::WithholdOutput)]
|
||||
#[case::read_users_file("read", json!({"filePath": "notes.secret"}), Verdict::WithholdOutput)]
|
||||
#[case::read_key("Read", json!({"file_path": "~/.ssh/id_ed25519"}), Verdict::WithholdOutput)]
|
||||
#[case::grep_credentials("Grep", json!({"pattern": "x", "path": "~/.aws/credentials"}), Verdict::WithholdOutput)]
|
||||
#[case::write_dotenv("Write", json!({"file_path": ".env", "content": "A=1"}), Verdict::WithholdWritten)]
|
||||
#[case::patch_dotenv("apply_patch", json!("*** Begin Patch\n*** Update File: .env\n+A=1\n*** End Patch"), Verdict::WithholdWritten)]
|
||||
#[case::shell_write("Bash", json!({"command": "printf '%s' 'violetmeadow' > .env"}), Verdict::WithholdWritten)]
|
||||
#[case::opencode_patch("patch", json!({"patchText": "*** Begin Patch\n*** Update File: .env\n*** End Patch"}), Verdict::WithholdWritten)]
|
||||
#[case::read_source("Read", json!({"file_path": "src/main.rs"}), Verdict::Keep)]
|
||||
#[case::read_example("Read", json!({"file_path": ".env.example"}), Verdict::Keep)]
|
||||
#[case::ordinary("Bash", json!({"command": "cargo test"}), Verdict::Keep)]
|
||||
#[case::search("WebSearch", json!({"query": "rust"}), Verdict::Keep)]
|
||||
fn what_a_call_is_kept_with(
|
||||
policy: CapturePolicy,
|
||||
#[case] name: &str,
|
||||
#[case] input: Value,
|
||||
#[case] verdict: Verdict,
|
||||
) {
|
||||
assert_eq!(policy.judge(&call(name, input), Some(Path::new("/w"))), verdict);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::ignored("/secret/project", true)]
|
||||
#[case::other("/w", false)]
|
||||
fn directories_history_ignores(
|
||||
policy: CapturePolicy,
|
||||
#[case] cwd: &str,
|
||||
#[case] ignored: bool,
|
||||
) {
|
||||
assert_eq!(policy.ignores_dir(Path::new(cwd)), ignored);
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::policy::CapturePolicy;
|
||||
|
||||
/// The most of one tool call's input, or of its output, capture keeps: bytes of it as stored
|
||||
/// (its JSON, strings escaped), past which strings are clipped, keeping their start and end, and
|
||||
/// lists lose their last items. Large enough for nearly every result whole (the harnesses clip
|
||||
@@ -39,13 +41,13 @@ impl ToolCapture {
|
||||
}
|
||||
}
|
||||
|
||||
/// Applies this policy to a tool call's input or output.
|
||||
pub(super) fn apply(self, payload: &mut Value) {
|
||||
/// Applies this policy to a tool call's input or output, redacting as `policy` does.
|
||||
pub(super) fn apply(self, payload: &mut Value, policy: &CapturePolicy) {
|
||||
match self {
|
||||
Self::Names => *payload = Value::Null,
|
||||
Self::Payloads => {
|
||||
let mut budget = TOOL_PAYLOAD_LIMIT;
|
||||
shape(payload, &mut budget);
|
||||
shape(payload, &mut budget, policy);
|
||||
// `shape` keeps the payload's shape, so a harness reads it back as the same kind
|
||||
// of input or output, and holds it to the limit (give or take the short strings
|
||||
// and first items it always keeps), but for an object of very many keys, which it
|
||||
@@ -83,7 +85,7 @@ pub(super) fn runs_output_unsafe(input: &Value) -> bool {
|
||||
/// clipped (but never one [`SHORT`] enough to be a name), and the items of a list past it
|
||||
/// dropped (but never its first); media blocks become a note. Its shape stays as it was, so a
|
||||
/// harness reads it back as the same kind of input or output.
|
||||
fn shape(value: &mut Value, budget: &mut usize) {
|
||||
fn shape(value: &mut Value, budget: &mut usize, policy: &CapturePolicy) {
|
||||
match value {
|
||||
Value::Object(object) => match media_note(object) {
|
||||
Some(note) => {
|
||||
@@ -100,7 +102,7 @@ fn shape(value: &mut Value, budget: &mut usize) {
|
||||
if let Some(name) = named {
|
||||
for key in ["value", "Value"] {
|
||||
if let Some(Value::String(text)) = object.get_mut(key) {
|
||||
redact_named(&name, text);
|
||||
redact_named(&name, text, policy);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -108,9 +110,9 @@ fn shape(value: &mut Value, budget: &mut usize) {
|
||||
for (key, value) in object.iter_mut() {
|
||||
*budget = budget.saturating_sub(stored_len(key) + 2);
|
||||
if let Value::String(text) = value {
|
||||
redact_named(key, text);
|
||||
redact_named(key, text, policy);
|
||||
}
|
||||
shape(value, budget);
|
||||
shape(value, budget, policy);
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -121,17 +123,19 @@ fn shape(value: &mut Value, budget: &mut usize) {
|
||||
if *budget == 0 && kept > 0 {
|
||||
break;
|
||||
}
|
||||
shape(item, budget);
|
||||
shape(item, budget, policy);
|
||||
kept += 1;
|
||||
}
|
||||
items.truncate(kept);
|
||||
}
|
||||
Value::String(text) => {
|
||||
if let Some(shaped) = shape_json_text(text, *budget) {
|
||||
if let Some(shaped) = shape_json_text(text, *budget, policy) {
|
||||
*text = shaped;
|
||||
} else {
|
||||
let redacted = atuin_common::secrets::redact(text);
|
||||
*text = fit(&redacted, (*budget).max(SHORT));
|
||||
*text = match policy.redact(text) {
|
||||
Some(redacted) => fit(&redacted, (*budget).max(SHORT)),
|
||||
None => NOT_REDACTED.to_owned(),
|
||||
};
|
||||
}
|
||||
*budget = budget.saturating_sub(stored_len(text) + 1);
|
||||
}
|
||||
@@ -145,13 +149,13 @@ fn shape(value: &mut Value, budget: &mut usize) {
|
||||
/// is so it stays JSON: as text, a quoted value's quotes are escaped, which hides it from
|
||||
/// redaction, and clipping would cut it mid-string. Kept as written when nothing in it changes;
|
||||
/// `None` when `text` isn't a JSON object or list.
|
||||
fn shape_json_text(text: &str, budget: usize) -> Option<String> {
|
||||
fn shape_json_text(text: &str, budget: usize, policy: &CapturePolicy) -> Option<String> {
|
||||
let parsed = json_text(text)?;
|
||||
let room = budget.max(SHORT);
|
||||
let mut allowance = budget;
|
||||
loop {
|
||||
let mut shaped = parsed.clone();
|
||||
shape(&mut shaped, &mut allowance.clone());
|
||||
shape(&mut shaped, &mut allowance.clone(), policy);
|
||||
let shaped = if shaped == parsed {
|
||||
text.to_owned()
|
||||
} else {
|
||||
@@ -199,9 +203,12 @@ fn fit(text: &str, room: usize) -> String {
|
||||
|
||||
/// `text`, the value of `key`, redacted as the assignment it is: a secret is often known by the
|
||||
/// name it is kept under (`{"AWS_SECRET_ACCESS_KEY": "..."}`), which the value alone doesn't say.
|
||||
fn redact_named(key: &str, text: &mut String) {
|
||||
fn redact_named(key: &str, text: &mut String, policy: &CapturePolicy) {
|
||||
let line = format!("{key}={text}");
|
||||
let redacted = atuin_common::secrets::redact(&line);
|
||||
let Some(redacted) = policy.redact(&line) else {
|
||||
NOT_REDACTED.clone_into(text);
|
||||
return;
|
||||
};
|
||||
if redacted != line {
|
||||
let value = redacted.strip_prefix(key).and_then(|rest| rest.strip_prefix('='));
|
||||
value.unwrap_or(atuin_common::secrets::REDACTED).clone_into(text);
|
||||
@@ -237,6 +244,56 @@ fn media_note(object: &Map<String, Value>) -> Option<Value> {
|
||||
])))
|
||||
}
|
||||
|
||||
/// In place of a string that took too long to redact (see
|
||||
/// [`REDACT_BUDGET`](atuin_common::secrets::REDACT_BUDGET)).
|
||||
pub(super) const NOT_REDACTED: &str = "[not captured: took too long to redact]";
|
||||
|
||||
/// In place of what a call writes to a file holding credentials.
|
||||
const WITHHELD: &str = "[not captured: a file holding credentials]";
|
||||
|
||||
/// `input`, a call writing to a file holding credentials, without what it writes: every string
|
||||
/// but the paths it names, and of a patch, all but its headers (`*** Update File: .env`). It keeps
|
||||
/// its shape, so a harness reads it back as the same call.
|
||||
pub(super) fn withhold_written(input: &mut Value) {
|
||||
fn blank(value: &mut Value) {
|
||||
match value {
|
||||
Value::String(text) => WITHHELD.clone_into(text),
|
||||
Value::Array(items) => items.iter_mut().for_each(blank),
|
||||
Value::Object(object) => object.values_mut().for_each(blank),
|
||||
Value::Null | Value::Bool(_) | Value::Number(_) => {}
|
||||
}
|
||||
}
|
||||
match input {
|
||||
Value::Object(object) => {
|
||||
for (key, value) in object.iter_mut() {
|
||||
if !PATH_KEYS.contains(&key.as_str()) {
|
||||
blank(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
Value::String(text) => match json_text(text) {
|
||||
Some(mut parsed) => {
|
||||
withhold_written(&mut parsed);
|
||||
*text = parsed.to_string();
|
||||
}
|
||||
None => {
|
||||
let headers: Vec<&str> =
|
||||
text.lines().filter(|line| line.trim_start().starts_with("*** ")).collect();
|
||||
*text = if headers.is_empty() {
|
||||
WITHHELD.to_owned()
|
||||
} else {
|
||||
headers.join("\n")
|
||||
};
|
||||
}
|
||||
},
|
||||
Value::Array(_) | Value::Null | Value::Bool(_) | Value::Number(_) => blank(input),
|
||||
}
|
||||
}
|
||||
|
||||
/// The keys a call names a file under, kept when what it writes isn't.
|
||||
const PATH_KEYS: &[&str] =
|
||||
&["file_path", "filePath", "path", "notebook_path", "filename", "file", "paths", "file_paths"];
|
||||
|
||||
/// About the length of [`clip`]'s note.
|
||||
const NOTE: usize = 40;
|
||||
|
||||
@@ -262,7 +319,7 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
fn kept(mut value: Value) -> Value {
|
||||
ToolCapture::Payloads.apply(&mut value);
|
||||
ToolCapture::Payloads.apply(&mut value, &CapturePolicy::default());
|
||||
value
|
||||
}
|
||||
|
||||
@@ -294,7 +351,7 @@ mod tests {
|
||||
#[rstest]
|
||||
fn names_only_keeps_nothing() {
|
||||
let mut value = json!({"command": "ls"});
|
||||
ToolCapture::Names.apply(&mut value);
|
||||
ToolCapture::Names.apply(&mut value, &CapturePolicy::default());
|
||||
assert!(value.is_null());
|
||||
}
|
||||
|
||||
@@ -392,6 +449,29 @@ mod tests {
|
||||
assert_eq!(runs_output_unsafe(&input), unsafe_);
|
||||
}
|
||||
|
||||
/// A write to a credential file keeps its shape and its paths, and nothing it writes.
|
||||
#[rstest]
|
||||
#[case::claude(
|
||||
json!({"file_path": ".env", "content": "A=1"}),
|
||||
json!({"file_path": ".env", "content": WITHHELD}),
|
||||
)]
|
||||
#[case::pi_edits(
|
||||
json!({"path": ".env", "edits": [{"oldText": "A=1", "newText": "A=2"}]}),
|
||||
json!({"path": ".env", "edits": [{"oldText": WITHHELD, "newText": WITHHELD}]}),
|
||||
)]
|
||||
#[case::codex_arguments(
|
||||
json!(r#"{"path":".env","content":"A=1"}"#),
|
||||
json!(json!({"path": ".env", "content": WITHHELD}).to_string()),
|
||||
)]
|
||||
#[case::patch(
|
||||
json!("*** Begin Patch\n*** Update File: .env\n@@\n-A=1\n+A=2\n*** End Patch"),
|
||||
json!("*** Begin Patch\n*** Update File: .env\n*** End Patch"),
|
||||
)]
|
||||
fn a_credential_files_writes_are_withheld(#[case] mut input: Value, #[case] want: Value) {
|
||||
withhold_written(&mut input);
|
||||
assert_eq!(input, want);
|
||||
}
|
||||
|
||||
/// A payload of many values each too short to clip is still held to the limit.
|
||||
#[rstest]
|
||||
#[case::listing(Value::Array((0..20_000).map(|n| json!({"name": format!("pkg-{n}"), "version": "1.0.0"})).collect()))]
|
||||
|
||||
@@ -554,6 +554,7 @@ async fn captures_never_outlive_their_entries_under_contention() {
|
||||
id,
|
||||
capture("out"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await;
|
||||
}));
|
||||
|
||||
@@ -320,6 +320,7 @@ async fn delete_forgets_captured_output(
|
||||
id,
|
||||
capture("secret output"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -378,6 +379,7 @@ async fn output_for_a_deleted_or_unknown_id_is_refused(
|
||||
id,
|
||||
capture("late"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
@@ -412,6 +414,7 @@ async fn cancel_discards_captured_output(#[future(awt)] env: TestEnv) {
|
||||
id,
|
||||
capture("boom"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
@@ -426,6 +429,7 @@ async fn cancel_discards_captured_output(#[future(awt)] env: TestEnv) {
|
||||
id,
|
||||
capture("late"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
@@ -454,6 +458,7 @@ async fn output_arriving_mid_delete_is_refused(#[future(awt)] env: TestEnv) {
|
||||
id,
|
||||
capture("late"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
@@ -501,6 +506,7 @@ async fn abandoned_delete_still_completes(#[future(awt)] env: TestEnv) {
|
||||
id,
|
||||
capture("late"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
@@ -162,6 +162,7 @@ async fn apply(env: &TestEnv, model: &mut Model, op: &Op) {
|
||||
id,
|
||||
capture("out"),
|
||||
&OutputCapture::Enabled(CaptureLimits::default()),
|
||||
&atuin_common::secrets::files::SensitiveFiles::default(),
|
||||
)
|
||||
.await;
|
||||
let state = model.slots[usize::from(*slot)].1;
|
||||
|
||||
@@ -58,9 +58,15 @@ async fn test_start_end_history(#[future(awt)] env: TestEnv) {
|
||||
#[rstest]
|
||||
#[case::key("atuin key", false)]
|
||||
#[case::account_login("atuin account login -u me", false)]
|
||||
#[case::gh_token("gh auth token", false)]
|
||||
#[case::kubectl_secret("kubectl get secret db -o yaml", false)]
|
||||
#[case::printenv("printenv", false)]
|
||||
#[case::dotenv("cat .env", false)]
|
||||
#[case::aws_credentials("cat ~/.aws/credentials", false)]
|
||||
#[case::ssh_key("cat id_ed25519", false)]
|
||||
#[case::ordinary("echo hello", true)]
|
||||
#[tokio::test]
|
||||
async fn output_is_stored_only_for_commands_that_cannot_print_the_key(
|
||||
async fn output_is_stored_only_for_commands_that_cannot_print_a_credential(
|
||||
#[future(awt)] env: TestEnv,
|
||||
#[case] command: &str,
|
||||
#[case] stored: bool,
|
||||
@@ -148,7 +154,7 @@ async fn disabling_output_on_a_live_daemon_stores_no_captures(
|
||||
..CaptureLimits::default()
|
||||
});
|
||||
env.handle.apply_settings(settings.clone()).await;
|
||||
settings.output = OutputCapture::Disabled;
|
||||
settings.output = OutputCapture::default();
|
||||
env.handle.apply_settings(settings).await;
|
||||
|
||||
let mut client = env.history_client().await;
|
||||
|
||||
@@ -127,23 +127,30 @@ fn semantic_command_capture_config() -> Option<atuin_pty_proxy::CaptureConfig> {
|
||||
return;
|
||||
};
|
||||
|
||||
let redactor = settings.security.redactor(settings.secrets_filter);
|
||||
while let Ok((history_id, capture)) = rx.recv() {
|
||||
// Output can carry credentials the command line never showed, e.g. `cat .env`.
|
||||
// Swap the string only when something was actually taken out, so that clean
|
||||
// output -- nearly all of it -- reaches the daemon without being copied.
|
||||
// output -- nearly all of it -- reaches the daemon without being copied. Output
|
||||
// that can't be redacted quickly is not kept at all.
|
||||
let redact = |output: &mut String| {
|
||||
if settings.secrets_filter
|
||||
&& let Cow::Owned(redacted) = atuin_common::secrets::redact(output)
|
||||
{
|
||||
*output = redacted;
|
||||
match redactor.redact_within(output, atuin_common::secrets::REDACT_BUDGET) {
|
||||
Some(Cow::Borrowed(_)) => true,
|
||||
Some(Cow::Owned(redacted)) => {
|
||||
*output = redacted;
|
||||
true
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
};
|
||||
|
||||
let mut output_start = capture.output_start;
|
||||
redact(&mut output_start);
|
||||
let mut output_end = capture.output_end;
|
||||
if let Some(output_end) = output_end.as_mut() {
|
||||
redact(output_end);
|
||||
let redacted = redact(&mut output_start)
|
||||
&& output_end.as_mut().is_none_or(redact);
|
||||
if !redacted {
|
||||
tracing::debug!(%history_id, "output took too long to redact; dropping it");
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Err(err) = client
|
||||
|
||||
@@ -19,12 +19,34 @@ in another agent (see [Resuming](#resuming)). Reasoning is kept only as a marker
|
||||
that the model reasoned.
|
||||
|
||||
Secrets Atuin recognises are replaced with `****`, in the conversation and the
|
||||
tool calls alike. Images are left out, and the input or output of a single call
|
||||
is clipped to 64 KiB, keeping its start and its end. As with
|
||||
[output capture](../guide/output-capture.md), the output of Atuin's own
|
||||
credential commands (`atuin key`, `atuin login`, `atuin register` and
|
||||
`atuin account change-password`) is never kept. Search covers a call's input,
|
||||
but not its output.
|
||||
tool calls alike: the [same patterns](../configuration/config.md#secrets_filter)
|
||||
as for captured command output, and your own
|
||||
[`redact_patterns`](../configuration/config.md#redact_patterns). Images are left
|
||||
out, and the input or output of a single call is clipped to 64 KiB, keeping its
|
||||
start and its end. Search covers a call's input, but not its output.
|
||||
|
||||
The rules that keep things out of your history and your captured output apply
|
||||
to your agents' tool calls too:
|
||||
|
||||
- A command [`history_filter`](../configuration/config.md#history_filter)
|
||||
excludes, or one [`secrets_filter`](../configuration/config.md#secrets_filter)
|
||||
recognises a credential in, is kept as the name of its tool only, without its
|
||||
input or output. Every call in a directory
|
||||
[`cwd_filter`](../configuration/config.md#cwd_filter) excludes is kept the
|
||||
same way.
|
||||
- A command whose output is never stored is kept without its output: one
|
||||
[`command_filter`](../configuration/config.md#command_filter) matches, one of
|
||||
Atuin's own credential commands (such as `atuin key` and `atuin login`), or
|
||||
another tool's that prints a credential, such as `gh auth token` or
|
||||
`kubectl get secret`.
|
||||
- A file holding credentials (`.env`, `~/.ssh/id_ed25519`,
|
||||
`~/.aws/credentials`, any of your
|
||||
[`sensitive_files`](../configuration/config.md#sensitive_files)) is never
|
||||
captured: a call that reads one, or a command naming one, is kept without its
|
||||
output, and a call that writes to one without what it wrote.
|
||||
|
||||
Text that can't be redacted quickly isn't kept. To keep more out, see
|
||||
[Keeping Secrets Out of Atuin](../guide/excluding-commands.md).
|
||||
|
||||
Tool calls can be a lot of your codebase, not just your conversations. To keep
|
||||
only the name of each tool called (and whether the call failed), set
|
||||
@@ -36,8 +58,8 @@ capture_sessions = true
|
||||
capture_tools = false
|
||||
```
|
||||
|
||||
The setting applies to sessions captured or imported after the daemon picks it
|
||||
up. Sessions already captured keep what they had.
|
||||
These settings, and the filters above, apply to sessions captured or imported
|
||||
after the daemon restarts. Sessions already captured keep what they had.
|
||||
|
||||
## Searching
|
||||
|
||||
|
||||
@@ -364,7 +364,9 @@ history_filter = [
|
||||
```
|
||||
|
||||
A filtered command's output isn't captured either. To keep a command in history
|
||||
but not its output, use [`command_filter`](#command_filter) in `[output]`.
|
||||
but not its output, use [`command_filter`](#command_filter) in `[output]`. The
|
||||
filter applies to the commands your AI agents run too: a matching call is
|
||||
[captured](../ai/sessions.md#whats-captured) with its tool's name only.
|
||||
|
||||
### `cwd_filter`
|
||||
|
||||
@@ -382,6 +384,9 @@ This supports regular expressions, so you can hide pretty much whatever you want
|
||||
|
||||
After updating that parameter, you can run [the prune command](../reference/prune.md) to remove old history entries that match the new filters.
|
||||
|
||||
An [AI session](../ai/sessions.md#whats-captured) captured in a matching
|
||||
directory keeps only the names of the tools its agent called there.
|
||||
|
||||
### `store_failed`
|
||||
|
||||
Default: `true`
|
||||
@@ -405,27 +410,42 @@ to save it if any of them match. The patterns currently cover:
|
||||
|
||||
| Service | Matches |
|
||||
|---------|---------|
|
||||
| AWS | Access key IDs, and commands setting `AWS_SECRET_ACCESS_KEY` or `AWS_SESSION_TOKEN` |
|
||||
| Azure | Commands setting `AZURE_*_KEY` |
|
||||
| Google Cloud | Commands setting `GOOGLE_SERVICE_ACCOUNT_KEY` |
|
||||
| AWS | Access key IDs, commands setting `AWS_SECRET_ACCESS_KEY` or `AWS_SESSION_TOKEN`, and Amazon Bedrock API keys |
|
||||
| Azure | Commands setting `AZURE_*_KEY`, and Entra ID client secrets |
|
||||
| Google Cloud | Commands setting `GOOGLE_SERVICE_ACCOUNT_KEY`, and API keys |
|
||||
| GitHub | Personal access tokens (old and new), OAuth access tokens (app and user), app installation tokens, and refresh tokens |
|
||||
| GitLab | Personal access tokens |
|
||||
| Slack | OAuth v2 bot and user tokens, and webhook URLs |
|
||||
| Stripe | Live and test keys |
|
||||
| Netlify | Authentication tokens |
|
||||
| npm | Tokens |
|
||||
| Pulumi | Personal access tokens |
|
||||
| GitLab | Personal, deploy, runner, CI/CD job, feed, SCIM, agent and OAuth tokens |
|
||||
| AI providers | Anthropic, OpenAI, OpenRouter, Groq, xAI, Perplexity, Pinecone, LangSmith and Hugging Face keys |
|
||||
| Slack | Bot, user, app and config tokens, and webhook URLs |
|
||||
| Stripe | Live, test and restricted keys |
|
||||
| Package registries | npm, PyPI, RubyGems, Clojars, Docker Hub and Artifactory tokens |
|
||||
| Infrastructure | DigitalOcean, Terraform Cloud, HashiCorp Vault, Fly.io, Heroku, Cloudflare origin CA, Databricks, PlanetScale, Supabase, Tailscale, Dynatrace, Grafana, Sentry, Doppler and OpenShift tokens |
|
||||
| Other services | Netlify, Pulumi, Atlassian, Linear, Notion, Postman, Shopify, SendGrid, Brevo, Square, EasyPost, Duffel, Frame.io, ReadMe, Prefect, Infracost and Telegram tokens, Discord and Microsoft Teams webhooks, 1Password service account tokens and age secret keys |
|
||||
| Atuin | `atuin login`, which takes your password and encryption key as arguments |
|
||||
|
||||
Many of these come from the rules of [gitleaks](https://github.com/gitleaks/gitleaks).
|
||||
For the exact expressions, see
|
||||
[`secrets.rs`](https://github.com/atuinsh/atuin/blob/main/crates/atuin-common/src/secrets.rs).
|
||||
|
||||
The same patterns are applied to captured command output. A command whose
|
||||
own text is clean can still print a credential — `cat .env`, `gh auth token`
|
||||
— so recognised values in the captured output are replaced with `****`
|
||||
before storage. Only the value is replaced; the variable name or flag
|
||||
beside it stays. A credential that color codes split apart in the output
|
||||
isn't recognised.
|
||||
The same patterns are applied to captured command output, and to
|
||||
[AI sessions](../ai/sessions.md#whats-captured). A command whose own text is
|
||||
clean can still print a credential, so recognised values in the captured
|
||||
output are replaced with `****` before storage. Only the value is replaced; the
|
||||
variable name or flag beside it stays. Output is also checked for credentials
|
||||
recognised by where they sit, which never keep a command out of history:
|
||||
|
||||
- private keys (`-----BEGIN ... PRIVATE KEY-----`), JSON web tokens,
|
||||
`Authorization` headers and bearer tokens, and `curl -u user:password`
|
||||
- the password in a web address or connection string (`postgres://app:pw@db`,
|
||||
`password=...`)
|
||||
- a value assigned to a name that holds a credential, such as
|
||||
`DB_PASSWORD=...`, `"client_secret": "..."` or `api_key: ...`. Placeholders
|
||||
like `${TOKEN}`, `<password>` and `changeme` stay.
|
||||
|
||||
A credential that color codes split apart in the output isn't recognised.
|
||||
Output that can't be redacted quickly (within a quarter of a second) isn't
|
||||
stored at all. Add patterns of your own with
|
||||
[`redact_patterns`](#redact_patterns).
|
||||
|
||||
!!! note
|
||||
|
||||
@@ -433,7 +453,7 @@ isn't recognised.
|
||||
recognized formats — use [`history_filter`](#history_filter) for anything
|
||||
else you need kept out, or [`command_filter`](#command_filter) to keep a
|
||||
command but not its output, and see
|
||||
[Excluding Commands from History](../guide/excluding-commands.md).
|
||||
[Keeping Secrets Out of Atuin](../guide/excluding-commands.md).
|
||||
|
||||
### macOS Ctrl-n key shortcuts
|
||||
|
||||
@@ -1109,6 +1129,19 @@ command_filter = [
|
||||
The filter applies to output captured after the daemon picks up the change; it
|
||||
doesn't remove output that's already stored.
|
||||
|
||||
Some commands' output is never stored, whatever this filter says, because it
|
||||
can hold a credential: Atuin's own `atuin key`, `atuin login`, `atuin register`
|
||||
and `atuin account change-password`; commands that print a token or a password,
|
||||
such as `gh auth token`, `gcloud auth print-access-token`,
|
||||
`aws configure get`, `kubectl get secret`, `vault kv get`, `op read`,
|
||||
`pass show` and `security find-generic-password -w`; commands that print the
|
||||
environment (`printenv`, `env`); and commands that name a
|
||||
[sensitive file](#sensitive_files), such as `cat .env`.
|
||||
|
||||
The filter also applies to the commands your AI agents run, whether or not
|
||||
output capture is enabled: a matching call is
|
||||
[captured](../ai/sessions.md#whats-captured) without its output.
|
||||
|
||||
### `sync`
|
||||
|
||||
Default: `false`
|
||||
@@ -1117,6 +1150,79 @@ Reserved for syncing captured output between machines, which isn't
|
||||
implemented yet: captured output never leaves your machine, whatever this is
|
||||
set to.
|
||||
|
||||
## Security
|
||||
|
||||
Settings under `[security]` add to Atuin's built-in rules for keeping
|
||||
credentials out of what it stores: captured command output and
|
||||
[AI sessions](../ai/sessions.md#whats-captured). For which setting to reach for,
|
||||
see [Keeping Secrets Out of Atuin](../guide/excluding-commands.md).
|
||||
|
||||
```toml
|
||||
[security]
|
||||
redact_patterns = []
|
||||
sensitive_files = []
|
||||
```
|
||||
|
||||
### `redact_patterns`
|
||||
|
||||
Default: `[]`
|
||||
|
||||
Regular expressions for credentials of your own, redacted wherever Atuin
|
||||
redacts, alongside the [built-in patterns](#secrets_filter). Atuin replaces a
|
||||
pattern's `secret` group with `****` if it has one, otherwise everything the
|
||||
pattern matches.
|
||||
|
||||
```toml
|
||||
[security]
|
||||
redact_patterns = [
|
||||
"ACME-[0-9A-F]{32}",
|
||||
"internal-api-key: (?<secret>\\S+)",
|
||||
]
|
||||
```
|
||||
|
||||
These patterns don't keep a command out of history; use
|
||||
[`history_filter`](#history_filter) for that. When
|
||||
[`secrets_filter`](#secrets_filter) is off, captured command output is redacted
|
||||
with these patterns only.
|
||||
|
||||
### `sensitive_files`
|
||||
|
||||
Default: `[]`
|
||||
|
||||
Files whose contents are credentials, on top of the built-in ones. The output
|
||||
of a command naming one isn't stored. An AI agent's reads of one aren't
|
||||
captured, and nor is what it writes to one.
|
||||
|
||||
A glob without a `/` matches a file's name wherever it is. A glob with a `/`
|
||||
matches its path, from `~` or `/`, or anywhere below a directory for a relative
|
||||
one.
|
||||
|
||||
```toml
|
||||
[security]
|
||||
sensitive_files = [
|
||||
"*.secret",
|
||||
"~/work/credentials/**",
|
||||
"config/master.key",
|
||||
]
|
||||
```
|
||||
|
||||
The built-in files are:
|
||||
|
||||
- `.env` and `.env.*` (but not `.env.example`, `.env.sample`, `.env.template`
|
||||
or `.env.dist`), `*.env` and `.envrc`
|
||||
- private keys: `id_rsa`, `id_dsa`, `id_ecdsa`, `id_ed25519` (not their
|
||||
`.pub` halves), `*.pem`, `*.key`, `*.p12`, `*.pfx`, `*.jks` and `*.keystore`
|
||||
- Terraform state and variables: `*.tfstate`, `*.tfvars`
|
||||
- `.netrc`, `.pgpass`, `.git-credentials`, `.npmrc`, `.pypirc`, `.htpasswd`
|
||||
and `.vault-token`
|
||||
- cloud and registry logins: `~/.aws/credentials` and its SSO cache, the
|
||||
gcloud credential files, `~/.kube/config`, `~/.docker/config.json`, the
|
||||
`gh` and `glab` logins, and Cargo, RubyGems and Terraform Cloud credentials
|
||||
- `~/.gnupg/private-keys-v1.d` and `~/.password-store`
|
||||
- the AI agents' own logins: `~/.claude/.credentials.json`,
|
||||
`~/.codex/auth.json`, opencode's and Pi's `auth.json`
|
||||
- Atuin's encryption key ([`key_path`](#key_path))
|
||||
|
||||
## logs
|
||||
|
||||
Behavior of log files.
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ history_filter = [
|
||||
|
||||
You can also exclude commands by directory with `cwd_filter`, or prefix individual commands with a space.
|
||||
|
||||
See [Excluding Commands from History](guide/excluding-commands.md) for more options.
|
||||
See [Keeping Secrets Out of Atuin](guide/excluding-commands.md) for more options.
|
||||
|
||||
## How do I remove the default up arrow binding?
|
||||
|
||||
|
||||
@@ -1,13 +1,31 @@
|
||||
# Excluding Commands from History
|
||||
# Keeping Secrets Out of Atuin
|
||||
|
||||
Sometimes you don't want a command in your history and Atuin gives you four ways
|
||||
to exclude the commands.
|
||||
Atuin records your commands, and can also record
|
||||
[what they print](output-capture.md) and your
|
||||
[AI agents' sessions](../ai/sessions.md). This page covers how to keep secrets,
|
||||
and anything else you don't want stored, out of all three.
|
||||
|
||||
## Prefix with a space
|
||||
| You want to keep out | Use |
|
||||
|----------------------|-----|
|
||||
| One command, just this once | [A leading space](#one-command-with-a-leading-space) |
|
||||
| Commands matching a pattern | [`history_filter`](#commands-history_filter) |
|
||||
| Everything run in a directory | [`cwd_filter`](#a-directory-cwd_filter) |
|
||||
| What a command prints, but not the command | [`command_filter`](#a-commands-output-command_filter) |
|
||||
| The contents of a file, read or written | [`sensitive_files`](#a-files-contents-sensitive_files) |
|
||||
| A secret of your own, wherever it appears | [`redact_patterns`](#your-own-secrets-redact_patterns) |
|
||||
| What your AI agents' tools read and run | [`capture_tools`](#ai-agents-tool-calls-capture_tools) |
|
||||
| Everything from a particular tool | [Skip Atuin for it](#everything-from-a-tool) |
|
||||
|
||||
Atuin already keeps a lot out without any setup; see
|
||||
[what's kept out by default](#whats-kept-out-by-default). Filters apply to what
|
||||
Atuin records from then on; to remove what it already has, see
|
||||
[cleaning up](#cleaning-up-what-you-already-recorded).
|
||||
|
||||
## One command, with a leading space
|
||||
|
||||
Most shells support "ignorespace": a command typed with a leading space isn't
|
||||
saved to history. Atuin honors this convention, and it's the quickest way to
|
||||
keep a single command out.
|
||||
keep a single command out. Its output isn't captured either.
|
||||
|
||||
```shell
|
||||
echo "this won't be saved" # note the leading space
|
||||
@@ -19,10 +37,10 @@ keep a single command out.
|
||||
still appear in your bash history. See [installation](installation.md) for
|
||||
details.
|
||||
|
||||
## Filter by command: `history_filter`
|
||||
## Commands: `history_filter`
|
||||
|
||||
[`history_filter`](../configuration/config.md#history_filter) excludes any
|
||||
command matching a regular expression:
|
||||
[`history_filter`](../configuration/config.md#history_filter) keeps any command
|
||||
matching a regular expression out of your history:
|
||||
|
||||
```toml
|
||||
history_filter = [
|
||||
@@ -35,9 +53,13 @@ history_filter = [
|
||||
Patterns are unanchored, so `secret` matches anywhere in the command. Use `^`
|
||||
and `$` when you want to match the whole command exactly.
|
||||
|
||||
## Filter by directory: `cwd_filter`
|
||||
A filtered command's output isn't captured. When an AI agent runs a matching
|
||||
command, its session keeps only the name of the tool it used, not the command
|
||||
or what it printed.
|
||||
|
||||
[`cwd_filter`](../configuration/config.md#cwd_filter) excludes every command
|
||||
## A directory: `cwd_filter`
|
||||
|
||||
[`cwd_filter`](../configuration/config.md#cwd_filter) keeps out every command
|
||||
run from a matching directory:
|
||||
|
||||
```toml
|
||||
@@ -51,26 +73,84 @@ cwd_filter = [
|
||||
These patterns are unanchored regular expressions too, matched against the
|
||||
working directory path.
|
||||
|
||||
## Keep the command, drop its output: `command_filter`
|
||||
Nothing run there has its output captured. An AI agent working in a matching
|
||||
directory has every tool call there kept as the tool's name only.
|
||||
|
||||
With [output capture](output-capture.md) on, both filters above also keep
|
||||
a command's output out of the store. To record a command in your history but
|
||||
never store what it prints, use
|
||||
## A command's output: `command_filter`
|
||||
|
||||
To keep a command in your history but never store what it prints, use
|
||||
[`command_filter`](../configuration/config.md#command_filter) in `[output]`:
|
||||
|
||||
```toml
|
||||
[output]
|
||||
enabled = true
|
||||
command_filter = [
|
||||
"^cat ", # file contents, like `cat .env`
|
||||
"^kubectl get secret",
|
||||
"^terraform plan",
|
||||
"^psql ",
|
||||
]
|
||||
```
|
||||
|
||||
Patterns work like `history_filter`'s: unanchored regular expressions matched
|
||||
against the command.
|
||||
|
||||
## Skip Atuin entirely for a tool
|
||||
The filter applies to your AI agents' commands too, even if you don't capture
|
||||
your own output: a matching command is kept in the session, but what it printed
|
||||
isn't.
|
||||
|
||||
## A file's contents: `sensitive_files`
|
||||
|
||||
Some files hold nothing but credentials: `.env` files, SSH keys, cloud logins.
|
||||
Atuin knows the common ones (see
|
||||
[`sensitive_files`](../configuration/config.md#sensitive_files) for the list)
|
||||
and keeps their contents out:
|
||||
|
||||
- the output of a command that names one, such as `cat .env` or
|
||||
`cat ~/.aws/credentials`
|
||||
- what an AI agent reads from one
|
||||
- what an AI agent writes to one, whether with its edit tool or a command like
|
||||
`echo KEY=value > .env`. The session keeps that the agent changed the file,
|
||||
not what it wrote.
|
||||
|
||||
Add your own by name or by path:
|
||||
|
||||
```toml
|
||||
[security]
|
||||
sensitive_files = [
|
||||
"*.secret", # any file with this name, anywhere
|
||||
"~/work/credentials/**", # everything under this directory
|
||||
"config/master.key", # this path within any project
|
||||
]
|
||||
```
|
||||
|
||||
## Your own secrets: `redact_patterns`
|
||||
|
||||
Atuin replaces the credentials it recognises with `****` in captured output and
|
||||
AI sessions. If you have secrets in a format it doesn't know, such as internal
|
||||
tokens, give it their pattern:
|
||||
|
||||
```toml
|
||||
[security]
|
||||
redact_patterns = [
|
||||
"ACME-[0-9A-F]{32}", # replace the whole match
|
||||
"internal-api-key: (?<secret>\\S+)", # replace only the `secret` group
|
||||
]
|
||||
```
|
||||
|
||||
Redaction replaces the value and keeps the text around it. To keep a whole
|
||||
command out instead, use [`history_filter`](#commands-history_filter).
|
||||
|
||||
## AI agents' tool calls: `capture_tools`
|
||||
|
||||
When Atuin [captures your AI agents' sessions](../ai/sessions.md), it keeps
|
||||
their tool calls with what each was given and what it returned, after applying
|
||||
everything on this page. To keep only the name of each tool the agent called,
|
||||
turn that off:
|
||||
|
||||
```toml
|
||||
[ai]
|
||||
capture_tools = false
|
||||
```
|
||||
|
||||
## Everything from a tool
|
||||
|
||||
If a tool spawns interactive shells and you'd rather it recorded nothing at
|
||||
all, guard the `atuin init` call in your shell config:
|
||||
@@ -89,12 +169,35 @@ what the plugin sets up.
|
||||
!!! tip "Commands from AI agents"
|
||||
You don't need to exclude AI agent commands to keep them out of your way.
|
||||
Atuin tags them with the agent that ran them and hides them from interactive
|
||||
search by default — see [AI Agent Hooks](agent-hooks.md).
|
||||
search by default; see [AI Agent Hooks](agent-hooks.md).
|
||||
|
||||
## Cleaning up commands you already recorded
|
||||
## What's kept out by default
|
||||
|
||||
Filters only apply going forward. To remove entries recorded *before* you added
|
||||
a filter, run [`atuin history prune`](../reference/prune.md):
|
||||
Without any of the settings above, Atuin:
|
||||
|
||||
- **doesn't record commands containing a credential** it recognises, such as
|
||||
AWS, GitHub, OpenAI or Anthropic keys, while
|
||||
[`secrets_filter`](../configuration/config.md#secrets_filter) is on (the
|
||||
default). It also leaves those commands out of AI sessions.
|
||||
- **redacts credentials in captured output and AI sessions**, replacing them
|
||||
with `****`: the same formats, plus private keys, passwords in connection
|
||||
strings, and values assigned to names like `DB_PASSWORD` or `api_key`.
|
||||
- **never stores the output of commands that print a credential**: Atuin's own
|
||||
`atuin key` and `atuin login`, and others such as `gh auth token`,
|
||||
`aws configure get`, `kubectl get secret` and `printenv`. See
|
||||
[`command_filter`](../configuration/config.md#command_filter) for the list.
|
||||
- **keeps the contents of credential files out**, as described under
|
||||
[`sensitive_files`](#a-files-contents-sensitive_files).
|
||||
- **doesn't store output it can't redact quickly**, rather than slow your shell
|
||||
down.
|
||||
|
||||
Recognising credentials is best-effort: it only knows common formats. For
|
||||
anything it might miss, use the settings above.
|
||||
|
||||
## Cleaning up what you already recorded
|
||||
|
||||
Filters only apply going forward. To remove history entries recorded *before*
|
||||
you added a filter, run [`atuin history prune`](../reference/prune.md):
|
||||
|
||||
```shell
|
||||
# See what would be removed
|
||||
@@ -108,9 +211,5 @@ This deletes existing entries matching your current `history_filter` and
|
||||
`cwd_filter`. For deleting entries that don't match a filter, see [Deleting
|
||||
History](delete-history.md).
|
||||
|
||||
## Secrets are filtered automatically
|
||||
|
||||
Independently of your own filters, Atuin refuses to record commands that look
|
||||
like they contain credentials — AWS keys, GitHub and npm tokens, Slack
|
||||
webhooks, Stripe keys, and more. This is on by default. For the full list,
|
||||
see [`secrets_filter`](../configuration/config.md#secrets_filter).
|
||||
AI sessions and captured output already stored keep what they had. Changes to
|
||||
these settings apply to AI sessions once the daemon restarts.
|
||||
|
||||
@@ -80,8 +80,15 @@ Some output is never stored:
|
||||
- **Commands in
|
||||
[`command_filter`](../configuration/config.md#command_filter).** These stay
|
||||
in your history; only their output is dropped.
|
||||
- **Atuin's own credential commands**: `atuin key`, `atuin login`,
|
||||
`atuin register`, and `atuin account change-password`.
|
||||
- **Commands whose output can hold a credential**: Atuin's own `atuin key`,
|
||||
`atuin login`, `atuin register` and `atuin account change-password`, and
|
||||
other tools' commands that print a token, a password or the environment,
|
||||
such as `gh auth token`, `aws configure get`, `kubectl get secret`,
|
||||
`vault kv get`, `op read` and `printenv`. See
|
||||
[`command_filter`](../configuration/config.md#command_filter) for the list.
|
||||
- **Commands that name a file holding credentials**, such as `cat .env` or
|
||||
`cat ~/.aws/credentials`. See
|
||||
[`sensitive_files`](../configuration/config.md#sensitive_files).
|
||||
|
||||
When a command prints more than
|
||||
[`max_output_size`](../configuration/config.md#max_output_size) (1MB by
|
||||
@@ -90,15 +97,18 @@ recording that it's missing.
|
||||
|
||||
## Privacy
|
||||
|
||||
- Recognised credentials in the output (API keys, tokens, and the like) are
|
||||
replaced with `****` before storage, while
|
||||
- Recognised credentials in the output (API keys, tokens, private keys,
|
||||
passwords in URLs and connection strings, values assigned to names like
|
||||
`DB_PASSWORD`) are replaced with `****` before storage, while
|
||||
[`secrets_filter`](../configuration/config.md#secrets_filter) is on (the
|
||||
default). This is best-effort: it only knows common formats, and misses a
|
||||
credential that colour codes split apart.
|
||||
default). Add patterns of your own with
|
||||
[`redact_patterns`](../configuration/config.md#redact_patterns). This is
|
||||
best-effort: it only knows common formats, and misses a credential that
|
||||
colour codes split apart. Output that can't be redacted quickly isn't stored.
|
||||
- For a command that prints secrets, don't rely on redaction: add it to
|
||||
[`command_filter`](../configuration/config.md#command_filter) to keep it in
|
||||
your history but never store its output. See
|
||||
[Excluding commands](excluding-commands.md#keep-the-command-drop-its-output-command_filter).
|
||||
[Keeping secrets out](excluding-commands.md#a-commands-output-command_filter).
|
||||
- Captured output stays on your machine. **It isn't synced**, even with
|
||||
[sync](sync.md) set up. We're actively working on supporting this.
|
||||
- Atuin AI only sends output to the LLM when it asks for a specific command's
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Atuin uses shell hooks to capture your command history. This page explains how the integration works, and why Atuin might not record commands in certain environments.
|
||||
|
||||
To keep specific commands *out* of your history on purpose, see [Excluding Commands from History](excluding-commands.md).
|
||||
To keep specific commands *out* of your history on purpose, see [Keeping Secrets Out of Atuin](excluding-commands.md).
|
||||
|
||||
## How Atuin's Shell Integration Works
|
||||
|
||||
|
||||
+2
-2
@@ -145,7 +145,7 @@ plugins:
|
||||
- guide/advanced-usage.md: Filter modes (global, host, session, directory, workspace) and search modes (prefix, fulltext, fuzzy, daemon-fuzzy).
|
||||
- guide/shell-integration.md: How shell hooks work, environment variables, and embedded terminals.
|
||||
- guide/agent-hooks.md: Capture commands from AI coding agents (Claude Code, Codex, opencode, pi) and filter by author.
|
||||
- guide/excluding-commands.md: Keep commands out of history with ignorespace, history_filter, and cwd_filter, or keep only their output out with command_filter.
|
||||
- guide/excluding-commands.md: Keep commands, their output, credential files and your own secrets out of history, captured output and AI sessions, by goal - ignorespace, history_filter, cwd_filter, command_filter, sensitive_files, redact_patterns, capture_tools.
|
||||
- guide/output-capture.md: Capture what each command printed - setup, reading and searching it, what's kept, privacy, and disk retention.
|
||||
- guide/delete-history.md: Delete single entries, bulk delete by query, remove duplicates, or wipe everything.
|
||||
- guide/theming.md: Customize the TUI with color themes.
|
||||
@@ -266,7 +266,7 @@ nav:
|
||||
- Advanced usage: guide/advanced-usage.md
|
||||
- Shell Integration: guide/shell-integration.md
|
||||
- AI Agent Hooks: guide/agent-hooks.md
|
||||
- Excluding commands: guide/excluding-commands.md
|
||||
- Keeping secrets out: guide/excluding-commands.md
|
||||
- Capturing command output: guide/output-capture.md
|
||||
- Deleting history: guide/delete-history.md
|
||||
- Theming: guide/theming.md
|
||||
|
||||
Reference in New Issue
Block a user