From 966af37f89341c8abe05c42fa15a916393219082 Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Fri, 10 Apr 2026 06:30:32 -0700 Subject: [PATCH] feat: add ecc2 dotenv memory connectors --- ecc2/src/config/mod.rs | 57 ++++++++ ecc2/src/main.rs | 297 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 354 insertions(+) diff --git a/ecc2/src/config/mod.rs b/ecc2/src/config/mod.rs index e058ba78..4ef420be 100644 --- a/ecc2/src/config/mod.rs +++ b/ecc2/src/config/mod.rs @@ -109,6 +109,7 @@ pub enum MemoryConnectorConfig { JsonlFile(MemoryConnectorJsonlFileConfig), JsonlDirectory(MemoryConnectorJsonlDirectoryConfig), MarkdownFile(MemoryConnectorMarkdownFileConfig), + DotenvFile(MemoryConnectorDotenvFileConfig), } #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -139,6 +140,19 @@ pub struct MemoryConnectorMarkdownFileConfig { pub default_observation_type: Option, } +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct MemoryConnectorDotenvFileConfig { + pub path: PathBuf, + pub session_id: Option, + pub default_entity_type: Option, + pub default_observation_type: Option, + pub key_prefixes: Vec, + pub include_keys: Vec, + pub exclude_keys: Vec, + pub include_safe_values: bool, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ResolvedOrchestrationTemplate { pub template_name: String, @@ -1368,6 +1382,49 @@ default_observation_type = "external_note" } } + #[test] + fn memory_dotenv_file_connectors_deserialize_from_toml() { + let config: Config = toml::from_str( + r#" +[memory_connectors.hermes_env] +kind = "dotenv_file" +path = "/tmp/hermes.env" +session_id = "latest" +default_entity_type = "service_config" +default_observation_type = "external_config" +key_prefixes = ["STRIPE_", "PUBLIC_"] +include_keys = ["PUBLIC_BASE_URL"] +exclude_keys = ["STRIPE_WEBHOOK_SECRET"] +include_safe_values = true +"#, + ) + .unwrap(); + + let connector = config + .memory_connectors + .get("hermes_env") + .expect("connector should deserialize"); + match connector { + crate::config::MemoryConnectorConfig::DotenvFile(settings) => { + assert_eq!(settings.path, PathBuf::from("/tmp/hermes.env")); + assert_eq!(settings.session_id.as_deref(), Some("latest")); + assert_eq!( + settings.default_entity_type.as_deref(), + Some("service_config") + ); + assert_eq!( + settings.default_observation_type.as_deref(), + Some("external_config") + ); + assert_eq!(settings.key_prefixes, vec!["STRIPE_", "PUBLIC_"]); + assert_eq!(settings.include_keys, vec!["PUBLIC_BASE_URL"]); + assert_eq!(settings.exclude_keys, vec!["STRIPE_WEBHOOK_SECRET"]); + assert!(settings.include_safe_values); + } + _ => panic!("expected dotenv_file connector"), + } + } + #[test] fn completion_summary_notifications_deserialize_from_toml() { let config: Config = toml::from_str( diff --git a/ecc2/src/main.rs b/ecc2/src/main.rs index acf3015c..7dcceb0d 100644 --- a/ecc2/src/main.rs +++ b/ecc2/src/main.rs @@ -590,6 +590,7 @@ struct JsonlMemoryConnectorRecord { const MARKDOWN_CONNECTOR_SUMMARY_LIMIT: usize = 160; const MARKDOWN_CONNECTOR_BODY_LIMIT: usize = 4000; +const DOTENV_CONNECTOR_VALUE_LIMIT: usize = 160; #[derive(Debug, Clone)] struct MarkdownMemorySection { @@ -600,6 +601,14 @@ struct MarkdownMemorySection { line_number: usize, } +#[derive(Debug, Clone)] +struct DotenvMemoryEntry { + key: String, + path: String, + summary: String, + details: BTreeMap, +} + #[tokio::main] async fn main() -> Result<()> { tracing_subscriber::fmt() @@ -1609,6 +1618,9 @@ fn sync_memory_connector( config::MemoryConnectorConfig::MarkdownFile(settings) => { sync_markdown_memory_connector(db, name, settings, limit) } + config::MemoryConnectorConfig::DotenvFile(settings) => { + sync_dotenv_memory_connector(db, name, settings, limit) + } } } @@ -1805,6 +1817,54 @@ fn sync_markdown_memory_connector( Ok(stats) } +fn sync_dotenv_memory_connector( + db: &session::store::StateStore, + name: &str, + settings: &config::MemoryConnectorDotenvFileConfig, + limit: usize, +) -> Result { + if settings.path.as_os_str().is_empty() { + anyhow::bail!("memory connector {name} has no path configured"); + } + + let body = std::fs::read_to_string(&settings.path) + .with_context(|| format!("read memory connector file {}", settings.path.display()))?; + let default_session_id = settings + .session_id + .as_deref() + .map(|value| resolve_session_id(db, value)) + .transpose()?; + let entries = parse_dotenv_memory_entries(&settings.path, &body, settings, limit); + let mut stats = GraphConnectorSyncStats { + connector_name: name.to_string(), + ..Default::default() + }; + + for entry in entries { + stats.records_read += 1; + import_memory_connector_record( + db, + &mut stats, + default_session_id.as_deref(), + settings.default_entity_type.as_deref(), + settings.default_observation_type.as_deref(), + JsonlMemoryConnectorRecord { + session_id: None, + entity_type: None, + entity_name: entry.key, + path: Some(entry.path), + entity_summary: Some(entry.summary.clone()), + metadata: BTreeMap::from([("connector".to_string(), "dotenv_file".to_string())]), + observation_type: None, + summary: entry.summary, + details: entry.details, + }, + )?; + } + + Ok(stats) +} + fn import_memory_connector_record( db: &session::store::StateStore, stats: &mut GraphConnectorSyncStats, @@ -1907,6 +1967,72 @@ fn collect_jsonl_paths_inner(root: &Path, recurse: bool, paths: &mut Vec Vec { + if limit == 0 { + return Vec::new(); + } + + let mut entries = Vec::new(); + let source_path = path.display().to_string(); + + for (index, raw_line) in body.lines().enumerate() { + if entries.len() >= limit { + break; + } + + let line = raw_line.trim(); + if line.is_empty() || line.starts_with('#') { + continue; + } + + let Some((key, value)) = parse_dotenv_assignment(line) else { + continue; + }; + if !dotenv_key_included(key, settings) { + continue; + } + + let value = parse_dotenv_value(value); + let secret_like = dotenv_key_is_secret(key); + let mut details = BTreeMap::new(); + details.insert("source_path".to_string(), source_path.clone()); + details.insert("line".to_string(), (index + 1).to_string()); + details.insert("key".to_string(), key.to_string()); + details.insert("secret_redacted".to_string(), secret_like.to_string()); + if settings.include_safe_values && !secret_like && !value.is_empty() { + details.insert( + "value".to_string(), + truncate_connector_text(&value, DOTENV_CONNECTOR_VALUE_LIMIT), + ); + } + + let summary = if secret_like { + format!("{key} configured (secret redacted)") + } else if settings.include_safe_values && !value.is_empty() { + format!( + "{key}={}", + truncate_connector_text(&value, DOTENV_CONNECTOR_VALUE_LIMIT) + ) + } else { + format!("{key} configured") + }; + + entries.push(DotenvMemoryEntry { + key: key.to_string(), + path: format!("{source_path}#{key}"), + summary, + details, + }); + } + + entries +} + fn parse_markdown_memory_sections( path: &Path, body: &str, @@ -2058,6 +2184,73 @@ fn truncate_connector_text(value: &str, max_chars: usize) -> String { format!("{truncated}…") } +fn parse_dotenv_assignment(line: &str) -> Option<(&str, &str)> { + let trimmed = line.strip_prefix("export ").unwrap_or(line).trim(); + let (key, value) = trimmed.split_once('=')?; + let key = key.trim(); + if key.is_empty() { + return None; + } + Some((key, value.trim())) +} + +fn parse_dotenv_value(raw: &str) -> String { + let trimmed = raw.trim(); + if let Some(unquoted) = trimmed + .strip_prefix('"') + .and_then(|value| value.strip_suffix('"')) + { + return unquoted.to_string(); + } + if let Some(unquoted) = trimmed + .strip_prefix('\'') + .and_then(|value| value.strip_suffix('\'')) + { + return unquoted.to_string(); + } + trimmed.to_string() +} + +fn dotenv_key_included(key: &str, settings: &config::MemoryConnectorDotenvFileConfig) -> bool { + if settings + .exclude_keys + .iter() + .any(|candidate| candidate == key) + { + return false; + } + if !settings.include_keys.is_empty() + && settings + .include_keys + .iter() + .any(|candidate| candidate == key) + { + return true; + } + if settings.key_prefixes.is_empty() { + return settings.include_keys.is_empty(); + } + settings + .key_prefixes + .iter() + .any(|prefix| !prefix.is_empty() && key.starts_with(prefix)) +} + +fn dotenv_key_is_secret(key: &str) -> bool { + let upper = key.to_ascii_uppercase(); + [ + "SECRET", + "TOKEN", + "PASSWORD", + "PRIVATE_KEY", + "API_KEY", + "CLIENT_SECRET", + "ACCESS_KEY", + ] + .iter() + .any(|marker| upper.contains(marker)) +} + fn build_message( kind: MessageKindArg, text: String, @@ -5477,6 +5670,110 @@ Guide users to repair before reinstall so wiped setups do not buy twice. Ok(()) } + #[test] + fn sync_memory_connector_imports_dotenv_entries_safely() -> Result<()> { + let tempdir = TestDir::new("graph-connector-sync-dotenv")?; + let db = session::store::StateStore::open(&tempdir.path().join("state.db"))?; + let now = chrono::Utc::now(); + db.insert_session(&session::Session { + id: "session-1".to_string(), + task: "service config import".to_string(), + project: "ecc-tools".to_string(), + task_group: "memory".to_string(), + agent_type: "claude".to_string(), + working_dir: PathBuf::from("/tmp"), + state: session::SessionState::Running, + pid: None, + worktree: None, + created_at: now, + updated_at: now, + last_heartbeat_at: now, + metrics: session::SessionMetrics::default(), + })?; + + let connector_path = tempdir.path().join("hermes.env"); + fs::write( + &connector_path, + r#"# Hermes service config +STRIPE_SECRET_KEY=sk_test_secret +STRIPE_PRO_PRICE_ID=price_pro_monthly +PUBLIC_BASE_URL="https://ecc.tools" +STRIPE_WEBHOOK_SECRET=whsec_secret +GITHUB_TOKEN=ghp_should_not_import +INVALID LINE +"#, + )?; + + let mut cfg = config::Config::default(); + cfg.memory_connectors.insert( + "hermes_env".to_string(), + config::MemoryConnectorConfig::DotenvFile(config::MemoryConnectorDotenvFileConfig { + path: connector_path.clone(), + session_id: Some("latest".to_string()), + default_entity_type: Some("service_config".to_string()), + default_observation_type: Some("external_config".to_string()), + key_prefixes: vec!["STRIPE_".to_string(), "PUBLIC_".to_string()], + include_keys: Vec::new(), + exclude_keys: vec!["STRIPE_WEBHOOK_SECRET".to_string()], + include_safe_values: true, + }), + ); + + let stats = sync_memory_connector(&db, &cfg, "hermes_env", 10)?; + assert_eq!(stats.records_read, 3); + assert_eq!(stats.entities_upserted, 3); + assert_eq!(stats.observations_added, 3); + assert_eq!(stats.skipped_records, 0); + + let recalled = db.recall_context_entities(None, "stripe ecc.tools", 10)?; + assert!(recalled + .iter() + .any(|entry| entry.entity.name == "STRIPE_SECRET_KEY")); + assert!(recalled + .iter() + .any(|entry| entry.entity.name == "STRIPE_PRO_PRICE_ID")); + assert!(recalled + .iter() + .any(|entry| entry.entity.name == "PUBLIC_BASE_URL")); + assert!(!recalled + .iter() + .any(|entry| entry.entity.name == "STRIPE_WEBHOOK_SECRET")); + assert!(!recalled + .iter() + .any(|entry| entry.entity.name == "GITHUB_TOKEN")); + + let secret = recalled + .iter() + .find(|entry| entry.entity.name == "STRIPE_SECRET_KEY") + .expect("secret entry should exist"); + let secret_observations = db.list_context_observations(Some(secret.entity.id), 5)?; + assert_eq!(secret_observations.len(), 1); + assert_eq!( + secret_observations[0] + .details + .get("secret_redacted") + .map(String::as_str), + Some("true") + ); + assert!(!secret_observations[0].details.contains_key("value")); + + let public_base = recalled + .iter() + .find(|entry| entry.entity.name == "PUBLIC_BASE_URL") + .expect("public base url should exist"); + let public_observations = db.list_context_observations(Some(public_base.entity.id), 5)?; + assert_eq!(public_observations.len(), 1); + assert_eq!( + public_observations[0] + .details + .get("value") + .map(String::as_str), + Some("https://ecc.tools") + ); + + Ok(()) + } + #[test] fn format_graph_sync_stats_human_renders_counts() { let text = format_graph_sync_stats_human(