zsp/everything-claude-code
1
0
Fork 0
mirror of https://github.com/affaan-m/everything-claude-code.git synced 2026-04-11 12:03:31 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: add ecc2 dotenv memory connectors

Browse Source
This commit is contained in:
Affaan Mustafa
2026-04-10 06:30:32 -07:00
parent 22a5a8de6d
commit 966af37f89
2 changed files with 354 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
ecc2/src/config/mod.rs
View File

@@ -109,6 +109,7 @@ pub enum MemoryConnectorConfig {
JsonlFile(MemoryConnectorJsonlFileConfig),
JsonlDirectory(MemoryConnectorJsonlDirectoryConfig),
MarkdownFile(MemoryConnectorMarkdownFileConfig),
DotenvFile(MemoryConnectorDotenvFileConfig),
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
@@ -139,6 +140,19 @@ pub struct MemoryConnectorMarkdownFileConfig {
pub default_observation_type: Option<String>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
#[serde(default)]
pub struct MemoryConnectorDotenvFileConfig {
pub path: PathBuf,
pub session_id: Option<String>,
pub default_entity_type: Option<String>,
pub default_observation_type: Option<String>,
pub key_prefixes: Vec<String>,
pub include_keys: Vec<String>,
pub exclude_keys: Vec<String>,
pub include_safe_values: bool,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResolvedOrchestrationTemplate {
pub template_name: String,
@@ -1368,6 +1382,49 @@ default_observation_type = "external_note"
}
}
#[test]
fn memory_dotenv_file_connectors_deserialize_from_toml() {
let config: Config = toml::from_str(
r#"
[memory_connectors.hermes_env]
kind = "dotenv_file"
path = "/tmp/hermes.env"
session_id = "latest"
default_entity_type = "service_config"
default_observation_type = "external_config"
key_prefixes = ["STRIPE_", "PUBLIC_"]
include_keys = ["PUBLIC_BASE_URL"]
exclude_keys = ["STRIPE_WEBHOOK_SECRET"]
include_safe_values = true
"#,
)
.unwrap();
let connector = config
.memory_connectors
.get("hermes_env")
.expect("connector should deserialize");
match connector {
crate::config::MemoryConnectorConfig::DotenvFile(settings) => {
assert_eq!(settings.path, PathBuf::from("/tmp/hermes.env"));
assert_eq!(settings.session_id.as_deref(), Some("latest"));
assert_eq!(
settings.default_entity_type.as_deref(),
Some("service_config")
);
assert_eq!(
settings.default_observation_type.as_deref(),
Some("external_config")
);
assert_eq!(settings.key_prefixes, vec!["STRIPE_", "PUBLIC_"]);
assert_eq!(settings.include_keys, vec!["PUBLIC_BASE_URL"]);
assert_eq!(settings.exclude_keys, vec!["STRIPE_WEBHOOK_SECRET"]);
assert!(settings.include_safe_values);
}
_ => panic!("expected dotenv_file connector"),
}
}
#[test]
fn completion_summary_notifications_deserialize_from_toml() {
let config: Config = toml::from_str(

297
ecc2/src/main.rs
View File

@@ -590,6 +590,7 @@ struct JsonlMemoryConnectorRecord {
const MARKDOWN_CONNECTOR_SUMMARY_LIMIT: usize = 160;
const MARKDOWN_CONNECTOR_BODY_LIMIT: usize = 4000;
const DOTENV_CONNECTOR_VALUE_LIMIT: usize = 160;
#[derive(Debug, Clone)]
struct MarkdownMemorySection {
@@ -600,6 +601,14 @@ struct MarkdownMemorySection {
line_number: usize,
}
#[derive(Debug, Clone)]
struct DotenvMemoryEntry {
key: String,
path: String,
summary: String,
details: BTreeMap<String, String>,
}
#[tokio::main]
async fn main() -> Result<()> {
tracing_subscriber::fmt()
@@ -1609,6 +1618,9 @@ fn sync_memory_connector(
config::MemoryConnectorConfig::MarkdownFile(settings) => {
sync_markdown_memory_connector(db, name, settings, limit)
}
config::MemoryConnectorConfig::DotenvFile(settings) => {
sync_dotenv_memory_connector(db, name, settings, limit)
}
}
}
@@ -1805,6 +1817,54 @@ fn sync_markdown_memory_connector(
Ok(stats)
}
fn sync_dotenv_memory_connector(
db: &session::store::StateStore,
name: &str,
settings: &config::MemoryConnectorDotenvFileConfig,
limit: usize,
) -> Result<GraphConnectorSyncStats> {
if settings.path.as_os_str().is_empty() {
anyhow::bail!("memory connector {name} has no path configured");
}
let body = std::fs::read_to_string(&settings.path)
.with_context(|| format!("read memory connector file {}", settings.path.display()))?;
let default_session_id = settings
.session_id
.as_deref()
.map(|value| resolve_session_id(db, value))
.transpose()?;
let entries = parse_dotenv_memory_entries(&settings.path, &body, settings, limit);
let mut stats = GraphConnectorSyncStats {
connector_name: name.to_string(),
..Default::default()
};
for entry in entries {
stats.records_read += 1;
import_memory_connector_record(
db,
&mut stats,
default_session_id.as_deref(),
settings.default_entity_type.as_deref(),
settings.default_observation_type.as_deref(),
JsonlMemoryConnectorRecord {
session_id: None,
entity_type: None,
entity_name: entry.key,
path: Some(entry.path),
entity_summary: Some(entry.summary.clone()),
metadata: BTreeMap::from([("connector".to_string(), "dotenv_file".to_string())]),
observation_type: None,
summary: entry.summary,
details: entry.details,
},
)?;
}
Ok(stats)
}
fn import_memory_connector_record(
db: &session::store::StateStore,
stats: &mut GraphConnectorSyncStats,
@@ -1907,6 +1967,72 @@ fn collect_jsonl_paths_inner(root: &Path, recurse: bool, paths: &mut Vec<PathBuf
Ok(())
}
fn parse_dotenv_memory_entries(
path: &Path,
body: &str,
settings: &config::MemoryConnectorDotenvFileConfig,
limit: usize,
) -> Vec<DotenvMemoryEntry> {
if limit == 0 {
return Vec::new();
}
let mut entries = Vec::new();
let source_path = path.display().to_string();
for (index, raw_line) in body.lines().enumerate() {
if entries.len() >= limit {
break;
}
let line = raw_line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
let Some((key, value)) = parse_dotenv_assignment(line) else {
continue;
};
if !dotenv_key_included(key, settings) {
continue;
}
let value = parse_dotenv_value(value);
let secret_like = dotenv_key_is_secret(key);
let mut details = BTreeMap::new();
details.insert("source_path".to_string(), source_path.clone());
details.insert("line".to_string(), (index + 1).to_string());
details.insert("key".to_string(), key.to_string());
details.insert("secret_redacted".to_string(), secret_like.to_string());
if settings.include_safe_values && !secret_like && !value.is_empty() {
details.insert(
"value".to_string(),
truncate_connector_text(&value, DOTENV_CONNECTOR_VALUE_LIMIT),
);
}
let summary = if secret_like {
format!("{key} configured (secret redacted)")
} else if settings.include_safe_values && !value.is_empty() {
format!(
"{key}={}",
truncate_connector_text(&value, DOTENV_CONNECTOR_VALUE_LIMIT)
)
} else {
format!("{key} configured")
};
entries.push(DotenvMemoryEntry {
key: key.to_string(),
path: format!("{source_path}#{key}"),
summary,
details,
});
}
entries
}
fn parse_markdown_memory_sections(
path: &Path,
body: &str,
@@ -2058,6 +2184,73 @@ fn truncate_connector_text(value: &str, max_chars: usize) -> String {
format!("{truncated}…")
}
fn parse_dotenv_assignment(line: &str) -> Option<(&str, &str)> {
let trimmed = line.strip_prefix("export ").unwrap_or(line).trim();
let (key, value) = trimmed.split_once('=')?;
let key = key.trim();
if key.is_empty() {
return None;
}
Some((key, value.trim()))
}
fn parse_dotenv_value(raw: &str) -> String {
let trimmed = raw.trim();
if let Some(unquoted) = trimmed
.strip_prefix('"')
.and_then(|value| value.strip_suffix('"'))
{
return unquoted.to_string();
}
if let Some(unquoted) = trimmed
.strip_prefix('\'')
.and_then(|value| value.strip_suffix('\''))
{
return unquoted.to_string();
}
trimmed.to_string()
}
fn dotenv_key_included(key: &str, settings: &config::MemoryConnectorDotenvFileConfig) -> bool {
if settings
.exclude_keys
.iter()
.any(|candidate| candidate == key)
{
return false;
}
if !settings.include_keys.is_empty()
&& settings
.include_keys
.iter()
.any(|candidate| candidate == key)
{
return true;
}
if settings.key_prefixes.is_empty() {
return settings.include_keys.is_empty();
}
settings
.key_prefixes
.iter()
.any(|prefix| !prefix.is_empty() && key.starts_with(prefix))
}
fn dotenv_key_is_secret(key: &str) -> bool {
let upper = key.to_ascii_uppercase();
[
"SECRET",
"TOKEN",
"PASSWORD",
"PRIVATE_KEY",
"API_KEY",
"CLIENT_SECRET",
"ACCESS_KEY",
]
.iter()
.any(|marker| upper.contains(marker))
}
fn build_message(
kind: MessageKindArg,
text: String,
@@ -5477,6 +5670,110 @@ Guide users to repair before reinstall so wiped setups do not buy twice.
Ok(())
}
#[test]
fn sync_memory_connector_imports_dotenv_entries_safely() -> Result<()> {
let tempdir = TestDir::new("graph-connector-sync-dotenv")?;
let db = session::store::StateStore::open(&tempdir.path().join("state.db"))?;
let now = chrono::Utc::now();
db.insert_session(&session::Session {
id: "session-1".to_string(),
task: "service config import".to_string(),
project: "ecc-tools".to_string(),
task_group: "memory".to_string(),
agent_type: "claude".to_string(),
working_dir: PathBuf::from("/tmp"),
state: session::SessionState::Running,
pid: None,
worktree: None,
created_at: now,
updated_at: now,
last_heartbeat_at: now,
metrics: session::SessionMetrics::default(),
})?;
let connector_path = tempdir.path().join("hermes.env");
fs::write(
&connector_path,
r#"# Hermes service config
STRIPE_SECRET_KEY=sk_test_secret
STRIPE_PRO_PRICE_ID=price_pro_monthly
PUBLIC_BASE_URL="https://ecc.tools"
STRIPE_WEBHOOK_SECRET=whsec_secret
GITHUB_TOKEN=ghp_should_not_import
INVALID LINE
"#,
)?;
let mut cfg = config::Config::default();
cfg.memory_connectors.insert(
"hermes_env".to_string(),
config::MemoryConnectorConfig::DotenvFile(config::MemoryConnectorDotenvFileConfig {
path: connector_path.clone(),
session_id: Some("latest".to_string()),
default_entity_type: Some("service_config".to_string()),
default_observation_type: Some("external_config".to_string()),
key_prefixes: vec!["STRIPE_".to_string(), "PUBLIC_".to_string()],
include_keys: Vec::new(),
exclude_keys: vec!["STRIPE_WEBHOOK_SECRET".to_string()],
include_safe_values: true,
}),
);
let stats = sync_memory_connector(&db, &cfg, "hermes_env", 10)?;
assert_eq!(stats.records_read, 3);
assert_eq!(stats.entities_upserted, 3);
assert_eq!(stats.observations_added, 3);
assert_eq!(stats.skipped_records, 0);
let recalled = db.recall_context_entities(None, "stripe ecc.tools", 10)?;
assert!(recalled
.iter()
.any(|entry| entry.entity.name == "STRIPE_SECRET_KEY"));
assert!(recalled
.iter()
.any(|entry| entry.entity.name == "STRIPE_PRO_PRICE_ID"));
assert!(recalled
.iter()
.any(|entry| entry.entity.name == "PUBLIC_BASE_URL"));
assert!(!recalled
.iter()
.any(|entry| entry.entity.name == "STRIPE_WEBHOOK_SECRET"));
assert!(!recalled
.iter()
.any(|entry| entry.entity.name == "GITHUB_TOKEN"));
let secret = recalled
.iter()
.find(|entry| entry.entity.name == "STRIPE_SECRET_KEY")
.expect("secret entry should exist");
let secret_observations = db.list_context_observations(Some(secret.entity.id), 5)?;
assert_eq!(secret_observations.len(), 1);
assert_eq!(
secret_observations[0]
.details
.get("secret_redacted")
.map(String::as_str),
Some("true")
);
assert!(!secret_observations[0].details.contains_key("value"));
let public_base = recalled
.iter()
.find(|entry| entry.entity.name == "PUBLIC_BASE_URL")
.expect("public base url should exist");
let public_observations = db.list_context_observations(Some(public_base.entity.id), 5)?;
assert_eq!(public_observations.len(), 1);
assert_eq!(
public_observations[0]
.details
.get("value")
.map(String::as_str),
Some("https://ecc.tools")
);
Ok(())
}
#[test]
fn format_graph_sync_stats_human_renders_counts() {
let text = format_graph_sync_stats_human(