mirror of
https://github.com/affaan-m/everything-claude-code.git
synced 2026-04-11 20:13:30 +08:00
feat: add ecc2 dotenv memory connectors
This commit is contained in:
@@ -109,6 +109,7 @@ pub enum MemoryConnectorConfig {
|
|||||||
JsonlFile(MemoryConnectorJsonlFileConfig),
|
JsonlFile(MemoryConnectorJsonlFileConfig),
|
||||||
JsonlDirectory(MemoryConnectorJsonlDirectoryConfig),
|
JsonlDirectory(MemoryConnectorJsonlDirectoryConfig),
|
||||||
MarkdownFile(MemoryConnectorMarkdownFileConfig),
|
MarkdownFile(MemoryConnectorMarkdownFileConfig),
|
||||||
|
DotenvFile(MemoryConnectorDotenvFileConfig),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
@@ -139,6 +140,19 @@ pub struct MemoryConnectorMarkdownFileConfig {
|
|||||||
pub default_observation_type: Option<String>,
|
pub default_observation_type: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(default)]
|
||||||
|
pub struct MemoryConnectorDotenvFileConfig {
|
||||||
|
pub path: PathBuf,
|
||||||
|
pub session_id: Option<String>,
|
||||||
|
pub default_entity_type: Option<String>,
|
||||||
|
pub default_observation_type: Option<String>,
|
||||||
|
pub key_prefixes: Vec<String>,
|
||||||
|
pub include_keys: Vec<String>,
|
||||||
|
pub exclude_keys: Vec<String>,
|
||||||
|
pub include_safe_values: bool,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct ResolvedOrchestrationTemplate {
|
pub struct ResolvedOrchestrationTemplate {
|
||||||
pub template_name: String,
|
pub template_name: String,
|
||||||
@@ -1368,6 +1382,49 @@ default_observation_type = "external_note"
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn memory_dotenv_file_connectors_deserialize_from_toml() {
|
||||||
|
let config: Config = toml::from_str(
|
||||||
|
r#"
|
||||||
|
[memory_connectors.hermes_env]
|
||||||
|
kind = "dotenv_file"
|
||||||
|
path = "/tmp/hermes.env"
|
||||||
|
session_id = "latest"
|
||||||
|
default_entity_type = "service_config"
|
||||||
|
default_observation_type = "external_config"
|
||||||
|
key_prefixes = ["STRIPE_", "PUBLIC_"]
|
||||||
|
include_keys = ["PUBLIC_BASE_URL"]
|
||||||
|
exclude_keys = ["STRIPE_WEBHOOK_SECRET"]
|
||||||
|
include_safe_values = true
|
||||||
|
"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let connector = config
|
||||||
|
.memory_connectors
|
||||||
|
.get("hermes_env")
|
||||||
|
.expect("connector should deserialize");
|
||||||
|
match connector {
|
||||||
|
crate::config::MemoryConnectorConfig::DotenvFile(settings) => {
|
||||||
|
assert_eq!(settings.path, PathBuf::from("/tmp/hermes.env"));
|
||||||
|
assert_eq!(settings.session_id.as_deref(), Some("latest"));
|
||||||
|
assert_eq!(
|
||||||
|
settings.default_entity_type.as_deref(),
|
||||||
|
Some("service_config")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
settings.default_observation_type.as_deref(),
|
||||||
|
Some("external_config")
|
||||||
|
);
|
||||||
|
assert_eq!(settings.key_prefixes, vec!["STRIPE_", "PUBLIC_"]);
|
||||||
|
assert_eq!(settings.include_keys, vec!["PUBLIC_BASE_URL"]);
|
||||||
|
assert_eq!(settings.exclude_keys, vec!["STRIPE_WEBHOOK_SECRET"]);
|
||||||
|
assert!(settings.include_safe_values);
|
||||||
|
}
|
||||||
|
_ => panic!("expected dotenv_file connector"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn completion_summary_notifications_deserialize_from_toml() {
|
fn completion_summary_notifications_deserialize_from_toml() {
|
||||||
let config: Config = toml::from_str(
|
let config: Config = toml::from_str(
|
||||||
|
|||||||
297
ecc2/src/main.rs
297
ecc2/src/main.rs
@@ -590,6 +590,7 @@ struct JsonlMemoryConnectorRecord {
|
|||||||
|
|
||||||
const MARKDOWN_CONNECTOR_SUMMARY_LIMIT: usize = 160;
|
const MARKDOWN_CONNECTOR_SUMMARY_LIMIT: usize = 160;
|
||||||
const MARKDOWN_CONNECTOR_BODY_LIMIT: usize = 4000;
|
const MARKDOWN_CONNECTOR_BODY_LIMIT: usize = 4000;
|
||||||
|
const DOTENV_CONNECTOR_VALUE_LIMIT: usize = 160;
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
struct MarkdownMemorySection {
|
struct MarkdownMemorySection {
|
||||||
@@ -600,6 +601,14 @@ struct MarkdownMemorySection {
|
|||||||
line_number: usize,
|
line_number: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct DotenvMemoryEntry {
|
||||||
|
key: String,
|
||||||
|
path: String,
|
||||||
|
summary: String,
|
||||||
|
details: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> Result<()> {
|
async fn main() -> Result<()> {
|
||||||
tracing_subscriber::fmt()
|
tracing_subscriber::fmt()
|
||||||
@@ -1609,6 +1618,9 @@ fn sync_memory_connector(
|
|||||||
config::MemoryConnectorConfig::MarkdownFile(settings) => {
|
config::MemoryConnectorConfig::MarkdownFile(settings) => {
|
||||||
sync_markdown_memory_connector(db, name, settings, limit)
|
sync_markdown_memory_connector(db, name, settings, limit)
|
||||||
}
|
}
|
||||||
|
config::MemoryConnectorConfig::DotenvFile(settings) => {
|
||||||
|
sync_dotenv_memory_connector(db, name, settings, limit)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1805,6 +1817,54 @@ fn sync_markdown_memory_connector(
|
|||||||
Ok(stats)
|
Ok(stats)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn sync_dotenv_memory_connector(
|
||||||
|
db: &session::store::StateStore,
|
||||||
|
name: &str,
|
||||||
|
settings: &config::MemoryConnectorDotenvFileConfig,
|
||||||
|
limit: usize,
|
||||||
|
) -> Result<GraphConnectorSyncStats> {
|
||||||
|
if settings.path.as_os_str().is_empty() {
|
||||||
|
anyhow::bail!("memory connector {name} has no path configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
let body = std::fs::read_to_string(&settings.path)
|
||||||
|
.with_context(|| format!("read memory connector file {}", settings.path.display()))?;
|
||||||
|
let default_session_id = settings
|
||||||
|
.session_id
|
||||||
|
.as_deref()
|
||||||
|
.map(|value| resolve_session_id(db, value))
|
||||||
|
.transpose()?;
|
||||||
|
let entries = parse_dotenv_memory_entries(&settings.path, &body, settings, limit);
|
||||||
|
let mut stats = GraphConnectorSyncStats {
|
||||||
|
connector_name: name.to_string(),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
|
||||||
|
for entry in entries {
|
||||||
|
stats.records_read += 1;
|
||||||
|
import_memory_connector_record(
|
||||||
|
db,
|
||||||
|
&mut stats,
|
||||||
|
default_session_id.as_deref(),
|
||||||
|
settings.default_entity_type.as_deref(),
|
||||||
|
settings.default_observation_type.as_deref(),
|
||||||
|
JsonlMemoryConnectorRecord {
|
||||||
|
session_id: None,
|
||||||
|
entity_type: None,
|
||||||
|
entity_name: entry.key,
|
||||||
|
path: Some(entry.path),
|
||||||
|
entity_summary: Some(entry.summary.clone()),
|
||||||
|
metadata: BTreeMap::from([("connector".to_string(), "dotenv_file".to_string())]),
|
||||||
|
observation_type: None,
|
||||||
|
summary: entry.summary,
|
||||||
|
details: entry.details,
|
||||||
|
},
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(stats)
|
||||||
|
}
|
||||||
|
|
||||||
fn import_memory_connector_record(
|
fn import_memory_connector_record(
|
||||||
db: &session::store::StateStore,
|
db: &session::store::StateStore,
|
||||||
stats: &mut GraphConnectorSyncStats,
|
stats: &mut GraphConnectorSyncStats,
|
||||||
@@ -1907,6 +1967,72 @@ fn collect_jsonl_paths_inner(root: &Path, recurse: bool, paths: &mut Vec<PathBuf
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_dotenv_memory_entries(
|
||||||
|
path: &Path,
|
||||||
|
body: &str,
|
||||||
|
settings: &config::MemoryConnectorDotenvFileConfig,
|
||||||
|
limit: usize,
|
||||||
|
) -> Vec<DotenvMemoryEntry> {
|
||||||
|
if limit == 0 {
|
||||||
|
return Vec::new();
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut entries = Vec::new();
|
||||||
|
let source_path = path.display().to_string();
|
||||||
|
|
||||||
|
for (index, raw_line) in body.lines().enumerate() {
|
||||||
|
if entries.len() >= limit {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
let line = raw_line.trim();
|
||||||
|
if line.is_empty() || line.starts_with('#') {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let Some((key, value)) = parse_dotenv_assignment(line) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if !dotenv_key_included(key, settings) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let value = parse_dotenv_value(value);
|
||||||
|
let secret_like = dotenv_key_is_secret(key);
|
||||||
|
let mut details = BTreeMap::new();
|
||||||
|
details.insert("source_path".to_string(), source_path.clone());
|
||||||
|
details.insert("line".to_string(), (index + 1).to_string());
|
||||||
|
details.insert("key".to_string(), key.to_string());
|
||||||
|
details.insert("secret_redacted".to_string(), secret_like.to_string());
|
||||||
|
if settings.include_safe_values && !secret_like && !value.is_empty() {
|
||||||
|
details.insert(
|
||||||
|
"value".to_string(),
|
||||||
|
truncate_connector_text(&value, DOTENV_CONNECTOR_VALUE_LIMIT),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let summary = if secret_like {
|
||||||
|
format!("{key} configured (secret redacted)")
|
||||||
|
} else if settings.include_safe_values && !value.is_empty() {
|
||||||
|
format!(
|
||||||
|
"{key}={}",
|
||||||
|
truncate_connector_text(&value, DOTENV_CONNECTOR_VALUE_LIMIT)
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
format!("{key} configured")
|
||||||
|
};
|
||||||
|
|
||||||
|
entries.push(DotenvMemoryEntry {
|
||||||
|
key: key.to_string(),
|
||||||
|
path: format!("{source_path}#{key}"),
|
||||||
|
summary,
|
||||||
|
details,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
entries
|
||||||
|
}
|
||||||
|
|
||||||
fn parse_markdown_memory_sections(
|
fn parse_markdown_memory_sections(
|
||||||
path: &Path,
|
path: &Path,
|
||||||
body: &str,
|
body: &str,
|
||||||
@@ -2058,6 +2184,73 @@ fn truncate_connector_text(value: &str, max_chars: usize) -> String {
|
|||||||
format!("{truncated}…")
|
format!("{truncated}…")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn parse_dotenv_assignment(line: &str) -> Option<(&str, &str)> {
|
||||||
|
let trimmed = line.strip_prefix("export ").unwrap_or(line).trim();
|
||||||
|
let (key, value) = trimmed.split_once('=')?;
|
||||||
|
let key = key.trim();
|
||||||
|
if key.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
Some((key, value.trim()))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_dotenv_value(raw: &str) -> String {
|
||||||
|
let trimmed = raw.trim();
|
||||||
|
if let Some(unquoted) = trimmed
|
||||||
|
.strip_prefix('"')
|
||||||
|
.and_then(|value| value.strip_suffix('"'))
|
||||||
|
{
|
||||||
|
return unquoted.to_string();
|
||||||
|
}
|
||||||
|
if let Some(unquoted) = trimmed
|
||||||
|
.strip_prefix('\'')
|
||||||
|
.and_then(|value| value.strip_suffix('\''))
|
||||||
|
{
|
||||||
|
return unquoted.to_string();
|
||||||
|
}
|
||||||
|
trimmed.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dotenv_key_included(key: &str, settings: &config::MemoryConnectorDotenvFileConfig) -> bool {
|
||||||
|
if settings
|
||||||
|
.exclude_keys
|
||||||
|
.iter()
|
||||||
|
.any(|candidate| candidate == key)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if !settings.include_keys.is_empty()
|
||||||
|
&& settings
|
||||||
|
.include_keys
|
||||||
|
.iter()
|
||||||
|
.any(|candidate| candidate == key)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if settings.key_prefixes.is_empty() {
|
||||||
|
return settings.include_keys.is_empty();
|
||||||
|
}
|
||||||
|
settings
|
||||||
|
.key_prefixes
|
||||||
|
.iter()
|
||||||
|
.any(|prefix| !prefix.is_empty() && key.starts_with(prefix))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dotenv_key_is_secret(key: &str) -> bool {
|
||||||
|
let upper = key.to_ascii_uppercase();
|
||||||
|
[
|
||||||
|
"SECRET",
|
||||||
|
"TOKEN",
|
||||||
|
"PASSWORD",
|
||||||
|
"PRIVATE_KEY",
|
||||||
|
"API_KEY",
|
||||||
|
"CLIENT_SECRET",
|
||||||
|
"ACCESS_KEY",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|marker| upper.contains(marker))
|
||||||
|
}
|
||||||
|
|
||||||
fn build_message(
|
fn build_message(
|
||||||
kind: MessageKindArg,
|
kind: MessageKindArg,
|
||||||
text: String,
|
text: String,
|
||||||
@@ -5477,6 +5670,110 @@ Guide users to repair before reinstall so wiped setups do not buy twice.
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sync_memory_connector_imports_dotenv_entries_safely() -> Result<()> {
|
||||||
|
let tempdir = TestDir::new("graph-connector-sync-dotenv")?;
|
||||||
|
let db = session::store::StateStore::open(&tempdir.path().join("state.db"))?;
|
||||||
|
let now = chrono::Utc::now();
|
||||||
|
db.insert_session(&session::Session {
|
||||||
|
id: "session-1".to_string(),
|
||||||
|
task: "service config import".to_string(),
|
||||||
|
project: "ecc-tools".to_string(),
|
||||||
|
task_group: "memory".to_string(),
|
||||||
|
agent_type: "claude".to_string(),
|
||||||
|
working_dir: PathBuf::from("/tmp"),
|
||||||
|
state: session::SessionState::Running,
|
||||||
|
pid: None,
|
||||||
|
worktree: None,
|
||||||
|
created_at: now,
|
||||||
|
updated_at: now,
|
||||||
|
last_heartbeat_at: now,
|
||||||
|
metrics: session::SessionMetrics::default(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let connector_path = tempdir.path().join("hermes.env");
|
||||||
|
fs::write(
|
||||||
|
&connector_path,
|
||||||
|
r#"# Hermes service config
|
||||||
|
STRIPE_SECRET_KEY=sk_test_secret
|
||||||
|
STRIPE_PRO_PRICE_ID=price_pro_monthly
|
||||||
|
PUBLIC_BASE_URL="https://ecc.tools"
|
||||||
|
STRIPE_WEBHOOK_SECRET=whsec_secret
|
||||||
|
GITHUB_TOKEN=ghp_should_not_import
|
||||||
|
INVALID LINE
|
||||||
|
"#,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
let mut cfg = config::Config::default();
|
||||||
|
cfg.memory_connectors.insert(
|
||||||
|
"hermes_env".to_string(),
|
||||||
|
config::MemoryConnectorConfig::DotenvFile(config::MemoryConnectorDotenvFileConfig {
|
||||||
|
path: connector_path.clone(),
|
||||||
|
session_id: Some("latest".to_string()),
|
||||||
|
default_entity_type: Some("service_config".to_string()),
|
||||||
|
default_observation_type: Some("external_config".to_string()),
|
||||||
|
key_prefixes: vec!["STRIPE_".to_string(), "PUBLIC_".to_string()],
|
||||||
|
include_keys: Vec::new(),
|
||||||
|
exclude_keys: vec!["STRIPE_WEBHOOK_SECRET".to_string()],
|
||||||
|
include_safe_values: true,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
|
let stats = sync_memory_connector(&db, &cfg, "hermes_env", 10)?;
|
||||||
|
assert_eq!(stats.records_read, 3);
|
||||||
|
assert_eq!(stats.entities_upserted, 3);
|
||||||
|
assert_eq!(stats.observations_added, 3);
|
||||||
|
assert_eq!(stats.skipped_records, 0);
|
||||||
|
|
||||||
|
let recalled = db.recall_context_entities(None, "stripe ecc.tools", 10)?;
|
||||||
|
assert!(recalled
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.entity.name == "STRIPE_SECRET_KEY"));
|
||||||
|
assert!(recalled
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.entity.name == "STRIPE_PRO_PRICE_ID"));
|
||||||
|
assert!(recalled
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.entity.name == "PUBLIC_BASE_URL"));
|
||||||
|
assert!(!recalled
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.entity.name == "STRIPE_WEBHOOK_SECRET"));
|
||||||
|
assert!(!recalled
|
||||||
|
.iter()
|
||||||
|
.any(|entry| entry.entity.name == "GITHUB_TOKEN"));
|
||||||
|
|
||||||
|
let secret = recalled
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.entity.name == "STRIPE_SECRET_KEY")
|
||||||
|
.expect("secret entry should exist");
|
||||||
|
let secret_observations = db.list_context_observations(Some(secret.entity.id), 5)?;
|
||||||
|
assert_eq!(secret_observations.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
secret_observations[0]
|
||||||
|
.details
|
||||||
|
.get("secret_redacted")
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("true")
|
||||||
|
);
|
||||||
|
assert!(!secret_observations[0].details.contains_key("value"));
|
||||||
|
|
||||||
|
let public_base = recalled
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.entity.name == "PUBLIC_BASE_URL")
|
||||||
|
.expect("public base url should exist");
|
||||||
|
let public_observations = db.list_context_observations(Some(public_base.entity.id), 5)?;
|
||||||
|
assert_eq!(public_observations.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
public_observations[0]
|
||||||
|
.details
|
||||||
|
.get("value")
|
||||||
|
.map(String::as_str),
|
||||||
|
Some("https://ecc.tools")
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn format_graph_sync_stats_human_renders_counts() {
|
fn format_graph_sync_stats_human_renders_counts() {
|
||||||
let text = format_graph_sync_stats_human(
|
let text = format_graph_sync_stats_human(
|
||||||
|
|||||||
Reference in New Issue
Block a user