mirror of
https://github.com/affaan-m/everything-claude-code.git
synced 2026-03-30 13:43:26 +08:00
Implements three roadmap features: - Agent description compression (#491): New `agent-compress` module with catalog/summary/full compression modes and lazy-loading. Reduces ~26k token agent descriptions to ~2-3k catalog entries for context efficiency. - Inspection logic (#485): New `inspection` module that detects recurring failure patterns in skill_runs. Groups by skill + normalized failure reason, generates structured reports with suggested remediation actions. Configurable threshold (default: 3 failures). - Governance event capture hook (#482): PreToolUse/PostToolUse hook that detects secrets, policy violations, approval-required commands, and elevated privilege usage. Gated behind ECC_GOVERNANCE_CAPTURE=1 flag. Writes to governance_events table via JSON-line stderr output. 59 new tests (16 + 16 + 27), all passing.
This commit is contained in:
@@ -74,6 +74,17 @@
|
||||
}
|
||||
],
|
||||
"description": "Optional InsAIts AI security monitor for Bash/Edit/Write flows. Enable with ECC_ENABLE_INSAITS=1. Requires: pip install insa-its"
|
||||
},
|
||||
{
|
||||
"matcher": "Bash|Write|Edit|MultiEdit",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags.js\" \"pre:governance-capture\" \"scripts/hooks/governance-capture.js\" \"standard,strict\"",
|
||||
"timeout": 10
|
||||
}
|
||||
],
|
||||
"description": "Capture governance events (secrets, policy violations, approval requests). Enable with ECC_GOVERNANCE_CAPTURE=1"
|
||||
}
|
||||
],
|
||||
"PreCompact": [
|
||||
@@ -165,6 +176,17 @@
|
||||
],
|
||||
"description": "Warn about console.log statements after edits"
|
||||
},
|
||||
{
|
||||
"matcher": "Bash|Write|Edit|MultiEdit",
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags.js\" \"post:governance-capture\" \"scripts/hooks/governance-capture.js\" \"standard,strict\"",
|
||||
"timeout": 10
|
||||
}
|
||||
],
|
||||
"description": "Capture governance events from tool outputs. Enable with ECC_GOVERNANCE_CAPTURE=1"
|
||||
},
|
||||
{
|
||||
"matcher": "*",
|
||||
"hooks": [
|
||||
|
||||
280
scripts/hooks/governance-capture.js
Normal file
280
scripts/hooks/governance-capture.js
Normal file
@@ -0,0 +1,280 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Governance Event Capture Hook
|
||||
*
|
||||
* PreToolUse/PostToolUse hook that detects governance-relevant events
|
||||
* and writes them to the governance_events table in the state store.
|
||||
*
|
||||
* Captured event types:
|
||||
* - secret_detected: Hardcoded secrets in tool input/output
|
||||
* - policy_violation: Actions that violate configured policies
|
||||
* - security_finding: Security-relevant tool invocations
|
||||
* - approval_requested: Operations requiring explicit approval
|
||||
*
|
||||
* Enable: Set ECC_GOVERNANCE_CAPTURE=1
|
||||
* Configure session: Set ECC_SESSION_ID for session correlation
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
|
||||
const MAX_STDIN = 1024 * 1024;
|
||||
|
||||
// Patterns that indicate potential hardcoded secrets
|
||||
const SECRET_PATTERNS = [
|
||||
{ name: 'aws_key', pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}/i },
|
||||
{ name: 'generic_secret', pattern: /(?:secret|password|token|api[_-]?key)\s*[:=]\s*["'][^"']{8,}/i },
|
||||
{ name: 'private_key', pattern: /-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----/ },
|
||||
{ name: 'jwt', pattern: /eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/ },
|
||||
{ name: 'github_token', pattern: /gh[pousr]_[A-Za-z0-9_]{36,}/ },
|
||||
];
|
||||
|
||||
// Tool names that represent security-relevant operations
|
||||
const SECURITY_RELEVANT_TOOLS = new Set([
|
||||
'Bash', // Could execute arbitrary commands
|
||||
]);
|
||||
|
||||
// Commands that require governance approval
|
||||
const APPROVAL_COMMANDS = [
|
||||
/git\s+push\s+.*--force/,
|
||||
/git\s+reset\s+--hard/,
|
||||
/rm\s+-rf?\s/,
|
||||
/DROP\s+(?:TABLE|DATABASE)/i,
|
||||
/DELETE\s+FROM\s+\w+\s*(?:;|$)/i,
|
||||
];
|
||||
|
||||
// File patterns that indicate policy-sensitive paths
|
||||
const SENSITIVE_PATHS = [
|
||||
/\.env(?:\.|$)/,
|
||||
/credentials/i,
|
||||
/secrets?\./i,
|
||||
/\.pem$/,
|
||||
/\.key$/,
|
||||
/id_rsa/,
|
||||
];
|
||||
|
||||
/**
|
||||
* Generate a unique event ID.
|
||||
*/
|
||||
function generateEventId() {
|
||||
return `gov-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scan text content for hardcoded secrets.
|
||||
* Returns array of { name, match } for each detected secret.
|
||||
*/
|
||||
function detectSecrets(text) {
|
||||
if (!text || typeof text !== 'string') return [];
|
||||
|
||||
const findings = [];
|
||||
for (const { name, pattern } of SECRET_PATTERNS) {
|
||||
if (pattern.test(text)) {
|
||||
findings.push({ name });
|
||||
}
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a command requires governance approval.
|
||||
*/
|
||||
function detectApprovalRequired(command) {
|
||||
if (!command || typeof command !== 'string') return [];
|
||||
|
||||
const findings = [];
|
||||
for (const pattern of APPROVAL_COMMANDS) {
|
||||
if (pattern.test(command)) {
|
||||
findings.push({ pattern: pattern.source });
|
||||
}
|
||||
}
|
||||
return findings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a file path is policy-sensitive.
|
||||
*/
|
||||
function detectSensitivePath(filePath) {
|
||||
if (!filePath || typeof filePath !== 'string') return false;
|
||||
|
||||
return SENSITIVE_PATHS.some(pattern => pattern.test(filePath));
|
||||
}
|
||||
|
||||
/**
|
||||
* Analyze a hook input payload and return governance events to capture.
|
||||
*
|
||||
* @param {Object} input - Parsed hook input (tool_name, tool_input, tool_output)
|
||||
* @param {Object} [context] - Additional context (sessionId, hookPhase)
|
||||
* @returns {Array<Object>} Array of governance event objects
|
||||
*/
|
||||
function analyzeForGovernanceEvents(input, context = {}) {
|
||||
const events = [];
|
||||
const toolName = input.tool_name || '';
|
||||
const toolInput = input.tool_input || {};
|
||||
const toolOutput = typeof input.tool_output === 'string' ? input.tool_output : '';
|
||||
const sessionId = context.sessionId || null;
|
||||
const hookPhase = context.hookPhase || 'unknown';
|
||||
|
||||
// 1. Secret detection in tool input content
|
||||
const inputText = typeof toolInput === 'object'
|
||||
? JSON.stringify(toolInput)
|
||||
: String(toolInput);
|
||||
|
||||
const inputSecrets = detectSecrets(inputText);
|
||||
const outputSecrets = detectSecrets(toolOutput);
|
||||
const allSecrets = [...inputSecrets, ...outputSecrets];
|
||||
|
||||
if (allSecrets.length > 0) {
|
||||
events.push({
|
||||
id: generateEventId(),
|
||||
sessionId,
|
||||
eventType: 'secret_detected',
|
||||
payload: {
|
||||
toolName,
|
||||
hookPhase,
|
||||
secretTypes: allSecrets.map(s => s.name),
|
||||
location: inputSecrets.length > 0 ? 'input' : 'output',
|
||||
severity: 'critical',
|
||||
},
|
||||
resolvedAt: null,
|
||||
resolution: null,
|
||||
});
|
||||
}
|
||||
|
||||
// 2. Approval-required commands (Bash only)
|
||||
if (toolName === 'Bash') {
|
||||
const command = toolInput.command || '';
|
||||
const approvalFindings = detectApprovalRequired(command);
|
||||
|
||||
if (approvalFindings.length > 0) {
|
||||
events.push({
|
||||
id: generateEventId(),
|
||||
sessionId,
|
||||
eventType: 'approval_requested',
|
||||
payload: {
|
||||
toolName,
|
||||
hookPhase,
|
||||
command: command.slice(0, 200),
|
||||
matchedPatterns: approvalFindings.map(f => f.pattern),
|
||||
severity: 'high',
|
||||
},
|
||||
resolvedAt: null,
|
||||
resolution: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Policy violation: writing to sensitive paths
|
||||
const filePath = toolInput.file_path || toolInput.path || '';
|
||||
if (filePath && detectSensitivePath(filePath)) {
|
||||
events.push({
|
||||
id: generateEventId(),
|
||||
sessionId,
|
||||
eventType: 'policy_violation',
|
||||
payload: {
|
||||
toolName,
|
||||
hookPhase,
|
||||
filePath: filePath.slice(0, 200),
|
||||
reason: 'sensitive_file_access',
|
||||
severity: 'warning',
|
||||
},
|
||||
resolvedAt: null,
|
||||
resolution: null,
|
||||
});
|
||||
}
|
||||
|
||||
// 4. Security-relevant tool usage tracking
|
||||
if (SECURITY_RELEVANT_TOOLS.has(toolName) && hookPhase === 'post') {
|
||||
const command = toolInput.command || '';
|
||||
const hasElevated = /sudo\s/.test(command) || /chmod\s/.test(command) || /chown\s/.test(command);
|
||||
|
||||
if (hasElevated) {
|
||||
events.push({
|
||||
id: generateEventId(),
|
||||
sessionId,
|
||||
eventType: 'security_finding',
|
||||
payload: {
|
||||
toolName,
|
||||
hookPhase,
|
||||
command: command.slice(0, 200),
|
||||
reason: 'elevated_privilege_command',
|
||||
severity: 'medium',
|
||||
},
|
||||
resolvedAt: null,
|
||||
resolution: null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
/**
|
||||
* Core hook logic — exported so run-with-flags.js can call directly.
|
||||
*
|
||||
* @param {string} rawInput - Raw JSON string from stdin
|
||||
* @returns {string} The original input (pass-through)
|
||||
*/
|
||||
function run(rawInput) {
|
||||
// Gate on feature flag
|
||||
if (String(process.env.ECC_GOVERNANCE_CAPTURE || '').toLowerCase() !== '1') {
|
||||
return rawInput;
|
||||
}
|
||||
|
||||
try {
|
||||
const input = JSON.parse(rawInput);
|
||||
const sessionId = process.env.ECC_SESSION_ID || null;
|
||||
const hookPhase = process.env.CLAUDE_HOOK_EVENT_NAME || 'unknown';
|
||||
|
||||
const events = analyzeForGovernanceEvents(input, {
|
||||
sessionId,
|
||||
hookPhase: hookPhase.startsWith('Pre') ? 'pre' : 'post',
|
||||
});
|
||||
|
||||
if (events.length > 0) {
|
||||
// Write events to stderr as JSON-lines for the caller to capture.
|
||||
// The state store write is async and handled by a separate process
|
||||
// to avoid blocking the hook pipeline.
|
||||
for (const event of events) {
|
||||
process.stderr.write(
|
||||
`[governance] ${JSON.stringify(event)}\n`
|
||||
);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Silently ignore parse errors — never block the tool pipeline.
|
||||
}
|
||||
|
||||
return rawInput;
|
||||
}
|
||||
|
||||
// ── stdin entry point ────────────────────────────────
|
||||
if (require.main === module) {
|
||||
let raw = '';
|
||||
process.stdin.setEncoding('utf8');
|
||||
process.stdin.on('data', chunk => {
|
||||
if (raw.length < MAX_STDIN) {
|
||||
const remaining = MAX_STDIN - raw.length;
|
||||
raw += chunk.substring(0, remaining);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdin.on('end', () => {
|
||||
const result = run(raw);
|
||||
process.stdout.write(result);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
APPROVAL_COMMANDS,
|
||||
SECRET_PATTERNS,
|
||||
SECURITY_RELEVANT_TOOLS,
|
||||
SENSITIVE_PATHS,
|
||||
analyzeForGovernanceEvents,
|
||||
detectApprovalRequired,
|
||||
detectSecrets,
|
||||
detectSensitivePath,
|
||||
generateEventId,
|
||||
run,
|
||||
};
|
||||
230
scripts/lib/agent-compress.js
Normal file
230
scripts/lib/agent-compress.js
Normal file
@@ -0,0 +1,230 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
/**
|
||||
* Parse YAML frontmatter from a markdown string.
|
||||
* Returns { frontmatter: {}, body: string }.
|
||||
*/
|
||||
function parseFrontmatter(content) {
|
||||
const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/);
|
||||
if (!match) {
|
||||
return { frontmatter: {}, body: content };
|
||||
}
|
||||
|
||||
const frontmatter = {};
|
||||
for (const line of match[1].split('\n')) {
|
||||
const colonIdx = line.indexOf(':');
|
||||
if (colonIdx === -1) continue;
|
||||
|
||||
const key = line.slice(0, colonIdx).trim();
|
||||
let value = line.slice(colonIdx + 1).trim();
|
||||
|
||||
// Handle JSON arrays (e.g. tools: ["Read", "Grep"])
|
||||
if (value.startsWith('[') && value.endsWith(']')) {
|
||||
try {
|
||||
value = JSON.parse(value);
|
||||
} catch {
|
||||
// keep as string
|
||||
}
|
||||
}
|
||||
|
||||
// Strip surrounding quotes
|
||||
if (typeof value === 'string' && value.startsWith('"') && value.endsWith('"')) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
|
||||
frontmatter[key] = value;
|
||||
}
|
||||
|
||||
return { frontmatter, body: match[2] };
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the first meaningful paragraph from agent body as a summary.
|
||||
* Skips headings and blank lines, returns up to maxSentences sentences.
|
||||
*/
|
||||
function extractSummary(body, maxSentences = 1) {
|
||||
const lines = body.split('\n');
|
||||
const paragraphs = [];
|
||||
let current = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
|
||||
if (trimmed === '') {
|
||||
if (current.length > 0) {
|
||||
paragraphs.push(current.join(' '));
|
||||
current = [];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip headings
|
||||
if (trimmed.startsWith('#')) {
|
||||
if (current.length > 0) {
|
||||
paragraphs.push(current.join(' '));
|
||||
current = [];
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip list items, code blocks, etc.
|
||||
if (trimmed.startsWith('```') || trimmed.startsWith('- **') || trimmed.startsWith('|')) {
|
||||
continue;
|
||||
}
|
||||
|
||||
current.push(trimmed);
|
||||
}
|
||||
if (current.length > 0) {
|
||||
paragraphs.push(current.join(' '));
|
||||
}
|
||||
|
||||
// Find first non-empty paragraph
|
||||
const firstParagraph = paragraphs.find(p => p.length > 0);
|
||||
if (!firstParagraph) {
|
||||
return '';
|
||||
}
|
||||
|
||||
// Extract up to maxSentences sentences
|
||||
const sentences = firstParagraph.match(/[^.!?]+[.!?]+/g) || [firstParagraph];
|
||||
return sentences.slice(0, maxSentences).join(' ').trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Load and parse a single agent file.
|
||||
* Returns the full agent object with frontmatter and body.
|
||||
*/
|
||||
function loadAgent(filePath) {
|
||||
const content = fs.readFileSync(filePath, 'utf8');
|
||||
const { frontmatter, body } = parseFrontmatter(content);
|
||||
const fileName = path.basename(filePath, '.md');
|
||||
|
||||
return {
|
||||
fileName,
|
||||
name: frontmatter.name || fileName,
|
||||
description: frontmatter.description || '',
|
||||
tools: Array.isArray(frontmatter.tools) ? frontmatter.tools : [],
|
||||
model: frontmatter.model || 'sonnet',
|
||||
body,
|
||||
byteSize: Buffer.byteLength(content, 'utf8'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Load all agents from a directory.
|
||||
*/
|
||||
function loadAgents(agentsDir) {
|
||||
if (!fs.existsSync(agentsDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs.readdirSync(agentsDir)
|
||||
.filter(f => f.endsWith('.md'))
|
||||
.sort()
|
||||
.map(f => loadAgent(path.join(agentsDir, f)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress an agent to its catalog entry (metadata only).
|
||||
* This is the minimal representation needed for agent selection.
|
||||
*/
|
||||
function compressToCatalog(agent) {
|
||||
return {
|
||||
name: agent.name,
|
||||
description: agent.description,
|
||||
tools: agent.tools,
|
||||
model: agent.model,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Compress an agent to a summary entry (metadata + first paragraph).
|
||||
* More context than catalog, less than full body.
|
||||
*/
|
||||
function compressToSummary(agent) {
|
||||
return {
|
||||
name: agent.name,
|
||||
description: agent.description,
|
||||
tools: agent.tools,
|
||||
model: agent.model,
|
||||
summary: extractSummary(agent.body),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a full compressed catalog from a directory of agents.
|
||||
*
|
||||
* Modes:
|
||||
* - 'catalog': name, description, tools, model only (~2-3k tokens for 27 agents)
|
||||
* - 'summary': catalog + first paragraph summary (~4-5k tokens)
|
||||
* - 'full': no compression, full body included
|
||||
*
|
||||
* Returns { agents: [], stats: { totalAgents, originalBytes, compressedTokenEstimate } }
|
||||
*/
|
||||
function buildAgentCatalog(agentsDir, options = {}) {
|
||||
const mode = options.mode || 'catalog';
|
||||
const filter = options.filter || null;
|
||||
|
||||
let agents = loadAgents(agentsDir);
|
||||
|
||||
if (typeof filter === 'function') {
|
||||
agents = agents.filter(filter);
|
||||
}
|
||||
|
||||
const originalBytes = agents.reduce((sum, a) => sum + a.byteSize, 0);
|
||||
|
||||
let compressed;
|
||||
if (mode === 'catalog') {
|
||||
compressed = agents.map(compressToCatalog);
|
||||
} else if (mode === 'summary') {
|
||||
compressed = agents.map(compressToSummary);
|
||||
} else {
|
||||
compressed = agents.map(a => ({
|
||||
name: a.name,
|
||||
description: a.description,
|
||||
tools: a.tools,
|
||||
model: a.model,
|
||||
body: a.body,
|
||||
}));
|
||||
}
|
||||
|
||||
const compressedJson = JSON.stringify(compressed);
|
||||
// Rough token estimate: ~4 chars per token for English text
|
||||
const compressedTokenEstimate = Math.ceil(compressedJson.length / 4);
|
||||
|
||||
return {
|
||||
agents: compressed,
|
||||
stats: {
|
||||
totalAgents: agents.length,
|
||||
originalBytes,
|
||||
compressedBytes: Buffer.byteLength(compressedJson, 'utf8'),
|
||||
compressedTokenEstimate,
|
||||
mode,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Lazy-load a single agent's full content by name from a directory.
|
||||
* Returns null if not found.
|
||||
*/
|
||||
function lazyLoadAgent(agentsDir, agentName) {
|
||||
const filePath = path.join(agentsDir, `${agentName}.md`);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return null;
|
||||
}
|
||||
return loadAgent(filePath);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
buildAgentCatalog,
|
||||
compressToCatalog,
|
||||
compressToSummary,
|
||||
extractSummary,
|
||||
lazyLoadAgent,
|
||||
loadAgent,
|
||||
loadAgents,
|
||||
parseFrontmatter,
|
||||
};
|
||||
212
scripts/lib/inspection.js
Normal file
212
scripts/lib/inspection.js
Normal file
@@ -0,0 +1,212 @@
|
||||
'use strict';
|
||||
|
||||
const DEFAULT_FAILURE_THRESHOLD = 3;
|
||||
const DEFAULT_WINDOW_SIZE = 50;
|
||||
|
||||
const FAILURE_OUTCOMES = new Set(['failure', 'failed', 'error']);
|
||||
|
||||
/**
|
||||
* Normalize a failure reason string for grouping.
|
||||
* Strips timestamps, UUIDs, file paths, and numeric suffixes.
|
||||
*/
|
||||
function normalizeFailureReason(reason) {
|
||||
if (!reason || typeof reason !== 'string') {
|
||||
return 'unknown';
|
||||
}
|
||||
|
||||
return reason
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
// Strip ISO timestamps (note: already lowercased, so t/z not T/Z)
|
||||
.replace(/\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}[.\dz]*/g, '<timestamp>')
|
||||
// Strip UUIDs (already lowercased)
|
||||
.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, '<uuid>')
|
||||
// Strip file paths
|
||||
.replace(/\/[\w./-]+/g, '<path>')
|
||||
// Collapse whitespace
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Group skill runs by skill ID and normalized failure reason.
|
||||
*
|
||||
* @param {Array} skillRuns - Array of skill run objects
|
||||
* @returns {Map<string, { skillId: string, normalizedReason: string, runs: Array }>}
|
||||
*/
|
||||
function groupFailures(skillRuns) {
|
||||
const groups = new Map();
|
||||
|
||||
for (const run of skillRuns) {
|
||||
const outcome = String(run.outcome || '').toLowerCase();
|
||||
if (!FAILURE_OUTCOMES.has(outcome)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const normalizedReason = normalizeFailureReason(run.failureReason);
|
||||
const key = `${run.skillId}::${normalizedReason}`;
|
||||
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, {
|
||||
skillId: run.skillId,
|
||||
normalizedReason,
|
||||
runs: [],
|
||||
});
|
||||
}
|
||||
|
||||
groups.get(key).runs.push(run);
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect recurring failure patterns from skill runs.
|
||||
*
|
||||
* @param {Array} skillRuns - Array of skill run objects (newest first)
|
||||
* @param {Object} [options]
|
||||
* @param {number} [options.threshold=3] - Minimum failure count to trigger pattern detection
|
||||
* @returns {Array<Object>} Array of detected patterns sorted by count descending
|
||||
*/
|
||||
function detectPatterns(skillRuns, options = {}) {
|
||||
const threshold = options.threshold ?? DEFAULT_FAILURE_THRESHOLD;
|
||||
const groups = groupFailures(skillRuns);
|
||||
const patterns = [];
|
||||
|
||||
for (const [, group] of groups) {
|
||||
if (group.runs.length < threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const sortedRuns = [...group.runs].sort(
|
||||
(a, b) => (b.createdAt || '').localeCompare(a.createdAt || '')
|
||||
);
|
||||
|
||||
const firstSeen = sortedRuns[sortedRuns.length - 1].createdAt || null;
|
||||
const lastSeen = sortedRuns[0].createdAt || null;
|
||||
const sessionIds = [...new Set(sortedRuns.map(r => r.sessionId).filter(Boolean))];
|
||||
const versions = [...new Set(sortedRuns.map(r => r.skillVersion).filter(Boolean))];
|
||||
|
||||
// Collect unique raw failure reasons for this normalized group
|
||||
const rawReasons = [...new Set(sortedRuns.map(r => r.failureReason).filter(Boolean))];
|
||||
|
||||
patterns.push({
|
||||
skillId: group.skillId,
|
||||
normalizedReason: group.normalizedReason,
|
||||
count: group.runs.length,
|
||||
firstSeen,
|
||||
lastSeen,
|
||||
sessionIds,
|
||||
versions,
|
||||
rawReasons,
|
||||
runIds: sortedRuns.map(r => r.id),
|
||||
});
|
||||
}
|
||||
|
||||
// Sort by count descending, then by lastSeen descending
|
||||
return patterns.sort((a, b) => {
|
||||
if (b.count !== a.count) return b.count - a.count;
|
||||
return (b.lastSeen || '').localeCompare(a.lastSeen || '');
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an inspection report from detected patterns.
|
||||
*
|
||||
* @param {Array} patterns - Output from detectPatterns()
|
||||
* @param {Object} [options]
|
||||
* @param {string} [options.generatedAt] - ISO timestamp for the report
|
||||
* @returns {Object} Inspection report
|
||||
*/
|
||||
function generateReport(patterns, options = {}) {
|
||||
const generatedAt = options.generatedAt || new Date().toISOString();
|
||||
|
||||
if (patterns.length === 0) {
|
||||
return {
|
||||
generatedAt,
|
||||
status: 'clean',
|
||||
patternCount: 0,
|
||||
patterns: [],
|
||||
summary: 'No recurring failure patterns detected.',
|
||||
};
|
||||
}
|
||||
|
||||
const totalFailures = patterns.reduce((sum, p) => sum + p.count, 0);
|
||||
const affectedSkills = [...new Set(patterns.map(p => p.skillId))];
|
||||
|
||||
return {
|
||||
generatedAt,
|
||||
status: 'attention_needed',
|
||||
patternCount: patterns.length,
|
||||
totalFailures,
|
||||
affectedSkills,
|
||||
patterns: patterns.map(p => ({
|
||||
skillId: p.skillId,
|
||||
normalizedReason: p.normalizedReason,
|
||||
count: p.count,
|
||||
firstSeen: p.firstSeen,
|
||||
lastSeen: p.lastSeen,
|
||||
sessionIds: p.sessionIds,
|
||||
versions: p.versions,
|
||||
rawReasons: p.rawReasons.slice(0, 5),
|
||||
suggestedAction: suggestAction(p),
|
||||
})),
|
||||
summary: `Found ${patterns.length} recurring failure pattern(s) across ${affectedSkills.length} skill(s) (${totalFailures} total failures).`,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Suggest a remediation action based on pattern characteristics.
|
||||
*/
|
||||
function suggestAction(pattern) {
|
||||
const reason = pattern.normalizedReason;
|
||||
|
||||
if (reason.includes('timeout')) {
|
||||
return 'Increase timeout or optimize skill execution time.';
|
||||
}
|
||||
if (reason.includes('permission') || reason.includes('denied') || reason.includes('auth')) {
|
||||
return 'Check tool permissions and authentication configuration.';
|
||||
}
|
||||
if (reason.includes('not found') || reason.includes('missing')) {
|
||||
return 'Verify required files/dependencies exist before skill execution.';
|
||||
}
|
||||
if (reason.includes('parse') || reason.includes('syntax') || reason.includes('json')) {
|
||||
return 'Review input/output format expectations and add validation.';
|
||||
}
|
||||
if (pattern.versions.length > 1) {
|
||||
return 'Failure spans multiple versions. Consider rollback to last stable version.';
|
||||
}
|
||||
|
||||
return 'Investigate root cause and consider adding error handling.';
|
||||
}
|
||||
|
||||
/**
|
||||
* Run full inspection pipeline: query skill runs, detect patterns, generate report.
|
||||
*
|
||||
* @param {Object} store - State store instance with listRecentSessions, getSessionDetail
|
||||
* @param {Object} [options]
|
||||
* @param {number} [options.threshold] - Minimum failure count
|
||||
* @param {number} [options.windowSize] - Number of recent skill runs to analyze
|
||||
* @returns {Object} Inspection report
|
||||
*/
|
||||
function inspect(store, options = {}) {
|
||||
const windowSize = options.windowSize ?? DEFAULT_WINDOW_SIZE;
|
||||
const threshold = options.threshold ?? DEFAULT_FAILURE_THRESHOLD;
|
||||
|
||||
const status = store.getStatus({ recentSkillRunLimit: windowSize });
|
||||
const skillRuns = status.skillRuns.recent || [];
|
||||
|
||||
const patterns = detectPatterns(skillRuns, { threshold });
|
||||
return generateReport(patterns, { generatedAt: status.generatedAt });
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DEFAULT_FAILURE_THRESHOLD,
|
||||
DEFAULT_WINDOW_SIZE,
|
||||
detectPatterns,
|
||||
generateReport,
|
||||
groupFailures,
|
||||
inspect,
|
||||
normalizeFailureReason,
|
||||
suggestAction,
|
||||
};
|
||||
294
tests/hooks/governance-capture.test.js
Normal file
294
tests/hooks/governance-capture.test.js
Normal file
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* Tests for governance event capture hook.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const {
|
||||
detectSecrets,
|
||||
detectApprovalRequired,
|
||||
detectSensitivePath,
|
||||
analyzeForGovernanceEvents,
|
||||
run,
|
||||
} = require('../../scripts/hooks/governance-capture');
|
||||
|
||||
async function test(name, fn) {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== Testing governance-capture ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
// ── detectSecrets ──────────────────────────────────────────
|
||||
|
||||
if (await test('detectSecrets finds AWS access keys', async () => {
|
||||
const findings = detectSecrets('my key is AKIAIOSFODNN7EXAMPLE');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'aws_key'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets finds generic secrets', async () => {
|
||||
const findings = detectSecrets('api_key = "sk-proj-abcdefghij1234567890"');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'generic_secret'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets finds private keys', async () => {
|
||||
const findings = detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMIIE...');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'private_key'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets finds GitHub tokens', async () => {
|
||||
const findings = detectSecrets('token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'github_token'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets returns empty array for clean text', async () => {
|
||||
const findings = detectSecrets('This is a normal log message with no secrets.');
|
||||
assert.strictEqual(findings.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets handles null and undefined', async () => {
|
||||
assert.deepStrictEqual(detectSecrets(null), []);
|
||||
assert.deepStrictEqual(detectSecrets(undefined), []);
|
||||
assert.deepStrictEqual(detectSecrets(''), []);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── detectApprovalRequired ─────────────────────────────────
|
||||
|
||||
if (await test('detectApprovalRequired flags force push', async () => {
|
||||
const findings = detectApprovalRequired('git push origin main --force');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired flags hard reset', async () => {
|
||||
const findings = detectApprovalRequired('git reset --hard HEAD~3');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired flags rm -rf', async () => {
|
||||
const findings = detectApprovalRequired('rm -rf /tmp/important');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired flags DROP TABLE', async () => {
|
||||
const findings = detectApprovalRequired('DROP TABLE users');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired allows safe commands', async () => {
|
||||
const findings = detectApprovalRequired('git status');
|
||||
assert.strictEqual(findings.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired handles null', async () => {
|
||||
assert.deepStrictEqual(detectApprovalRequired(null), []);
|
||||
assert.deepStrictEqual(detectApprovalRequired(''), []);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── detectSensitivePath ────────────────────────────────────
|
||||
|
||||
if (await test('detectSensitivePath identifies .env files', async () => {
|
||||
assert.ok(detectSensitivePath('.env'));
|
||||
assert.ok(detectSensitivePath('.env.local'));
|
||||
assert.ok(detectSensitivePath('/project/.env.production'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSensitivePath identifies credential files', async () => {
|
||||
assert.ok(detectSensitivePath('credentials.json'));
|
||||
assert.ok(detectSensitivePath('/home/user/.ssh/id_rsa'));
|
||||
assert.ok(detectSensitivePath('server.key'));
|
||||
assert.ok(detectSensitivePath('cert.pem'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSensitivePath returns false for normal files', async () => {
|
||||
assert.ok(!detectSensitivePath('index.js'));
|
||||
assert.ok(!detectSensitivePath('README.md'));
|
||||
assert.ok(!detectSensitivePath('package.json'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSensitivePath handles null', async () => {
|
||||
assert.ok(!detectSensitivePath(null));
|
||||
assert.ok(!detectSensitivePath(''));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── analyzeForGovernanceEvents ─────────────────────────────
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects secrets in tool input', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: {
|
||||
file_path: '/tmp/config.js',
|
||||
content: 'const key = "AKIAIOSFODNN7EXAMPLE";',
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
const secretEvent = events.find(e => e.eventType === 'secret_detected');
|
||||
assert.ok(secretEvent);
|
||||
assert.strictEqual(secretEvent.payload.severity, 'critical');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects approval-required commands', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Bash',
|
||||
tool_input: {
|
||||
command: 'git push origin main --force',
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
const approvalEvent = events.find(e => e.eventType === 'approval_requested');
|
||||
assert.ok(approvalEvent);
|
||||
assert.strictEqual(approvalEvent.payload.severity, 'high');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects sensitive file access', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Edit',
|
||||
tool_input: {
|
||||
file_path: '/project/.env.production',
|
||||
old_string: 'DB_URL=old',
|
||||
new_string: 'DB_URL=new',
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
const policyEvent = events.find(e => e.eventType === 'policy_violation');
|
||||
assert.ok(policyEvent);
|
||||
assert.strictEqual(policyEvent.payload.reason, 'sensitive_file_access');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects elevated privilege commands', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Bash',
|
||||
tool_input: { command: 'sudo rm -rf /etc/something' },
|
||||
}, {
|
||||
hookPhase: 'post',
|
||||
});
|
||||
|
||||
const securityEvent = events.find(e => e.eventType === 'security_finding');
|
||||
assert.ok(securityEvent);
|
||||
assert.strictEqual(securityEvent.payload.reason, 'elevated_privilege_command');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents returns empty for clean inputs', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Read',
|
||||
tool_input: { file_path: '/project/src/index.js' },
|
||||
});
|
||||
assert.strictEqual(events.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents populates session ID from context', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: {
|
||||
file_path: '/project/.env',
|
||||
content: 'DB_URL=test',
|
||||
},
|
||||
}, {
|
||||
sessionId: 'test-session-123',
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
assert.strictEqual(events[0].sessionId, 'test-session-123');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents generates unique event IDs', async () => {
|
||||
const events1 = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: { file_path: '.env', content: '' },
|
||||
});
|
||||
const events2 = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: { file_path: '.env.local', content: '' },
|
||||
});
|
||||
|
||||
if (events1.length > 0 && events2.length > 0) {
|
||||
assert.notStrictEqual(events1[0].id, events2[0].id);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── run() function ─────────────────────────────────────────
|
||||
|
||||
if (await test('run() passes through input when feature flag is off', async () => {
|
||||
const original = process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
delete process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
|
||||
try {
|
||||
const input = JSON.stringify({ tool_name: 'Bash', tool_input: { command: 'git push --force' } });
|
||||
const result = run(input);
|
||||
assert.strictEqual(result, input);
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = original;
|
||||
}
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('run() passes through input when feature flag is on', async () => {
|
||||
const original = process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = '1';
|
||||
|
||||
try {
|
||||
const input = JSON.stringify({ tool_name: 'Read', tool_input: { file_path: 'index.js' } });
|
||||
const result = run(input);
|
||||
assert.strictEqual(result, input);
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = original;
|
||||
} else {
|
||||
delete process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
}
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('run() handles invalid JSON gracefully', async () => {
|
||||
const original = process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = '1';
|
||||
|
||||
try {
|
||||
const result = run('not valid json');
|
||||
assert.strictEqual(result, 'not valid json');
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = original;
|
||||
} else {
|
||||
delete process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
}
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('run() can detect multiple event types in one input', async () => {
|
||||
// Bash command with force push AND secret in command
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Bash',
|
||||
tool_input: {
|
||||
command: 'API_KEY="AKIAIOSFODNN7EXAMPLE" git push --force',
|
||||
},
|
||||
});
|
||||
|
||||
const eventTypes = events.map(e => e.eventType);
|
||||
assert.ok(eventTypes.includes('secret_detected'));
|
||||
assert.ok(eventTypes.includes('approval_requested'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
293
tests/lib/agent-compress.test.js
Normal file
293
tests/lib/agent-compress.test.js
Normal file
@@ -0,0 +1,293 @@
|
||||
/**
|
||||
* Tests for agent description compression and lazy loading.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
const {
|
||||
parseFrontmatter,
|
||||
extractSummary,
|
||||
loadAgent,
|
||||
loadAgents,
|
||||
compressToCatalog,
|
||||
compressToSummary,
|
||||
buildAgentCatalog,
|
||||
lazyLoadAgent,
|
||||
} = require('../../scripts/lib/agent-compress');
|
||||
|
||||
function createTempDir(prefix) {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function cleanupTempDir(dirPath) {
|
||||
fs.rmSync(dirPath, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function writeAgent(dir, name, content) {
|
||||
fs.writeFileSync(path.join(dir, `${name}.md`), content, 'utf8');
|
||||
}
|
||||
|
||||
const SAMPLE_AGENT = `---
|
||||
name: test-agent
|
||||
description: A test agent for unit testing purposes.
|
||||
tools: ["Read", "Grep", "Glob"]
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
You are a test agent that validates compression logic.
|
||||
|
||||
## Your Role
|
||||
|
||||
- Run unit tests
|
||||
- Validate compression output
|
||||
- Ensure correctness
|
||||
|
||||
## Process
|
||||
|
||||
### 1. Setup
|
||||
- Prepare test fixtures
|
||||
- Load agent files
|
||||
|
||||
### 2. Validate
|
||||
Check the output format and content.
|
||||
`;
|
||||
|
||||
const MINIMAL_AGENT = `---
|
||||
name: minimal
|
||||
description: Minimal agent.
|
||||
tools: ["Read"]
|
||||
model: haiku
|
||||
---
|
||||
|
||||
Short body.
|
||||
`;
|
||||
|
||||
async function test(name, fn) {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== Testing agent-compress ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
if (await test('parseFrontmatter extracts YAML frontmatter and body', async () => {
|
||||
const { frontmatter, body } = parseFrontmatter(SAMPLE_AGENT);
|
||||
assert.strictEqual(frontmatter.name, 'test-agent');
|
||||
assert.strictEqual(frontmatter.description, 'A test agent for unit testing purposes.');
|
||||
assert.deepStrictEqual(frontmatter.tools, ['Read', 'Grep', 'Glob']);
|
||||
assert.strictEqual(frontmatter.model, 'sonnet');
|
||||
assert.ok(body.includes('You are a test agent'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('parseFrontmatter handles content without frontmatter', async () => {
|
||||
const { frontmatter, body } = parseFrontmatter('Just a plain document.');
|
||||
assert.deepStrictEqual(frontmatter, {});
|
||||
assert.strictEqual(body, 'Just a plain document.');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('extractSummary returns the first paragraph of the body', async () => {
|
||||
const { body } = parseFrontmatter(SAMPLE_AGENT);
|
||||
const summary = extractSummary(body);
|
||||
assert.ok(summary.includes('test agent'));
|
||||
assert.ok(summary.includes('compression logic'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('extractSummary returns empty string for empty body', async () => {
|
||||
assert.strictEqual(extractSummary(''), '');
|
||||
assert.strictEqual(extractSummary('# Just a heading'), '');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('loadAgent reads and parses a single agent file', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
|
||||
assert.strictEqual(agent.name, 'test-agent');
|
||||
assert.strictEqual(agent.fileName, 'test-agent');
|
||||
assert.deepStrictEqual(agent.tools, ['Read', 'Grep', 'Glob']);
|
||||
assert.strictEqual(agent.model, 'sonnet');
|
||||
assert.ok(agent.byteSize > 0);
|
||||
assert.ok(agent.body.includes('You are a test agent'));
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('loadAgents reads all .md files from a directory', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
|
||||
const agents = loadAgents(tmpDir);
|
||||
assert.strictEqual(agents.length, 2);
|
||||
assert.strictEqual(agents[0].fileName, 'agent-a');
|
||||
assert.strictEqual(agents[1].fileName, 'agent-b');
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('loadAgents returns empty array for non-existent directory', async () => {
|
||||
const agents = loadAgents('/tmp/nonexistent-ecc-dir-12345');
|
||||
assert.deepStrictEqual(agents, []);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('compressToCatalog strips body and keeps only metadata', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
|
||||
const catalog = compressToCatalog(agent);
|
||||
|
||||
assert.strictEqual(catalog.name, 'test-agent');
|
||||
assert.strictEqual(catalog.description, 'A test agent for unit testing purposes.');
|
||||
assert.deepStrictEqual(catalog.tools, ['Read', 'Grep', 'Glob']);
|
||||
assert.strictEqual(catalog.model, 'sonnet');
|
||||
assert.strictEqual(catalog.body, undefined);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('compressToSummary includes first paragraph summary', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
|
||||
const summary = compressToSummary(agent);
|
||||
|
||||
assert.strictEqual(summary.name, 'test-agent');
|
||||
assert.ok(summary.summary.length > 0);
|
||||
assert.strictEqual(summary.body, undefined);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog in catalog mode produces minimal output with stats', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, { mode: 'catalog' });
|
||||
assert.strictEqual(result.agents.length, 2);
|
||||
assert.strictEqual(result.stats.totalAgents, 2);
|
||||
assert.strictEqual(result.stats.mode, 'catalog');
|
||||
assert.ok(result.stats.originalBytes > 0);
|
||||
assert.ok(result.stats.compressedBytes > 0);
|
||||
assert.ok(result.stats.compressedBytes < result.stats.originalBytes);
|
||||
assert.ok(result.stats.compressedTokenEstimate > 0);
|
||||
|
||||
// Catalog entries should not have body
|
||||
for (const agent of result.agents) {
|
||||
assert.strictEqual(agent.body, undefined);
|
||||
assert.ok(agent.name);
|
||||
assert.ok(agent.description);
|
||||
}
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog in summary mode includes summaries', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, { mode: 'summary' });
|
||||
assert.strictEqual(result.agents.length, 1);
|
||||
assert.ok(result.agents[0].summary);
|
||||
assert.strictEqual(result.agents[0].body, undefined);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog in full mode preserves body', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, { mode: 'full' });
|
||||
assert.strictEqual(result.agents.length, 1);
|
||||
assert.ok(result.agents[0].body.includes('You are a test agent'));
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog supports filter function', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, {
|
||||
mode: 'catalog',
|
||||
filter: agent => agent.model === 'haiku',
|
||||
});
|
||||
assert.strictEqual(result.agents.length, 1);
|
||||
assert.strictEqual(result.agents[0].name, 'minimal');
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('lazyLoadAgent loads a single agent by name', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'other', MINIMAL_AGENT);
|
||||
|
||||
const agent = lazyLoadAgent(tmpDir, 'test-agent');
|
||||
assert.ok(agent);
|
||||
assert.strictEqual(agent.name, 'test-agent');
|
||||
assert.ok(agent.body.includes('You are a test agent'));
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('lazyLoadAgent returns null for non-existent agent', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
const agent = lazyLoadAgent(tmpDir, 'nonexistent');
|
||||
assert.strictEqual(agent, null);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog works with real agents directory', async () => {
|
||||
const agentsDir = path.join(__dirname, '..', '..', 'agents');
|
||||
if (!fs.existsSync(agentsDir)) {
|
||||
// Skip if agents dir doesn't exist (shouldn't happen in this repo)
|
||||
return;
|
||||
}
|
||||
|
||||
const result = buildAgentCatalog(agentsDir, { mode: 'catalog' });
|
||||
assert.ok(result.agents.length > 0, 'Should find at least one agent');
|
||||
assert.ok(result.stats.originalBytes > 0);
|
||||
assert.ok(result.stats.compressedBytes < result.stats.originalBytes,
|
||||
'Catalog mode should be smaller than full agent files');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
232
tests/lib/inspection.test.js
Normal file
232
tests/lib/inspection.test.js
Normal file
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* Tests for inspection logic — pattern detection from failures.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const {
|
||||
normalizeFailureReason,
|
||||
groupFailures,
|
||||
detectPatterns,
|
||||
generateReport,
|
||||
suggestAction,
|
||||
DEFAULT_FAILURE_THRESHOLD,
|
||||
} = require('../../scripts/lib/inspection');
|
||||
|
||||
async function test(name, fn) {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function makeSkillRun(overrides = {}) {
|
||||
return {
|
||||
id: overrides.id || `run-${Math.random().toString(36).slice(2, 8)}`,
|
||||
skillId: overrides.skillId || 'test-skill',
|
||||
skillVersion: overrides.skillVersion || '1.0.0',
|
||||
sessionId: overrides.sessionId || 'session-1',
|
||||
taskDescription: overrides.taskDescription || 'test task',
|
||||
outcome: overrides.outcome || 'failure',
|
||||
failureReason: overrides.failureReason || 'generic error',
|
||||
tokensUsed: overrides.tokensUsed || 500,
|
||||
durationMs: overrides.durationMs || 1000,
|
||||
userFeedback: overrides.userFeedback || null,
|
||||
createdAt: overrides.createdAt || '2026-03-15T08:00:00.000Z',
|
||||
};
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== Testing inspection ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
if (await test('normalizeFailureReason strips timestamps and UUIDs', async () => {
|
||||
const normalized = normalizeFailureReason(
|
||||
'Error at 2026-03-15T08:00:00.000Z for id 550e8400-e29b-41d4-a716-446655440000'
|
||||
);
|
||||
assert.ok(!normalized.includes('2026'));
|
||||
assert.ok(!normalized.includes('550e8400'));
|
||||
assert.ok(normalized.includes('<timestamp>'));
|
||||
assert.ok(normalized.includes('<uuid>'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('normalizeFailureReason strips file paths', async () => {
|
||||
const normalized = normalizeFailureReason('File not found: /usr/local/bin/node');
|
||||
assert.ok(!normalized.includes('/usr/local'));
|
||||
assert.ok(normalized.includes('<path>'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('normalizeFailureReason handles null and empty values', async () => {
|
||||
assert.strictEqual(normalizeFailureReason(null), 'unknown');
|
||||
assert.strictEqual(normalizeFailureReason(''), 'unknown');
|
||||
assert.strictEqual(normalizeFailureReason(undefined), 'unknown');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('groupFailures groups by skillId and normalized reason', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', skillId: 'skill-a', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r2', skillId: 'skill-a', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r3', skillId: 'skill-b', failureReason: 'parse error' }),
|
||||
makeSkillRun({ id: 'r4', skillId: 'skill-a', outcome: 'success' }), // should be excluded
|
||||
];
|
||||
|
||||
const groups = groupFailures(runs);
|
||||
assert.strictEqual(groups.size, 2);
|
||||
|
||||
const skillAGroup = groups.get('skill-a::timeout');
|
||||
assert.ok(skillAGroup);
|
||||
assert.strictEqual(skillAGroup.runs.length, 2);
|
||||
|
||||
const skillBGroup = groups.get('skill-b::parse error');
|
||||
assert.ok(skillBGroup);
|
||||
assert.strictEqual(skillBGroup.runs.length, 1);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('groupFailures handles mixed outcome casing', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', outcome: 'FAILURE', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r2', outcome: 'Failed', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r3', outcome: 'error', failureReason: 'timeout' }),
|
||||
];
|
||||
|
||||
const groups = groupFailures(runs);
|
||||
assert.strictEqual(groups.size, 1);
|
||||
const group = groups.values().next().value;
|
||||
assert.strictEqual(group.runs.length, 3);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns returns empty array when below threshold', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r2', failureReason: 'timeout' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns detects patterns at or above threshold', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', failureReason: 'timeout', createdAt: '2026-03-15T08:00:00Z' }),
|
||||
makeSkillRun({ id: 'r2', failureReason: 'timeout', createdAt: '2026-03-15T08:01:00Z' }),
|
||||
makeSkillRun({ id: 'r3', failureReason: 'timeout', createdAt: '2026-03-15T08:02:00Z' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
assert.strictEqual(patterns[0].count, 3);
|
||||
assert.strictEqual(patterns[0].skillId, 'test-skill');
|
||||
assert.strictEqual(patterns[0].normalizedReason, 'timeout');
|
||||
assert.strictEqual(patterns[0].firstSeen, '2026-03-15T08:00:00Z');
|
||||
assert.strictEqual(patterns[0].lastSeen, '2026-03-15T08:02:00Z');
|
||||
assert.strictEqual(patterns[0].runIds.length, 3);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns uses default threshold', async () => {
|
||||
const runs = Array.from({ length: DEFAULT_FAILURE_THRESHOLD }, (_, i) =>
|
||||
makeSkillRun({ id: `r${i}`, failureReason: 'permission denied' })
|
||||
);
|
||||
|
||||
const patterns = detectPatterns(runs);
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns sorts by count descending', async () => {
|
||||
const runs = [
|
||||
// 4 timeouts
|
||||
...Array.from({ length: 4 }, (_, i) =>
|
||||
makeSkillRun({ id: `t${i}`, skillId: 'skill-a', failureReason: 'timeout' })
|
||||
),
|
||||
// 3 parse errors
|
||||
...Array.from({ length: 3 }, (_, i) =>
|
||||
makeSkillRun({ id: `p${i}`, skillId: 'skill-b', failureReason: 'parse error' })
|
||||
),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 2);
|
||||
assert.strictEqual(patterns[0].count, 4);
|
||||
assert.strictEqual(patterns[0].skillId, 'skill-a');
|
||||
assert.strictEqual(patterns[1].count, 3);
|
||||
assert.strictEqual(patterns[1].skillId, 'skill-b');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns groups similar failure reasons with different timestamps', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', failureReason: 'Error at 2026-03-15T08:00:00Z in /tmp/foo' }),
|
||||
makeSkillRun({ id: 'r2', failureReason: 'Error at 2026-03-15T09:00:00Z in /tmp/bar' }),
|
||||
makeSkillRun({ id: 'r3', failureReason: 'Error at 2026-03-15T10:00:00Z in /tmp/baz' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
assert.ok(patterns[0].normalizedReason.includes('<timestamp>'));
|
||||
assert.ok(patterns[0].normalizedReason.includes('<path>'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns tracks unique session IDs and versions', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', sessionId: 'sess-1', skillVersion: '1.0.0', failureReason: 'err' }),
|
||||
makeSkillRun({ id: 'r2', sessionId: 'sess-2', skillVersion: '1.0.0', failureReason: 'err' }),
|
||||
makeSkillRun({ id: 'r3', sessionId: 'sess-1', skillVersion: '1.1.0', failureReason: 'err' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
assert.deepStrictEqual(patterns[0].sessionIds.sort(), ['sess-1', 'sess-2']);
|
||||
assert.deepStrictEqual(patterns[0].versions.sort(), ['1.0.0', '1.1.0']);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('generateReport returns clean status with no patterns', async () => {
|
||||
const report = generateReport([]);
|
||||
assert.strictEqual(report.status, 'clean');
|
||||
assert.strictEqual(report.patternCount, 0);
|
||||
assert.ok(report.summary.includes('No recurring'));
|
||||
assert.ok(report.generatedAt);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('generateReport produces structured report from patterns', async () => {
|
||||
const runs = [
|
||||
...Array.from({ length: 3 }, (_, i) =>
|
||||
makeSkillRun({ id: `r${i}`, skillId: 'my-skill', failureReason: 'timeout' })
|
||||
),
|
||||
];
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
const report = generateReport(patterns, { generatedAt: '2026-03-15T09:00:00Z' });
|
||||
|
||||
assert.strictEqual(report.status, 'attention_needed');
|
||||
assert.strictEqual(report.patternCount, 1);
|
||||
assert.strictEqual(report.totalFailures, 3);
|
||||
assert.deepStrictEqual(report.affectedSkills, ['my-skill']);
|
||||
assert.strictEqual(report.patterns[0].skillId, 'my-skill');
|
||||
assert.ok(report.patterns[0].suggestedAction);
|
||||
assert.strictEqual(report.generatedAt, '2026-03-15T09:00:00Z');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('suggestAction returns timeout-specific advice', async () => {
|
||||
const action = suggestAction({ normalizedReason: 'timeout after 30s', versions: ['1.0.0'] });
|
||||
assert.ok(action.toLowerCase().includes('timeout'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('suggestAction returns permission-specific advice', async () => {
|
||||
const action = suggestAction({ normalizedReason: 'permission denied', versions: ['1.0.0'] });
|
||||
assert.ok(action.toLowerCase().includes('permission'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('suggestAction returns version-span advice when multiple versions affected', async () => {
|
||||
const action = suggestAction({ normalizedReason: 'something broke', versions: ['1.0.0', '1.1.0'] });
|
||||
assert.ok(action.toLowerCase().includes('version'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
Reference in New Issue
Block a user