mirror of
https://github.com/affaan-m/everything-claude-code.git
synced 2026-03-30 21:53:28 +08:00
Implements three roadmap features: - Agent description compression (#491): New `agent-compress` module with catalog/summary/full compression modes and lazy-loading. Reduces ~26k token agent descriptions to ~2-3k catalog entries for context efficiency. - Inspection logic (#485): New `inspection` module that detects recurring failure patterns in skill_runs. Groups by skill + normalized failure reason, generates structured reports with suggested remediation actions. Configurable threshold (default: 3 failures). - Governance event capture hook (#482): PreToolUse/PostToolUse hook that detects secrets, policy violations, approval-required commands, and elevated privilege usage. Gated behind ECC_GOVERNANCE_CAPTURE=1 flag. Writes to governance_events table via JSON-line stderr output. 59 new tests (16 + 16 + 27), all passing.
This commit is contained in:
Affaan Mustafa
294
tests/hooks/governance-capture.test.js
Normal file
294
tests/hooks/governance-capture.test.js
Normal file
@@ -0,0 +1,294 @@
|
||||
/**
|
||||
* Tests for governance event capture hook.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const {
|
||||
detectSecrets,
|
||||
detectApprovalRequired,
|
||||
detectSensitivePath,
|
||||
analyzeForGovernanceEvents,
|
||||
run,
|
||||
} = require('../../scripts/hooks/governance-capture');
|
||||
|
||||
async function test(name, fn) {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== Testing governance-capture ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
// ── detectSecrets ──────────────────────────────────────────
|
||||
|
||||
if (await test('detectSecrets finds AWS access keys', async () => {
|
||||
const findings = detectSecrets('my key is AKIAIOSFODNN7EXAMPLE');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'aws_key'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets finds generic secrets', async () => {
|
||||
const findings = detectSecrets('api_key = "sk-proj-abcdefghij1234567890"');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'generic_secret'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets finds private keys', async () => {
|
||||
const findings = detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMIIE...');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'private_key'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets finds GitHub tokens', async () => {
|
||||
const findings = detectSecrets('token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij');
|
||||
assert.ok(findings.length > 0);
|
||||
assert.ok(findings.some(f => f.name === 'github_token'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets returns empty array for clean text', async () => {
|
||||
const findings = detectSecrets('This is a normal log message with no secrets.');
|
||||
assert.strictEqual(findings.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSecrets handles null and undefined', async () => {
|
||||
assert.deepStrictEqual(detectSecrets(null), []);
|
||||
assert.deepStrictEqual(detectSecrets(undefined), []);
|
||||
assert.deepStrictEqual(detectSecrets(''), []);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── detectApprovalRequired ─────────────────────────────────
|
||||
|
||||
if (await test('detectApprovalRequired flags force push', async () => {
|
||||
const findings = detectApprovalRequired('git push origin main --force');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired flags hard reset', async () => {
|
||||
const findings = detectApprovalRequired('git reset --hard HEAD~3');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired flags rm -rf', async () => {
|
||||
const findings = detectApprovalRequired('rm -rf /tmp/important');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired flags DROP TABLE', async () => {
|
||||
const findings = detectApprovalRequired('DROP TABLE users');
|
||||
assert.ok(findings.length > 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired allows safe commands', async () => {
|
||||
const findings = detectApprovalRequired('git status');
|
||||
assert.strictEqual(findings.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectApprovalRequired handles null', async () => {
|
||||
assert.deepStrictEqual(detectApprovalRequired(null), []);
|
||||
assert.deepStrictEqual(detectApprovalRequired(''), []);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── detectSensitivePath ────────────────────────────────────
|
||||
|
||||
if (await test('detectSensitivePath identifies .env files', async () => {
|
||||
assert.ok(detectSensitivePath('.env'));
|
||||
assert.ok(detectSensitivePath('.env.local'));
|
||||
assert.ok(detectSensitivePath('/project/.env.production'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSensitivePath identifies credential files', async () => {
|
||||
assert.ok(detectSensitivePath('credentials.json'));
|
||||
assert.ok(detectSensitivePath('/home/user/.ssh/id_rsa'));
|
||||
assert.ok(detectSensitivePath('server.key'));
|
||||
assert.ok(detectSensitivePath('cert.pem'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSensitivePath returns false for normal files', async () => {
|
||||
assert.ok(!detectSensitivePath('index.js'));
|
||||
assert.ok(!detectSensitivePath('README.md'));
|
||||
assert.ok(!detectSensitivePath('package.json'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectSensitivePath handles null', async () => {
|
||||
assert.ok(!detectSensitivePath(null));
|
||||
assert.ok(!detectSensitivePath(''));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── analyzeForGovernanceEvents ─────────────────────────────
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects secrets in tool input', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: {
|
||||
file_path: '/tmp/config.js',
|
||||
content: 'const key = "AKIAIOSFODNN7EXAMPLE";',
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
const secretEvent = events.find(e => e.eventType === 'secret_detected');
|
||||
assert.ok(secretEvent);
|
||||
assert.strictEqual(secretEvent.payload.severity, 'critical');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects approval-required commands', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Bash',
|
||||
tool_input: {
|
||||
command: 'git push origin main --force',
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
const approvalEvent = events.find(e => e.eventType === 'approval_requested');
|
||||
assert.ok(approvalEvent);
|
||||
assert.strictEqual(approvalEvent.payload.severity, 'high');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects sensitive file access', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Edit',
|
||||
tool_input: {
|
||||
file_path: '/project/.env.production',
|
||||
old_string: 'DB_URL=old',
|
||||
new_string: 'DB_URL=new',
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
const policyEvent = events.find(e => e.eventType === 'policy_violation');
|
||||
assert.ok(policyEvent);
|
||||
assert.strictEqual(policyEvent.payload.reason, 'sensitive_file_access');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents detects elevated privilege commands', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Bash',
|
||||
tool_input: { command: 'sudo rm -rf /etc/something' },
|
||||
}, {
|
||||
hookPhase: 'post',
|
||||
});
|
||||
|
||||
const securityEvent = events.find(e => e.eventType === 'security_finding');
|
||||
assert.ok(securityEvent);
|
||||
assert.strictEqual(securityEvent.payload.reason, 'elevated_privilege_command');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents returns empty for clean inputs', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Read',
|
||||
tool_input: { file_path: '/project/src/index.js' },
|
||||
});
|
||||
assert.strictEqual(events.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents populates session ID from context', async () => {
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: {
|
||||
file_path: '/project/.env',
|
||||
content: 'DB_URL=test',
|
||||
},
|
||||
}, {
|
||||
sessionId: 'test-session-123',
|
||||
});
|
||||
|
||||
assert.ok(events.length > 0);
|
||||
assert.strictEqual(events[0].sessionId, 'test-session-123');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('analyzeForGovernanceEvents generates unique event IDs', async () => {
|
||||
const events1 = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: { file_path: '.env', content: '' },
|
||||
});
|
||||
const events2 = analyzeForGovernanceEvents({
|
||||
tool_name: 'Write',
|
||||
tool_input: { file_path: '.env.local', content: '' },
|
||||
});
|
||||
|
||||
if (events1.length > 0 && events2.length > 0) {
|
||||
assert.notStrictEqual(events1[0].id, events2[0].id);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
// ── run() function ─────────────────────────────────────────
|
||||
|
||||
if (await test('run() passes through input when feature flag is off', async () => {
|
||||
const original = process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
delete process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
|
||||
try {
|
||||
const input = JSON.stringify({ tool_name: 'Bash', tool_input: { command: 'git push --force' } });
|
||||
const result = run(input);
|
||||
assert.strictEqual(result, input);
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = original;
|
||||
}
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('run() passes through input when feature flag is on', async () => {
|
||||
const original = process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = '1';
|
||||
|
||||
try {
|
||||
const input = JSON.stringify({ tool_name: 'Read', tool_input: { file_path: 'index.js' } });
|
||||
const result = run(input);
|
||||
assert.strictEqual(result, input);
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = original;
|
||||
} else {
|
||||
delete process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
}
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('run() handles invalid JSON gracefully', async () => {
|
||||
const original = process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = '1';
|
||||
|
||||
try {
|
||||
const result = run('not valid json');
|
||||
assert.strictEqual(result, 'not valid json');
|
||||
} finally {
|
||||
if (original !== undefined) {
|
||||
process.env.ECC_GOVERNANCE_CAPTURE = original;
|
||||
} else {
|
||||
delete process.env.ECC_GOVERNANCE_CAPTURE;
|
||||
}
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('run() can detect multiple event types in one input', async () => {
|
||||
// Bash command with force push AND secret in command
|
||||
const events = analyzeForGovernanceEvents({
|
||||
tool_name: 'Bash',
|
||||
tool_input: {
|
||||
command: 'API_KEY="AKIAIOSFODNN7EXAMPLE" git push --force',
|
||||
},
|
||||
});
|
||||
|
||||
const eventTypes = events.map(e => e.eventType);
|
||||
assert.ok(eventTypes.includes('secret_detected'));
|
||||
assert.ok(eventTypes.includes('approval_requested'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
293
tests/lib/agent-compress.test.js
Normal file
293
tests/lib/agent-compress.test.js
Normal file
@@ -0,0 +1,293 @@
|
||||
/**
|
||||
* Tests for agent description compression and lazy loading.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
const {
|
||||
parseFrontmatter,
|
||||
extractSummary,
|
||||
loadAgent,
|
||||
loadAgents,
|
||||
compressToCatalog,
|
||||
compressToSummary,
|
||||
buildAgentCatalog,
|
||||
lazyLoadAgent,
|
||||
} = require('../../scripts/lib/agent-compress');
|
||||
|
||||
function createTempDir(prefix) {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function cleanupTempDir(dirPath) {
|
||||
fs.rmSync(dirPath, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
function writeAgent(dir, name, content) {
|
||||
fs.writeFileSync(path.join(dir, `${name}.md`), content, 'utf8');
|
||||
}
|
||||
|
||||
const SAMPLE_AGENT = `---
|
||||
name: test-agent
|
||||
description: A test agent for unit testing purposes.
|
||||
tools: ["Read", "Grep", "Glob"]
|
||||
model: sonnet
|
||||
---
|
||||
|
||||
You are a test agent that validates compression logic.
|
||||
|
||||
## Your Role
|
||||
|
||||
- Run unit tests
|
||||
- Validate compression output
|
||||
- Ensure correctness
|
||||
|
||||
## Process
|
||||
|
||||
### 1. Setup
|
||||
- Prepare test fixtures
|
||||
- Load agent files
|
||||
|
||||
### 2. Validate
|
||||
Check the output format and content.
|
||||
`;
|
||||
|
||||
const MINIMAL_AGENT = `---
|
||||
name: minimal
|
||||
description: Minimal agent.
|
||||
tools: ["Read"]
|
||||
model: haiku
|
||||
---
|
||||
|
||||
Short body.
|
||||
`;
|
||||
|
||||
async function test(name, fn) {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== Testing agent-compress ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
if (await test('parseFrontmatter extracts YAML frontmatter and body', async () => {
|
||||
const { frontmatter, body } = parseFrontmatter(SAMPLE_AGENT);
|
||||
assert.strictEqual(frontmatter.name, 'test-agent');
|
||||
assert.strictEqual(frontmatter.description, 'A test agent for unit testing purposes.');
|
||||
assert.deepStrictEqual(frontmatter.tools, ['Read', 'Grep', 'Glob']);
|
||||
assert.strictEqual(frontmatter.model, 'sonnet');
|
||||
assert.ok(body.includes('You are a test agent'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('parseFrontmatter handles content without frontmatter', async () => {
|
||||
const { frontmatter, body } = parseFrontmatter('Just a plain document.');
|
||||
assert.deepStrictEqual(frontmatter, {});
|
||||
assert.strictEqual(body, 'Just a plain document.');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('extractSummary returns the first paragraph of the body', async () => {
|
||||
const { body } = parseFrontmatter(SAMPLE_AGENT);
|
||||
const summary = extractSummary(body);
|
||||
assert.ok(summary.includes('test agent'));
|
||||
assert.ok(summary.includes('compression logic'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('extractSummary returns empty string for empty body', async () => {
|
||||
assert.strictEqual(extractSummary(''), '');
|
||||
assert.strictEqual(extractSummary('# Just a heading'), '');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('loadAgent reads and parses a single agent file', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
|
||||
assert.strictEqual(agent.name, 'test-agent');
|
||||
assert.strictEqual(agent.fileName, 'test-agent');
|
||||
assert.deepStrictEqual(agent.tools, ['Read', 'Grep', 'Glob']);
|
||||
assert.strictEqual(agent.model, 'sonnet');
|
||||
assert.ok(agent.byteSize > 0);
|
||||
assert.ok(agent.body.includes('You are a test agent'));
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('loadAgents reads all .md files from a directory', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
|
||||
const agents = loadAgents(tmpDir);
|
||||
assert.strictEqual(agents.length, 2);
|
||||
assert.strictEqual(agents[0].fileName, 'agent-a');
|
||||
assert.strictEqual(agents[1].fileName, 'agent-b');
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('loadAgents returns empty array for non-existent directory', async () => {
|
||||
const agents = loadAgents('/tmp/nonexistent-ecc-dir-12345');
|
||||
assert.deepStrictEqual(agents, []);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('compressToCatalog strips body and keeps only metadata', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
|
||||
const catalog = compressToCatalog(agent);
|
||||
|
||||
assert.strictEqual(catalog.name, 'test-agent');
|
||||
assert.strictEqual(catalog.description, 'A test agent for unit testing purposes.');
|
||||
assert.deepStrictEqual(catalog.tools, ['Read', 'Grep', 'Glob']);
|
||||
assert.strictEqual(catalog.model, 'sonnet');
|
||||
assert.strictEqual(catalog.body, undefined);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('compressToSummary includes first paragraph summary', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
const agent = loadAgent(path.join(tmpDir, 'test-agent.md'));
|
||||
const summary = compressToSummary(agent);
|
||||
|
||||
assert.strictEqual(summary.name, 'test-agent');
|
||||
assert.ok(summary.summary.length > 0);
|
||||
assert.strictEqual(summary.body, undefined);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog in catalog mode produces minimal output with stats', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, { mode: 'catalog' });
|
||||
assert.strictEqual(result.agents.length, 2);
|
||||
assert.strictEqual(result.stats.totalAgents, 2);
|
||||
assert.strictEqual(result.stats.mode, 'catalog');
|
||||
assert.ok(result.stats.originalBytes > 0);
|
||||
assert.ok(result.stats.compressedBytes > 0);
|
||||
assert.ok(result.stats.compressedBytes < result.stats.originalBytes);
|
||||
assert.ok(result.stats.compressedTokenEstimate > 0);
|
||||
|
||||
// Catalog entries should not have body
|
||||
for (const agent of result.agents) {
|
||||
assert.strictEqual(agent.body, undefined);
|
||||
assert.ok(agent.name);
|
||||
assert.ok(agent.description);
|
||||
}
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog in summary mode includes summaries', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, { mode: 'summary' });
|
||||
assert.strictEqual(result.agents.length, 1);
|
||||
assert.ok(result.agents[0].summary);
|
||||
assert.strictEqual(result.agents[0].body, undefined);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog in full mode preserves body', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, { mode: 'full' });
|
||||
assert.strictEqual(result.agents.length, 1);
|
||||
assert.ok(result.agents[0].body.includes('You are a test agent'));
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog supports filter function', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT);
|
||||
|
||||
const result = buildAgentCatalog(tmpDir, {
|
||||
mode: 'catalog',
|
||||
filter: agent => agent.model === 'haiku',
|
||||
});
|
||||
assert.strictEqual(result.agents.length, 1);
|
||||
assert.strictEqual(result.agents[0].name, 'minimal');
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('lazyLoadAgent loads a single agent by name', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT);
|
||||
writeAgent(tmpDir, 'other', MINIMAL_AGENT);
|
||||
|
||||
const agent = lazyLoadAgent(tmpDir, 'test-agent');
|
||||
assert.ok(agent);
|
||||
assert.strictEqual(agent.name, 'test-agent');
|
||||
assert.ok(agent.body.includes('You are a test agent'));
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('lazyLoadAgent returns null for non-existent agent', async () => {
|
||||
const tmpDir = createTempDir('ecc-agent-compress-');
|
||||
try {
|
||||
const agent = lazyLoadAgent(tmpDir, 'nonexistent');
|
||||
assert.strictEqual(agent, null);
|
||||
} finally {
|
||||
cleanupTempDir(tmpDir);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('buildAgentCatalog works with real agents directory', async () => {
|
||||
const agentsDir = path.join(__dirname, '..', '..', 'agents');
|
||||
if (!fs.existsSync(agentsDir)) {
|
||||
// Skip if agents dir doesn't exist (shouldn't happen in this repo)
|
||||
return;
|
||||
}
|
||||
|
||||
const result = buildAgentCatalog(agentsDir, { mode: 'catalog' });
|
||||
assert.ok(result.agents.length > 0, 'Should find at least one agent');
|
||||
assert.ok(result.stats.originalBytes > 0);
|
||||
assert.ok(result.stats.compressedBytes < result.stats.originalBytes,
|
||||
'Catalog mode should be smaller than full agent files');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
232
tests/lib/inspection.test.js
Normal file
232
tests/lib/inspection.test.js
Normal file
@@ -0,0 +1,232 @@
|
||||
/**
|
||||
* Tests for inspection logic — pattern detection from failures.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
|
||||
const {
|
||||
normalizeFailureReason,
|
||||
groupFailures,
|
||||
detectPatterns,
|
||||
generateReport,
|
||||
suggestAction,
|
||||
DEFAULT_FAILURE_THRESHOLD,
|
||||
} = require('../../scripts/lib/inspection');
|
||||
|
||||
async function test(name, fn) {
|
||||
try {
|
||||
await fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function makeSkillRun(overrides = {}) {
|
||||
return {
|
||||
id: overrides.id || `run-${Math.random().toString(36).slice(2, 8)}`,
|
||||
skillId: overrides.skillId || 'test-skill',
|
||||
skillVersion: overrides.skillVersion || '1.0.0',
|
||||
sessionId: overrides.sessionId || 'session-1',
|
||||
taskDescription: overrides.taskDescription || 'test task',
|
||||
outcome: overrides.outcome || 'failure',
|
||||
failureReason: overrides.failureReason || 'generic error',
|
||||
tokensUsed: overrides.tokensUsed || 500,
|
||||
durationMs: overrides.durationMs || 1000,
|
||||
userFeedback: overrides.userFeedback || null,
|
||||
createdAt: overrides.createdAt || '2026-03-15T08:00:00.000Z',
|
||||
};
|
||||
}
|
||||
|
||||
async function runTests() {
|
||||
console.log('\n=== Testing inspection ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
if (await test('normalizeFailureReason strips timestamps and UUIDs', async () => {
|
||||
const normalized = normalizeFailureReason(
|
||||
'Error at 2026-03-15T08:00:00.000Z for id 550e8400-e29b-41d4-a716-446655440000'
|
||||
);
|
||||
assert.ok(!normalized.includes('2026'));
|
||||
assert.ok(!normalized.includes('550e8400'));
|
||||
assert.ok(normalized.includes('<timestamp>'));
|
||||
assert.ok(normalized.includes('<uuid>'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('normalizeFailureReason strips file paths', async () => {
|
||||
const normalized = normalizeFailureReason('File not found: /usr/local/bin/node');
|
||||
assert.ok(!normalized.includes('/usr/local'));
|
||||
assert.ok(normalized.includes('<path>'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('normalizeFailureReason handles null and empty values', async () => {
|
||||
assert.strictEqual(normalizeFailureReason(null), 'unknown');
|
||||
assert.strictEqual(normalizeFailureReason(''), 'unknown');
|
||||
assert.strictEqual(normalizeFailureReason(undefined), 'unknown');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('groupFailures groups by skillId and normalized reason', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', skillId: 'skill-a', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r2', skillId: 'skill-a', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r3', skillId: 'skill-b', failureReason: 'parse error' }),
|
||||
makeSkillRun({ id: 'r4', skillId: 'skill-a', outcome: 'success' }), // should be excluded
|
||||
];
|
||||
|
||||
const groups = groupFailures(runs);
|
||||
assert.strictEqual(groups.size, 2);
|
||||
|
||||
const skillAGroup = groups.get('skill-a::timeout');
|
||||
assert.ok(skillAGroup);
|
||||
assert.strictEqual(skillAGroup.runs.length, 2);
|
||||
|
||||
const skillBGroup = groups.get('skill-b::parse error');
|
||||
assert.ok(skillBGroup);
|
||||
assert.strictEqual(skillBGroup.runs.length, 1);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('groupFailures handles mixed outcome casing', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', outcome: 'FAILURE', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r2', outcome: 'Failed', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r3', outcome: 'error', failureReason: 'timeout' }),
|
||||
];
|
||||
|
||||
const groups = groupFailures(runs);
|
||||
assert.strictEqual(groups.size, 1);
|
||||
const group = groups.values().next().value;
|
||||
assert.strictEqual(group.runs.length, 3);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns returns empty array when below threshold', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', failureReason: 'timeout' }),
|
||||
makeSkillRun({ id: 'r2', failureReason: 'timeout' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 0);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns detects patterns at or above threshold', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', failureReason: 'timeout', createdAt: '2026-03-15T08:00:00Z' }),
|
||||
makeSkillRun({ id: 'r2', failureReason: 'timeout', createdAt: '2026-03-15T08:01:00Z' }),
|
||||
makeSkillRun({ id: 'r3', failureReason: 'timeout', createdAt: '2026-03-15T08:02:00Z' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
assert.strictEqual(patterns[0].count, 3);
|
||||
assert.strictEqual(patterns[0].skillId, 'test-skill');
|
||||
assert.strictEqual(patterns[0].normalizedReason, 'timeout');
|
||||
assert.strictEqual(patterns[0].firstSeen, '2026-03-15T08:00:00Z');
|
||||
assert.strictEqual(patterns[0].lastSeen, '2026-03-15T08:02:00Z');
|
||||
assert.strictEqual(patterns[0].runIds.length, 3);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns uses default threshold', async () => {
|
||||
const runs = Array.from({ length: DEFAULT_FAILURE_THRESHOLD }, (_, i) =>
|
||||
makeSkillRun({ id: `r${i}`, failureReason: 'permission denied' })
|
||||
);
|
||||
|
||||
const patterns = detectPatterns(runs);
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns sorts by count descending', async () => {
|
||||
const runs = [
|
||||
// 4 timeouts
|
||||
...Array.from({ length: 4 }, (_, i) =>
|
||||
makeSkillRun({ id: `t${i}`, skillId: 'skill-a', failureReason: 'timeout' })
|
||||
),
|
||||
// 3 parse errors
|
||||
...Array.from({ length: 3 }, (_, i) =>
|
||||
makeSkillRun({ id: `p${i}`, skillId: 'skill-b', failureReason: 'parse error' })
|
||||
),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 2);
|
||||
assert.strictEqual(patterns[0].count, 4);
|
||||
assert.strictEqual(patterns[0].skillId, 'skill-a');
|
||||
assert.strictEqual(patterns[1].count, 3);
|
||||
assert.strictEqual(patterns[1].skillId, 'skill-b');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns groups similar failure reasons with different timestamps', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', failureReason: 'Error at 2026-03-15T08:00:00Z in /tmp/foo' }),
|
||||
makeSkillRun({ id: 'r2', failureReason: 'Error at 2026-03-15T09:00:00Z in /tmp/bar' }),
|
||||
makeSkillRun({ id: 'r3', failureReason: 'Error at 2026-03-15T10:00:00Z in /tmp/baz' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
assert.ok(patterns[0].normalizedReason.includes('<timestamp>'));
|
||||
assert.ok(patterns[0].normalizedReason.includes('<path>'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('detectPatterns tracks unique session IDs and versions', async () => {
|
||||
const runs = [
|
||||
makeSkillRun({ id: 'r1', sessionId: 'sess-1', skillVersion: '1.0.0', failureReason: 'err' }),
|
||||
makeSkillRun({ id: 'r2', sessionId: 'sess-2', skillVersion: '1.0.0', failureReason: 'err' }),
|
||||
makeSkillRun({ id: 'r3', sessionId: 'sess-1', skillVersion: '1.1.0', failureReason: 'err' }),
|
||||
];
|
||||
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
assert.strictEqual(patterns.length, 1);
|
||||
assert.deepStrictEqual(patterns[0].sessionIds.sort(), ['sess-1', 'sess-2']);
|
||||
assert.deepStrictEqual(patterns[0].versions.sort(), ['1.0.0', '1.1.0']);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('generateReport returns clean status with no patterns', async () => {
|
||||
const report = generateReport([]);
|
||||
assert.strictEqual(report.status, 'clean');
|
||||
assert.strictEqual(report.patternCount, 0);
|
||||
assert.ok(report.summary.includes('No recurring'));
|
||||
assert.ok(report.generatedAt);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('generateReport produces structured report from patterns', async () => {
|
||||
const runs = [
|
||||
...Array.from({ length: 3 }, (_, i) =>
|
||||
makeSkillRun({ id: `r${i}`, skillId: 'my-skill', failureReason: 'timeout' })
|
||||
),
|
||||
];
|
||||
const patterns = detectPatterns(runs, { threshold: 3 });
|
||||
const report = generateReport(patterns, { generatedAt: '2026-03-15T09:00:00Z' });
|
||||
|
||||
assert.strictEqual(report.status, 'attention_needed');
|
||||
assert.strictEqual(report.patternCount, 1);
|
||||
assert.strictEqual(report.totalFailures, 3);
|
||||
assert.deepStrictEqual(report.affectedSkills, ['my-skill']);
|
||||
assert.strictEqual(report.patterns[0].skillId, 'my-skill');
|
||||
assert.ok(report.patterns[0].suggestedAction);
|
||||
assert.strictEqual(report.generatedAt, '2026-03-15T09:00:00Z');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('suggestAction returns timeout-specific advice', async () => {
|
||||
const action = suggestAction({ normalizedReason: 'timeout after 30s', versions: ['1.0.0'] });
|
||||
assert.ok(action.toLowerCase().includes('timeout'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('suggestAction returns permission-specific advice', async () => {
|
||||
const action = suggestAction({ normalizedReason: 'permission denied', versions: ['1.0.0'] });
|
||||
assert.ok(action.toLowerCase().includes('permission'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (await test('suggestAction returns version-span advice when multiple versions affected', async () => {
|
||||
const action = suggestAction({ normalizedReason: 'something broke', versions: ['1.0.0', '1.1.0'] });
|
||||
assert.ok(action.toLowerCase().includes('version'));
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
Reference in New Issue
Block a user