From 0b0b66c02fa1d6c341cb20fdc36fb9003d0ee4eb Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Fri, 20 Mar 2026 01:38:13 -0700 Subject: [PATCH] feat: agent compression, inspection logic, governance hooks (#491, #485, #482) (#688) Implements three roadmap features: - Agent description compression (#491): New `agent-compress` module with catalog/summary/full compression modes and lazy-loading. Reduces ~26k token agent descriptions to ~2-3k catalog entries for context efficiency. - Inspection logic (#485): New `inspection` module that detects recurring failure patterns in skill_runs. Groups by skill + normalized failure reason, generates structured reports with suggested remediation actions. Configurable threshold (default: 3 failures). - Governance event capture hook (#482): PreToolUse/PostToolUse hook that detects secrets, policy violations, approval-required commands, and elevated privilege usage. Gated behind ECC_GOVERNANCE_CAPTURE=1 flag. Writes to governance_events table via JSON-line stderr output. 59 new tests (16 + 16 + 27), all passing. --- hooks/hooks.json | 22 ++ scripts/hooks/governance-capture.js | 280 +++++++++++++++++++++++ scripts/lib/agent-compress.js | 230 +++++++++++++++++++ scripts/lib/inspection.js | 212 ++++++++++++++++++ tests/hooks/governance-capture.test.js | 294 +++++++++++++++++++++++++ tests/lib/agent-compress.test.js | 293 ++++++++++++++++++++++++ tests/lib/inspection.test.js | 232 +++++++++++++++++++ 7 files changed, 1563 insertions(+) create mode 100644 scripts/hooks/governance-capture.js create mode 100644 scripts/lib/agent-compress.js create mode 100644 scripts/lib/inspection.js create mode 100644 tests/hooks/governance-capture.test.js create mode 100644 tests/lib/agent-compress.test.js create mode 100644 tests/lib/inspection.test.js diff --git a/hooks/hooks.json b/hooks/hooks.json index 24bc248a..a3d23262 100644 --- a/hooks/hooks.json +++ b/hooks/hooks.json @@ -74,6 +74,17 @@ } ], "description": "Optional InsAIts AI security monitor for Bash/Edit/Write flows. Enable with ECC_ENABLE_INSAITS=1. Requires: pip install insa-its" + }, + { + "matcher": "Bash|Write|Edit|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags.js\" \"pre:governance-capture\" \"scripts/hooks/governance-capture.js\" \"standard,strict\"", + "timeout": 10 + } + ], + "description": "Capture governance events (secrets, policy violations, approval requests). Enable with ECC_GOVERNANCE_CAPTURE=1" } ], "PreCompact": [ @@ -165,6 +176,17 @@ ], "description": "Warn about console.log statements after edits" }, + { + "matcher": "Bash|Write|Edit|MultiEdit", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/scripts/hooks/run-with-flags.js\" \"post:governance-capture\" \"scripts/hooks/governance-capture.js\" \"standard,strict\"", + "timeout": 10 + } + ], + "description": "Capture governance events from tool outputs. Enable with ECC_GOVERNANCE_CAPTURE=1" + }, { "matcher": "*", "hooks": [ diff --git a/scripts/hooks/governance-capture.js b/scripts/hooks/governance-capture.js new file mode 100644 index 00000000..0efec36c --- /dev/null +++ b/scripts/hooks/governance-capture.js @@ -0,0 +1,280 @@ +#!/usr/bin/env node +/** + * Governance Event Capture Hook + * + * PreToolUse/PostToolUse hook that detects governance-relevant events + * and writes them to the governance_events table in the state store. + * + * Captured event types: + * - secret_detected: Hardcoded secrets in tool input/output + * - policy_violation: Actions that violate configured policies + * - security_finding: Security-relevant tool invocations + * - approval_requested: Operations requiring explicit approval + * + * Enable: Set ECC_GOVERNANCE_CAPTURE=1 + * Configure session: Set ECC_SESSION_ID for session correlation + */ + +'use strict'; + +const crypto = require('crypto'); + +const MAX_STDIN = 1024 * 1024; + +// Patterns that indicate potential hardcoded secrets +const SECRET_PATTERNS = [ + { name: 'aws_key', pattern: /(?:AKIA|ASIA)[A-Z0-9]{16}/i }, + { name: 'generic_secret', pattern: /(?:secret|password|token|api[_-]?key)\s*[:=]\s*["'][^"']{8,}/i }, + { name: 'private_key', pattern: /-----BEGIN (?:RSA |EC |DSA )?PRIVATE KEY-----/ }, + { name: 'jwt', pattern: /eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/ }, + { name: 'github_token', pattern: /gh[pousr]_[A-Za-z0-9_]{36,}/ }, +]; + +// Tool names that represent security-relevant operations +const SECURITY_RELEVANT_TOOLS = new Set([ + 'Bash', // Could execute arbitrary commands +]); + +// Commands that require governance approval +const APPROVAL_COMMANDS = [ + /git\s+push\s+.*--force/, + /git\s+reset\s+--hard/, + /rm\s+-rf?\s/, + /DROP\s+(?:TABLE|DATABASE)/i, + /DELETE\s+FROM\s+\w+\s*(?:;|$)/i, +]; + +// File patterns that indicate policy-sensitive paths +const SENSITIVE_PATHS = [ + /\.env(?:\.|$)/, + /credentials/i, + /secrets?\./i, + /\.pem$/, + /\.key$/, + /id_rsa/, +]; + +/** + * Generate a unique event ID. + */ +function generateEventId() { + return `gov-${Date.now()}-${crypto.randomBytes(4).toString('hex')}`; +} + +/** + * Scan text content for hardcoded secrets. + * Returns array of { name, match } for each detected secret. + */ +function detectSecrets(text) { + if (!text || typeof text !== 'string') return []; + + const findings = []; + for (const { name, pattern } of SECRET_PATTERNS) { + if (pattern.test(text)) { + findings.push({ name }); + } + } + return findings; +} + +/** + * Check if a command requires governance approval. + */ +function detectApprovalRequired(command) { + if (!command || typeof command !== 'string') return []; + + const findings = []; + for (const pattern of APPROVAL_COMMANDS) { + if (pattern.test(command)) { + findings.push({ pattern: pattern.source }); + } + } + return findings; +} + +/** + * Check if a file path is policy-sensitive. + */ +function detectSensitivePath(filePath) { + if (!filePath || typeof filePath !== 'string') return false; + + return SENSITIVE_PATHS.some(pattern => pattern.test(filePath)); +} + +/** + * Analyze a hook input payload and return governance events to capture. + * + * @param {Object} input - Parsed hook input (tool_name, tool_input, tool_output) + * @param {Object} [context] - Additional context (sessionId, hookPhase) + * @returns {Array} Array of governance event objects + */ +function analyzeForGovernanceEvents(input, context = {}) { + const events = []; + const toolName = input.tool_name || ''; + const toolInput = input.tool_input || {}; + const toolOutput = typeof input.tool_output === 'string' ? input.tool_output : ''; + const sessionId = context.sessionId || null; + const hookPhase = context.hookPhase || 'unknown'; + + // 1. Secret detection in tool input content + const inputText = typeof toolInput === 'object' + ? JSON.stringify(toolInput) + : String(toolInput); + + const inputSecrets = detectSecrets(inputText); + const outputSecrets = detectSecrets(toolOutput); + const allSecrets = [...inputSecrets, ...outputSecrets]; + + if (allSecrets.length > 0) { + events.push({ + id: generateEventId(), + sessionId, + eventType: 'secret_detected', + payload: { + toolName, + hookPhase, + secretTypes: allSecrets.map(s => s.name), + location: inputSecrets.length > 0 ? 'input' : 'output', + severity: 'critical', + }, + resolvedAt: null, + resolution: null, + }); + } + + // 2. Approval-required commands (Bash only) + if (toolName === 'Bash') { + const command = toolInput.command || ''; + const approvalFindings = detectApprovalRequired(command); + + if (approvalFindings.length > 0) { + events.push({ + id: generateEventId(), + sessionId, + eventType: 'approval_requested', + payload: { + toolName, + hookPhase, + command: command.slice(0, 200), + matchedPatterns: approvalFindings.map(f => f.pattern), + severity: 'high', + }, + resolvedAt: null, + resolution: null, + }); + } + } + + // 3. Policy violation: writing to sensitive paths + const filePath = toolInput.file_path || toolInput.path || ''; + if (filePath && detectSensitivePath(filePath)) { + events.push({ + id: generateEventId(), + sessionId, + eventType: 'policy_violation', + payload: { + toolName, + hookPhase, + filePath: filePath.slice(0, 200), + reason: 'sensitive_file_access', + severity: 'warning', + }, + resolvedAt: null, + resolution: null, + }); + } + + // 4. Security-relevant tool usage tracking + if (SECURITY_RELEVANT_TOOLS.has(toolName) && hookPhase === 'post') { + const command = toolInput.command || ''; + const hasElevated = /sudo\s/.test(command) || /chmod\s/.test(command) || /chown\s/.test(command); + + if (hasElevated) { + events.push({ + id: generateEventId(), + sessionId, + eventType: 'security_finding', + payload: { + toolName, + hookPhase, + command: command.slice(0, 200), + reason: 'elevated_privilege_command', + severity: 'medium', + }, + resolvedAt: null, + resolution: null, + }); + } + } + + return events; +} + +/** + * Core hook logic — exported so run-with-flags.js can call directly. + * + * @param {string} rawInput - Raw JSON string from stdin + * @returns {string} The original input (pass-through) + */ +function run(rawInput) { + // Gate on feature flag + if (String(process.env.ECC_GOVERNANCE_CAPTURE || '').toLowerCase() !== '1') { + return rawInput; + } + + try { + const input = JSON.parse(rawInput); + const sessionId = process.env.ECC_SESSION_ID || null; + const hookPhase = process.env.CLAUDE_HOOK_EVENT_NAME || 'unknown'; + + const events = analyzeForGovernanceEvents(input, { + sessionId, + hookPhase: hookPhase.startsWith('Pre') ? 'pre' : 'post', + }); + + if (events.length > 0) { + // Write events to stderr as JSON-lines for the caller to capture. + // The state store write is async and handled by a separate process + // to avoid blocking the hook pipeline. + for (const event of events) { + process.stderr.write( + `[governance] ${JSON.stringify(event)}\n` + ); + } + } + } catch { + // Silently ignore parse errors — never block the tool pipeline. + } + + return rawInput; +} + +// ── stdin entry point ──────────────────────────────── +if (require.main === module) { + let raw = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', chunk => { + if (raw.length < MAX_STDIN) { + const remaining = MAX_STDIN - raw.length; + raw += chunk.substring(0, remaining); + } + }); + + process.stdin.on('end', () => { + const result = run(raw); + process.stdout.write(result); + }); +} + +module.exports = { + APPROVAL_COMMANDS, + SECRET_PATTERNS, + SECURITY_RELEVANT_TOOLS, + SENSITIVE_PATHS, + analyzeForGovernanceEvents, + detectApprovalRequired, + detectSecrets, + detectSensitivePath, + generateEventId, + run, +}; diff --git a/scripts/lib/agent-compress.js b/scripts/lib/agent-compress.js new file mode 100644 index 00000000..c605a053 --- /dev/null +++ b/scripts/lib/agent-compress.js @@ -0,0 +1,230 @@ +'use strict'; + +const fs = require('fs'); +const path = require('path'); + +/** + * Parse YAML frontmatter from a markdown string. + * Returns { frontmatter: {}, body: string }. + */ +function parseFrontmatter(content) { + const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n([\s\S]*)$/); + if (!match) { + return { frontmatter: {}, body: content }; + } + + const frontmatter = {}; + for (const line of match[1].split('\n')) { + const colonIdx = line.indexOf(':'); + if (colonIdx === -1) continue; + + const key = line.slice(0, colonIdx).trim(); + let value = line.slice(colonIdx + 1).trim(); + + // Handle JSON arrays (e.g. tools: ["Read", "Grep"]) + if (value.startsWith('[') && value.endsWith(']')) { + try { + value = JSON.parse(value); + } catch { + // keep as string + } + } + + // Strip surrounding quotes + if (typeof value === 'string' && value.startsWith('"') && value.endsWith('"')) { + value = value.slice(1, -1); + } + + frontmatter[key] = value; + } + + return { frontmatter, body: match[2] }; +} + +/** + * Extract the first meaningful paragraph from agent body as a summary. + * Skips headings and blank lines, returns up to maxSentences sentences. + */ +function extractSummary(body, maxSentences = 1) { + const lines = body.split('\n'); + const paragraphs = []; + let current = []; + + for (const line of lines) { + const trimmed = line.trim(); + + if (trimmed === '') { + if (current.length > 0) { + paragraphs.push(current.join(' ')); + current = []; + } + continue; + } + + // Skip headings + if (trimmed.startsWith('#')) { + if (current.length > 0) { + paragraphs.push(current.join(' ')); + current = []; + } + continue; + } + + // Skip list items, code blocks, etc. + if (trimmed.startsWith('```') || trimmed.startsWith('- **') || trimmed.startsWith('|')) { + continue; + } + + current.push(trimmed); + } + if (current.length > 0) { + paragraphs.push(current.join(' ')); + } + + // Find first non-empty paragraph + const firstParagraph = paragraphs.find(p => p.length > 0); + if (!firstParagraph) { + return ''; + } + + // Extract up to maxSentences sentences + const sentences = firstParagraph.match(/[^.!?]+[.!?]+/g) || [firstParagraph]; + return sentences.slice(0, maxSentences).join(' ').trim(); +} + +/** + * Load and parse a single agent file. + * Returns the full agent object with frontmatter and body. + */ +function loadAgent(filePath) { + const content = fs.readFileSync(filePath, 'utf8'); + const { frontmatter, body } = parseFrontmatter(content); + const fileName = path.basename(filePath, '.md'); + + return { + fileName, + name: frontmatter.name || fileName, + description: frontmatter.description || '', + tools: Array.isArray(frontmatter.tools) ? frontmatter.tools : [], + model: frontmatter.model || 'sonnet', + body, + byteSize: Buffer.byteLength(content, 'utf8'), + }; +} + +/** + * Load all agents from a directory. + */ +function loadAgents(agentsDir) { + if (!fs.existsSync(agentsDir)) { + return []; + } + + return fs.readdirSync(agentsDir) + .filter(f => f.endsWith('.md')) + .sort() + .map(f => loadAgent(path.join(agentsDir, f))); +} + +/** + * Compress an agent to its catalog entry (metadata only). + * This is the minimal representation needed for agent selection. + */ +function compressToCatalog(agent) { + return { + name: agent.name, + description: agent.description, + tools: agent.tools, + model: agent.model, + }; +} + +/** + * Compress an agent to a summary entry (metadata + first paragraph). + * More context than catalog, less than full body. + */ +function compressToSummary(agent) { + return { + name: agent.name, + description: agent.description, + tools: agent.tools, + model: agent.model, + summary: extractSummary(agent.body), + }; +} + +/** + * Build a full compressed catalog from a directory of agents. + * + * Modes: + * - 'catalog': name, description, tools, model only (~2-3k tokens for 27 agents) + * - 'summary': catalog + first paragraph summary (~4-5k tokens) + * - 'full': no compression, full body included + * + * Returns { agents: [], stats: { totalAgents, originalBytes, compressedTokenEstimate } } + */ +function buildAgentCatalog(agentsDir, options = {}) { + const mode = options.mode || 'catalog'; + const filter = options.filter || null; + + let agents = loadAgents(agentsDir); + + if (typeof filter === 'function') { + agents = agents.filter(filter); + } + + const originalBytes = agents.reduce((sum, a) => sum + a.byteSize, 0); + + let compressed; + if (mode === 'catalog') { + compressed = agents.map(compressToCatalog); + } else if (mode === 'summary') { + compressed = agents.map(compressToSummary); + } else { + compressed = agents.map(a => ({ + name: a.name, + description: a.description, + tools: a.tools, + model: a.model, + body: a.body, + })); + } + + const compressedJson = JSON.stringify(compressed); + // Rough token estimate: ~4 chars per token for English text + const compressedTokenEstimate = Math.ceil(compressedJson.length / 4); + + return { + agents: compressed, + stats: { + totalAgents: agents.length, + originalBytes, + compressedBytes: Buffer.byteLength(compressedJson, 'utf8'), + compressedTokenEstimate, + mode, + }, + }; +} + +/** + * Lazy-load a single agent's full content by name from a directory. + * Returns null if not found. + */ +function lazyLoadAgent(agentsDir, agentName) { + const filePath = path.join(agentsDir, `${agentName}.md`); + if (!fs.existsSync(filePath)) { + return null; + } + return loadAgent(filePath); +} + +module.exports = { + buildAgentCatalog, + compressToCatalog, + compressToSummary, + extractSummary, + lazyLoadAgent, + loadAgent, + loadAgents, + parseFrontmatter, +}; diff --git a/scripts/lib/inspection.js b/scripts/lib/inspection.js new file mode 100644 index 00000000..e6f2cdf3 --- /dev/null +++ b/scripts/lib/inspection.js @@ -0,0 +1,212 @@ +'use strict'; + +const DEFAULT_FAILURE_THRESHOLD = 3; +const DEFAULT_WINDOW_SIZE = 50; + +const FAILURE_OUTCOMES = new Set(['failure', 'failed', 'error']); + +/** + * Normalize a failure reason string for grouping. + * Strips timestamps, UUIDs, file paths, and numeric suffixes. + */ +function normalizeFailureReason(reason) { + if (!reason || typeof reason !== 'string') { + return 'unknown'; + } + + return reason + .trim() + .toLowerCase() + // Strip ISO timestamps (note: already lowercased, so t/z not T/Z) + .replace(/\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}[.\dz]*/g, '') + // Strip UUIDs (already lowercased) + .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, '') + // Strip file paths + .replace(/\/[\w./-]+/g, '') + // Collapse whitespace + .replace(/\s+/g, ' ') + .trim(); +} + +/** + * Group skill runs by skill ID and normalized failure reason. + * + * @param {Array} skillRuns - Array of skill run objects + * @returns {Map} + */ +function groupFailures(skillRuns) { + const groups = new Map(); + + for (const run of skillRuns) { + const outcome = String(run.outcome || '').toLowerCase(); + if (!FAILURE_OUTCOMES.has(outcome)) { + continue; + } + + const normalizedReason = normalizeFailureReason(run.failureReason); + const key = `${run.skillId}::${normalizedReason}`; + + if (!groups.has(key)) { + groups.set(key, { + skillId: run.skillId, + normalizedReason, + runs: [], + }); + } + + groups.get(key).runs.push(run); + } + + return groups; +} + +/** + * Detect recurring failure patterns from skill runs. + * + * @param {Array} skillRuns - Array of skill run objects (newest first) + * @param {Object} [options] + * @param {number} [options.threshold=3] - Minimum failure count to trigger pattern detection + * @returns {Array} Array of detected patterns sorted by count descending + */ +function detectPatterns(skillRuns, options = {}) { + const threshold = options.threshold ?? DEFAULT_FAILURE_THRESHOLD; + const groups = groupFailures(skillRuns); + const patterns = []; + + for (const [, group] of groups) { + if (group.runs.length < threshold) { + continue; + } + + const sortedRuns = [...group.runs].sort( + (a, b) => (b.createdAt || '').localeCompare(a.createdAt || '') + ); + + const firstSeen = sortedRuns[sortedRuns.length - 1].createdAt || null; + const lastSeen = sortedRuns[0].createdAt || null; + const sessionIds = [...new Set(sortedRuns.map(r => r.sessionId).filter(Boolean))]; + const versions = [...new Set(sortedRuns.map(r => r.skillVersion).filter(Boolean))]; + + // Collect unique raw failure reasons for this normalized group + const rawReasons = [...new Set(sortedRuns.map(r => r.failureReason).filter(Boolean))]; + + patterns.push({ + skillId: group.skillId, + normalizedReason: group.normalizedReason, + count: group.runs.length, + firstSeen, + lastSeen, + sessionIds, + versions, + rawReasons, + runIds: sortedRuns.map(r => r.id), + }); + } + + // Sort by count descending, then by lastSeen descending + return patterns.sort((a, b) => { + if (b.count !== a.count) return b.count - a.count; + return (b.lastSeen || '').localeCompare(a.lastSeen || ''); + }); +} + +/** + * Generate an inspection report from detected patterns. + * + * @param {Array} patterns - Output from detectPatterns() + * @param {Object} [options] + * @param {string} [options.generatedAt] - ISO timestamp for the report + * @returns {Object} Inspection report + */ +function generateReport(patterns, options = {}) { + const generatedAt = options.generatedAt || new Date().toISOString(); + + if (patterns.length === 0) { + return { + generatedAt, + status: 'clean', + patternCount: 0, + patterns: [], + summary: 'No recurring failure patterns detected.', + }; + } + + const totalFailures = patterns.reduce((sum, p) => sum + p.count, 0); + const affectedSkills = [...new Set(patterns.map(p => p.skillId))]; + + return { + generatedAt, + status: 'attention_needed', + patternCount: patterns.length, + totalFailures, + affectedSkills, + patterns: patterns.map(p => ({ + skillId: p.skillId, + normalizedReason: p.normalizedReason, + count: p.count, + firstSeen: p.firstSeen, + lastSeen: p.lastSeen, + sessionIds: p.sessionIds, + versions: p.versions, + rawReasons: p.rawReasons.slice(0, 5), + suggestedAction: suggestAction(p), + })), + summary: `Found ${patterns.length} recurring failure pattern(s) across ${affectedSkills.length} skill(s) (${totalFailures} total failures).`, + }; +} + +/** + * Suggest a remediation action based on pattern characteristics. + */ +function suggestAction(pattern) { + const reason = pattern.normalizedReason; + + if (reason.includes('timeout')) { + return 'Increase timeout or optimize skill execution time.'; + } + if (reason.includes('permission') || reason.includes('denied') || reason.includes('auth')) { + return 'Check tool permissions and authentication configuration.'; + } + if (reason.includes('not found') || reason.includes('missing')) { + return 'Verify required files/dependencies exist before skill execution.'; + } + if (reason.includes('parse') || reason.includes('syntax') || reason.includes('json')) { + return 'Review input/output format expectations and add validation.'; + } + if (pattern.versions.length > 1) { + return 'Failure spans multiple versions. Consider rollback to last stable version.'; + } + + return 'Investigate root cause and consider adding error handling.'; +} + +/** + * Run full inspection pipeline: query skill runs, detect patterns, generate report. + * + * @param {Object} store - State store instance with listRecentSessions, getSessionDetail + * @param {Object} [options] + * @param {number} [options.threshold] - Minimum failure count + * @param {number} [options.windowSize] - Number of recent skill runs to analyze + * @returns {Object} Inspection report + */ +function inspect(store, options = {}) { + const windowSize = options.windowSize ?? DEFAULT_WINDOW_SIZE; + const threshold = options.threshold ?? DEFAULT_FAILURE_THRESHOLD; + + const status = store.getStatus({ recentSkillRunLimit: windowSize }); + const skillRuns = status.skillRuns.recent || []; + + const patterns = detectPatterns(skillRuns, { threshold }); + return generateReport(patterns, { generatedAt: status.generatedAt }); +} + +module.exports = { + DEFAULT_FAILURE_THRESHOLD, + DEFAULT_WINDOW_SIZE, + detectPatterns, + generateReport, + groupFailures, + inspect, + normalizeFailureReason, + suggestAction, +}; diff --git a/tests/hooks/governance-capture.test.js b/tests/hooks/governance-capture.test.js new file mode 100644 index 00000000..d7b11e40 --- /dev/null +++ b/tests/hooks/governance-capture.test.js @@ -0,0 +1,294 @@ +/** + * Tests for governance event capture hook. + */ + +const assert = require('assert'); + +const { + detectSecrets, + detectApprovalRequired, + detectSensitivePath, + analyzeForGovernanceEvents, + run, +} = require('../../scripts/hooks/governance-capture'); + +async function test(name, fn) { + try { + await fn(); + console.log(` \u2713 ${name}`); + return true; + } catch (error) { + console.log(` \u2717 ${name}`); + console.log(` Error: ${error.message}`); + return false; + } +} + +async function runTests() { + console.log('\n=== Testing governance-capture ===\n'); + + let passed = 0; + let failed = 0; + + // ── detectSecrets ────────────────────────────────────────── + + if (await test('detectSecrets finds AWS access keys', async () => { + const findings = detectSecrets('my key is AKIAIOSFODNN7EXAMPLE'); + assert.ok(findings.length > 0); + assert.ok(findings.some(f => f.name === 'aws_key')); + })) passed += 1; else failed += 1; + + if (await test('detectSecrets finds generic secrets', async () => { + const findings = detectSecrets('api_key = "sk-proj-abcdefghij1234567890"'); + assert.ok(findings.length > 0); + assert.ok(findings.some(f => f.name === 'generic_secret')); + })) passed += 1; else failed += 1; + + if (await test('detectSecrets finds private keys', async () => { + const findings = detectSecrets('-----BEGIN RSA PRIVATE KEY-----\nMIIE...'); + assert.ok(findings.length > 0); + assert.ok(findings.some(f => f.name === 'private_key')); + })) passed += 1; else failed += 1; + + if (await test('detectSecrets finds GitHub tokens', async () => { + const findings = detectSecrets('token: ghp_ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghij'); + assert.ok(findings.length > 0); + assert.ok(findings.some(f => f.name === 'github_token')); + })) passed += 1; else failed += 1; + + if (await test('detectSecrets returns empty array for clean text', async () => { + const findings = detectSecrets('This is a normal log message with no secrets.'); + assert.strictEqual(findings.length, 0); + })) passed += 1; else failed += 1; + + if (await test('detectSecrets handles null and undefined', async () => { + assert.deepStrictEqual(detectSecrets(null), []); + assert.deepStrictEqual(detectSecrets(undefined), []); + assert.deepStrictEqual(detectSecrets(''), []); + })) passed += 1; else failed += 1; + + // ── detectApprovalRequired ───────────────────────────────── + + if (await test('detectApprovalRequired flags force push', async () => { + const findings = detectApprovalRequired('git push origin main --force'); + assert.ok(findings.length > 0); + })) passed += 1; else failed += 1; + + if (await test('detectApprovalRequired flags hard reset', async () => { + const findings = detectApprovalRequired('git reset --hard HEAD~3'); + assert.ok(findings.length > 0); + })) passed += 1; else failed += 1; + + if (await test('detectApprovalRequired flags rm -rf', async () => { + const findings = detectApprovalRequired('rm -rf /tmp/important'); + assert.ok(findings.length > 0); + })) passed += 1; else failed += 1; + + if (await test('detectApprovalRequired flags DROP TABLE', async () => { + const findings = detectApprovalRequired('DROP TABLE users'); + assert.ok(findings.length > 0); + })) passed += 1; else failed += 1; + + if (await test('detectApprovalRequired allows safe commands', async () => { + const findings = detectApprovalRequired('git status'); + assert.strictEqual(findings.length, 0); + })) passed += 1; else failed += 1; + + if (await test('detectApprovalRequired handles null', async () => { + assert.deepStrictEqual(detectApprovalRequired(null), []); + assert.deepStrictEqual(detectApprovalRequired(''), []); + })) passed += 1; else failed += 1; + + // ── detectSensitivePath ──────────────────────────────────── + + if (await test('detectSensitivePath identifies .env files', async () => { + assert.ok(detectSensitivePath('.env')); + assert.ok(detectSensitivePath('.env.local')); + assert.ok(detectSensitivePath('/project/.env.production')); + })) passed += 1; else failed += 1; + + if (await test('detectSensitivePath identifies credential files', async () => { + assert.ok(detectSensitivePath('credentials.json')); + assert.ok(detectSensitivePath('/home/user/.ssh/id_rsa')); + assert.ok(detectSensitivePath('server.key')); + assert.ok(detectSensitivePath('cert.pem')); + })) passed += 1; else failed += 1; + + if (await test('detectSensitivePath returns false for normal files', async () => { + assert.ok(!detectSensitivePath('index.js')); + assert.ok(!detectSensitivePath('README.md')); + assert.ok(!detectSensitivePath('package.json')); + })) passed += 1; else failed += 1; + + if (await test('detectSensitivePath handles null', async () => { + assert.ok(!detectSensitivePath(null)); + assert.ok(!detectSensitivePath('')); + })) passed += 1; else failed += 1; + + // ── analyzeForGovernanceEvents ───────────────────────────── + + if (await test('analyzeForGovernanceEvents detects secrets in tool input', async () => { + const events = analyzeForGovernanceEvents({ + tool_name: 'Write', + tool_input: { + file_path: '/tmp/config.js', + content: 'const key = "AKIAIOSFODNN7EXAMPLE";', + }, + }); + + assert.ok(events.length > 0); + const secretEvent = events.find(e => e.eventType === 'secret_detected'); + assert.ok(secretEvent); + assert.strictEqual(secretEvent.payload.severity, 'critical'); + })) passed += 1; else failed += 1; + + if (await test('analyzeForGovernanceEvents detects approval-required commands', async () => { + const events = analyzeForGovernanceEvents({ + tool_name: 'Bash', + tool_input: { + command: 'git push origin main --force', + }, + }); + + assert.ok(events.length > 0); + const approvalEvent = events.find(e => e.eventType === 'approval_requested'); + assert.ok(approvalEvent); + assert.strictEqual(approvalEvent.payload.severity, 'high'); + })) passed += 1; else failed += 1; + + if (await test('analyzeForGovernanceEvents detects sensitive file access', async () => { + const events = analyzeForGovernanceEvents({ + tool_name: 'Edit', + tool_input: { + file_path: '/project/.env.production', + old_string: 'DB_URL=old', + new_string: 'DB_URL=new', + }, + }); + + assert.ok(events.length > 0); + const policyEvent = events.find(e => e.eventType === 'policy_violation'); + assert.ok(policyEvent); + assert.strictEqual(policyEvent.payload.reason, 'sensitive_file_access'); + })) passed += 1; else failed += 1; + + if (await test('analyzeForGovernanceEvents detects elevated privilege commands', async () => { + const events = analyzeForGovernanceEvents({ + tool_name: 'Bash', + tool_input: { command: 'sudo rm -rf /etc/something' }, + }, { + hookPhase: 'post', + }); + + const securityEvent = events.find(e => e.eventType === 'security_finding'); + assert.ok(securityEvent); + assert.strictEqual(securityEvent.payload.reason, 'elevated_privilege_command'); + })) passed += 1; else failed += 1; + + if (await test('analyzeForGovernanceEvents returns empty for clean inputs', async () => { + const events = analyzeForGovernanceEvents({ + tool_name: 'Read', + tool_input: { file_path: '/project/src/index.js' }, + }); + assert.strictEqual(events.length, 0); + })) passed += 1; else failed += 1; + + if (await test('analyzeForGovernanceEvents populates session ID from context', async () => { + const events = analyzeForGovernanceEvents({ + tool_name: 'Write', + tool_input: { + file_path: '/project/.env', + content: 'DB_URL=test', + }, + }, { + sessionId: 'test-session-123', + }); + + assert.ok(events.length > 0); + assert.strictEqual(events[0].sessionId, 'test-session-123'); + })) passed += 1; else failed += 1; + + if (await test('analyzeForGovernanceEvents generates unique event IDs', async () => { + const events1 = analyzeForGovernanceEvents({ + tool_name: 'Write', + tool_input: { file_path: '.env', content: '' }, + }); + const events2 = analyzeForGovernanceEvents({ + tool_name: 'Write', + tool_input: { file_path: '.env.local', content: '' }, + }); + + if (events1.length > 0 && events2.length > 0) { + assert.notStrictEqual(events1[0].id, events2[0].id); + } + })) passed += 1; else failed += 1; + + // ── run() function ───────────────────────────────────────── + + if (await test('run() passes through input when feature flag is off', async () => { + const original = process.env.ECC_GOVERNANCE_CAPTURE; + delete process.env.ECC_GOVERNANCE_CAPTURE; + + try { + const input = JSON.stringify({ tool_name: 'Bash', tool_input: { command: 'git push --force' } }); + const result = run(input); + assert.strictEqual(result, input); + } finally { + if (original !== undefined) { + process.env.ECC_GOVERNANCE_CAPTURE = original; + } + } + })) passed += 1; else failed += 1; + + if (await test('run() passes through input when feature flag is on', async () => { + const original = process.env.ECC_GOVERNANCE_CAPTURE; + process.env.ECC_GOVERNANCE_CAPTURE = '1'; + + try { + const input = JSON.stringify({ tool_name: 'Read', tool_input: { file_path: 'index.js' } }); + const result = run(input); + assert.strictEqual(result, input); + } finally { + if (original !== undefined) { + process.env.ECC_GOVERNANCE_CAPTURE = original; + } else { + delete process.env.ECC_GOVERNANCE_CAPTURE; + } + } + })) passed += 1; else failed += 1; + + if (await test('run() handles invalid JSON gracefully', async () => { + const original = process.env.ECC_GOVERNANCE_CAPTURE; + process.env.ECC_GOVERNANCE_CAPTURE = '1'; + + try { + const result = run('not valid json'); + assert.strictEqual(result, 'not valid json'); + } finally { + if (original !== undefined) { + process.env.ECC_GOVERNANCE_CAPTURE = original; + } else { + delete process.env.ECC_GOVERNANCE_CAPTURE; + } + } + })) passed += 1; else failed += 1; + + if (await test('run() can detect multiple event types in one input', async () => { + // Bash command with force push AND secret in command + const events = analyzeForGovernanceEvents({ + tool_name: 'Bash', + tool_input: { + command: 'API_KEY="AKIAIOSFODNN7EXAMPLE" git push --force', + }, + }); + + const eventTypes = events.map(e => e.eventType); + assert.ok(eventTypes.includes('secret_detected')); + assert.ok(eventTypes.includes('approval_requested')); + })) passed += 1; else failed += 1; + + console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); + process.exit(failed > 0 ? 1 : 0); +} + +runTests(); diff --git a/tests/lib/agent-compress.test.js b/tests/lib/agent-compress.test.js new file mode 100644 index 00000000..45da6668 --- /dev/null +++ b/tests/lib/agent-compress.test.js @@ -0,0 +1,293 @@ +/** + * Tests for agent description compression and lazy loading. + */ + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + parseFrontmatter, + extractSummary, + loadAgent, + loadAgents, + compressToCatalog, + compressToSummary, + buildAgentCatalog, + lazyLoadAgent, +} = require('../../scripts/lib/agent-compress'); + +function createTempDir(prefix) { + return fs.mkdtempSync(path.join(os.tmpdir(), prefix)); +} + +function cleanupTempDir(dirPath) { + fs.rmSync(dirPath, { recursive: true, force: true }); +} + +function writeAgent(dir, name, content) { + fs.writeFileSync(path.join(dir, `${name}.md`), content, 'utf8'); +} + +const SAMPLE_AGENT = `--- +name: test-agent +description: A test agent for unit testing purposes. +tools: ["Read", "Grep", "Glob"] +model: sonnet +--- + +You are a test agent that validates compression logic. + +## Your Role + +- Run unit tests +- Validate compression output +- Ensure correctness + +## Process + +### 1. Setup +- Prepare test fixtures +- Load agent files + +### 2. Validate +Check the output format and content. +`; + +const MINIMAL_AGENT = `--- +name: minimal +description: Minimal agent. +tools: ["Read"] +model: haiku +--- + +Short body. +`; + +async function test(name, fn) { + try { + await fn(); + console.log(` \u2713 ${name}`); + return true; + } catch (error) { + console.log(` \u2717 ${name}`); + console.log(` Error: ${error.message}`); + return false; + } +} + +async function runTests() { + console.log('\n=== Testing agent-compress ===\n'); + + let passed = 0; + let failed = 0; + + if (await test('parseFrontmatter extracts YAML frontmatter and body', async () => { + const { frontmatter, body } = parseFrontmatter(SAMPLE_AGENT); + assert.strictEqual(frontmatter.name, 'test-agent'); + assert.strictEqual(frontmatter.description, 'A test agent for unit testing purposes.'); + assert.deepStrictEqual(frontmatter.tools, ['Read', 'Grep', 'Glob']); + assert.strictEqual(frontmatter.model, 'sonnet'); + assert.ok(body.includes('You are a test agent')); + })) passed += 1; else failed += 1; + + if (await test('parseFrontmatter handles content without frontmatter', async () => { + const { frontmatter, body } = parseFrontmatter('Just a plain document.'); + assert.deepStrictEqual(frontmatter, {}); + assert.strictEqual(body, 'Just a plain document.'); + })) passed += 1; else failed += 1; + + if (await test('extractSummary returns the first paragraph of the body', async () => { + const { body } = parseFrontmatter(SAMPLE_AGENT); + const summary = extractSummary(body); + assert.ok(summary.includes('test agent')); + assert.ok(summary.includes('compression logic')); + })) passed += 1; else failed += 1; + + if (await test('extractSummary returns empty string for empty body', async () => { + assert.strictEqual(extractSummary(''), ''); + assert.strictEqual(extractSummary('# Just a heading'), ''); + })) passed += 1; else failed += 1; + + if (await test('loadAgent reads and parses a single agent file', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT); + const agent = loadAgent(path.join(tmpDir, 'test-agent.md')); + assert.strictEqual(agent.name, 'test-agent'); + assert.strictEqual(agent.fileName, 'test-agent'); + assert.deepStrictEqual(agent.tools, ['Read', 'Grep', 'Glob']); + assert.strictEqual(agent.model, 'sonnet'); + assert.ok(agent.byteSize > 0); + assert.ok(agent.body.includes('You are a test agent')); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('loadAgents reads all .md files from a directory', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT); + writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT); + const agents = loadAgents(tmpDir); + assert.strictEqual(agents.length, 2); + assert.strictEqual(agents[0].fileName, 'agent-a'); + assert.strictEqual(agents[1].fileName, 'agent-b'); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('loadAgents returns empty array for non-existent directory', async () => { + const agents = loadAgents('/tmp/nonexistent-ecc-dir-12345'); + assert.deepStrictEqual(agents, []); + })) passed += 1; else failed += 1; + + if (await test('compressToCatalog strips body and keeps only metadata', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT); + const agent = loadAgent(path.join(tmpDir, 'test-agent.md')); + const catalog = compressToCatalog(agent); + + assert.strictEqual(catalog.name, 'test-agent'); + assert.strictEqual(catalog.description, 'A test agent for unit testing purposes.'); + assert.deepStrictEqual(catalog.tools, ['Read', 'Grep', 'Glob']); + assert.strictEqual(catalog.model, 'sonnet'); + assert.strictEqual(catalog.body, undefined); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('compressToSummary includes first paragraph summary', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT); + const agent = loadAgent(path.join(tmpDir, 'test-agent.md')); + const summary = compressToSummary(agent); + + assert.strictEqual(summary.name, 'test-agent'); + assert.ok(summary.summary.length > 0); + assert.strictEqual(summary.body, undefined); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('buildAgentCatalog in catalog mode produces minimal output with stats', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT); + writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT); + + const result = buildAgentCatalog(tmpDir, { mode: 'catalog' }); + assert.strictEqual(result.agents.length, 2); + assert.strictEqual(result.stats.totalAgents, 2); + assert.strictEqual(result.stats.mode, 'catalog'); + assert.ok(result.stats.originalBytes > 0); + assert.ok(result.stats.compressedBytes > 0); + assert.ok(result.stats.compressedBytes < result.stats.originalBytes); + assert.ok(result.stats.compressedTokenEstimate > 0); + + // Catalog entries should not have body + for (const agent of result.agents) { + assert.strictEqual(agent.body, undefined); + assert.ok(agent.name); + assert.ok(agent.description); + } + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('buildAgentCatalog in summary mode includes summaries', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT); + + const result = buildAgentCatalog(tmpDir, { mode: 'summary' }); + assert.strictEqual(result.agents.length, 1); + assert.ok(result.agents[0].summary); + assert.strictEqual(result.agents[0].body, undefined); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('buildAgentCatalog in full mode preserves body', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT); + + const result = buildAgentCatalog(tmpDir, { mode: 'full' }); + assert.strictEqual(result.agents.length, 1); + assert.ok(result.agents[0].body.includes('You are a test agent')); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('buildAgentCatalog supports filter function', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'agent-a', SAMPLE_AGENT); + writeAgent(tmpDir, 'agent-b', MINIMAL_AGENT); + + const result = buildAgentCatalog(tmpDir, { + mode: 'catalog', + filter: agent => agent.model === 'haiku', + }); + assert.strictEqual(result.agents.length, 1); + assert.strictEqual(result.agents[0].name, 'minimal'); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('lazyLoadAgent loads a single agent by name', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + writeAgent(tmpDir, 'test-agent', SAMPLE_AGENT); + writeAgent(tmpDir, 'other', MINIMAL_AGENT); + + const agent = lazyLoadAgent(tmpDir, 'test-agent'); + assert.ok(agent); + assert.strictEqual(agent.name, 'test-agent'); + assert.ok(agent.body.includes('You are a test agent')); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('lazyLoadAgent returns null for non-existent agent', async () => { + const tmpDir = createTempDir('ecc-agent-compress-'); + try { + const agent = lazyLoadAgent(tmpDir, 'nonexistent'); + assert.strictEqual(agent, null); + } finally { + cleanupTempDir(tmpDir); + } + })) passed += 1; else failed += 1; + + if (await test('buildAgentCatalog works with real agents directory', async () => { + const agentsDir = path.join(__dirname, '..', '..', 'agents'); + if (!fs.existsSync(agentsDir)) { + // Skip if agents dir doesn't exist (shouldn't happen in this repo) + return; + } + + const result = buildAgentCatalog(agentsDir, { mode: 'catalog' }); + assert.ok(result.agents.length > 0, 'Should find at least one agent'); + assert.ok(result.stats.originalBytes > 0); + assert.ok(result.stats.compressedBytes < result.stats.originalBytes, + 'Catalog mode should be smaller than full agent files'); + })) passed += 1; else failed += 1; + + console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); + process.exit(failed > 0 ? 1 : 0); +} + +runTests(); diff --git a/tests/lib/inspection.test.js b/tests/lib/inspection.test.js new file mode 100644 index 00000000..f8725f37 --- /dev/null +++ b/tests/lib/inspection.test.js @@ -0,0 +1,232 @@ +/** + * Tests for inspection logic — pattern detection from failures. + */ + +const assert = require('assert'); + +const { + normalizeFailureReason, + groupFailures, + detectPatterns, + generateReport, + suggestAction, + DEFAULT_FAILURE_THRESHOLD, +} = require('../../scripts/lib/inspection'); + +async function test(name, fn) { + try { + await fn(); + console.log(` \u2713 ${name}`); + return true; + } catch (error) { + console.log(` \u2717 ${name}`); + console.log(` Error: ${error.message}`); + return false; + } +} + +function makeSkillRun(overrides = {}) { + return { + id: overrides.id || `run-${Math.random().toString(36).slice(2, 8)}`, + skillId: overrides.skillId || 'test-skill', + skillVersion: overrides.skillVersion || '1.0.0', + sessionId: overrides.sessionId || 'session-1', + taskDescription: overrides.taskDescription || 'test task', + outcome: overrides.outcome || 'failure', + failureReason: overrides.failureReason || 'generic error', + tokensUsed: overrides.tokensUsed || 500, + durationMs: overrides.durationMs || 1000, + userFeedback: overrides.userFeedback || null, + createdAt: overrides.createdAt || '2026-03-15T08:00:00.000Z', + }; +} + +async function runTests() { + console.log('\n=== Testing inspection ===\n'); + + let passed = 0; + let failed = 0; + + if (await test('normalizeFailureReason strips timestamps and UUIDs', async () => { + const normalized = normalizeFailureReason( + 'Error at 2026-03-15T08:00:00.000Z for id 550e8400-e29b-41d4-a716-446655440000' + ); + assert.ok(!normalized.includes('2026')); + assert.ok(!normalized.includes('550e8400')); + assert.ok(normalized.includes('')); + assert.ok(normalized.includes('')); + })) passed += 1; else failed += 1; + + if (await test('normalizeFailureReason strips file paths', async () => { + const normalized = normalizeFailureReason('File not found: /usr/local/bin/node'); + assert.ok(!normalized.includes('/usr/local')); + assert.ok(normalized.includes('')); + })) passed += 1; else failed += 1; + + if (await test('normalizeFailureReason handles null and empty values', async () => { + assert.strictEqual(normalizeFailureReason(null), 'unknown'); + assert.strictEqual(normalizeFailureReason(''), 'unknown'); + assert.strictEqual(normalizeFailureReason(undefined), 'unknown'); + })) passed += 1; else failed += 1; + + if (await test('groupFailures groups by skillId and normalized reason', async () => { + const runs = [ + makeSkillRun({ id: 'r1', skillId: 'skill-a', failureReason: 'timeout' }), + makeSkillRun({ id: 'r2', skillId: 'skill-a', failureReason: 'timeout' }), + makeSkillRun({ id: 'r3', skillId: 'skill-b', failureReason: 'parse error' }), + makeSkillRun({ id: 'r4', skillId: 'skill-a', outcome: 'success' }), // should be excluded + ]; + + const groups = groupFailures(runs); + assert.strictEqual(groups.size, 2); + + const skillAGroup = groups.get('skill-a::timeout'); + assert.ok(skillAGroup); + assert.strictEqual(skillAGroup.runs.length, 2); + + const skillBGroup = groups.get('skill-b::parse error'); + assert.ok(skillBGroup); + assert.strictEqual(skillBGroup.runs.length, 1); + })) passed += 1; else failed += 1; + + if (await test('groupFailures handles mixed outcome casing', async () => { + const runs = [ + makeSkillRun({ id: 'r1', outcome: 'FAILURE', failureReason: 'timeout' }), + makeSkillRun({ id: 'r2', outcome: 'Failed', failureReason: 'timeout' }), + makeSkillRun({ id: 'r3', outcome: 'error', failureReason: 'timeout' }), + ]; + + const groups = groupFailures(runs); + assert.strictEqual(groups.size, 1); + const group = groups.values().next().value; + assert.strictEqual(group.runs.length, 3); + })) passed += 1; else failed += 1; + + if (await test('detectPatterns returns empty array when below threshold', async () => { + const runs = [ + makeSkillRun({ id: 'r1', failureReason: 'timeout' }), + makeSkillRun({ id: 'r2', failureReason: 'timeout' }), + ]; + + const patterns = detectPatterns(runs, { threshold: 3 }); + assert.strictEqual(patterns.length, 0); + })) passed += 1; else failed += 1; + + if (await test('detectPatterns detects patterns at or above threshold', async () => { + const runs = [ + makeSkillRun({ id: 'r1', failureReason: 'timeout', createdAt: '2026-03-15T08:00:00Z' }), + makeSkillRun({ id: 'r2', failureReason: 'timeout', createdAt: '2026-03-15T08:01:00Z' }), + makeSkillRun({ id: 'r3', failureReason: 'timeout', createdAt: '2026-03-15T08:02:00Z' }), + ]; + + const patterns = detectPatterns(runs, { threshold: 3 }); + assert.strictEqual(patterns.length, 1); + assert.strictEqual(patterns[0].count, 3); + assert.strictEqual(patterns[0].skillId, 'test-skill'); + assert.strictEqual(patterns[0].normalizedReason, 'timeout'); + assert.strictEqual(patterns[0].firstSeen, '2026-03-15T08:00:00Z'); + assert.strictEqual(patterns[0].lastSeen, '2026-03-15T08:02:00Z'); + assert.strictEqual(patterns[0].runIds.length, 3); + })) passed += 1; else failed += 1; + + if (await test('detectPatterns uses default threshold', async () => { + const runs = Array.from({ length: DEFAULT_FAILURE_THRESHOLD }, (_, i) => + makeSkillRun({ id: `r${i}`, failureReason: 'permission denied' }) + ); + + const patterns = detectPatterns(runs); + assert.strictEqual(patterns.length, 1); + })) passed += 1; else failed += 1; + + if (await test('detectPatterns sorts by count descending', async () => { + const runs = [ + // 4 timeouts + ...Array.from({ length: 4 }, (_, i) => + makeSkillRun({ id: `t${i}`, skillId: 'skill-a', failureReason: 'timeout' }) + ), + // 3 parse errors + ...Array.from({ length: 3 }, (_, i) => + makeSkillRun({ id: `p${i}`, skillId: 'skill-b', failureReason: 'parse error' }) + ), + ]; + + const patterns = detectPatterns(runs, { threshold: 3 }); + assert.strictEqual(patterns.length, 2); + assert.strictEqual(patterns[0].count, 4); + assert.strictEqual(patterns[0].skillId, 'skill-a'); + assert.strictEqual(patterns[1].count, 3); + assert.strictEqual(patterns[1].skillId, 'skill-b'); + })) passed += 1; else failed += 1; + + if (await test('detectPatterns groups similar failure reasons with different timestamps', async () => { + const runs = [ + makeSkillRun({ id: 'r1', failureReason: 'Error at 2026-03-15T08:00:00Z in /tmp/foo' }), + makeSkillRun({ id: 'r2', failureReason: 'Error at 2026-03-15T09:00:00Z in /tmp/bar' }), + makeSkillRun({ id: 'r3', failureReason: 'Error at 2026-03-15T10:00:00Z in /tmp/baz' }), + ]; + + const patterns = detectPatterns(runs, { threshold: 3 }); + assert.strictEqual(patterns.length, 1); + assert.ok(patterns[0].normalizedReason.includes('')); + assert.ok(patterns[0].normalizedReason.includes('')); + })) passed += 1; else failed += 1; + + if (await test('detectPatterns tracks unique session IDs and versions', async () => { + const runs = [ + makeSkillRun({ id: 'r1', sessionId: 'sess-1', skillVersion: '1.0.0', failureReason: 'err' }), + makeSkillRun({ id: 'r2', sessionId: 'sess-2', skillVersion: '1.0.0', failureReason: 'err' }), + makeSkillRun({ id: 'r3', sessionId: 'sess-1', skillVersion: '1.1.0', failureReason: 'err' }), + ]; + + const patterns = detectPatterns(runs, { threshold: 3 }); + assert.strictEqual(patterns.length, 1); + assert.deepStrictEqual(patterns[0].sessionIds.sort(), ['sess-1', 'sess-2']); + assert.deepStrictEqual(patterns[0].versions.sort(), ['1.0.0', '1.1.0']); + })) passed += 1; else failed += 1; + + if (await test('generateReport returns clean status with no patterns', async () => { + const report = generateReport([]); + assert.strictEqual(report.status, 'clean'); + assert.strictEqual(report.patternCount, 0); + assert.ok(report.summary.includes('No recurring')); + assert.ok(report.generatedAt); + })) passed += 1; else failed += 1; + + if (await test('generateReport produces structured report from patterns', async () => { + const runs = [ + ...Array.from({ length: 3 }, (_, i) => + makeSkillRun({ id: `r${i}`, skillId: 'my-skill', failureReason: 'timeout' }) + ), + ]; + const patterns = detectPatterns(runs, { threshold: 3 }); + const report = generateReport(patterns, { generatedAt: '2026-03-15T09:00:00Z' }); + + assert.strictEqual(report.status, 'attention_needed'); + assert.strictEqual(report.patternCount, 1); + assert.strictEqual(report.totalFailures, 3); + assert.deepStrictEqual(report.affectedSkills, ['my-skill']); + assert.strictEqual(report.patterns[0].skillId, 'my-skill'); + assert.ok(report.patterns[0].suggestedAction); + assert.strictEqual(report.generatedAt, '2026-03-15T09:00:00Z'); + })) passed += 1; else failed += 1; + + if (await test('suggestAction returns timeout-specific advice', async () => { + const action = suggestAction({ normalizedReason: 'timeout after 30s', versions: ['1.0.0'] }); + assert.ok(action.toLowerCase().includes('timeout')); + })) passed += 1; else failed += 1; + + if (await test('suggestAction returns permission-specific advice', async () => { + const action = suggestAction({ normalizedReason: 'permission denied', versions: ['1.0.0'] }); + assert.ok(action.toLowerCase().includes('permission')); + })) passed += 1; else failed += 1; + + if (await test('suggestAction returns version-span advice when multiple versions affected', async () => { + const action = suggestAction({ normalizedReason: 'something broke', versions: ['1.0.0', '1.1.0'] }); + assert.ok(action.toLowerCase().includes('version')); + })) passed += 1; else failed += 1; + + console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`); + process.exit(failed > 0 ? 1 : 0); +} + +runTests();