2 Commits

Author SHA1 Message Date
Affaan Mustafa
c53bba9e02 feat: self-improving skills loop — observe, inspect, amend, evaluate
- Add skill health observation layer (execution logging, success/failure tracking)
- Add skill health inspector (trace recurring failures across runs)
- Add amendify mechanism (propose SKILL.md patches from failure evidence)
- Add evaluation scaffolding (compare amended vs original performance)
- Wire into session-inspect CLI: skills:health, skills:amendify, skills:evaluate
- 1145/1145 tests passing (+3 new)
2026-03-14 23:21:18 -07:00
Affaan Mustafa
2b2777915e feat: expand session adapter registry with structured targets
- Registry accepts { type, value } structured targets
- Add --list-adapters and --target-type CLI flags to session-inspect
- Export adapter type from claude-history and dmux-tmux adapters
- 71 new session adapter tests, 34 new session-inspect tests
- All 1142 tests passing
2026-03-14 19:09:26 -07:00
12 changed files with 970 additions and 26 deletions

36
package-lock.json generated
View File

@@ -10,6 +10,7 @@
"hasInstallScript": true,
"license": "MIT",
"bin": {
"ecc": "scripts/ecc.js",
"ecc-install": "install.sh"
},
"devDependencies": {
@@ -18,7 +19,7 @@
"c8": "^10.1.2",
"eslint": "^9.39.2",
"globals": "^17.1.0",
"markdownlint-cli": "^0.48.0"
"markdownlint-cli": "^0.47.0"
},
"engines": {
"node": ">=18"
@@ -1655,22 +1656,23 @@
}
},
"node_modules/markdownlint-cli": {
"version": "0.48.0",
"resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz",
"integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==",
"version": "0.47.0",
"resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz",
"integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==",
"dev": true,
"license": "MIT",
"dependencies": {
"commander": "~14.0.3",
"commander": "~14.0.2",
"deep-extend": "~0.6.0",
"ignore": "~7.0.5",
"js-yaml": "~4.1.1",
"jsonc-parser": "~3.3.1",
"jsonpointer": "~5.0.1",
"markdown-it": "~14.1.1",
"markdown-it": "~14.1.0",
"markdownlint": "~0.40.0",
"minimatch": "~10.2.4",
"minimatch": "~10.1.1",
"run-con": "~1.3.2",
"smol-toml": "~1.6.0",
"smol-toml": "~1.5.2",
"tinyglobby": "~0.2.15"
},
"bin": {
@@ -1685,6 +1687,7 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
"integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
"dev": true,
"license": "MIT",
"engines": {
"node": "18 || 20 || >=22"
}
@@ -1694,6 +1697,7 @@
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz",
"integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^4.0.2"
},
@@ -1712,15 +1716,16 @@
}
},
"node_modules/markdownlint-cli/node_modules/minimatch": {
"version": "10.2.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
"version": "10.1.3",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.3.tgz",
"integrity": "sha512-IF6URNyBX7Z6XfvjpaNy5meRxPZiIf2OqtOoSLs+hLJ9pJAScnM1RjrFcbCaD85y42KcI+oZmKjFIJKYDFjQfg==",
"dev": true,
"license": "BlueOak-1.0.0",
"dependencies": {
"brace-expansion": "^5.0.2"
},
"engines": {
"node": "18 || 20 || >=22"
"node": "20 || >=22"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
@@ -2579,10 +2584,11 @@
}
},
"node_modules/smol-toml": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz",
"integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==",
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz",
"integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==",
"dev": true,
"license": "BSD-3-Clause",
"engines": {
"node": ">= 18"
},

View File

@@ -114,6 +114,8 @@ function resolveSessionRecord(target, cwd) {
function createClaudeHistoryAdapter() {
return {
id: 'claude-history',
description: 'Claude local session history and session-file snapshots',
targetTypes: ['claude-history', 'claude-alias', 'session-file'],
canOpen(target, context = {}) {
if (context.adapterId && context.adapterId !== 'claude-history') {
return false;

View File

@@ -45,6 +45,8 @@ function createDmuxTmuxAdapter(options = {}) {
return {
id: 'dmux-tmux',
description: 'Tmux/worktree orchestration snapshots from plan files or session names',
targetTypes: ['plan', 'session'],
canOpen(target, context = {}) {
if (context.adapterId && context.adapterId !== 'dmux-tmux') {
return false;

View File

@@ -3,6 +3,14 @@
const { createClaudeHistoryAdapter } = require('./claude-history');
const { createDmuxTmuxAdapter } = require('./dmux-tmux');
const TARGET_TYPE_TO_ADAPTER_ID = Object.freeze({
plan: 'dmux-tmux',
session: 'dmux-tmux',
'claude-history': 'claude-history',
'claude-alias': 'claude-history',
'session-file': 'claude-history'
});
function createDefaultAdapters() {
return [
createClaudeHistoryAdapter(),
@@ -10,13 +18,72 @@ function createDefaultAdapters() {
];
}
function coerceTargetValue(value) {
if (typeof value !== 'string' || value.trim().length === 0) {
throw new Error('Structured session targets require a non-empty string value');
}
return value.trim();
}
function normalizeStructuredTarget(target, context = {}) {
if (!target || typeof target !== 'object' || Array.isArray(target)) {
return {
target,
context: { ...context }
};
}
const value = coerceTargetValue(target.value);
const type = typeof target.type === 'string' ? target.type.trim() : '';
if (type.length === 0) {
throw new Error('Structured session targets require a non-empty type');
}
const adapterId = target.adapterId || TARGET_TYPE_TO_ADAPTER_ID[type] || context.adapterId || null;
const nextContext = {
...context,
adapterId
};
if (type === 'claude-history' || type === 'claude-alias') {
return {
target: `claude:${value}`,
context: nextContext
};
}
return {
target: value,
context: nextContext
};
}
function createAdapterRegistry(options = {}) {
const adapters = options.adapters || createDefaultAdapters();
return {
adapters,
getAdapter(id) {
const adapter = adapters.find(candidate => candidate.id === id);
if (!adapter) {
throw new Error(`Unknown session adapter: ${id}`);
}
return adapter;
},
listAdapters() {
return adapters.map(adapter => ({
id: adapter.id,
description: adapter.description || '',
targetTypes: Array.isArray(adapter.targetTypes) ? [...adapter.targetTypes] : []
}));
},
select(target, context = {}) {
const adapter = adapters.find(candidate => candidate.canOpen(target, context));
const normalized = normalizeStructuredTarget(target, context);
const adapter = normalized.context.adapterId
? this.getAdapter(normalized.context.adapterId)
: adapters.find(candidate => candidate.canOpen(normalized.target, normalized.context));
if (!adapter) {
throw new Error(`No session adapter matched target: ${target}`);
}
@@ -24,8 +91,9 @@ function createAdapterRegistry(options = {}) {
return adapter;
},
open(target, context = {}) {
const adapter = this.select(target, context);
return adapter.open(target, context);
const normalized = normalizeStructuredTarget(target, context);
const adapter = this.select(normalized.target, normalized.context);
return adapter.open(normalized.target, normalized.context);
}
};
}
@@ -38,5 +106,6 @@ function inspectSessionTarget(target, options = {}) {
module.exports = {
createAdapterRegistry,
createDefaultAdapters,
inspectSessionTarget
inspectSessionTarget,
normalizeStructuredTarget
};

View File

@@ -0,0 +1,89 @@
'use strict';
const { buildSkillHealthReport } = require('./health');
const AMENDMENT_SCHEMA_VERSION = 'ecc.skill-amendment-proposal.v1';
function createProposalId(skillId) {
return `amend-${skillId}-${Date.now()}`;
}
function summarizePatchPreview(skillId, health) {
const lines = [
'## Failure-Driven Amendments',
'',
`- Focus skill routing for \`${skillId}\` when tasks match the proven success cases.`,
];
if (health.recurringErrors[0]) {
lines.push(`- Add explicit guardrails for recurring failure: ${health.recurringErrors[0].error}.`);
}
if (health.recurringTasks[0]) {
lines.push(`- Add an example workflow for task pattern: ${health.recurringTasks[0].task}.`);
}
if (health.recurringFeedback[0]) {
lines.push(`- Address repeated user feedback: ${health.recurringFeedback[0].feedback}.`);
}
lines.push('- Add a verification checklist before declaring the skill output complete.');
return lines.join('\n');
}
function proposeSkillAmendment(skillId, records, options = {}) {
const report = buildSkillHealthReport(records, {
...options,
skillId,
minFailureCount: options.minFailureCount || 1
});
const [health] = report.skills;
if (!health || health.failures === 0) {
return {
schemaVersion: AMENDMENT_SCHEMA_VERSION,
skill: {
id: skillId,
path: null
},
status: 'insufficient-evidence',
rationale: ['No failed observations were available for this skill.'],
patch: null
};
}
const preview = summarizePatchPreview(skillId, health);
return {
schemaVersion: AMENDMENT_SCHEMA_VERSION,
proposalId: createProposalId(skillId),
generatedAt: new Date().toISOString(),
status: 'proposed',
skill: {
id: skillId,
path: health.skill.path || null
},
evidence: {
totalRuns: health.totalRuns,
failures: health.failures,
successRate: health.successRate,
recurringErrors: health.recurringErrors,
recurringTasks: health.recurringTasks,
recurringFeedback: health.recurringFeedback
},
rationale: [
'Proposals are generated from repeated failed runs rather than a single anecdotal error.',
'The suggested patch is additive so the original SKILL.md intent remains auditable.'
],
patch: {
format: 'markdown-fragment',
targetPath: health.skill.path || `skills/${skillId}/SKILL.md`,
preview
}
};
}
module.exports = {
AMENDMENT_SCHEMA_VERSION,
proposeSkillAmendment
};

View File

@@ -0,0 +1,59 @@
'use strict';
const EVALUATION_SCHEMA_VERSION = 'ecc.skill-evaluation.v1';
function roundRate(value) {
return Math.round(value * 1000) / 1000;
}
function summarize(records) {
const runs = records.length;
const successes = records.filter(record => record.outcome && record.outcome.success).length;
const failures = runs - successes;
return {
runs,
successes,
failures,
successRate: runs > 0 ? roundRate(successes / runs) : 0
};
}
function buildSkillEvaluationScaffold(skillId, records, options = {}) {
const minimumRunsPerVariant = options.minimumRunsPerVariant || 2;
const amendmentId = options.amendmentId || null;
const filtered = records.filter(record => record.skill && record.skill.id === skillId);
const baseline = filtered.filter(record => !record.run || record.run.variant !== 'amended');
const amended = filtered.filter(record => record.run && record.run.variant === 'amended')
.filter(record => !amendmentId || record.run.amendmentId === amendmentId);
const baselineSummary = summarize(baseline);
const amendedSummary = summarize(amended);
const delta = {
successRate: roundRate(amendedSummary.successRate - baselineSummary.successRate),
failures: amendedSummary.failures - baselineSummary.failures
};
let recommendation = 'insufficient-data';
if (baselineSummary.runs >= minimumRunsPerVariant && amendedSummary.runs >= minimumRunsPerVariant) {
recommendation = delta.successRate > 0 ? 'promote-amendment' : 'keep-baseline';
}
return {
schemaVersion: EVALUATION_SCHEMA_VERSION,
generatedAt: new Date().toISOString(),
skillId,
amendmentId,
gate: {
minimumRunsPerVariant
},
baseline: baselineSummary,
amended: amendedSummary,
delta,
recommendation
};
}
module.exports = {
EVALUATION_SCHEMA_VERSION,
buildSkillEvaluationScaffold
};

View File

@@ -0,0 +1,118 @@
'use strict';
const HEALTH_SCHEMA_VERSION = 'ecc.skill-health.v1';
function roundRate(value) {
return Math.round(value * 1000) / 1000;
}
function rankCounts(values) {
return Array.from(values.entries())
.map(([value, count]) => ({ value, count }))
.sort((left, right) => right.count - left.count || left.value.localeCompare(right.value));
}
function summarizeVariantRuns(records) {
return records.reduce((accumulator, record) => {
const key = record.run && record.run.variant ? record.run.variant : 'baseline';
if (!accumulator[key]) {
accumulator[key] = { runs: 0, successes: 0, failures: 0 };
}
accumulator[key].runs += 1;
if (record.outcome && record.outcome.success) {
accumulator[key].successes += 1;
} else {
accumulator[key].failures += 1;
}
return accumulator;
}, {});
}
function deriveSkillStatus(skillSummary, options = {}) {
const minFailureCount = options.minFailureCount || 2;
if (skillSummary.failures >= minFailureCount) {
return 'failing';
}
if (skillSummary.failures > 0) {
return 'watch';
}
return 'healthy';
}
function buildSkillHealthReport(records, options = {}) {
const filterSkillId = options.skillId || null;
const filtered = filterSkillId
? records.filter(record => record.skill && record.skill.id === filterSkillId)
: records.slice();
const grouped = filtered.reduce((accumulator, record) => {
const skillId = record.skill.id;
if (!accumulator.has(skillId)) {
accumulator.set(skillId, []);
}
accumulator.get(skillId).push(record);
return accumulator;
}, new Map());
const skills = Array.from(grouped.entries())
.map(([skillId, skillRecords]) => {
const successes = skillRecords.filter(record => record.outcome && record.outcome.success).length;
const failures = skillRecords.length - successes;
const recurringErrors = new Map();
const recurringTasks = new Map();
const recurringFeedback = new Map();
skillRecords.forEach(record => {
if (!record.outcome || record.outcome.success) {
return;
}
if (record.outcome.error) {
recurringErrors.set(record.outcome.error, (recurringErrors.get(record.outcome.error) || 0) + 1);
}
if (record.task) {
recurringTasks.set(record.task, (recurringTasks.get(record.task) || 0) + 1);
}
if (record.outcome.feedback) {
recurringFeedback.set(record.outcome.feedback, (recurringFeedback.get(record.outcome.feedback) || 0) + 1);
}
});
const summary = {
skill: {
id: skillId,
path: skillRecords[0].skill.path || null
},
totalRuns: skillRecords.length,
successes,
failures,
successRate: skillRecords.length > 0 ? roundRate(successes / skillRecords.length) : 0,
status: 'healthy',
recurringErrors: rankCounts(recurringErrors).map(entry => ({ error: entry.value, count: entry.count })),
recurringTasks: rankCounts(recurringTasks).map(entry => ({ task: entry.value, count: entry.count })),
recurringFeedback: rankCounts(recurringFeedback).map(entry => ({ feedback: entry.value, count: entry.count })),
variants: summarizeVariantRuns(skillRecords)
};
summary.status = deriveSkillStatus(summary, options);
return summary;
})
.sort((left, right) => right.failures - left.failures || left.skill.id.localeCompare(right.skill.id));
return {
schemaVersion: HEALTH_SCHEMA_VERSION,
generatedAt: new Date().toISOString(),
totalObservations: filtered.length,
skillCount: skills.length,
skills
};
}
module.exports = {
HEALTH_SCHEMA_VERSION,
buildSkillHealthReport
};

View File

@@ -0,0 +1,108 @@
'use strict';
const fs = require('fs');
const path = require('path');
const os = require('os');
const OBSERVATION_SCHEMA_VERSION = 'ecc.skill-observation.v1';
function resolveProjectRoot(options = {}) {
return path.resolve(options.projectRoot || options.cwd || process.cwd());
}
function getSkillTelemetryRoot(options = {}) {
return path.join(resolveProjectRoot(options), '.claude', 'ecc', 'skills');
}
function getSkillObservationsPath(options = {}) {
return path.join(getSkillTelemetryRoot(options), 'observations.jsonl');
}
function ensureString(value, label) {
if (typeof value !== 'string' || value.trim().length === 0) {
throw new Error(`${label} must be a non-empty string`);
}
return value.trim();
}
function createObservationId() {
return `obs-${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2, 8)}`;
}
function createSkillObservation(input) {
const task = ensureString(input.task, 'task');
const skillId = ensureString(input.skill && input.skill.id, 'skill.id');
const skillPath = typeof input.skill.path === 'string' && input.skill.path.trim().length > 0
? input.skill.path.trim()
: null;
const success = Boolean(input.success);
const error = input.error == null ? null : String(input.error);
const feedback = input.feedback == null ? null : String(input.feedback);
const variant = typeof input.variant === 'string' && input.variant.trim().length > 0
? input.variant.trim()
: 'baseline';
return {
schemaVersion: OBSERVATION_SCHEMA_VERSION,
observationId: typeof input.observationId === 'string' && input.observationId.length > 0
? input.observationId
: createObservationId(),
timestamp: typeof input.timestamp === 'string' && input.timestamp.length > 0
? input.timestamp
: new Date().toISOString(),
task,
skill: {
id: skillId,
path: skillPath
},
outcome: {
success,
status: success ? 'success' : 'failure',
error,
feedback
},
run: {
variant,
amendmentId: input.amendmentId || null,
sessionId: input.sessionId || null,
source: input.source || 'manual'
}
};
}
function appendSkillObservation(observation, options = {}) {
const outputPath = getSkillObservationsPath(options);
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
fs.appendFileSync(outputPath, `${JSON.stringify(observation)}${os.EOL}`, 'utf8');
return outputPath;
}
function readSkillObservations(options = {}) {
const observationPath = path.resolve(options.observationsPath || getSkillObservationsPath(options));
if (!fs.existsSync(observationPath)) {
return [];
}
return fs.readFileSync(observationPath, 'utf8')
.split(/\r?\n/)
.filter(Boolean)
.map(line => {
try {
return JSON.parse(line);
} catch {
return null;
}
})
.filter(record => record && record.schemaVersion === OBSERVATION_SCHEMA_VERSION);
}
module.exports = {
OBSERVATION_SCHEMA_VERSION,
appendSkillObservation,
createSkillObservation,
getSkillObservationsPath,
getSkillTelemetryRoot,
readSkillObservations,
resolveProjectRoot
};

View File

@@ -4,12 +4,17 @@
const fs = require('fs');
const path = require('path');
const { inspectSessionTarget } = require('./lib/session-adapters/registry');
const { createAdapterRegistry, inspectSessionTarget } = require('./lib/session-adapters/registry');
const { readSkillObservations } = require('./lib/skill-improvement/observations');
const { buildSkillHealthReport } = require('./lib/skill-improvement/health');
const { proposeSkillAmendment } = require('./lib/skill-improvement/amendify');
const { buildSkillEvaluationScaffold } = require('./lib/skill-improvement/evaluate');
function usage() {
console.log([
'Usage:',
' node scripts/session-inspect.js <target> [--adapter <id>] [--write <output.json>]',
' node scripts/session-inspect.js <target> [--adapter <id>] [--target-type <type>] [--write <output.json>]',
' node scripts/session-inspect.js --list-adapters',
'',
'Targets:',
' <plan.json> Dmux/orchestration plan file',
@@ -17,11 +22,17 @@ function usage() {
' claude:latest Most recent Claude session history entry',
' claude:<id|alias> Specific Claude session or alias',
' <session.tmp> Direct path to a Claude session file',
' skills:health Inspect skill failure/success patterns from observations',
' skills:amendify Propose a SKILL.md patch from failure evidence',
' skills:evaluate Compare baseline vs amended skill outcomes',
'',
'Examples:',
' node scripts/session-inspect.js .claude/plan/workflow.json',
' node scripts/session-inspect.js workflow-visual-proof',
' node scripts/session-inspect.js claude:latest',
' node scripts/session-inspect.js latest --target-type claude-history',
' node scripts/session-inspect.js skills:health',
' node scripts/session-inspect.js skills:amendify --skill api-design',
' node scripts/session-inspect.js claude:a1b2c3d4 --write /tmp/session.json'
].join('\n'));
}
@@ -29,29 +40,91 @@ function usage() {
function parseArgs(argv) {
const args = argv.slice(2);
const target = args.find(argument => !argument.startsWith('--'));
const listAdapters = args.includes('--list-adapters');
const adapterIndex = args.indexOf('--adapter');
const adapterId = adapterIndex >= 0 ? args[adapterIndex + 1] : null;
const targetTypeIndex = args.indexOf('--target-type');
const targetType = targetTypeIndex >= 0 ? args[targetTypeIndex + 1] : null;
const skillIndex = args.indexOf('--skill');
const skillId = skillIndex >= 0 ? args[skillIndex + 1] : null;
const amendmentIndex = args.indexOf('--amendment-id');
const amendmentId = amendmentIndex >= 0 ? args[amendmentIndex + 1] : null;
const observationsIndex = args.indexOf('--observations');
const observationsPath = observationsIndex >= 0 ? args[observationsIndex + 1] : null;
const writeIndex = args.indexOf('--write');
const writePath = writeIndex >= 0 ? args[writeIndex + 1] : null;
return { target, adapterId, writePath };
return { target, adapterId, targetType, writePath, listAdapters, skillId, amendmentId, observationsPath };
}
function inspectSkillLoopTarget(target, options = {}) {
const observations = readSkillObservations({
cwd: options.cwd,
projectRoot: options.cwd,
observationsPath: options.observationsPath
});
if (target === 'skills:health') {
return buildSkillHealthReport(observations, {
skillId: options.skillId || null
});
}
if (target === 'skills:amendify') {
if (!options.skillId) {
throw new Error('skills:amendify requires --skill <id>');
}
return proposeSkillAmendment(options.skillId, observations);
}
if (target === 'skills:evaluate') {
if (!options.skillId) {
throw new Error('skills:evaluate requires --skill <id>');
}
return buildSkillEvaluationScaffold(options.skillId, observations, {
amendmentId: options.amendmentId || null
});
}
return null;
}
function main() {
const { target, adapterId, writePath } = parseArgs(process.argv);
const { target, adapterId, targetType, writePath, listAdapters, skillId, amendmentId, observationsPath } = parseArgs(process.argv);
if (listAdapters) {
const registry = createAdapterRegistry();
console.log(JSON.stringify({ adapters: registry.listAdapters() }, null, 2));
return;
}
if (!target) {
usage();
process.exit(1);
}
const snapshot = inspectSessionTarget(target, {
const skillLoopPayload = inspectSkillLoopTarget(target, {
cwd: process.cwd(),
adapterId
skillId,
amendmentId,
observationsPath
});
const payload = JSON.stringify(snapshot, null, 2);
const payloadObject = skillLoopPayload || inspectSessionTarget(
targetType ? { type: targetType, value: target } : target,
{
cwd: process.cwd(),
adapterId
}
);
const payload = JSON.stringify(payloadObject, null, 2);
if (writePath) {
const absoluteWritePath = path.resolve(writePath);

View File

@@ -210,5 +210,76 @@ test('adapter registry routes plan files to dmux and explicit claude targets to
}
});
test('adapter registry resolves structured target types into the correct adapter', () => {
const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-typed-repo-'));
const planPath = path.join(repoRoot, 'workflow.json');
fs.writeFileSync(planPath, JSON.stringify({
sessionName: 'workflow-typed-proof',
repoRoot,
coordinationRoot: path.join(repoRoot, '.claude', 'orchestration')
}));
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-typed-home-'));
const sessionsDir = path.join(homeDir, '.claude', 'sessions');
fs.mkdirSync(sessionsDir, { recursive: true });
fs.writeFileSync(
path.join(sessionsDir, '2026-03-13-z9y8x7w6-session.tmp'),
'# Typed History Session\n\n**Branch:** feat/typed-targets\n'
);
try {
withHome(homeDir, () => {
const registry = createAdapterRegistry({
adapters: [
createDmuxTmuxAdapter({
collectSessionSnapshotImpl: () => ({
sessionName: 'workflow-typed-proof',
coordinationDir: path.join(repoRoot, '.claude', 'orchestration', 'workflow-typed-proof'),
repoRoot,
targetType: 'plan',
sessionActive: true,
paneCount: 0,
workerCount: 0,
workerStates: {},
panes: [],
workers: []
})
}),
createClaudeHistoryAdapter()
]
});
const dmuxSnapshot = registry.open({ type: 'plan', value: planPath }, { cwd: repoRoot }).getSnapshot();
const claudeSnapshot = registry.open({ type: 'claude-history', value: 'latest' }, { cwd: repoRoot }).getSnapshot();
assert.strictEqual(dmuxSnapshot.adapterId, 'dmux-tmux');
assert.strictEqual(dmuxSnapshot.session.sourceTarget.type, 'plan');
assert.strictEqual(claudeSnapshot.adapterId, 'claude-history');
assert.strictEqual(claudeSnapshot.session.sourceTarget.type, 'claude-history');
assert.strictEqual(claudeSnapshot.workers[0].branch, 'feat/typed-targets');
});
} finally {
fs.rmSync(repoRoot, { recursive: true, force: true });
fs.rmSync(homeDir, { recursive: true, force: true });
}
});
test('adapter registry lists adapter metadata and target types', () => {
const registry = createAdapterRegistry();
const adapters = registry.listAdapters();
const ids = adapters.map(adapter => adapter.id);
assert.ok(ids.includes('claude-history'));
assert.ok(ids.includes('dmux-tmux'));
assert.ok(
adapters.some(adapter => adapter.id === 'claude-history' && adapter.targetTypes.includes('claude-history')),
'claude-history should advertise its canonical target type'
);
assert.ok(
adapters.some(adapter => adapter.id === 'dmux-tmux' && adapter.targetTypes.includes('plan')),
'dmux-tmux should advertise plan targets'
);
});
console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
if (failed > 0) process.exit(1);

View File

@@ -0,0 +1,186 @@
'use strict';
const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');
const {
appendSkillObservation,
createSkillObservation,
getSkillObservationsPath,
readSkillObservations
} = require('../../scripts/lib/skill-improvement/observations');
const { buildSkillHealthReport } = require('../../scripts/lib/skill-improvement/health');
const { proposeSkillAmendment } = require('../../scripts/lib/skill-improvement/amendify');
const { buildSkillEvaluationScaffold } = require('../../scripts/lib/skill-improvement/evaluate');
console.log('=== Testing skill-improvement ===\n');
let passed = 0;
let failed = 0;
function test(name, fn) {
try {
fn();
console.log(`${name}`);
passed += 1;
} catch (error) {
console.log(`${name}: ${error.message}`);
failed += 1;
}
}
function makeProjectRoot(prefix) {
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
}
function cleanup(dirPath) {
fs.rmSync(dirPath, { recursive: true, force: true });
}
test('observation layer writes and reads structured skill outcomes', () => {
const projectRoot = makeProjectRoot('ecc-skill-observe-');
try {
const observation = createSkillObservation({
task: 'Fix flaky Playwright test',
skill: {
id: 'e2e-testing',
path: 'skills/e2e-testing/SKILL.md'
},
success: false,
error: 'playwright timeout',
feedback: 'Timed out waiting for locator',
sessionId: 'sess-1234'
});
appendSkillObservation(observation, { projectRoot });
const records = readSkillObservations({ projectRoot });
assert.strictEqual(records.length, 1);
assert.strictEqual(records[0].schemaVersion, 'ecc.skill-observation.v1');
assert.strictEqual(records[0].task, 'Fix flaky Playwright test');
assert.strictEqual(records[0].skill.id, 'e2e-testing');
assert.strictEqual(records[0].outcome.success, false);
assert.strictEqual(records[0].outcome.error, 'playwright timeout');
assert.strictEqual(getSkillObservationsPath({ projectRoot }), path.join(projectRoot, '.claude', 'ecc', 'skills', 'observations.jsonl'));
} finally {
cleanup(projectRoot);
}
});
test('health inspector traces recurring failures for a skill across runs', () => {
const projectRoot = makeProjectRoot('ecc-skill-health-');
try {
[
createSkillObservation({
task: 'Ship Next.js auth middleware',
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
success: false,
error: 'missing csrf guidance',
feedback: 'Did not mention CSRF'
}),
createSkillObservation({
task: 'Harden Next.js auth middleware',
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
success: false,
error: 'missing csrf guidance',
feedback: 'Repeated omission'
}),
createSkillObservation({
task: 'Review payment webhook security',
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
success: true
})
].forEach(record => appendSkillObservation(record, { projectRoot }));
const report = buildSkillHealthReport(readSkillObservations({ projectRoot }), {
minFailureCount: 2
});
const skill = report.skills.find(entry => entry.skill.id === 'security-review');
assert.ok(skill, 'security-review should appear in the report');
assert.strictEqual(skill.totalRuns, 3);
assert.strictEqual(skill.failures, 2);
assert.strictEqual(skill.status, 'failing');
assert.strictEqual(skill.recurringErrors[0].error, 'missing csrf guidance');
assert.strictEqual(skill.recurringErrors[0].count, 2);
} finally {
cleanup(projectRoot);
}
});
test('amendify proposes SKILL.md patch content from failure evidence', () => {
const records = [
createSkillObservation({
task: 'Add API rate limiting',
skill: { id: 'api-design', path: 'skills/api-design/SKILL.md' },
success: false,
error: 'missing rate limiting guidance',
feedback: 'No rate-limit section'
}),
createSkillObservation({
task: 'Design public API error envelopes',
skill: { id: 'api-design', path: 'skills/api-design/SKILL.md' },
success: false,
error: 'missing error response examples',
feedback: 'Need explicit examples'
})
];
const proposal = proposeSkillAmendment('api-design', records);
assert.strictEqual(proposal.schemaVersion, 'ecc.skill-amendment-proposal.v1');
assert.strictEqual(proposal.skill.id, 'api-design');
assert.strictEqual(proposal.status, 'proposed');
assert.ok(proposal.patch.preview.includes('## Failure-Driven Amendments'));
assert.ok(proposal.patch.preview.includes('rate limiting'));
assert.ok(proposal.patch.preview.includes('error response'));
});
test('evaluation scaffold compares amended and baseline performance', () => {
const records = [
createSkillObservation({
task: 'Fix flaky login test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
success: false,
variant: 'baseline'
}),
createSkillObservation({
task: 'Fix flaky checkout test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
success: true,
variant: 'baseline'
}),
createSkillObservation({
task: 'Fix flaky login test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
success: true,
variant: 'amended',
amendmentId: 'amend-1'
}),
createSkillObservation({
task: 'Fix flaky checkout test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
success: true,
variant: 'amended',
amendmentId: 'amend-1'
})
];
const evaluation = buildSkillEvaluationScaffold('e2e-testing', records, {
amendmentId: 'amend-1',
minimumRunsPerVariant: 2
});
assert.strictEqual(evaluation.schemaVersion, 'ecc.skill-evaluation.v1');
assert.strictEqual(evaluation.baseline.runs, 2);
assert.strictEqual(evaluation.amended.runs, 2);
assert.strictEqual(evaluation.delta.successRate, 0.5);
assert.strictEqual(evaluation.recommendation, 'promote-amendment');
});
console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
if (failed > 0) process.exit(1);

View File

@@ -56,6 +56,15 @@ function runTests() {
assert.ok(result.stdout.includes('Usage:'));
})) passed++; else failed++;
if (test('lists registered adapters', () => {
const result = run(['--list-adapters']);
assert.strictEqual(result.code, 0, result.stderr);
const payload = JSON.parse(result.stdout);
assert.ok(Array.isArray(payload.adapters));
assert.ok(payload.adapters.some(adapter => adapter.id === 'claude-history'));
assert.ok(payload.adapters.some(adapter => adapter.id === 'dmux-tmux'));
})) passed++; else failed++;
if (test('prints canonical JSON for claude history targets', () => {
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-home-'));
const sessionsDir = path.join(homeDir, '.claude', 'sessions');
@@ -81,6 +90,31 @@ function runTests() {
}
})) passed++; else failed++;
if (test('supports explicit target types for structured registry routing', () => {
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-home-'));
const sessionsDir = path.join(homeDir, '.claude', 'sessions');
fs.mkdirSync(sessionsDir, { recursive: true });
try {
fs.writeFileSync(
path.join(sessionsDir, '2026-03-13-a1b2c3d4-session.tmp'),
'# Inspect Session\n\n**Branch:** feat/typed-inspect\n'
);
const result = run(['latest', '--target-type', 'claude-history'], {
env: { HOME: homeDir }
});
assert.strictEqual(result.code, 0, result.stderr);
const payload = JSON.parse(result.stdout);
assert.strictEqual(payload.adapterId, 'claude-history');
assert.strictEqual(payload.session.sourceTarget.type, 'claude-history');
assert.strictEqual(payload.workers[0].branch, 'feat/typed-inspect');
} finally {
fs.rmSync(homeDir, { recursive: true, force: true });
}
})) passed++; else failed++;
if (test('writes snapshot JSON to disk when --write is provided', () => {
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-home-'));
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-out-'));
@@ -109,6 +143,133 @@ function runTests() {
}
})) passed++; else failed++;
if (test('inspects skill health from recorded observations', () => {
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-skills-'));
const observationsDir = path.join(projectRoot, '.claude', 'ecc', 'skills');
fs.mkdirSync(observationsDir, { recursive: true });
fs.writeFileSync(
path.join(observationsDir, 'observations.jsonl'),
[
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-1',
timestamp: '2026-03-14T12:00:00.000Z',
task: 'Review auth middleware',
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
outcome: { success: false, status: 'failure', error: 'missing csrf guidance', feedback: 'Need CSRF coverage' },
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-1' }
}),
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-2',
timestamp: '2026-03-14T12:05:00.000Z',
task: 'Review auth middleware',
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
outcome: { success: false, status: 'failure', error: 'missing csrf guidance', feedback: null },
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-2' }
})
].join('\n') + '\n'
);
try {
const result = run(['skills:health'], { cwd: projectRoot });
assert.strictEqual(result.code, 0, result.stderr);
const payload = JSON.parse(result.stdout);
assert.strictEqual(payload.schemaVersion, 'ecc.skill-health.v1');
assert.ok(payload.skills.some(skill => skill.skill.id === 'security-review'));
} finally {
fs.rmSync(projectRoot, { recursive: true, force: true });
}
})) passed++; else failed++;
if (test('proposes skill amendments through session-inspect', () => {
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-amend-'));
const observationsDir = path.join(projectRoot, '.claude', 'ecc', 'skills');
fs.mkdirSync(observationsDir, { recursive: true });
fs.writeFileSync(
path.join(observationsDir, 'observations.jsonl'),
[
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-1',
timestamp: '2026-03-14T12:00:00.000Z',
task: 'Add rate limiting',
skill: { id: 'api-design', path: 'skills/api-design/SKILL.md' },
outcome: { success: false, status: 'failure', error: 'missing rate limiting guidance', feedback: 'Need rate limiting examples' },
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-1' }
})
].join('\n') + '\n'
);
try {
const result = run(['skills:amendify', '--skill', 'api-design'], { cwd: projectRoot });
assert.strictEqual(result.code, 0, result.stderr);
const payload = JSON.parse(result.stdout);
assert.strictEqual(payload.schemaVersion, 'ecc.skill-amendment-proposal.v1');
assert.strictEqual(payload.skill.id, 'api-design');
assert.ok(payload.patch.preview.includes('Failure-Driven Amendments'));
} finally {
fs.rmSync(projectRoot, { recursive: true, force: true });
}
})) passed++; else failed++;
if (test('builds skill evaluation scaffolding through session-inspect', () => {
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-eval-'));
const observationsDir = path.join(projectRoot, '.claude', 'ecc', 'skills');
fs.mkdirSync(observationsDir, { recursive: true });
fs.writeFileSync(
path.join(observationsDir, 'observations.jsonl'),
[
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-1',
timestamp: '2026-03-14T12:00:00.000Z',
task: 'Fix flaky login test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
outcome: { success: false, status: 'failure', error: null, feedback: null },
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-1' }
}),
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-2',
timestamp: '2026-03-14T12:10:00.000Z',
task: 'Fix flaky checkout test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
outcome: { success: true, status: 'success', error: null, feedback: null },
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-2' }
}),
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-3',
timestamp: '2026-03-14T12:20:00.000Z',
task: 'Fix flaky login test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
outcome: { success: true, status: 'success', error: null, feedback: null },
run: { variant: 'amended', amendmentId: 'amend-1', sessionId: 'sess-3' }
}),
JSON.stringify({
schemaVersion: 'ecc.skill-observation.v1',
observationId: 'obs-4',
timestamp: '2026-03-14T12:30:00.000Z',
task: 'Fix flaky checkout test',
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
outcome: { success: true, status: 'success', error: null, feedback: null },
run: { variant: 'amended', amendmentId: 'amend-1', sessionId: 'sess-4' }
})
].join('\n') + '\n'
);
try {
const result = run(['skills:evaluate', '--skill', 'e2e-testing', '--amendment-id', 'amend-1'], { cwd: projectRoot });
assert.strictEqual(result.code, 0, result.stderr);
const payload = JSON.parse(result.stdout);
assert.strictEqual(payload.schemaVersion, 'ecc.skill-evaluation.v1');
assert.strictEqual(payload.recommendation, 'promote-amendment');
} finally {
fs.rmSync(projectRoot, { recursive: true, force: true });
}
})) passed++; else failed++;
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
process.exit(failed > 0 ? 1 : 0);
}