mirror of
https://github.com/affaan-m/everything-claude-code.git
synced 2026-04-12 20:53:34 +08:00
Compare commits
2 Commits
fcaf78e449
...
c53bba9e02
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c53bba9e02 | ||
|
|
2b2777915e |
36
package-lock.json
generated
36
package-lock.json
generated
@@ -10,6 +10,7 @@
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"ecc": "scripts/ecc.js",
|
||||
"ecc-install": "install.sh"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -18,7 +19,7 @@
|
||||
"c8": "^10.1.2",
|
||||
"eslint": "^9.39.2",
|
||||
"globals": "^17.1.0",
|
||||
"markdownlint-cli": "^0.48.0"
|
||||
"markdownlint-cli": "^0.47.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
@@ -1655,22 +1656,23 @@
|
||||
}
|
||||
},
|
||||
"node_modules/markdownlint-cli": {
|
||||
"version": "0.48.0",
|
||||
"resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz",
|
||||
"integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==",
|
||||
"version": "0.47.0",
|
||||
"resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz",
|
||||
"integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"commander": "~14.0.3",
|
||||
"commander": "~14.0.2",
|
||||
"deep-extend": "~0.6.0",
|
||||
"ignore": "~7.0.5",
|
||||
"js-yaml": "~4.1.1",
|
||||
"jsonc-parser": "~3.3.1",
|
||||
"jsonpointer": "~5.0.1",
|
||||
"markdown-it": "~14.1.1",
|
||||
"markdown-it": "~14.1.0",
|
||||
"markdownlint": "~0.40.0",
|
||||
"minimatch": "~10.2.4",
|
||||
"minimatch": "~10.1.1",
|
||||
"run-con": "~1.3.2",
|
||||
"smol-toml": "~1.6.0",
|
||||
"smol-toml": "~1.5.2",
|
||||
"tinyglobby": "~0.2.15"
|
||||
},
|
||||
"bin": {
|
||||
@@ -1685,6 +1687,7 @@
|
||||
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
|
||||
"integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
}
|
||||
@@ -1694,6 +1697,7 @@
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz",
|
||||
"integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"balanced-match": "^4.0.2"
|
||||
},
|
||||
@@ -1712,15 +1716,16 @@
|
||||
}
|
||||
},
|
||||
"node_modules/markdownlint-cli/node_modules/minimatch": {
|
||||
"version": "10.2.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
|
||||
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
|
||||
"version": "10.1.3",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.3.tgz",
|
||||
"integrity": "sha512-IF6URNyBX7Z6XfvjpaNy5meRxPZiIf2OqtOoSLs+hLJ9pJAScnM1RjrFcbCaD85y42KcI+oZmKjFIJKYDFjQfg==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^5.0.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
"node": "20 || >=22"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/sponsors/isaacs"
|
||||
@@ -2579,10 +2584,11 @@
|
||||
}
|
||||
},
|
||||
"node_modules/smol-toml": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz",
|
||||
"integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==",
|
||||
"version": "1.5.2",
|
||||
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz",
|
||||
"integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==",
|
||||
"dev": true,
|
||||
"license": "BSD-3-Clause",
|
||||
"engines": {
|
||||
"node": ">= 18"
|
||||
},
|
||||
|
||||
@@ -114,6 +114,8 @@ function resolveSessionRecord(target, cwd) {
|
||||
function createClaudeHistoryAdapter() {
|
||||
return {
|
||||
id: 'claude-history',
|
||||
description: 'Claude local session history and session-file snapshots',
|
||||
targetTypes: ['claude-history', 'claude-alias', 'session-file'],
|
||||
canOpen(target, context = {}) {
|
||||
if (context.adapterId && context.adapterId !== 'claude-history') {
|
||||
return false;
|
||||
|
||||
@@ -45,6 +45,8 @@ function createDmuxTmuxAdapter(options = {}) {
|
||||
|
||||
return {
|
||||
id: 'dmux-tmux',
|
||||
description: 'Tmux/worktree orchestration snapshots from plan files or session names',
|
||||
targetTypes: ['plan', 'session'],
|
||||
canOpen(target, context = {}) {
|
||||
if (context.adapterId && context.adapterId !== 'dmux-tmux') {
|
||||
return false;
|
||||
|
||||
@@ -3,6 +3,14 @@
|
||||
const { createClaudeHistoryAdapter } = require('./claude-history');
|
||||
const { createDmuxTmuxAdapter } = require('./dmux-tmux');
|
||||
|
||||
const TARGET_TYPE_TO_ADAPTER_ID = Object.freeze({
|
||||
plan: 'dmux-tmux',
|
||||
session: 'dmux-tmux',
|
||||
'claude-history': 'claude-history',
|
||||
'claude-alias': 'claude-history',
|
||||
'session-file': 'claude-history'
|
||||
});
|
||||
|
||||
function createDefaultAdapters() {
|
||||
return [
|
||||
createClaudeHistoryAdapter(),
|
||||
@@ -10,13 +18,72 @@ function createDefaultAdapters() {
|
||||
];
|
||||
}
|
||||
|
||||
function coerceTargetValue(value) {
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
throw new Error('Structured session targets require a non-empty string value');
|
||||
}
|
||||
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function normalizeStructuredTarget(target, context = {}) {
|
||||
if (!target || typeof target !== 'object' || Array.isArray(target)) {
|
||||
return {
|
||||
target,
|
||||
context: { ...context }
|
||||
};
|
||||
}
|
||||
|
||||
const value = coerceTargetValue(target.value);
|
||||
const type = typeof target.type === 'string' ? target.type.trim() : '';
|
||||
if (type.length === 0) {
|
||||
throw new Error('Structured session targets require a non-empty type');
|
||||
}
|
||||
|
||||
const adapterId = target.adapterId || TARGET_TYPE_TO_ADAPTER_ID[type] || context.adapterId || null;
|
||||
const nextContext = {
|
||||
...context,
|
||||
adapterId
|
||||
};
|
||||
|
||||
if (type === 'claude-history' || type === 'claude-alias') {
|
||||
return {
|
||||
target: `claude:${value}`,
|
||||
context: nextContext
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
target: value,
|
||||
context: nextContext
|
||||
};
|
||||
}
|
||||
|
||||
function createAdapterRegistry(options = {}) {
|
||||
const adapters = options.adapters || createDefaultAdapters();
|
||||
|
||||
return {
|
||||
adapters,
|
||||
getAdapter(id) {
|
||||
const adapter = adapters.find(candidate => candidate.id === id);
|
||||
if (!adapter) {
|
||||
throw new Error(`Unknown session adapter: ${id}`);
|
||||
}
|
||||
|
||||
return adapter;
|
||||
},
|
||||
listAdapters() {
|
||||
return adapters.map(adapter => ({
|
||||
id: adapter.id,
|
||||
description: adapter.description || '',
|
||||
targetTypes: Array.isArray(adapter.targetTypes) ? [...adapter.targetTypes] : []
|
||||
}));
|
||||
},
|
||||
select(target, context = {}) {
|
||||
const adapter = adapters.find(candidate => candidate.canOpen(target, context));
|
||||
const normalized = normalizeStructuredTarget(target, context);
|
||||
const adapter = normalized.context.adapterId
|
||||
? this.getAdapter(normalized.context.adapterId)
|
||||
: adapters.find(candidate => candidate.canOpen(normalized.target, normalized.context));
|
||||
if (!adapter) {
|
||||
throw new Error(`No session adapter matched target: ${target}`);
|
||||
}
|
||||
@@ -24,8 +91,9 @@ function createAdapterRegistry(options = {}) {
|
||||
return adapter;
|
||||
},
|
||||
open(target, context = {}) {
|
||||
const adapter = this.select(target, context);
|
||||
return adapter.open(target, context);
|
||||
const normalized = normalizeStructuredTarget(target, context);
|
||||
const adapter = this.select(normalized.target, normalized.context);
|
||||
return adapter.open(normalized.target, normalized.context);
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -38,5 +106,6 @@ function inspectSessionTarget(target, options = {}) {
|
||||
module.exports = {
|
||||
createAdapterRegistry,
|
||||
createDefaultAdapters,
|
||||
inspectSessionTarget
|
||||
inspectSessionTarget,
|
||||
normalizeStructuredTarget
|
||||
};
|
||||
|
||||
89
scripts/lib/skill-improvement/amendify.js
Normal file
89
scripts/lib/skill-improvement/amendify.js
Normal file
@@ -0,0 +1,89 @@
|
||||
'use strict';
|
||||
|
||||
const { buildSkillHealthReport } = require('./health');
|
||||
|
||||
const AMENDMENT_SCHEMA_VERSION = 'ecc.skill-amendment-proposal.v1';
|
||||
|
||||
function createProposalId(skillId) {
|
||||
return `amend-${skillId}-${Date.now()}`;
|
||||
}
|
||||
|
||||
function summarizePatchPreview(skillId, health) {
|
||||
const lines = [
|
||||
'## Failure-Driven Amendments',
|
||||
'',
|
||||
`- Focus skill routing for \`${skillId}\` when tasks match the proven success cases.`,
|
||||
];
|
||||
|
||||
if (health.recurringErrors[0]) {
|
||||
lines.push(`- Add explicit guardrails for recurring failure: ${health.recurringErrors[0].error}.`);
|
||||
}
|
||||
|
||||
if (health.recurringTasks[0]) {
|
||||
lines.push(`- Add an example workflow for task pattern: ${health.recurringTasks[0].task}.`);
|
||||
}
|
||||
|
||||
if (health.recurringFeedback[0]) {
|
||||
lines.push(`- Address repeated user feedback: ${health.recurringFeedback[0].feedback}.`);
|
||||
}
|
||||
|
||||
lines.push('- Add a verification checklist before declaring the skill output complete.');
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
function proposeSkillAmendment(skillId, records, options = {}) {
|
||||
const report = buildSkillHealthReport(records, {
|
||||
...options,
|
||||
skillId,
|
||||
minFailureCount: options.minFailureCount || 1
|
||||
});
|
||||
const [health] = report.skills;
|
||||
|
||||
if (!health || health.failures === 0) {
|
||||
return {
|
||||
schemaVersion: AMENDMENT_SCHEMA_VERSION,
|
||||
skill: {
|
||||
id: skillId,
|
||||
path: null
|
||||
},
|
||||
status: 'insufficient-evidence',
|
||||
rationale: ['No failed observations were available for this skill.'],
|
||||
patch: null
|
||||
};
|
||||
}
|
||||
|
||||
const preview = summarizePatchPreview(skillId, health);
|
||||
|
||||
return {
|
||||
schemaVersion: AMENDMENT_SCHEMA_VERSION,
|
||||
proposalId: createProposalId(skillId),
|
||||
generatedAt: new Date().toISOString(),
|
||||
status: 'proposed',
|
||||
skill: {
|
||||
id: skillId,
|
||||
path: health.skill.path || null
|
||||
},
|
||||
evidence: {
|
||||
totalRuns: health.totalRuns,
|
||||
failures: health.failures,
|
||||
successRate: health.successRate,
|
||||
recurringErrors: health.recurringErrors,
|
||||
recurringTasks: health.recurringTasks,
|
||||
recurringFeedback: health.recurringFeedback
|
||||
},
|
||||
rationale: [
|
||||
'Proposals are generated from repeated failed runs rather than a single anecdotal error.',
|
||||
'The suggested patch is additive so the original SKILL.md intent remains auditable.'
|
||||
],
|
||||
patch: {
|
||||
format: 'markdown-fragment',
|
||||
targetPath: health.skill.path || `skills/${skillId}/SKILL.md`,
|
||||
preview
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
AMENDMENT_SCHEMA_VERSION,
|
||||
proposeSkillAmendment
|
||||
};
|
||||
59
scripts/lib/skill-improvement/evaluate.js
Normal file
59
scripts/lib/skill-improvement/evaluate.js
Normal file
@@ -0,0 +1,59 @@
|
||||
'use strict';
|
||||
|
||||
const EVALUATION_SCHEMA_VERSION = 'ecc.skill-evaluation.v1';
|
||||
|
||||
function roundRate(value) {
|
||||
return Math.round(value * 1000) / 1000;
|
||||
}
|
||||
|
||||
function summarize(records) {
|
||||
const runs = records.length;
|
||||
const successes = records.filter(record => record.outcome && record.outcome.success).length;
|
||||
const failures = runs - successes;
|
||||
return {
|
||||
runs,
|
||||
successes,
|
||||
failures,
|
||||
successRate: runs > 0 ? roundRate(successes / runs) : 0
|
||||
};
|
||||
}
|
||||
|
||||
function buildSkillEvaluationScaffold(skillId, records, options = {}) {
|
||||
const minimumRunsPerVariant = options.minimumRunsPerVariant || 2;
|
||||
const amendmentId = options.amendmentId || null;
|
||||
const filtered = records.filter(record => record.skill && record.skill.id === skillId);
|
||||
const baseline = filtered.filter(record => !record.run || record.run.variant !== 'amended');
|
||||
const amended = filtered.filter(record => record.run && record.run.variant === 'amended')
|
||||
.filter(record => !amendmentId || record.run.amendmentId === amendmentId);
|
||||
|
||||
const baselineSummary = summarize(baseline);
|
||||
const amendedSummary = summarize(amended);
|
||||
const delta = {
|
||||
successRate: roundRate(amendedSummary.successRate - baselineSummary.successRate),
|
||||
failures: amendedSummary.failures - baselineSummary.failures
|
||||
};
|
||||
|
||||
let recommendation = 'insufficient-data';
|
||||
if (baselineSummary.runs >= minimumRunsPerVariant && amendedSummary.runs >= minimumRunsPerVariant) {
|
||||
recommendation = delta.successRate > 0 ? 'promote-amendment' : 'keep-baseline';
|
||||
}
|
||||
|
||||
return {
|
||||
schemaVersion: EVALUATION_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
skillId,
|
||||
amendmentId,
|
||||
gate: {
|
||||
minimumRunsPerVariant
|
||||
},
|
||||
baseline: baselineSummary,
|
||||
amended: amendedSummary,
|
||||
delta,
|
||||
recommendation
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
EVALUATION_SCHEMA_VERSION,
|
||||
buildSkillEvaluationScaffold
|
||||
};
|
||||
118
scripts/lib/skill-improvement/health.js
Normal file
118
scripts/lib/skill-improvement/health.js
Normal file
@@ -0,0 +1,118 @@
|
||||
'use strict';
|
||||
|
||||
const HEALTH_SCHEMA_VERSION = 'ecc.skill-health.v1';
|
||||
|
||||
function roundRate(value) {
|
||||
return Math.round(value * 1000) / 1000;
|
||||
}
|
||||
|
||||
function rankCounts(values) {
|
||||
return Array.from(values.entries())
|
||||
.map(([value, count]) => ({ value, count }))
|
||||
.sort((left, right) => right.count - left.count || left.value.localeCompare(right.value));
|
||||
}
|
||||
|
||||
function summarizeVariantRuns(records) {
|
||||
return records.reduce((accumulator, record) => {
|
||||
const key = record.run && record.run.variant ? record.run.variant : 'baseline';
|
||||
if (!accumulator[key]) {
|
||||
accumulator[key] = { runs: 0, successes: 0, failures: 0 };
|
||||
}
|
||||
|
||||
accumulator[key].runs += 1;
|
||||
if (record.outcome && record.outcome.success) {
|
||||
accumulator[key].successes += 1;
|
||||
} else {
|
||||
accumulator[key].failures += 1;
|
||||
}
|
||||
|
||||
return accumulator;
|
||||
}, {});
|
||||
}
|
||||
|
||||
function deriveSkillStatus(skillSummary, options = {}) {
|
||||
const minFailureCount = options.minFailureCount || 2;
|
||||
if (skillSummary.failures >= minFailureCount) {
|
||||
return 'failing';
|
||||
}
|
||||
|
||||
if (skillSummary.failures > 0) {
|
||||
return 'watch';
|
||||
}
|
||||
|
||||
return 'healthy';
|
||||
}
|
||||
|
||||
function buildSkillHealthReport(records, options = {}) {
|
||||
const filterSkillId = options.skillId || null;
|
||||
const filtered = filterSkillId
|
||||
? records.filter(record => record.skill && record.skill.id === filterSkillId)
|
||||
: records.slice();
|
||||
|
||||
const grouped = filtered.reduce((accumulator, record) => {
|
||||
const skillId = record.skill.id;
|
||||
if (!accumulator.has(skillId)) {
|
||||
accumulator.set(skillId, []);
|
||||
}
|
||||
accumulator.get(skillId).push(record);
|
||||
return accumulator;
|
||||
}, new Map());
|
||||
|
||||
const skills = Array.from(grouped.entries())
|
||||
.map(([skillId, skillRecords]) => {
|
||||
const successes = skillRecords.filter(record => record.outcome && record.outcome.success).length;
|
||||
const failures = skillRecords.length - successes;
|
||||
const recurringErrors = new Map();
|
||||
const recurringTasks = new Map();
|
||||
const recurringFeedback = new Map();
|
||||
|
||||
skillRecords.forEach(record => {
|
||||
if (!record.outcome || record.outcome.success) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (record.outcome.error) {
|
||||
recurringErrors.set(record.outcome.error, (recurringErrors.get(record.outcome.error) || 0) + 1);
|
||||
}
|
||||
if (record.task) {
|
||||
recurringTasks.set(record.task, (recurringTasks.get(record.task) || 0) + 1);
|
||||
}
|
||||
if (record.outcome.feedback) {
|
||||
recurringFeedback.set(record.outcome.feedback, (recurringFeedback.get(record.outcome.feedback) || 0) + 1);
|
||||
}
|
||||
});
|
||||
|
||||
const summary = {
|
||||
skill: {
|
||||
id: skillId,
|
||||
path: skillRecords[0].skill.path || null
|
||||
},
|
||||
totalRuns: skillRecords.length,
|
||||
successes,
|
||||
failures,
|
||||
successRate: skillRecords.length > 0 ? roundRate(successes / skillRecords.length) : 0,
|
||||
status: 'healthy',
|
||||
recurringErrors: rankCounts(recurringErrors).map(entry => ({ error: entry.value, count: entry.count })),
|
||||
recurringTasks: rankCounts(recurringTasks).map(entry => ({ task: entry.value, count: entry.count })),
|
||||
recurringFeedback: rankCounts(recurringFeedback).map(entry => ({ feedback: entry.value, count: entry.count })),
|
||||
variants: summarizeVariantRuns(skillRecords)
|
||||
};
|
||||
|
||||
summary.status = deriveSkillStatus(summary, options);
|
||||
return summary;
|
||||
})
|
||||
.sort((left, right) => right.failures - left.failures || left.skill.id.localeCompare(right.skill.id));
|
||||
|
||||
return {
|
||||
schemaVersion: HEALTH_SCHEMA_VERSION,
|
||||
generatedAt: new Date().toISOString(),
|
||||
totalObservations: filtered.length,
|
||||
skillCount: skills.length,
|
||||
skills
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
HEALTH_SCHEMA_VERSION,
|
||||
buildSkillHealthReport
|
||||
};
|
||||
108
scripts/lib/skill-improvement/observations.js
Normal file
108
scripts/lib/skill-improvement/observations.js
Normal file
@@ -0,0 +1,108 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const os = require('os');
|
||||
|
||||
const OBSERVATION_SCHEMA_VERSION = 'ecc.skill-observation.v1';
|
||||
|
||||
function resolveProjectRoot(options = {}) {
|
||||
return path.resolve(options.projectRoot || options.cwd || process.cwd());
|
||||
}
|
||||
|
||||
function getSkillTelemetryRoot(options = {}) {
|
||||
return path.join(resolveProjectRoot(options), '.claude', 'ecc', 'skills');
|
||||
}
|
||||
|
||||
function getSkillObservationsPath(options = {}) {
|
||||
return path.join(getSkillTelemetryRoot(options), 'observations.jsonl');
|
||||
}
|
||||
|
||||
function ensureString(value, label) {
|
||||
if (typeof value !== 'string' || value.trim().length === 0) {
|
||||
throw new Error(`${label} must be a non-empty string`);
|
||||
}
|
||||
|
||||
return value.trim();
|
||||
}
|
||||
|
||||
function createObservationId() {
|
||||
return `obs-${Date.now()}-${process.pid}-${Math.random().toString(16).slice(2, 8)}`;
|
||||
}
|
||||
|
||||
function createSkillObservation(input) {
|
||||
const task = ensureString(input.task, 'task');
|
||||
const skillId = ensureString(input.skill && input.skill.id, 'skill.id');
|
||||
const skillPath = typeof input.skill.path === 'string' && input.skill.path.trim().length > 0
|
||||
? input.skill.path.trim()
|
||||
: null;
|
||||
const success = Boolean(input.success);
|
||||
const error = input.error == null ? null : String(input.error);
|
||||
const feedback = input.feedback == null ? null : String(input.feedback);
|
||||
const variant = typeof input.variant === 'string' && input.variant.trim().length > 0
|
||||
? input.variant.trim()
|
||||
: 'baseline';
|
||||
|
||||
return {
|
||||
schemaVersion: OBSERVATION_SCHEMA_VERSION,
|
||||
observationId: typeof input.observationId === 'string' && input.observationId.length > 0
|
||||
? input.observationId
|
||||
: createObservationId(),
|
||||
timestamp: typeof input.timestamp === 'string' && input.timestamp.length > 0
|
||||
? input.timestamp
|
||||
: new Date().toISOString(),
|
||||
task,
|
||||
skill: {
|
||||
id: skillId,
|
||||
path: skillPath
|
||||
},
|
||||
outcome: {
|
||||
success,
|
||||
status: success ? 'success' : 'failure',
|
||||
error,
|
||||
feedback
|
||||
},
|
||||
run: {
|
||||
variant,
|
||||
amendmentId: input.amendmentId || null,
|
||||
sessionId: input.sessionId || null,
|
||||
source: input.source || 'manual'
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function appendSkillObservation(observation, options = {}) {
|
||||
const outputPath = getSkillObservationsPath(options);
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.appendFileSync(outputPath, `${JSON.stringify(observation)}${os.EOL}`, 'utf8');
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
function readSkillObservations(options = {}) {
|
||||
const observationPath = path.resolve(options.observationsPath || getSkillObservationsPath(options));
|
||||
if (!fs.existsSync(observationPath)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs.readFileSync(observationPath, 'utf8')
|
||||
.split(/\r?\n/)
|
||||
.filter(Boolean)
|
||||
.map(line => {
|
||||
try {
|
||||
return JSON.parse(line);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter(record => record && record.schemaVersion === OBSERVATION_SCHEMA_VERSION);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
OBSERVATION_SCHEMA_VERSION,
|
||||
appendSkillObservation,
|
||||
createSkillObservation,
|
||||
getSkillObservationsPath,
|
||||
getSkillTelemetryRoot,
|
||||
readSkillObservations,
|
||||
resolveProjectRoot
|
||||
};
|
||||
@@ -4,12 +4,17 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { inspectSessionTarget } = require('./lib/session-adapters/registry');
|
||||
const { createAdapterRegistry, inspectSessionTarget } = require('./lib/session-adapters/registry');
|
||||
const { readSkillObservations } = require('./lib/skill-improvement/observations');
|
||||
const { buildSkillHealthReport } = require('./lib/skill-improvement/health');
|
||||
const { proposeSkillAmendment } = require('./lib/skill-improvement/amendify');
|
||||
const { buildSkillEvaluationScaffold } = require('./lib/skill-improvement/evaluate');
|
||||
|
||||
function usage() {
|
||||
console.log([
|
||||
'Usage:',
|
||||
' node scripts/session-inspect.js <target> [--adapter <id>] [--write <output.json>]',
|
||||
' node scripts/session-inspect.js <target> [--adapter <id>] [--target-type <type>] [--write <output.json>]',
|
||||
' node scripts/session-inspect.js --list-adapters',
|
||||
'',
|
||||
'Targets:',
|
||||
' <plan.json> Dmux/orchestration plan file',
|
||||
@@ -17,11 +22,17 @@ function usage() {
|
||||
' claude:latest Most recent Claude session history entry',
|
||||
' claude:<id|alias> Specific Claude session or alias',
|
||||
' <session.tmp> Direct path to a Claude session file',
|
||||
' skills:health Inspect skill failure/success patterns from observations',
|
||||
' skills:amendify Propose a SKILL.md patch from failure evidence',
|
||||
' skills:evaluate Compare baseline vs amended skill outcomes',
|
||||
'',
|
||||
'Examples:',
|
||||
' node scripts/session-inspect.js .claude/plan/workflow.json',
|
||||
' node scripts/session-inspect.js workflow-visual-proof',
|
||||
' node scripts/session-inspect.js claude:latest',
|
||||
' node scripts/session-inspect.js latest --target-type claude-history',
|
||||
' node scripts/session-inspect.js skills:health',
|
||||
' node scripts/session-inspect.js skills:amendify --skill api-design',
|
||||
' node scripts/session-inspect.js claude:a1b2c3d4 --write /tmp/session.json'
|
||||
].join('\n'));
|
||||
}
|
||||
@@ -29,29 +40,91 @@ function usage() {
|
||||
function parseArgs(argv) {
|
||||
const args = argv.slice(2);
|
||||
const target = args.find(argument => !argument.startsWith('--'));
|
||||
const listAdapters = args.includes('--list-adapters');
|
||||
|
||||
const adapterIndex = args.indexOf('--adapter');
|
||||
const adapterId = adapterIndex >= 0 ? args[adapterIndex + 1] : null;
|
||||
|
||||
const targetTypeIndex = args.indexOf('--target-type');
|
||||
const targetType = targetTypeIndex >= 0 ? args[targetTypeIndex + 1] : null;
|
||||
|
||||
const skillIndex = args.indexOf('--skill');
|
||||
const skillId = skillIndex >= 0 ? args[skillIndex + 1] : null;
|
||||
|
||||
const amendmentIndex = args.indexOf('--amendment-id');
|
||||
const amendmentId = amendmentIndex >= 0 ? args[amendmentIndex + 1] : null;
|
||||
|
||||
const observationsIndex = args.indexOf('--observations');
|
||||
const observationsPath = observationsIndex >= 0 ? args[observationsIndex + 1] : null;
|
||||
|
||||
const writeIndex = args.indexOf('--write');
|
||||
const writePath = writeIndex >= 0 ? args[writeIndex + 1] : null;
|
||||
|
||||
return { target, adapterId, writePath };
|
||||
return { target, adapterId, targetType, writePath, listAdapters, skillId, amendmentId, observationsPath };
|
||||
}
|
||||
|
||||
function inspectSkillLoopTarget(target, options = {}) {
|
||||
const observations = readSkillObservations({
|
||||
cwd: options.cwd,
|
||||
projectRoot: options.cwd,
|
||||
observationsPath: options.observationsPath
|
||||
});
|
||||
|
||||
if (target === 'skills:health') {
|
||||
return buildSkillHealthReport(observations, {
|
||||
skillId: options.skillId || null
|
||||
});
|
||||
}
|
||||
|
||||
if (target === 'skills:amendify') {
|
||||
if (!options.skillId) {
|
||||
throw new Error('skills:amendify requires --skill <id>');
|
||||
}
|
||||
|
||||
return proposeSkillAmendment(options.skillId, observations);
|
||||
}
|
||||
|
||||
if (target === 'skills:evaluate') {
|
||||
if (!options.skillId) {
|
||||
throw new Error('skills:evaluate requires --skill <id>');
|
||||
}
|
||||
|
||||
return buildSkillEvaluationScaffold(options.skillId, observations, {
|
||||
amendmentId: options.amendmentId || null
|
||||
});
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const { target, adapterId, writePath } = parseArgs(process.argv);
|
||||
const { target, adapterId, targetType, writePath, listAdapters, skillId, amendmentId, observationsPath } = parseArgs(process.argv);
|
||||
|
||||
if (listAdapters) {
|
||||
const registry = createAdapterRegistry();
|
||||
console.log(JSON.stringify({ adapters: registry.listAdapters() }, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!target) {
|
||||
usage();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const snapshot = inspectSessionTarget(target, {
|
||||
const skillLoopPayload = inspectSkillLoopTarget(target, {
|
||||
cwd: process.cwd(),
|
||||
adapterId
|
||||
skillId,
|
||||
amendmentId,
|
||||
observationsPath
|
||||
});
|
||||
const payload = JSON.stringify(snapshot, null, 2);
|
||||
const payloadObject = skillLoopPayload || inspectSessionTarget(
|
||||
targetType ? { type: targetType, value: target } : target,
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
adapterId
|
||||
}
|
||||
);
|
||||
const payload = JSON.stringify(payloadObject, null, 2);
|
||||
|
||||
if (writePath) {
|
||||
const absoluteWritePath = path.resolve(writePath);
|
||||
|
||||
@@ -210,5 +210,76 @@ test('adapter registry routes plan files to dmux and explicit claude targets to
|
||||
}
|
||||
});
|
||||
|
||||
test('adapter registry resolves structured target types into the correct adapter', () => {
|
||||
const repoRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-typed-repo-'));
|
||||
const planPath = path.join(repoRoot, 'workflow.json');
|
||||
fs.writeFileSync(planPath, JSON.stringify({
|
||||
sessionName: 'workflow-typed-proof',
|
||||
repoRoot,
|
||||
coordinationRoot: path.join(repoRoot, '.claude', 'orchestration')
|
||||
}));
|
||||
|
||||
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-typed-home-'));
|
||||
const sessionsDir = path.join(homeDir, '.claude', 'sessions');
|
||||
fs.mkdirSync(sessionsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(sessionsDir, '2026-03-13-z9y8x7w6-session.tmp'),
|
||||
'# Typed History Session\n\n**Branch:** feat/typed-targets\n'
|
||||
);
|
||||
|
||||
try {
|
||||
withHome(homeDir, () => {
|
||||
const registry = createAdapterRegistry({
|
||||
adapters: [
|
||||
createDmuxTmuxAdapter({
|
||||
collectSessionSnapshotImpl: () => ({
|
||||
sessionName: 'workflow-typed-proof',
|
||||
coordinationDir: path.join(repoRoot, '.claude', 'orchestration', 'workflow-typed-proof'),
|
||||
repoRoot,
|
||||
targetType: 'plan',
|
||||
sessionActive: true,
|
||||
paneCount: 0,
|
||||
workerCount: 0,
|
||||
workerStates: {},
|
||||
panes: [],
|
||||
workers: []
|
||||
})
|
||||
}),
|
||||
createClaudeHistoryAdapter()
|
||||
]
|
||||
});
|
||||
|
||||
const dmuxSnapshot = registry.open({ type: 'plan', value: planPath }, { cwd: repoRoot }).getSnapshot();
|
||||
const claudeSnapshot = registry.open({ type: 'claude-history', value: 'latest' }, { cwd: repoRoot }).getSnapshot();
|
||||
|
||||
assert.strictEqual(dmuxSnapshot.adapterId, 'dmux-tmux');
|
||||
assert.strictEqual(dmuxSnapshot.session.sourceTarget.type, 'plan');
|
||||
assert.strictEqual(claudeSnapshot.adapterId, 'claude-history');
|
||||
assert.strictEqual(claudeSnapshot.session.sourceTarget.type, 'claude-history');
|
||||
assert.strictEqual(claudeSnapshot.workers[0].branch, 'feat/typed-targets');
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(repoRoot, { recursive: true, force: true });
|
||||
fs.rmSync(homeDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('adapter registry lists adapter metadata and target types', () => {
|
||||
const registry = createAdapterRegistry();
|
||||
const adapters = registry.listAdapters();
|
||||
const ids = adapters.map(adapter => adapter.id);
|
||||
|
||||
assert.ok(ids.includes('claude-history'));
|
||||
assert.ok(ids.includes('dmux-tmux'));
|
||||
assert.ok(
|
||||
adapters.some(adapter => adapter.id === 'claude-history' && adapter.targetTypes.includes('claude-history')),
|
||||
'claude-history should advertise its canonical target type'
|
||||
);
|
||||
assert.ok(
|
||||
adapters.some(adapter => adapter.id === 'dmux-tmux' && adapter.targetTypes.includes('plan')),
|
||||
'dmux-tmux should advertise plan targets'
|
||||
);
|
||||
});
|
||||
|
||||
console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
|
||||
if (failed > 0) process.exit(1);
|
||||
|
||||
186
tests/lib/skill-improvement.test.js
Normal file
186
tests/lib/skill-improvement.test.js
Normal file
@@ -0,0 +1,186 @@
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
const {
|
||||
appendSkillObservation,
|
||||
createSkillObservation,
|
||||
getSkillObservationsPath,
|
||||
readSkillObservations
|
||||
} = require('../../scripts/lib/skill-improvement/observations');
|
||||
const { buildSkillHealthReport } = require('../../scripts/lib/skill-improvement/health');
|
||||
const { proposeSkillAmendment } = require('../../scripts/lib/skill-improvement/amendify');
|
||||
const { buildSkillEvaluationScaffold } = require('../../scripts/lib/skill-improvement/evaluate');
|
||||
|
||||
console.log('=== Testing skill-improvement ===\n');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` ✓ ${name}`);
|
||||
passed += 1;
|
||||
} catch (error) {
|
||||
console.log(` ✗ ${name}: ${error.message}`);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
function makeProjectRoot(prefix) {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
||||
}
|
||||
|
||||
function cleanup(dirPath) {
|
||||
fs.rmSync(dirPath, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
test('observation layer writes and reads structured skill outcomes', () => {
|
||||
const projectRoot = makeProjectRoot('ecc-skill-observe-');
|
||||
|
||||
try {
|
||||
const observation = createSkillObservation({
|
||||
task: 'Fix flaky Playwright test',
|
||||
skill: {
|
||||
id: 'e2e-testing',
|
||||
path: 'skills/e2e-testing/SKILL.md'
|
||||
},
|
||||
success: false,
|
||||
error: 'playwright timeout',
|
||||
feedback: 'Timed out waiting for locator',
|
||||
sessionId: 'sess-1234'
|
||||
});
|
||||
|
||||
appendSkillObservation(observation, { projectRoot });
|
||||
const records = readSkillObservations({ projectRoot });
|
||||
|
||||
assert.strictEqual(records.length, 1);
|
||||
assert.strictEqual(records[0].schemaVersion, 'ecc.skill-observation.v1');
|
||||
assert.strictEqual(records[0].task, 'Fix flaky Playwright test');
|
||||
assert.strictEqual(records[0].skill.id, 'e2e-testing');
|
||||
assert.strictEqual(records[0].outcome.success, false);
|
||||
assert.strictEqual(records[0].outcome.error, 'playwright timeout');
|
||||
assert.strictEqual(getSkillObservationsPath({ projectRoot }), path.join(projectRoot, '.claude', 'ecc', 'skills', 'observations.jsonl'));
|
||||
} finally {
|
||||
cleanup(projectRoot);
|
||||
}
|
||||
});
|
||||
|
||||
test('health inspector traces recurring failures for a skill across runs', () => {
|
||||
const projectRoot = makeProjectRoot('ecc-skill-health-');
|
||||
|
||||
try {
|
||||
[
|
||||
createSkillObservation({
|
||||
task: 'Ship Next.js auth middleware',
|
||||
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
|
||||
success: false,
|
||||
error: 'missing csrf guidance',
|
||||
feedback: 'Did not mention CSRF'
|
||||
}),
|
||||
createSkillObservation({
|
||||
task: 'Harden Next.js auth middleware',
|
||||
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
|
||||
success: false,
|
||||
error: 'missing csrf guidance',
|
||||
feedback: 'Repeated omission'
|
||||
}),
|
||||
createSkillObservation({
|
||||
task: 'Review payment webhook security',
|
||||
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
|
||||
success: true
|
||||
})
|
||||
].forEach(record => appendSkillObservation(record, { projectRoot }));
|
||||
|
||||
const report = buildSkillHealthReport(readSkillObservations({ projectRoot }), {
|
||||
minFailureCount: 2
|
||||
});
|
||||
const skill = report.skills.find(entry => entry.skill.id === 'security-review');
|
||||
|
||||
assert.ok(skill, 'security-review should appear in the report');
|
||||
assert.strictEqual(skill.totalRuns, 3);
|
||||
assert.strictEqual(skill.failures, 2);
|
||||
assert.strictEqual(skill.status, 'failing');
|
||||
assert.strictEqual(skill.recurringErrors[0].error, 'missing csrf guidance');
|
||||
assert.strictEqual(skill.recurringErrors[0].count, 2);
|
||||
} finally {
|
||||
cleanup(projectRoot);
|
||||
}
|
||||
});
|
||||
|
||||
test('amendify proposes SKILL.md patch content from failure evidence', () => {
|
||||
const records = [
|
||||
createSkillObservation({
|
||||
task: 'Add API rate limiting',
|
||||
skill: { id: 'api-design', path: 'skills/api-design/SKILL.md' },
|
||||
success: false,
|
||||
error: 'missing rate limiting guidance',
|
||||
feedback: 'No rate-limit section'
|
||||
}),
|
||||
createSkillObservation({
|
||||
task: 'Design public API error envelopes',
|
||||
skill: { id: 'api-design', path: 'skills/api-design/SKILL.md' },
|
||||
success: false,
|
||||
error: 'missing error response examples',
|
||||
feedback: 'Need explicit examples'
|
||||
})
|
||||
];
|
||||
|
||||
const proposal = proposeSkillAmendment('api-design', records);
|
||||
|
||||
assert.strictEqual(proposal.schemaVersion, 'ecc.skill-amendment-proposal.v1');
|
||||
assert.strictEqual(proposal.skill.id, 'api-design');
|
||||
assert.strictEqual(proposal.status, 'proposed');
|
||||
assert.ok(proposal.patch.preview.includes('## Failure-Driven Amendments'));
|
||||
assert.ok(proposal.patch.preview.includes('rate limiting'));
|
||||
assert.ok(proposal.patch.preview.includes('error response'));
|
||||
});
|
||||
|
||||
test('evaluation scaffold compares amended and baseline performance', () => {
|
||||
const records = [
|
||||
createSkillObservation({
|
||||
task: 'Fix flaky login test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
success: false,
|
||||
variant: 'baseline'
|
||||
}),
|
||||
createSkillObservation({
|
||||
task: 'Fix flaky checkout test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
success: true,
|
||||
variant: 'baseline'
|
||||
}),
|
||||
createSkillObservation({
|
||||
task: 'Fix flaky login test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
success: true,
|
||||
variant: 'amended',
|
||||
amendmentId: 'amend-1'
|
||||
}),
|
||||
createSkillObservation({
|
||||
task: 'Fix flaky checkout test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
success: true,
|
||||
variant: 'amended',
|
||||
amendmentId: 'amend-1'
|
||||
})
|
||||
];
|
||||
|
||||
const evaluation = buildSkillEvaluationScaffold('e2e-testing', records, {
|
||||
amendmentId: 'amend-1',
|
||||
minimumRunsPerVariant: 2
|
||||
});
|
||||
|
||||
assert.strictEqual(evaluation.schemaVersion, 'ecc.skill-evaluation.v1');
|
||||
assert.strictEqual(evaluation.baseline.runs, 2);
|
||||
assert.strictEqual(evaluation.amended.runs, 2);
|
||||
assert.strictEqual(evaluation.delta.successRate, 0.5);
|
||||
assert.strictEqual(evaluation.recommendation, 'promote-amendment');
|
||||
});
|
||||
|
||||
console.log(`\n=== Results: ${passed} passed, ${failed} failed ===`);
|
||||
if (failed > 0) process.exit(1);
|
||||
@@ -56,6 +56,15 @@ function runTests() {
|
||||
assert.ok(result.stdout.includes('Usage:'));
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('lists registered adapters', () => {
|
||||
const result = run(['--list-adapters']);
|
||||
assert.strictEqual(result.code, 0, result.stderr);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.ok(Array.isArray(payload.adapters));
|
||||
assert.ok(payload.adapters.some(adapter => adapter.id === 'claude-history'));
|
||||
assert.ok(payload.adapters.some(adapter => adapter.id === 'dmux-tmux'));
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('prints canonical JSON for claude history targets', () => {
|
||||
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-home-'));
|
||||
const sessionsDir = path.join(homeDir, '.claude', 'sessions');
|
||||
@@ -81,6 +90,31 @@ function runTests() {
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('supports explicit target types for structured registry routing', () => {
|
||||
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-home-'));
|
||||
const sessionsDir = path.join(homeDir, '.claude', 'sessions');
|
||||
fs.mkdirSync(sessionsDir, { recursive: true });
|
||||
|
||||
try {
|
||||
fs.writeFileSync(
|
||||
path.join(sessionsDir, '2026-03-13-a1b2c3d4-session.tmp'),
|
||||
'# Inspect Session\n\n**Branch:** feat/typed-inspect\n'
|
||||
);
|
||||
|
||||
const result = run(['latest', '--target-type', 'claude-history'], {
|
||||
env: { HOME: homeDir }
|
||||
});
|
||||
|
||||
assert.strictEqual(result.code, 0, result.stderr);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.strictEqual(payload.adapterId, 'claude-history');
|
||||
assert.strictEqual(payload.session.sourceTarget.type, 'claude-history');
|
||||
assert.strictEqual(payload.workers[0].branch, 'feat/typed-inspect');
|
||||
} finally {
|
||||
fs.rmSync(homeDir, { recursive: true, force: true });
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('writes snapshot JSON to disk when --write is provided', () => {
|
||||
const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-home-'));
|
||||
const outputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-out-'));
|
||||
@@ -109,6 +143,133 @@ function runTests() {
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('inspects skill health from recorded observations', () => {
|
||||
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-skills-'));
|
||||
const observationsDir = path.join(projectRoot, '.claude', 'ecc', 'skills');
|
||||
fs.mkdirSync(observationsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(observationsDir, 'observations.jsonl'),
|
||||
[
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-1',
|
||||
timestamp: '2026-03-14T12:00:00.000Z',
|
||||
task: 'Review auth middleware',
|
||||
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
|
||||
outcome: { success: false, status: 'failure', error: 'missing csrf guidance', feedback: 'Need CSRF coverage' },
|
||||
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-1' }
|
||||
}),
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-2',
|
||||
timestamp: '2026-03-14T12:05:00.000Z',
|
||||
task: 'Review auth middleware',
|
||||
skill: { id: 'security-review', path: 'skills/security-review/SKILL.md' },
|
||||
outcome: { success: false, status: 'failure', error: 'missing csrf guidance', feedback: null },
|
||||
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-2' }
|
||||
})
|
||||
].join('\n') + '\n'
|
||||
);
|
||||
|
||||
try {
|
||||
const result = run(['skills:health'], { cwd: projectRoot });
|
||||
assert.strictEqual(result.code, 0, result.stderr);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.strictEqual(payload.schemaVersion, 'ecc.skill-health.v1');
|
||||
assert.ok(payload.skills.some(skill => skill.skill.id === 'security-review'));
|
||||
} finally {
|
||||
fs.rmSync(projectRoot, { recursive: true, force: true });
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('proposes skill amendments through session-inspect', () => {
|
||||
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-amend-'));
|
||||
const observationsDir = path.join(projectRoot, '.claude', 'ecc', 'skills');
|
||||
fs.mkdirSync(observationsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(observationsDir, 'observations.jsonl'),
|
||||
[
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-1',
|
||||
timestamp: '2026-03-14T12:00:00.000Z',
|
||||
task: 'Add rate limiting',
|
||||
skill: { id: 'api-design', path: 'skills/api-design/SKILL.md' },
|
||||
outcome: { success: false, status: 'failure', error: 'missing rate limiting guidance', feedback: 'Need rate limiting examples' },
|
||||
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-1' }
|
||||
})
|
||||
].join('\n') + '\n'
|
||||
);
|
||||
|
||||
try {
|
||||
const result = run(['skills:amendify', '--skill', 'api-design'], { cwd: projectRoot });
|
||||
assert.strictEqual(result.code, 0, result.stderr);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.strictEqual(payload.schemaVersion, 'ecc.skill-amendment-proposal.v1');
|
||||
assert.strictEqual(payload.skill.id, 'api-design');
|
||||
assert.ok(payload.patch.preview.includes('Failure-Driven Amendments'));
|
||||
} finally {
|
||||
fs.rmSync(projectRoot, { recursive: true, force: true });
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
if (test('builds skill evaluation scaffolding through session-inspect', () => {
|
||||
const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-session-inspect-eval-'));
|
||||
const observationsDir = path.join(projectRoot, '.claude', 'ecc', 'skills');
|
||||
fs.mkdirSync(observationsDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(observationsDir, 'observations.jsonl'),
|
||||
[
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-1',
|
||||
timestamp: '2026-03-14T12:00:00.000Z',
|
||||
task: 'Fix flaky login test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
outcome: { success: false, status: 'failure', error: null, feedback: null },
|
||||
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-1' }
|
||||
}),
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-2',
|
||||
timestamp: '2026-03-14T12:10:00.000Z',
|
||||
task: 'Fix flaky checkout test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
outcome: { success: true, status: 'success', error: null, feedback: null },
|
||||
run: { variant: 'baseline', amendmentId: null, sessionId: 'sess-2' }
|
||||
}),
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-3',
|
||||
timestamp: '2026-03-14T12:20:00.000Z',
|
||||
task: 'Fix flaky login test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
outcome: { success: true, status: 'success', error: null, feedback: null },
|
||||
run: { variant: 'amended', amendmentId: 'amend-1', sessionId: 'sess-3' }
|
||||
}),
|
||||
JSON.stringify({
|
||||
schemaVersion: 'ecc.skill-observation.v1',
|
||||
observationId: 'obs-4',
|
||||
timestamp: '2026-03-14T12:30:00.000Z',
|
||||
task: 'Fix flaky checkout test',
|
||||
skill: { id: 'e2e-testing', path: 'skills/e2e-testing/SKILL.md' },
|
||||
outcome: { success: true, status: 'success', error: null, feedback: null },
|
||||
run: { variant: 'amended', amendmentId: 'amend-1', sessionId: 'sess-4' }
|
||||
})
|
||||
].join('\n') + '\n'
|
||||
);
|
||||
|
||||
try {
|
||||
const result = run(['skills:evaluate', '--skill', 'e2e-testing', '--amendment-id', 'amend-1'], { cwd: projectRoot });
|
||||
assert.strictEqual(result.code, 0, result.stderr);
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.strictEqual(payload.schemaVersion, 'ecc.skill-evaluation.v1');
|
||||
assert.strictEqual(payload.recommendation, 'promote-amendment');
|
||||
} finally {
|
||||
fs.rmSync(projectRoot, { recursive: true, force: true });
|
||||
}
|
||||
})) passed++; else failed++;
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user