From 01ed1b3b03064b6d9e7d74e19b4f9ffc559b7858 Mon Sep 17 00:00:00 2001 From: Justin Philpott <8027086+justinphilpott@users.noreply.github.com> Date: Mon, 16 Mar 2026 20:37:51 +0000 Subject: [PATCH] fix(ci): enforce catalog count integrity (#525) * fix(ci): enforce catalog count integrity * test: harden catalog structure parsing --- .github/workflows/ci.yml | 4 ++ AGENTS.md | 8 +-- README.md | 14 ++--- package.json | 2 +- scripts/ci/catalog.js | 45 ++++++++++++-- tests/ci/validators.test.js | 116 ++++++++++++++++++++++++++++++++++++ 6 files changed, 173 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7498eee2..cbc9f0d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -179,6 +179,10 @@ jobs: run: node scripts/ci/validate-rules.js continue-on-error: false + - name: Validate catalog counts + run: node scripts/ci/catalog.js --text + continue-on-error: false + security: name: Security Scan runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 56e7bb45..533e79e9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # Everything Claude Code (ECC) — Agent Instructions -This is a **production-ready AI coding plugin** providing 20 specialized agents, 65+ skills, 51 commands, and automated hook workflows for software development. +This is a **production-ready AI coding plugin** providing 21 specialized agents, 102 skills, 52 commands, and automated hook workflows for software development. ## Core Principles @@ -132,9 +132,9 @@ Troubleshoot failures: check test isolation → verify mocks → fix implementat ## Project Structure ``` -agents/ — 20 specialized subagents -skills/ — 65+ workflow skills and domain knowledge -commands/ — 51 slash commands +agents/ — 21 specialized subagents +skills/ — 102 workflow skills and domain knowledge +commands/ — 52 slash commands hooks/ — Trigger-based automations rules/ — Always-follow guidelines (common + per-language) scripts/ — Cross-platform Node.js utilities diff --git a/README.md b/README.md index 5d0b2605..b0f3aabc 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ For manual install instructions see the README in the `rules/` folder. /plugin list everything-claude-code@everything-claude-code ``` -✨ **That's it!** You now have access to 16 agents, 65 skills, and 40 commands. +✨ **That's it!** You now have access to 21 agents, 102 skills, and 52 commands. --- @@ -1039,9 +1039,9 @@ The configuration is automatically detected from `.opencode/opencode.json`. | Feature | Claude Code | OpenCode | Status | |---------|-------------|----------|--------| -| Agents | ✅ 16 agents | ✅ 12 agents | **Claude Code leads** | -| Commands | ✅ 40 commands | ✅ 31 commands | **Claude Code leads** | -| Skills | ✅ 65 skills | ✅ 37 skills | **Claude Code leads** | +| Agents | ✅ 21 agents | ✅ 12 agents | **Claude Code leads** | +| Commands | ✅ 52 commands | ✅ 31 commands | **Claude Code leads** | +| Skills | ✅ 102 skills | ✅ 37 skills | **Claude Code leads** | | Hooks | ✅ 8 event types | ✅ 11 events | **OpenCode has more!** | | Rules | ✅ 29 rules | ✅ 13 instructions | **Claude Code leads** | | MCP Servers | ✅ 14 servers | ✅ Full | **Full parity** | @@ -1147,9 +1147,9 @@ ECC is the **first plugin to maximize every major AI coding tool**. Here's how e | Feature | Claude Code | Cursor IDE | Codex CLI | OpenCode | |---------|------------|------------|-----------|----------| -| **Agents** | 16 | Shared (AGENTS.md) | Shared (AGENTS.md) | 12 | -| **Commands** | 40 | Shared | Instruction-based | 31 | -| **Skills** | 65 | Shared | 10 (native format) | 37 | +| **Agents** | 21 | Shared (AGENTS.md) | Shared (AGENTS.md) | 12 | +| **Commands** | 52 | Shared | Instruction-based | 31 | +| **Skills** | 102 | Shared | 10 (native format) | 37 | | **Hook Events** | 8 types | 15 types | None yet | 11 types | | **Hook Scripts** | 20+ scripts | 16 scripts (DRY adapter) | N/A | Plugin hooks | | **Rules** | 34 (common + lang) | 34 (YAML frontmatter) | Instruction-based | 13 instructions | diff --git a/package.json b/package.json index 4cb62772..cc3814f5 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,7 @@ "orchestrate:status": "node scripts/orchestration-status.js", "orchestrate:worker": "bash scripts/orchestrate-codex-worker.sh", "orchestrate:tmux": "node scripts/orchestrate-worktrees.js", - "test": "node scripts/ci/validate-agents.js && node scripts/ci/validate-commands.js && node scripts/ci/validate-rules.js && node scripts/ci/validate-skills.js && node scripts/ci/validate-hooks.js && node scripts/ci/validate-install-manifests.js && node scripts/ci/validate-no-personal-paths.js && node tests/run-all.js", + "test": "node scripts/ci/validate-agents.js && node scripts/ci/validate-commands.js && node scripts/ci/validate-rules.js && node scripts/ci/validate-skills.js && node scripts/ci/validate-hooks.js && node scripts/ci/validate-install-manifests.js && node scripts/ci/validate-no-personal-paths.js && node scripts/ci/catalog.js --text && node tests/run-all.js", "coverage": "c8 --all --include=\"scripts/**/*.js\" --check-coverage --lines 80 --functions 80 --branches 80 --statements 80 --reporter=text --reporter=lcov node tests/run-all.js" }, "dependencies": { diff --git a/scripts/ci/catalog.js b/scripts/ci/catalog.js index f15192c1..ed0ff74c 100644 --- a/scripts/ci/catalog.js +++ b/scripts/ci/catalog.js @@ -76,9 +76,9 @@ function parseReadmeExpectations(readmeContent) { ); const tablePatterns = [ - { category: 'agents', regex: /\|\s*Agents\s*\|\s*✅\s*(\d+)\s+agents\s*\|/i, source: 'README.md comparison table' }, - { category: 'commands', regex: /\|\s*Commands\s*\|\s*✅\s*(\d+)\s+commands\s*\|/i, source: 'README.md comparison table' }, - { category: 'skills', regex: /\|\s*Skills\s*\|\s*✅\s*(\d+)\s+skills\s*\|/i, source: 'README.md comparison table' } + { category: 'agents', regex: /\|\s*(?:\*\*)?Agents(?:\*\*)?\s*\|\s*✅\s*(\d+)\s+agents\s*\|/i, source: 'README.md comparison table' }, + { category: 'commands', regex: /\|\s*(?:\*\*)?Commands(?:\*\*)?\s*\|\s*✅\s*(\d+)\s+commands\s*\|/i, source: 'README.md comparison table' }, + { category: 'skills', regex: /\|\s*(?:\*\*)?Skills(?:\*\*)?\s*\|\s*✅\s*(\d+)\s+skills\s*\|/i, source: 'README.md comparison table' } ]; for (const pattern of tablePatterns) { @@ -104,7 +104,7 @@ function parseAgentsDocExpectations(agentsContent) { throw new Error('AGENTS.md is missing the catalog summary line'); } - return [ + const expectations = [ { category: 'agents', mode: 'exact', expected: Number(summaryMatch[1]), source: 'AGENTS.md summary' }, { category: 'skills', @@ -114,6 +114,43 @@ function parseAgentsDocExpectations(agentsContent) { }, { category: 'commands', mode: 'exact', expected: Number(summaryMatch[4]), source: 'AGENTS.md summary' } ]; + + const structurePatterns = [ + { + category: 'agents', + mode: 'exact', + regex: /^\s*agents\/\s*[—–-]\s*(\d+)\s+specialized subagents\s*$/im, + source: 'AGENTS.md project structure' + }, + { + category: 'skills', + mode: 'minimum', + regex: /^\s*skills\/\s*[—–-]\s*(\d+)(\+)?\s+workflow skills and domain knowledge\s*$/im, + source: 'AGENTS.md project structure' + }, + { + category: 'commands', + mode: 'exact', + regex: /^\s*commands\/\s*[—–-]\s*(\d+)\s+slash commands\s*$/im, + source: 'AGENTS.md project structure' + } + ]; + + for (const pattern of structurePatterns) { + const match = agentsContent.match(pattern.regex); + if (!match) { + throw new Error(`${pattern.source} is missing the ${pattern.category} entry`); + } + + expectations.push({ + category: pattern.category, + mode: pattern.mode === 'minimum' && match[2] ? 'minimum' : pattern.mode, + expected: Number(match[1]), + source: `${pattern.source} (${pattern.category})` + }); + } + + return expectations; } function evaluateExpectations(catalog, expectations) { diff --git a/tests/ci/validators.test.js b/tests/ci/validators.test.js index ca52941b..0e0fc378 100644 --- a/tests/ci/validators.test.js +++ b/tests/ci/validators.test.js @@ -140,6 +140,68 @@ function runValidator(validatorName) { } } +function runCatalogValidator(overrides = {}) { + const validatorPath = path.join(validatorsDir, 'catalog.js'); + let source = fs.readFileSync(validatorPath, 'utf8'); + source = source.replace(/^#!.*\n/, ''); + source = `process.argv.push('--text');\n${source}`; + + const resolvedOverrides = { + ROOT: repoRoot, + README_PATH: path.join(repoRoot, 'README.md'), + AGENTS_PATH: path.join(repoRoot, 'AGENTS.md'), + ...overrides, + }; + + for (const [constant, overridePath] of Object.entries(resolvedOverrides)) { + const dirRegex = new RegExp(`const ${constant} = .*?;`); + source = source.replace(dirRegex, `const ${constant} = ${JSON.stringify(overridePath)};`); + } + + try { + const stdout = execFileSync('node', ['-e', source], { + encoding: 'utf8', + stdio: ['pipe', 'pipe', 'pipe'], + timeout: 10000, + }); + return { code: 0, stdout, stderr: '' }; + } catch (err) { + return { + code: err.status || 1, + stdout: err.stdout || '', + stderr: err.stderr || '', + }; + } +} + +function writeCatalogFixture(testDir, options = {}) { + const { + readmeCounts = { agents: 1, skills: 1, commands: 1 }, + summaryCounts = { agents: 1, skills: 1, commands: 1 }, + structureLines = [ + 'agents/ — 1 specialized subagents', + 'skills/ — 1 workflow skills and domain knowledge', + 'commands/ — 1 slash commands', + ], + } = options; + + const readmePath = path.join(testDir, 'README.md'); + const agentsPath = path.join(testDir, 'AGENTS.md'); + + fs.mkdirSync(path.join(testDir, 'agents'), { recursive: true }); + fs.mkdirSync(path.join(testDir, 'commands'), { recursive: true }); + fs.mkdirSync(path.join(testDir, 'skills', 'demo-skill'), { recursive: true }); + + fs.writeFileSync(path.join(testDir, 'agents', 'planner.md'), '---\nmodel: sonnet\ntools: Read\n---\n# Planner'); + fs.writeFileSync(path.join(testDir, 'commands', 'plan.md'), '---\ndescription: Plan\n---\n# Plan'); + fs.writeFileSync(path.join(testDir, 'skills', 'demo-skill', 'SKILL.md'), '---\nname: demo-skill\ndescription: Demo skill\norigin: ECC\n---\n# Demo Skill'); + + fs.writeFileSync(readmePath, `Access to ${readmeCounts.agents} agents, ${readmeCounts.skills} skills, and ${readmeCounts.commands} commands.\n| Feature | Claude Code | Cursor IDE | Codex CLI | OpenCode |\n|---------|------------|------------|-----------|----------|\n| Agents | ✅ ${readmeCounts.agents} agents | Shared | Shared | 1 |\n| Commands | ✅ ${readmeCounts.commands} commands | Shared | Shared | 1 |\n| Skills | ✅ ${readmeCounts.skills} skills | Shared | Shared | 1 |\n`); + fs.writeFileSync(agentsPath, `This is a **production-ready AI coding plugin** providing ${summaryCounts.agents} specialized agents, ${summaryCounts.skills} skills, ${summaryCounts.commands} commands, and automated hook workflows for software development.\n\n\`\`\`\n${structureLines.join('\n')}\n\`\`\`\n`); + + return { readmePath, agentsPath }; +} + function runTests() { console.log('\n=== Testing CI Validators ===\n'); @@ -262,6 +324,60 @@ function runTests() { assert.ok(result.stdout.includes('Validated'), 'Should output validation count'); })) passed++; else failed++; + // ========================================== + // catalog.js + // ========================================== + console.log('\ncatalog.js:'); + + if (test('passes on real project catalog counts', () => { + const result = runCatalogValidator(); + assert.strictEqual(result.code, 0, `Should pass, got stderr: ${result.stderr}`); + assert.ok(result.stdout.includes('Documentation counts match the repository catalog.'), 'Should report matching counts'); + })) passed++; else failed++; + + if (test('fails when README and AGENTS catalog counts drift', () => { + const testDir = createTestDir(); + const { readmePath, agentsPath } = writeCatalogFixture(testDir, { + readmeCounts: { agents: 99, skills: 99, commands: 99 }, + summaryCounts: { agents: 99, skills: 99, commands: 99 }, + structureLines: [ + 'agents/ — 99 specialized subagents', + 'skills/ — 99 workflow skills and domain knowledge', + 'commands/ — 99 slash commands', + ], + }); + + const result = runCatalogValidator({ + ROOT: testDir, + README_PATH: readmePath, + AGENTS_PATH: agentsPath, + }); + + assert.strictEqual(result.code, 1, 'Should fail when catalog counts drift'); + assert.ok((result.stdout + result.stderr).includes('Documentation count mismatches found:'), 'Should report mismatches'); + cleanupTestDir(testDir); + })) passed++; else failed++; + + if (test('accepts AGENTS project structure entries with varied spacing and dash styles', () => { + const testDir = createTestDir(); + const { readmePath, agentsPath } = writeCatalogFixture(testDir, { + structureLines: [ + ' agents/ - 1 specialized subagents ', + '\tskills/\t–\t1+ workflow skills and domain knowledge\t', + ' commands/ — 1 slash commands ', + ], + }); + + const result = runCatalogValidator({ + ROOT: testDir, + README_PATH: readmePath, + AGENTS_PATH: agentsPath, + }); + + assert.strictEqual(result.code, 0, `Should accept formatting variations, got stderr: ${result.stderr}`); + cleanupTestDir(testDir); + })) passed++; else failed++; + if (test('exits 0 when hooks.json does not exist', () => { const result = runValidatorWithDir('validate-hooks', 'HOOKS_FILE', '/nonexistent/hooks.json'); assert.strictEqual(result.code, 0, 'Should skip when no hooks.json');