fix: harden unicode safety checks

This commit is contained in:
Affaan Mustafa
2026-03-29 08:59:06 -04:00
parent 527c79350c
commit 1e0de43ef2
239 changed files with 3780 additions and 3962 deletions

View File

@@ -76,9 +76,9 @@ function parseReadmeExpectations(readmeContent) {
);
const tablePatterns = [
{ category: 'agents', regex: /\|\s*(?:\*\*)?Agents(?:\*\*)?\s*\|\s*\s*(\d+)\s+agents\s*\|/i, source: 'README.md comparison table' },
{ category: 'commands', regex: /\|\s*(?:\*\*)?Commands(?:\*\*)?\s*\|\s*\s*(\d+)\s+commands\s*\|/i, source: 'README.md comparison table' },
{ category: 'skills', regex: /\|\s*(?:\*\*)?Skills(?:\*\*)?\s*\|\s*\s*(\d+)\s+skills\s*\|/i, source: 'README.md comparison table' }
{ category: 'agents', regex: /\|\s*(?:\*\*)?Agents(?:\*\*)?\s*\|\s*PASS:\s*(\d+)\s+agents\s*\|/i, source: 'README.md comparison table' },
{ category: 'commands', regex: /\|\s*(?:\*\*)?Commands(?:\*\*)?\s*\|\s*PASS:\s*(\d+)\s+commands\s*\|/i, source: 'README.md comparison table' },
{ category: 'skills', regex: /\|\s*(?:\*\*)?Skills(?:\*\*)?\s*\|\s*PASS:\s*(\d+)\s+skills\s*\|/i, source: 'README.md comparison table' }
];
for (const pattern of tablePatterns) {

View File

@@ -0,0 +1,178 @@
#!/usr/bin/env node
const fs = require('fs');
const path = require('path');
const repoRoot = process.env.ECC_UNICODE_SCAN_ROOT
? path.resolve(process.env.ECC_UNICODE_SCAN_ROOT)
: path.resolve(__dirname, '..', '..');
const writeMode = process.argv.includes('--write');
const ignoredDirs = new Set([
'.git',
'node_modules',
'.dmux',
'.next',
'coverage',
]);
const textExtensions = new Set([
'.md',
'.mdx',
'.txt',
'.js',
'.cjs',
'.mjs',
'.ts',
'.tsx',
'.jsx',
'.json',
'.toml',
'.yml',
'.yaml',
'.sh',
'.bash',
'.zsh',
'.ps1',
'.py',
'.rs',
]);
const writeModeSkip = new Set([
path.normalize('scripts/ci/check-unicode-safety.js'),
path.normalize('tests/scripts/check-unicode-safety.test.js'),
]);
const dangerousInvisibleRe =
/[\u200B-\u200D\u2060\uFEFF\u202A-\u202E\u2066-\u2069\uFE00-\uFE0F\u{E0100}-\u{E01EF}]/gu;
const emojiRe = /[\p{Extended_Pictographic}\p{Regional_Indicator}]/gu;
const targetedReplacements = [
[new RegExp(`${String.fromCodePoint(0x26A0)}(?:\\uFE0F)?`, 'gu'), 'WARNING:'],
[new RegExp(`${String.fromCodePoint(0x23ED)}(?:\\uFE0F)?`, 'gu'), 'SKIPPED:'],
[new RegExp(String.fromCodePoint(0x2705), 'gu'), 'PASS:'],
[new RegExp(String.fromCodePoint(0x274C), 'gu'), 'FAIL:'],
[new RegExp(String.fromCodePoint(0x2728), 'gu'), ''],
];
function shouldSkip(entryPath) {
return entryPath.split(path.sep).some(part => ignoredDirs.has(part));
}
function isTextFile(filePath) {
return textExtensions.has(path.extname(filePath).toLowerCase());
}
function listFiles(dirPath) {
const results = [];
for (const entry of fs.readdirSync(dirPath, { withFileTypes: true })) {
const entryPath = path.join(dirPath, entry.name);
if (shouldSkip(entryPath)) continue;
if (entry.isDirectory()) {
results.push(...listFiles(entryPath));
continue;
}
if (entry.isFile() && isTextFile(entryPath)) {
results.push(entryPath);
}
}
return results;
}
function lineAndColumn(text, index) {
const line = text.slice(0, index).split('\n').length;
const lastNewline = text.lastIndexOf('\n', index - 1);
const column = index - lastNewline;
return { line, column };
}
function sanitizeText(text) {
let next = text;
next = next.replace(dangerousInvisibleRe, '');
for (const [pattern, replacement] of targetedReplacements) {
next = next.replace(pattern, replacement);
}
next = next.replace(emojiRe, '');
next = next.replace(/^ +(?=\*\*)/gm, '');
next = next.replace(/^(\*\*)\s+/gm, '$1');
next = next.replace(/^(#+)\s{2,}/gm, '$1 ');
next = next.replace(/^>\s{2,}/gm, '> ');
next = next.replace(/^-\s{2,}/gm, '- ');
next = next.replace(/^(\d+\.)\s{2,}/gm, '$1 ');
next = next.replace(/[ \t]+$/gm, '');
return next;
}
function collectMatches(text, regex, kind) {
const matches = [];
for (const match of text.matchAll(regex)) {
const char = match[0];
const index = match.index ?? 0;
const { line, column } = lineAndColumn(text, index);
matches.push({
kind,
char,
codePoint: `U+${char.codePointAt(0).toString(16).toUpperCase()}`,
line,
column,
});
}
return matches;
}
const changedFiles = [];
const violations = [];
for (const filePath of listFiles(repoRoot)) {
const relativePath = path.relative(repoRoot, filePath);
let text;
try {
text = fs.readFileSync(filePath, 'utf8');
} catch {
continue;
}
if (writeMode && !writeModeSkip.has(path.normalize(relativePath))) {
const sanitized = sanitizeText(text);
if (sanitized !== text) {
fs.writeFileSync(filePath, sanitized, 'utf8');
changedFiles.push(relativePath);
text = sanitized;
}
}
const fileViolations = [
...collectMatches(text, dangerousInvisibleRe, 'dangerous-invisible'),
...collectMatches(text, emojiRe, 'emoji'),
];
for (const violation of fileViolations) {
violations.push({
file: relativePath,
...violation,
});
}
}
if (changedFiles.length > 0) {
console.log(`Sanitized ${changedFiles.length} files:`);
for (const file of changedFiles) {
console.log(`- ${file}`);
}
}
if (violations.length > 0) {
console.error('Unicode safety violations detected:');
for (const violation of violations) {
console.error(
`${violation.file}:${violation.line}:${violation.column} ${violation.kind} ${violation.codePoint}`
);
}
process.exit(1);
}
console.log('Unicode safety check passed.');

View File

@@ -3,8 +3,8 @@ set -euo pipefail
# Install ECC git safety hooks globally via core.hooksPath.
# Usage:
# ./scripts/codex/install-global-git-hooks.sh
# ./scripts/codex/install-global-git-hooks.sh --dry-run
# ./scripts/codex/install-global-git-hooks.sh
# ./scripts/codex/install-global-git-hooks.sh --dry-run
MODE="apply"
if [[ "${1:-}" == "--dry-run" ]]; then

View File

@@ -133,7 +133,7 @@ def write_audit(event: Dict[str, Any]) -> None:
"""
try:
enriched: Dict[str, Any] = {
**event,
**event,
"timestamp": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
}
enriched["hash"] = hashlib.sha256(

View File

@@ -49,7 +49,7 @@ function getStagedFileContent(filePath) {
/**
* Check if a file should be quality-checked
* @param {string} filePath
* @param {string} filePath
* @returns {boolean}
*/
function shouldCheckFile(filePath) {
@@ -59,22 +59,22 @@ function shouldCheckFile(filePath) {
/**
* Find issues in file content
* @param {string} filePath
* @param {string} filePath
* @returns {object[]} Array of issues found
*/
function findFileIssues(filePath) {
const issues = [];
try {
const content = getStagedFileContent(filePath);
if (content == null) {
return issues;
}
const lines = content.split('\n');
lines.forEach((line, index) => {
const lineNum = index + 1;
// Check for console.log
if (line.includes('console.log') && !line.trim().startsWith('//') && !line.trim().startsWith('*')) {
issues.push({
@@ -84,7 +84,7 @@ function findFileIssues(filePath) {
severity: 'warning'
});
}
// Check for debugger statements
if (/\bdebugger\b/.test(line) && !line.trim().startsWith('//')) {
issues.push({
@@ -94,7 +94,7 @@ function findFileIssues(filePath) {
severity: 'error'
});
}
// Check for TODO/FIXME without issue reference
const todoMatch = line.match(/\/\/\s*(TODO|FIXME):?\s*(.+)/);
if (todoMatch && !todoMatch[2].match(/#\d+|issue/i)) {
@@ -105,7 +105,7 @@ function findFileIssues(filePath) {
severity: 'info'
});
}
// Check for hardcoded secrets (basic patterns)
const secretPatterns = [
{ pattern: /sk-[a-zA-Z0-9]{20,}/, name: 'OpenAI API key' },
@@ -113,7 +113,7 @@ function findFileIssues(filePath) {
{ pattern: /AKIA[A-Z0-9]{16}/, name: 'AWS Access Key' },
{ pattern: /api[_-]?key\s*[=:]\s*['"][^'"]+['"]/i, name: 'API key' }
];
for (const { pattern, name } of secretPatterns) {
if (pattern.test(line)) {
issues.push({
@@ -128,23 +128,23 @@ function findFileIssues(filePath) {
} catch {
// File not readable, skip
}
return issues;
}
/**
* Validate commit message format
* @param {string} command
* @param {string} command
* @returns {object|null} Validation result or null if no message to validate
*/
function validateCommitMessage(command) {
// Extract commit message from command
const messageMatch = command.match(/(?:-m|--message)[=\s]+["']?([^"']+)["']?/);
if (!messageMatch) return null;
const message = messageMatch[1];
const issues = [];
// Check conventional commit format
const conventionalCommit = /^(feat|fix|docs|style|refactor|test|chore|build|ci|perf|revert)(\(.+\))?:\s*.+/;
if (!conventionalCommit.test(message)) {
@@ -154,7 +154,7 @@ function validateCommitMessage(command) {
suggestion: 'Use format: type(scope): description (e.g., "feat(auth): add login flow")'
});
}
// Check message length
if (message.length > 72) {
issues.push({
@@ -163,7 +163,7 @@ function validateCommitMessage(command) {
suggestion: 'Keep the first line under 72 characters'
});
}
// Check for lowercase first letter (conventional)
if (conventionalCommit.test(message)) {
const afterColon = message.split(':')[1];
@@ -175,7 +175,7 @@ function validateCommitMessage(command) {
});
}
}
// Check for trailing period
if (message.endsWith('.')) {
issues.push({
@@ -184,26 +184,26 @@ function validateCommitMessage(command) {
suggestion: 'Remove the trailing period'
});
}
return { message, issues };
}
/**
* Run linter on staged files
* @param {string[]} files
* @param {string[]} files
* @returns {object} Lint results
*/
function runLinter(files) {
const jsFiles = files.filter(f => /\.(js|jsx|ts|tsx)$/.test(f));
const pyFiles = files.filter(f => f.endsWith('.py'));
const goFiles = files.filter(f => f.endsWith('.go'));
const results = {
eslint: null,
pylint: null,
golint: null
};
// Run ESLint if available
if (jsFiles.length > 0) {
const eslintBin = process.platform === 'win32' ? 'eslint.cmd' : 'eslint';
@@ -220,7 +220,7 @@ function runLinter(files) {
};
}
}
// Run Pylint if available
if (pyFiles.length > 0) {
try {
@@ -241,7 +241,7 @@ function runLinter(files) {
// Pylint not available
}
}
// Run golint if available
if (goFiles.length > 0) {
try {
@@ -262,7 +262,7 @@ function runLinter(files) {
// golint not available
}
}
return results;
}
@@ -275,40 +275,40 @@ function evaluate(rawInput) {
try {
const input = JSON.parse(rawInput);
const command = input.tool_input?.command || '';
// Only run for git commit commands
if (!command.includes('git commit')) {
return { output: rawInput, exitCode: 0 };
}
// Check if this is an amend (skip checks for amends to avoid blocking)
if (command.includes('--amend')) {
return { output: rawInput, exitCode: 0 };
}
// Get staged files
const stagedFiles = getStagedFiles();
if (stagedFiles.length === 0) {
console.error('[Hook] No staged files found. Use "git add" to stage files first.');
return { output: rawInput, exitCode: 0 };
}
console.error(`[Hook] Checking ${stagedFiles.length} staged file(s)...`);
// Check each staged file
const filesToCheck = stagedFiles.filter(shouldCheckFile);
let totalIssues = 0;
let errorCount = 0;
let warningCount = 0;
let infoCount = 0;
for (const file of filesToCheck) {
const fileIssues = findFileIssues(file);
if (fileIssues.length > 0) {
console.error(`\n📁 ${file}`);
console.error(`\n ${file}`);
for (const issue of fileIssues) {
const icon = issue.severity === 'error' ? '' : issue.severity === 'warning' ? '⚠️' : '';
const icon = issue.severity === 'error' ? 'FAIL:' : issue.severity === 'warning' ? 'WARNING:' : '';
console.error(` ${icon} Line ${issue.line}: ${issue.message}`);
totalIssues++;
if (issue.severity === 'error') errorCount++;
@@ -317,65 +317,65 @@ function evaluate(rawInput) {
}
}
}
// Validate commit message if provided
const messageValidation = validateCommitMessage(command);
if (messageValidation && messageValidation.issues.length > 0) {
console.error('\n📝 Commit Message Issues:');
console.error('\n Commit Message Issues:');
for (const issue of messageValidation.issues) {
console.error(` ⚠️ ${issue.message}`);
console.error(` WARNING: ${issue.message}`);
if (issue.suggestion) {
console.error(` 💡 ${issue.suggestion}`);
console.error(` ${issue.suggestion}`);
}
totalIssues++;
warningCount++;
}
}
// Run linter
const lintResults = runLinter(filesToCheck);
if (lintResults.eslint && !lintResults.eslint.success) {
console.error('\n🔍 ESLint Issues:');
console.error('\n ESLint Issues:');
console.error(lintResults.eslint.output);
totalIssues++;
errorCount++;
}
if (lintResults.pylint && !lintResults.pylint.success) {
console.error('\n🔍 Pylint Issues:');
console.error('\n Pylint Issues:');
console.error(lintResults.pylint.output);
totalIssues++;
errorCount++;
}
if (lintResults.golint && !lintResults.golint.success) {
console.error('\n🔍 golint Issues:');
console.error('\n golint Issues:');
console.error(lintResults.golint.output);
totalIssues++;
errorCount++;
}
// Summary
if (totalIssues > 0) {
console.error(`\n📊 Summary: ${totalIssues} issue(s) found (${errorCount} error(s), ${warningCount} warning(s), ${infoCount} info)`);
console.error(`\n Summary: ${totalIssues} issue(s) found (${errorCount} error(s), ${warningCount} warning(s), ${infoCount} info)`);
if (errorCount > 0) {
console.error('\n[Hook] Commit blocked due to critical issues. Fix them before committing.');
console.error('\n[Hook] FAIL: Commit blocked due to critical issues. Fix them before committing.');
return { output: rawInput, exitCode: 2 };
} else {
console.error('\n[Hook] ⚠️ Warnings found. Consider fixing them, but commit is allowed.');
console.error('\n[Hook] WARNING: Warnings found. Consider fixing them, but commit is allowed.');
console.error('[Hook] To bypass these checks, use: git commit --no-verify');
}
} else {
console.error('\n[Hook] All checks passed!');
console.error('\n[Hook] PASS: All checks passed!');
}
} catch (error) {
console.error(`[Hook] Error: ${error.message}`);
// Non-blocking on error
}
return { output: rawInput, exitCode: 0 };
}
@@ -387,14 +387,14 @@ function run(rawInput) {
if (require.main === module) {
let data = '';
process.stdin.setEncoding('utf8');
process.stdin.on('data', chunk => {
if (data.length < MAX_STDIN) {
const remaining = MAX_STDIN - data.length;
data += chunk.substring(0, remaining);
}
});
process.stdin.on('end', () => {
const result = evaluate(data);
process.stdout.write(result.output);

View File

@@ -64,7 +64,7 @@ function sleep(ms) {
}
async function animateProgress(label, steps, callback) {
process.stdout.write(`\n${chalk.cyan('')} ${label}...\n`);
process.stdout.write(`\n${chalk.cyan('')} ${label}...\n`);
for (let i = 0; i < steps.length; i++) {
const step = steps[i];
@@ -90,7 +90,7 @@ class SkillCreateOutput {
console.log('\n');
console.log(chalk.bold(chalk.magenta('╔════════════════════════════════════════════════════════════════╗')));
console.log(chalk.bold(chalk.magenta('║')) + chalk.bold(' 🔮 ECC Skill Creator ') + chalk.bold(chalk.magenta('║')));
console.log(chalk.bold(chalk.magenta('║')) + chalk.bold(' ECC Skill Creator ') + chalk.bold(chalk.magenta('║')));
console.log(chalk.bold(chalk.magenta('║')) + ` ${subtitle}${' '.repeat(Math.max(0, 59 - stripAnsi(subtitle).length))}` + chalk.bold(chalk.magenta('║')));
console.log(chalk.bold(chalk.magenta('╚════════════════════════════════════════════════════════════════╝')));
console.log('');
@@ -111,7 +111,7 @@ class SkillCreateOutput {
analysisResults(data) {
console.log('\n');
console.log(box('📊 Analysis Results', `
console.log(box(' Analysis Results', `
${chalk.bold('Commits Analyzed:')} ${chalk.yellow(data.commits)}
${chalk.bold('Time Range:')} ${chalk.gray(data.timeRange)}
${chalk.bold('Contributors:')} ${chalk.cyan(data.contributors)}
@@ -121,7 +121,7 @@ ${chalk.bold('Files Tracked:')} ${chalk.green(data.files)}
patterns(patterns) {
console.log('\n');
console.log(chalk.bold(chalk.cyan('🔍 Key Patterns Discovered:')));
console.log(chalk.bold(chalk.cyan(' Key Patterns Discovered:')));
console.log(chalk.gray('─'.repeat(50)));
patterns.forEach((pattern, i) => {
@@ -137,26 +137,26 @@ ${chalk.bold('Files Tracked:')} ${chalk.green(data.files)}
instincts(instincts) {
console.log('\n');
console.log(box('🧠 Instincts Generated', instincts.map((inst, i) =>
console.log(box(' Instincts Generated', instincts.map((inst, i) =>
`${chalk.yellow(`${i + 1}.`)} ${chalk.bold(inst.name)} ${chalk.gray(`(${Math.round(inst.confidence * 100)}%)`)}`
).join('\n')));
}
output(skillPath, instinctsPath) {
console.log('\n');
console.log(chalk.bold(chalk.green(' Generation Complete!')));
console.log(chalk.bold(chalk.green(' Generation Complete!')));
console.log(chalk.gray('─'.repeat(50)));
console.log(`
${chalk.green('📄')} ${chalk.bold('Skill File:')}
${chalk.green('')} ${chalk.bold('Skill File:')}
${chalk.cyan(skillPath)}
${chalk.green('🧠')} ${chalk.bold('Instincts File:')}
${chalk.green('')} ${chalk.bold('Instincts File:')}
${chalk.cyan(instinctsPath)}
`);
}
nextSteps() {
console.log(box('📋 Next Steps', `
console.log(box(' Next Steps', `
${chalk.yellow('1.')} Review the generated SKILL.md
${chalk.yellow('2.')} Import instincts: ${chalk.cyan('/instinct-import <path>')}
${chalk.yellow('3.')} View learned patterns: ${chalk.cyan('/instinct-status')}