fix: narrow unicode cleanup scope

This commit is contained in:
Affaan Mustafa
2026-03-29 09:06:44 -04:00
parent 6325b197c1
commit e22ab5e5cb
29 changed files with 249 additions and 180 deletions

View File

@@ -7,7 +7,9 @@
# ./install.sh # Install to current directory
# ./install.sh /path/to/dir # Install to specific directory
# ./install.sh ~ # Install globally to ~/.kiro/
# set -euo pipefail
#
set -euo pipefail
# When globs match nothing, expand to empty list instead of the literal pattern
shopt -s nullglob

View File

@@ -9,7 +9,9 @@
#
# Environment:
# TRAE_ENV=cn # Force use .trae-cn directory
# set -euo pipefail
#
set -euo pipefail
# When globs match nothing, expand to empty list instead of the literal pattern
shopt -s nullglob

View File

@@ -9,7 +9,9 @@
#
# Environment:
# TRAE_ENV=cn # Force use .trae-cn directory
# set -euo pipefail
#
set -euo pipefail
# Resolve the directory where this script lives
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"

View File

@@ -76,9 +76,9 @@ function parseReadmeExpectations(readmeContent) {
);
const tablePatterns = [
{ category: 'agents', regex: /\|\s*(?:\*\*)?Agents(?:\*\*)?\s*\|\s*PASS:\s*(\d+)\s+agents\s*\|/i, source: 'README.md comparison table' },
{ category: 'commands', regex: /\|\s*(?:\*\*)?Commands(?:\*\*)?\s*\|\s*PASS:\s*(\d+)\s+commands\s*\|/i, source: 'README.md comparison table' },
{ category: 'skills', regex: /\|\s*(?:\*\*)?Skills(?:\*\*)?\s*\|\s*PASS:\s*(\d+)\s+skills\s*\|/i, source: 'README.md comparison table' }
{ category: 'agents', regex: /\|\s*(?:\*\*)?Agents(?:\*\*)?\s*\|\s*(?:(?:PASS:|\u2705)\s*)?(\d+)\s+agents\s*\|/i, source: 'README.md comparison table' },
{ category: 'commands', regex: /\|\s*(?:\*\*)?Commands(?:\*\*)?\s*\|\s*(?:(?:PASS:|\u2705)\s*)?(\d+)\s+commands\s*\|/i, source: 'README.md comparison table' },
{ category: 'skills', regex: /\|\s*(?:\*\*)?Skills(?:\*\*)?\s*\|\s*(?:(?:PASS:|\u2705)\s*)?(\d+)\s+skills\s*\|/i, source: 'README.md comparison table' }
];
for (const pattern of tablePatterns) {

View File

@@ -39,6 +39,12 @@ const textExtensions = new Set([
'.rs',
]);
const writableExtensions = new Set([
'.md',
'.mdx',
'.txt',
]);
const writeModeSkip = new Set([
path.normalize('scripts/ci/check-unicode-safety.js'),
path.normalize('tests/scripts/check-unicode-safety.test.js'),
@@ -47,6 +53,11 @@ const writeModeSkip = new Set([
const dangerousInvisibleRe =
/[\u200B-\u200D\u2060\uFEFF\u202A-\u202E\u2066-\u2069\uFE00-\uFE0F\u{E0100}-\u{E01EF}]/gu;
const emojiRe = /[\p{Extended_Pictographic}\p{Regional_Indicator}]/gu;
const allowedSymbolCodePoints = new Set([
0x00A9,
0x00AE,
0x2122,
]);
const targetedReplacements = [
[new RegExp(`${String.fromCodePoint(0x26A0)}(?:\\uFE0F)?`, 'gu'), 'WARNING:'],
@@ -64,6 +75,10 @@ function isTextFile(filePath) {
return textExtensions.has(path.extname(filePath).toLowerCase());
}
function canAutoWrite(relativePath) {
return writableExtensions.has(path.extname(relativePath).toLowerCase());
}
function listFiles(dirPath) {
const results = [];
for (const entry of fs.readdirSync(dirPath, { withFileTypes: true })) {
@@ -87,6 +102,10 @@ function lineAndColumn(text, index) {
return { line, column };
}
function isAllowedEmojiLikeSymbol(char) {
return allowedSymbolCodePoints.has(char.codePointAt(0));
}
function sanitizeText(text) {
let next = text;
next = next.replace(dangerousInvisibleRe, '');
@@ -95,7 +114,7 @@ function sanitizeText(text) {
next = next.replace(pattern, replacement);
}
next = next.replace(emojiRe, '');
next = next.replace(emojiRe, match => (isAllowedEmojiLikeSymbol(match) ? match : ''));
next = next.replace(/^ +(?=\*\*)/gm, '');
next = next.replace(/^(\*\*)\s+/gm, '$1');
next = next.replace(/^(#+)\s{2,}/gm, '$1 ');
@@ -111,6 +130,9 @@ function collectMatches(text, regex, kind) {
const matches = [];
for (const match of text.matchAll(regex)) {
const char = match[0];
if (kind === 'emoji' && isAllowedEmojiLikeSymbol(char)) {
continue;
}
const index = match.index ?? 0;
const { line, column } = lineAndColumn(text, index);
matches.push({
@@ -136,7 +158,11 @@ for (const filePath of listFiles(repoRoot)) {
continue;
}
if (writeMode && !writeModeSkip.has(path.normalize(relativePath))) {
if (
writeMode &&
!writeModeSkip.has(path.normalize(relativePath)) &&
canAutoWrite(relativePath)
) {
const sanitized = sanitizeText(text);
if (sanitized !== text) {
fs.writeFileSync(filePath, sanitized, 'utf8');

View File

@@ -306,10 +306,10 @@ function evaluate(rawInput) {
for (const file of filesToCheck) {
const fileIssues = findFileIssues(file);
if (fileIssues.length > 0) {
console.error(`\n ${file}`);
console.error(`\n[FILE] ${file}`);
for (const issue of fileIssues) {
const icon = issue.severity === 'error' ? 'FAIL:' : issue.severity === 'warning' ? 'WARNING:' : '';
console.error(` ${icon} Line ${issue.line}: ${issue.message}`);
const label = issue.severity === 'error' ? 'ERROR' : issue.severity === 'warning' ? 'WARNING' : 'INFO';
console.error(` ${label} Line ${issue.line}: ${issue.message}`);
totalIssues++;
if (issue.severity === 'error') errorCount++;
if (issue.severity === 'warning') warningCount++;
@@ -323,9 +323,9 @@ function evaluate(rawInput) {
if (messageValidation && messageValidation.issues.length > 0) {
console.error('\nCommit Message Issues:');
for (const issue of messageValidation.issues) {
console.error(` WARNING: ${issue.message}`);
console.error(` WARNING ${issue.message}`);
if (issue.suggestion) {
console.error(` ${issue.suggestion}`);
console.error(` TIP ${issue.suggestion}`);
}
totalIssues++;
warningCount++;
@@ -361,7 +361,7 @@ function evaluate(rawInput) {
console.error(`\nSummary: ${totalIssues} issue(s) found (${errorCount} error(s), ${warningCount} warning(s), ${infoCount} info)`);
if (errorCount > 0) {
console.error('\n[Hook] FAIL: Commit blocked due to critical issues. Fix them before committing.');
console.error('\n[Hook] ERROR: Commit blocked due to critical issues. Fix them before committing.');
return { output: rawInput, exitCode: 2 };
} else {
console.error('\n[Hook] WARNING: Warnings found. Consider fixing them, but commit is allowed.');

View File

@@ -64,7 +64,7 @@ function sleep(ms) {
}
async function animateProgress(label, steps, callback) {
process.stdout.write(`\n${chalk.cyan('')} ${label}...\n`);
process.stdout.write(`\n${chalk.cyan('[RUN]')} ${label}...\n`);
for (let i = 0; i < steps.length; i++) {
const step = steps[i];
@@ -72,7 +72,7 @@ async function animateProgress(label, steps, callback) {
await sleep(step.duration || 500);
process.stdout.clearLine?.(0) || process.stdout.write('\r');
process.stdout.cursorTo?.(0) || process.stdout.write('\r');
process.stdout.write(` ${chalk.green('')} ${step.name}\n`);
process.stdout.write(` ${chalk.green('[DONE]')} ${step.name}\n`);
if (callback) callback(step, i);
}
}
@@ -147,10 +147,10 @@ ${chalk.bold('Files Tracked:')} ${chalk.green(data.files)}
console.log(chalk.bold(chalk.green('Generation Complete!')));
console.log(chalk.gray('─'.repeat(50)));
console.log(`
${chalk.green('')} ${chalk.bold('Skill File:')}
${chalk.green('-')} ${chalk.bold('Skill File:')}
${chalk.cyan(skillPath)}
${chalk.green('')} ${chalk.bold('Instincts File:')}
${chalk.green('-')} ${chalk.bold('Instincts File:')}
${chalk.cyan(instinctsPath)}
`);
}

View File

@@ -28,7 +28,7 @@ if (projectPath && projectPath !== cwd) {
if (existsSync(projectPath)) {
console.log(`→ cd ${projectPath}`);
} else {
console.log(`WARNING: Path not found: ${projectPath}`);
console.log(`WARNING Path not found: ${projectPath}`);
}
}

View File

@@ -130,7 +130,7 @@ function main() {
// Check if previous session ID exists in sessions array
const alreadySaved = context.sessions?.some(s => s.id === prevSession.sessionId);
if (!alreadySaved) {
summaryLines.push(`WARNING: Last session wasn't saved — run /ck:save to capture it`);
summaryLines.push(`WARNING Last session wasn't saved — run /ck:save to capture it`);
}
}
@@ -142,7 +142,7 @@ function main() {
const claudeMdGoal = extractClaudeMdGoal(cwd);
if (claudeMdGoal && context.goal &&
claudeMdGoal.toLowerCase().trim() !== context.goal.toLowerCase().trim()) {
summaryLines.push(`WARNING: Goal mismatch — ck: "${context.goal.slice(0, 40)}" · CLAUDE.md: "${claudeMdGoal.slice(0, 40)}"`);
summaryLines.push(`WARNING Goal mismatch — ck: "${context.goal.slice(0, 40)}" · CLAUDE.md: "${claudeMdGoal.slice(0, 40)}"`);
summaryLines.push(` Run /ck:save with updated goal to sync`);
}
@@ -165,7 +165,7 @@ function main() {
'```',
``,
`After the block, add one line: "Ready — what are we working on?"`,
`If you see WARNING: warnings above, mention them briefly after the block.`,
`If you see WARNING lines above, mention them briefly after the block.`,
].join('\n'));
return parts;

View File

@@ -48,6 +48,7 @@ function resetAliases() {
}
function runTests() {
const rocketEmoji = String.fromCodePoint(0x1F680);
console.log('\n=== Testing session-aliases.js ===\n');
let passed = 0;
@@ -1441,7 +1442,7 @@ function runTests() {
'CJK characters should be rejected');
// Emoji
const emojiResult = aliases.resolveAlias('rocket-');
const emojiResult = aliases.resolveAlias(`rocket-${rocketEmoji}`);
assert.strictEqual(emojiResult, null,
'Emoji should be rejected by the ASCII-only regex');

View File

@@ -27,6 +27,8 @@ function test(name, fn) {
// Test suite
function runTests() {
const rocketParty = String.fromCodePoint(0x1F680, 0x1F389);
const partyEmoji = String.fromCodePoint(0x1F389);
console.log('\n=== Testing utils.js ===\n');
let passed = 0;
@@ -166,9 +168,12 @@ function runTests() {
if (test('sanitizeSessionId returns stable hashes for non-ASCII values', () => {
const chinese = utils.sanitizeSessionId('我的项目');
const cyrillic = utils.sanitizeSessionId('проект');
const emoji = utils.sanitizeSessionId(rocketParty);
assert.ok(/^[a-f0-9]{8}$/.test(chinese), `Expected 8-char hash, got: ${chinese}`);
assert.ok(/^[a-f0-9]{8}$/.test(cyrillic), `Expected 8-char hash, got: ${cyrillic}`);
assert.ok(/^[a-f0-9]{8}$/.test(emoji), `Expected 8-char hash, got: ${emoji}`);
assert.notStrictEqual(chinese, cyrillic);
assert.notStrictEqual(chinese, emoji);
assert.strictEqual(utils.sanitizeSessionId('日本語プロジェクト'), utils.sanitizeSessionId('日本語プロジェクト'));
})) passed++; else failed++;
@@ -704,7 +709,7 @@ function runTests() {
if (test('writeFile handles unicode content', () => {
const testFile = path.join(utils.getTempDir(), `utils-test-${Date.now()}.txt`);
try {
const unicode = '日本語テスト 中文 émojis';
const unicode = `日本語テスト ${String.fromCodePoint(0x1F680)} émojis`;
utils.writeFile(testFile, unicode);
const content = utils.readFile(testFile);
assert.strictEqual(content, unicode);
@@ -1868,18 +1873,18 @@ function runTests() {
}
})) passed++; else failed++;
// ── Round 108: grepFile with Unicode content — UTF-16 string matching on split lines ──
console.log('\nRound 108: grepFile (Unicode — regex matching on UTF-16 split lines):');
if (test('grepFile finds Unicode patterns across lines', () => {
// ── Round 108: grepFile with Unicode/emoji content — UTF-16 string matching on split lines ──
console.log('\nRound 108: grepFile (Unicode/emoji — regex matching on UTF-16 split lines):');
if (test('grepFile finds Unicode emoji patterns across lines', () => {
const tmpDir = fs.mkdtempSync(path.join(utils.getTempDir(), 'r108-grep-unicode-'));
const testFile = path.join(tmpDir, 'test.txt');
try {
fs.writeFileSync(testFile, '猫 celebration\nnormal line\n猫 party\n日本語テスト');
const unicodeResults = utils.grepFile(testFile, /猫/);
assert.strictEqual(unicodeResults.length, 2,
'Should find Unicode matches on 2 lines (lines 1 and 3)');
assert.strictEqual(unicodeResults[0].lineNumber, 1);
assert.strictEqual(unicodeResults[1].lineNumber, 3);
fs.writeFileSync(testFile, `${partyEmoji} celebration\nnormal line\n${partyEmoji} party\n日本語テスト`);
const emojiResults = utils.grepFile(testFile, new RegExp(partyEmoji, 'u'));
assert.strictEqual(emojiResults.length, 2,
'Should find emoji on 2 lines (lines 1 and 3)');
assert.strictEqual(emojiResults[0].lineNumber, 1);
assert.strictEqual(emojiResults[1].lineNumber, 3);
const cjkResults = utils.grepFile(testFile, /日本語/);
assert.strictEqual(cjkResults.length, 1,
'Should find CJK characters on line 4');

View File

@@ -67,7 +67,7 @@ for (const testFile of testFiles) {
const displayPath = testFile.split(path.sep).join('/');
if (!fs.existsSync(testPath)) {
console.log(`WARNING: Skipping ${displayPath} (file not found)`);
console.log(`WARNING Skipping ${displayPath} (file not found)`);
continue;
}

View File

@@ -35,6 +35,7 @@ function makeTempRoot(prefix) {
const warningEmoji = String.fromCodePoint(0x26A0, 0xFE0F);
const toolsEmoji = String.fromCodePoint(0x1F6E0, 0xFE0F);
const zeroWidthSpace = String.fromCodePoint(0x200B);
const rocketEmoji = String.fromCodePoint(0x1F680);
let passed = 0;
let failed = 0;
@@ -78,6 +79,36 @@ if (
passed++;
else failed++;
if (
test('write mode does not rewrite executable files', () => {
const root = makeTempRoot('ecc-unicode-code-');
fs.mkdirSync(path.join(root, 'scripts'), { recursive: true });
const scriptFile = path.join(root, 'scripts', 'sample.js');
const original = `const label = "Launch ${rocketEmoji}";\n`;
fs.writeFileSync(scriptFile, original);
const result = runCheck(root, ['--write']);
assert.notStrictEqual(result.status, 0, result.stdout + result.stderr);
assert.match(result.stderr, /scripts\/sample\.js:1:23 emoji U\+1F680/);
assert.strictEqual(fs.readFileSync(scriptFile, 'utf8'), original);
})
)
passed++;
else failed++;
if (
test('plain symbols like copyright remain allowed', () => {
const root = makeTempRoot('ecc-unicode-symbols-');
fs.mkdirSync(path.join(root, 'docs'), { recursive: true });
fs.writeFileSync(path.join(root, 'docs', 'legal.md'), 'Copyright © ECC\nTrademark ® ECC\n');
const result = runCheck(root);
assert.strictEqual(result.status, 0, result.stdout + result.stderr);
})
)
passed++;
else failed++;
console.log(`\nPassed: ${passed}`);
console.log(`Failed: ${failed}`);
process.exit(failed > 0 ? 1 : 0);