fix: replace better-sqlite3 with sql.js to eliminate native module CI failures

better-sqlite3 requires native C++ compilation (node-gyp, prebuild-install)
which fails in CI across npm/pnpm on all platforms:
- npm ci: lock file out of sync with native transitive deps
- pnpm: native bindings not found at runtime
- Windows: native compilation fails entirely

sql.js is a pure JavaScript/WASM SQLite implementation with zero native
dependencies. The adapter in index.js wraps the sql.js API to match the
better-sqlite3 interface used by migrations.js and queries.js.

Key implementation detail: sql.js db.export() implicitly ends active
transactions, so the adapter defers disk writes (saveToDisk) until
after transaction commit via an inTransaction guard flag.

createStateStore is now async (sql.js requires async WASM init).
Updated status.js, sessions-cli.js, and tests accordingly.
This commit is contained in:
Affaan Mustafa
2026-03-16 01:28:45 -07:00
parent 9799f3d2a8
commit d4ed5976cd
6 changed files with 188 additions and 49 deletions

45
package-lock.json generated
View File

@@ -9,7 +9,11 @@
"version": "1.8.0", "version": "1.8.0",
"hasInstallScript": true, "hasInstallScript": true,
"license": "MIT", "license": "MIT",
"dependencies": {
"sql.js": "^1.14.1"
},
"bin": { "bin": {
"ecc": "scripts/ecc.js",
"ecc-install": "install.sh" "ecc-install": "install.sh"
}, },
"devDependencies": { "devDependencies": {
@@ -18,7 +22,7 @@
"c8": "^10.1.2", "c8": "^10.1.2",
"eslint": "^9.39.2", "eslint": "^9.39.2",
"globals": "^17.1.0", "globals": "^17.1.0",
"markdownlint-cli": "^0.48.0" "markdownlint-cli": "^0.47.0"
}, },
"engines": { "engines": {
"node": ">=18" "node": ">=18"
@@ -1655,22 +1659,23 @@
} }
}, },
"node_modules/markdownlint-cli": { "node_modules/markdownlint-cli": {
"version": "0.48.0", "version": "0.47.0",
"resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz", "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz",
"integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==", "integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"commander": "~14.0.3", "commander": "~14.0.2",
"deep-extend": "~0.6.0", "deep-extend": "~0.6.0",
"ignore": "~7.0.5", "ignore": "~7.0.5",
"js-yaml": "~4.1.1", "js-yaml": "~4.1.1",
"jsonc-parser": "~3.3.1", "jsonc-parser": "~3.3.1",
"jsonpointer": "~5.0.1", "jsonpointer": "~5.0.1",
"markdown-it": "~14.1.1", "markdown-it": "~14.1.0",
"markdownlint": "~0.40.0", "markdownlint": "~0.40.0",
"minimatch": "~10.2.4", "minimatch": "~10.1.1",
"run-con": "~1.3.2", "run-con": "~1.3.2",
"smol-toml": "~1.6.0", "smol-toml": "~1.5.2",
"tinyglobby": "~0.2.15" "tinyglobby": "~0.2.15"
}, },
"bin": { "bin": {
@@ -1685,6 +1690,7 @@
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz",
"integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==",
"dev": true, "dev": true,
"license": "MIT",
"engines": { "engines": {
"node": "18 || 20 || >=22" "node": "18 || 20 || >=22"
} }
@@ -1694,6 +1700,7 @@
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz",
"integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==",
"dev": true, "dev": true,
"license": "MIT",
"dependencies": { "dependencies": {
"balanced-match": "^4.0.2" "balanced-match": "^4.0.2"
}, },
@@ -1712,15 +1719,16 @@
} }
}, },
"node_modules/markdownlint-cli/node_modules/minimatch": { "node_modules/markdownlint-cli/node_modules/minimatch": {
"version": "10.2.4", "version": "10.1.3",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.3.tgz",
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", "integrity": "sha512-IF6URNyBX7Z6XfvjpaNy5meRxPZiIf2OqtOoSLs+hLJ9pJAScnM1RjrFcbCaD85y42KcI+oZmKjFIJKYDFjQfg==",
"dev": true, "dev": true,
"license": "BlueOak-1.0.0",
"dependencies": { "dependencies": {
"brace-expansion": "^5.0.2" "brace-expansion": "^5.0.2"
}, },
"engines": { "engines": {
"node": "18 || 20 || >=22" "node": "20 || >=22"
}, },
"funding": { "funding": {
"url": "https://github.com/sponsors/isaacs" "url": "https://github.com/sponsors/isaacs"
@@ -2579,10 +2587,11 @@
} }
}, },
"node_modules/smol-toml": { "node_modules/smol-toml": {
"version": "1.6.0", "version": "1.5.2",
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz",
"integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==",
"dev": true, "dev": true,
"license": "BSD-3-Clause",
"engines": { "engines": {
"node": ">= 18" "node": ">= 18"
}, },
@@ -2590,6 +2599,12 @@
"url": "https://github.com/sponsors/cyyynthia" "url": "https://github.com/sponsors/cyyynthia"
} }
}, },
"node_modules/sql.js": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.14.1.tgz",
"integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==",
"license": "MIT"
},
"node_modules/string-width": { "node_modules/string-width": {
"version": "8.1.0", "version": "8.1.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz",

View File

@@ -105,7 +105,7 @@
"coverage": "c8 --all --include=\"scripts/**/*.js\" --check-coverage --lines 80 --functions 80 --branches 80 --statements 80 --reporter=text --reporter=lcov node tests/run-all.js" "coverage": "c8 --all --include=\"scripts/**/*.js\" --check-coverage --lines 80 --functions 80 --branches 80 --statements 80 --reporter=text --reporter=lcov node tests/run-all.js"
}, },
"dependencies": { "dependencies": {
"better-sqlite3": "^11.8.1" "sql.js": "^1.14.1"
}, },
"devDependencies": { "devDependencies": {
"@eslint/js": "^9.39.2", "@eslint/js": "^9.39.2",

View File

@@ -3,7 +3,7 @@
const fs = require('fs'); const fs = require('fs');
const os = require('os'); const os = require('os');
const path = require('path'); const path = require('path');
const Database = require('better-sqlite3'); const initSqlJs = require('sql.js');
const { applyMigrations, getAppliedMigrations } = require('./migrations'); const { applyMigrations, getAppliedMigrations } = require('./migrations');
const { createQueryApi } = require('./queries'); const { createQueryApi } = require('./queries');
@@ -23,12 +23,135 @@ function resolveStateStorePath(options = {}) {
return path.join(homeDir, DEFAULT_STATE_STORE_RELATIVE_PATH); return path.join(homeDir, DEFAULT_STATE_STORE_RELATIVE_PATH);
} }
function openDatabase(dbPath) { /**
* Wraps a sql.js Database with a better-sqlite3-compatible API surface so
* that the rest of the state-store code (migrations.js, queries.js) can
* operate without knowing which driver is in use.
*
* IMPORTANT: sql.js db.export() implicitly ends any active transaction, so
* we must defer all disk writes until after the transaction commits.
*/
function wrapSqlJsDatabase(rawDb, dbPath) {
let inTransaction = false;
function saveToDisk() {
if (dbPath === ':memory:' || inTransaction) {
return;
}
const data = rawDb.export();
const buffer = Buffer.from(data);
fs.writeFileSync(dbPath, buffer);
}
const db = {
exec(sql) {
rawDb.run(sql);
saveToDisk();
},
pragma(pragmaStr) {
try {
rawDb.run(`PRAGMA ${pragmaStr}`);
} catch (_error) {
// Ignore unsupported pragmas (e.g. WAL for in-memory databases).
}
},
prepare(sql) {
return {
all(...positionalArgs) {
const stmt = rawDb.prepare(sql);
if (positionalArgs.length === 1 && typeof positionalArgs[0] !== 'object') {
stmt.bind([positionalArgs[0]]);
} else if (positionalArgs.length > 1) {
stmt.bind(positionalArgs);
}
const rows = [];
while (stmt.step()) {
rows.push(stmt.getAsObject());
}
stmt.free();
return rows;
},
get(...positionalArgs) {
const stmt = rawDb.prepare(sql);
if (positionalArgs.length === 1 && typeof positionalArgs[0] !== 'object') {
stmt.bind([positionalArgs[0]]);
} else if (positionalArgs.length > 1) {
stmt.bind(positionalArgs);
}
let row = null;
if (stmt.step()) {
row = stmt.getAsObject();
}
stmt.free();
return row;
},
run(namedParams) {
const stmt = rawDb.prepare(sql);
if (namedParams && typeof namedParams === 'object' && !Array.isArray(namedParams)) {
const sqlJsParams = {};
for (const [key, value] of Object.entries(namedParams)) {
sqlJsParams[`@${key}`] = value === undefined ? null : value;
}
stmt.bind(sqlJsParams);
}
stmt.step();
stmt.free();
saveToDisk();
},
};
},
transaction(fn) {
return (...args) => {
rawDb.run('BEGIN');
inTransaction = true;
try {
const result = fn(...args);
rawDb.run('COMMIT');
inTransaction = false;
saveToDisk();
return result;
} catch (error) {
try {
rawDb.run('ROLLBACK');
} catch (_rollbackError) {
// Transaction may already be rolled back.
}
inTransaction = false;
throw error;
}
};
},
close() {
saveToDisk();
rawDb.close();
},
};
return db;
}
async function openDatabase(SQL, dbPath) {
if (dbPath !== ':memory:') { if (dbPath !== ':memory:') {
fs.mkdirSync(path.dirname(dbPath), { recursive: true }); fs.mkdirSync(path.dirname(dbPath), { recursive: true });
} }
const db = new Database(dbPath); let rawDb;
if (dbPath !== ':memory:' && fs.existsSync(dbPath)) {
const fileBuffer = fs.readFileSync(dbPath);
rawDb = new SQL.Database(fileBuffer);
} else {
rawDb = new SQL.Database();
}
const db = wrapSqlJsDatabase(rawDb, dbPath);
db.pragma('foreign_keys = ON'); db.pragma('foreign_keys = ON');
try { try {
db.pragma('journal_mode = WAL'); db.pragma('journal_mode = WAL');
@@ -38,9 +161,10 @@ function openDatabase(dbPath) {
return db; return db;
} }
function createStateStore(options = {}) { async function createStateStore(options = {}) {
const dbPath = resolveStateStorePath(options); const dbPath = resolveStateStorePath(options);
const db = openDatabase(dbPath); const SQL = await initSqlJs();
const db = await openDatabase(SQL, dbPath);
const appliedMigrations = applyMigrations(db); const appliedMigrations = applyMigrations(db);
const queryApi = createQueryApi(db); const queryApi = createQueryApi(db);

View File

@@ -123,7 +123,7 @@ function printSessionDetail(payload) {
printDecisions(payload.decisions); printDecisions(payload.decisions);
} }
function main() { async function main() {
let store = null; let store = null;
try { try {
@@ -132,7 +132,7 @@ function main() {
showHelp(0); showHelp(0);
} }
store = createStateStore({ store = await createStateStore({
dbPath: options.dbPath, dbPath: options.dbPath,
homeDir: process.env.HOME, homeDir: process.env.HOME,
}); });

View File

@@ -128,7 +128,7 @@ function printHuman(payload) {
printGovernance(payload.governance); printGovernance(payload.governance);
} }
function main() { async function main() {
let store = null; let store = null;
try { try {
@@ -137,7 +137,7 @@ function main() {
showHelp(0); showHelp(0);
} }
store = createStateStore({ store = await createStateStore({
dbPath: options.dbPath, dbPath: options.dbPath,
homeDir: process.env.HOME, homeDir: process.env.HOME,
}); });

View File

@@ -17,9 +17,9 @@ const ECC_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'ecc.js');
const STATUS_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'status.js'); const STATUS_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'status.js');
const SESSIONS_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'sessions-cli.js'); const SESSIONS_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'sessions-cli.js');
function test(name, fn) { async function test(name, fn) {
try { try {
fn(); await fn();
console.log(` \u2713 ${name}`); console.log(` \u2713 ${name}`);
return true; return true;
} catch (error) { } catch (error) {
@@ -52,8 +52,8 @@ function parseJson(stdout) {
return JSON.parse(stdout.trim()); return JSON.parse(stdout.trim());
} }
function seedStore(dbPath) { async function seedStore(dbPath) {
const store = createStateStore({ dbPath }); const store = await createStateStore({ dbPath });
store.upsertSession({ store.upsertSession({
id: 'session-active', id: 'session-active',
@@ -252,20 +252,20 @@ function seedStore(dbPath) {
store.close(); store.close();
} }
function runTests() { async function runTests() {
console.log('\n=== Testing state-store ===\n'); console.log('\n=== Testing state-store ===\n');
let passed = 0; let passed = 0;
let failed = 0; let failed = 0;
if (test('creates the default state.db path and applies migrations idempotently', () => { if (await test('creates the default state.db path and applies migrations idempotently', async () => {
const homeDir = createTempDir('ecc-state-home-'); const homeDir = createTempDir('ecc-state-home-');
try { try {
const expectedPath = path.join(homeDir, '.claude', 'ecc', 'state.db'); const expectedPath = path.join(homeDir, '.claude', 'ecc', 'state.db');
assert.strictEqual(resolveStateStorePath({ homeDir }), expectedPath); assert.strictEqual(resolveStateStorePath({ homeDir }), expectedPath);
const firstStore = createStateStore({ homeDir }); const firstStore = await createStateStore({ homeDir });
const firstMigrations = firstStore.getAppliedMigrations(); const firstMigrations = firstStore.getAppliedMigrations();
firstStore.close(); firstStore.close();
@@ -273,7 +273,7 @@ function runTests() {
assert.strictEqual(firstMigrations[0].version, 1); assert.strictEqual(firstMigrations[0].version, 1);
assert.ok(fs.existsSync(expectedPath)); assert.ok(fs.existsSync(expectedPath));
const secondStore = createStateStore({ homeDir }); const secondStore = await createStateStore({ homeDir });
const secondMigrations = secondStore.getAppliedMigrations(); const secondMigrations = secondStore.getAppliedMigrations();
secondStore.close(); secondStore.close();
@@ -284,7 +284,7 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('preserves SQLite special database names like :memory:', () => { if (await test('preserves SQLite special database names like :memory:', async () => {
const tempDir = createTempDir('ecc-state-memory-'); const tempDir = createTempDir('ecc-state-memory-');
const previousCwd = process.cwd(); const previousCwd = process.cwd();
@@ -292,7 +292,7 @@ function runTests() {
process.chdir(tempDir); process.chdir(tempDir);
assert.strictEqual(resolveStateStorePath({ dbPath: ':memory:' }), ':memory:'); assert.strictEqual(resolveStateStorePath({ dbPath: ':memory:' }), ':memory:');
const store = createStateStore({ dbPath: ':memory:' }); const store = await createStateStore({ dbPath: ':memory:' });
assert.strictEqual(store.dbPath, ':memory:'); assert.strictEqual(store.dbPath, ':memory:');
assert.strictEqual(store.getAppliedMigrations().length, 1); assert.strictEqual(store.getAppliedMigrations().length, 1);
store.close(); store.close();
@@ -304,14 +304,14 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('stores sessions and returns detailed session views with workers, skill runs, and decisions', () => { if (await test('stores sessions and returns detailed session views with workers, skill runs, and decisions', async () => {
const testDir = createTempDir('ecc-state-db-'); const testDir = createTempDir('ecc-state-db-');
const dbPath = path.join(testDir, 'state.db'); const dbPath = path.join(testDir, 'state.db');
try { try {
seedStore(dbPath); await seedStore(dbPath);
const store = createStateStore({ dbPath }); const store = await createStateStore({ dbPath });
const listResult = store.listRecentSessions({ limit: 10 }); const listResult = store.listRecentSessions({ limit: 10 });
const detail = store.getSessionDetail('session-active'); const detail = store.getSessionDetail('session-active');
store.close(); store.close();
@@ -328,14 +328,14 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('builds a status snapshot with active sessions, skill rates, install health, and pending governance', () => { if (await test('builds a status snapshot with active sessions, skill rates, install health, and pending governance', async () => {
const testDir = createTempDir('ecc-state-db-'); const testDir = createTempDir('ecc-state-db-');
const dbPath = path.join(testDir, 'state.db'); const dbPath = path.join(testDir, 'state.db');
try { try {
seedStore(dbPath); await seedStore(dbPath);
const store = createStateStore({ dbPath }); const store = await createStateStore({ dbPath });
const status = store.getStatus(); const status = store.getStatus();
store.close(); store.close();
@@ -354,12 +354,12 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('validates entity payloads before writing to the database', () => { if (await test('validates entity payloads before writing to the database', async () => {
const testDir = createTempDir('ecc-state-db-'); const testDir = createTempDir('ecc-state-db-');
const dbPath = path.join(testDir, 'state.db'); const dbPath = path.join(testDir, 'state.db');
try { try {
const store = createStateStore({ dbPath }); const store = await createStateStore({ dbPath });
assert.throws(() => { assert.throws(() => {
store.upsertSession({ store.upsertSession({
id: '', id: '',
@@ -404,12 +404,12 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('status CLI supports human-readable and --json output', () => { if (await test('status CLI supports human-readable and --json output', async () => {
const testDir = createTempDir('ecc-state-cli-'); const testDir = createTempDir('ecc-state-cli-');
const dbPath = path.join(testDir, 'state.db'); const dbPath = path.join(testDir, 'state.db');
try { try {
seedStore(dbPath); await seedStore(dbPath);
const jsonResult = runNode(STATUS_SCRIPT, ['--db', dbPath, '--json']); const jsonResult = runNode(STATUS_SCRIPT, ['--db', dbPath, '--json']);
assert.strictEqual(jsonResult.status, 0, jsonResult.stderr); assert.strictEqual(jsonResult.status, 0, jsonResult.stderr);
@@ -428,12 +428,12 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('sessions CLI supports list and detail views in human-readable and --json output', () => { if (await test('sessions CLI supports list and detail views in human-readable and --json output', async () => {
const testDir = createTempDir('ecc-state-cli-'); const testDir = createTempDir('ecc-state-cli-');
const dbPath = path.join(testDir, 'state.db'); const dbPath = path.join(testDir, 'state.db');
try { try {
seedStore(dbPath); await seedStore(dbPath);
const listJsonResult = runNode(SESSIONS_SCRIPT, ['--db', dbPath, '--json']); const listJsonResult = runNode(SESSIONS_SCRIPT, ['--db', dbPath, '--json']);
assert.strictEqual(listJsonResult.status, 0, listJsonResult.stderr); assert.strictEqual(listJsonResult.status, 0, listJsonResult.stderr);
@@ -460,12 +460,12 @@ function runTests() {
} }
})) passed += 1; else failed += 1; })) passed += 1; else failed += 1;
if (test('ecc CLI delegates the new status and sessions subcommands', () => { if (await test('ecc CLI delegates the new status and sessions subcommands', async () => {
const testDir = createTempDir('ecc-state-cli-'); const testDir = createTempDir('ecc-state-cli-');
const dbPath = path.join(testDir, 'state.db'); const dbPath = path.join(testDir, 'state.db');
try { try {
seedStore(dbPath); await seedStore(dbPath);
const statusResult = runNode(ECC_SCRIPT, ['status', '--db', dbPath, '--json']); const statusResult = runNode(ECC_SCRIPT, ['status', '--db', dbPath, '--json']);
assert.strictEqual(statusResult.status, 0, statusResult.stderr); assert.strictEqual(statusResult.status, 0, statusResult.stderr);