From d4ed5976cd0777d4602a1b9687c57a2b8e9c112b Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Mon, 16 Mar 2026 01:28:45 -0700 Subject: [PATCH] fix: replace better-sqlite3 with sql.js to eliminate native module CI failures better-sqlite3 requires native C++ compilation (node-gyp, prebuild-install) which fails in CI across npm/pnpm on all platforms: - npm ci: lock file out of sync with native transitive deps - pnpm: native bindings not found at runtime - Windows: native compilation fails entirely sql.js is a pure JavaScript/WASM SQLite implementation with zero native dependencies. The adapter in index.js wraps the sql.js API to match the better-sqlite3 interface used by migrations.js and queries.js. Key implementation detail: sql.js db.export() implicitly ends active transactions, so the adapter defers disk writes (saveToDisk) until after transaction commit via an inTransaction guard flag. createStateStore is now async (sql.js requires async WASM init). Updated status.js, sessions-cli.js, and tests accordingly. --- package-lock.json | 45 +++++++---- package.json | 2 +- scripts/lib/state-store/index.js | 134 +++++++++++++++++++++++++++++-- scripts/sessions-cli.js | 4 +- scripts/status.js | 4 +- tests/lib/state-store.test.js | 48 +++++------ 6 files changed, 188 insertions(+), 49 deletions(-) diff --git a/package-lock.json b/package-lock.json index 60bb15f7..2e18fa41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,11 @@ "version": "1.8.0", "hasInstallScript": true, "license": "MIT", + "dependencies": { + "sql.js": "^1.14.1" + }, "bin": { + "ecc": "scripts/ecc.js", "ecc-install": "install.sh" }, "devDependencies": { @@ -18,7 +22,7 @@ "c8": "^10.1.2", "eslint": "^9.39.2", "globals": "^17.1.0", - "markdownlint-cli": "^0.48.0" + "markdownlint-cli": "^0.47.0" }, "engines": { "node": ">=18" @@ -1655,22 +1659,23 @@ } }, "node_modules/markdownlint-cli": { - "version": "0.48.0", - "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.48.0.tgz", - "integrity": "sha512-NkZQNu2E0Q5qLEEHwWj674eYISTLD4jMHkBzDobujXd1kv+yCxi8jOaD/rZoQNW1FBBMMGQpuW5So8B51N/e0A==", + "version": "0.47.0", + "resolved": "https://registry.npmjs.org/markdownlint-cli/-/markdownlint-cli-0.47.0.tgz", + "integrity": "sha512-HOcxeKFAdDoldvoYDofd85vI8LgNWy8vmYpCwnlLV46PJcodmGzD7COSSBlhHwsfT4o9KrAStGodImVBus31Bg==", "dev": true, + "license": "MIT", "dependencies": { - "commander": "~14.0.3", + "commander": "~14.0.2", "deep-extend": "~0.6.0", "ignore": "~7.0.5", "js-yaml": "~4.1.1", "jsonc-parser": "~3.3.1", "jsonpointer": "~5.0.1", - "markdown-it": "~14.1.1", + "markdown-it": "~14.1.0", "markdownlint": "~0.40.0", - "minimatch": "~10.2.4", + "minimatch": "~10.1.1", "run-con": "~1.3.2", - "smol-toml": "~1.6.0", + "smol-toml": "~1.5.2", "tinyglobby": "~0.2.15" }, "bin": { @@ -1685,6 +1690,7 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", "integrity": "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA==", "dev": true, + "license": "MIT", "engines": { "node": "18 || 20 || >=22" } @@ -1694,6 +1700,7 @@ "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.4.tgz", "integrity": "sha512-h+DEnpVvxmfVefa4jFbCf5HdH5YMDXRsmKflpf1pILZWRFlTbJpxeU55nJl4Smt5HQaGzg1o6RHFPJaOqnmBDg==", "dev": true, + "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, @@ -1712,15 +1719,16 @@ } }, "node_modules/markdownlint-cli/node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "10.1.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.1.3.tgz", + "integrity": "sha512-IF6URNyBX7Z6XfvjpaNy5meRxPZiIf2OqtOoSLs+hLJ9pJAScnM1RjrFcbCaD85y42KcI+oZmKjFIJKYDFjQfg==", "dev": true, + "license": "BlueOak-1.0.0", "dependencies": { "brace-expansion": "^5.0.2" }, "engines": { - "node": "18 || 20 || >=22" + "node": "20 || >=22" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -2579,10 +2587,11 @@ } }, "node_modules/smol-toml": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.6.0.tgz", - "integrity": "sha512-4zemZi0HvTnYwLfrpk/CF9LOd9Lt87kAt50GnqhMpyF9U3poDAP2+iukq2bZsO/ufegbYehBkqINbsWxj4l4cw==", + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.5.2.tgz", + "integrity": "sha512-QlaZEqcAH3/RtNyet1IPIYPsEWAaYyXXv1Krsi+1L/QHppjX4Ifm8MQsBISz9vE8cHicIq3clogsheili5vhaQ==", "dev": true, + "license": "BSD-3-Clause", "engines": { "node": ">= 18" }, @@ -2590,6 +2599,12 @@ "url": "https://github.com/sponsors/cyyynthia" } }, + "node_modules/sql.js": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/sql.js/-/sql.js-1.14.1.tgz", + "integrity": "sha512-gcj8zBWU5cFsi9WUP+4bFNXAyF1iRpA3LLyS/DP5xlrNzGmPIizUeBggKa8DbDwdqaKwUcTEnChtd2grWo/x/A==", + "license": "MIT" + }, "node_modules/string-width": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/string-width/-/string-width-8.1.0.tgz", diff --git a/package.json b/package.json index a504825f..baf2e410 100644 --- a/package.json +++ b/package.json @@ -105,7 +105,7 @@ "coverage": "c8 --all --include=\"scripts/**/*.js\" --check-coverage --lines 80 --functions 80 --branches 80 --statements 80 --reporter=text --reporter=lcov node tests/run-all.js" }, "dependencies": { - "better-sqlite3": "^11.8.1" + "sql.js": "^1.14.1" }, "devDependencies": { "@eslint/js": "^9.39.2", diff --git a/scripts/lib/state-store/index.js b/scripts/lib/state-store/index.js index 259c31c4..bf60992e 100644 --- a/scripts/lib/state-store/index.js +++ b/scripts/lib/state-store/index.js @@ -3,7 +3,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); -const Database = require('better-sqlite3'); +const initSqlJs = require('sql.js'); const { applyMigrations, getAppliedMigrations } = require('./migrations'); const { createQueryApi } = require('./queries'); @@ -23,12 +23,135 @@ function resolveStateStorePath(options = {}) { return path.join(homeDir, DEFAULT_STATE_STORE_RELATIVE_PATH); } -function openDatabase(dbPath) { +/** + * Wraps a sql.js Database with a better-sqlite3-compatible API surface so + * that the rest of the state-store code (migrations.js, queries.js) can + * operate without knowing which driver is in use. + * + * IMPORTANT: sql.js db.export() implicitly ends any active transaction, so + * we must defer all disk writes until after the transaction commits. + */ +function wrapSqlJsDatabase(rawDb, dbPath) { + let inTransaction = false; + + function saveToDisk() { + if (dbPath === ':memory:' || inTransaction) { + return; + } + const data = rawDb.export(); + const buffer = Buffer.from(data); + fs.writeFileSync(dbPath, buffer); + } + + const db = { + exec(sql) { + rawDb.run(sql); + saveToDisk(); + }, + + pragma(pragmaStr) { + try { + rawDb.run(`PRAGMA ${pragmaStr}`); + } catch (_error) { + // Ignore unsupported pragmas (e.g. WAL for in-memory databases). + } + }, + + prepare(sql) { + return { + all(...positionalArgs) { + const stmt = rawDb.prepare(sql); + if (positionalArgs.length === 1 && typeof positionalArgs[0] !== 'object') { + stmt.bind([positionalArgs[0]]); + } else if (positionalArgs.length > 1) { + stmt.bind(positionalArgs); + } + + const rows = []; + while (stmt.step()) { + rows.push(stmt.getAsObject()); + } + stmt.free(); + return rows; + }, + + get(...positionalArgs) { + const stmt = rawDb.prepare(sql); + if (positionalArgs.length === 1 && typeof positionalArgs[0] !== 'object') { + stmt.bind([positionalArgs[0]]); + } else if (positionalArgs.length > 1) { + stmt.bind(positionalArgs); + } + + let row = null; + if (stmt.step()) { + row = stmt.getAsObject(); + } + stmt.free(); + return row; + }, + + run(namedParams) { + const stmt = rawDb.prepare(sql); + if (namedParams && typeof namedParams === 'object' && !Array.isArray(namedParams)) { + const sqlJsParams = {}; + for (const [key, value] of Object.entries(namedParams)) { + sqlJsParams[`@${key}`] = value === undefined ? null : value; + } + stmt.bind(sqlJsParams); + } + stmt.step(); + stmt.free(); + saveToDisk(); + }, + }; + }, + + transaction(fn) { + return (...args) => { + rawDb.run('BEGIN'); + inTransaction = true; + try { + const result = fn(...args); + rawDb.run('COMMIT'); + inTransaction = false; + saveToDisk(); + return result; + } catch (error) { + try { + rawDb.run('ROLLBACK'); + } catch (_rollbackError) { + // Transaction may already be rolled back. + } + inTransaction = false; + throw error; + } + }; + }, + + close() { + saveToDisk(); + rawDb.close(); + }, + }; + + return db; +} + +async function openDatabase(SQL, dbPath) { if (dbPath !== ':memory:') { fs.mkdirSync(path.dirname(dbPath), { recursive: true }); } - const db = new Database(dbPath); + let rawDb; + if (dbPath !== ':memory:' && fs.existsSync(dbPath)) { + const fileBuffer = fs.readFileSync(dbPath); + rawDb = new SQL.Database(fileBuffer); + } else { + rawDb = new SQL.Database(); + } + + const db = wrapSqlJsDatabase(rawDb, dbPath); db.pragma('foreign_keys = ON'); try { db.pragma('journal_mode = WAL'); @@ -38,9 +161,10 @@ function openDatabase(dbPath) { return db; } -function createStateStore(options = {}) { +async function createStateStore(options = {}) { const dbPath = resolveStateStorePath(options); - const db = openDatabase(dbPath); + const SQL = await initSqlJs(); + const db = await openDatabase(SQL, dbPath); const appliedMigrations = applyMigrations(db); const queryApi = createQueryApi(db); diff --git a/scripts/sessions-cli.js b/scripts/sessions-cli.js index 435b4b3c..436796f5 100644 --- a/scripts/sessions-cli.js +++ b/scripts/sessions-cli.js @@ -123,7 +123,7 @@ function printSessionDetail(payload) { printDecisions(payload.decisions); } -function main() { +async function main() { let store = null; try { @@ -132,7 +132,7 @@ function main() { showHelp(0); } - store = createStateStore({ + store = await createStateStore({ dbPath: options.dbPath, homeDir: process.env.HOME, }); diff --git a/scripts/status.js b/scripts/status.js index 7574a6a1..2e5349df 100644 --- a/scripts/status.js +++ b/scripts/status.js @@ -128,7 +128,7 @@ function printHuman(payload) { printGovernance(payload.governance); } -function main() { +async function main() { let store = null; try { @@ -137,7 +137,7 @@ function main() { showHelp(0); } - store = createStateStore({ + store = await createStateStore({ dbPath: options.dbPath, homeDir: process.env.HOME, }); diff --git a/tests/lib/state-store.test.js b/tests/lib/state-store.test.js index 0bc6efc4..440b8b65 100644 --- a/tests/lib/state-store.test.js +++ b/tests/lib/state-store.test.js @@ -17,9 +17,9 @@ const ECC_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'ecc.js'); const STATUS_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'status.js'); const SESSIONS_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'sessions-cli.js'); -function test(name, fn) { +async function test(name, fn) { try { - fn(); + await fn(); console.log(` \u2713 ${name}`); return true; } catch (error) { @@ -52,8 +52,8 @@ function parseJson(stdout) { return JSON.parse(stdout.trim()); } -function seedStore(dbPath) { - const store = createStateStore({ dbPath }); +async function seedStore(dbPath) { + const store = await createStateStore({ dbPath }); store.upsertSession({ id: 'session-active', @@ -252,20 +252,20 @@ function seedStore(dbPath) { store.close(); } -function runTests() { +async function runTests() { console.log('\n=== Testing state-store ===\n'); let passed = 0; let failed = 0; - if (test('creates the default state.db path and applies migrations idempotently', () => { + if (await test('creates the default state.db path and applies migrations idempotently', async () => { const homeDir = createTempDir('ecc-state-home-'); try { const expectedPath = path.join(homeDir, '.claude', 'ecc', 'state.db'); assert.strictEqual(resolveStateStorePath({ homeDir }), expectedPath); - const firstStore = createStateStore({ homeDir }); + const firstStore = await createStateStore({ homeDir }); const firstMigrations = firstStore.getAppliedMigrations(); firstStore.close(); @@ -273,7 +273,7 @@ function runTests() { assert.strictEqual(firstMigrations[0].version, 1); assert.ok(fs.existsSync(expectedPath)); - const secondStore = createStateStore({ homeDir }); + const secondStore = await createStateStore({ homeDir }); const secondMigrations = secondStore.getAppliedMigrations(); secondStore.close(); @@ -284,7 +284,7 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('preserves SQLite special database names like :memory:', () => { + if (await test('preserves SQLite special database names like :memory:', async () => { const tempDir = createTempDir('ecc-state-memory-'); const previousCwd = process.cwd(); @@ -292,7 +292,7 @@ function runTests() { process.chdir(tempDir); assert.strictEqual(resolveStateStorePath({ dbPath: ':memory:' }), ':memory:'); - const store = createStateStore({ dbPath: ':memory:' }); + const store = await createStateStore({ dbPath: ':memory:' }); assert.strictEqual(store.dbPath, ':memory:'); assert.strictEqual(store.getAppliedMigrations().length, 1); store.close(); @@ -304,14 +304,14 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('stores sessions and returns detailed session views with workers, skill runs, and decisions', () => { + if (await test('stores sessions and returns detailed session views with workers, skill runs, and decisions', async () => { const testDir = createTempDir('ecc-state-db-'); const dbPath = path.join(testDir, 'state.db'); try { - seedStore(dbPath); + await seedStore(dbPath); - const store = createStateStore({ dbPath }); + const store = await createStateStore({ dbPath }); const listResult = store.listRecentSessions({ limit: 10 }); const detail = store.getSessionDetail('session-active'); store.close(); @@ -328,14 +328,14 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('builds a status snapshot with active sessions, skill rates, install health, and pending governance', () => { + if (await test('builds a status snapshot with active sessions, skill rates, install health, and pending governance', async () => { const testDir = createTempDir('ecc-state-db-'); const dbPath = path.join(testDir, 'state.db'); try { - seedStore(dbPath); + await seedStore(dbPath); - const store = createStateStore({ dbPath }); + const store = await createStateStore({ dbPath }); const status = store.getStatus(); store.close(); @@ -354,12 +354,12 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('validates entity payloads before writing to the database', () => { + if (await test('validates entity payloads before writing to the database', async () => { const testDir = createTempDir('ecc-state-db-'); const dbPath = path.join(testDir, 'state.db'); try { - const store = createStateStore({ dbPath }); + const store = await createStateStore({ dbPath }); assert.throws(() => { store.upsertSession({ id: '', @@ -404,12 +404,12 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('status CLI supports human-readable and --json output', () => { + if (await test('status CLI supports human-readable and --json output', async () => { const testDir = createTempDir('ecc-state-cli-'); const dbPath = path.join(testDir, 'state.db'); try { - seedStore(dbPath); + await seedStore(dbPath); const jsonResult = runNode(STATUS_SCRIPT, ['--db', dbPath, '--json']); assert.strictEqual(jsonResult.status, 0, jsonResult.stderr); @@ -428,12 +428,12 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('sessions CLI supports list and detail views in human-readable and --json output', () => { + if (await test('sessions CLI supports list and detail views in human-readable and --json output', async () => { const testDir = createTempDir('ecc-state-cli-'); const dbPath = path.join(testDir, 'state.db'); try { - seedStore(dbPath); + await seedStore(dbPath); const listJsonResult = runNode(SESSIONS_SCRIPT, ['--db', dbPath, '--json']); assert.strictEqual(listJsonResult.status, 0, listJsonResult.stderr); @@ -460,12 +460,12 @@ function runTests() { } })) passed += 1; else failed += 1; - if (test('ecc CLI delegates the new status and sessions subcommands', () => { + if (await test('ecc CLI delegates the new status and sessions subcommands', async () => { const testDir = createTempDir('ecc-state-cli-'); const dbPath = path.join(testDir, 'state.db'); try { - seedStore(dbPath); + await seedStore(dbPath); const statusResult = runNode(ECC_SCRIPT, ['status', '--db', dbPath, '--json']); assert.strictEqual(statusResult.status, 0, statusResult.stderr);