diff --git a/docs/sqlite-databases.md b/docs/sqlite-databases.md index 4a9a43e9ca..2d9ad6c76c 100644 --- a/docs/sqlite-databases.md +++ b/docs/sqlite-databases.md @@ -57,7 +57,12 @@ triggers or arbitrary SQL. Rows are optional. `table` selects one exact ordinary-table name. `columns` (CLI repeated `--column`) selects exact visible column names in the requested order; omission selects all visible columns, including generated columns. -Unselected cells are not evaluated or projected. Malformed column-name bytes +Unselected cells are not evaluated or projected. A selected virtual generated +column whose expression needs a function REA's connection lacks (extensions and +application functions are not loaded), or raises an SQL error for a sampled +row, is refused as `capability_unavailable`. The error names the table and +each failing selected column with SQLite's message, and recommends only columns +whose sampled values were read; stored and other generated cells stay readable. Malformed column-name bytes cause a format refusal when all columns are requested. In inventory scope, an explicit valid subset can still be sampled without decoding unrelated malformed column names. Column selection requires a diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index a41ce65592..12e3030aa5 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -10,9 +10,14 @@ import { } from "../domain/sqlite/sqliteDatabase.js"; import { SQLITE_DATABASE_LIMITS, + SqliteColumnEvaluationFailure, SqliteInspectionFailure, + sqlitePrimaryResultCode, } from "./SqliteDatabaseLimits.js"; +/** SQLite's generic SQL error, raised while compiling or evaluating an expression. */ +const SQLITE_ERROR = 1; + const natural = z .bigint() .nonnegative() @@ -419,6 +424,107 @@ const readSchema = ( return { schema, selectedTable }; }; +/** Byte length of one cell's projected representation, without copying its value. */ +const cellByteLength = (name: string): string => { + const column = quoteIdentifier(name); + return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`; +}; + +/** Ordinary and stored generated columns, read without evaluating an expression; skips undecodable names. */ +const storedColumnNames = ( + db: DatabaseSync, + table: string, + encoding: DatabaseEncoding, +): string[] => { + const names: string[] = []; + for (const raw of db + .prepare( + "SELECT CAST(name AS BLOB) AS name FROM temp.pragma_table_xinfo(?, 'main') WHERE hidden IN (0, 3) ORDER BY cid", + ) + .iterate(table)) { + const { name } = z.object({ name: z.instanceof(Uint8Array) }).parse(raw); + try { + names.push(decodeSchemaText(name, encoding, "column name")); + } catch (cause: unknown) { + // An unrelated malformed name cannot be selected exactly, so it is not recommended. + if (!(cause instanceof SqliteInspectionFailure)) throw cause; + } + } + return names; +}; + +/** + * Attribute an SQL error from selected cells to the virtual generated columns + * that raise it on this connection. Each column is probed alone over the same + * sampled rows, so only columns whose probe succeeds are recommended; any + * other cause keeps its original classification. + */ +const columnEvaluationFailure = ( + db: DatabaseSync, + cause: unknown, + selection: { + readonly table: SelectedTableFacts; + readonly columns: readonly string[]; + readonly source: string; + readonly limit: number; + readonly encoding: DatabaseEncoding; + }, +): unknown => { + if (sqlitePrimaryResultCode(cause) !== SQLITE_ERROR) return cause; + const { table, columns, source, limit, encoding } = selection; + // The aggregate evaluates exactly the sampled cells, as the length pass does. + const probe = (name: string): { readonly failure: unknown } | undefined => { + try { + db.prepare( + `SELECT count(${cellByteLength(name)}) FROM (SELECT ${quoteIdentifier(name)} FROM ${source} LIMIT ?)`, + ).get(limit); + return undefined; + } catch (failure: unknown) { + return { failure }; + } + }; + const failures = new Map(); + for (const name of new Set(columns)) { + const failed = probe(name); + if (failed !== undefined) failures.set(name, failed.failure); + } + const generated = new Set( + table.columns + .filter((column) => column.hidden === 2) + .map((column) => column.name), + ); + if ( + ![...failures].some( + ([name, failure]) => + generated.has(name) && + sqlitePrimaryResultCode(failure) === SQLITE_ERROR, + ) + ) + return cause; + const remaining = columns.filter((name) => !failures.has(name)); + const readableStoredColumns = (): string[] => { + try { + return storedColumnNames(db, table.name, encoding).filter( + (name) => !failures.has(name) && probe(name) === undefined, + ); + } catch { + // Discovery only refines recovery; it must not replace the attributed failure. + return []; + } + }; + return new SqliteColumnEvaluationFailure( + `SQLite cannot evaluate selected column(s) of table ${JSON.stringify(table.name)} on REA's read-only connection: ${[ + ...failures, + ] + .map( + ([name, failure]) => + `${JSON.stringify(name)}: ${failure instanceof Error ? failure.message : String(failure)}`, + ) + .join("; ")}`, + remaining.length > 0 ? remaining : readableStoredColumns(), + ); +}; + const readRows = ( db: DatabaseSync, input: InspectSqliteDatabaseInput, @@ -460,19 +566,6 @@ const readRows = ( }) .join(", "); const source = `main.${quoteIdentifier(input.table)} NOT INDEXED`; - const lengths = db.prepare( - `SELECT ${columns - .map((name) => { - const column = quoteIdentifier(name); - return `CASE typeof(${column}) WHEN 'text' THEN length(CAST(${column} AS BLOB)) WHEN 'blob' THEN length(${column}) ELSE 0 END`; - }) - .join(", ")} FROM ${source} LIMIT ?`, - ); - lengths.setReturnArrays(true); - for (const raw of lengths.iterate(limit)) - budget.beforeProjection(cellByteLengths.parse(raw)); - const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`); - statement.setReturnArrays(true); const rows: NonNullable = { table: input.table, columns, @@ -482,13 +575,33 @@ const readRows = ( truncated: false, order: "unspecified", }; - for (const raw of statement.iterate(limit)) { - const rawRow = arrayRow.parse(raw); - // Hex expansion already belongs to the retained-output budget, before decoding/copying cells. - budget.beforeDecode(rawRow); - const values = rawRow.map((value) => cell(value, encoding)); - budget.retain(values); - rows.values.push(values); + // SQLite compiles and evaluates only selected cells, so an SQL error here is + // attributed to the selected generated expressions rather than the file. + try { + const lengths = db.prepare( + `SELECT ${columns.map(cellByteLength).join(", ")} FROM ${source} LIMIT ?`, + ); + lengths.setReturnArrays(true); + for (const raw of lengths.iterate(limit)) + budget.beforeProjection(cellByteLengths.parse(raw)); + const statement = db.prepare(`SELECT ${projection} FROM ${source} LIMIT ?`); + statement.setReturnArrays(true); + for (const raw of statement.iterate(limit)) { + const rawRow = arrayRow.parse(raw); + // Hex expansion already belongs to the retained-output budget, before decoding/copying cells. + budget.beforeDecode(rawRow); + const values = rawRow.map((value) => cell(value, encoding)); + budget.retain(values); + rows.values.push(values); + } + } catch (cause: unknown) { + throw columnEvaluationFailure(db, cause, { + table, + columns, + source, + limit, + encoding, + }); } rows.returned_rows = rows.values.length; rows.truncated = diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 50ab3978f2..3b6fbd6c20 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -12,6 +12,7 @@ export class SqliteInspectionFailure extends Error { | "format" | "selection" | "column-selection" + | "column-evaluation" | "unsupported" | "unavailable" | "output-limit" @@ -21,3 +22,20 @@ export class SqliteInspectionFailure extends Error { super(message); } } +/** Selected generated cells the private connection cannot compute; the database stays readable. */ +export class SqliteColumnEvaluationFailure extends SqliteInspectionFailure { + constructor( + message: string, + /** Selected or stored columns that remain readable without the failed expressions. */ + readonly readableColumns: readonly string[], + ) { + super("column-evaluation", message); + } +} +/** SQLite's primary result code for a native node:sqlite failure, if it carries one. */ +export const sqlitePrimaryResultCode = (cause: unknown): number | undefined => + cause instanceof Error && + "errcode" in cause && + typeof cause.errcode === "number" + ? cause.errcode & 0xff + : undefined; diff --git a/src/sqlite/SqliteDatabaseWorker.ts b/src/sqlite/SqliteDatabaseWorker.ts index ebd2d5bac1..c55528a5d3 100644 --- a/src/sqlite/SqliteDatabaseWorker.ts +++ b/src/sqlite/SqliteDatabaseWorker.ts @@ -4,7 +4,9 @@ import { inspectSqliteDatabaseInputSchema } from "../domain/sqlite/sqliteDatabas import { inspectSqliteDatabaseSnapshot } from "./SqliteDatabaseInspection.js"; import { SQLITE_DATABASE_LIMITS, + SqliteColumnEvaluationFailure, SqliteInspectionFailure, + sqlitePrimaryResultCode, } from "./SqliteDatabaseLimits.js"; const requestSchema = z.strictObject({ @@ -29,20 +31,28 @@ const main = async (): Promise => { ), }; } catch (cause: unknown) { - const code = - cause instanceof Error && "errcode" in cause ? cause.errcode : undefined; - reply = { - ok: false, - reason: - cause instanceof SqliteInspectionFailure - ? cause.reason - : code === 7 - ? "memory" - : code === 18 - ? "output-limit" - : "format", - message: cause instanceof Error ? cause.message : String(cause), - }; + const code = sqlitePrimaryResultCode(cause); + const message = cause instanceof Error ? cause.message : String(cause); + reply = + cause instanceof SqliteColumnEvaluationFailure + ? { + ok: false, + reason: cause.reason, + message, + readable_columns: cause.readableColumns, + } + : { + ok: false, + reason: + cause instanceof SqliteInspectionFailure + ? cause.reason + : code === 7 + ? "memory" + : code === 18 + ? "output-limit" + : "format", + message, + }; } let encoded = JSON.stringify(reply); if (Buffer.byteLength(encoded) > SQLITE_DATABASE_LIMITS.outputBytes) diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index bdd82ad1ee..acabd5666a 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -39,7 +39,7 @@ const { ...inspectionShape } = sqliteDatabaseSchema.shape; const inspectionSchema = z.strictObject(inspectionShape); -const replySchema = z.discriminatedUnion("ok", [ +const replySchema = z.union([ z.strictObject({ ok: z.literal(true), inspection: inspectionSchema }), z.strictObject({ ok: z.literal(false), @@ -54,6 +54,12 @@ const replySchema = z.discriminatedUnion("ok", [ ]), message: z.string(), }), + z.strictObject({ + ok: z.literal(false), + reason: z.literal("column-evaluation"), + message: z.string(), + readable_columns: z.array(z.string()), + }), ]); /** Replaceable launcher retains the shared owned-process supervision boundary. */ export type SqliteWorkerLauncher = NonNullable< @@ -80,6 +86,23 @@ const rejectedReply = ( reply.message, { capturedOutput, userMessage: reply.message }, ); + if (reply.reason === "column-evaluation") { + const readable = reply.readable_columns; + return new AnalysisCapabilityUnavailableError( + "sqlite", + operation, + reply.message, + { + capturedOutput, + userMessage: reply.message, + remediationAction: `The database is readable; only the listed column values failed. REA's SQLite connection loads no extensions or application functions and disables schema trust. ${ + readable.length === 0 + ? "No other readable column was found" + : `Select columns whose values were read: columns: ${JSON.stringify(readable)} (CLI: one --column per name)` + }; omit table and use schema_scope: database to read the stored declarations without evaluating rows. Obtaining the failed values needs a SQLite connection able to evaluate them. Do not repeat the unchanged request.`, + }, + ); + } if (reply.reason === "memory") return new AnalysisResourceConstraintError( operation, diff --git a/tests/acceptance/sqliteDatabaseCli.test.ts b/tests/acceptance/sqliteDatabaseCli.test.ts index d871fe31ab..5ce76ea826 100644 --- a/tests/acceptance/sqliteDatabaseCli.test.ts +++ b/tests/acceptance/sqliteDatabaseCli.test.ts @@ -796,3 +796,98 @@ cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( } }, ); + +cliTest.runIf(SQLITE_NATIVE_LIMITS_AVAILABLE)( + "attributes an unevaluable generated column to its selection, not the readable database path", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-sqlite-generated-"); + const path = join(root, "hashes.db"); + const producer = new DatabaseSync(path); + try { + // The producing application supplies a function REA's connection lacks. + producer.function( + "application_digest", + { deterministic: true }, + (value) => (typeof value === "string" ? `digest:${value}` : null), + ); + producer.exec(` + CREATE TABLE t(a TEXT, b TEXT GENERATED ALWAYS AS (application_digest(a)) VIRTUAL); + INSERT INTO t(a) VALUES ('hello'), ('world'); + `); + } finally { + producer.close(); + } + const before = await readFile(path); + const { call } = await connectLocalToolsMcp(); + const message = `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "b": unknown function: application_digest()`; + const expectedError = { + code: "capability_unavailable", + retryable: false, + message, + details: { provider_id: "sqlite", reason: message }, + remediation: { + action: expect.stringContaining( + 'Select columns whose values were read: columns: ["a"] (CLI: one --column per name)', + ), + }, + }; + for (const columns of [["b", "a"], undefined]) { + const result = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "t", + ...(columns ?? []).flatMap((column) => ["--column", column]), + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode, result.stdout + result.stderr).toBe(1); + expect(result.json).toMatchObject(expectedError); + expect(result.stdout).not.toContain("invalid_format"); + const response = await call("inspect_sqlite_database", { + path, + table: "t", + ...(columns === undefined ? {} : { columns }), + }); + expect(parseMcpToolError(response).error).toMatchObject(expectedError); + } + const base = await cli.run({ + arguments: [ + "inspect-sqlite-database", + path, + "--table", + "t", + "--column", + "a", + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(base.exitCode, base.stdout + base.stderr).toBe(0); + expect(base.json).toMatchObject({ + normalized_result: { rows: { columns: ["a"], returned_rows: 2 } }, + }); + const schema = await call("inspect_sqlite_database", { + path, + schema_scope: "database", + }); + expect(schema.isError).not.toBe(true); + expect(schema.structuredContent).toMatchObject({ + normalized_result: { + rows: null, + schema: { + tables: [ + { + name: "t", + sql: expect.stringContaining("application_digest(a)"), + columns: [{ name: "a" }, { name: "b", hidden: 2 }], + }, + ], + }, + }, + }); + expect(await readFile(path)).toEqual(before); + }, +); diff --git a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts index 26fab7d6a4..17af7d5519 100644 --- a/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseInspection.test.ts @@ -364,6 +364,179 @@ it("rejects the combined representation of individually small generated cells", } }); +it("attributes compile and row evaluation errors to each failing generated column", async () => { + const workspace = await createTestWorkspace("rea-sqlite-evaluation-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("private.db"); + // Writable schema plants an expression that fails only for the second + // stored row; virtual cells are not stored, so record layout is unchanged. + const producer = new DatabaseSync(path, { defensive: false }); + try { + producer.function("application_digest", { deterministic: true }, (value) => + String(value), + ); + producer.exec(` + CREATE TABLE t( + id INTEGER, + payload TEXT, + digest TEXT GENERATED ALWAYS AS (application_digest(payload)) VIRTUAL, + stored_digest TEXT GENERATED ALWAYS AS (application_digest(payload)) STORED, + field GENERATED ALWAYS AS (payload) VIRTUAL, + upper_payload GENERATED ALWAYS AS (upper(payload)) VIRTUAL + ); + INSERT INTO t(id, payload) VALUES (1, '{"k":1}'), (2, 'not json'); + PRAGMA writable_schema = ON; + UPDATE sqlite_schema + SET sql = replace(sql, 'field GENERATED ALWAYS AS (payload)', 'field GENERATED ALWAYS AS (json_extract(payload, ''$.k''))') + WHERE name = 't'; + PRAGMA writable_schema = OFF; + `); + } finally { + producer.close(); + } + const failure = (input: { columns?: string[]; row_limit?: number }) => { + try { + inspectSqliteDatabaseSnapshot(path, { path, table: "t", ...input }); + } catch (cause: unknown) { + return cause; + } + throw new Error("Expected a generated-column evaluation failure"); + }; + const prefix = `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: `; + expect(failure({})).toMatchObject({ + reason: "column-evaluation", + message: `${prefix}"digest": unknown function: application_digest(); "field": malformed JSON`, + readableColumns: ["id", "payload", "stored_digest", "upper_payload"], + }); + expect(failure({ columns: ["field", "id", "field"] })).toMatchObject({ + message: `${prefix}"field": malformed JSON`, + readableColumns: ["id"], + }); + // With no readable selection left, stored columns remain available without evaluation. + expect(failure({ columns: ["digest"] })).toMatchObject({ + message: `${prefix}"digest": unknown function: application_digest()`, + readableColumns: ["id", "payload", "stored_digest"], + }); + // Only sampled rows are evaluated, and stored or built-in generated cells are read. + expect( + inspectSqliteDatabaseSnapshot(path, { + path, + table: "t", + columns: ["field", "stored_digest", "upper_payload"], + row_limit: 1, + }).rows, + ).toMatchObject({ + values: [ + [ + { type: "integer", value: "1" }, + { type: "text", value: '{"k":1}' }, + { type: "text", value: '{"K":1}' }, + ], + ], + truncated: true, + }); +}); + +const createApplicationDatabase = async ( + sql: string, + edit?: (bytes: Buffer) => void, +): Promise => { + const workspace = await createTestWorkspace("rea-sqlite-evaluation-"); + onTestFinished(() => removeTestWorkspace(workspace.root)); + const path = workspace.path("private.db"); + const producer = new DatabaseSync(path); + try { + // The producing application supplies a function REA's connection lacks. + producer.function("application_digest", { deterministic: true }, (value) => + String(value), + ); + producer.exec(sql); + } finally { + producer.close(); + } + if (edit !== undefined) { + const bytes = await readFile(path); + edit(bytes); + await writeFile(path, bytes); + } + return path; +}; + +it("never recommends a selected column whose own probe fails for another reason", async () => { + const path = await createApplicationDatabase(` + CREATE TABLE t( + id INTEGER, + digest TEXT GENERATED ALWAYS AS (application_digest(id)) VIRTUAL, + payload BLOB GENERATED ALWAYS AS (zeroblob(72 * 1024 * 1024)) VIRTUAL + ); + INSERT INTO t(id) VALUES (1); + `); + expect(() => + inspectSqliteDatabaseSnapshot(path, { + path, + table: "t", + columns: ["payload", "digest", "id"], + }), + ).toThrow( + expect.objectContaining({ + reason: "column-evaluation", + message: `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "payload": string or blob too big; "digest": unknown function: application_digest()`, + readableColumns: ["id"], + }), + ); +}); + +it("recommends stored columns outside an inventory-scope selection without decoding malformed names", async () => { + const path = await createApplicationDatabase( + ` + CREATE TABLE t( + id INTEGER, + "col" TEXT, + payload TEXT, + digest TEXT GENERATED ALWAYS AS (application_digest(payload)) VIRTUAL + ); + INSERT INTO t(id, "col", payload) VALUES (1, 'kept', 'value'); + `, + (bytes) => { + const marker = Buffer.from('"col"'); + const offset = bytes.indexOf(marker); + if (offset < 0 || bytes.indexOf(marker, offset + 1) !== -1) + throw new Error("Expected unique authored column marker"); + bytes[offset + 1] = 0xff; + }, + ); + expect(() => + inspectSqliteDatabaseSnapshot(path, { + path, + table: "t", + schema_scope: "inventory", + columns: ["digest"], + }), + ).toThrow( + expect.objectContaining({ + reason: "column-evaluation", + message: `SQLite cannot evaluate selected column(s) of table "t" on REA's read-only connection: "digest": unknown function: application_digest()`, + readableColumns: ["id", "payload"], + }), + ); + // The recommended stored columns are themselves selectable in the same scope. + expect( + inspectSqliteDatabaseSnapshot(path, { + path, + table: "t", + schema_scope: "inventory", + columns: ["id", "payload"], + }).rows, + ).toMatchObject({ + values: [ + [ + { type: "integer", value: "1" }, + { type: "text", value: "value" }, + ], + ], + }); +}); + it("samples above 1000 rows and safe integer limits within the byte budget", async () => { const path = await createDatabase(`CREATE TABLE items(value INTEGER); WITH RECURSIVE numbers(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM numbers WHERE value < 1002)