diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1af432690..28373f89d2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: nativeaot: ${{ steps.plan.outputs.nativeaot }} website: ${{ steps.plan.outputs.website }} matrix: ${{ steps.plan.outputs.matrix }} + test_selection: ${{ steps.plan.outputs.test_selection }} steps: - uses: actions/checkout@v4 with: @@ -315,6 +316,8 @@ jobs: SHARD_RESULT: ${{ needs.test-shard.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" + CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: node scripts/ci/test-report.mjs - uses: actions/upload-artifact@v4 if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' @@ -325,6 +328,8 @@ jobs: test-results.json test-timings.json coverage/ + .cache/ci-test-selection/report.json + include-hidden-files: true if-no-files-found: error retention-days: 30 - name: Publish successful timing history diff --git a/docs/testing.md b/docs/testing.md index ac1a01f84b..03121e172e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -805,6 +805,27 @@ remain present and fail if classification fails. Every code change retains the complete six-shard Linux suite; individual test files are not selected by imports or filenames. +Ordinary PRs also produce an **observational** test-selection plan. The candidate +leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately +from provider-job scopes. It proposes keeping affected owners plus every unowned +test, including all CLI/MCP, acceptance, composition, conformance, and process +tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall +back to the complete inventory. Renames account for both paths. Main, scheduled, +manual, and `ci:full` runs always propose the full suite. + +All six shards still execute the complete suite. The existing report merge +compares the proposal with actual module outcomes and retains +`.cache/ci-test-selection/report.json` in `test-results`, including selected and +omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and +executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate, +skipped, interrupted, or unhandled results cannot establish a complete comparison; +even a complete passing comparison is not proof of ownership completeness. +Before enabling omissions, review representative full-run comparisons and +cross-owner consumers, including failures, and measure completed CI feedback and +runner demand. There is no automatic promotion or switch to skip tests. The +initial proposal measures only leaf producer-test omissions; it does not promise +the savings of a broader affected-test planner. + `scripts/ci/plan.mjs` selects lanes using the ownership table in `scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider verifiers and the Linux x64 installed-package/Inspector baseline. The `apple` diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 3c711a1f9c..86879216d4 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { matchesGlob } from "node:path"; import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs"; +import { planTestSelection } from "./test-selection.mjs"; // Test files select the host/runtime lanes that execute their focused suites. const HOST_TEST_OWNERS = ["apple", "package", "sqlite"]; @@ -250,11 +251,17 @@ const matrix = { ] : [linux], }; -const plan = { ...selected, tier, matrix, files, reasons }; +const testSelection = planTestSelection(files, event, tier); +const plan = { ...selected, tier, matrix, files, reasons, testSelection }; if (process.env.GITHUB_OUTPUT) appendFileSync( process.env.GITHUB_OUTPUT, - Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) }) + Object.entries({ + ...selected, + tier, + matrix: JSON.stringify(matrix), + test_selection: JSON.stringify(testSelection), + }) .map(([key, value]) => `${key}=${value}\n`) .join(""), ); diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 86419ae9f9..035a019908 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -4,25 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { z } from "zod"; +import { compareTestSelection } from "./test-selection.mjs"; + const root = process.cwd(); const historyDirectory = join(root, ".cache/ci-test-history"); const historyPath = join(historyDirectory, "test-results.json"); +const selectionDirectory = join(root, ".cache/ci-test-selection"); +const selectionPath = join(selectionDirectory, "report.json"); // Never let a failed or partial rerun leave a previous successful history file. await rm(historyPath, { force: true }); +await rm(selectionPath, { force: true }); const report = z .object({ version: z.literal(1), success: z.boolean(), + reason: z.string().optional(), + unhandledErrors: z.number().int().nonnegative().optional(), files: z.array( z.object({ name: z.string().min(1), durationMs: z.number().finite().nonnegative(), + state: z.string().optional(), }), ), }) .parse(JSON.parse(await readFile("test-timings.json", "utf8"))); -const files = report.files.map(({ name, durationMs }) => { +const files = report.files.map(({ name, durationMs, state }) => { const path = relative(root, resolve(root, name)); if ( path === "" || @@ -31,21 +39,47 @@ const files = report.files.map(({ name, durationMs }) => { isAbsolute(path) ) throw new Error(`Test report path is outside the checkout: ${name}`); - return { name: path.split(sep).join("/"), durationMs }; + return { name: path.split(sep).join("/"), durationMs, state }; }); const complete = process.env.SHARD_RESULT === "success"; const passed = complete && process.env.MERGE_RESULT === "success" && report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. -if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { - const expected = execFileSync( - "git", - ["ls-files", "-z", "--", "src", "tests"], - { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }, - ) - .split("\0") - .filter((path) => path.endsWith(".test.ts")); +const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; +const expected = + verifyInventory || process.env.CI_TEST_SELECTION + ? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }) + .split("\0") + .filter((path) => path.endsWith(".test.ts")) + : []; +let selection; +if (process.env.CI_TEST_SELECTION) { + const comparison = compareTestSelection( + JSON.parse(process.env.CI_TEST_SELECTION), + expected, + files, + ); + selection = { + ...comparison, + complete: + comparison.complete && + report.unhandledErrors === 0 && + ["passed", "failed"].includes(report.reason) && + ["success", "failure"].includes(process.env.SHARD_RESULT) && + ["success", "failure"].includes(process.env.MERGE_RESULT), + executionSha: process.env.GITHUB_SHA ?? null, + prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, + runId: process.env.GITHUB_RUN_ID ?? null, + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null, + }; + await mkdir(selectionDirectory, { recursive: true }); + await writeFile(selectionPath, JSON.stringify(selection)); +} +if (passed && verifyInventory) { const actual = files.map(({ name }) => name); const expectedSet = new Set(expected); const actualSet = new Set(actual); @@ -59,6 +93,16 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { const summary = [ complete ? "## Merged test diagnostics" : "## Partial test diagnostics", "", + ...(selection + ? [ + "### Observational PR test selection", + "", + `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, + `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, + "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "", + ] + : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, passed ? "Successful timing history is available." @@ -85,6 +129,10 @@ if (passed) { await mkdir(historyDirectory, { recursive: true }); await writeFile( historyPath, - JSON.stringify({ version: 1, success: true, files }), + JSON.stringify({ + version: 1, + success: true, + files: files.map(({ name, durationMs }) => ({ name, durationMs })), + }), ); } diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts new file mode 100644 index 0000000000..c194143984 --- /dev/null +++ b/scripts/ci/test-selection.d.mts @@ -0,0 +1,35 @@ +export interface TestSelectionPlan { + readonly version: 1; + readonly mode: "full" | "candidate"; + readonly owners: readonly string[]; + readonly changedTests: readonly string[]; + readonly changedPaths: readonly string[]; + readonly reason: string; +} +export function planTestSelection( + files: readonly string[], + event: string, + tier: string, +): TestSelectionPlan; +export function compareTestSelection( + plan: unknown, + inventory: readonly string[], + results: readonly { + readonly name: string; + readonly durationMs: number; + readonly state?: string; + }[], +): { + readonly version: 1; + readonly observational: true; + readonly plan: TestSelectionPlan; + readonly selected: readonly string[]; + readonly omitted: readonly string[]; + readonly missing: readonly string[]; + readonly unexpected: readonly string[]; + readonly duplicateFiles: number; + readonly unobserved: readonly string[]; + readonly missedFailures: readonly string[]; + readonly complete: boolean; + readonly omittedDurationMs: number; +}; diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs new file mode 100644 index 0000000000..8165e3658c --- /dev/null +++ b/scripts/ci/test-selection.mjs @@ -0,0 +1,151 @@ +import { matchesGlob } from "node:path"; + +// Candidate ownership only: these leaf adapters and their producer tests can be +// compared with full-suite outcomes. This table never changes test execution. +// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners. +const OWNERS = { + adb: ["src/adb/**", "tests/boundary/providers/adb/**"], + apktool: ["src/apktool/**"], + dotnet: ["src/dotnet/**"], + evm: ["src/evm/**", "tests/boundary/evm/**"], + firmware: ["src/firmware/**", "tests/boundary/firmware/**"], + flutter: ["src/flutter/**"], + ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"], + hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"], + ida: ["src/ida/**", "tests/boundary/providers/ida/**"], + jeb: ["src/jeb/**"], + pcap: ["src/pcap/**", "tests/boundary/pcap/**"], + sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"], + wasm: ["src/wasm/**"], +}; +const ownersOf = (path) => + Object.entries(OWNERS) + .filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob))) + .map(([owner]) => owner); +const validPath = (path) => + typeof path === "string" && + /^(src|tests)\//.test(path) && + !/[\\\0\r\n]/.test(path) && + !path.split("/").some((part) => ["", ".", ".."].includes(part)); +const full = (reason, changedPaths = []) => ({ + version: 1, + mode: "full", + owners: [], + changedTests: [], + changedPaths, + reason, +}); + +/** Plan an observational PR subset from the classifier's existing Git diff. */ +export function planTestSelection(files, event, tier) { + if (event !== "pull_request" || tier !== "fast") + return full("Only ordinary PRs are candidates", files); + if (!files.length) return full("No changed paths available"); + const owners = new Set(); + const changedTests = []; + for (const path of files) { + if (!validPath(path)) + return full(`Shared or unknown input: ${path}`, files); + const matches = ownersOf(path); + if (!matches.length) return full(`Shared or unknown input: ${path}`, files); + for (const owner of matches) owners.add(owner); + if (path.endsWith(".test.ts")) changedTests.push(path); + } + return { + version: 1, + mode: "candidate", + owners: [...owners].sort(), + changedTests: [...new Set(changedTests)].sort(), + changedPaths: files, + reason: + "Observe affected leaf owners; retain integration and unowned tests", + }; +} + +/** Compare a proposal with the full run; missing observations never prove parity. */ +export function compareTestSelection(plan, inventory, results) { + if ( + plan?.version !== 1 || + !["full", "candidate"].includes(plan.mode) || + !Array.isArray(plan.owners) || + plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) || + !Array.isArray(plan.changedTests) || + !Array.isArray(plan.changedPaths) || + plan.changedPaths.some((path) => typeof path !== "string") || + plan.changedTests.some( + (path) => !validPath(path) || !path.endsWith(".test.ts"), + ) || + typeof plan.reason !== "string" || + (plan.mode === "candidate" && !plan.owners.length) || + (plan.mode === "full" && (plan.owners.length || plan.changedTests.length)) + ) + throw new Error("Invalid observational test selection plan"); + const expected = new Set(inventory); + if (!expected.size || expected.size !== inventory.length) + throw new Error("Test selection requires a nonempty unique inventory"); + const ownersByFile = new Map( + plan.mode === "candidate" + ? inventory.map((path) => [path, ownersOf(path)]) + : [], + ); + // Removed/renamed tests have no current executable identity. Keep all tests. + const removed = plan.changedTests.filter((path) => !expected.has(path)); + const missingOwners = plan.owners.filter( + (owner) => + ![...ownersByFile.values()].some((owners) => owners.includes(owner)), + ); + const effective = removed.length + ? full( + `Changed tests absent from current inventory: ${removed.join(", ")}`, + plan.changedPaths, + ) + : missingOwners.length + ? full( + `Owners without executable tests: ${missingOwners.join(", ")}`, + plan.changedPaths, + ) + : plan; + const selected = []; + const omitted = []; + for (const path of inventory.toSorted()) { + const owners = ownersByFile.get(path) ?? []; + const keep = + effective.mode === "full" || + !owners.length || + effective.changedTests.includes(path) || + owners.some((owner) => effective.owners.includes(owner)); + (keep ? selected : omitted).push(path); + } + const actual = new Set(results.map((file) => file.name)); + const missing = inventory.filter((path) => !actual.has(path)); + const unexpected = [...actual].filter((path) => !expected.has(path)); + const duplicateFiles = results.length - actual.size; + const unobserved = results + .filter((file) => !["passed", "failed"].includes(file.state)) + .map((file) => file.name); + const omittedSet = new Set(omitted); + const missedFailures = results + .filter((file) => omittedSet.has(file.name) && file.state === "failed") + .map((file) => file.name); + return { + version: 1, + observational: true, + plan: effective, + selected, + omitted, + missing, + unexpected, + duplicateFiles, + unobserved, + missedFailures, + complete: + !missing.length && + !unexpected.length && + !duplicateFiles && + !unobserved.length, + // Phase totals are scheduling weights, not an estimate of wall-time savings. + omittedDurationMs: results + .filter((file) => omittedSet.has(file.name)) + .reduce((sum, file) => sum + file.durationMs, 0), + }; +} diff --git a/scripts/ci/test-timing-reporter.mjs b/scripts/ci/test-timing-reporter.mjs index 29f0cc582d..ce0b1c5dc4 100644 --- a/scripts/ci/test-timing-reporter.mjs +++ b/scripts/ci/test-timing-reporter.mjs @@ -21,6 +21,7 @@ export default class TestTimingReporter { }; return { name: module.moduleId, + state: module.state(), durationMs: Object.values(phases).reduce( (total, value) => total + value, 0, @@ -33,6 +34,8 @@ export default class TestTimingReporter { JSON.stringify({ version: 1, success: reason === "passed" && errors.length === 0, + reason, + unhandledErrors: errors.length, files, }), ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 714d2f980b..fa4b71ebf2 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -178,6 +178,23 @@ it("keeps unknown CI scripts on complete qualification", async () => { for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); }); +it("emits an observational owner proposal without changing ordinary or full portable execution", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["src/evm/Example.ts"]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + testSelection: { mode: "candidate", owners: ["evm"] }, + }); + expect( + await classify(directory, base, head, "pull_request", true), + ).toMatchObject({ + code: true, + testSelection: { mode: "full", owners: [] }, + }); + const output = await readFile(join(directory, "scope-output"), "utf8"); + expect(output).toContain('test_selection={"version":1,"mode":"full"'); +}); + it("does not charge a documentation PR for unrelated implementation updates on main", async () => { const { directory, base } = await fixture(); await git(directory, ["switch", "-c", "docs/change"]); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 333e93f08f..06212ad075 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -8,6 +8,8 @@ import { expect, it } from "vitest"; import { parse } from "yaml"; import { z } from "zod"; +import { planTestSelection } from "../../../scripts/ci/test-selection.mjs"; + import { parseTestDurations } from "../../ciTestSequencer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { workflowGit } from "../../support/workflowGit.js"; @@ -61,6 +63,16 @@ it("collects failed-shard diagnostics before enforcing the shard result", async "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(steps[upload]?.with?.path).toContain("test-timings.json"); + expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); + expect(steps[upload]?.with?.path).toContain( + ".cache/ci-test-selection/report.json", + ); + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.CI_TEST_SELECTION).toBe( + "${{ needs.changes.outputs.test_selection }}", + ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); for (const result of ["failure", "cancelled", "skipped", ""]) { @@ -304,6 +316,13 @@ it("recovers complete module costs from real blob reports without executing test ), }) .parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8"))); + expect( + JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")), + ).toMatchObject({ + reason: "passed", + unhandledErrors: 0, + files: [{ state: "passed" }], + }); expect(report.files).toHaveLength(1); const file = report.files[0]!; const json = z @@ -349,7 +368,13 @@ it("recovers complete module costs from real blob reports without executing test const failed = JSON.parse( await readFile(join(root, "test-timings.json"), "utf8"), ) as unknown; - expect(failed).toMatchObject({ version: 1, success: false }); + expect(failed).toMatchObject({ + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: [{ state: "failed" }], + }); // Even a caller accidentally marking the merge successful cannot publish this report. await execute(process.execPath, [fileURLToPath(script)], { cwd: root, @@ -359,3 +384,76 @@ it("recovers complete module costs from real blob reports without executing test readFile(join(root, ".cache/ci-test-history/test-results.json")), ).rejects.toMatchObject({ code: "ENOENT" }); }); + +it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => { + const root = await createTestTempDirectory("rea-ci-selection-report-"); + await workflowGit(root, ["init"]); + const paths = [ + "src/sqlite/provider.test.ts", + "src/evm/provider.test.ts", + "tests/boundary/mcp/workflow.test.ts", + ]; + for (const path of paths) { + await mkdir(join(root, path, ".."), { recursive: true }); + await writeFile(join(root, path), "// fixture\n"); + } + await workflowGit(root, ["add", "src", "tests"]); + const timing = { + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: paths.map((name, index) => ({ + name, + durationMs: 10, + state: index === 1 ? "failed" : "passed", + })), + }; + const run = async (report: unknown) => { + await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); + await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "failure", + MERGE_RESULT: "failure", + CI_VERIFY_TEST_INVENTORY: "true", + CI_TEST_SELECTION: JSON.stringify( + planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"), + ), + GITHUB_SHA: "merge-sha", + CI_PR_HEAD_SHA: "pr-head-sha", + }, + }); + return JSON.parse( + await readFile( + join(root, ".cache/ci-test-selection/report.json"), + "utf8", + ), + ) as unknown; + }; + expect(await run(timing)).toMatchObject({ + observational: true, + complete: true, + selected: [paths[0], paths[2]], + omitted: [paths[1]], + missedFailures: [paths[1]], + executionSha: "merge-sha", + prHeadSha: "pr-head-sha", + }); + for (const report of [ + { ...timing, files: timing.files.slice(0, 1) }, + { ...timing, unhandledErrors: 1 }, + { ...timing, reason: "interrupted" }, + ]) + expect(await run(report)).toMatchObject({ complete: false }); + await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ + code: 1, + }); + await expect( + readFile(join(root, ".cache/ci-test-selection/report.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts new file mode 100644 index 0000000000..1dff7f9e3a --- /dev/null +++ b/tests/evaluation/ciTestSelection.test.ts @@ -0,0 +1,240 @@ +import { expect, it } from "vitest"; + +import { + compareTestSelection, + planTestSelection, +} from "../../scripts/ci/test-selection.mjs"; + +const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts"; +const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts"; +const adbSource = "src/adb/AdbProvider.test.ts"; +const adbBoundary = + "tests/boundary/providers/adb/adbBugreportPublication.test.ts"; +const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts"; +const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts"; +const processBoundary = + "tests/boundary/process/processCaptureLifecycle.test.ts"; +const acceptance = "tests/acceptance/tools/localToolContracts.test.ts"; +const composition = "tests/composition/directAnalysisCleanup.test.ts"; +const evaluation = "tests/evaluation/testDiscovery.test.ts"; + +const passed = (name: string) => ({ name, durationMs: 10, state: "passed" }); + +it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(plan).toMatchObject({ + mode: "candidate", + owners: ["adb", "sqlite"], + }); + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([]); +}); + +it("omits only unrelated owned tests while retaining dynamic boundary suites", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([adbSource, adbBoundary]); + expect(report.complete).toBe(true); +}); + +it.each([ + ["unowned shared source", "src/application/Workflow.ts"], + ["fixture input", "tests/conformance/c/fixture.c"], + ["configuration", "vitest.config.ts"], + ["documentation", "README.md"], + ["CLI implementation", "src/cli/apktoolCommands.ts"], + ["MCP implementation", "src/server/registerSqliteTools.ts"], + ["process ownership", "src/process/ProcessOwnership.ts"], + ["acceptance test", acceptance], + ["composition test", composition], +])("uses full fallback for %s changes", (_description, path) => { + expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({ + mode: "full", + owners: [], + changedTests: [], + }); +}); + +it("uses full fallback outside ordinary fast pull requests or without diff paths", () => { + expect( + planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast") + .mode, + ).toBe("full"); + expect( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "full", + ).mode, + ).toBe("full"); + expect(planTestSelection([], "pull_request", "fast").mode).toBe("full"); +}); + +it("keeps new changed tests and falls back when a changed test was removed", () => { + const addedTest = "src/sqlite/NewDatabaseRule.test.ts"; + const candidate = planTestSelection([addedTest], "pull_request", "fast"); + expect(candidate.changedTests).toEqual([addedTest]); + const inventory = [addedTest, adbSource]; + expect( + compareTestSelection(candidate, inventory, inventory.map(passed)).selected, + ).toContain(addedTest); + + const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts"; + const report = compareTestSelection( + planTestSelection([removedTest], "pull_request", "fast"), + [sqliteSource, adbSource], + [passed(sqliteSource), passed(adbSource)], + ); + expect(report.plan.mode).toBe("full"); + expect(report.selected).toEqual([adbSource, sqliteSource]); + expect(report.omitted).toEqual([]); +}); + +it("marks incomplete inventories and nonterminal outcomes as incomplete", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [sqliteSource, adbSource]; + const completeResults = inventory.map(passed); + + expect( + compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete, + ).toBe(false); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed("tests/evaluation/unexpected.test.ts"), + ]), + ).toMatchObject({ + complete: false, + unexpected: ["tests/evaluation/unexpected.test.ts"], + }); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed(adbSource), + ]), + ).toMatchObject({ complete: false, duplicateFiles: 1 }); + for (const state of ["pending", "skipped"]) + expect( + compareTestSelection(plan, inventory, [ + passed(sqliteSource), + { name: adbSource, durationMs: 0, state }, + ]), + ).toMatchObject({ complete: false, unobserved: [adbSource] }); +}); + +it("reports a failure from an omitted test, including collection or module failures", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const report = compareTestSelection( + plan, + [sqliteSource, adbSource], + [passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }], + ); + + expect(report.omitted).toContain(adbSource); + expect(report.missedFailures).toEqual([adbSource]); +}); + +it("rejects plans that name unowned owners or malformed changed tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + expect(() => + compareTestSelection( + { ...plan, owners: ["unknown"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); + expect(() => + compareTestSelection( + { ...plan, changedTests: ["src/sqlite/not-a-test.ts"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); +}); + +it("uses full fallback when a changed owner has no executable tests", () => { + const report = compareTestSelection( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ), + [adbSource], + [passed(adbSource)], + ); + + expect(report.plan.mode).toBe("full"); + expect(report.omitted).toEqual([]); +});