From 993356b85adc72036ba44cd87b9335e2ec172a32 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Sun, 12 Jul 2026 22:09:34 +0000 Subject: [PATCH] test: add package installation end-to-end coverage --- .github/workflows/ci.yml | 17 +++++++++++ .github/workflows/real-hopper.yml | 43 +++++++++++++++++++++++++++ .github/workflows/release.yml | 16 ++++++++++ package.json | 1 + scripts/verify-package.mjs | 44 ++++++++++++++++++++++++++-- scripts/verify-published-package.mjs | 44 ++++++++++++++++++++++++++++ scripts/verify-real-hopper.mjs | 22 ++++++++++++-- tests/artifactProvider.test.ts | 2 +- tests/fixtures/fakeHopper.mjs | 35 ++++++++++++++++++++++ 9 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/real-hopper.yml create mode 100644 scripts/verify-published-package.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4bb07d4cb1..912129e836 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,23 @@ jobs: path: test-results.xml retention-days: 30 + package-e2e: + name: Package E2E (${{ matrix.os }}) + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-14] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - run: npm run rebuild:native + - run: npm run verify:package + docs: name: Docs Generation runs-on: ubuntu-latest diff --git a/.github/workflows/real-hopper.yml b/.github/workflows/real-hopper.yml new file mode 100644 index 0000000000..09303ba487 --- /dev/null +++ b/.github/workflows/real-hopper.yml @@ -0,0 +1,43 @@ +name: Real Hopper E2E + +on: + workflow_dispatch: + inputs: + first_target: + description: Absolute path to first test binary on the runner + required: true + type: string + second_target: + description: Absolute path to distinct second test binary on the runner + required: true + type: string + +permissions: + contents: read + +jobs: + verify: + name: Installed package against real Hopper + runs-on: [self-hosted, macOS, hopper] + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - run: npm run rebuild:native + - name: Install packed CLI in isolation + shell: bash + run: | + tarball="$(npm pack --silent)" + prefix="${RUNNER_TEMP}/rea-prefix" + npm install --global --prefix "${prefix}" "./${tarball}" + echo "REA_VERIFY_SERVER_COMMAND=${prefix}/bin/rea" >> "${GITHUB_ENV}" + echo 'REA_VERIFY_SERVER_ARGS_JSON=["mcp"]' >> "${GITHUB_ENV}" + - name: Verify two real binaries + env: + HOPPER_TARGET_PATH: ${{ inputs.first_target }} + HOPPER_SECOND_TARGET_PATH: ${{ inputs.second_target }} + run: npm run verify:hopper diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index cf7ecec48b..fbb603c351 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -43,3 +43,19 @@ jobs: - name: Publish run: npm publish --access public + + - name: Verify published CLI from npm + shell: bash + run: | + version="$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)')" + for attempt in 1 2 3 4 5 6; do + if npm view "rea-agents@${version}" version --json >/dev/null 2>&1; then + npx --yes --package "rea-agents@${version}" rea --help + npx --yes --package "rea-agents@${version}" rea --llms + npm run verify:published -- "${version}" + exit 0 + fi + if [ "${attempt}" -lt 6 ]; then sleep 10; fi + done + echo "Published rea-agents@${version} was not available from npm" >&2 + exit 1 diff --git a/package.json b/package.json index fba3daf302..97c2fe0a1c 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "build:fixtures": "node scripts/build-conformance-fixtures.mjs", "verify:fixtures": "npm run build:fixtures && node scripts/verify-conformance-fixtures.mjs", "verify:package": "npm run build && node scripts/verify-package.mjs", + "verify:published": "node scripts/verify-published-package.mjs", "config:print": "node scripts/print-mcp-config.mjs", "docs:generate": "typedoc", "prepack": "npm run build", diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index cf0c740328..e1fa3f0b57 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -96,7 +96,10 @@ try { HOME: home, XDG_CONFIG_HOME: join(home, ".config"), PATH: `${fakeBin}:${process.env.PATH ?? ""}`, - HOPPER_LAUNCHER_PATH: hopper, + HOPPER_LAUNCHER_PATH: process.execPath, + HOPPER_LOADER_ARGS_JSON: JSON.stringify([ + join(root, "tests/fixtures/fakeLauncher.mjs"), + ]), REA_NPX_LOG: npxLog, REA_EVIDENCE_ROOTS_JSON: JSON.stringify([evidenceRoot]), REA_REFERENCE_ROOTS_JSON: JSON.stringify([referenceRoot]), @@ -142,6 +145,16 @@ try { artifactInventory.normalized_result?.manifest?.root_format !== "zip" ) throw new Error("packaged artifact inventory CLI failed"); + const overview = json( + await run(cli, ["analyze", process.execPath, "--json"], environment), + ); + if ( + overview.operation !== "binary_overview" || + overview.normalized_result?.procedure_count < 1 + ) + throw new Error( + `packaged Hopper-backed analyze CLI failed: ${JSON.stringify(overview)}`, + ); const referenceImport = json( await run( cli, @@ -292,6 +305,27 @@ try { }); if (result.isError !== true) throw new Error("packaged target-free MCP omitted no-target error"); + const opened = await client.callTool({ + name: "open_binary", + arguments: { path: process.execPath }, + }); + if (opened.isError === true) + throw new Error("packaged MCP could not open a binary"); + const overviewResult = await client.callTool({ + name: "binary_overview", + arguments: {}, + }); + if ( + overviewResult.isError === true || + json(text(overviewResult)).normalized_result?.procedure_count < 1 + ) + throw new Error("packaged MCP analysis workflow failed"); + const closed = await client.callTool({ + name: "close_binary", + arguments: {}, + }); + if (closed.isError === true) + throw new Error("packaged MCP could not close its binary"); const mcpBundlePath = join(evidenceRoot, "mcp.json"); const mcpExport = await client.callTool({ name: "export_evidence_bundle", @@ -313,7 +347,7 @@ try { } process.stdout.write( - `${JSON.stringify({ cli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: process.platform === "darwin" ? "idempotent" : "unsupported-host-rejected", clients: process.platform === "darwin" ? 2 : 0, backupReadback: process.platform === "darwin", failureRecovery: process.platform === "darwin", skill: process.platform === "darwin", mcpTools: expectedToolCount, evidenceMcp: true, targetFree: true })}\n`, + `${JSON.stringify({ cli: true, analysisCli: true, artifactCli: true, evidenceCli: true, incurMcpCommand: "npx -y rea-agents mcp", doctor: "platform-appropriate", setup: process.platform === "darwin" ? "idempotent" : "unsupported-host-rejected", clients: process.platform === "darwin" ? 2 : 0, backupReadback: process.platform === "darwin", failureRecovery: process.platform === "darwin", skill: process.platform === "darwin", mcpTools: expectedToolCount, evidenceMcp: true, targetFree: true, targetLifecycle: true })}\n`, ); } finally { if (tarball) await rm(join(root, tarball), { force: true }); @@ -326,3 +360,9 @@ async function run(command, args, env) { function json(text) { return JSON.parse(text); } + +function text(result) { + const value = result.content?.find((item) => item.type === "text")?.text; + if (typeof value !== "string") throw new Error("MCP result omitted text"); + return value; +} diff --git a/scripts/verify-published-package.mjs b/scripts/verify-published-package.mjs new file mode 100644 index 0000000000..316237969e --- /dev/null +++ b/scripts/verify-published-package.mjs @@ -0,0 +1,44 @@ +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; + +const version = process.argv[2]; +if (!/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/u.test(version ?? "")) + throw new Error("Usage: node scripts/verify-published-package.mjs "); + +const serverEnvironment = { ...process.env }; +delete serverEnvironment.HOPPER_TARGET_PATH; +const transport = new StdioClientTransport({ + command: "npx", + args: ["--yes", "--package", `rea-agents@${version}`, "rea", "mcp"], + env: serverEnvironment, + stderr: "pipe", +}); +const client = new Client({ name: "published-package-canary", version: "1" }); +let stderr = ""; +transport.stderr?.on("data", (chunk) => { + if (stderr.length < 16_384) stderr += chunk.toString("utf8"); +}); + +try { + await client.connect(transport); + const tools = await client.listTools(); + if (tools.tools.length !== 68) + throw new Error( + `Published MCP exposed ${String(tools.tools.length)} tools`, + ); + const status = await client.callTool({ + name: "binary_session", + arguments: {}, + }); + if (status.isError === true) + throw new Error("Published target-free MCP session check failed"); +} catch (cause) { + throw new Error(`Published MCP canary failed: ${stderr}`, { cause }); +} finally { + await client.close(); + await transport.close(); +} + +process.stdout.write( + `${JSON.stringify({ package: "rea-agents", version, mcpTools: 68 })}\n`, +); diff --git a/scripts/verify-real-hopper.mjs b/scripts/verify-real-hopper.mjs index ff18bf4c79..c45a79db80 100644 --- a/scripts/verify-real-hopper.mjs +++ b/scripts/verify-real-hopper.mjs @@ -15,6 +15,17 @@ import { const execFileAsync = promisify(execFile); const timeout = 180_000; + +const parseServerArgs = (encoded) => { + const parsed = JSON.parse(encoded); + if ( + !Array.isArray(parsed) || + parsed.some((value) => typeof value !== "string") + ) + throw new Error("REA_VERIFY_SERVER_ARGS_JSON must encode string arguments"); + return parsed; +}; + const sessionsBefore = new Set( (await readdir("/tmp")).filter((name) => name.startsWith("rea-")), ); @@ -245,10 +256,17 @@ if (targetA === targetB) const serverEnvironment = { ...process.env }; delete serverEnvironment.HOPPER_TARGET_PATH; delete serverEnvironment.HOPPER_SECOND_TARGET_PATH; +delete serverEnvironment.REA_VERIFY_SERVER_COMMAND; +delete serverEnvironment.REA_VERIFY_SERVER_ARGS_JSON; + +const serverCommand = process.env.REA_VERIFY_SERVER_COMMAND ?? process.execPath; +const serverArgs = parseServerArgs( + process.env.REA_VERIFY_SERVER_ARGS_JSON ?? '["dist/main.js"]', +); const transport = new StdioClientTransport({ - command: process.execPath, - args: ["dist/main.js"], + command: serverCommand, + args: serverArgs, cwd: process.cwd(), env: serverEnvironment, stderr: "pipe", diff --git a/tests/artifactProvider.test.ts b/tests/artifactProvider.test.ts index f6a2a9417f..656614dea6 100644 --- a/tests/artifactProvider.test.ts +++ b/tests/artifactProvider.test.ts @@ -294,7 +294,7 @@ describe("artifact graph provider", () => { offset: 500, next_offset: null, }); - }); + }, 15_000); it("uses bounded native lipo metadata for universal slice ranges", async () => { const root = await mkdtemp(join(tmpdir(), "rea-slices-")); diff --git a/tests/fixtures/fakeHopper.mjs b/tests/fixtures/fakeHopper.mjs index cea23a29c2..1d39effd15 100644 --- a/tests/fixtures/fakeHopper.mjs +++ b/tests/fixtures/fakeHopper.mjs @@ -1,6 +1,39 @@ import { createServer } from "node:net"; const [socketPath, token, runId] = process.argv.slice(2); +const enhancedFixtureResult = (method) => { + switch (method) { + case "list_segments": + return [ + { + name: "__TEXT", + start: "0x1000", + end: "0x2000", + readable: true, + writable: false, + executable: true, + }, + ]; + case "list_documents": + return ["fixture"]; + case "list_procedures": + return { + items: [{ address: "0x1000", value: "fixture" }], + offset: 0, + limit: 500, + total: 1, + next_offset: null, + has_more: false, + }; + case "list_strings": + return ["fixture"]; + case "procedure_pseudo_code": + return "return 0;"; + default: + return undefined; + } +}; + const server = createServer((socket) => { socket.on("error", () => undefined); socket.setEncoding("utf8"); @@ -50,6 +83,8 @@ const server = createServer((socket) => { }); } else if (request.method === "current_document") { send({ id: request.id, result: "fixture" }); + } else if (enhancedFixtureResult(request.method) !== undefined) { + send({ id: request.id, result: enhancedFixtureResult(request.method) }); } else if (request.method === "resolve_containing_procedure") { send({ id: request.id,