diff --git a/docs/python-packaged-applications.md b/docs/python-packaged-applications.md index fd3e0feec9..b3488a70fa 100644 --- a/docs/python-packaged-applications.md +++ b/docs/python-packaged-applications.md @@ -6,6 +6,16 @@ MCP. They read archive bytes without launching the application. Bytecode source decompilation, encrypted PYZ recovery, zipapp and PEX analysis are outside this capability. +`extract_artifact.coverage` preserves the inventory producer's source traversal +scope and certainty. Successful publication proves that the selected inventoried +regular files were written and verified; it does not prove that every source +entry was representable or every embedded payload was decoded. For example, a +directory containing an invalid UTF-8 filename can publish its representable +files successfully while coverage remains `unknown`. Extraction retains the +inventory's command provenance and limitations, plus the selected occurrences' +path-qualified derivation notices. A reconstructed `.pyc` header therefore stays +identified as derived bytes in the extraction result. + ## Facts and focused queries The root occurrence's `format_metadata` records the CArchive layout (2.0 or diff --git a/src/artifacts/extraction/ArtifactExtraction.ts b/src/artifacts/extraction/ArtifactExtraction.ts index d27e36b07a..6fee7dc9f2 100644 --- a/src/artifacts/extraction/ArtifactExtraction.ts +++ b/src/artifacts/extraction/ArtifactExtraction.ts @@ -16,6 +16,7 @@ import { ArtifactReaderFailure, type ArtifactEntry, type ArtifactReader, + type ArtifactReaderTraversal, } from "../ArtifactReader.js"; import { abortIfNeeded } from "../ArtifactHash.js"; import { DirectoryArtifactReader } from "../DirectoryArtifactReader.js"; @@ -30,9 +31,9 @@ import { artifactExtractionResultSchema, type ArtifactExtractionResult, type ArtifactGraphManifest, + type ArtifactCommand, type ArtifactNode, type ArtifactOccurrence, - type IntegrityContradiction, } from "../../domain/artifactGraph.js"; import { AnalysisUnsupportedTargetError } from "../../domain/analysisErrorCore.js"; import type { @@ -142,21 +143,12 @@ const materializeArtifactInventoryInScope = async ( } return true; }); - const selectedIds = new Set( - activeOccurrences.map(({ occurrence_id: id }) => id), - ); const neededNodes = new Set( activeOccurrences.map(({ artifact_id: id }) => id), ); const nodes = new Map(); for (const node of snapshot.nodes) if (neededNodes.has(node.artifact_id)) nodes.set(node.artifact_id, node); - const inventory: ExtractionInventory = { - manifest: snapshot.manifest, - integrityContradictions: snapshot.integrity_contradictions.filter( - ({ occurrence_id: id }) => selectedIds.has(id), - ), - }; const selected = activeOccurrences.map((occurrence) => { // REA never decrypts archive entries, so an encrypted entry makes the // complete extraction unsupported rather than the archive invalid. @@ -204,7 +196,7 @@ const materializeArtifactInventoryInScope = async ( return materializeSelection({ input, sourcePath, - inventory, + inventory: snapshot, selected, signal, }); @@ -232,7 +224,7 @@ const materializeSelection = async ({ }: { readonly input: ArtifactExtractionInput; readonly sourcePath: string; - readonly inventory: ExtractionInventory; + readonly inventory: ArtifactInventorySnapshot; readonly selected: readonly SelectedOccurrence[]; readonly signal: AbortSignal | undefined; }): Promise => { @@ -269,7 +261,13 @@ const materializeSelection = async ({ if (cause instanceof SafeOutputTreeCreationFailure) output = cause.tree; throw cause; } - await writeSelectedEntries({ reader, output, byPath, extracted, signal }); + const traversal = await writeSelectedEntries({ + reader, + output, + byPath, + extracted, + signal, + }); localReaderOwner = undefined; const closeAttempt = await input.resourceScope.release(readerOwner); if (closeAttempt.kind === "failed") @@ -279,7 +277,14 @@ const materializeSelection = async ({ extracted.sort((left, right) => compareUnicodeCodePoints(left.relative_path, right.relative_path), ); - completed = createExtractionResult(input, inventory, selected, extracted); + completed = createExtractionResult( + input, + inventory, + selected, + extracted, + traversal, + reader.provenance(), + ); await output.commit(signal); return completed; } catch (cause: unknown) { @@ -316,9 +321,22 @@ const writeSelectedEntries = async ({ readonly byPath: Map; readonly extracted: ExtractedOccurrence[]; readonly signal: AbortSignal | undefined; -}): Promise => { +}): Promise => { const registry = new ArtifactPathRegistry(); - for await (const entry of reader.entries(signal)) { + let status: ArtifactReaderTraversal["status"] = "complete"; + const limitations: string[] = []; + for await (const entry of reader.entries(signal, { + onCoverage: (coverage) => { + if (coverage.status !== "complete") status = coverage.status; + limitations.push(...coverage.limitations); + }, + onOmission: ({ path, kind }) => { + status = "unknown"; + limitations.push( + `Materialization traversal omitted ${kind} at ${JSON.stringify(path)}.`, + ); + }, + })) { const path = normalizeArtifactPath(entry.path); registry.add(path, entry.kind); // Logical declarations may share a query path with a real file. They do @@ -356,6 +374,7 @@ const writeSelectedEntries = async ({ "integrity", `Inventoried regular artifact entries were not materialized: ${[...byPath.keys()].sort(compareUnicodeCodePoints).join(", ")}`, ); + return { status, limitations }; }; const releaseExtractionOwners = async ( @@ -386,10 +405,18 @@ const releaseExtractionOwners = async ( const createExtractionResult = ( input: ArtifactExtractionInput, - inventory: ExtractionInventory, + inventory: ArtifactInventorySnapshot, selected: readonly SelectedOccurrence[], extracted: readonly ExtractedOccurrence[], + traversal: ArtifactReaderTraversal, + materializationProvenance: readonly ArtifactCommand[], ): ArtifactExtractionResult => { + const selectedIds = new Set( + selected.map(({ occurrence }) => occurrence.occurrence_id), + ); + const integrityContradictions = inventory.integrity_contradictions.filter( + ({ occurrence_id: id }) => selectedIds.has(id), + ); const extractionSemantic = { source_manifest_id: inventory.manifest.manifest_id, selected_occurrence_ids: selected @@ -408,24 +435,36 @@ const createExtractionResult = ( artifacts: extracted, containment_verified: true, cleanup: { attempted: false, verified: true, residual_paths: [] }, - provenance: [], - integrity_contradictions: inventory.integrityContradictions, + provenance: [...inventory.provenance, ...materializationProvenance], + coverage: { + ...inventory.coverage, + status: + inventory.coverage.status === "complete" && + traversal.status === "complete" + ? "complete" + : "unknown", + }, + integrity_contradictions: integrityContradictions, limitations: [ - "All regular files in the active artifact were materialized; nested archive contents remain represented by their containing file.", - ...(inventory.integrityContradictions.length === 0 + ...inventory.limitations, + ...traversal.limitations.filter( + (limitation) => !inventory.limitations.includes(limitation), + ), + ...selected.flatMap(({ occurrence }) => + occurrence.limitations.map( + (limitation) => `${occurrence.logical_path}: ${limitation}`, + ), + ), + "All selected inventoried regular files exposed by the active reader were materialized; nested archive contents remain represented by their containing file. Publication does not establish complete source traversal or content decoding.", + ...(integrityContradictions.length === 0 ? [] : [ - `${String(inventory.integrityContradictions.length)} extracted file(s) contradict declared integrity metadata; their bytes are observed-untrusted.`, + `${String(integrityContradictions.length)} extracted file(s) contradict declared integrity metadata; their bytes are observed-untrusted.`, ]), ], }); }; -interface ExtractionInventory { - readonly manifest: ArtifactGraphManifest; - readonly integrityContradictions: readonly IntegrityContradiction[]; -} - /** * Refuse a format without an extraction reader before inventory work starts. * The target kind, not the host, is unsupported by extraction. The error keeps diff --git a/src/domain/artifactGraph.ts b/src/domain/artifactGraph.ts index 364a5a4a65..70ddbd52a1 100644 --- a/src/domain/artifactGraph.ts +++ b/src/domain/artifactGraph.ts @@ -256,6 +256,7 @@ export const artifactExtractionResultSchema = z.object({ verified: z.boolean(), residual_paths: z.array(boundedRelativePathSchema), }), + coverage: artifactInventoryCoverageSchema, provenance: z.array(artifactCommandSchema), integrity_contradictions: z.array(integrityContradictionSchema), limitations: z.array(z.string()), diff --git a/tests/acceptance/pyinstallerArchive.test.ts b/tests/acceptance/pyinstallerArchive.test.ts index e330de9d0d..4cf529c266 100644 --- a/tests/acceptance/pyinstallerArchive.test.ts +++ b/tests/acceptance/pyinstallerArchive.test.ts @@ -360,6 +360,13 @@ it("extracts CArchive members through extract-artifact", async () => { const extracted = artifactExtractionResultSchema.parse( parseEvidence(JSON.parse(result.stdout)).normalized_result, ); + expect(extracted.limitations).toContain( + "PYZ.pyz/app.pyc: Python bytecode header was reconstructed from the archive Python magic; this digest covers the derived file.", + ); + expect(extracted.coverage).toEqual({ + scope: "container-entries", + status: "complete", + }); const hello = extracted.artifacts.find( (artifact) => artifact.relative_path === "hello", ); diff --git a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts index 53ffcd7d38..e75efcb2fb 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts @@ -15,6 +15,70 @@ import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjec import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; import { DirectoryArtifactReader } from "../../../../src/artifacts/DirectoryArtifactReader.js"; +it.for([false, true])( + "preserves source traversal coverage when invalid-byte names are present: %s", + async (invalidName, context) => { + if (invalidName && process.platform !== "linux") context.skip(); + const root = await createTestTempDirectory("rea-extract-coverage-"); + const source = join(root, "source"); + await mkdir(source); + await writeFile(join(source, "visible.txt"), "retained"); + if (invalidName) + await writeFile( + Buffer.concat([Buffer.from(`${source}/`), Buffer.from([0xff])]), + "unrepresented", + ); + const snapshot = await scanArtifactInventory(source); + const result = await extract(source, join(root, "output")); + if (!result.ok) throw result.error; + const extracted = artifactExtractionResultSchema.parse(result.value.result); + expect(extracted.coverage).toEqual(snapshot.coverage); + expect(extracted.coverage.status).toBe( + invalidName ? "unknown" : "complete", + ); + expect(extracted.limitations).toEqual( + expect.arrayContaining([...snapshot.limitations]), + ); + expect( + extracted.artifacts.map(({ relative_path }) => relative_path), + ).toEqual(["visible.txt"]); + expect( + await readFile(join(extracted.output_root, "visible.txt"), "utf8"), + ).toBe("retained"); + }, +); + +it.skipIf(process.platform !== "linux")( + "retains newly unknown materialization traversal after a complete inventory", + async () => { + const root = await createTestTempDirectory("rea-extract-late-coverage-"); + const source = join(root, "source"); + await mkdir(source); + await writeFile(join(source, "visible.txt"), "retained"); + const snapshot = await scanArtifactInventory(source); + expect(snapshot.coverage.status).toBe("complete"); + await writeFile( + Buffer.concat([Buffer.from(`${source}/`), Buffer.from([0xff])]), + "unrepresented", + ); + const extracted = await materializeArtifactInventory( + { + inputPath: source, + inputFormat: "asar", + outputRoot: join(root, "output"), + environment: {}, + integrityPolicy: "fail", + }, + source, + snapshot, + ); + expect(extracted.coverage.status).toBe("unknown"); + expect(extracted.limitations).toContainEqual( + expect.stringContaining("not valid UTF-8"), + ); + }, +); + it.each(["file", "directory"] as const)( "rolls back when an inventoried %s disappears before extraction", async (removedKind) => { diff --git a/tests/fixtures/machoArmSlices.ts b/tests/fixtures/machoArmSlices.ts index a7ff070af4..d22631faf6 100644 --- a/tests/fixtures/machoArmSlices.ts +++ b/tests/fixtures/machoArmSlices.ts @@ -86,6 +86,13 @@ export const expectArmSliceExtraction = async ( ) => { const extracted = artifactExtractionResultSchema.parse(value); expect(extracted.containment_verified).toBe(true); + // Inventory and materialization each acquire their own lipo observation. + expect(extracted.provenance).toHaveLength(2); + expect(extracted.provenance).toEqual( + expect.arrayContaining([ + expect.objectContaining({ tool: "lipo", effects: ["read"] }), + ]), + ); expect(extracted.artifacts).toHaveLength(fixture.slices.length); for (const slice of fixture.slices) { const relativePath = `slices/${slice.name}`;