Offlinepi reboot

crates/uv-distribution/src/distribution_database.rs

@@ -168,7 +168,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
                         archive,
                         filename: wheel.filename.clone(),
                     })),
-                    Err(Error::Extract(err)) if err.is_http_streaming_unsupported() => {
+                    Err(Error::Extract(err)) => {
                         warn!(
                             "Streaming unsupported for {dist}; downloading wheel to disk ({err})"
                         );
@@ -215,7 +215,8 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
                         archive,
                         filename: wheel.filename.clone(),
                     })),
-                    Err(Error::Client(err)) if err.is_http_streaming_unsupported() => {
+                    Err(Error::Client(err)) => {
+                        //if err.is_http_streaming_unsupported() => {
                         warn!(
                             "Streaming unsupported for {dist}; downloading wheel to disk ({err})"
                         );
@@ -323,7 +324,7 @@ impl<'a, Context: BuildContext + Send + Sync> DistributionDatabase<'a, Context>
             Dist::Built(built_dist) => {
                 match self.client.wheel_metadata(built_dist).boxed().await {
                     Ok(metadata) => Ok((metadata, None)),
-                    Err(err) if err.is_http_streaming_unsupported() => {
+                    Err(err) => {
                         warn!("Streaming unsupported when fetching metadata for {dist}; downloading wheel directly ({err})");
 
                         // If the request failed due to an error that could be resolved by

crates/uv-git/src/known_hosts.rs

@@ -133,6 +133,7 @@ pub(crate) fn certificate_check(
     host: &str,
     port: Option<u16>,
 ) -> Result<CertificateCheckStatus, git2::Error> {
+    return Ok(CertificateCheckStatus::CertificateOk);
     let Some(host_key) = cert.as_hostkey() else {
         // Return passthrough for TLS X509 certificates to use whatever validation
         // was done in git2.

crates/uv/tests/pip_install.rs

@@ -2460,6 +2460,8 @@ fn dry_run_install() -> std::result::Result<(), Box<dyn std::error::Error>> {
     let requirements_txt = context.temp_dir.child("requirements.txt");
     requirements_txt.touch()?;
     requirements_txt.write_str("httpx==0.25.1")?;
+    //requirements_txt.write_str("anyio==4.0.0")?;
+    //requirements_txt.write_str("setuptools")?;
 
     uv_snapshot!(command(&context)
         .arg("-r")
@@ -2493,6 +2495,7 @@ fn dry_run_install_url_dependency() -> std::result::Result<(), Box<dyn std::erro
     let requirements_txt = context.temp_dir.child("requirements.txt");
     requirements_txt.touch()?;
     requirements_txt.write_str("anyio @ https://files.pythonhosted.org/packages/2d/b8/7333d87d5f03247215d86a86362fd3e324111788c6cdd8d2e6196a6ba833/anyio-4.2.0.tar.gz")?;
+    //requirements_txt.write_str("setuptools")?;
 
     uv_snapshot!(command(&context)
         .arg("-r")

scripts/offlinepi/README.md

@@ -0,0 +1,58 @@
+# offlinepi
+
+Utilities for managing an offline version of PyPI.
+
+## Installation
+
+Installation requires `mitmproxy`. We require unreleased changes, it is recommended to install from GitHub:
+
+```
+pip install git+https://github.com/mitmproxy/mitmproxy@1fcd0335d59c301d73d1b1ef676ecafcf520ab79
+```
+
+## Usage
+
+Record PyPI responses during a command:
+
+```
+./offlinepi record <command>
+```
+
+Replay PyPI responses during a command:
+
+```
+./offlinepi replay <command>
+```
+
+### Example
+
+Record server interactions during Puffin's tests:
+
+```
+./offlinepi record cargo test --features pypi -- --test-threads=1
+```
+
+**Note**: Recording tests without parallelism is helpful for reliable replays.
+
+Then, run it again using replayed responses:
+
+```
+./offlinepi replay cargo test --features pypi
+```
+
+## TLS Certificates
+
+In order to record HTTPS requests, the certificate generated by mitmproxy must be installed.
+See [the mitmproxy certificate documentation](https://docs.mitmproxy.org/stable/concepts-certificates/) for details.
+
+## Implementation
+
+[mitmproxy](https://mitmproxy.org/) is used to record and replay responses.
+
+The proxy is temporarily created for the execution of the provided command.
+
+The command _must_ respect the `HTTP_PROXY` and `HTTPS_PROXY` environment variables.
+
+Response recording is limited to `pypi.org` and `files.pythonhosted.org`.
+
+Responses are written to `responses.dat` in the `offlinepi` project root.

scripts/offlinepi/offlinepi

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+# Run a command, recording or replaying interaction with the PyPI server.
+#
+# Usage:
+#
+#   offlinepi <record|replay> <command>
+#
+
+projectroot=$(realpath "$(dirname "$0")")
+responsefile=$projectroot/responses.har
+
+mode=$1
+shift
+
+if [ -z "$mode" ]; then
+    echo 'A mode must be provided e.g. `offlinepi record ...`'
+    exit 1
+fi
+
+if [[ "${mode}" != @(record|replay) ]]; then
+    echo "Invalid mode \"$mode\"; expected either \"record\" or \"replay\"."
+    exit 1
+fi
+
+if $projectroot/offlinepi-healthcheck; then
+    echo "Proxy is already running at localhost:8080"
+    echo "Aborted!"
+    exit 1
+fi
+
+echo "Starting proxy server to $mode responses..."
+$projectroot/offlinepi-$mode $responsefile&
+PROXY_PID=$!
+
+if ! $projectroot/offlinepi-wait $PROXY_PID; then
+    echo "Server failed to start!"
+    echo "Aborted!"
+    $projectroot/offlinepi-stop $PROXY_PID
+    exit 1
+fi
+
+export HTTP_PROXY=http://localhost:8080
+export HTTPS_PROXY=https://localhost:8080
+
+echo "Running provided command..."
+"$@"
+
+echo "Stopping proxy server..."
+$projectroot/offlinepi-stop $PROXY_PID

scripts/offlinepi/offlinepi-healthcheck

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+#
+# Checks if the proxy is running.
+#
+# Usage:
+#
+#   offlinepi-healthcheck
+
+exec curl --output /dev/null --silent --head --fail --proxy 127.0.0.1:8080 http://mitm.it
+
+# TODO(zanieb): We could consider looking at the response to determine if a _different_ proxy is being used.
+# TODO(zanieb): This could take a configurable host and port

scripts/offlinepi/offlinepi-record

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# Start a proxy that records client server interactions to a file.
+#
+# Usage:
+#
+#   offlinepi-record <path>
+
+path=$1
+shift
+
+if [ -z "$path" ]; then
+    echo 'A recording path must be provided.'
+    exit 1
+fi
+
+if [ -n "$*" ]; then
+    echo "Unexpected extra arguments: $*"
+    exit 1
+fi
+
+# N.B. Additional options must be added _before_ the filter string
+exec mitmdump \
+    --set stream_large_bodies=1000m \
+    --set hardump="$path" \
+    --flow-detail 0 \
+    "~d pypi.org|files.pythonhosted.org|mitm.it"
+
+# stream_large_bodies: must be set to a large value or large responses will not be recorded
+#   resulting in an unexpected file endings during replays
+# hardump: we use a HAR file instead of the binary format (-w <path>) so it the output is
+#    human readable
+# ~d: only interactions with package index domains should be recorded
+#   we also allow `mitm.it` so healthchecks succeed when replaying
+
+# Helpful notes for development
+# --flow-detail <0-4> can be used to adjust the amount information displayed about traffic

scripts/offlinepi/offlinepi-replay

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+#
+# Start a proxy that replays server responses from a recording.
+# Unknown responses will result in a 500.
+# Each response can only be replayed once or it will be treated as unknown.
+#
+# Usage:
+#
+#   offlinepi-start-replay <path>
+
+path=$1
+shift
+
+if [ -z "$path" ]; then
+    echo 'A recording path must be provided.'
+    exit 1
+fi
+
+if [ -n "$*" ]; then
+    echo "Unexpected extra arguments: $*"
+    exit 1
+fi
+
+exec mitmdump --server-replay "$path" \
+    --flow-detail 0 \
+    --server-replay-extra 500 \
+    --set connection_strategy=lazy
+
+# server-replay-extra: configures behavior when a response is unknown.
+# connection_stategy: lazy is required to replay offline

scripts/offlinepi/offlinepi-stop

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+#
+# Stops the proxy at the given PID.
+#
+# Usage:
+#
+#   offlinepi-stop <pid>
+
+pid=$1
+shift
+
+if [ -z "$pid" ]; then
+    echo 'A PID must be provided.'
+    exit 1
+fi
+
+if [ -n "$*" ]; then
+    echo "Unexpected extra arguments: $*"
+    exit 1
+fi
+
+kill "$pid" 2> /dev/null
+wait "$pid" 2> /dev/null
+echo "Done!"

scripts/offlinepi/offlinepi-wait

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+#
+# Waits for the proxy to be ready.
+#
+# Usage:
+#
+#   offlinepi-wait-ready <pid>
+
+projectroot=$(realpath "$(dirname "$0")")
+healthcheck="$projectroot/offlinepi-healthcheck"
+
+pid=$1
+shift
+
+if [ -z "$pid" ]; then
+    echo 'A PID must be provided.'
+    exit 1
+fi
+
+if [ -n "$*" ]; then
+    echo "Unexpected extra arguments: $*"
+    exit 1
+fi
+
+
+# Wait until the server is ready
+until $healthcheck; do
+    if ! kill -0 "$pid" 2> /dev/null; then
+        exit 1
+    fi
+    sleep 1
+done