Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ac4945ca97 | ||
|
|
2ca2c7aff8 | ||
|
|
a18df74946 | ||
|
|
2bc8414cc1 | ||
|
|
c4569343a4 | ||
|
|
5f5820c71a | ||
|
|
4719a6fecf | ||
|
|
c999359b9f | ||
|
|
f22e2b6e68 | ||
|
|
31a9550f5b | ||
|
|
201f2d61b9 | ||
|
|
3ae4dfae8e | ||
|
|
7b095fe4ff | ||
|
|
890bcb1bb6 | ||
|
|
aa97ea9f82 | ||
|
|
9b40dbbf27 | ||
|
|
289f3e801b | ||
|
|
edacd09dc8 | ||
|
|
5682863725 | ||
|
|
4304d7a638 | ||
|
|
f56f88da94 | ||
|
|
87c8b361ea | ||
|
|
cd505ddb6c | ||
|
|
eeea617fb1 | ||
|
|
cc6dbddb49 | ||
|
|
9d3df2cdc6 | ||
|
|
ab601c3830 | ||
|
|
3738be2b6d | ||
|
|
53160f01c7 | ||
|
|
594ad55bd8 | ||
|
|
d2615f51dc | ||
|
|
c097733ae7 | ||
|
|
67d4b7dafc | ||
|
|
b1d6bbce0c | ||
|
|
20124f4891 | ||
|
|
0dd540afaf | ||
|
|
df71083359 | ||
|
|
349c7bb3ea | ||
|
|
5a30c6b44b | ||
|
|
929924accd | ||
|
|
812b46960c | ||
|
|
874080dbda | ||
|
|
93dd9d4ed4 | ||
|
|
3f0ced0143 | ||
|
|
8112ab6d04 | ||
|
|
e5fc05f5cd | ||
|
|
1068ff659a | ||
|
|
d4d9bbe424 | ||
|
|
cf3a8c8ede | ||
|
|
920d992459 | ||
|
|
c61b3ba858 | ||
|
|
dc6e564ea2 | ||
|
|
24536b5e18 | ||
|
|
908fd59019 | ||
|
|
a19aa37ea8 | ||
|
|
c46bd5900b | ||
|
|
5f98ed23b3 | ||
|
|
c6b135398a | ||
|
|
791e44796e | ||
|
|
b428dd8471 | ||
|
|
b88479446c | ||
|
|
1d6217ef5a | ||
|
|
746c7f05de | ||
|
|
29836d979a | ||
|
|
5ba6e33fa8 | ||
|
|
643c4ce7ef | ||
|
|
c011f90b76 | ||
|
|
875481b9a2 | ||
|
|
05985583f0 | ||
|
|
651fa716b4 | ||
|
|
9be3982dc6 | ||
|
|
27c9fb4cd3 |
131
.appveyor.yml
131
.appveyor.yml
@@ -1,131 +0,0 @@
|
||||
# Appveyor configuration template for Rust using rustup for Rust installation
|
||||
# https://github.com/starkat99/appveyor-rust
|
||||
|
||||
## Operating System (VM environment) ##
|
||||
|
||||
# Rust needs at least Visual Studio 2013 Appveyor OS for MSVC targets.
|
||||
os: Visual Studio 2015
|
||||
|
||||
## Build Matrix ##
|
||||
|
||||
# This configuration will setup a build for each channel & target combination (12 windows
|
||||
# combinations in all).
|
||||
#
|
||||
# There are 3 channels: stable, beta, and nightly.
|
||||
#
|
||||
# Alternatively, the full version may be specified for the channel to build using that specific
|
||||
# version (e.g. channel: 1.5.0)
|
||||
#
|
||||
# The values for target are the set of windows Rust build targets. Each value is of the form
|
||||
#
|
||||
# ARCH-pc-windows-TOOLCHAIN
|
||||
#
|
||||
# Where ARCH is the target architecture, either x86_64 or i686, and TOOLCHAIN is the linker
|
||||
# toolchain to use, either msvc or gnu. See https://www.rust-lang.org/downloads.html#win-foot for
|
||||
# a description of the toolchain differences.
|
||||
# See https://github.com/rust-lang-nursery/rustup.rs/#toolchain-specification for description of
|
||||
# toolchains and host triples.
|
||||
#
|
||||
# Comment out channel/target combos you do not wish to build in CI.
|
||||
#
|
||||
# You may use the `cargoflags` and `RUSTFLAGS` variables to set additional flags for cargo commands
|
||||
# and rustc, respectively. For instance, you can uncomment the cargoflags lines in the nightly
|
||||
# channels to enable unstable features when building for nightly. Or you could add additional
|
||||
# matrix entries to test different combinations of features.
|
||||
environment:
|
||||
matrix:
|
||||
|
||||
### MSVC Toolchains ###
|
||||
|
||||
# Stable 64-bit MSVC
|
||||
- channel: stable
|
||||
target: x86_64-pc-windows-msvc
|
||||
# Stable 32-bit MSVC
|
||||
- channel: stable
|
||||
target: i686-pc-windows-msvc
|
||||
# Beta 64-bit MSVC
|
||||
- channel: beta
|
||||
target: x86_64-pc-windows-msvc
|
||||
# Beta 32-bit MSVC
|
||||
- channel: beta
|
||||
target: i686-pc-windows-msvc
|
||||
# Nightly 64-bit MSVC
|
||||
- channel: nightly
|
||||
target: x86_64-pc-windows-msvc
|
||||
cargoflags: --features "unstable"
|
||||
# Nightly 32-bit MSVC
|
||||
- channel: nightly
|
||||
target: i686-pc-windows-msvc
|
||||
cargoflags: --features "unstable"
|
||||
|
||||
### GNU Toolchains ###
|
||||
|
||||
# Stable 64-bit GNU
|
||||
- channel: stable
|
||||
target: x86_64-pc-windows-gnu
|
||||
MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
|
||||
# Stable 32-bit GNU
|
||||
- channel: stable
|
||||
target: i686-pc-windows-gnu
|
||||
MINGW_PATH: 'C:\MinGW\bin'
|
||||
# Beta 64-bit GNU
|
||||
- channel: beta
|
||||
target: x86_64-pc-windows-gnu
|
||||
MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
|
||||
# Beta 32-bit GNU
|
||||
- channel: beta
|
||||
target: i686-pc-windows-gnu
|
||||
MINGW_PATH: 'C:\MinGW\bin'
|
||||
# Nightly 64-bit GNU
|
||||
- channel: nightly
|
||||
target: x86_64-pc-windows-gnu
|
||||
MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
|
||||
cargoflags: --features "unstable"
|
||||
# Nightly 32-bit GNU
|
||||
- channel: nightly
|
||||
target: i686-pc-windows-gnu
|
||||
MINGW_PATH: 'C:\MinGW\bin'
|
||||
cargoflags: --features "unstable"
|
||||
|
||||
### Allowed failures ###
|
||||
|
||||
# See Appveyor documentation for specific details. In short, place any channel or targets you wish
|
||||
# to allow build failures on (usually nightly at least is a wise choice). This will prevent a build
|
||||
# or test failure in the matching channels/targets from failing the entire build.
|
||||
matrix:
|
||||
allow_failures:
|
||||
- channel: nightly
|
||||
- channel: beta
|
||||
|
||||
# If you only care about stable channel build failures, uncomment the following line:
|
||||
#- channel: beta
|
||||
|
||||
## Install Script ##
|
||||
|
||||
# This is the most important part of the Appveyor configuration. This installs the version of Rust
|
||||
# specified by the 'channel' and 'target' environment variables from the build matrix. This uses
|
||||
# rustup to install Rust.
|
||||
#
|
||||
# For simple configurations, instead of using the build matrix, you can simply set the
|
||||
# default-toolchain and default-host manually here.
|
||||
install:
|
||||
- appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
|
||||
- rustup-init -yv --default-toolchain %channel% --default-host %target%
|
||||
- set PATH=%PATH%;%USERPROFILE%\.cargo\bin
|
||||
- if defined MINGW_PATH set PATH=%PATH%;%MINGW_PATH%
|
||||
- rustc -vV
|
||||
- cargo -vV
|
||||
- rustup component add rustfmt
|
||||
|
||||
## Build Script ##
|
||||
|
||||
# 'cargo test' takes care of building for us, so disable Appveyor's build stage. This prevents
|
||||
# the "directory does not contain a project or solution file" error.
|
||||
build: false
|
||||
|
||||
# Uses 'cargo test' to run tests and build. Alternatively, the project may call compiled programs
|
||||
# directly or perform other testing commands. Rust will automatically be placed in the PATH
|
||||
# environment variable.
|
||||
test_script:
|
||||
- cargo build --all --locked --verbose %cargoflags%
|
||||
- cargo test --all --locked --verbose %cargoflags%
|
||||
22
.github/workflows/build_gnu_linux.yml
vendored
Normal file
22
.github/workflows/build_gnu_linux.yml
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
name: GNU/Linux
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
rust:
|
||||
- stable
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- run: git config --global core.autocrlf false
|
||||
- uses: actions/checkout@v2
|
||||
- name: Build
|
||||
run: cargo build --all --locked --verbose
|
||||
22
.github/workflows/build_macos.yml
vendored
Normal file
22
.github/workflows/build_macos.yml
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
name: macOS
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- macos-latest
|
||||
rust:
|
||||
- stable
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- run: git config --global core.autocrlf false
|
||||
- uses: actions/checkout@v2
|
||||
- name: Build
|
||||
run: cargo build --all --locked --verbose
|
||||
22
.github/workflows/build_windows.yml
vendored
Normal file
22
.github/workflows/build_windows.yml
vendored
Normal file
@@ -0,0 +1,22 @@
|
||||
name: Windows
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- windows-latest
|
||||
rust:
|
||||
- stable
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- run: git config --global core.autocrlf false
|
||||
- uses: actions/checkout@v2
|
||||
- name: Build
|
||||
run: cargo build --all --locked --verbose
|
||||
53
.github/workflows/cd.yml
vendored
Normal file
53
.github/workflows/cd.yml
vendored
Normal file
@@ -0,0 +1,53 @@
|
||||
# CD GitHub Actions workflow for Monolith
|
||||
|
||||
name: CD
|
||||
|
||||
on:
|
||||
release:
|
||||
types:
|
||||
- created
|
||||
|
||||
jobs:
|
||||
|
||||
windows:
|
||||
runs-on: windows-2019
|
||||
steps:
|
||||
- run: git config --global core.autocrlf false
|
||||
- name: Checkout the repository
|
||||
uses: actions/checkout@v2
|
||||
- name: Build and install the executable
|
||||
run: cargo build --release
|
||||
- uses: Shopify/upload-to-release@1.0.0
|
||||
with:
|
||||
name: monolith.exe
|
||||
path: target\release\monolith.exe
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
gnu_linux_armhf:
|
||||
runs-on: ubuntu-18.04
|
||||
steps:
|
||||
- name: Checkout the repository
|
||||
uses: actions/checkout@v2
|
||||
- name: Prepare cross-platform environment
|
||||
run: |
|
||||
sudo mkdir -p /cross-build-arm
|
||||
sudo echo "deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic main" >> /etc/apt/sources.list
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y gcc-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross
|
||||
sudo apt-get download libssl1.1:armhf libssl-dev:armhf
|
||||
sudo dpkg -x libssl1.1*.deb /cross-build-arm
|
||||
sudo dpkg -x libssl-dev*.deb /cross-build-arm
|
||||
rustup target add arm-unknown-linux-gnueabihf
|
||||
echo "::set-env name=C_INCLUDE_PATH::/cross-build-arm/usr/include"
|
||||
echo "::set-env name=OPENSSL_INCLUDE_DIR::/cross-build-arm/usr/include/arm-linux-gnueabihf"
|
||||
echo "::set-env name=OPENSSL_LIB_DIR::/cross-build-arm/usr/lib/arm-linux-gnueabihf"
|
||||
echo "::set-env name=PKG_CONFIG_ALLOW_CROSS::1"
|
||||
echo "::set-env name=RUSTFLAGS::-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build-arm/usr/lib/arm-linux-gnueabihf -L/cross-build-arm/lib/arm-linux-gnueabihf"
|
||||
- name: Build the executable
|
||||
run: cargo build --release --target=arm-unknown-linux-gnueabihf
|
||||
- name: Attach artifact to the release
|
||||
uses: Shopify/upload-to-release@1.0.0
|
||||
with:
|
||||
name: monolith-gnu-linux-armhf
|
||||
path: target/arm-unknown-linux-gnueabihf/release/monolith
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
32
.github/workflows/ci.yml
vendored
Normal file
32
.github/workflows/ci.yml
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
build_and_test:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
os:
|
||||
- ubuntu-latest
|
||||
- macos-latest
|
||||
- windows-latest
|
||||
rust:
|
||||
- stable
|
||||
- beta
|
||||
- nightly
|
||||
runs-on: ${{ matrix.os }}
|
||||
|
||||
steps:
|
||||
- run: git config --global core.autocrlf false
|
||||
- uses: actions/checkout@v2
|
||||
- name: Build
|
||||
run: cargo build --all --locked --verbose
|
||||
- name: Run tests
|
||||
run: cargo test --all --locked --verbose
|
||||
- name: Check code formatting
|
||||
run: |
|
||||
rustup component add rustfmt
|
||||
cargo fmt --all -- --check
|
||||
37
.travis.yml
37
.travis.yml
@@ -1,37 +0,0 @@
|
||||
language: rust
|
||||
cache: cargo
|
||||
|
||||
os:
|
||||
- linux
|
||||
- osx
|
||||
- windows
|
||||
|
||||
rust:
|
||||
- stable
|
||||
- beta
|
||||
- nightly
|
||||
|
||||
git:
|
||||
autocrlf: false # don't mangle LF into CRLF on windows
|
||||
|
||||
before_script:
|
||||
- rustup component add rustfmt
|
||||
|
||||
script:
|
||||
- cargo build --all --locked --verbose
|
||||
- cargo test --all --locked --verbose
|
||||
- |
|
||||
if [[ "$TRAVIS_RUST_VERSION" == "stable" ]]; then
|
||||
cargo fmt --all -- --check
|
||||
fi
|
||||
|
||||
after_success: |
|
||||
if [ "${TRAVIS_OS_NAME}" == linux ] && [ "${TRAVIS_RUST_VERSION}" == stable ]; then
|
||||
docker build -t monolith .
|
||||
docker run monolith monolith -V
|
||||
fi
|
||||
|
||||
jobs:
|
||||
allow_failures:
|
||||
- rust: nightly
|
||||
fast_finish: true
|
||||
93
Cargo.lock
generated
93
Cargo.lock
generated
@@ -61,16 +61,50 @@ name = "base64"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "base64"
|
||||
version = "0.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-padding"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bumpalo"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "byte-tools"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "0.5.3"
|
||||
@@ -168,6 +202,14 @@ name = "difference"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "doc-comment"
|
||||
version = "0.3.1"
|
||||
@@ -205,6 +247,11 @@ dependencies = [
|
||||
"serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fake-simd"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.0.13"
|
||||
@@ -303,6 +350,14 @@ dependencies = [
|
||||
"slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.1.13"
|
||||
@@ -568,15 +623,17 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "monolith"
|
||||
version = "2.2.1"
|
||||
version = "2.2.5"
|
||||
dependencies = [
|
||||
"assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"cssparser 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"html5ever 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"reqwest 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
@@ -630,6 +687,11 @@ dependencies = [
|
||||
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "openssl"
|
||||
version = "0.10.26"
|
||||
@@ -1096,6 +1158,17 @@ dependencies = [
|
||||
"url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
dependencies = [
|
||||
"block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
"opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "0.2.3"
|
||||
@@ -1261,6 +1334,11 @@ name = "try-lock"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.6.0"
|
||||
@@ -1501,8 +1579,13 @@ dependencies = [
|
||||
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
|
||||
"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
|
||||
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||
"checksum base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d5ca2cd0adc3f48f9e9ea5a6bbdf9ccc0bfade884847e484d452414c7ccffb3"
|
||||
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
|
||||
"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
|
||||
"checksum bumpalo 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fe2567a8d8a3aedb4e39aa39e186d5673acfd56393c6ac83b2bc5bd82f4369c"
|
||||
"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
|
||||
"checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
|
||||
"checksum bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10004c15deb332055f7a4a208190aed362cf9a7c2f6ab70a305fba50e1105f38"
|
||||
"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
|
||||
"checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8"
|
||||
@@ -1515,11 +1598,13 @@ dependencies = [
|
||||
"checksum cssparser 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)" = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
|
||||
"checksum cssparser-macros 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
|
||||
"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
|
||||
"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
|
||||
"checksum doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "923dea538cea0aa3025e8685b20d6ee21ef99c4f77e954a30febbaac5ec73a97"
|
||||
"checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
|
||||
"checksum dtoa-short 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "59020b8513b76630c49d918c33db9f4c91638e7d3404a28084083b87e33f76f2"
|
||||
"checksum encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)" = "87240518927716f79692c2ed85bfe6e98196d18c6401ec75355760233a7e12e9"
|
||||
"checksum escargot 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "74cf96bec282dcdb07099f7e31d9fed323bca9435a09aba7b6d99b7617bca96d"
|
||||
"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
|
||||
"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
|
||||
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
|
||||
"checksum foreign-types 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
|
||||
@@ -1534,6 +1619,7 @@ dependencies = [
|
||||
"checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16"
|
||||
"checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9"
|
||||
"checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76"
|
||||
"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
|
||||
"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
|
||||
"checksum h2 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9433d71e471c1736fd5a61b671fc0b148d7a2992f666c958d03cd8feb3b88d1"
|
||||
"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
|
||||
@@ -1567,6 +1653,7 @@ dependencies = [
|
||||
"checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30"
|
||||
"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
|
||||
"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
|
||||
"checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
|
||||
"checksum openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)" = "3a3cc5799d98e1088141b8e01ff760112bbd9f19d850c124500566ca6901a585"
|
||||
"checksum openssl-probe 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de"
|
||||
"checksum openssl-sys 0.9.53 (registry+https://github.com/rust-lang/crates.io-index)" = "465d16ae7fc0e313318f7de5cecf57b2fbe7511fd213978b457e1c96ff46736f"
|
||||
@@ -1619,6 +1706,7 @@ dependencies = [
|
||||
"checksum serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c6faef9a2e64b0064f48570289b4bf8823b7581f1d6157c1b52152306651d0"
|
||||
"checksum serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "1a3351dcbc1f067e2c92ab7c3c1f288ad1a4cffc470b5aaddb4c2e0a3ae80043"
|
||||
"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"
|
||||
"checksum sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0"
|
||||
"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
|
||||
"checksum siphasher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e88f89a550c01e4cd809f3df4f52dc9e939f3273a2017eabd5c6d12fd98bb23"
|
||||
"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
|
||||
@@ -1639,6 +1727,7 @@ dependencies = [
|
||||
"checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
|
||||
"checksum treeline 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41"
|
||||
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
|
||||
"checksum typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
|
||||
"checksum unicase 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6"
|
||||
"checksum unicode-bidi 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5"
|
||||
"checksum unicode-normalization 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "b561e267b2326bb4cebfc0ef9e68355c7abe6c6f522aeac2f5bf95d56c59bdcf"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "monolith"
|
||||
version = "2.2.1"
|
||||
version = "2.2.5"
|
||||
edition = "2018"
|
||||
authors = [
|
||||
"Sunshine <sunshine@uberspace.net>",
|
||||
@@ -12,10 +12,12 @@ authors = [
|
||||
description = "CLI tool for saving web pages as a single HTML file"
|
||||
|
||||
[dependencies]
|
||||
base64 = "0.11.0"
|
||||
base64 = "0.12.0"
|
||||
clap = "2.33.0"
|
||||
cssparser = "0.27.2"
|
||||
html5ever = "0.24.1"
|
||||
sha2 = "0.8.1" # Used in calculating checksums during integrity checks
|
||||
time = "0.1.42" # Used to render comments indicating the time the page was saved
|
||||
url = "2.1.1"
|
||||
|
||||
[dependencies.reqwest]
|
||||
|
||||
18
Makefile
18
Makefile
@@ -1,21 +1,25 @@
|
||||
#!/usr/bin/make -f
|
||||
# Makefile for monolith
|
||||
|
||||
all: test
|
||||
all: build
|
||||
.PHONY: all
|
||||
|
||||
build:
|
||||
@cargo build --locked
|
||||
.PHONY: build
|
||||
|
||||
install:
|
||||
@cargo install --force --locked --path .
|
||||
.PHONY: install
|
||||
|
||||
test: build
|
||||
@cargo test --locked
|
||||
@cargo fmt --all -- --check
|
||||
.PHONY: test
|
||||
.PHONY: test_code_formatting
|
||||
|
||||
lint:
|
||||
@cargo fmt --all --
|
||||
.PHONY: lint
|
||||
|
||||
install:
|
||||
@cargo install --force --locked --path .
|
||||
.PHONY: install
|
||||
|
||||
uninstall:
|
||||
@cargo uninstall
|
||||
.PHONY: uninstall
|
||||
|
||||
29
README.md
29
README.md
@@ -1,5 +1,6 @@
|
||||
[](https://travis-ci.org/Y2Z/monolith)
|
||||
[](https://ci.appveyor.com/project/snshn/monolith/branch/master)
|
||||
[](https://github.com/Y2Z/monolith/actions?query=workflow%3AGNU%2FLinux)
|
||||
[](https://github.com/Y2Z/monolith/actions?query=workflow%3AmacOS)
|
||||
[](https://github.com/Y2Z/monolith/actions?query=workflow%3AWindows)
|
||||
|
||||
```
|
||||
___ ___________ __________ ___________________ ___
|
||||
@@ -11,9 +12,9 @@
|
||||
|___| |__________| \____________________| |___| |___| |___|
|
||||
```
|
||||
|
||||
A data hoarder's dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
|
||||
A data hoarder’s dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
|
||||
|
||||
Unlike the conventional "Save page as", `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.
|
||||
Unlike the conventional “Save page as”, `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.
|
||||
|
||||
If compared to saving websites with `wget -mpk`, this tool embeds all assets as data URLs and therefore lets browsers render the saved page exactly the way it was on the Internet, even when no network connection is available.
|
||||
|
||||
@@ -21,18 +22,21 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
|
||||
|
||||
## Installation
|
||||
|
||||
#### From source
|
||||
$ git clone https://github.com/Y2Z/monolith.git
|
||||
$ cd monolith
|
||||
$ make install
|
||||
|
||||
#### With Homebrew (on macOS and GNU/Linux)
|
||||
#### Via Homebrew (on macOS and GNU/Linux)
|
||||
$ brew install monolith
|
||||
|
||||
#### Using Snapcraft (on GNU/Linux)
|
||||
$ snap install monolith
|
||||
|
||||
#### Via Docker
|
||||
#### From source
|
||||
|
||||
Dependency: `libssl-dev`
|
||||
|
||||
$ git clone https://github.com/Y2Z/monolith.git
|
||||
$ cd monolith
|
||||
$ make install
|
||||
|
||||
#### With Docker
|
||||
The guide can be found [here](docs/containers.md)
|
||||
|
||||
---------------------------------------------------
|
||||
@@ -45,6 +49,7 @@ The guide can be found [here](docs/containers.md)
|
||||
## Options
|
||||
- `-c`: Ignore styles
|
||||
- `-f`: Exclude frames and iframes
|
||||
- `-F`: Omit web fonts
|
||||
- `-i`: Remove images
|
||||
- `-I`: Isolate the document
|
||||
- `-j`: Exclude JavaScript
|
||||
@@ -79,4 +84,4 @@ The Unlicense
|
||||
---------------------------------------------------
|
||||
|
||||
<!-- Microtext -->
|
||||
<sub>Keep in mind that `monolith` is not aware of your browser's session</sub>
|
||||
<sub>Keep in mind that `monolith` is not aware of your browser’s session</sub>
|
||||
|
||||
19
docs/arch/0002-noscript-nodes.md
Normal file
19
docs/arch/0002-noscript-nodes.md
Normal file
@@ -0,0 +1,19 @@
|
||||
# 2. NOSCRIPT nodes
|
||||
|
||||
Date: 2020-04-16
|
||||
|
||||
## Status
|
||||
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
|
||||
HTML pages sometimes contain NOSCRIPT nodes, which reveal their contents only in case when JavaScript is not available. Most of the time they contain hidden messages that inform about certain JavaScript-dependent features not being operational, however sometimes can also feature media assets or even iframes.
|
||||
|
||||
## Decision
|
||||
|
||||
When the document is being saved with or without JavaScript, each NOSCRIPT node should be preserved while its children need to be processed exactly the same way as the rest of the document. This approach will ensure that even hidden remote assets are embedded — since those hidden elements may have to be displayed later in a browser that has JavaScript turned off. An option should be available to "unwrap" all NOSCRIPT nodes in order to make their contents always visible in the document, complimenting the "disable JS" function of the program.
|
||||
|
||||
## Consequences
|
||||
|
||||
Saved documents will have contents of all NOSCRIPT nodes processed as if they are part of the document's DOM, therefore properly display images encapsulated within NOSCRIPT nodes when being viewed in browsers that have JavaScript turned off (or have no JavaScript support in the first place). The new option to "unwrap" NOSCRIPT elements will help the user ensure that the resulting document always represents what the original web page looked like in a browser that had JavaScript turned off.
|
||||
@@ -21,5 +21,5 @@ saved by monolith, if needed.
|
||||
|
||||
## Consequences
|
||||
|
||||
Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only a minimal
|
||||
Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only minimal
|
||||
amount of modifications to the original web page — whatever is needed to provide security or exclude unwanted asset types.
|
||||
|
||||
18
src/args.rs
18
src/args.rs
@@ -2,8 +2,9 @@ use clap::{App, Arg};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct AppArgs {
|
||||
pub url_target: String,
|
||||
pub target: String,
|
||||
pub no_css: bool,
|
||||
pub no_fonts: bool,
|
||||
pub no_frames: bool,
|
||||
pub no_images: bool,
|
||||
pub no_js: bool,
|
||||
@@ -13,6 +14,7 @@ pub struct AppArgs {
|
||||
pub silent: bool,
|
||||
pub timeout: u64,
|
||||
pub user_agent: String,
|
||||
pub no_metadata: bool,
|
||||
}
|
||||
|
||||
const DEFAULT_NETWORK_TIMEOUT: u64 = 120;
|
||||
@@ -26,19 +28,21 @@ impl AppArgs {
|
||||
.author(crate_authors!("\n"))
|
||||
.about(crate_description!())
|
||||
.arg(
|
||||
Arg::with_name("url")
|
||||
Arg::with_name("target")
|
||||
.required(true)
|
||||
.takes_value(true)
|
||||
.index(1)
|
||||
.help("URL to download"),
|
||||
.help("URL or file path"),
|
||||
)
|
||||
// .args_from_usage("-a, --include-audio 'Removes audio sources'")
|
||||
.args_from_usage("-c, --no-css 'Removes CSS'")
|
||||
.args_from_usage("-f, --no-frames 'Removes frames and iframes'")
|
||||
.args_from_usage("-F, --no-fonts 'Removes fonts'")
|
||||
.args_from_usage("-i, --no-images 'Removes images'")
|
||||
.args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
|
||||
.args_from_usage("-j, --no-js 'Removes JavaScript'")
|
||||
.args_from_usage("-k, --insecure 'Allows invalid X.509 (TLS) certificates'")
|
||||
.args_from_usage("-M, --no-metadata 'Excludes metadata information from the document'")
|
||||
.args_from_usage("-o, --output=[document.html] 'Writes output to <file>'")
|
||||
.args_from_usage("-s, --silent 'Suppresses verbosity'")
|
||||
.args_from_usage("-t, --timeout=[60] 'Adjusts network request timeout'")
|
||||
@@ -47,15 +51,17 @@ impl AppArgs {
|
||||
.get_matches();
|
||||
let mut app_args = AppArgs::default();
|
||||
// Process the command
|
||||
app_args.url_target = app
|
||||
.value_of("url")
|
||||
.expect("please set target url")
|
||||
app_args.target = app
|
||||
.value_of("target")
|
||||
.expect("please set target")
|
||||
.to_string();
|
||||
app_args.no_css = app.is_present("no-css");
|
||||
app_args.no_fonts = app.is_present("no-fonts");
|
||||
app_args.no_frames = app.is_present("no-frames");
|
||||
app_args.no_images = app.is_present("no-images");
|
||||
app_args.no_js = app.is_present("no-js");
|
||||
app_args.insecure = app.is_present("insecure");
|
||||
app_args.no_metadata = app.is_present("no-metadata");
|
||||
app_args.isolate = app.is_present("isolate");
|
||||
app_args.silent = app.is_present("silent");
|
||||
app_args.timeout = app
|
||||
|
||||
225
src/css.rs
225
src/css.rs
@@ -2,12 +2,12 @@ use cssparser::{ParseError, Parser, ParserInput, SourcePosition, Token};
|
||||
use reqwest::blocking::Client;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::utils::{data_to_data_url, decode_url, get_url_fragment, resolve_url, retrieve_asset};
|
||||
use crate::utils::{data_to_data_url, get_url_fragment, is_http_url, resolve_url, retrieve_asset};
|
||||
|
||||
const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
|
||||
// Universal
|
||||
"background",
|
||||
"background-image",
|
||||
"border",
|
||||
"border-image",
|
||||
"border-image-source",
|
||||
"content",
|
||||
@@ -16,10 +16,15 @@ const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
|
||||
"list-style-image",
|
||||
"mask",
|
||||
"mask-image",
|
||||
// Specific to @counter-style
|
||||
"additive-symbols",
|
||||
"negative",
|
||||
"pad",
|
||||
"prefix",
|
||||
"suffix",
|
||||
"symbols",
|
||||
];
|
||||
|
||||
const TRANSPARENT_PIXEL: &str = "data:image/png;base64,\
|
||||
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
|
||||
const CSS_SPECIAL_CHARS: &str = "~!@$%^&*()+=,./'\";:?><[]{}|`#";
|
||||
|
||||
pub fn is_image_url_prop(prop_name: &str) -> bool {
|
||||
CSS_PROPS_WITH_IMAGE_URLS
|
||||
@@ -36,14 +41,27 @@ pub fn enquote(input: String, double: bool) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn escape(value: &str) -> String {
|
||||
let mut res = str!(&value);
|
||||
|
||||
res = res.replace("\\", "\\\\");
|
||||
|
||||
for c in CSS_SPECIAL_CHARS.chars() {
|
||||
res = res.replace(c, format!("\\{}", c).as_str());
|
||||
}
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
pub fn process_css<'a>(
|
||||
cache: &mut HashMap<String, String>,
|
||||
cache: &mut HashMap<String, Vec<u8>>,
|
||||
client: &Client,
|
||||
parent_url: &str,
|
||||
parser: &mut Parser,
|
||||
rule_name: &str,
|
||||
prop_name: &str,
|
||||
func_name: &str,
|
||||
opt_no_fonts: bool,
|
||||
opt_no_images: bool,
|
||||
opt_silent: bool,
|
||||
) -> Result<String, ParseError<'a, String>> {
|
||||
@@ -72,6 +90,10 @@ pub fn process_css<'a>(
|
||||
Token::Colon => result.push_str(":"),
|
||||
Token::Comma => result.push_str(","),
|
||||
Token::ParenthesisBlock | Token::SquareBracketBlock | Token::CurlyBracketBlock => {
|
||||
if opt_no_fonts && curr_rule == "font-face" {
|
||||
continue;
|
||||
}
|
||||
|
||||
let closure: &str;
|
||||
if token == &Token::ParenthesisBlock {
|
||||
result.push_str("(");
|
||||
@@ -94,6 +116,7 @@ pub fn process_css<'a>(
|
||||
rule_name,
|
||||
curr_prop.as_str(),
|
||||
func_name,
|
||||
opt_no_fonts,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
)
|
||||
@@ -116,12 +139,18 @@ pub fn process_css<'a>(
|
||||
Token::WhiteSpace(ref value) => {
|
||||
result.push_str(value);
|
||||
}
|
||||
// div...
|
||||
Token::Ident(ref value) => {
|
||||
curr_rule = str!();
|
||||
curr_prop = str!(value);
|
||||
result.push_str(value);
|
||||
result.push_str(&escape(value));
|
||||
}
|
||||
// @import, @font-face, @charset, @media...
|
||||
Token::AtKeyword(ref value) => {
|
||||
curr_rule = str!(value);
|
||||
if opt_no_fonts && curr_rule == "font-face" {
|
||||
continue;
|
||||
}
|
||||
result.push_str("@");
|
||||
result.push_str(value);
|
||||
}
|
||||
@@ -130,53 +159,49 @@ pub fn process_css<'a>(
|
||||
result.push_str(value);
|
||||
}
|
||||
Token::QuotedString(ref value) => {
|
||||
let is_import: bool = curr_rule == "import";
|
||||
if is_import {
|
||||
if curr_rule == "import" {
|
||||
// Reset current at-rule value
|
||||
curr_rule = str!();
|
||||
}
|
||||
|
||||
if is_import {
|
||||
// Skip empty import values
|
||||
if value.len() < 1 {
|
||||
result.push_str("''");
|
||||
continue;
|
||||
}
|
||||
|
||||
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
|
||||
let url_fragment = get_url_fragment(full_url.clone());
|
||||
let full_url_decoded = decode_url(full_url);
|
||||
let (css, final_url) = retrieve_asset(
|
||||
cache,
|
||||
client,
|
||||
&parent_url,
|
||||
&full_url_decoded,
|
||||
false,
|
||||
"",
|
||||
opt_silent,
|
||||
)
|
||||
.unwrap_or_default();
|
||||
|
||||
result.push_str(
|
||||
enquote(
|
||||
data_to_data_url(
|
||||
"text/css",
|
||||
embed_css(
|
||||
cache,
|
||||
client,
|
||||
final_url.as_str(),
|
||||
&css,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
let import_full_url = resolve_url(&parent_url, value).unwrap_or_default();
|
||||
let import_url_fragment = get_url_fragment(import_full_url.clone());
|
||||
match retrieve_asset(cache, client, &parent_url, &import_full_url, opt_silent) {
|
||||
Ok((import_contents, import_final_url, _import_media_type)) => {
|
||||
result.push_str(
|
||||
enquote(
|
||||
data_to_data_url(
|
||||
"text/css",
|
||||
embed_css(
|
||||
cache,
|
||||
client,
|
||||
&import_final_url,
|
||||
&String::from_utf8_lossy(&import_contents),
|
||||
opt_no_fonts,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
)
|
||||
.as_bytes(),
|
||||
&import_final_url,
|
||||
&import_url_fragment,
|
||||
),
|
||||
false,
|
||||
)
|
||||
.as_bytes(),
|
||||
&final_url,
|
||||
url_fragment.as_str(),
|
||||
),
|
||||
false,
|
||||
)
|
||||
.as_str(),
|
||||
);
|
||||
.as_str(),
|
||||
);
|
||||
}
|
||||
Err(_) => {
|
||||
// Keep remote reference if unable to retrieve the asset
|
||||
if is_http_url(import_full_url.clone()) {
|
||||
result.push_str(enquote(import_full_url, false).as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if func_name == "url" {
|
||||
// Skip empty url()'s
|
||||
@@ -185,20 +210,33 @@ pub fn process_css<'a>(
|
||||
}
|
||||
|
||||
if opt_no_images && is_image_url_prop(curr_prop.as_str()) {
|
||||
result.push_str(enquote(str!(TRANSPARENT_PIXEL), false).as_str());
|
||||
result.push_str(enquote(str!(empty_image!()), false).as_str());
|
||||
} else {
|
||||
let resolved_url = resolve_url(&parent_url, value).unwrap_or_default();
|
||||
let (data_url, _final_url) = retrieve_asset(
|
||||
let url_fragment = get_url_fragment(resolved_url.clone());
|
||||
match retrieve_asset(
|
||||
cache,
|
||||
client,
|
||||
&parent_url,
|
||||
&resolved_url,
|
||||
true,
|
||||
"",
|
||||
opt_silent,
|
||||
)
|
||||
.unwrap_or_default();
|
||||
result.push_str(enquote(data_url, false).as_str());
|
||||
) {
|
||||
Ok((data, final_url, media_type)) => {
|
||||
let data_url = data_to_data_url(
|
||||
&media_type,
|
||||
&data,
|
||||
&final_url,
|
||||
&url_fragment,
|
||||
);
|
||||
result.push_str(enquote(data_url, false).as_str());
|
||||
}
|
||||
Err(_) => {
|
||||
// Keep remote reference if unable to retrieve the asset
|
||||
if is_http_url(resolved_url.clone()) {
|
||||
result.push_str(enquote(resolved_url, false).as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
result.push_str(enquote(str!(value), false).as_str());
|
||||
@@ -220,26 +258,33 @@ pub fn process_css<'a>(
|
||||
ref unit_value,
|
||||
..
|
||||
} => {
|
||||
if *has_sign {
|
||||
result.push_str("-");
|
||||
if *has_sign && *unit_value >= 0. {
|
||||
result.push_str("+");
|
||||
}
|
||||
result.push_str(str!(unit_value * 100.).as_str());
|
||||
result.push_str("%");
|
||||
}
|
||||
Token::Dimension {
|
||||
ref has_sign,
|
||||
ref value,
|
||||
ref unit,
|
||||
..
|
||||
} => {
|
||||
if *has_sign && *value >= 0. {
|
||||
result.push_str("+");
|
||||
}
|
||||
result.push_str(str!(value).as_str());
|
||||
result.push_str(str!(unit).as_str());
|
||||
}
|
||||
// #selector, #id...
|
||||
Token::IDHash(ref value) => {
|
||||
curr_rule = str!();
|
||||
result.push_str("#");
|
||||
result.push_str(value);
|
||||
result.push_str(&escape(value));
|
||||
}
|
||||
Token::UnquotedUrl(ref value) => {
|
||||
let is_import: bool = curr_rule == "import";
|
||||
|
||||
if is_import {
|
||||
// Reset current at-rule value
|
||||
curr_rule = str!();
|
||||
@@ -260,54 +305,51 @@ pub fn process_css<'a>(
|
||||
if is_import {
|
||||
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
|
||||
let url_fragment = get_url_fragment(full_url.clone());
|
||||
let full_url_decoded = decode_url(full_url);
|
||||
let (css, final_url) = retrieve_asset(
|
||||
cache,
|
||||
client,
|
||||
&parent_url,
|
||||
&full_url_decoded,
|
||||
false,
|
||||
"",
|
||||
opt_silent,
|
||||
)
|
||||
.unwrap_or_default();
|
||||
|
||||
result.push_str(
|
||||
enquote(
|
||||
data_to_data_url(
|
||||
match retrieve_asset(cache, client, &parent_url, &full_url, opt_silent) {
|
||||
Ok((css, final_url, _media_type)) => {
|
||||
let data_url = data_to_data_url(
|
||||
"text/css",
|
||||
embed_css(
|
||||
cache,
|
||||
client,
|
||||
final_url.as_str(),
|
||||
&css,
|
||||
&final_url,
|
||||
&String::from_utf8_lossy(&css),
|
||||
opt_no_fonts,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
)
|
||||
.as_bytes(),
|
||||
&final_url,
|
||||
url_fragment.as_str(),
|
||||
),
|
||||
false,
|
||||
)
|
||||
.as_str(),
|
||||
);
|
||||
&url_fragment,
|
||||
);
|
||||
result.push_str(enquote(data_url, false).as_str());
|
||||
}
|
||||
Err(_) => {
|
||||
// Keep remote reference if unable to retrieve the asset
|
||||
if is_http_url(full_url.clone()) {
|
||||
result.push_str(enquote(full_url, false).as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if opt_no_images && is_image_url_prop(curr_prop.as_str()) {
|
||||
result.push_str(enquote(str!(TRANSPARENT_PIXEL), false).as_str());
|
||||
result.push_str(enquote(str!(empty_image!()), false).as_str());
|
||||
} else {
|
||||
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
|
||||
let (data_url, _final_url) = retrieve_asset(
|
||||
cache,
|
||||
client,
|
||||
&parent_url,
|
||||
&full_url,
|
||||
true,
|
||||
"",
|
||||
opt_silent,
|
||||
)
|
||||
.unwrap_or_default();
|
||||
result.push_str(enquote(data_url, false).as_str());
|
||||
let url_fragment = get_url_fragment(full_url.clone());
|
||||
match retrieve_asset(cache, client, &parent_url, &full_url, opt_silent) {
|
||||
Ok((data, final_url, media_type)) => {
|
||||
let data_url =
|
||||
data_to_data_url(&media_type, &data, &final_url, &url_fragment);
|
||||
result.push_str(enquote(data_url, false).as_str());
|
||||
}
|
||||
Err(_) => {
|
||||
// Keep remote reference if unable to retrieve the asset
|
||||
if is_http_url(full_url.clone()) {
|
||||
result.push_str(enquote(full_url, false).as_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
result.push_str(")");
|
||||
@@ -328,6 +370,7 @@ pub fn process_css<'a>(
|
||||
curr_rule.as_str(),
|
||||
curr_prop.as_str(),
|
||||
function_name,
|
||||
opt_no_fonts,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
)
|
||||
@@ -345,10 +388,11 @@ pub fn process_css<'a>(
|
||||
}
|
||||
|
||||
pub fn embed_css(
|
||||
cache: &mut HashMap<String, String>,
|
||||
cache: &mut HashMap<String, Vec<u8>>,
|
||||
client: &Client,
|
||||
parent_url: &str,
|
||||
css: &str,
|
||||
opt_no_fonts: bool,
|
||||
opt_no_images: bool,
|
||||
opt_silent: bool,
|
||||
) -> String {
|
||||
@@ -363,6 +407,7 @@ pub fn embed_css(
|
||||
"",
|
||||
"",
|
||||
"",
|
||||
opt_no_fonts,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
)
|
||||
|
||||
794
src/html.rs
794
src/html.rs
File diff suppressed because it is too large
Load Diff
@@ -7,3 +7,11 @@ macro_rules! str {
|
||||
ToString::to_string(&$val)
|
||||
};
|
||||
}
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! empty_image {
|
||||
() => {
|
||||
"data:image/png;base64,\
|
||||
iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII="
|
||||
};
|
||||
}
|
||||
|
||||
68
src/main.rs
68
src/main.rs
@@ -1,7 +1,8 @@
|
||||
use monolith::html::{html_to_dom, stringify_document, walk_and_embed_assets};
|
||||
use monolith::utils::{data_url_to_text, is_data_url, is_file_url, is_http_url, retrieve_asset};
|
||||
use monolith::utils::{data_url_to_data, is_data_url, is_file_url, is_http_url, retrieve_asset};
|
||||
use reqwest::blocking::Client;
|
||||
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
|
||||
use reqwest::Url;
|
||||
use std::collections::HashMap;
|
||||
use std::env;
|
||||
use std::fs;
|
||||
@@ -47,7 +48,7 @@ impl Output {
|
||||
|
||||
fn main() {
|
||||
let app_args = AppArgs::get();
|
||||
let original_target: &str = &app_args.url_target;
|
||||
let original_target: &str = &app_args.target;
|
||||
let target_url: &str;
|
||||
let base_url;
|
||||
let dom;
|
||||
@@ -109,43 +110,44 @@ fn main() {
|
||||
|
||||
// Retrieve root document
|
||||
if is_file_url(target_url) || is_http_url(target_url) {
|
||||
let (data, final_url) = retrieve_asset(
|
||||
&mut cache,
|
||||
&client,
|
||||
target_url,
|
||||
target_url,
|
||||
false,
|
||||
"",
|
||||
app_args.silent,
|
||||
)
|
||||
.expect("Could not retrieve target document");
|
||||
base_url = final_url;
|
||||
dom = html_to_dom(&data);
|
||||
match retrieve_asset(&mut cache, &client, target_url, target_url, app_args.silent) {
|
||||
Ok((data, final_url, _media_type)) => {
|
||||
base_url = final_url;
|
||||
dom = html_to_dom(&String::from_utf8_lossy(&data));
|
||||
}
|
||||
Err(_) => {
|
||||
eprintln!("Could not retrieve target document");
|
||||
process::exit(1);
|
||||
}
|
||||
}
|
||||
} else if is_data_url(target_url) {
|
||||
let text: String = data_url_to_text(target_url);
|
||||
if text.len() == 0 {
|
||||
eprintln!("Unsupported data URL input");
|
||||
let (media_type, data): (String, Vec<u8>) = data_url_to_data(target_url);
|
||||
if !media_type.eq_ignore_ascii_case("text/html") {
|
||||
eprintln!("Unsupported data URL media type");
|
||||
process::exit(1);
|
||||
}
|
||||
base_url = str!(target_url);
|
||||
dom = html_to_dom(&text);
|
||||
dom = html_to_dom(&String::from_utf8_lossy(&data));
|
||||
} else {
|
||||
process::exit(1);
|
||||
}
|
||||
|
||||
let time_saved = time::now_utc();
|
||||
|
||||
walk_and_embed_assets(
|
||||
&mut cache,
|
||||
&client,
|
||||
&base_url,
|
||||
&dom.document,
|
||||
app_args.no_css,
|
||||
app_args.no_fonts,
|
||||
app_args.no_frames,
|
||||
app_args.no_js,
|
||||
app_args.no_images,
|
||||
app_args.silent,
|
||||
app_args.no_frames,
|
||||
);
|
||||
|
||||
let html: String = stringify_document(
|
||||
let mut html: String = stringify_document(
|
||||
&dom.document,
|
||||
app_args.no_css,
|
||||
app_args.no_frames,
|
||||
@@ -154,6 +156,32 @@ fn main() {
|
||||
app_args.isolate,
|
||||
);
|
||||
|
||||
if !app_args.no_metadata {
|
||||
// Safe to unwrap (we just put this through an HTTP request)
|
||||
let mut clean_url = Url::parse(&base_url).unwrap();
|
||||
clean_url.set_fragment(None);
|
||||
// Don't include credentials
|
||||
clean_url.set_username("").unwrap();
|
||||
clean_url.set_password(None).unwrap();
|
||||
let metadata_comment = if is_http_url(&base_url) {
|
||||
format!(
|
||||
"<!-- Saved from {} at {} using {} v{} -->\n",
|
||||
&clean_url,
|
||||
time_saved.rfc3339(),
|
||||
env!("CARGO_PKG_NAME"),
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
)
|
||||
} else {
|
||||
format!(
|
||||
"<!-- Saved from local source at {} using {} v{} -->\n",
|
||||
time_saved.rfc3339(),
|
||||
env!("CARGO_PKG_NAME"),
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
)
|
||||
};
|
||||
html.insert_str(0, &metadata_comment);
|
||||
}
|
||||
|
||||
output
|
||||
.writeln_str(&html)
|
||||
.expect("Could not write HTML output");
|
||||
|
||||
@@ -62,7 +62,7 @@ fn passing_bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// STDERR should contain error description
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stderr).unwrap(),
|
||||
"Unsupported data URL input\n"
|
||||
"Unsupported data URL media type\n"
|
||||
);
|
||||
|
||||
// The exit code should be 1
|
||||
@@ -75,6 +75,7 @@ fn passing_bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
fn passing_isolate_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-I")
|
||||
.arg("data:text/html,Hello%2C%20World!")
|
||||
.output()
|
||||
@@ -101,6 +102,7 @@ fn passing_isolate_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
fn passing_remove_css_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-c")
|
||||
.arg("data:text/html,<style>body{background-color:pink}</style>Hello")
|
||||
.output()
|
||||
@@ -128,6 +130,7 @@ fn passing_remove_css_from_data_url() -> Result<(), Box<dyn std::error::Error>>
|
||||
fn passing_remove_frames_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-f")
|
||||
.arg("data:text/html,<iframe src=\"https://google.com\"></iframe>Hi")
|
||||
.output()
|
||||
@@ -154,6 +157,7 @@ fn passing_remove_frames_from_data_url() -> Result<(), Box<dyn std::error::Error
|
||||
fn passing_remove_images_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-i")
|
||||
.arg("data:text/html,<img src=\"https://google.com\"/>Hi")
|
||||
.output()
|
||||
@@ -162,15 +166,18 @@ fn passing_remove_images_from_data_url() -> Result<(), Box<dyn std::error::Error
|
||||
// STDOUT should contain HTML with no images
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stdout).unwrap(),
|
||||
"<html>\
|
||||
format!(
|
||||
"<html>\
|
||||
<head>\
|
||||
<meta http-equiv=\"Content-Security-Policy\" content=\"img-src data:;\"></meta>\
|
||||
</head>\
|
||||
<body>\
|
||||
<img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\
|
||||
<img src=\"{empty_image}\">\
|
||||
Hi\
|
||||
</body>\
|
||||
</html>\n"
|
||||
</html>\n",
|
||||
empty_image = empty_image!()
|
||||
)
|
||||
);
|
||||
|
||||
// STDERR should be empty
|
||||
@@ -186,6 +193,7 @@ Hi\
|
||||
fn passing_remove_js_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-j")
|
||||
.arg("data:text/html,<script>alert(2)</script>Hi")
|
||||
.output()
|
||||
@@ -217,6 +225,7 @@ fn passing_local_file_target_input() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let cwd_normalized: String =
|
||||
str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg(if cfg!(windows) {
|
||||
"src\\tests\\data\\local-file.html"
|
||||
} else {
|
||||
@@ -229,25 +238,29 @@ fn passing_local_file_target_input() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// STDOUT should contain HTML from the local file
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stdout).unwrap(),
|
||||
"<!DOCTYPE html><html lang=\"en\"><head>\n \
|
||||
"\
|
||||
<!DOCTYPE html><html lang=\"en\"><head>\n \
|
||||
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n \
|
||||
<title>Local HTML file</title>\n \
|
||||
<link href=\"data:text/css;base64,Ym9keSB7CiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjMDAwOwogICAgY29sb3I6ICNmZmY7Cn0K\" rel=\"stylesheet\" type=\"text/css\">\n \
|
||||
<link href=\"data:text/css;base64,\" rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
|
||||
<img alt=\"\" src=\"\">\n \
|
||||
<link rel=\"stylesheet\" type=\"text/css\" href=\"data:text/css;base64,Ym9keSB7CiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjMDAwOwogICAgY29sb3I6ICNmZmY7Cn0K\">\n \
|
||||
<link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
|
||||
<img alt=\"\">\n \
|
||||
<a href=\"file://local-file.html/\">Tricky href</a>\n \
|
||||
<a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n \
|
||||
<script src=\"data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==\"></script>\n\n\n\n\
|
||||
</body></html>\n"
|
||||
</body></html>\n\
|
||||
"
|
||||
);
|
||||
|
||||
// STDERR should contain list of retrieved file URLs
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stderr).unwrap(),
|
||||
format!(
|
||||
"{file}{cwd}/src/tests/data/local-file.html\n\
|
||||
"\
|
||||
{file}{cwd}/src/tests/data/local-file.html\n\
|
||||
{file}{cwd}/src/tests/data/local-style.css\n\
|
||||
{file}{cwd}/src/tests/data/local-script.js\n",
|
||||
{file}{cwd}/src/tests/data/local-script.js\n\
|
||||
",
|
||||
file = file_url_protocol,
|
||||
cwd = cwd_normalized
|
||||
)
|
||||
@@ -267,6 +280,7 @@ fn passing_local_file_target_input_absolute_target_path() -> Result<(), Box<dyn
|
||||
str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-jciI")
|
||||
.arg(if cfg!(windows) {
|
||||
format!(
|
||||
@@ -286,17 +300,22 @@ fn passing_local_file_target_input_absolute_target_path() -> Result<(), Box<dyn
|
||||
// STDOUT should contain HTML from the local file
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stdout).unwrap(),
|
||||
"<!DOCTYPE html><html lang=\"en\"><head>\
|
||||
format!(
|
||||
"\
|
||||
<!DOCTYPE html><html lang=\"en\"><head>\
|
||||
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:; style-src 'none'; script-src 'none'; img-src data:;\"></meta>\n \
|
||||
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n \
|
||||
<title>Local HTML file</title>\n \
|
||||
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n \
|
||||
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
|
||||
<img alt=\"\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\n \
|
||||
<link rel=\"stylesheet\" type=\"text/css\">\n \
|
||||
<link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
|
||||
<img alt=\"\" src=\"{empty_image}\">\n \
|
||||
<a href=\"file://local-file.html/\">Tricky href</a>\n \
|
||||
<a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n \
|
||||
<script src=\"\"></script>\n\n\n\n\
|
||||
</body></html>\n"
|
||||
<script></script>\n\n\n\n\
|
||||
</body></html>\n\
|
||||
",
|
||||
empty_image = empty_image!()
|
||||
)
|
||||
);
|
||||
|
||||
// STDERR should contain only the target file
|
||||
@@ -322,6 +341,7 @@ fn passing_local_file_url_target_input() -> Result<(), Box<dyn std::error::Error
|
||||
str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
|
||||
let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("-cji")
|
||||
.arg(if cfg!(windows) {
|
||||
format!(
|
||||
@@ -342,17 +362,22 @@ fn passing_local_file_url_target_input() -> Result<(), Box<dyn std::error::Error
|
||||
// STDOUT should contain HTML from the local file
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stdout).unwrap(),
|
||||
"<!DOCTYPE html><html lang=\"en\"><head>\
|
||||
format!(
|
||||
"\
|
||||
<!DOCTYPE html><html lang=\"en\"><head>\
|
||||
<meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none'; script-src 'none'; img-src data:;\"></meta>\n \
|
||||
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n \
|
||||
<title>Local HTML file</title>\n \
|
||||
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n \
|
||||
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
|
||||
<img alt=\"\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\n \
|
||||
<link rel=\"stylesheet\" type=\"text/css\">\n \
|
||||
<link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
|
||||
<img alt=\"\" src=\"{empty_image}\">\n \
|
||||
<a href=\"file://local-file.html/\">Tricky href</a>\n \
|
||||
<a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n \
|
||||
<script src=\"\"></script>\n\n\n\n\
|
||||
</body></html>\n"
|
||||
<script></script>\n\n\n\n\
|
||||
</body></html>\n\
|
||||
",
|
||||
empty_image = empty_image!()
|
||||
)
|
||||
);
|
||||
|
||||
// STDERR should contain list of retrieved file URLs
|
||||
@@ -384,6 +409,7 @@ fn passing_security_disallow_local_assets_within_data_url_targets(
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
|
||||
let out = cmd
|
||||
.arg("-M")
|
||||
.arg("data:text/html,%3Cscript%20src=\"src/tests/data/local-script.js\"%3E%3C/script%3E")
|
||||
.output()
|
||||
.unwrap();
|
||||
@@ -391,7 +417,7 @@ fn passing_security_disallow_local_assets_within_data_url_targets(
|
||||
// STDOUT should contain HTML with no JS in it
|
||||
assert_eq!(
|
||||
std::str::from_utf8(&out.stdout).unwrap(),
|
||||
"<html><head><script src=\"\"></script></head><body></body></html>\n"
|
||||
"<html><head><script></script></head><body></body></html>\n"
|
||||
);
|
||||
|
||||
// STDERR should be empty
|
||||
@@ -421,7 +447,7 @@ fn passing_embed_file_url_local_asset_within_style_attribute(
|
||||
file = file_url_prefix,
|
||||
path = str!(file_svg.path().to_str().unwrap()).replace("\\", "/"),
|
||||
)?;
|
||||
let out = cmd.arg(file_html.path()).output().unwrap();
|
||||
let out = cmd.arg("-M").arg(file_html.path()).output().unwrap();
|
||||
|
||||
// STDOUT should contain HTML with date URL for background-image in it
|
||||
assert_eq!(
|
||||
@@ -458,7 +484,8 @@ fn passing_css_import_string() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let mut file_html = NamedTempFile::new()?;
|
||||
writeln!(
|
||||
file_html,
|
||||
"<style>\n\
|
||||
"\
|
||||
<style>\n\
|
||||
@charset 'UTF-8';\n\
|
||||
\n\
|
||||
@import '{file}{css_path}';\n\
|
||||
@@ -466,11 +493,12 @@ fn passing_css_import_string() -> Result<(), Box<dyn std::error::Error>> {
|
||||
@import url({file}{css_path});\n\
|
||||
\n\
|
||||
@import url('{file}{css_path}')\n\
|
||||
</style>\n",
|
||||
</style>\n\
|
||||
",
|
||||
file = file_url_prefix,
|
||||
css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
|
||||
)?;
|
||||
let out = cmd.arg(file_html.path()).output().unwrap();
|
||||
let out = cmd.arg("-M").arg(file_html.path()).output().unwrap();
|
||||
|
||||
// STDOUT should contain embedded CSS url()'s
|
||||
assert_eq!(
|
||||
|
||||
@@ -15,7 +15,10 @@ fn passing_empty_input() {
|
||||
let cache = &mut HashMap::new();
|
||||
let client = Client::new();
|
||||
|
||||
assert_eq!(css::embed_css(cache, &client, "", "", false, false,), "");
|
||||
assert_eq!(
|
||||
css::embed_css(cache, &client, "", "", false, false, false,),
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -37,16 +40,20 @@ height: calc(100vh - 10pt)";
|
||||
&client,
|
||||
"https://doesntmatter.local/",
|
||||
&STYLE,
|
||||
false,
|
||||
true,
|
||||
true,
|
||||
),
|
||||
"/* border: none;*/\
|
||||
background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='); \
|
||||
list-style: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=');\
|
||||
format!(
|
||||
"/* border: none;*/\
|
||||
background-image: url('{empty_image}'); \
|
||||
list-style: url('{empty_image}');\
|
||||
width:99.998%; \
|
||||
margin-top: -20px; \
|
||||
line-height: -1; \
|
||||
height: calc(100vh - 10pt)"
|
||||
height: calc(100vh - 10pt)",
|
||||
empty_image = empty_image!()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -64,21 +71,17 @@ line-height: -1; \
|
||||
height: calc(100vh - 10pt)";
|
||||
|
||||
assert_eq!(
|
||||
css::embed_css(
|
||||
cache,
|
||||
&client,
|
||||
"",
|
||||
&STYLE,
|
||||
true,
|
||||
true,
|
||||
),
|
||||
"/* border: none;*/\
|
||||
background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='); \
|
||||
list-style: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=');\
|
||||
css::embed_css(cache, &client, "", &STYLE, false, true, true,),
|
||||
format!(
|
||||
"/* border: none;*/\
|
||||
background-image: url('{empty_image}'); \
|
||||
list-style: url('{empty_image}');\
|
||||
width:99.998%; \
|
||||
margin-top: -20px; \
|
||||
line-height: -1; \
|
||||
height: calc(100vh - 10pt)"
|
||||
height: calc(100vh - 10pt)",
|
||||
empty_image = empty_image!()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -96,7 +99,7 @@ fn passing_style_block() {
|
||||
html > body {}";
|
||||
|
||||
assert_eq!(
|
||||
css::embed_css(cache, &client, "file:///", &CSS, false, true,),
|
||||
css::embed_css(cache, &client, "file:///", &CSS, false, false, true,),
|
||||
CSS
|
||||
);
|
||||
}
|
||||
@@ -136,7 +139,10 @@ fn passing_attribute_selectors() {
|
||||
}
|
||||
";
|
||||
|
||||
assert_eq!(css::embed_css(cache, &client, "", &CSS, false, false,), CSS);
|
||||
assert_eq!(
|
||||
css::embed_css(cache, &client, "", &CSS, false, false, false,),
|
||||
CSS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -159,14 +165,15 @@ fn passing_import_string() {
|
||||
"https://doesntmatter.local/",
|
||||
&CSS,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
"\
|
||||
@charset 'UTF-8';\n\
|
||||
\n\
|
||||
@import 'data:text/css;base64,ZGF0YTp0ZXh0L2NzcyxodG1se2JhY2tncm91bmQtY29sb3I6IzAwMH0=';\n\
|
||||
@import 'data:text/css;base64,aHRtbHtiYWNrZ3JvdW5kLWNvbG9yOiMwMDB9';\n\
|
||||
\n\
|
||||
@import url('data:text/css;base64,ZGF0YTp0ZXh0L2NzcyxodG1se2NvbG9yOiNmZmZ9')\n\
|
||||
@import url('data:text/css;base64,aHRtbHtjb2xvcjojZmZmfQ==')\n\
|
||||
"
|
||||
);
|
||||
}
|
||||
@@ -193,8 +200,118 @@ body {\n \
|
||||
"https://doesntmatter.local/",
|
||||
&CSS,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
CSS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_transform_percentages_and_degrees() {
|
||||
let cache = &mut HashMap::new();
|
||||
let client = Client::new();
|
||||
|
||||
const CSS: &str = "\
|
||||
div {\n \
|
||||
transform: translate(-50%, -50%) rotate(-45deg);\n\
|
||||
transform: translate(50%, 50%) rotate(45deg);\n\
|
||||
transform: translate(+50%, +50%) rotate(+45deg);\n\
|
||||
}\n\
|
||||
";
|
||||
|
||||
assert_eq!(
|
||||
css::embed_css(
|
||||
cache,
|
||||
&client,
|
||||
"https://doesntmatter.local/",
|
||||
&CSS,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
CSS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_unusual_indents() {
|
||||
let cache = &mut HashMap::new();
|
||||
let client = Client::new();
|
||||
|
||||
const CSS: &str = "\
|
||||
.is\\:good:hover {\n \
|
||||
color: green\n\
|
||||
}\n\
|
||||
\n\
|
||||
#\\~\\!\\@\\$\\%\\^\\&\\*\\(\\)\\+\\=\\,\\.\\/\\\\\\'\\\"\\;\\:\\?\\>\\<\\[\\]\\{\\}\\|\\`\\# {\n \
|
||||
color: black\n\
|
||||
}\n\
|
||||
";
|
||||
|
||||
assert_eq!(
|
||||
css::embed_css(
|
||||
cache,
|
||||
&client,
|
||||
"https://doesntmatter.local/",
|
||||
&CSS,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
CSS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_exclude_fonts() {
|
||||
let cache = &mut HashMap::new();
|
||||
let client = Client::new();
|
||||
|
||||
const CSS: &str = "\
|
||||
@font-face {\n \
|
||||
font-family: 'My Font';\n \
|
||||
src: url(my_font.woff);\n\
|
||||
}\n\
|
||||
\n\
|
||||
#identifier {\n \
|
||||
font-family: 'My Font' Arial\n\
|
||||
}\n\
|
||||
\n\
|
||||
@font-face {\n \
|
||||
font-family: 'My Font';\n \
|
||||
src: url(my_font.woff);\n\
|
||||
}\n\
|
||||
\n\
|
||||
div {\n \
|
||||
font-family: 'My Font' Verdana\n\
|
||||
}\n\
|
||||
";
|
||||
|
||||
const CSS_OUT: &str = " \
|
||||
\n\
|
||||
\n\
|
||||
#identifier {\n \
|
||||
font-family: 'My Font' Arial\n\
|
||||
}\n\
|
||||
\n \
|
||||
\n\
|
||||
\n\
|
||||
div {\n \
|
||||
font-family: 'My Font' Verdana\n\
|
||||
}\n\
|
||||
";
|
||||
|
||||
assert_eq!(
|
||||
css::embed_css(
|
||||
cache,
|
||||
&client,
|
||||
"https://doesntmatter.local/",
|
||||
&CSS,
|
||||
true,
|
||||
false,
|
||||
true,
|
||||
),
|
||||
CSS_OUT
|
||||
);
|
||||
}
|
||||
|
||||
@@ -28,11 +28,11 @@ fn get_node_name() {
|
||||
let parent = html::get_parent_node(node);
|
||||
let parent_node_name = html::get_node_name(&parent);
|
||||
if node_name == "head" || node_name == "body" {
|
||||
assert_eq!(parent_node_name, "html");
|
||||
assert_eq!(parent_node_name, Some("html"));
|
||||
} else if node_name == "div" {
|
||||
assert_eq!(parent_node_name, "body");
|
||||
assert_eq!(parent_node_name, Some("body"));
|
||||
} else if node_name == "p" {
|
||||
assert_eq!(parent_node_name, "div");
|
||||
assert_eq!(parent_node_name, Some("div"));
|
||||
}
|
||||
|
||||
for child in node.children.borrow().iter() {
|
||||
|
||||
92
src/tests/html/has_proper_integrity.rs
Normal file
92
src/tests/html/has_proper_integrity.rs
Normal file
@@ -0,0 +1,92 @@
|
||||
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
|
||||
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
|
||||
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
|
||||
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
|
||||
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
|
||||
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
|
||||
|
||||
#[cfg(test)]
|
||||
mod passing {
|
||||
use crate::html;
|
||||
|
||||
#[test]
|
||||
fn empty_input_sha256() {
|
||||
assert!(html::has_proper_integrity(
|
||||
"".as_bytes(),
|
||||
"sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha256() {
|
||||
assert!(html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
"sha256-9EWAHgy4mSYsm54hmDaIDXPKLRsLnBX7lZyQ6xISNOM="
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha384() {
|
||||
assert!(html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
"sha384-gc9l7omltke8C33bedgh15E12M7RrAQa5t63Yb8APlpe7ZhiqV23+oqiulSJl3Kw"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha512() {
|
||||
assert!(html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
"sha512-zG5B88cYMqcdiMi9gz0XkOFYw2BpjeYdn5V6+oFrMgSNjRpqL7EF8JEwl17ztZbK3N7I/tTwp3kxQbN1RgFBww=="
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗
|
||||
// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝
|
||||
// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗
|
||||
// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║
|
||||
// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝
|
||||
// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
|
||||
|
||||
#[cfg(test)]
|
||||
mod failing {
|
||||
use crate::html;
|
||||
|
||||
#[test]
|
||||
fn empty_hash() {
|
||||
assert!(!html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
""
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_input_empty_hash() {
|
||||
assert!(!html::has_proper_integrity("".as_bytes(), ""));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha256() {
|
||||
assert!(!html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
"sha256-badhash"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha384() {
|
||||
assert!(!html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
"sha384-badhash"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sha512() {
|
||||
assert!(!html::has_proper_integrity(
|
||||
"abcdef0123456789".as_bytes(),
|
||||
"sha512-badhash"
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
mod get_node_name;
|
||||
mod has_proper_integrity;
|
||||
mod is_icon;
|
||||
mod stringify_document;
|
||||
mod walk_and_embed_assets;
|
||||
|
||||
@@ -19,6 +19,7 @@ fn passing_basic() {
|
||||
let url = "http://localhost";
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -32,10 +33,11 @@ fn passing_basic() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -55,6 +57,7 @@ fn passing_ensure_no_recursive_iframe() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -68,10 +71,11 @@ fn passing_ensure_no_recursive_iframe() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -91,6 +95,7 @@ fn passing_ensure_no_recursive_frame() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -104,10 +109,11 @@ fn passing_ensure_no_recursive_frame() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -129,6 +135,7 @@ fn passing_no_css() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = true;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -141,10 +148,11 @@ fn passing_no_css() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -154,7 +162,7 @@ fn passing_no_css() {
|
||||
buf.iter().map(|&c| c as char).collect::<String>(),
|
||||
"<html>\
|
||||
<head>\
|
||||
<link rel=\"stylesheet\" href=\"\">\
|
||||
<link rel=\"stylesheet\">\
|
||||
<style></style>\
|
||||
</head>\
|
||||
<body>\
|
||||
@@ -173,6 +181,7 @@ fn passing_no_images() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = true;
|
||||
@@ -186,10 +195,11 @@ fn passing_no_images() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -197,18 +207,19 @@ fn passing_no_images() {
|
||||
|
||||
assert_eq!(
|
||||
buf.iter().map(|&c| c as char).collect::<String>(),
|
||||
"<html>\
|
||||
format!(
|
||||
"<html>\
|
||||
<head>\
|
||||
<link rel=\"icon\" href=\"\">\
|
||||
<link rel=\"icon\">\
|
||||
</head>\
|
||||
<body>\
|
||||
<div>\
|
||||
<img src=\"data:image/png;base64,\
|
||||
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0\
|
||||
lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\
|
||||
<img src=\"{empty_image}\">\
|
||||
</div>\
|
||||
</body>\
|
||||
</html>"
|
||||
</html>",
|
||||
empty_image = empty_image!()
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
@@ -220,6 +231,7 @@ fn passing_no_body_background_images() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = true;
|
||||
@@ -233,10 +245,11 @@ fn passing_no_body_background_images() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -256,6 +269,7 @@ fn passing_no_frames() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = true;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -268,10 +282,11 @@ fn passing_no_frames() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -291,6 +306,7 @@ fn passing_no_iframes() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = true;
|
||||
let opt_no_js: bool = false;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -303,10 +319,11 @@ fn passing_no_iframes() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -329,6 +346,7 @@ fn passing_no_js() {
|
||||
let cache = &mut HashMap::new();
|
||||
|
||||
let opt_no_css: bool = false;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = false;
|
||||
let opt_no_js: bool = true;
|
||||
let opt_no_images: bool = false;
|
||||
@@ -342,10 +360,11 @@ fn passing_no_js() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -353,7 +372,7 @@ fn passing_no_js() {
|
||||
|
||||
assert_eq!(
|
||||
buf.iter().map(|&c| c as char).collect::<String>(),
|
||||
"<html><head></head><body><div><script src=\"\"></script>\
|
||||
"<html><head></head><body><div><script></script>\
|
||||
<script></script></div></body></html>"
|
||||
);
|
||||
}
|
||||
@@ -368,6 +387,7 @@ fn passing_with_no_integrity() {
|
||||
let cache = &mut HashMap::new();
|
||||
let client = Client::new();
|
||||
let opt_no_css: bool = true;
|
||||
let opt_no_fonts: bool = false;
|
||||
let opt_no_frames: bool = true;
|
||||
let opt_no_js: bool = true;
|
||||
let opt_no_images: bool = true;
|
||||
@@ -379,10 +399,11 @@ fn passing_with_no_integrity() {
|
||||
&url,
|
||||
&dom.document,
|
||||
opt_no_css,
|
||||
opt_no_fonts,
|
||||
opt_no_frames,
|
||||
opt_no_js,
|
||||
opt_no_images,
|
||||
opt_silent,
|
||||
opt_no_frames,
|
||||
);
|
||||
|
||||
let mut buf: Vec<u8> = Vec::new();
|
||||
@@ -391,7 +412,7 @@ fn passing_with_no_integrity() {
|
||||
assert_eq!(
|
||||
buf.iter().map(|&c| c as char).collect::<String>(),
|
||||
"<html>\
|
||||
<head><title>No integrity</title><link rel=\"something\"><script src=\"\"></script></head>\
|
||||
<head><title>No integrity</title><link rel=\"something\"><script></script></head>\
|
||||
<body></body>\
|
||||
</html>"
|
||||
);
|
||||
|
||||
14
src/tests/macros/empty_image.rs
Normal file
14
src/tests/macros/empty_image.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
|
||||
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
|
||||
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
|
||||
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
|
||||
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
|
||||
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
|
||||
|
||||
#[cfg(test)]
|
||||
mod passing {
|
||||
#[test]
|
||||
fn contains_correct_image_data() {
|
||||
assert_eq!(empty_image!(), "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII=");
|
||||
}
|
||||
}
|
||||
2
src/tests/macros/mod.rs
Normal file
2
src/tests/macros/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
mod empty_image;
|
||||
mod str;
|
||||
24
src/tests/macros/str.rs
Normal file
24
src/tests/macros/str.rs
Normal file
@@ -0,0 +1,24 @@
|
||||
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
|
||||
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
|
||||
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
|
||||
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
|
||||
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
|
||||
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
|
||||
|
||||
#[cfg(test)]
|
||||
mod passing {
|
||||
#[test]
|
||||
fn returns_empty_string() {
|
||||
assert_eq!(str!(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn converts_integer_into_string() {
|
||||
assert_eq!(str!(123), "123");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn converts_str_into_string() {
|
||||
assert_eq!(str!("abc"), "abc");
|
||||
}
|
||||
}
|
||||
@@ -2,4 +2,5 @@ mod cli;
|
||||
mod css;
|
||||
mod html;
|
||||
mod js;
|
||||
mod macros;
|
||||
mod utils;
|
||||
|
||||
@@ -30,3 +30,19 @@ fn passing_removes_empty_query_and_empty_fragment() {
|
||||
"https://somewhere.com/font.eot"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_removes_empty_query_amp_and_empty_fragment() {
|
||||
assert_eq!(
|
||||
utils::clean_url("https://somewhere.com/font.eot?a=b&#"),
|
||||
"https://somewhere.com/font.eot?a=b"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_keeps_credentials() {
|
||||
assert_eq!(
|
||||
utils::clean_url("https://cookie:monster@gibson.internet/"),
|
||||
"https://cookie:monster@gibson.internet/"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9,42 +9,76 @@ use crate::utils;
|
||||
|
||||
#[test]
|
||||
fn passing_parse_text_html_base64() {
|
||||
let (media_type, data) = utils::data_url_to_data("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
|
||||
|
||||
assert_eq!(media_type, "text/html");
|
||||
assert_eq!(
|
||||
utils::data_url_to_text("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg=="),
|
||||
String::from_utf8_lossy(&data),
|
||||
"Work expands so as to fill the time available for its completion"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_parse_text_html_utf8() {
|
||||
let (media_type, data) = utils::data_url_to_data(
|
||||
"data:text/html;utf8,Work expands so as to fill the time available for its completion",
|
||||
);
|
||||
|
||||
assert_eq!(media_type, "text/html");
|
||||
assert_eq!(
|
||||
utils::data_url_to_text(
|
||||
"data:text/html;utf8,Work expands so as to fill the time available for its completion"
|
||||
),
|
||||
String::from_utf8_lossy(&data),
|
||||
"Work expands so as to fill the time available for its completion"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_parse_text_html_plaintext() {
|
||||
let (media_type, data) = utils::data_url_to_data(
|
||||
"data:text/html,Work expands so as to fill the time available for its completion",
|
||||
);
|
||||
|
||||
assert_eq!(media_type, "text/html");
|
||||
assert_eq!(
|
||||
utils::data_url_to_text(
|
||||
"data:text/html,Work expands so as to fill the time available for its completion"
|
||||
),
|
||||
String::from_utf8_lossy(&data),
|
||||
"Work expands so as to fill the time available for its completion"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_parse_text_html_charset_utf_8_between_two_whitespaces() {
|
||||
let (media_type, data) = utils::data_url_to_data(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
|
||||
|
||||
assert_eq!(media_type, "text/html");
|
||||
assert_eq!(
|
||||
utils::data_url_to_text(
|
||||
" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion "
|
||||
),
|
||||
String::from_utf8_lossy(&data),
|
||||
"Work expands so as to fill the time available for its completion"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_parse_text_css_url_encoded() {
|
||||
let (media_type, data) = utils::data_url_to_data("data:text/css,div{background-color:%23000}");
|
||||
|
||||
assert_eq!(media_type, "text/css");
|
||||
assert_eq!(String::from_utf8_lossy(&data), "div{background-color:#000}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_parse_no_media_type_base64() {
|
||||
let (media_type, data) = utils::data_url_to_data("data:;base64,dGVzdA==");
|
||||
|
||||
assert_eq!(media_type, "");
|
||||
assert_eq!(String::from_utf8_lossy(&data), "test");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_parse_no_media_type_no_encoding() {
|
||||
let (media_type, data) = utils::data_url_to_data("data:;,test%20test");
|
||||
|
||||
assert_eq!(media_type, "");
|
||||
assert_eq!(String::from_utf8_lossy(&data), "test test");
|
||||
}
|
||||
|
||||
// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗
|
||||
// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝
|
||||
// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗
|
||||
@@ -54,5 +88,8 @@ fn passing_parse_text_html_charset_utf_8_between_two_whitespaces() {
|
||||
|
||||
#[test]
|
||||
fn failing_just_word_data() {
|
||||
assert_eq!(utils::data_url_to_text("data"), "");
|
||||
let (media_type, data) = utils::data_url_to_data("data");
|
||||
|
||||
assert_eq!(media_type, "");
|
||||
assert_eq!(String::from_utf8_lossy(&data), "");
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
use crate::utils;
|
||||
|
||||
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
|
||||
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
|
||||
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
|
||||
@@ -7,20 +5,35 @@ use crate::utils;
|
||||
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
|
||||
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
|
||||
|
||||
#[test]
|
||||
fn passing_decode_unicode_characters() {
|
||||
assert_eq!(
|
||||
utils::decode_url(str!(
|
||||
"%E6%A4%9C%E3%83%92%E3%83%A0%E8%A7%A3%E5%A1%97%E3%82%83%E3%83%83%20%3D%20%E3%82%B5"
|
||||
)),
|
||||
"検ヒム解塗ゃッ = サ"
|
||||
);
|
||||
}
|
||||
#[cfg(test)]
|
||||
mod passing {
|
||||
use crate::utils;
|
||||
|
||||
#[test]
|
||||
fn passing_decode_file_url() {
|
||||
assert_eq!(
|
||||
utils::decode_url(str!("file:///tmp/space%20here/test%231.html")),
|
||||
"file:///tmp/space here/test#1.html"
|
||||
);
|
||||
#[test]
|
||||
fn decode_unicode_characters() {
|
||||
assert_eq!(
|
||||
utils::decode_url(str!(
|
||||
"%E6%A4%9C%E3%83%92%E3%83%A0%E8%A7%A3%E5%A1%97%E3%82%83%E3%83%83%20%3D%20%E3%82%B5"
|
||||
)),
|
||||
"検ヒム解塗ゃッ = サ"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_file_url() {
|
||||
assert_eq!(
|
||||
utils::decode_url(str!("file:///tmp/space%20here/test%231.html")),
|
||||
"file:///tmp/space here/test#1.html"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn plus_sign() {
|
||||
assert_eq!(
|
||||
utils::decode_url(str!(
|
||||
"fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
|
||||
)),
|
||||
"fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,3 +21,18 @@ fn passing_remove_protocl_and_fragment() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_decodes_urls() {
|
||||
if cfg!(windows) {
|
||||
assert_eq!(
|
||||
utils::file_url_to_fs_path("file:///C:/Documents%20and%20Settings/some-file.html"),
|
||||
"C:\\Documents and Settings\\some-file.html"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(
|
||||
utils::file_url_to_fs_path("file:///home/user/My%20Documents"),
|
||||
"/home/user/My Documents"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
mod clean_url;
|
||||
mod data_to_data_url;
|
||||
mod data_url_to_text;
|
||||
mod data_url_to_data;
|
||||
mod decode_url;
|
||||
mod detect_media_type;
|
||||
mod file_url_to_fs_path;
|
||||
|
||||
@@ -17,38 +17,23 @@ fn passing_read_data_url() {
|
||||
|
||||
// If both source and target are data URLs,
|
||||
// ensure the result contains target data URL
|
||||
let (retrieved_data, final_url) = utils::retrieve_asset(
|
||||
let (data, final_url, media_type) = utils::retrieve_asset(
|
||||
cache,
|
||||
&client,
|
||||
"data:text/html;base64,SoUrCe",
|
||||
"data:text/html;base64,TaRgEt",
|
||||
true,
|
||||
"",
|
||||
"data:text/html;base64,c291cmNl",
|
||||
"data:text/html;base64,dGFyZ2V0",
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(&retrieved_data, "data:text/html;base64,TaRgEt");
|
||||
assert_eq!(&final_url, "data:text/html;base64,TaRgEt");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passing_read_data_url_ignore_suggested_media_type() {
|
||||
let cache = &mut HashMap::new();
|
||||
let client = Client::new();
|
||||
|
||||
// Media type parameter should not influence data URLs
|
||||
let (data, final_url) = utils::retrieve_asset(
|
||||
cache,
|
||||
&client,
|
||||
"data:text/html;base64,SoUrCe",
|
||||
"data:text/html;base64,TaRgEt",
|
||||
true,
|
||||
"image/png",
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(&data, "data:text/html;base64,TaRgEt");
|
||||
assert_eq!(&final_url, "data:text/html;base64,TaRgEt");
|
||||
assert_eq!(
|
||||
utils::data_to_data_url(&media_type, &data, &final_url, ""),
|
||||
utils::data_to_data_url("text/html", "target".as_bytes(), "", "")
|
||||
);
|
||||
assert_eq!(
|
||||
final_url,
|
||||
utils::data_to_data_url("text/html", "target".as_bytes(), "", "")
|
||||
);
|
||||
assert_eq!(&media_type, "text/html");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -60,7 +45,7 @@ fn passing_read_local_file_with_file_url_parent() {
|
||||
|
||||
// Inclusion of local assets from local sources should be allowed
|
||||
let cwd = env::current_dir().unwrap();
|
||||
let (data, final_url) = utils::retrieve_asset(
|
||||
let (data, final_url, _media_type) = utils::retrieve_asset(
|
||||
cache,
|
||||
&client,
|
||||
&format!(
|
||||
@@ -73,12 +58,10 @@ fn passing_read_local_file_with_file_url_parent() {
|
||||
file = file_url_protocol,
|
||||
cwd = cwd.to_str().unwrap()
|
||||
),
|
||||
true,
|
||||
"application/javascript",
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(&data, "data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==");
|
||||
assert_eq!(utils::data_to_data_url("application/javascript", &data, &final_url, ""), "data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==");
|
||||
assert_eq!(
|
||||
&final_url,
|
||||
&format!(
|
||||
@@ -102,18 +85,20 @@ fn failing_read_local_file_with_data_url_parent() {
|
||||
let client = Client::new();
|
||||
|
||||
// Inclusion of local assets from data URL sources should not be allowed
|
||||
let (data, final_url) = utils::retrieve_asset(
|
||||
match utils::retrieve_asset(
|
||||
cache,
|
||||
&client,
|
||||
"data:text/html;base64,SoUrCe",
|
||||
"file:///etc/passwd",
|
||||
true,
|
||||
"",
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(&data, "");
|
||||
assert_eq!(&final_url, "");
|
||||
) {
|
||||
Ok((..)) => {
|
||||
assert!(false);
|
||||
}
|
||||
Err(_) => {
|
||||
assert!(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -122,16 +107,18 @@ fn failing_read_local_file_with_https_parent() {
|
||||
let client = Client::new();
|
||||
|
||||
// Inclusion of local assets from remote sources should not be allowed
|
||||
let (data, final_url) = utils::retrieve_asset(
|
||||
match utils::retrieve_asset(
|
||||
cache,
|
||||
&client,
|
||||
"https://kernel.org/",
|
||||
"file:///etc/passwd",
|
||||
true,
|
||||
"",
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(&data, "");
|
||||
assert_eq!(&final_url, "");
|
||||
) {
|
||||
Ok((..)) => {
|
||||
assert!(false);
|
||||
}
|
||||
Err(_) => {
|
||||
assert!(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
165
src/utils.rs
165
src/utils.rs
@@ -30,6 +30,14 @@ const MAGIC: [[&[u8]; 2]; 18] = [
|
||||
[b"\x1A\x45\xDF\xA3", b"video/webm"],
|
||||
];
|
||||
|
||||
const PLAINTEXT_MEDIA_TYPES: &[&str] = &[
|
||||
"image/svg+xml",
|
||||
"text/css",
|
||||
"text/html",
|
||||
"text/javascript",
|
||||
"text/plain",
|
||||
];
|
||||
|
||||
pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str, fragment: &str) -> String {
|
||||
let media_type: String = if media_type.is_empty() {
|
||||
detect_media_type(data, &url)
|
||||
@@ -88,6 +96,10 @@ pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
pub fn is_plaintext_media_type(media_type: &str) -> bool {
|
||||
PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
|
||||
}
|
||||
|
||||
pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
|
||||
let result = if is_http_url(to.as_ref()) {
|
||||
to.as_ref().to_string()
|
||||
@@ -108,65 +120,68 @@ pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clean_url<T: AsRef<str>>(url: T) -> String {
|
||||
let mut result = Url::parse(url.as_ref()).unwrap();
|
||||
pub fn clean_url<T: AsRef<str>>(input: T) -> String {
|
||||
let mut url = Url::parse(input.as_ref()).unwrap();
|
||||
|
||||
// Clear fragment
|
||||
result.set_fragment(None);
|
||||
url.set_fragment(None);
|
||||
|
||||
// Get rid of stray question mark
|
||||
if result.query() == Some("") {
|
||||
result.set_query(None);
|
||||
if url.query() == Some("") {
|
||||
url.set_query(None);
|
||||
}
|
||||
result.to_string()
|
||||
|
||||
// Remove empty trailing ampersand(s)
|
||||
let mut result: String = url.to_string();
|
||||
while result.ends_with("&") {
|
||||
result.pop();
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn data_url_to_text<T: AsRef<str>>(url: T) -> String {
|
||||
let parsed_url = Url::parse(url.as_ref()).unwrap_or(Url::parse("http://[::1]").unwrap());
|
||||
pub fn data_url_to_data<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
|
||||
let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
|
||||
let path: String = parsed_url.path().to_string();
|
||||
let comma_loc: usize = path.find(',').unwrap_or(path.len());
|
||||
|
||||
if comma_loc == path.len() {
|
||||
return str!();
|
||||
}
|
||||
|
||||
let meta_data: String = path.chars().take(comma_loc).collect();
|
||||
let raw_data: String = path.chars().skip(comma_loc + 1).collect();
|
||||
|
||||
let data: String = decode_url(raw_data);
|
||||
let text: String = decode_url(raw_data);
|
||||
|
||||
let meta_data_items: Vec<&str> = meta_data.split(';').collect();
|
||||
let mut media_type: &str = "";
|
||||
let mut media_type: String = str!();
|
||||
let mut encoding: &str = "";
|
||||
|
||||
let mut i: i8 = 0;
|
||||
for item in &meta_data_items {
|
||||
if i == 0 {
|
||||
if item.eq_ignore_ascii_case("text/html") {
|
||||
media_type = item;
|
||||
continue;
|
||||
media_type = str!(item);
|
||||
} else {
|
||||
if item.eq_ignore_ascii_case("base64")
|
||||
|| item.eq_ignore_ascii_case("utf8")
|
||||
|| item.eq_ignore_ascii_case("charset=UTF-8")
|
||||
{
|
||||
encoding = item;
|
||||
}
|
||||
}
|
||||
|
||||
if item.eq_ignore_ascii_case("base64") || item.eq_ignore_ascii_case("utf8") {
|
||||
encoding = item;
|
||||
}
|
||||
|
||||
i = i + 1;
|
||||
}
|
||||
|
||||
if media_type.eq_ignore_ascii_case("text/html") {
|
||||
if encoding.eq_ignore_ascii_case("base64") {
|
||||
String::from_utf8(base64::decode(&data).unwrap_or(vec![])).unwrap_or(str!())
|
||||
} else {
|
||||
data
|
||||
}
|
||||
let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
|
||||
base64::decode(&text).unwrap_or(vec![])
|
||||
} else {
|
||||
str!()
|
||||
}
|
||||
text.as_bytes().to_vec()
|
||||
};
|
||||
|
||||
(media_type, data)
|
||||
}
|
||||
|
||||
pub fn decode_url(input: String) -> String {
|
||||
let input: String = input.replace("+", "%2B");
|
||||
|
||||
form_urlencoded::parse(input.as_bytes())
|
||||
.map(|(key, val)| {
|
||||
[
|
||||
@@ -200,66 +215,61 @@ pub fn file_url_to_fs_path(url: &str) -> String {
|
||||
fs_file_path = fs_file_path.replace("/", "\\");
|
||||
}
|
||||
|
||||
fs_file_path
|
||||
// File paths should not be %-encoded
|
||||
decode_url(fs_file_path)
|
||||
}
|
||||
|
||||
pub fn retrieve_asset(
|
||||
cache: &mut HashMap<String, String>,
|
||||
cache: &mut HashMap<String, Vec<u8>>,
|
||||
client: &Client,
|
||||
parent_url: &str,
|
||||
url: &str,
|
||||
as_data_url: bool,
|
||||
media_type: &str,
|
||||
opt_silent: bool,
|
||||
) -> Result<(String, String), reqwest::Error> {
|
||||
) -> Result<(Vec<u8>, String, String), reqwest::Error> {
|
||||
if url.len() == 0 {
|
||||
return Ok((str!(), str!()));
|
||||
// Provoke error
|
||||
client.get("").send()?;
|
||||
}
|
||||
|
||||
let cache_key = clean_url(&url);
|
||||
|
||||
if is_data_url(&url) {
|
||||
Ok((url.to_string(), url.to_string()))
|
||||
let (media_type, data) = data_url_to_data(url);
|
||||
Ok((data, url.to_string(), media_type))
|
||||
} else if is_file_url(&url) {
|
||||
// Check if parent_url is also file:///
|
||||
// (if not, then we don't embed the asset)
|
||||
if !is_file_url(&parent_url) {
|
||||
return Ok((str!(), str!()));
|
||||
// Provoke error
|
||||
client.get("").send()?;
|
||||
}
|
||||
|
||||
let fs_file_path: String = file_url_to_fs_path(url);
|
||||
let path = Path::new(&fs_file_path);
|
||||
let url_fragment = get_url_fragment(url);
|
||||
if path.exists() {
|
||||
if !opt_silent {
|
||||
eprintln!("{}", &url);
|
||||
}
|
||||
|
||||
if as_data_url {
|
||||
let data_url: String = data_to_data_url(
|
||||
&media_type,
|
||||
&fs::read(&fs_file_path).unwrap(),
|
||||
&fs_file_path,
|
||||
&url_fragment,
|
||||
);
|
||||
Ok((data_url, url.to_string()))
|
||||
} else {
|
||||
let data: String = fs::read_to_string(&fs_file_path).expect(url);
|
||||
Ok((data, url.to_string()))
|
||||
}
|
||||
Ok((fs::read(&fs_file_path).expect(""), url.to_string(), str!()))
|
||||
} else {
|
||||
Ok((str!(), url.to_string()))
|
||||
// Provoke error
|
||||
Err(client.get("").send().unwrap_err())
|
||||
}
|
||||
} else {
|
||||
let cache_key: String = clean_url(&url);
|
||||
|
||||
if cache.contains_key(&cache_key) {
|
||||
// URL is in cache
|
||||
// URL is in cache, we get and return it
|
||||
if !opt_silent {
|
||||
eprintln!("{} (from cache)", &url);
|
||||
}
|
||||
let data = cache.get(&cache_key).unwrap();
|
||||
Ok((data.to_string(), url.to_string()))
|
||||
|
||||
Ok((
|
||||
cache.get(&cache_key).unwrap().to_vec(),
|
||||
url.to_string(),
|
||||
str!(),
|
||||
))
|
||||
} else {
|
||||
// URL not in cache, we request it
|
||||
// URL not in cache, we retrieve the file
|
||||
let mut response = client.get(url).send()?;
|
||||
let res_url = response.url().to_string();
|
||||
|
||||
@@ -271,34 +281,23 @@ pub fn retrieve_asset(
|
||||
}
|
||||
}
|
||||
|
||||
let new_cache_key = clean_url(&res_url);
|
||||
let new_cache_key: String = clean_url(&res_url);
|
||||
|
||||
if as_data_url {
|
||||
// Convert response into a byte array
|
||||
let mut data: Vec<u8> = vec![];
|
||||
response.copy_to(&mut data)?;
|
||||
// Convert response into a byte array
|
||||
let mut data: Vec<u8> = vec![];
|
||||
response.copy_to(&mut data)?;
|
||||
|
||||
// Attempt to obtain media type by reading the Content-Type header
|
||||
let media_type = if media_type == "" {
|
||||
response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|header| header.to_str().ok())
|
||||
.unwrap_or(&media_type)
|
||||
} else {
|
||||
media_type
|
||||
};
|
||||
let url_fragment = get_url_fragment(url);
|
||||
let data_url = data_to_data_url(&media_type, &data, url, &url_fragment);
|
||||
// Add to cache
|
||||
cache.insert(new_cache_key, data_url.clone());
|
||||
Ok((data_url, res_url))
|
||||
} else {
|
||||
let content = response.text().unwrap();
|
||||
// Add to cache
|
||||
cache.insert(new_cache_key, content.clone());
|
||||
Ok((content, res_url))
|
||||
}
|
||||
// Attempt to obtain media type by reading the Content-Type header
|
||||
let media_type = response
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|header| header.to_str().ok())
|
||||
.unwrap_or("");
|
||||
|
||||
// Add to cache
|
||||
cache.insert(new_cache_key, data.clone());
|
||||
|
||||
Ok((data, res_url, media_type.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user