56 Commits

Author SHA1 Message Date
Sunshine
7b095fe4ff Merge pull request #167 from snshn/version-bump
version bump
2020-04-25 03:50:10 -04:00
Sunshine
890bcb1bb6 version bump 2020-04-25 01:03:49 -04:00
Sunshine
aa97ea9f82 Merge pull request #165 from snshn/no-fonts
Add flag for excluding web fonts
2020-04-22 09:16:30 -04:00
Sunshine
9b40dbbf27 add option to exclude web fonts 2020-04-22 09:11:20 -04:00
Sunshine
289f3e801b Merge pull request #161 from snshn/cache-blob
Store blobs instead of data URLs in cache
2020-04-19 13:33:03 -04:00
Sunshine
edacd09dc8 store blobs instead of data URLs in cache 2020-04-19 13:26:14 -04:00
Sunshine
5682863725 Merge pull request #164 from snshn/raspberry-pi-artifact-update
Update GitHub Action for assembling ARM artifacts
2020-04-18 13:46:44 -04:00
Sunshine
4304d7a638 update GitHub Action for assembling ARM artifacts 2020-04-18 13:44:26 -04:00
Sunshine
f56f88da94 Merge pull request #91 from snshn/unwrap-noscript-if-no-js
Propose ADR-0002 (NOSCRIPT nodes)
2020-04-16 23:24:30 -04:00
Sunshine
87c8b361ea add ADR-0002 (NOSCRIPT nodes) 2020-04-16 23:24:03 -04:00
Sunshine
cd505ddb6c Merge pull request #163 from snshn/proper-css-ident-escaping
Escape all special chars within #id and .class CSS selectors
2020-04-11 18:33:41 -04:00
Sunshine
eeea617fb1 escape all special chars within #id and .class CSS selectors 2020-04-11 17:50:23 -04:00
Sunshine
cc6dbddb49 Merge pull request #162 from snshn/colons-in-css-class-names
Escape colons within CSS idents
2020-04-10 21:20:37 -04:00
Sunshine
9d3df2cdc6 escape colons within CSS idents 2020-04-10 20:59:56 -04:00
Sunshine
ab601c3830 Merge pull request #160 from snshn/more-css-image-url-detection-props
Treat url()'s found in @counter-style rules as images
2020-04-10 07:28:55 -04:00
Sunshine
3738be2b6d treat url()'s found in @counter-style rules as images 2020-04-10 07:22:02 -04:00
Sunshine
53160f01c7 Merge pull request #159 from snshn/implement-data-url-media-type-detection
Improve data URL media type detection
2020-04-10 06:04:49 -04:00
Sunshine
594ad55bd8 improve data URL media type detection 2020-04-10 05:50:33 -04:00
Sunshine
d2615f51dc Merge pull request #158 from snshn/improve-data-url-support
Improve parsing of data URLs
2020-04-10 01:49:34 -04:00
Sunshine
c097733ae7 improve parsing of data URLs 2020-04-09 20:27:07 -04:00
Sunshine
67d4b7dafc Merge pull request #157 from snshn/2-2-3
Upgrade base64 crate & version bump (2.2.2 → 2.2.3)
2020-04-08 19:56:24 -04:00
Sunshine
b1d6bbce0c upgrade base64 crate & version bump (2.2.2 → 2.2.3) 2020-04-08 19:49:46 -04:00
Sunshine
20124f4891 Merge pull request #156 from snshn/raspberry-pi-artifact
Make the pipeline build and upload armhf executable with every new release
2020-04-08 19:40:41 -04:00
Sunshine
0dd540afaf make the pipeline build and upload armhf executable with every new release 2020-04-08 19:29:17 -04:00
Sunshine
df71083359 Merge pull request #155 from snshn/fix-css-unit-sign-bug
Fix css unit sign bug
2020-04-08 18:19:32 -04:00
Sunshine
349c7bb3ea properly parse negative units in CSS 2020-04-08 18:07:39 -04:00
Sunshine
5a30c6b44b Merge branch 'master' of github.com:snshn/monolith 2020-04-08 10:53:29 -04:00
Sunshine
929924accd Merge pull request #153 from snshn/proper-quotation-marks
use proper quotation marks in the README
2020-04-05 16:25:40 -04:00
Sunshine
812b46960c use proper quotation marks in the README 2020-04-05 16:24:18 -04:00
Sunshine
874080dbda Merge pull request #152 from snshn/separate-ci-build-jobs
Separate OS build jobs
2020-04-05 15:34:21 -04:00
Sunshine
93dd9d4ed4 separate build job per OS 2020-04-05 15:32:25 -04:00
Sunshine
3f0ced0143 Merge pull request #151 from snshn/2-2-2
version bump (2.2.1 → 2.2.2)
2020-04-05 14:44:48 -04:00
Sunshine
8112ab6d04 version bump (2.2.1 → 2.2.2) 2020-04-05 14:38:40 -04:00
Sunshine
e5fc05f5cd Merge pull request #150 from snshn/cd-windows-executable
Make the pipeline upload windows build to every new release
2020-04-05 14:35:44 -04:00
Sunshine
1068ff659a make the pipeline upload windows build to every new release 2020-04-05 14:29:06 -04:00
Sunshine
d4d9bbe424 update cd.yml 2020-04-04 22:12:35 -04:00
Sunshine
cf3a8c8ede Merge pull request #149 from snshn/remove-travis-ci-and-appveyor
Remove TravisCI and AppVeyor from the project
2020-04-04 19:38:03 -04:00
Sunshine
920d992459 remove TravisCI and AppVeyor from the project 2020-04-04 19:26:58 -04:00
Sunshine
c61b3ba858 Merge pull request #148 from snshn/github-actions-build
Improve GitHub Actions integration
2020-04-04 19:14:52 -04:00
Sunshine
dc6e564ea2 integrate GitHub Actions CI further 2020-04-04 19:05:49 -04:00
Sunshine
24536b5e18 Merge pull request #147 from Y2Z/github-actions-ci
Implement CI using GitHub Actions
2020-04-04 17:51:28 -04:00
Sunshine
908fd59019 Update ci.yml 2020-04-04 17:08:19 -04:00
Sunshine
a19aa37ea8 Merge pull request #145 from snshn/no-images-svg
Empty SVG nodes when excluding images
2020-04-04 15:55:26 -04:00
Sunshine
c46bd5900b Merge pull request #146 from snshn/image-map-area-href
Resolve hrefs of <area> image-map tags
2020-04-04 15:51:45 -04:00
Sunshine
5f98ed23b3 set autocrlf to false to let windows builds pass 2020-04-04 15:42:53 -04:00
Sunshine
c6b135398a Implement CI using GitHub Actions 2020-04-04 15:30:13 -04:00
Sunshine
791e44796e resolve hrefs of <area> image-map tags 2020-04-04 14:55:45 -04:00
Sunshine
b428dd8471 Merge pull request #144 from snshn/macros-unit-test
Implement unit tests for macros
2020-04-04 13:11:19 -04:00
Sunshine
b88479446c implement unit tests for macros 2020-04-04 08:21:41 -04:00
Sunshine
1d6217ef5a empty SVG nodes if --no-images 2020-04-03 21:56:46 -04:00
Sunshine
746c7f05de Merge pull request #143 from snshn/embed-input-images
Add support for image inputs
2020-04-03 04:12:06 -04:00
Sunshine
29836d979a add support for image inputs 2020-04-03 03:30:52 -04:00
Sunshine
5ba6e33fa8 Merge pull request #142 from snshn/robatipoors-improvements
Revamp is_icon() and get_node_name()
2020-04-03 01:39:45 -04:00
Sunshine
643c4ce7ef implement improvements suggested by @robatipoor 2020-04-03 00:00:08 -04:00
Sunshine
c011f90b76 Merge pull request #141 from snshn/update-help-dialog
Update help dialog
2020-04-02 22:49:59 -04:00
Sunshine
875481b9a2 update help dialog 2020-04-02 03:04:21 -04:00
31 changed files with 777 additions and 347 deletions

View File

@@ -1,131 +0,0 @@
# Appveyor configuration template for Rust using rustup for Rust installation
# https://github.com/starkat99/appveyor-rust
## Operating System (VM environment) ##
# Rust needs at least Visual Studio 2013 Appveyor OS for MSVC targets.
os: Visual Studio 2015
## Build Matrix ##
# This configuration will setup a build for each channel & target combination (12 windows
# combinations in all).
#
# There are 3 channels: stable, beta, and nightly.
#
# Alternatively, the full version may be specified for the channel to build using that specific
# version (e.g. channel: 1.5.0)
#
# The values for target are the set of windows Rust build targets. Each value is of the form
#
# ARCH-pc-windows-TOOLCHAIN
#
# Where ARCH is the target architecture, either x86_64 or i686, and TOOLCHAIN is the linker
# toolchain to use, either msvc or gnu. See https://www.rust-lang.org/downloads.html#win-foot for
# a description of the toolchain differences.
# See https://github.com/rust-lang-nursery/rustup.rs/#toolchain-specification for description of
# toolchains and host triples.
#
# Comment out channel/target combos you do not wish to build in CI.
#
# You may use the `cargoflags` and `RUSTFLAGS` variables to set additional flags for cargo commands
# and rustc, respectively. For instance, you can uncomment the cargoflags lines in the nightly
# channels to enable unstable features when building for nightly. Or you could add additional
# matrix entries to test different combinations of features.
environment:
matrix:
### MSVC Toolchains ###
# Stable 64-bit MSVC
- channel: stable
target: x86_64-pc-windows-msvc
# Stable 32-bit MSVC
- channel: stable
target: i686-pc-windows-msvc
# Beta 64-bit MSVC
- channel: beta
target: x86_64-pc-windows-msvc
# Beta 32-bit MSVC
- channel: beta
target: i686-pc-windows-msvc
# Nightly 64-bit MSVC
- channel: nightly
target: x86_64-pc-windows-msvc
cargoflags: --features "unstable"
# Nightly 32-bit MSVC
- channel: nightly
target: i686-pc-windows-msvc
cargoflags: --features "unstable"
### GNU Toolchains ###
# Stable 64-bit GNU
- channel: stable
target: x86_64-pc-windows-gnu
MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
# Stable 32-bit GNU
- channel: stable
target: i686-pc-windows-gnu
MINGW_PATH: 'C:\MinGW\bin'
# Beta 64-bit GNU
- channel: beta
target: x86_64-pc-windows-gnu
MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
# Beta 32-bit GNU
- channel: beta
target: i686-pc-windows-gnu
MINGW_PATH: 'C:\MinGW\bin'
# Nightly 64-bit GNU
- channel: nightly
target: x86_64-pc-windows-gnu
MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
cargoflags: --features "unstable"
# Nightly 32-bit GNU
- channel: nightly
target: i686-pc-windows-gnu
MINGW_PATH: 'C:\MinGW\bin'
cargoflags: --features "unstable"
### Allowed failures ###
# See Appveyor documentation for specific details. In short, place any channel or targets you wish
# to allow build failures on (usually nightly at least is a wise choice). This will prevent a build
# or test failure in the matching channels/targets from failing the entire build.
matrix:
allow_failures:
- channel: nightly
- channel: beta
# If you only care about stable channel build failures, uncomment the following line:
#- channel: beta
## Install Script ##
# This is the most important part of the Appveyor configuration. This installs the version of Rust
# specified by the 'channel' and 'target' environment variables from the build matrix. This uses
# rustup to install Rust.
#
# For simple configurations, instead of using the build matrix, you can simply set the
# default-toolchain and default-host manually here.
install:
- appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
- rustup-init -yv --default-toolchain %channel% --default-host %target%
- set PATH=%PATH%;%USERPROFILE%\.cargo\bin
- if defined MINGW_PATH set PATH=%PATH%;%MINGW_PATH%
- rustc -vV
- cargo -vV
- rustup component add rustfmt
## Build Script ##
# 'cargo test' takes care of building for us, so disable Appveyor's build stage. This prevents
# the "directory does not contain a project or solution file" error.
build: false
# Uses 'cargo test' to run tests and build. Alternatively, the project may call compiled programs
# directly or perform other testing commands. Rust will automatically be placed in the PATH
# environment variable.
test_script:
- cargo build --all --locked --verbose %cargoflags%
- cargo test --all --locked --verbose %cargoflags%

22
.github/workflows/build_gnu_linux.yml vendored Normal file
View File

@@ -0,0 +1,22 @@
name: GNU/Linux
on:
push:
branches: [ master ]
jobs:
build:
strategy:
matrix:
os:
- ubuntu-latest
rust:
- stable
runs-on: ${{ matrix.os }}
steps:
- run: git config --global core.autocrlf false
- uses: actions/checkout@v2
- name: Build
run: cargo build --all --locked --verbose

22
.github/workflows/build_macos.yml vendored Normal file
View File

@@ -0,0 +1,22 @@
name: macOS
on:
push:
branches: [ master ]
jobs:
build:
strategy:
matrix:
os:
- macos-latest
rust:
- stable
runs-on: ${{ matrix.os }}
steps:
- run: git config --global core.autocrlf false
- uses: actions/checkout@v2
- name: Build
run: cargo build --all --locked --verbose

22
.github/workflows/build_windows.yml vendored Normal file
View File

@@ -0,0 +1,22 @@
name: Windows
on:
push:
branches: [ master ]
jobs:
build:
strategy:
matrix:
os:
- windows-latest
rust:
- stable
runs-on: ${{ matrix.os }}
steps:
- run: git config --global core.autocrlf false
- uses: actions/checkout@v2
- name: Build
run: cargo build --all --locked --verbose

54
.github/workflows/cd.yml vendored Normal file
View File

@@ -0,0 +1,54 @@
# CD GitHub Actions workflow for Monolith
name: CD
on:
release:
types:
- created
jobs:
windows:
runs-on: windows-2019
steps:
- run: git config --global core.autocrlf false
- name: Checkout the repository
uses: actions/checkout@v2
- name: Build and install the executable
run: cargo build --release
- uses: Shopify/upload-to-release@1.0.0
with:
name: monolith.exe
path: target\release\monolith.exe
repo-token: ${{ secrets.GITHUB_TOKEN }}
gnu_linux_armhf:
runs-on: ubuntu-18.04
steps:
- name: Checkout the repository
uses: actions/checkout@v2
- name: Prepare cross-platform environment
run: |
sudo mkdir -p /cross-build-arm
sudo echo "deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic main" >> /etc/apt/sources.list
sudo apt-get update
sudo apt-get install -y gcc-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross
sudo apt-get download libssl1.1:armhf libssl-dev:armhf
sudo dpkg -x libssl1.1*.deb /cross-build-arm
sudo dpkg -x libssl-dev*.deb /cross-build-arm
rustup target add arm-unknown-linux-gnueabihf
echo "::set-env name=C_INCLUDE_PATH::/cross-build-arm/usr/include"
echo "::set-env name=OPENSSL_INCLUDE_DIR::/cross-build-arm/usr/include/arm-linux-gnueabihf"
echo "::set-env name=OPENSSL_LIB_DIR::/cross-build-arm/usr/lib/arm-linux-gnueabihf"
echo "::set-env name=PKG_CONFIG_ALLOW_CROSS::1"
echo "::set-env name=RUSTFLAGS::-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build-arm/usr/lib/arm-linux-gnueabihf -L/cross-build-arm/lib/arm-linux-gnueabihf"
- name: Build the executable
run: |
cargo build --release --target=arm-unknown-linux-gnueabihf
- name: Attach artifact to the release
uses: Shopify/upload-to-release@1.0.0
with:
name: monolith-gnu-linux-armhf
path: target/arm-unknown-linux-gnueabihf/release/monolith
repo-token: ${{ secrets.GITHUB_TOKEN }}

30
.github/workflows/ci.yml vendored Normal file
View File

@@ -0,0 +1,30 @@
name: CI
on:
pull_request:
branches: [ master ]
jobs:
build_and_test:
strategy:
matrix:
os:
- ubuntu-latest
- macos-latest
- windows-latest
rust:
- stable
- beta
- nightly
runs-on: ${{ matrix.os }}
steps:
- run: git config --global core.autocrlf false
- uses: actions/checkout@v2
- name: Build
run: cargo build --all --locked --verbose
- name: Run tests
run: cargo test --all --locked --verbose
- name: Check code formatting
run: cargo fmt --all -- --check

View File

@@ -1,37 +0,0 @@
language: rust
cache: cargo
os:
- linux
- osx
- windows
rust:
- stable
- beta
- nightly
git:
autocrlf: false # don't mangle LF into CRLF on windows
before_script:
- rustup component add rustfmt
script:
- cargo build --all --locked --verbose
- cargo test --all --locked --verbose
- |
if [[ "$TRAVIS_RUST_VERSION" == "stable" ]]; then
cargo fmt --all -- --check
fi
after_success: |
if [ "${TRAVIS_OS_NAME}" == linux ] && [ "${TRAVIS_RUST_VERSION}" == stable ]; then
docker build -t monolith .
docker run monolith monolith -V
fi
jobs:
allow_failures:
- rust: nightly
fast_finish: true

10
Cargo.lock generated
View File

@@ -61,6 +61,11 @@ name = "base64"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "base64"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "bitflags"
version = "1.2.1"
@@ -568,10 +573,10 @@ dependencies = [
[[package]]
name = "monolith"
version = "2.2.1"
version = "2.2.4"
dependencies = [
"assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"cssparser 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)",
"html5ever 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1501,6 +1506,7 @@ dependencies = [
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
"checksum base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d5ca2cd0adc3f48f9e9ea5a6bbdf9ccc0bfade884847e484d452414c7ccffb3"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
"checksum bumpalo 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fe2567a8d8a3aedb4e39aa39e186d5673acfd56393c6ac83b2bc5bd82f4369c"
"checksum bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "10004c15deb332055f7a4a208190aed362cf9a7c2f6ab70a305fba50e1105f38"

View File

@@ -1,6 +1,6 @@
[package]
name = "monolith"
version = "2.2.1"
version = "2.2.4"
edition = "2018"
authors = [
"Sunshine <sunshine@uberspace.net>",
@@ -12,7 +12,7 @@ authors = [
description = "CLI tool for saving web pages as a single HTML file"
[dependencies]
base64 = "0.11.0"
base64 = "0.12.0"
clap = "2.33.0"
cssparser = "0.27.2"
html5ever = "0.24.1"

View File

@@ -1,21 +1,25 @@
#!/usr/bin/make -f
# Makefile for monolith
all: test
all: build
.PHONY: all
build:
@cargo build --locked
.PHONY: build
install:
@cargo install --force --locked --path .
.PHONY: install
test: build
@cargo test --locked
@cargo fmt --all -- --check
.PHONY: test
.PHONY: test_code_formatting
lint:
@cargo fmt --all --
.PHONY: lint
install:
@cargo install --force --locked --path .
.PHONY: install
uninstall:
@cargo uninstall
.PHONY: uninstall

View File

@@ -1,5 +1,6 @@
[![Travis CI Build Status](https://travis-ci.org/Y2Z/monolith.svg?branch=master)](https://travis-ci.org/Y2Z/monolith)
[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/ae7soyjih8jg2bv7/branch/master?svg=true)](https://ci.appveyor.com/project/snshn/monolith/branch/master)
[![Monolith Build Status for GNU/Linux](https://github.com/Y2Z/monolith/workflows/GNU%2FLinux/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AGNU%2FLinux)
[![Monolith Build Status for macOS](https://github.com/Y2Z/monolith/workflows/macOS/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AmacOS)
[![Monolith Build Status for Windows](https://github.com/Y2Z/monolith/workflows/Windows/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AWindows)
```
___ ___________ __________ ___________________ ___
@@ -11,9 +12,9 @@
|___| |__________| \____________________| |___| |___| |___|
```
A data hoarder's dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
A data hoarders dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
Unlike the conventional "Save page as", `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.
Unlike the conventional Save page as, `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.
If compared to saving websites with `wget -mpk`, this tool embeds all assets as data URLs and therefore lets browsers render the saved page exactly the way it was on the Internet, even when no network connection is available.
@@ -21,18 +22,21 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
## Installation
#### From source
$ git clone https://github.com/Y2Z/monolith.git
$ cd monolith
$ make install
#### With Homebrew (on macOS and GNU/Linux)
#### Via Homebrew (on macOS and GNU/Linux)
$ brew install monolith
#### Using Snapcraft (on GNU/Linux)
$ snap install monolith
#### Via Docker
#### From source
Dependency: `libssl-dev`
$ git clone https://github.com/Y2Z/monolith.git
$ cd monolith
$ make install
#### With Docker
The guide can be found [here](docs/containers.md)
---------------------------------------------------
@@ -45,6 +49,7 @@ The guide can be found [here](docs/containers.md)
## Options
- `-c`: Ignore styles
- `-f`: Exclude frames and iframes
- `-F`: Omit web fonts
- `-i`: Remove images
- `-I`: Isolate the document
- `-j`: Exclude JavaScript
@@ -79,4 +84,4 @@ The Unlicense
---------------------------------------------------
<!-- Microtext -->
<sub>Keep in mind that `monolith` is not aware of your browser's session</sub>
<sub>Keep in mind that `monolith` is not aware of your browsers session</sub>

View File

@@ -0,0 +1,19 @@
# 2. NOSCRIPT nodes
Date: 2020-04-16
## Status
Accepted
## Context
HTML pages sometimes contain NOSCRIPT nodes, which reveal their contents only in case when JavaScript is not available. Most of the time they contain hidden messages that inform about certain JavaScript-dependent features not being operational, however sometimes can also feature media assets or even iframes.
## Decision
When the document is being saved with or without JavaScript, each NOSCRIPT node should be preserved while its children need to be processed exactly the same way as the rest of the document. This approach will ensure that even hidden remote assets are embedded — since those hidden elements may have to be displayed later in a browser that has JavaScript turned off. An option should be available to "unwrap" all NOSCRIPT nodes in order to make their contents always visible in the document, complimenting the "disable JS" function of the program.
## Consequences
Saved documents will have contents of all NOSCRIPT nodes processed as if they are part of the document's DOM, therefore properly display images encapsulated within NOSCRIPT nodes when being viewed in browsers that have JavaScript turned off (or have no JavaScript support in the first place). The new option to "unwrap" NOSCRIPT elements will help the user ensure that the resulting document always represents what the original web page looked like in a browser that had JavaScript turned off.

View File

@@ -21,5 +21,5 @@ saved by monolith, if needed.
## Consequences
Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only a minimal
Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only minimal
amount of modifications to the original web page — whatever is needed to provide security or exclude unwanted asset types.

View File

@@ -2,8 +2,9 @@ use clap::{App, Arg};
#[derive(Default)]
pub struct AppArgs {
pub url_target: String,
pub target: String,
pub no_css: bool,
pub no_fonts: bool,
pub no_frames: bool,
pub no_images: bool,
pub no_js: bool,
@@ -26,15 +27,16 @@ impl AppArgs {
.author(crate_authors!("\n"))
.about(crate_description!())
.arg(
Arg::with_name("url")
Arg::with_name("target")
.required(true)
.takes_value(true)
.index(1)
.help("URL to download"),
.help("URL or file path"),
)
// .args_from_usage("-a, --include-audio 'Removes audio sources'")
.args_from_usage("-c, --no-css 'Removes CSS'")
.args_from_usage("-f, --no-frames 'Removes frames and iframes'")
.args_from_usage("-F, --no-fonts 'Removes fonts'")
.args_from_usage("-i, --no-images 'Removes images'")
.args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
.args_from_usage("-j, --no-js 'Removes JavaScript'")
@@ -47,11 +49,12 @@ impl AppArgs {
.get_matches();
let mut app_args = AppArgs::default();
// Process the command
app_args.url_target = app
.value_of("url")
.expect("please set target url")
app_args.target = app
.value_of("target")
.expect("please set target")
.to_string();
app_args.no_css = app.is_present("no-css");
app_args.no_fonts = app.is_present("no-fonts");
app_args.no_frames = app.is_present("no-frames");
app_args.no_images = app.is_present("no-images");
app_args.no_js = app.is_present("no-js");

View File

@@ -2,12 +2,12 @@ use cssparser::{ParseError, Parser, ParserInput, SourcePosition, Token};
use reqwest::blocking::Client;
use std::collections::HashMap;
use crate::utils::{data_to_data_url, decode_url, get_url_fragment, resolve_url, retrieve_asset};
use crate::utils::{data_to_data_url, get_url_fragment, resolve_url, retrieve_asset};
const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
// Universal
"background",
"background-image",
"border",
"border-image",
"border-image-source",
"content",
@@ -16,10 +16,15 @@ const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
"list-style-image",
"mask",
"mask-image",
// Specific to @counter-style
"additive-symbols",
"negative",
"pad",
"prefix",
"suffix",
"symbols",
];
const TRANSPARENT_PIXEL: &str = "data:image/png;base64,\
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
const CSS_SPECIAL_CHARS: &str = "~!@$%^&*()+=,./'\";:?><[]{}|`#";
pub fn is_image_url_prop(prop_name: &str) -> bool {
CSS_PROPS_WITH_IMAGE_URLS
@@ -36,14 +41,27 @@ pub fn enquote(input: String, double: bool) -> String {
}
}
pub fn escape(value: &str) -> String {
let mut res = str!(&value);
res = res.replace("\\", "\\\\");
for c in CSS_SPECIAL_CHARS.chars() {
res = res.replace(c, format!("\\{}", c).as_str());
}
res
}
pub fn process_css<'a>(
cache: &mut HashMap<String, String>,
cache: &mut HashMap<String, Vec<u8>>,
client: &Client,
parent_url: &str,
parser: &mut Parser,
rule_name: &str,
prop_name: &str,
func_name: &str,
opt_no_fonts: bool,
opt_no_images: bool,
opt_silent: bool,
) -> Result<String, ParseError<'a, String>> {
@@ -72,6 +90,10 @@ pub fn process_css<'a>(
Token::Colon => result.push_str(":"),
Token::Comma => result.push_str(","),
Token::ParenthesisBlock | Token::SquareBracketBlock | Token::CurlyBracketBlock => {
if opt_no_fonts && curr_rule == "font-face" {
continue;
}
let closure: &str;
if token == &Token::ParenthesisBlock {
result.push_str("(");
@@ -94,6 +116,7 @@ pub fn process_css<'a>(
rule_name,
curr_prop.as_str(),
func_name,
opt_no_fonts,
opt_no_images,
opt_silent,
)
@@ -116,12 +139,18 @@ pub fn process_css<'a>(
Token::WhiteSpace(ref value) => {
result.push_str(value);
}
// div...
Token::Ident(ref value) => {
curr_rule = str!();
curr_prop = str!(value);
result.push_str(value);
result.push_str(&escape(value));
}
// @import, @font-face, @charset, @media...
Token::AtKeyword(ref value) => {
curr_rule = str!(value);
if opt_no_fonts && curr_rule == "font-face" {
continue;
}
result.push_str("@");
result.push_str(value);
}
@@ -130,13 +159,10 @@ pub fn process_css<'a>(
result.push_str(value);
}
Token::QuotedString(ref value) => {
let is_import: bool = curr_rule == "import";
if is_import {
if curr_rule == "import" {
// Reset current at-rule value
curr_rule = str!();
}
if is_import {
// Skip empty import values
if value.len() < 1 {
result.push_str("''");
@@ -145,12 +171,11 @@ pub fn process_css<'a>(
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
let url_fragment = get_url_fragment(full_url.clone());
let full_url_decoded = decode_url(full_url);
let (css, final_url) = retrieve_asset(
cache,
client,
&parent_url,
&full_url_decoded,
&full_url,
false,
"",
opt_silent,
@@ -166,6 +191,7 @@ pub fn process_css<'a>(
client,
final_url.as_str(),
&css,
opt_no_fonts,
opt_no_images,
opt_silent,
)
@@ -185,7 +211,7 @@ pub fn process_css<'a>(
}
if opt_no_images && is_image_url_prop(curr_prop.as_str()) {
result.push_str(enquote(str!(TRANSPARENT_PIXEL), false).as_str());
result.push_str(enquote(str!(empty_image!()), false).as_str());
} else {
let resolved_url = resolve_url(&parent_url, value).unwrap_or_default();
let (data_url, _final_url) = retrieve_asset(
@@ -220,26 +246,33 @@ pub fn process_css<'a>(
ref unit_value,
..
} => {
if *has_sign {
result.push_str("-");
if *has_sign && *unit_value >= 0. {
result.push_str("+");
}
result.push_str(str!(unit_value * 100.).as_str());
result.push_str("%");
}
Token::Dimension {
ref has_sign,
ref value,
ref unit,
..
} => {
if *has_sign && *value >= 0. {
result.push_str("+");
}
result.push_str(str!(value).as_str());
result.push_str(str!(unit).as_str());
}
// #selector, #id...
Token::IDHash(ref value) => {
curr_rule = str!();
result.push_str("#");
result.push_str(value);
result.push_str(&escape(value));
}
Token::UnquotedUrl(ref value) => {
let is_import: bool = curr_rule == "import";
if is_import {
// Reset current at-rule value
curr_rule = str!();
@@ -260,12 +293,11 @@ pub fn process_css<'a>(
if is_import {
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
let url_fragment = get_url_fragment(full_url.clone());
let full_url_decoded = decode_url(full_url);
let (css, final_url) = retrieve_asset(
cache,
client,
&parent_url,
&full_url_decoded,
&full_url,
false,
"",
opt_silent,
@@ -281,6 +313,7 @@ pub fn process_css<'a>(
client,
final_url.as_str(),
&css,
opt_no_fonts,
opt_no_images,
opt_silent,
)
@@ -294,7 +327,7 @@ pub fn process_css<'a>(
);
} else {
if opt_no_images && is_image_url_prop(curr_prop.as_str()) {
result.push_str(enquote(str!(TRANSPARENT_PIXEL), false).as_str());
result.push_str(enquote(str!(empty_image!()), false).as_str());
} else {
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
let (data_url, _final_url) = retrieve_asset(
@@ -328,6 +361,7 @@ pub fn process_css<'a>(
curr_rule.as_str(),
curr_prop.as_str(),
function_name,
opt_no_fonts,
opt_no_images,
opt_silent,
)
@@ -345,10 +379,11 @@ pub fn process_css<'a>(
}
pub fn embed_css(
cache: &mut HashMap<String, String>,
cache: &mut HashMap<String, Vec<u8>>,
client: &Client,
parent_url: &str,
css: &str,
opt_no_fonts: bool,
opt_no_images: bool,
opt_silent: bool,
) -> String {
@@ -363,6 +398,7 @@ pub fn embed_css(
"",
"",
"",
opt_no_fonts,
opt_no_images,
opt_silent,
)

View File

@@ -20,38 +20,33 @@ const ICON_VALUES: &[&str] = &[
"fluid-icon",
];
const TRANSPARENT_PIXEL: &str = "data:image/png;base64,\
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=";
pub fn get_parent_node(node: &Handle) -> Handle {
let parent = node.parent.take().clone();
parent.and_then(|node| node.upgrade()).unwrap()
}
pub fn get_node_name(node: &Handle) -> &'_ str {
pub fn get_node_name(node: &Handle) -> Option<&'_ str> {
match &node.data {
NodeData::Element { ref name, .. } => name.local.as_ref(),
_ => "",
NodeData::Element { ref name, .. } => Some(name.local.as_ref()),
_ => None,
}
}
pub fn is_icon(attr_value: &str) -> bool {
ICON_VALUES
.iter()
.find(|a| attr_value.eq_ignore_ascii_case(a))
.is_some()
ICON_VALUES.contains(&attr_value.to_lowercase().as_str())
}
pub fn walk_and_embed_assets(
cache: &mut HashMap<String, String>,
cache: &mut HashMap<String, Vec<u8>>,
client: &Client,
url: &str,
node: &Handle,
opt_no_css: bool,
opt_no_fonts: bool,
opt_no_frames: bool,
opt_no_js: bool,
opt_no_images: bool,
opt_silent: bool,
opt_no_frames: bool,
) {
match node.data {
NodeData::Document => {
@@ -63,10 +58,11 @@ pub fn walk_and_embed_assets(
&url,
child,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
}
}
@@ -163,17 +159,18 @@ pub fn walk_and_embed_assets(
) {
// On successful retrieval, traverse CSS
Ok((css_data, final_url)) => {
let x: String = embed_css(
let css: String = embed_css(
cache,
client,
&final_url,
&css_data,
opt_no_fonts,
opt_no_images,
opt_silent,
);
data_to_data_url(
"text/css",
x.as_bytes(),
css.as_bytes(),
&final_url,
"",
)
@@ -273,7 +270,7 @@ pub fn walk_and_embed_assets(
if opt_no_images {
attrs_mut.push(Attribute {
name: QualName::new(None, ns!(), local_name!("src")),
value: Tendril::from_slice(TRANSPARENT_PIXEL),
value: Tendril::from_slice(empty_image!()),
});
} else if let Some((data_url, _)) = found_datasrc
.iter()
@@ -300,6 +297,63 @@ pub fn walk_and_embed_assets(
});
}
}
"input" => {
let mut is_image: bool = false;
for attr in attrs_mut.iter_mut() {
let attr_name: &str = &attr.name.local;
if attr_name == "type" {
is_image = attr.value.to_string().eq_ignore_ascii_case("image");
}
}
if is_image {
let mut found_src: Option<Attribute> = None;
let mut i = 0;
while i < attrs_mut.len() {
let attr_name = attrs_mut[i].name.local.as_ref();
if attr_name.eq_ignore_ascii_case("src") {
found_src = Some(attrs_mut.remove(i));
} else {
i += 1;
}
}
// If images are disabled, clear both sources
if opt_no_images {
attrs_mut.push(Attribute {
name: QualName::new(None, ns!(), local_name!("src")),
value: Tendril::from_slice(empty_image!()),
});
} else if let Some((data_url, _)) = found_src
.iter()
.map(|attr| attr.value.trim())
.filter(|src| !src.is_empty()) // Skip if empty
.next()
.and_then(|src| resolve_url(&url, src).ok()) // Make absolute
.and_then(|abs_src| // Download and convert to data_url
retrieve_asset(
cache,
client,
&url,
&abs_src,
true,
"",
opt_silent,
).ok())
{
// Add new data_url src attribute
attrs_mut.push(Attribute {
name: QualName::new(None, ns!(), local_name!("src")),
value: Tendril::from_slice(data_url.as_ref()),
});
}
}
}
"svg" => {
if opt_no_images {
node.children.borrow_mut().clear();
}
}
"source" => {
for attr in attrs_mut.iter_mut() {
let attr_name: &str = &attr.name.local;
@@ -310,10 +364,10 @@ pub fn walk_and_embed_assets(
attr.value.clear();
attr.value.push_slice(src_full_url.as_str());
} else if attr_name == "srcset" {
if get_node_name(&get_parent_node(&node)) == "picture" {
if get_node_name(&get_parent_node(&node)) == Some("picture") {
if opt_no_images {
attr.value.clear();
attr.value.push_slice(TRANSPARENT_PIXEL);
attr.value.push_slice(empty_image!());
} else {
let srcset_full_url =
resolve_url(&url, attr.value.trim()).unwrap_or_default();
@@ -334,7 +388,7 @@ pub fn walk_and_embed_assets(
}
}
}
"a" => {
"a" | "area" => {
for attr in attrs_mut.iter_mut() {
if &attr.name.local == "href" {
let attr_value = attr.value.trim();
@@ -411,6 +465,7 @@ pub fn walk_and_embed_assets(
client,
&url,
tendril.as_ref(),
opt_no_fonts,
opt_no_images,
opt_silent,
);
@@ -468,10 +523,11 @@ pub fn walk_and_embed_assets(
&frame_final_url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -539,6 +595,7 @@ pub fn walk_and_embed_assets(
client,
&url,
attribute.value.as_ref(),
opt_no_fonts,
opt_no_images,
opt_silent,
);
@@ -570,10 +627,11 @@ pub fn walk_and_embed_assets(
&url,
child,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
}
}

View File

@@ -7,3 +7,11 @@ macro_rules! str {
ToString::to_string(&$val)
};
}
#[macro_export]
macro_rules! empty_image {
() => {
"data:image/png;base64,\
iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII="
};
}

View File

@@ -47,7 +47,7 @@ impl Output {
fn main() {
let app_args = AppArgs::get();
let original_target: &str = &app_args.url_target;
let original_target: &str = &app_args.target;
let target_url: &str;
let base_url;
let dom;
@@ -122,9 +122,9 @@ fn main() {
base_url = final_url;
dom = html_to_dom(&data);
} else if is_data_url(target_url) {
let text: String = data_url_to_text(target_url);
if text.len() == 0 {
eprintln!("Unsupported data URL input");
let (media_type, text): (String, String) = data_url_to_text(target_url);
if !media_type.eq_ignore_ascii_case("text/html") {
eprintln!("Unsupported data URL media type");
process::exit(1);
}
base_url = str!(target_url);
@@ -139,10 +139,11 @@ fn main() {
&base_url,
&dom.document,
app_args.no_css,
app_args.no_fonts,
app_args.no_frames,
app_args.no_js,
app_args.no_images,
app_args.silent,
app_args.no_frames,
);
let html: String = stringify_document(

View File

@@ -62,7 +62,7 @@ fn passing_bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
// STDERR should contain error description
assert_eq!(
std::str::from_utf8(&out.stderr).unwrap(),
"Unsupported data URL input\n"
"Unsupported data URL media type\n"
);
// The exit code should be 1
@@ -162,15 +162,18 @@ fn passing_remove_images_from_data_url() -> Result<(), Box<dyn std::error::Error
// STDOUT should contain HTML with no images
assert_eq!(
std::str::from_utf8(&out.stdout).unwrap(),
"<html>\
format!(
"<html>\
<head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"img-src data:;\"></meta>\
</head>\
<body>\
<img src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\
<img src=\"{empty_image}\">\
Hi\
</body>\
</html>\n"
</html>\n",
empty_image = empty_image!()
)
);
// STDERR should be empty
@@ -229,7 +232,8 @@ fn passing_local_file_target_input() -> Result<(), Box<dyn std::error::Error>> {
// STDOUT should contain HTML from the local file
assert_eq!(
std::str::from_utf8(&out.stdout).unwrap(),
"<!DOCTYPE html><html lang=\"en\"><head>\n \
"\
<!DOCTYPE html><html lang=\"en\"><head>\n \
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n \
<title>Local HTML file</title>\n \
<link href=\"data:text/css;base64,Ym9keSB7CiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjMDAwOwogICAgY29sb3I6ICNmZmY7Cn0K\" rel=\"stylesheet\" type=\"text/css\">\n \
@@ -238,16 +242,19 @@ fn passing_local_file_target_input() -> Result<(), Box<dyn std::error::Error>> {
<a href=\"file://local-file.html/\">Tricky href</a>\n \
<a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n \
<script src=\"data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==\"></script>\n\n\n\n\
</body></html>\n"
</body></html>\n\
"
);
// STDERR should contain list of retrieved file URLs
assert_eq!(
std::str::from_utf8(&out.stderr).unwrap(),
format!(
"{file}{cwd}/src/tests/data/local-file.html\n\
"\
{file}{cwd}/src/tests/data/local-file.html\n\
{file}{cwd}/src/tests/data/local-style.css\n\
{file}{cwd}/src/tests/data/local-script.js\n",
{file}{cwd}/src/tests/data/local-script.js\n\
",
file = file_url_protocol,
cwd = cwd_normalized
)
@@ -286,17 +293,22 @@ fn passing_local_file_target_input_absolute_target_path() -> Result<(), Box<dyn
// STDOUT should contain HTML from the local file
assert_eq!(
std::str::from_utf8(&out.stdout).unwrap(),
"<!DOCTYPE html><html lang=\"en\"><head>\
format!(
"\
<!DOCTYPE html><html lang=\"en\"><head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:; style-src 'none'; script-src 'none'; img-src data:;\"></meta>\n \
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n \
<title>Local HTML file</title>\n \
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n \
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
<img alt=\"\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\n \
<img alt=\"\" src=\"{empty_image}\">\n \
<a href=\"file://local-file.html/\">Tricky href</a>\n \
<a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n \
<script src=\"\"></script>\n\n\n\n\
</body></html>\n"
</body></html>\n\
",
empty_image = empty_image!()
)
);
// STDERR should contain only the target file
@@ -342,17 +354,22 @@ fn passing_local_file_url_target_input() -> Result<(), Box<dyn std::error::Error
// STDOUT should contain HTML from the local file
assert_eq!(
std::str::from_utf8(&out.stdout).unwrap(),
"<!DOCTYPE html><html lang=\"en\"><head>\
format!(
"\
<!DOCTYPE html><html lang=\"en\"><head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none'; script-src 'none'; img-src data:;\"></meta>\n \
<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n \
<title>Local HTML file</title>\n \
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n \
<link href=\"\" rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n \
<img alt=\"\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\n \
<img alt=\"\" src=\"{empty_image}\">\n \
<a href=\"file://local-file.html/\">Tricky href</a>\n \
<a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n \
<script src=\"\"></script>\n\n\n\n\
</body></html>\n"
</body></html>\n\
",
empty_image = empty_image!()
)
);
// STDERR should contain list of retrieved file URLs
@@ -458,7 +475,8 @@ fn passing_css_import_string() -> Result<(), Box<dyn std::error::Error>> {
let mut file_html = NamedTempFile::new()?;
writeln!(
file_html,
"<style>\n\
"\
<style>\n\
@charset 'UTF-8';\n\
\n\
@import '{file}{css_path}';\n\
@@ -466,7 +484,8 @@ fn passing_css_import_string() -> Result<(), Box<dyn std::error::Error>> {
@import url({file}{css_path});\n\
\n\
@import url('{file}{css_path}')\n\
</style>\n",
</style>\n\
",
file = file_url_prefix,
css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
)?;

View File

@@ -15,7 +15,10 @@ fn passing_empty_input() {
let cache = &mut HashMap::new();
let client = Client::new();
assert_eq!(css::embed_css(cache, &client, "", "", false, false,), "");
assert_eq!(
css::embed_css(cache, &client, "", "", false, false, false,),
""
);
}
#[test]
@@ -37,16 +40,20 @@ height: calc(100vh - 10pt)";
&client,
"https://doesntmatter.local/",
&STYLE,
false,
true,
true,
),
"/* border: none;*/\
background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='); \
list-style: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=');\
format!(
"/* border: none;*/\
background-image: url('{empty_image}'); \
list-style: url('{empty_image}');\
width:99.998%; \
margin-top: -20px; \
line-height: -1; \
height: calc(100vh - 10pt)"
height: calc(100vh - 10pt)",
empty_image = empty_image!()
)
);
}
@@ -64,21 +71,17 @@ line-height: -1; \
height: calc(100vh - 10pt)";
assert_eq!(
css::embed_css(
cache,
&client,
"",
&STYLE,
true,
true,
),
"/* border: none;*/\
background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII='); \
list-style: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=');\
css::embed_css(cache, &client, "", &STYLE, false, true, true,),
format!(
"/* border: none;*/\
background-image: url('{empty_image}'); \
list-style: url('{empty_image}');\
width:99.998%; \
margin-top: -20px; \
line-height: -1; \
height: calc(100vh - 10pt)"
height: calc(100vh - 10pt)",
empty_image = empty_image!()
)
);
}
@@ -96,7 +99,7 @@ fn passing_style_block() {
html > body {}";
assert_eq!(
css::embed_css(cache, &client, "file:///", &CSS, false, true,),
css::embed_css(cache, &client, "file:///", &CSS, false, false, true,),
CSS
);
}
@@ -136,7 +139,10 @@ fn passing_attribute_selectors() {
}
";
assert_eq!(css::embed_css(cache, &client, "", &CSS, false, false,), CSS);
assert_eq!(
css::embed_css(cache, &client, "", &CSS, false, false, false,),
CSS
);
}
#[test]
@@ -159,14 +165,15 @@ fn passing_import_string() {
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
),
"\
@charset 'UTF-8';\n\
\n\
@import 'data:text/css;base64,ZGF0YTp0ZXh0L2NzcyxodG1se2JhY2tncm91bmQtY29sb3I6IzAwMH0=';\n\
@import 'data:text/css;base64,aHRtbHtiYWNrZ3JvdW5kLWNvbG9yOiMwMDB9';\n\
\n\
@import url('data:text/css;base64,ZGF0YTp0ZXh0L2NzcyxodG1se2NvbG9yOiNmZmZ9')\n\
@import url('data:text/css;base64,aHRtbHtjb2xvcjojZmZmfQ==')\n\
"
);
}
@@ -193,8 +200,118 @@ body {\n \
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
),
CSS
);
}
#[test]
fn passing_transform_percentages_and_degrees() {
let cache = &mut HashMap::new();
let client = Client::new();
const CSS: &str = "\
div {\n \
transform: translate(-50%, -50%) rotate(-45deg);\n\
transform: translate(50%, 50%) rotate(45deg);\n\
transform: translate(+50%, +50%) rotate(+45deg);\n\
}\n\
";
assert_eq!(
css::embed_css(
cache,
&client,
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
),
CSS
);
}
#[test]
fn passing_unusual_indents() {
let cache = &mut HashMap::new();
let client = Client::new();
const CSS: &str = "\
.is\\:good:hover {\n \
color: green\n\
}\n\
\n\
#\\~\\!\\@\\$\\%\\^\\&\\*\\(\\)\\+\\=\\,\\.\\/\\\\\\'\\\"\\;\\:\\?\\>\\<\\[\\]\\{\\}\\|\\`\\# {\n \
color: black\n\
}\n\
";
assert_eq!(
css::embed_css(
cache,
&client,
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
),
CSS
);
}
#[test]
fn passing_exclude_fonts() {
let cache = &mut HashMap::new();
let client = Client::new();
const CSS: &str = "\
@font-face {\n \
font-family: 'My Font';\n \
src: url(my_font.woff);\n\
}\n\
\n\
#identifier {\n \
font-family: 'My Font' Arial\n\
}\n\
\n\
@font-face {\n \
font-family: 'My Font';\n \
src: url(my_font.woff);\n\
}\n\
\n\
div {\n \
font-family: 'My Font' Verdana\n\
}\n\
";
const CSS_OUT: &str = " \
\n\
\n\
#identifier {\n \
font-family: 'My Font' Arial\n\
}\n\
\n \
\n\
\n\
div {\n \
font-family: 'My Font' Verdana\n\
}\n\
";
assert_eq!(
css::embed_css(
cache,
&client,
"https://doesntmatter.local/",
&CSS,
true,
false,
true,
),
CSS_OUT
);
}

View File

@@ -28,11 +28,11 @@ fn get_node_name() {
let parent = html::get_parent_node(node);
let parent_node_name = html::get_node_name(&parent);
if node_name == "head" || node_name == "body" {
assert_eq!(parent_node_name, "html");
assert_eq!(parent_node_name, Some("html"));
} else if node_name == "div" {
assert_eq!(parent_node_name, "body");
assert_eq!(parent_node_name, Some("body"));
} else if node_name == "p" {
assert_eq!(parent_node_name, "div");
assert_eq!(parent_node_name, Some("div"));
}
for child in node.children.borrow().iter() {

View File

@@ -19,6 +19,7 @@ fn passing_basic() {
let url = "http://localhost";
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
@@ -32,10 +33,11 @@ fn passing_basic() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -55,6 +57,7 @@ fn passing_ensure_no_recursive_iframe() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
@@ -68,10 +71,11 @@ fn passing_ensure_no_recursive_iframe() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -91,6 +95,7 @@ fn passing_ensure_no_recursive_frame() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
@@ -104,10 +109,11 @@ fn passing_ensure_no_recursive_frame() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -129,6 +135,7 @@ fn passing_no_css() {
let cache = &mut HashMap::new();
let opt_no_css: bool = true;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
@@ -141,10 +148,11 @@ fn passing_no_css() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -173,6 +181,7 @@ fn passing_no_images() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = true;
@@ -186,10 +195,11 @@ fn passing_no_images() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -197,18 +207,19 @@ fn passing_no_images() {
assert_eq!(
buf.iter().map(|&c| c as char).collect::<String>(),
"<html>\
format!(
"<html>\
<head>\
<link rel=\"icon\" href=\"\">\
</head>\
<body>\
<div>\
<img src=\"data:image/png;base64,\
iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0\
lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\
<img src=\"{empty_image}\">\
</div>\
</body>\
</html>"
</html>",
empty_image = empty_image!()
)
);
}
@@ -220,6 +231,7 @@ fn passing_no_body_background_images() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = true;
@@ -233,10 +245,11 @@ fn passing_no_body_background_images() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -256,6 +269,7 @@ fn passing_no_frames() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
@@ -268,10 +282,11 @@ fn passing_no_frames() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -291,6 +306,7 @@ fn passing_no_iframes() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
@@ -303,10 +319,11 @@ fn passing_no_iframes() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -329,6 +346,7 @@ fn passing_no_js() {
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = true;
let opt_no_images: bool = false;
@@ -342,10 +360,11 @@ fn passing_no_js() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();
@@ -368,6 +387,7 @@ fn passing_with_no_integrity() {
let cache = &mut HashMap::new();
let client = Client::new();
let opt_no_css: bool = true;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = true;
let opt_no_images: bool = true;
@@ -379,10 +399,11 @@ fn passing_with_no_integrity() {
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
opt_no_frames,
);
let mut buf: Vec<u8> = Vec::new();

View File

@@ -0,0 +1,14 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
#[test]
fn contains_correct_image_data() {
assert_eq!(empty_image!(), "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII=");
}
}

2
src/tests/macros/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
mod empty_image;
mod str;

24
src/tests/macros/str.rs Normal file
View File

@@ -0,0 +1,24 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
#[test]
fn returns_empty_string() {
assert_eq!(str!(), "");
}
#[test]
fn converts_integer_into_string() {
assert_eq!(str!(123), "123");
}
#[test]
fn converts_str_into_string() {
assert_eq!(str!("abc"), "abc");
}
}

View File

@@ -2,4 +2,5 @@ mod cli;
mod css;
mod html;
mod js;
mod macros;
mod utils;

View File

@@ -30,3 +30,11 @@ fn passing_removes_empty_query_and_empty_fragment() {
"https://somewhere.com/font.eot"
);
}
#[test]
fn passing_removes_empty_query_amp_and_empty_fragment() {
assert_eq!(
utils::clean_url("https://somewhere.com/font.eot?a=b&#"),
"https://somewhere.com/font.eot?a=b"
);
}

View File

@@ -9,42 +9,76 @@ use crate::utils;
#[test]
fn passing_parse_text_html_base64() {
let (media_type, text) = utils::data_url_to_text("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
assert_eq!(media_type, "text/html");
assert_eq!(
utils::data_url_to_text("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg=="),
text,
"Work expands so as to fill the time available for its completion"
);
}
#[test]
fn passing_parse_text_html_utf8() {
let (media_type, text) = utils::data_url_to_text(
"data:text/html;utf8,Work expands so as to fill the time available for its completion",
);
assert_eq!(media_type, "text/html");
assert_eq!(
utils::data_url_to_text(
"data:text/html;utf8,Work expands so as to fill the time available for its completion"
),
text,
"Work expands so as to fill the time available for its completion"
);
}
#[test]
fn passing_parse_text_html_plaintext() {
let (media_type, text) = utils::data_url_to_text(
"data:text/html,Work expands so as to fill the time available for its completion",
);
assert_eq!(media_type, "text/html");
assert_eq!(
utils::data_url_to_text(
"data:text/html,Work expands so as to fill the time available for its completion"
),
text,
"Work expands so as to fill the time available for its completion"
);
}
#[test]
fn passing_parse_text_html_charset_utf_8_between_two_whitespaces() {
let (media_type, text) = utils::data_url_to_text(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
assert_eq!(media_type, "text/html");
assert_eq!(
utils::data_url_to_text(
" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion "
),
text,
"Work expands so as to fill the time available for its completion"
);
}
#[test]
fn passing_parse_text_css_url_encoded() {
let (media_type, text) = utils::data_url_to_text("data:text/css,div{background-color:%23000}");
assert_eq!(media_type, "text/css");
assert_eq!(text, "div{background-color:#000}");
}
#[test]
fn passing_parse_no_media_type_base64() {
let (media_type, text) = utils::data_url_to_text("data:;base64,dGVzdA==");
assert_eq!(media_type, "");
assert_eq!(text, "test");
}
#[test]
fn passing_parse_no_media_type_no_encoding() {
let (media_type, text) = utils::data_url_to_text("data:;,test%20test");
assert_eq!(media_type, "");
assert_eq!(text, "test test");
}
// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗
// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝
// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗
@@ -54,5 +88,8 @@ fn passing_parse_text_html_charset_utf_8_between_two_whitespaces() {
#[test]
fn failing_just_word_data() {
assert_eq!(utils::data_url_to_text("data"), "");
let (media_type, text) = utils::data_url_to_text("data");
assert_eq!(media_type, "");
assert_eq!(text, "");
}

View File

@@ -24,3 +24,13 @@ fn passing_decode_file_url() {
"file:///tmp/space here/test#1.html"
);
}
#[test]
fn passing_plus_sign() {
assert_eq!(
utils::decode_url(str!(
"fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
)),
"fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
);
}

View File

@@ -21,3 +21,18 @@ fn passing_remove_protocl_and_fragment() {
);
}
}
#[test]
fn passing_decodes_urls() {
if cfg!(windows) {
assert_eq!(
utils::file_url_to_fs_path("file:///C:/Documents%20and%20Settings/some-file.html"),
"C:\\Documents and Settings\\some-file.html"
);
} else {
assert_eq!(
utils::file_url_to_fs_path("file:///home/user/My%20Documents"),
"/home/user/My Documents"
);
}
}

View File

@@ -30,6 +30,14 @@ const MAGIC: [[&[u8]; 2]; 18] = [
[b"\x1A\x45\xDF\xA3", b"video/webm"],
];
const PLAINTEXT_MEDIA_TYPES: &[&str] = &[
"image/svg+xml",
"text/css",
"text/html",
"text/javascript",
"text/plain",
];
pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str, fragment: &str) -> String {
let media_type: String = if media_type.is_empty() {
detect_media_type(data, &url)
@@ -88,6 +96,10 @@ pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
.unwrap_or(false)
}
pub fn is_plaintext_media_type(media_type: &str) -> bool {
PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
}
pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
let result = if is_http_url(to.as_ref()) {
to.as_ref().to_string()
@@ -108,65 +120,72 @@ pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
}
}
pub fn clean_url<T: AsRef<str>>(url: T) -> String {
let mut result = Url::parse(url.as_ref()).unwrap();
pub fn clean_url<T: AsRef<str>>(input: T) -> String {
let mut url = Url::parse(input.as_ref()).unwrap();
// Clear fragment
result.set_fragment(None);
url.set_fragment(None);
// Get rid of stray question mark
if result.query() == Some("") {
result.set_query(None);
if url.query() == Some("") {
url.set_query(None);
}
result.to_string()
// Remove empty trailing ampersand(s)
let mut result: String = url.to_string();
while result.ends_with("&") {
result.pop();
}
result
}
pub fn data_url_to_text<T: AsRef<str>>(url: T) -> String {
let parsed_url = Url::parse(url.as_ref()).unwrap_or(Url::parse("http://[::1]").unwrap());
pub fn data_url_to_text<T: AsRef<str>>(url: T) -> (String, String) {
let parsed_url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
let path: String = parsed_url.path().to_string();
let comma_loc: usize = path.find(',').unwrap_or(path.len());
if comma_loc == path.len() {
return str!();
}
let meta_data: String = path.chars().take(comma_loc).collect();
let raw_data: String = path.chars().skip(comma_loc + 1).collect();
let data: String = decode_url(raw_data);
let meta_data_items: Vec<&str> = meta_data.split(';').collect();
let mut media_type: &str = "";
let mut encoding: &str = "";
let mut media_type: String = str!();
let mut text: String = str!();
let mut i: i8 = 0;
for item in &meta_data_items {
if i == 0 {
if item.eq_ignore_ascii_case("text/html") {
media_type = item;
continue;
media_type = str!(item);
} else {
if item.eq_ignore_ascii_case("base64")
|| item.eq_ignore_ascii_case("utf8")
|| item.eq_ignore_ascii_case("charset=UTF-8")
{
encoding = item;
}
}
if item.eq_ignore_ascii_case("base64") || item.eq_ignore_ascii_case("utf8") {
encoding = item;
}
i = i + 1;
}
if media_type.eq_ignore_ascii_case("text/html") {
if is_plaintext_media_type(&media_type) || media_type.is_empty() {
if encoding.eq_ignore_ascii_case("base64") {
String::from_utf8(base64::decode(&data).unwrap_or(vec![])).unwrap_or(str!())
text = String::from_utf8(base64::decode(&data).unwrap_or(vec![])).unwrap_or(str!())
} else {
data
text = data
}
} else {
str!()
}
(media_type, text)
}
pub fn decode_url(input: String) -> String {
let input: String = input.replace("+", "%2B");
form_urlencoded::parse(input.as_bytes())
.map(|(key, val)| {
[
@@ -200,11 +219,12 @@ pub fn file_url_to_fs_path(url: &str) -> String {
fs_file_path = fs_file_path.replace("/", "\\");
}
fs_file_path
// File paths should not be %-encoded
decode_url(fs_file_path)
}
pub fn retrieve_asset(
cache: &mut HashMap<String, String>,
cache: &mut HashMap<String, Vec<u8>>,
client: &Client,
parent_url: &str,
url: &str,
@@ -216,10 +236,14 @@ pub fn retrieve_asset(
return Ok((str!(), str!()));
}
let cache_key = clean_url(&url);
if is_data_url(&url) {
Ok((url.to_string(), url.to_string()))
if as_data_url {
Ok((url.to_string(), url.to_string()))
} else {
let (_media_type, text) = data_url_to_text(url);
Ok((text, url.to_string()))
}
} else if is_file_url(&url) {
// Check if parent_url is also file:///
// (if not, then we don't embed the asset)
@@ -251,13 +275,25 @@ pub fn retrieve_asset(
Ok((str!(), url.to_string()))
}
} else {
let cache_key: String = clean_url(&url);
if cache.contains_key(&cache_key) {
// URL is in cache
// URL is in cache, we retrieve it
let data = cache.get(&cache_key).unwrap();
if !opt_silent {
eprintln!("{} (from cache)", &url);
}
let data = cache.get(&cache_key).unwrap();
Ok((data.to_string(), url.to_string()))
if as_data_url {
let url_fragment = get_url_fragment(url);
Ok((
data_to_data_url(media_type, data, url, &url_fragment),
url.to_string(),
))
} else {
Ok((String::from_utf8_lossy(data).to_string(), url.to_string()))
}
} else {
// URL not in cache, we request it
let mut response = client.get(url).send()?;
@@ -271,7 +307,7 @@ pub fn retrieve_asset(
}
}
let new_cache_key = clean_url(&res_url);
let new_cache_key: String = clean_url(&res_url);
if as_data_url {
// Convert response into a byte array
@@ -290,13 +326,17 @@ pub fn retrieve_asset(
};
let url_fragment = get_url_fragment(url);
let data_url = data_to_data_url(&media_type, &data, url, &url_fragment);
// Add to cache
cache.insert(new_cache_key, data_url.clone());
cache.insert(new_cache_key, data);
Ok((data_url, res_url))
} else {
let content = response.text().unwrap();
// Add to cache
cache.insert(new_cache_key, content.clone());
cache.insert(new_cache_key, content.as_bytes().to_vec());
Ok((content, res_url))
}
}