Merge pull request #233 from snshn/cargo-fix-license-identifier

Fix license identifier in Cargo.toml

.adr-dir

@@ -0,0 +1 @@
+docs/arch

.appveyor.yml

@@ -1,130 +0,0 @@
-# Appveyor configuration template for Rust using rustup for Rust installation
-# https://github.com/starkat99/appveyor-rust
-
-## Operating System (VM environment) ##
-
-# Rust needs at least Visual Studio 2013 Appveyor OS for MSVC targets.
-os: Visual Studio 2015
-
-## Build Matrix ##
-
-# This configuration will setup a build for each channel & target combination (12 windows
-# combinations in all).
-#
-# There are 3 channels: stable, beta, and nightly.
-#
-# Alternatively, the full version may be specified for the channel to build using that specific
-# version (e.g. channel: 1.5.0)
-#
-# The values for target are the set of windows Rust build targets. Each value is of the form
-#
-# ARCH-pc-windows-TOOLCHAIN
-#
-# Where ARCH is the target architecture, either x86_64 or i686, and TOOLCHAIN is the linker
-# toolchain to use, either msvc or gnu. See https://www.rust-lang.org/downloads.html#win-foot for
-# a description of the toolchain differences.
-# See https://github.com/rust-lang-nursery/rustup.rs/#toolchain-specification for description of
-# toolchains and host triples.
-#
-# Comment out channel/target combos you do not wish to build in CI.
-#
-# You may use the `cargoflags` and `RUSTFLAGS` variables to set additional flags for cargo commands
-# and rustc, respectively. For instance, you can uncomment the cargoflags lines in the nightly
-# channels to enable unstable features when building for nightly. Or you could add additional
-# matrix entries to test different combinations of features.
-environment:
-  matrix:
-
-### MSVC Toolchains ###
-
-  # Stable 64-bit MSVC
-    - channel: stable
-      target: x86_64-pc-windows-msvc
-  # Stable 32-bit MSVC
-    - channel: stable
-      target: i686-pc-windows-msvc
-  # Beta 64-bit MSVC
-    - channel: beta
-      target: x86_64-pc-windows-msvc
-  # Beta 32-bit MSVC
-    - channel: beta
-      target: i686-pc-windows-msvc
-  # Nightly 64-bit MSVC
-    - channel: nightly
-      target: x86_64-pc-windows-msvc
-      #cargoflags: --features "unstable"
-  # Nightly 32-bit MSVC
-    - channel: nightly
-      target: i686-pc-windows-msvc
-      #cargoflags: --features "unstable"
-
-### GNU Toolchains ###
-
-  # Stable 64-bit GNU
-    - channel: stable
-      target: x86_64-pc-windows-gnu
-      MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
-  # Stable 32-bit GNU
-    - channel: stable
-      target: i686-pc-windows-gnu
-      MINGW_PATH: 'C:\MinGW\bin'
-  # Beta 64-bit GNU
-    - channel: beta
-      target: x86_64-pc-windows-gnu
-      MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
-  # Beta 32-bit GNU
-    - channel: beta
-      target: i686-pc-windows-gnu
-      MINGW_PATH: 'C:\MinGW\bin'
-  # Nightly 64-bit GNU
-    - channel: nightly
-      target: x86_64-pc-windows-gnu
-      MINGW_PATH: 'C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin'
-      #cargoflags: --features "unstable"
-  # Nightly 32-bit GNU
-    - channel: nightly
-      target: i686-pc-windows-gnu
-      MINGW_PATH: 'C:\MinGW\bin'
-      #cargoflags: --features "unstable"
-
-### Allowed failures ###
-
-# See Appveyor documentation for specific details. In short, place any channel or targets you wish
-# to allow build failures on (usually nightly at least is a wise choice). This will prevent a build
-# or test failure in the matching channels/targets from failing the entire build.
-matrix:
-  allow_failures:
-    - channel: nightly
-    - channel: beta
-
-# If you only care about stable channel build failures, uncomment the following line:
-    #- channel: beta
-
-## Install Script ##
-
-# This is the most important part of the Appveyor configuration. This installs the version of Rust
-# specified by the 'channel' and 'target' environment variables from the build matrix. This uses
-# rustup to install Rust.
-#
-# For simple configurations, instead of using the build matrix, you can simply set the
-# default-toolchain and default-host manually here.
-install:
-  - appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
-  - rustup-init -yv --default-toolchain %channel% --default-host %target%
-  - set PATH=%PATH%;%USERPROFILE%\.cargo\bin
-  - if defined MINGW_PATH set PATH=%PATH%;%MINGW_PATH%
-  - rustc -vV
-  - cargo -vV
-  - rustup component add rustfmt
-
-## Build Script ##
-
-# 'cargo test' takes care of building for us, so disable Appveyor's build stage. This prevents
-# the "directory does not contain a project or solution file" error.
-build: false
-
-# Uses 'cargo test' to run tests and build. Alternatively, the project may call compiled programs
-#directly or perform other testing commands. Rust will automatically be placed in the PATH
-# environment variable.
-test_script:
-  - cargo test --verbose %cargoflags%

.github/workflows/build_gnu_linux.yml

@@ -0,0 +1,22 @@
+name: GNU/Linux
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+        rust:
+          - stable
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - run: git config --global core.autocrlf false
+    - uses: actions/checkout@v2
+    - name: Build
+      run: cargo build --all --locked --verbose

.github/workflows/build_macos.yml

@@ -0,0 +1,22 @@
+name: macOS
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    strategy:
+      matrix:
+        os:
+          - macos-latest
+        rust:
+          - stable
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - run: git config --global core.autocrlf false
+    - uses: actions/checkout@v2
+    - name: Build
+      run: cargo build --all --locked --verbose

.github/workflows/build_windows.yml

@@ -0,0 +1,22 @@
+name: Windows
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    strategy:
+      matrix:
+        os:
+          - windows-latest
+        rust:
+          - stable
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - run: git config --global core.autocrlf false
+    - uses: actions/checkout@v2
+    - name: Build
+      run: cargo build --all --locked --verbose

.github/workflows/cd.yml

@@ -0,0 +1,67 @@
+# CD GitHub Actions workflow for Monolith
+
+name: CD
+
+on:
+  release:
+    types:
+    - created
+
+jobs:
+
+  windows:
+    runs-on: windows-2019
+    steps:
+    - run: git config --global core.autocrlf false
+    - name: Checkout the repository
+      uses: actions/checkout@v2
+    - name: Build the executable
+      run: cargo build --release
+    - uses: Shopify/upload-to-release@1.0.0
+      with:
+        name: monolith.exe
+        path: target\release\monolith.exe
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+  gnu_linux_armhf:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout the repository
+      uses: actions/checkout@v2
+    - name: Prepare cross-platform environment
+      run: |
+        sudo mkdir -p /cross-build-arm
+        sudo touch /etc/apt/sources.list.d/armhf.list
+        echo "deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic main" | sudo tee -a /etc/apt/sources.list.d/armhf.list
+        sudo apt-get update
+        sudo apt-get install -y gcc-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross
+        sudo apt-get download libssl1.1:armhf libssl-dev:armhf
+        sudo dpkg -x libssl1.1*.deb /cross-build-arm
+        sudo dpkg -x libssl-dev*.deb /cross-build-arm
+        rustup target add arm-unknown-linux-gnueabihf
+        echo "::set-env name=C_INCLUDE_PATH::/cross-build-arm/usr/include"
+        echo "::set-env name=OPENSSL_INCLUDE_DIR::/cross-build-arm/usr/include/arm-linux-gnueabihf"
+        echo "::set-env name=OPENSSL_LIB_DIR::/cross-build-arm/usr/lib/arm-linux-gnueabihf"
+        echo "::set-env name=PKG_CONFIG_ALLOW_CROSS::1"
+        echo "::set-env name=RUSTFLAGS::-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build-arm/usr/lib/arm-linux-gnueabihf -L/cross-build-arm/lib/arm-linux-gnueabihf"
+    - name: Build the executable
+      run: cargo build --release --target=arm-unknown-linux-gnueabihf
+    - name: Attach artifact to the release
+      uses: Shopify/upload-to-release@1.0.0
+      with:
+        name: monolith-gnu-linux-armhf
+        path: target/arm-unknown-linux-gnueabihf/release/monolith
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+  gnu_linux_x86_64:
+    runs-on: ubuntu-18.04
+    steps:
+    - name: Checkout the repository
+      uses: actions/checkout@v2
+    - name: Build the executable
+      run: cargo build --release
+    - uses: Shopify/upload-to-release@1.0.0
+      with:
+        name: monolith-gnu-linux-x86_64
+        path: target/release/monolith
+        repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -0,0 +1,32 @@
+name: CI
+
+on:
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build_and_test:
+
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+          - windows-latest
+        rust:
+          - stable
+          - beta
+          - nightly
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - run: git config --global core.autocrlf false
+    - uses: actions/checkout@v2
+    - name: Build
+      run: cargo build --all --locked --verbose
+    - name: Run tests
+      run: cargo test --all --locked --verbose
+    - name: Check code formatting
+      run: |
+        rustup component add rustfmt
+        cargo fmt --all -- --check

.gitignore

@@ -4,6 +4,3 @@
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
-
-# Exclude accidental HTML files
-*.html

.travis.yml

@@ -1,29 +0,0 @@
-language: rust
-cache: cargo
-
-sudo: false
-
-os:
-  - linux
-  - osx
-
-rust:
-  - stable
-  - beta
-  - nightly
-
-before_script:
-  - rustup component add rustfmt
-
-script:
-  - cargo build --all --locked --verbose
-  - cargo test --all --locked --verbose
-  - |
-    if [[ "$TRAVIS_RUST_VERSION" == "stable" ]]; then
-      cargo fmt --all -- --check
-    fi
-
-jobs:
-  allow_failures:
-    - rust: nightly
-  fast_finish: true

Cargo.toml

@@ -1,7 +1,6 @@
 [package]
 name = "monolith"
-version = "2.1.2"
-edition = "2018"
+version = "2.4.0"
 authors = [
     "Sunshine <sunshine@uberspace.net>",
     "Mahdi Robatipoor <mahdi.robatipoor@gmail.com>",
@@ -9,17 +8,35 @@ authors = [
     "Emi Simpson <emi@alchemi.dev>",
     "rhysd <lin90162@yahoo.co.jp>",
 ]
+edition = "2018"
 description = "CLI tool for saving web pages as a single HTML file"
+homepage = "https://github.com/Y2Z/monolith"
+repository = "https://github.com/Y2Z/monolith"
+readme = "README.md"
+keywords = ["web", "http", "html", "download", "command-line"]
+categories = ["command-line-utilities", "web-programming"]
+include = [
+    "src/*.rs",
+    "Cargo.toml",
+]
+license = "Unlicense"
+license-file = "LICENSE"
 
 [dependencies]
-base64 = "0.10.1"
-clap = "2.33.0"
+base64 = "0.13.0"
+chrono = "0.4.19" # Used for formatting creation timestamp
+clap = "2.33.3"
+cssparser = "0.27.2"
 html5ever = "0.24.1"
-lazy_static = "1.4.0"
-regex = "1.3.1"
-url = "2.1.0"
+regex = "1.4.2" # Used for parsing srcset
+sha2 = "0.9.2" # Used for calculating checksums during integrity checks
+url = "2.2.0"
 
 [dependencies.reqwest]
-version = "0.10.*"
+version = "0.10.9"
 default-features = false
 features = ["default-tls", "blocking", "gzip"]
+
+[dev-dependencies]
+assert_cmd = "1.0.1"
+tempfile = "3.1.0"

Dockerfile

@@ -0,0 +1,18 @@
+FROM rust
+
+WORKDIR /usr/local/src/
+RUN curl -s https://api.github.com/repos/y2z/monolith/releases/latest \
+| grep "tarball_url.*\"," \
+| cut -d '"' -f 4 \
+| wget -qi - -O monolith.tar.gz
+
+RUN tar xfz monolith.tar.gz \
+&& mv Y2Z-monolith-* monolith \
+&& rm monolith.tar.gz
+
+WORKDIR /usr/local/src/monolith
+RUN ls -a
+RUN make install
+
+WORKDIR /tmp
+CMD ["/usr/local/cargo/bin/monolith"]

Makefile

@@ -1,16 +1,29 @@
-.PHONY: all build install run test lint
+# Makefile for monolith
 
-all: test build
+all: build
+.PHONY: all
 
 build:
 	@cargo build --locked
+.PHONY: build
 
-install:
-	@cargo install --force --locked --path .
-
-test:
+test: build
 	@cargo test --locked
 	@cargo fmt --all -- --check
+.PHONY: test_code_formatting
 
 lint:
 	@cargo fmt --all --
+.PHONY: lint
+
+install:
+	@cargo install --force --locked --path .
+.PHONY: install
+
+uninstall:
+	@cargo uninstall
+.PHONY: uninstall
+
+clean:
+	@cargo clean
+.PHONY: clean

README.md

@@ -1,63 +1,103 @@
-[![Travis CI Build Status](https://travis-ci.org/Y2Z/monolith.svg?branch=master)](https://travis-ci.org/Y2Z/monolith)
-[![AppVeyor Build status](https://ci.appveyor.com/api/projects/status/ae7soyjih8jg2bv7/branch/master?svg=true)](https://ci.appveyor.com/project/snshn/monolith/branch/master)
+[![Monolith Build Status for GNU/Linux](https://github.com/Y2Z/monolith/workflows/GNU%2FLinux/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AGNU%2FLinux)
+[![Monolith Build Status for macOS](https://github.com/Y2Z/monolith/workflows/macOS/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AmacOS)
+[![Monolith Build Status for Windows](https://github.com/Y2Z/monolith/workflows/Windows/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AWindows)
 
 ```
- ___     ___________    __________      ___________________    ___
-|   \   /           \  |          |    |                   |  |   |
-|    \_/     __      \_|    __    |    |    ___     ___    |__|   |
-|           |  |           |  |   |    |   |   |   |   |          |
-|           |__|    _      |__|   |____|   |   |   |   |    __    |
-|   |\_/|          | \                     |   |   |   |   |  |   |
-|___|   |__________|  \____________________|   |___|   |___|  |___|
+ _____     ______________    __________      ___________________    ___
+|     \   /              \  |          |    |                   |  |   |
+|      \_/       __       \_|    __    |    |    ___     ___    |__|   |
+|               |  |            |  |   |    |   |   |   |   |          |
+|   |\     /|   |__|    _       |__|   |____|   |   |   |   |    __    |
+|   | \___/ |          | \                      |   |   |   |   |  |   |
+|___|       |__________|  \_____________________|   |___|   |___|  |___|
 ```
 
-A data hoarder's dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
+A data hoarder’s dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
 
-Unlike the conventional "Save page as", `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.
+Unlike the conventional “Save page as”, `monolith` not only saves the target document, it embeds CSS, image, and JavaScript assets **all at once**, producing a single HTML5 document that is a joy to store and share.
 
 If compared to saving websites with `wget -mpk`, this tool embeds all assets as data URLs and therefore lets browsers render the saved page exactly the way it was on the Internet, even when no network connection is available.
 
+---------------------------------------------------
+
 ## Installation
 
-### From source
-    $ git clone https://github.com/Y2Z/monolith.git
-    $ cd monolith
-    $ cargo install --path .
+### Using Cargo
+    $ cargo install monolith
 
-### With Homebrew (on macOS and GNU/Linux)
+#### Via Homebrew (on macOS and GNU/Linux)
     $ brew install monolith
 
-### Using Snapcraft (on GNU/Linux)
+#### Using Snapcraft (on GNU/Linux)
     $ snap install monolith
 
+#### Using Ports collection (on FreeBSD and TrueOS)
+    $ cd /usr/ports/www/monolith/
+    $ make install clean
+
+#### Using pre-built binaries (Windows, ARM-based devices, etc)
+Every [release](https://github.com/Y2Z/monolith/releases) contains pre-built binaries for Windows, GNU/Linux, as well as platforms with non-standart CPU architecture.
+
+#### From source
+
+Dependency: `libssl-dev`
+
+    $ git clone https://github.com/Y2Z/monolith.git
+    $ cd monolith
+    $ make install
+
+#### Using Containers
+The guide can be found [here](docs/containers.md)
+
+---------------------------------------------------
+
 ## Usage
     $ monolith https://lyrics.github.io/db/P/Portishead/Dummy/Roads/ -o portishead-roads-lyrics.html
 
+---------------------------------------------------
+
 ## Options
- - `-c`: Ignore styles
- - `-f`: Exclude iframes
+ - `-a`: Exclude audio sources
+ - `-b`: Use custom base URL
+ - `-c`: Exclude CSS
+ - `-e`: Ignore network errors
+ - `-f`: Omit frames
+ - `-F`: Exclude web fonts
  - `-i`: Remove images
- - `-I`: Isolate document
+ - `-I`: Isolate the document
  - `-j`: Exclude JavaScript
  - `-k`: Accept invalid X.509 (TLS) certificates
+ - `-M`: Don't add timestamp and URL information
  - `-o`: Write output to file
- - `-s`: Silent mode
- - `-u`: Specify custom User-Agent
+ - `-s`: Be quiet
+ - `-t`: Adjust network request timeout
+ - `-u`: Provide custom User-Agent
+ - `-v`: Exclude videos
 
-## HTTPS and HTTP proxies
-Please set `https_proxy`, `http_proxy` and `no_proxy` environment variables.
+---------------------------------------------------
+
+## Proxies
+Please set `https_proxy`, `http_proxy`, and `no_proxy` environment variables.
+
+---------------------------------------------------
 
 ## Contributing
 Please open an issue if something is wrong, that helps make this project better.
 
+---------------------------------------------------
+
 ## Related projects
  - `Monolith Chrome Extension`: https://github.com/rhysd/monolith-of-web
  - `Pagesaver`: https://github.com/distributed-mind/pagesaver
  - `Personal WayBack Machine`: https://github.com/popey/pwbm
- - `SingleFile`: https://github.com/gildas-lormeau/SingleFile
+ - `Hako`: https://github.com/dmpop/hako
+
+---------------------------------------------------
 
 ## License
 The Unlicense
 
+---------------------------------------------------
+
 <!-- Microtext -->
-<sub>Keep in mind that `monolith` is not aware of your browser's session</sub>
+<sub>Keep in mind that `monolith` is not aware of your browser’s session</sub>

docs/arch/0001-record-architecture-decisions.md

@@ -0,0 +1,19 @@
+# 1. Record architecture decisions
+
+Date: 2019-12-25
+
+## Status
+
+Accepted
+
+## Context
+
+We need to record the architectural decisions made on this project.
+
+## Decision
+
+We will use Architecture Decision Records, as [described by Michael Nygard](http://thinkrelevance.com/blog/2011/11/15/documenting-architecture-decisions).
+
+## Consequences
+
+See Michael Nygard's article, linked above. For a lightweight ADR toolset, see Nat Pryce's [adr-tools](https://github.com/npryce/adr-tools).

docs/arch/0002-noscript-nodes.md

@@ -0,0 +1,19 @@
+# 2. NOSCRIPT nodes
+
+Date: 2020-04-16
+
+## Status
+
+Accepted
+
+## Context
+
+HTML pages can contain `noscript` nodes, which reveal their contents only in case when JavaScript is not available. Most of the time they contain hidden messages that inform about certain JavaScript-dependent features not being operational, however sometimes can also feature media assets or even iframes.
+
+## Decision
+
+When the document is being saved with or without JavaScript, each `noscript` node should be preserved while its children need to be processed exactly the same way as the rest of the document. This approach will ensure that even hidden remote assets are embedded — since those hidden elements may have to be displayed later in a browser that has JavaScript turned off. An option should be available to "unwrap" all `noscript` nodes in order to make their contents always visible in the document, complimenting the "disable JS" function of the program.
+
+## Consequences
+
+Saved documents will have contents of all `noscript` nodes processed as if they are part of the document's DOM, therefore properly display images encapsulated within `noscript` nodes when being viewed in browsers that have JavaScript turned off (or have no JavaScript support in the first place). The new option to "unwrap" `noscript` elements will help the user ensure that the resulting document always represents what the original web page looked like in a browser that had JavaScript turned off.

docs/arch/0003-network-request-timeout.md

@@ -0,0 +1,21 @@
+# 3. Network request timeout
+
+Date: 2020-02-15
+
+## Status
+
+Accepted
+
+## Context
+
+A slow network connection and overloaded server may negatively impact network response time.
+
+## Decision
+
+Make the program simulate behavior of popular web browsers and CLI tools, where the default network response timeout is most often set to 120 seconds.
+
+Instead of featuring retries for timed out network requests, the program should have an option to adjust the timeout length, along with making it indefinite when given "0" as its value.
+
+## Consequences
+
+The user is able to retrieve resources that have long response time, as well as obtain full control over how soon, and if at all, network requests should time out.

docs/arch/0004-asset-integrity-check.md

@@ -0,0 +1,21 @@
+# 4. Asset integrity check
+
+Date: 2020-02-23
+
+## Status
+
+Accepted
+
+## Context
+
+In HTML5, `link` and `script` nodes have an attribute named `integrity`, which lets the browser check if the remote file is valid, mostly for the purpose of enhancing page security.
+
+## Decision
+
+In order to replicate the browser's behavior, the program should perform integrity check the same way it does, excluding the linked asset from the final result if such check fails.
+
+The `integrity` attribute should be removed from nodes, as it bears no benefit for resources embedded as data URLs.
+
+## Consequences
+
+Assets that fail to pass the check get excluded from the saved document. Meanwhile, saved documents no longer contain integrity attributes on all `link` and `script` nodes.

docs/arch/0005-asset-minimization.md

@@ -0,0 +1,19 @@
+# 5. Asset Minimization
+
+Date: 2020-03-14
+
+## Status
+
+Accepted
+
+## Context
+
+It may look like a good idea to make monolith compress retrieved assets while saving the page for the purpose of reducing the resulting document's file size.
+
+## Decision
+
+Given that the main purpose of this program is to save pages in a convenient to store and share manner — it's mostly an archiving tool, aside from being able to tell monolith to exclude certain types of asests (e.g. images, CSS, JavaScript), it would be outside of scope of this program to implement code for compressing assets. Minimizing files before embedding them does not reduce the amount of data that needs to be transferred either. A separate tool can be used later to compress and minimize pages saved by monolith, if needed.
+
+## Consequences
+
+Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only minimal amount of modifications to the original web page — whatever is needed to provide security or exclude unwanted asset types.

docs/arch/0006-reload-and-location-meta-tags.md

@@ -0,0 +1,19 @@
+# 6. Reload and location `meta` tags
+
+Date: 2020-06-25
+
+## Status
+
+Accepted
+
+## Context
+
+HTML documents may contain `meta` tags capable of automatically refreshing the page or redirecting to another location.
+
+## Decision
+
+Since the resulting document is saved to disk and generally not intended to be served over the network, it only makes sense to remove `meta` tags that have `http-equiv` attribute equal to "Refresh" or "Location", in order to prevent them from reloading the page or redirecting to another location.
+
+## Consequences
+
+Monolith will ensure that saved documents do not contain `meta` tags capable of changing location or reloading the page.

docs/arch/0007-network-errors.md

@@ -0,0 +1,19 @@
+# 7. Network errors
+
+Date: 2020-11-22
+
+## Status
+
+Accepted
+
+## Context
+
+Servers may return information with HTTP response codes other than `200`, however those responses may still contain useful data.
+
+## Decision
+
+Fail by default, notifying of the network error. Add option to continue retrieving assets by treating all response codes as `200`.
+
+## Consequences
+
+Monolith will fail to obtain resources with status other than `200`, unless told to ignore network errors.

docs/arch/0008-base-tag.md

@@ -0,0 +1,40 @@
+# 8. Base Tag
+
+Date: 2020-12-25
+
+## Status
+
+Accepted
+
+## Context
+
+HTML documents may contain `base` tag, which influences resolution of anchor links and relative URLs as well as dynamically loaded resources.
+
+Sometimes, in order to make certain saved documents function closer to how they operate while being served from a remote server, the `base` tag specifying the source page's URL may need to be added to the document.
+
+There can be only one such tag. If multiple `base` tags are present, only the first encountered tag ends up being used.
+
+## Decision
+
+Adding the `base` tag should be optional — saved documents should not contain the `base` tag unless it was specified by the user, or the document originally had the `base` tag in it.
+
+Existing `href` attribute's value of the original `base` tag should be used for resolving the document's relative links instead of document's own URL (precisely the way browsers do it).
+
+## Consequences
+
+#### If the base tag does not exist in the source document
+
+- If the base tag does not exist in the source document
+  - With base URL option provided
+    - use the specified base URL value to retrieve assets, keep original base URL value in the document
+  - Without base URL option provided
+    - download document as usual, do not add base tag
+-  If the base tag already exists in the source document
+   - With base URL option provided
+     - we overwrite the original base URL before retrieving assets, keep new base URL value in the document
+   - Without base URL option provided:
+     - use the base URL from the original document to retrieve assets, keep original base URL value in the document
+
+The program will obtain ability to retrieve remote assets for non-remote sources (such as data URLs and local files).
+
+The program will obatin ability to get rid of existing base tag values (by provind an empty one).

docs/containers.md

@@ -0,0 +1,15 @@
+1. Run `docker build -t y2z/monolith .` to create a Docker image
+
+2. Create a file named `monolith` which contains:
+```sh
+#!/bin/sh
+
+docker run --rm \
+    y2z/monolith \
+    monolith \
+    "$@"
+```
+3. Make the file executable (`chmod +x monolith`) and include it into your `$PATH`
+4. Now you should be able to run a containerized build of monolith like this:  
+`monolith -I https://github.com > document.html`
+

docs/references.md

@@ -0,0 +1,3 @@
+# References
+
+- https://content-security-policy.com/

docs/web-apps.md

@@ -0,0 +1,10 @@
+# Web apps that can be saved with Monolith
+
+These apps retain most or all of their functionality when saved with Monolith
+
+|Converse|https://conversejs.org|
+|:--|:--|
+|Description|An XMPP client built using web technologies|
+|Functionality retained|**full**|
+|Command to use|`monolith https://conversejs.org/fullscreen.html > conversejs.html`|
+|Monolith version used|2.2.7|

snap/snapcraft.yaml

@@ -18,11 +18,11 @@ description: |
 
 confinement: strict
 
-# Building on armhf fails, so we specify all supported non-armhf architectures
 architectures:
   - build-on: amd64
-  - build-on: i386
   - build-on: arm64
+  - build-on: armhf
+  - build-on: i386
   - build-on: ppc64el
   - build-on: s390x
 

src/args.rs

@@ -1,65 +0,0 @@
-use clap::{App, Arg};
-
-#[derive(Default)]
-pub struct AppArgs {
-    pub url_target: String,
-    pub no_css: bool,
-    pub no_frames: bool,
-    pub no_images: bool,
-    pub no_js: bool,
-    pub insecure: bool,
-    pub isolate: bool,
-    pub output: String,
-    pub silent: bool,
-    pub user_agent: String,
-}
-
-const DEFAULT_USER_AGENT: &str =
-    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.14; rv:66.0) Gecko/20100101 Firefox/66.0";
-
-impl AppArgs {
-    pub fn get() -> AppArgs {
-        let app = App::new("monolith")
-            .version(crate_version!())
-            .author(crate_authors!("\n"))
-            .about(crate_description!())
-            .arg(
-                Arg::with_name("url")
-                    .required(true)
-                    .takes_value(true)
-                    .index(1)
-                    .help("URL to download"),
-            )
-            // .args_from_usage("-a, --include-audio 'Embed audio sources'")
-            .args_from_usage("-c, --no-css 'Ignore styles'")
-            .args_from_usage("-f, --no-frames 'Exclude iframes'")
-            .args_from_usage("-i, --no-images 'Remove images'")
-            .args_from_usage("-I, --isolate 'Cut off from the Internet'")
-            .args_from_usage("-j, --no-js 'Exclude JavaScript'")
-            .args_from_usage("-k, --insecure 'Accept invalid X.509 (TLS) certificates'")
-            .args_from_usage("-o, --output=[document.html] 'Write output to <file>'")
-            .args_from_usage("-s, --silent 'Suppress verbosity'")
-            .args_from_usage("-u, --user-agent=[Iceweasel] 'Custom User-Agent string'")
-            // .args_from_usage("-v, --include-video 'Embed video sources'")
-            .get_matches();
-        let mut app_args = AppArgs::default();
-        // Process the command
-        app_args.url_target = app
-            .value_of("url")
-            .expect("please set target url")
-            .to_string();
-        app_args.no_css = app.is_present("no-css");
-        app_args.no_frames = app.is_present("no-frames");
-        app_args.no_images = app.is_present("no-images");
-        app_args.no_js = app.is_present("no-js");
-        app_args.insecure = app.is_present("insecure");
-        app_args.isolate = app.is_present("isolate");
-        app_args.silent = app.is_present("silent");
-        app_args.output = app.value_of("output").unwrap_or("").to_string();
-        app_args.user_agent = app
-            .value_of("user-agent")
-            .unwrap_or(DEFAULT_USER_AGENT)
-            .to_string();
-        app_args
-    }
-}

src/css.rs

@@ -0,0 +1,440 @@
+use cssparser::{ParseError, Parser, ParserInput, SourcePosition, Token};
+use reqwest::blocking::Client;
+use std::collections::HashMap;
+
+use crate::opts::Options;
+use crate::url::{data_to_data_url, get_url_fragment, is_http_url, resolve_url, url_with_fragment};
+use crate::utils::retrieve_asset;
+
+const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
+    // Universal
+    "background",
+    "background-image",
+    "border-image",
+    "border-image-source",
+    "content",
+    "cursor",
+    "list-style",
+    "list-style-image",
+    "mask",
+    "mask-image",
+    // Specific to @counter-style
+    "additive-symbols",
+    "negative",
+    "pad",
+    "prefix",
+    "suffix",
+    "symbols",
+];
+const CSS_SPECIAL_CHARS: &str = "~!@$%^&*()+=,./'\";:?><[]{}|`#";
+
+pub fn is_image_url_prop(prop_name: &str) -> bool {
+    CSS_PROPS_WITH_IMAGE_URLS
+        .iter()
+        .find(|p| prop_name.eq_ignore_ascii_case(p))
+        .is_some()
+}
+
+pub fn enquote(input: String, double: bool) -> String {
+    if double {
+        format!("\"{}\"", input.replace("\"", "\\\""))
+    } else {
+        format!("'{}'", input.replace("'", "\\'"))
+    }
+}
+
+pub fn escape(value: &str) -> String {
+    let mut res = str!(&value);
+
+    res = res.replace("\\", "\\\\");
+
+    for c in CSS_SPECIAL_CHARS.chars() {
+        res = res.replace(c, format!("\\{}", c).as_str());
+    }
+
+    res
+}
+
+pub fn process_css<'a>(
+    cache: &mut HashMap<String, Vec<u8>>,
+    client: &Client,
+    parent_url: &str,
+    parser: &mut Parser,
+    options: &Options,
+    depth: u32,
+    rule_name: &str,
+    prop_name: &str,
+    func_name: &str,
+) -> Result<String, ParseError<'a, String>> {
+    let mut result: String = str!();
+
+    let mut curr_rule: String = str!(rule_name.clone());
+    let mut curr_prop: String = str!(prop_name.clone());
+    let mut token: &Token;
+    let mut token_offset: SourcePosition;
+
+    loop {
+        token_offset = parser.position();
+        token = match parser.next_including_whitespace_and_comments() {
+            Ok(token) => token,
+            Err(_) => {
+                break;
+            }
+        };
+
+        match *token {
+            Token::Comment(_) => {
+                let token_slice = parser.slice_from(token_offset);
+                result.push_str(str!(token_slice).as_str());
+            }
+            Token::Semicolon => result.push_str(";"),
+            Token::Colon => result.push_str(":"),
+            Token::Comma => result.push_str(","),
+            Token::ParenthesisBlock | Token::SquareBracketBlock | Token::CurlyBracketBlock => {
+                if options.no_fonts && curr_rule == "font-face" {
+                    continue;
+                }
+
+                let closure: &str;
+                if token == &Token::ParenthesisBlock {
+                    result.push_str("(");
+                    closure = ")";
+                } else if token == &Token::SquareBracketBlock {
+                    result.push_str("[");
+                    closure = "]";
+                } else {
+                    result.push_str("{");
+                    closure = "}";
+                }
+
+                let block_css: String = parser
+                    .parse_nested_block(|parser| {
+                        process_css(
+                            cache,
+                            client,
+                            parent_url,
+                            parser,
+                            options,
+                            depth,
+                            rule_name,
+                            curr_prop.as_str(),
+                            func_name,
+                        )
+                    })
+                    .unwrap();
+                result.push_str(block_css.as_str());
+
+                result.push_str(closure);
+            }
+            Token::CloseParenthesis => result.push_str(")"),
+            Token::CloseSquareBracket => result.push_str("]"),
+            Token::CloseCurlyBracket => result.push_str("}"),
+            Token::IncludeMatch => result.push_str("~="),
+            Token::DashMatch => result.push_str("|="),
+            Token::PrefixMatch => result.push_str("^="),
+            Token::SuffixMatch => result.push_str("$="),
+            Token::SubstringMatch => result.push_str("*="),
+            Token::CDO => result.push_str("<!--"),
+            Token::CDC => result.push_str("-->"),
+            Token::WhiteSpace(ref value) => {
+                result.push_str(value);
+            }
+            // div...
+            Token::Ident(ref value) => {
+                curr_rule = str!();
+                curr_prop = str!(value);
+                result.push_str(&escape(value));
+            }
+            // @import, @font-face, @charset, @media...
+            Token::AtKeyword(ref value) => {
+                curr_rule = str!(value);
+                if options.no_fonts && curr_rule == "font-face" {
+                    continue;
+                }
+                result.push_str("@");
+                result.push_str(value);
+            }
+            Token::Hash(ref value) => {
+                result.push_str("#");
+                result.push_str(value);
+            }
+            Token::QuotedString(ref value) => {
+                if curr_rule == "import" {
+                    // Reset current at-rule value
+                    curr_rule = str!();
+
+                    // Skip empty import values
+                    if value.len() < 1 {
+                        result.push_str("''");
+                        continue;
+                    }
+
+                    let import_full_url = resolve_url(&parent_url, value).unwrap_or_default();
+                    let import_url_fragment = get_url_fragment(import_full_url.clone());
+                    match retrieve_asset(
+                        cache,
+                        client,
+                        &parent_url,
+                        &import_full_url,
+                        options,
+                        depth + 1,
+                    ) {
+                        Ok((import_contents, import_final_url, _import_media_type)) => {
+                            let import_data_url = data_to_data_url(
+                                "text/css",
+                                embed_css(
+                                    cache,
+                                    client,
+                                    &import_final_url,
+                                    &String::from_utf8_lossy(&import_contents),
+                                    options,
+                                    depth + 1,
+                                )
+                                .as_bytes(),
+                                &import_final_url,
+                            );
+                            let assembled_url: String = url_with_fragment(
+                                import_data_url.as_str(),
+                                import_url_fragment.as_str(),
+                            );
+                            result.push_str(enquote(assembled_url, false).as_str());
+                        }
+                        Err(_) => {
+                            // Keep remote reference if unable to retrieve the asset
+                            if is_http_url(import_full_url.clone()) {
+                                let assembled_url: String = url_with_fragment(
+                                    import_full_url.as_str(),
+                                    import_url_fragment.as_str(),
+                                );
+                                result.push_str(enquote(assembled_url, false).as_str());
+                            }
+                        }
+                    }
+                } else {
+                    if func_name == "url" {
+                        // Skip empty url()'s
+                        if value.len() < 1 {
+                            continue;
+                        }
+
+                        if options.no_images && is_image_url_prop(curr_prop.as_str()) {
+                            result.push_str(enquote(str!(empty_image!()), false).as_str());
+                        } else {
+                            let resolved_url = resolve_url(&parent_url, value).unwrap_or_default();
+                            let url_fragment = get_url_fragment(resolved_url.clone());
+                            match retrieve_asset(
+                                cache,
+                                client,
+                                &parent_url,
+                                &resolved_url,
+                                options,
+                                depth + 1,
+                            ) {
+                                Ok((data, final_url, media_type)) => {
+                                    let data_url = data_to_data_url(&media_type, &data, &final_url);
+                                    let assembled_url: String =
+                                        url_with_fragment(data_url.as_str(), url_fragment.as_str());
+                                    result.push_str(enquote(assembled_url, false).as_str());
+                                }
+                                Err(_) => {
+                                    // Keep remote reference if unable to retrieve the asset
+                                    if is_http_url(resolved_url.clone()) {
+                                        let assembled_url: String = url_with_fragment(
+                                            resolved_url.as_str(),
+                                            url_fragment.as_str(),
+                                        );
+                                        result.push_str(enquote(assembled_url, false).as_str());
+                                    }
+                                }
+                            }
+                        }
+                    } else {
+                        result.push_str(enquote(str!(value), false).as_str());
+                    }
+                }
+            }
+            Token::Number {
+                ref has_sign,
+                ref value,
+                ..
+            } => {
+                if *has_sign && *value >= 0. {
+                    result.push_str("+");
+                }
+                result.push_str(&value.to_string())
+            }
+            Token::Percentage {
+                ref has_sign,
+                ref unit_value,
+                ..
+            } => {
+                if *has_sign && *unit_value >= 0. {
+                    result.push_str("+");
+                }
+                result.push_str(str!(unit_value * 100.0).as_str());
+                result.push_str("%");
+            }
+            Token::Dimension {
+                ref has_sign,
+                ref value,
+                ref unit,
+                ..
+            } => {
+                if *has_sign && *value >= 0. {
+                    result.push_str("+");
+                }
+                result.push_str(str!(value).as_str());
+                result.push_str(str!(unit).as_str());
+            }
+            // #selector, #id...
+            Token::IDHash(ref value) => {
+                curr_rule = str!();
+                result.push_str("#");
+                result.push_str(&escape(value));
+            }
+            Token::UnquotedUrl(ref value) => {
+                let is_import: bool = curr_rule == "import";
+
+                if is_import {
+                    // Reset current at-rule value
+                    curr_rule = str!();
+                }
+
+                // Skip empty url()'s
+                if value.len() < 1 {
+                    result.push_str("url()");
+                    continue;
+                } else if value.starts_with("#") {
+                    result.push_str("url(");
+                    result.push_str(value);
+                    result.push_str(")");
+                    continue;
+                }
+
+                result.push_str("url(");
+                if is_import {
+                    let full_url = resolve_url(&parent_url, value).unwrap_or_default();
+                    let url_fragment = get_url_fragment(full_url.clone());
+                    match retrieve_asset(cache, client, &parent_url, &full_url, options, depth + 1)
+                    {
+                        Ok((css, final_url, _media_type)) => {
+                            let data_url = data_to_data_url(
+                                "text/css",
+                                embed_css(
+                                    cache,
+                                    client,
+                                    &final_url,
+                                    &String::from_utf8_lossy(&css),
+                                    options,
+                                    depth + 1,
+                                )
+                                .as_bytes(),
+                                &final_url,
+                            );
+                            let assembled_url: String =
+                                url_with_fragment(data_url.as_str(), url_fragment.as_str());
+                            result.push_str(enquote(assembled_url, false).as_str());
+                        }
+                        Err(_) => {
+                            // Keep remote reference if unable to retrieve the asset
+                            if is_http_url(full_url.clone()) {
+                                let assembled_url: String =
+                                    url_with_fragment(full_url.as_str(), url_fragment.as_str());
+                                result.push_str(enquote(assembled_url, false).as_str());
+                            }
+                        }
+                    }
+                } else {
+                    if is_image_url_prop(curr_prop.as_str()) && options.no_images {
+                        result.push_str(enquote(str!(empty_image!()), false).as_str());
+                    } else {
+                        let full_url = resolve_url(&parent_url, value).unwrap_or_default();
+                        let url_fragment = get_url_fragment(full_url.clone());
+                        match retrieve_asset(
+                            cache,
+                            client,
+                            &parent_url,
+                            &full_url,
+                            options,
+                            depth + 1,
+                        ) {
+                            Ok((data, final_url, media_type)) => {
+                                let data_url = data_to_data_url(&media_type, &data, &final_url);
+                                let assembled_url: String =
+                                    url_with_fragment(data_url.as_str(), url_fragment.as_str());
+                                result.push_str(enquote(assembled_url, false).as_str());
+                            }
+                            Err(_) => {
+                                // Keep remote reference if unable to retrieve the asset
+                                if is_http_url(full_url.clone()) {
+                                    let assembled_url: String =
+                                        url_with_fragment(full_url.as_str(), url_fragment.as_str());
+                                    result.push_str(enquote(assembled_url, false).as_str());
+                                }
+                            }
+                        }
+                    }
+                }
+                result.push_str(")");
+            }
+            Token::Delim(ref value) => result.push_str(&value.to_string()),
+            Token::Function(ref name) => {
+                let function_name: &str = &name.clone();
+                result.push_str(function_name);
+                result.push_str("(");
+
+                let block_css: String = parser
+                    .parse_nested_block(|parser| {
+                        process_css(
+                            cache,
+                            client,
+                            parent_url,
+                            parser,
+                            options,
+                            depth,
+                            curr_rule.as_str(),
+                            curr_prop.as_str(),
+                            function_name,
+                        )
+                    })
+                    .unwrap();
+                result.push_str(block_css.as_str());
+
+                result.push_str(")");
+            }
+            Token::BadUrl(_) | Token::BadString(_) => {}
+        }
+    }
+
+    // Ensure empty CSS is really empty
+    if result.len() > 0 && result.trim().len() == 0 {
+        result = result.trim().to_string()
+    }
+
+    Ok(result)
+}
+
+pub fn embed_css(
+    cache: &mut HashMap<String, Vec<u8>>,
+    client: &Client,
+    parent_url: &str,
+    css: &str,
+    options: &Options,
+    depth: u32,
+) -> String {
+    let mut input = ParserInput::new(&css);
+    let mut parser = Parser::new(&mut input);
+
+    process_css(
+        cache,
+        client,
+        parent_url,
+        &mut parser,
+        options,
+        depth,
+        "",
+        "",
+        "",
+    )
+    .unwrap()
+}

src/http.rs

@@ -1,68 +0,0 @@
-use crate::utils::{clean_url, data_to_dataurl, is_data_url};
-use reqwest::blocking::Client;
-use reqwest::header::CONTENT_TYPE;
-use std::collections::HashMap;
-
-pub fn retrieve_asset(
-    cache: &mut HashMap<String, String>,
-    client: &Client,
-    url: &str,
-    as_dataurl: bool,
-    mime: &str,
-    opt_silent: bool,
-) -> Result<(String, String), reqwest::Error> {
-    let cache_key = clean_url(&url);
-
-    if is_data_url(&url).unwrap() {
-        Ok((url.to_string(), url.to_string()))
-    } else {
-        if cache.contains_key(&cache_key) {
-            // url is in cache
-            if !opt_silent {
-                eprintln!("{} (from cache)", &url);
-            }
-            let data = cache.get(&cache_key).unwrap();
-            Ok((data.to_string(), url.to_string()))
-        } else {
-            // url not in cache, we request it
-            let mut response = client.get(url).send()?;
-            let res_url = response.url().to_string();
-
-            if !opt_silent {
-                if url == res_url {
-                    eprintln!("{}", &url);
-                } else {
-                    eprintln!("{} -> {}", &url, &res_url);
-                }
-            }
-
-            let new_cache_key = clean_url(&res_url);
-
-            if as_dataurl {
-                // Convert response into a byte array
-                let mut data: Vec<u8> = vec![];
-                response.copy_to(&mut data)?;
-
-                // Attempt to obtain MIME type by reading the Content-Type header
-                let mimetype = if mime == "" {
-                    response
-                        .headers()
-                        .get(CONTENT_TYPE)
-                        .and_then(|header| header.to_str().ok())
-                        .unwrap_or(&mime)
-                } else {
-                    mime
-                };
-                let dataurl = data_to_dataurl(&mimetype, &data);
-                // insert in cache
-                cache.insert(new_cache_key, dataurl.clone());
-                Ok((dataurl, res_url))
-            } else {
-                let content = response.text().unwrap();
-                // insert in cache
-                cache.insert(new_cache_key, content.clone());
-                Ok((content, res_url))
-            }
-        }
-    }
-}

src/js.rs

@@ -1,7 +1,7 @@
 const JS_DOM_EVENT_ATTRS: &[&str] = &[
-    // From WHATWG HTML spec 8.1.5.2 'Event handlers on elements, Document objects, and Window objects':
+    // From WHATWG HTML spec 8.1.5.2 "Event handlers on elements, Document objects, and Window objects":
     //   https://html.spec.whatwg.org/#event-handlers-on-elements,-document-objects,-and-window-objects
-    //   https://html.spec.whatwg.org/#attributes-3 (table 'List of event handler content attributes')
+    //   https://html.spec.whatwg.org/#attributes-3 (table "List of event handler content attributes")
 
     // Global event handlers
     "onabort",

src/lib.rs

@@ -1,12 +1,14 @@
 #[macro_use]
-extern crate lazy_static;
+extern crate clap;
 
 #[macro_use]
 mod macros;
 
+pub mod css;
 pub mod html;
-pub mod http;
 pub mod js;
+pub mod opts;
+pub mod url;
 pub mod utils;
 
 #[cfg(test)]

src/macros.rs

@@ -7,3 +7,11 @@ macro_rules! str {
         ToString::to_string(&$val)
     };
 }
+
+#[macro_export]
+macro_rules! empty_image {
+    () => {
+"data:image/png;base64,\
+iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII="
+    };
+}

src/main.rs

@@ -1,32 +1,36 @@
-#[macro_use]
-extern crate clap;
-
-mod args;
-mod macros;
-
-use crate::args::AppArgs;
-use monolith::html::{html_to_dom, stringify_document, walk_and_embed_assets};
-use monolith::http::retrieve_asset;
-use monolith::utils::is_valid_url;
 use reqwest::blocking::Client;
 use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
 use std::collections::HashMap;
-use std::fs::File;
-use std::io::{self, Error, Write};
+use std::env;
+use std::fs;
+use std::io::{self, prelude::*, Error, Write};
+use std::path::Path;
 use std::process;
 use std::time::Duration;
 
+use monolith::html::{
+    add_favicon, create_metadata_tag, get_base_url, has_favicon, html_to_dom, set_base_url,
+    stringify_document, walk_and_embed_assets,
+};
+use monolith::opts::Options;
+use monolith::url::{
+    data_to_data_url, is_data_url, is_file_url, is_http_url, parse_data_url, resolve_url,
+};
+use monolith::utils::retrieve_asset;
+
+mod macros;
+
 enum Output {
     Stdout(io::Stdout),
-    File(File),
+    File(fs::File),
 }
 
 impl Output {
     fn new(file_path: &str) -> Result<Output, Error> {
-        if file_path.is_empty() {
+        if file_path.is_empty() || file_path.eq("-") {
             Ok(Output::Stdout(io::stdout()))
         } else {
-            Ok(Output::File(File::create(file_path)?))
+            Ok(Output::File(fs::File::create(file_path)?))
         }
     }
 
@@ -44,68 +48,182 @@ impl Output {
     }
 }
 
-fn main() {
-    let app_args = AppArgs::get();
+pub fn read_stdin() -> String {
+    let mut buffer = String::new();
+    for line in io::stdin().lock().lines() {
+        buffer += line.unwrap_or_default().as_str();
+        buffer += "\n";
+    }
+    buffer
+}
 
-    if !is_valid_url(app_args.url_target.as_str()) {
-        eprintln!(
-            "Only HTTP and HTTPS URLs are allowed but got: {}",
-            &app_args.url_target
-        );
+fn main() {
+    let options = Options::from_args();
+    let original_target: &str = &options.target;
+    let target_url: &str;
+    let mut base_url: String;
+    let mut dom;
+    let mut use_stdin: bool = false;
+
+    // Pre-process the input
+    let cwd_normalized: String =
+        str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
+    let path = Path::new(original_target);
+    let mut target: String = str!(original_target.clone()).replace("\\", "/");
+    let path_is_relative: bool = path.is_relative();
+
+    // Determine exact target URL
+    if target.clone().len() == 0 {
+        if !options.silent {
+            eprintln!("No target specified");
+        }
         process::exit(1);
+    } else if target.clone() == "-" {
+        // Read from pipe (stdin)
+        use_stdin = true;
+        // Default target URL to empty data URL; the user can control it via --base-url
+        target_url = "data:text/html,"
+    } else if is_http_url(target.clone()) || is_data_url(target.clone()) {
+        target_url = target.as_str();
+    } else if is_file_url(target.clone()) {
+        target_url = target.as_str();
+    } else if path.exists() {
+        if !path.is_file() {
+            if !options.silent {
+                eprintln!("Local target is not a file: {}", original_target);
+            }
+            process::exit(1);
+        }
+        target.insert_str(0, if cfg!(windows) { "file:///" } else { "file://" });
+        if path_is_relative {
+            target.insert_str(if cfg!(windows) { 8 } else { 7 }, &cwd_normalized);
+            target.insert_str(
+                if cfg!(windows) { 8 } else { 7 } + &cwd_normalized.len(),
+                "/",
+            );
+        }
+        target_url = target.as_str();
+    } else {
+        target.insert_str(0, "http://");
+        target_url = target.as_str();
     }
 
-    let mut output = Output::new(&app_args.output).expect("Could not prepare output");
+    // Define output
+    let mut output = Output::new(&options.output).expect("Could not prepare output");
 
     // Initialize client
     let mut cache = HashMap::new();
     let mut header_map = HeaderMap::new();
     header_map.insert(
         USER_AGENT,
-        HeaderValue::from_str(&app_args.user_agent).expect("Invalid User-Agent header specified"),
+        HeaderValue::from_str(&options.user_agent).expect("Invalid User-Agent header specified"),
     );
-
+    let timeout: u64 = if options.timeout > 0 {
+        options.timeout
+    } else {
+        std::u64::MAX / 4
+    };
     let client = Client::builder()
-        .timeout(Duration::from_secs(10))
-        .danger_accept_invalid_certs(app_args.insecure)
+        .timeout(Duration::from_secs(timeout))
+        .danger_accept_invalid_certs(options.insecure)
         .default_headers(header_map)
         .build()
         .expect("Failed to initialize HTTP client");
 
-    // Retrieve root document
-    let (data, final_url) = retrieve_asset(
-        &mut cache,
-        &client,
-        app_args.url_target.as_str(),
-        false,
-        "",
-        app_args.silent,
-    )
-    .expect("Could not retrieve assets in HTML");
-    let dom = html_to_dom(&data);
+    // At this stage we assume that the base URL is the same as the target URL
+    base_url = str!(target_url);
 
-    walk_and_embed_assets(
-        &mut cache,
-        &client,
-        &final_url,
-        &dom.document,
-        app_args.no_css,
-        app_args.no_js,
-        app_args.no_images,
-        app_args.silent,
-        app_args.no_frames,
-    );
+    // Retrieve target document
+    if use_stdin {
+        dom = html_to_dom(&read_stdin());
+    } else if is_file_url(target_url) || is_http_url(target_url) {
+        match retrieve_asset(&mut cache, &client, target_url, target_url, &options, 0) {
+            Ok((data, final_url, _media_type)) => {
+                if options.base_url.clone().unwrap_or(str!()).is_empty() {
+                    base_url = final_url
+                }
+                dom = html_to_dom(&String::from_utf8_lossy(&data));
+            }
+            Err(_) => {
+                if !options.silent {
+                    eprintln!("Could not retrieve target document");
+                }
+                process::exit(1);
+            }
+        }
+    } else if is_data_url(target_url) {
+        let (media_type, data): (String, Vec<u8>) = parse_data_url(target_url);
+        if !media_type.eq_ignore_ascii_case("text/html") {
+            if !options.silent {
+                eprintln!("Unsupported data URL media type");
+            }
+            process::exit(1);
+        }
+        dom = html_to_dom(&String::from_utf8_lossy(&data));
+    } else {
+        process::exit(1);
+    }
 
-    let html: String = stringify_document(
-        &dom.document,
-        app_args.no_css,
-        app_args.no_frames,
-        app_args.no_js,
-        app_args.no_images,
-        app_args.isolate,
-    );
+    // Use custom base URL if specified, read and use what's in the DOM otherwise
+    if !options.base_url.clone().unwrap_or(str!()).is_empty() {
+        if is_data_url(options.base_url.clone().unwrap()) {
+            if !options.silent {
+                eprintln!("Data URLs cannot be used as base URL");
+            }
+            process::exit(1);
+        } else {
+            base_url = options.base_url.clone().unwrap();
+        }
+    } else {
+        if let Some(existing_base_url) = get_base_url(&dom.document) {
+            base_url = resolve_url(target_url, existing_base_url).unwrap();
+        }
+    }
 
+    // Embed remote assets
+    walk_and_embed_assets(&mut cache, &client, &base_url, &dom.document, &options, 0);
+
+    // Update or add new BASE tag to reroute network requests and hash-links in the final document
+    if let Some(new_base_url) = options.base_url.clone() {
+        dom = set_base_url(&dom.document, new_base_url);
+    }
+
+    // Request and embed /favicon.ico (unless it's already linked in the document)
+    if !options.no_images && is_http_url(target_url) && !has_favicon(&dom.document) {
+        let favicon_ico_url: String = resolve_url(&base_url, "/favicon.ico").unwrap();
+
+        match retrieve_asset(
+            &mut cache,
+            &client,
+            &base_url,
+            &favicon_ico_url,
+            &options,
+            0,
+        ) {
+            Ok((data, final_url, media_type)) => {
+                let favicon_data_url: String = data_to_data_url(&media_type, &data, &final_url);
+                dom = add_favicon(&dom.document, favicon_data_url);
+            }
+            Err(_) => {
+                // Failed to retrieve favicon.ico
+            }
+        }
+    }
+
+    // Serialize DOM tree
+    let mut result: String = stringify_document(&dom.document, &options);
+
+    // Add metadata tag
+    if !options.no_metadata {
+        let metadata_comment: String = create_metadata_tag(&target_url);
+        result.insert_str(0, &metadata_comment);
+        if metadata_comment.len() > 0 {
+            result.insert_str(metadata_comment.len(), "\n");
+        }
+    }
+
+    // Write result into stdout or file
     output
-        .writeln_str(&html)
+        .writeln_str(&result)
         .expect("Could not write HTML output");
 }

src/opts.rs

@@ -0,0 +1,102 @@
+use clap::{App, Arg};
+
+#[derive(Default)]
+pub struct Options {
+    pub no_audio: bool,
+    pub base_url: Option<String>,
+    pub no_css: bool,
+    pub ignore_errors: bool,
+    pub no_frames: bool,
+    pub no_fonts: bool,
+    pub no_images: bool,
+    pub isolate: bool,
+    pub no_js: bool,
+    pub insecure: bool,
+    pub no_metadata: bool,
+    pub output: String,
+    pub silent: bool,
+    pub timeout: u64,
+    pub user_agent: String,
+    pub no_video: bool,
+    pub target: String,
+}
+
+const ASCII: &str = " \
+ _____     ______________    __________      ___________________    ___
+|     \\   /              \\  |          |    |                   |  |   |
+|      \\_/       __       \\_|    __    |    |    ___     ___    |__|   |
+|               |  |            |  |   |    |   |   |   |   |          |
+|   |\\     /|   |__|    _       |__|   |____|   |   |   |   |    __    |
+|   | \\___/ |          | \\                      |   |   |   |   |  |   |
+|___|       |__________|  \\_____________________|   |___|   |___|  |___|
+";
+const DEFAULT_NETWORK_TIMEOUT: u64 = 120;
+const DEFAULT_USER_AGENT: &str =
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0";
+
+impl Options {
+    pub fn from_args() -> Options {
+        let app = App::new(env!("CARGO_PKG_NAME"))
+            .version(crate_version!())
+            .author(crate_authors!("\n"))
+            .about(format!("{}\n{}", ASCII, crate_description!()).as_str())
+            .args_from_usage("-a, --no-audio 'Removes audio sources'")
+            .args_from_usage("-b, --base-url=[http://localhost/] 'Sets custom base URL'")
+            .args_from_usage("-c, --no-css 'Removes CSS'")
+            .args_from_usage("-e, --ignore-errors 'Ignore network errors'")
+            .args_from_usage("-f, --no-frames 'Removes frames and iframes'")
+            .args_from_usage("-F, --no-fonts 'Removes fonts'")
+            .args_from_usage("-i, --no-images 'Removes images'")
+            .args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
+            .args_from_usage("-j, --no-js 'Removes JavaScript'")
+            .args_from_usage("-k, --insecure 'Allows invalid X.509 (TLS) certificates'")
+            .args_from_usage("-M, --no-metadata 'Excludes timestamp and source information'")
+            .args_from_usage("-o, --output=[document.html] 'Writes output to <file>'")
+            .args_from_usage("-s, --silent 'Suppresses verbosity'")
+            .args_from_usage("-t, --timeout=[60] 'Adjusts network request timeout'")
+            .args_from_usage("-u, --user-agent=[Firefox] 'Sets custom User-Agent string'")
+            .args_from_usage("-v, --no-video 'Removes video sources'")
+            .arg(
+                Arg::with_name("target")
+                    .required(true)
+                    .takes_value(true)
+                    .index(1)
+                    .help("URL or file path"),
+            )
+            .get_matches();
+        let mut options: Options = Options::default();
+
+        // Process the command
+        options.target = app
+            .value_of("target")
+            .expect("please set target")
+            .to_string();
+        options.no_audio = app.is_present("no-audio");
+        if let Some(base_url) = app.value_of("base-url") {
+            options.base_url = Some(str!(base_url));
+        }
+        options.no_css = app.is_present("no-css");
+        options.ignore_errors = app.is_present("ignore-errors");
+        options.no_frames = app.is_present("no-frames");
+        options.no_fonts = app.is_present("no-fonts");
+        options.no_images = app.is_present("no-images");
+        options.isolate = app.is_present("isolate");
+        options.no_js = app.is_present("no-js");
+        options.insecure = app.is_present("insecure");
+        options.no_metadata = app.is_present("no-metadata");
+        options.output = app.value_of("output").unwrap_or("").to_string();
+        options.silent = app.is_present("silent");
+        options.timeout = app
+            .value_of("timeout")
+            .unwrap_or(&DEFAULT_NETWORK_TIMEOUT.to_string())
+            .parse::<u64>()
+            .unwrap();
+        options.user_agent = app
+            .value_of("user-agent")
+            .unwrap_or(DEFAULT_USER_AGENT)
+            .to_string();
+        options.no_video = app.is_present("no-video");
+
+        options
+    }
+}

src/tests/cli/base_url.rs

@@ -0,0 +1,123 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn add_new_when_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("http://localhost:8000/")
+            .arg("data:text/html,Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost:8000/\"></base>\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn keep_existing_when_none_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost:8000/\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn override_existing_when_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("http://localhost/")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost/\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_existing_when_empty_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}

src/tests/cli/basic.rs

@@ -0,0 +1,559 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::io::Write;
+    use std::process::Command;
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn print_version() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd.arg("-V").output().unwrap();
+
+        // STDOUT should contain program name and version
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            format!("{} {}\n", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"))
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn bad_input_empty_target() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd.arg("").output().unwrap();
+
+        // STDOUT should be empty
+        assert_eq!(std::str::from_utf8(&out.stdout).unwrap(), "");
+
+        // STDERR should contain error description
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            "No target specified\n"
+        );
+
+        // The exit code should be 1
+        out.assert().code(1);
+
+        Ok(())
+    }
+
+    #[test]
+    fn bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd.arg("data:,Hello%2C%20World!").output().unwrap();
+
+        // STDOUT should contain HTML
+        assert_eq!(std::str::from_utf8(&out.stdout).unwrap(), "");
+
+        // STDERR should contain error description
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            "Unsupported data URL media type\n"
+        );
+
+        // The exit code should be 1
+        out.assert().code(1);
+
+        Ok(())
+    }
+
+    #[test]
+    fn isolate_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-I")
+            .arg("data:text/html,Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain isolated HTML
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:;\"></meta>\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_css_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-c")
+            .arg("data:text/html,<style>body{background-color:pink}</style>Hello")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no CSS
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none';\"></meta>\
+            <style></style>\
+            </head><body>Hello</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_fonts_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-F")
+            .arg("data:text/html,<style>@font-face { font-family: myFont; src: url(font.woff); }</style>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no web fonts
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"font-src 'none';\"></meta>\
+            <style></style>\
+            </head><body>Hi</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_frames_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-f")
+            .arg("data:text/html,<iframe src=\"https://duckduckgo.com\"></iframe>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no iframes
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none'; child-src 'none';\"></meta>\
+            </head><body><iframe src=\"\"></iframe>Hi</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_images_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-i")
+            .arg("data:text/html,<img src=\"https://google.com\"/>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no images
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            format!(
+                "<html>\
+                <head>\
+                <meta http-equiv=\"Content-Security-Policy\" content=\"img-src data:;\"></meta>\
+                </head>\
+                <body>\
+                <img src=\"{empty_image}\">\
+                Hi\
+                </body>\
+                </html>\n",
+                empty_image = empty_image!()
+            )
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_js_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-j")
+            .arg("data:text/html,<script>alert(2)</script>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no JS
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html>\
+            <head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"script-src 'none';\"></meta>\
+            <script></script></head>\
+            <body>Hi</body>\
+            </html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn local_file_target_input() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let cwd_normalized: String =
+            str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
+        let out = cmd
+            .arg("-M")
+            .arg(if cfg!(windows) {
+                "src\\tests\\data\\basic\\local-file.html"
+            } else {
+                "src/tests/data/basic/local-file.html"
+            })
+            .output()
+            .unwrap();
+        let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
+
+        // STDOUT should contain HTML from the local file
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "\
+            <!DOCTYPE html><html lang=\"en\"><head>\n  \
+            <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  \
+            <title>Local HTML file</title>\n  \
+            <link rel=\"stylesheet\" type=\"text/css\" href=\"data:text/css;base64,Ym9keSB7CiAgICBiYWNrZ3JvdW5kLWNvbG9yOiAjMDAwOwogICAgY29sb3I6ICNmZmY7Cn0K\">\n  \
+            <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
+            <img alt=\"\">\n  \
+            <a href=\"file://local-file.html/\">Tricky href</a>\n  \
+            <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
+            <script src=\"data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==\"></script>\n\n\n\n\
+            </body></html>\n\
+            "
+        );
+
+        // STDERR should contain list of retrieved file URLs
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            format!(
+                "\
+                {file}{cwd}/src/tests/data/basic/local-file.html\n \
+                {file}{cwd}/src/tests/data/basic/local-style.css\n \
+                {file}{cwd}/src/tests/data/basic/local-script.js\n\
+                ",
+                file = file_url_protocol,
+                cwd = cwd_normalized
+            )
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn local_file_target_input_absolute_target_path() -> Result<(), Box<dyn std::error::Error>> {
+        let cwd = env::current_dir().unwrap();
+        let cwd_normalized: String =
+            str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-jciI")
+            .arg(if cfg!(windows) {
+                format!(
+                    "{cwd}\\src\\tests\\data\\basic\\local-file.html",
+                    cwd = cwd.to_str().unwrap()
+                )
+            } else {
+                format!(
+                    "{cwd}/src/tests/data/basic/local-file.html",
+                    cwd = cwd.to_str().unwrap()
+                )
+            })
+            .output()
+            .unwrap();
+        let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
+
+        // STDOUT should contain HTML from the local file
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            format!(
+                "\
+                <!DOCTYPE html><html lang=\"en\"><head>\
+                <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:; style-src 'none'; script-src 'none'; img-src data:;\"></meta>\n  \
+                <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  \
+                <title>Local HTML file</title>\n  \
+                <link rel=\"stylesheet\" type=\"text/css\">\n  \
+                <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
+                <img src=\"{empty_image}\" alt=\"\">\n  \
+                <a href=\"file://local-file.html/\">Tricky href</a>\n  \
+                <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
+                <script></script>\n\n\n\n\
+                </body></html>\n\
+                ",
+                empty_image = empty_image!()
+            )
+        );
+
+        // STDERR should contain only the target file
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            format!(
+                "{file}{cwd}/src/tests/data/basic/local-file.html\n",
+                file = file_url_protocol,
+                cwd = cwd_normalized,
+            )
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn local_file_url_target_input() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let cwd_normalized: String =
+            str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
+        let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
+        let out = cmd
+            .arg("-M")
+            .arg("-cji")
+            .arg(if cfg!(windows) {
+                format!(
+                    "{file}{cwd}/src/tests/data/basic/local-file.html",
+                    file = file_url_protocol,
+                    cwd = cwd_normalized,
+                )
+            } else {
+                format!(
+                    "{file}{cwd}/src/tests/data/basic/local-file.html",
+                    file = file_url_protocol,
+                    cwd = cwd_normalized,
+                )
+            })
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML from the local file
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            format!(
+                "\
+                <!DOCTYPE html><html lang=\"en\"><head>\
+                <meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none'; script-src 'none'; img-src data:;\"></meta>\n  \
+                <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n  \
+                <title>Local HTML file</title>\n  \
+                <link rel=\"stylesheet\" type=\"text/css\">\n  \
+                <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
+                <img src=\"{empty_image}\" alt=\"\">\n  \
+                <a href=\"file://local-file.html/\">Tricky href</a>\n  \
+                <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
+                <script></script>\n\n\n\n\
+                </body></html>\n\
+                ",
+                empty_image = empty_image!()
+            )
+        );
+
+        // STDERR should contain list of retrieved file URLs
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            if cfg!(windows) {
+                format!(
+                    "{file}{cwd}/src/tests/data/basic/local-file.html\n",
+                    file = file_url_protocol,
+                    cwd = cwd_normalized,
+                )
+            } else {
+                format!(
+                    "{file}{cwd}/src/tests/data/basic/local-file.html\n",
+                    file = file_url_protocol,
+                    cwd = cwd_normalized,
+                )
+            }
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn security_disallow_local_assets_within_data_url_targets(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("data:text/html,%3Cscript%20src=\"src/tests/data/basic/local-script.js\"%3E%3C/script%3E")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no JS in it
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head><script></script></head><body></body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn embed_file_url_local_asset_within_style_attribute() -> Result<(), Box<dyn std::error::Error>>
+    {
+        let file_url_prefix: &str = if cfg!(windows) { "file:///" } else { "file://" };
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let mut file_svg = NamedTempFile::new()?;
+        writeln!(file_svg, "<svg version=\"1.1\" baseProfile=\"full\" width=\"300\" height=\"200\" xmlns=\"http://www.w3.org/2000/svg\">\
+                            <rect width=\"100%\" height=\"100%\" fill=\"red\" />\
+                            <circle cx=\"150\" cy=\"100\" r=\"80\" fill=\"green\" />\
+                            <text x=\"150\" y=\"125\" font-size=\"60\" text-anchor=\"middle\" fill=\"white\">SVG</text>\
+                            </svg>\n")?;
+        let mut file_html = NamedTempFile::new()?;
+        writeln!(
+            file_html,
+            "<div style='background-image: url(\"{file}{path}\")'></div>\n",
+            file = file_url_prefix,
+            path = str!(file_svg.path().to_str().unwrap()).replace("\\", "/"),
+        )?;
+        let out = cmd.arg("-M").arg(file_html.path()).output().unwrap();
+
+        // STDOUT should contain HTML with date URL for background-image in it
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head></head><body><div style=\"background-image: url('data:image/svg+xml;base64,PHN2ZyB2ZXJzaW9uPSIxLjEiIGJhc2VQcm9maWxlPSJmdWxsIiB3aWR0aD0iMzAwIiBoZWlnaHQ9IjIwMCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48cmVjdCB3aWR0aD0iMTAwJSIgaGVpZ2h0PSIxMDAlIiBmaWxsPSJyZWQiIC8+PGNpcmNsZSBjeD0iMTUwIiBjeT0iMTAwIiByPSI4MCIgZmlsbD0iZ3JlZW4iIC8+PHRleHQgeD0iMTUwIiB5PSIxMjUiIGZvbnQtc2l6ZT0iNjAiIHRleHQtYW5jaG9yPSJtaWRkbGUiIGZpbGw9IndoaXRlIj5TVkc8L3RleHQ+PC9zdmc+Cgo=')\"></div>\n\n</body></html>\n"
+        );
+
+        // STDERR should list temporary files that got retrieved
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            format!(
+                "\
+                {file}{html_path}\n \
+                {file}{svg_path}\n\
+                ",
+                file = file_url_prefix,
+                html_path = str!(file_html.path().to_str().unwrap()).replace("\\", "/"),
+                svg_path = str!(file_svg.path().to_str().unwrap()).replace("\\", "/"),
+            )
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn css_import_string() -> Result<(), Box<dyn std::error::Error>> {
+        let file_url_prefix: &str = if cfg!(windows) { "file:///" } else { "file://" };
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let mut file_css = NamedTempFile::new()?;
+        writeln!(file_css, "body{{background-color:#000;color:#fff}}")?;
+        let mut file_html = NamedTempFile::new()?;
+        writeln!(
+            file_html,
+            "\
+            <style>\n\
+            @charset 'UTF-8';\n\
+            \n\
+            @import '{file}{css_path}';\n\
+            \n\
+            @import url({file}{css_path});\n\
+            \n\
+            @import url('{file}{css_path}')\n\
+            </style>\n\
+            ",
+            file = file_url_prefix,
+            css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
+        )?;
+        let out = cmd.arg("-M").arg(file_html.path()).output().unwrap();
+
+        // STDOUT should contain embedded CSS url()'s
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head><style>\n@charset 'UTF-8';\n\n@import 'data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K';\n\n@import url('data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K');\n\n@import url('data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K')\n</style>\n\n</head><body></body></html>\n"
+        );
+
+        // STDERR should list temporary files that got retrieved
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            format!(
+                "\
+                {file}{html_path}\n \
+                {file}{css_path}\n \
+                {file}{css_path}\n \
+                {file}{css_path}\n\
+                ",
+                file = file_url_prefix,
+                html_path = str!(file_html.path().to_str().unwrap()).replace("\\", "/"),
+                css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
+            )
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}

src/tests/cli/mod.rs

@@ -0,0 +1,2 @@
+mod base_url;
+mod basic;

src/tests/css/embed_css.rs

@@ -0,0 +1,350 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::css;
+    use crate::opts::Options;
+
+    #[test]
+    fn empty_input() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let options = Options::default();
+
+        assert_eq!(css::embed_css(cache, &client, "", "", &options, 0), "");
+    }
+
+    #[test]
+    fn trim_if_empty() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let options = Options::default();
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                "\t     \t   ",
+                &options,
+                0,
+            ),
+            ""
+        );
+    }
+
+    #[test]
+    fn style_exclude_unquoted_images() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+
+        const STYLE: &str = "/* border: none;*/\
+            background-image: url(https://somewhere.com/bg.png); \
+            list-style: url(/assets/images/bullet.svg);\
+            width:99.998%; \
+            margin-top: -20px; \
+            line-height: -1; \
+            height: calc(100vh - 10pt)";
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                &STYLE,
+                &options,
+                0,
+            ),
+            format!(
+                "/* border: none;*/\
+                background-image: url('{empty_image}'); \
+                list-style: url('{empty_image}');\
+                width:99.998%; \
+                margin-top: -20px; \
+                line-height: -1; \
+                height: calc(100vh - 10pt)",
+                empty_image = empty_image!()
+            )
+        );
+    }
+
+    #[test]
+    fn style_exclude_single_quoted_images() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+
+        const STYLE: &str = "/* border: none;*/\
+            background-image: url('https://somewhere.com/bg.png'); \
+            list-style: url('/assets/images/bullet.svg');\
+            width:99.998%; \
+            margin-top: -20px; \
+            line-height: -1; \
+            height: calc(100vh - 10pt)";
+
+        assert_eq!(
+            css::embed_css(cache, &client, "", &STYLE, &options, 0),
+            format!(
+                "/* border: none;*/\
+                background-image: url('{empty_image}'); \
+                list-style: url('{empty_image}');\
+                width:99.998%; \
+                margin-top: -20px; \
+                line-height: -1; \
+                height: calc(100vh - 10pt)",
+                empty_image = empty_image!()
+            )
+        );
+    }
+
+    #[test]
+    fn style_block() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.silent = true;
+
+        const CSS: &str = "\
+            #id.class-name:not(:nth-child(3n+0)) {\n  \
+            // border: none;\n  \
+            background-image: url('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=');\n\
+            }\n\
+            \n\
+            html > body {}";
+
+        assert_eq!(
+            css::embed_css(cache, &client, "file:///", &CSS, &options, 0),
+            CSS
+        );
+    }
+
+    #[test]
+    fn attribute_selectors() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.silent = true;
+
+        const CSS: &str = "\
+            [data-value] {
+                /* Attribute exists */
+            }
+
+            [data-value='foo'] {
+                /* Attribute has this exact value */
+            }
+
+            [data-value*='foo'] {
+                /* Attribute value contains this value somewhere in it */
+            }
+
+            [data-value~='foo'] {
+                /* Attribute has this value in a space-separated list somewhere */
+            }
+
+            [data-value^='foo'] {
+                /* Attribute value starts with this */
+            }
+
+            [data-value|='foo'] {
+                /* Attribute value starts with this in a dash-separated list */
+            }
+
+            [data-value$='foo'] {
+                /* Attribute value ends with this */
+            }
+            ";
+
+        assert_eq!(css::embed_css(cache, &client, "", &CSS, &options, 0), CSS);
+    }
+
+    #[test]
+    fn import_string() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.silent = true;
+
+        const CSS: &str = "\
+            @charset 'UTF-8';\n\
+            \n\
+            @import 'data:text/css,html{background-color:%23000}';\n\
+            \n\
+            @import url('data:text/css,html{color:%23fff}')\n\
+            ";
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                &CSS,
+                &options,
+                0,
+            ),
+            "\
+            @charset 'UTF-8';\n\
+            \n\
+            @import 'data:text/css;base64,aHRtbHtiYWNrZ3JvdW5kLWNvbG9yOiMwMDB9';\n\
+            \n\
+            @import url('data:text/css;base64,aHRtbHtjb2xvcjojZmZmfQ==')\n\
+            "
+        );
+    }
+
+    #[test]
+    fn hash_urls() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.silent = true;
+
+        const CSS: &str = "\
+            body {\n    \
+                behavior: url(#default#something);\n\
+            }\n\
+            \n\
+            .scissorHalf {\n    \
+                offset-path: url(#somePath);\n\
+            }\n\
+            ";
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                &CSS,
+                &options,
+                0,
+            ),
+            CSS
+        );
+    }
+
+    #[test]
+    fn transform_percentages_and_degrees() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.silent = true;
+
+        const CSS: &str = "\
+            div {\n    \
+                transform: translate(-50%, -50%) rotate(-45deg);\n\
+                transform: translate(50%, 50%) rotate(45deg);\n\
+                transform: translate(+50%, +50%) rotate(+45deg);\n\
+            }\n\
+            ";
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                &CSS,
+                &options,
+                0,
+            ),
+            CSS
+        );
+    }
+
+    #[test]
+    fn unusual_indents() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.silent = true;
+
+        const CSS: &str = "\
+            .is\\:good:hover {\n    \
+                color: green\n\
+            }\n\
+            \n\
+            #\\~\\!\\@\\$\\%\\^\\&\\*\\(\\)\\+\\=\\,\\.\\/\\\\\\'\\\"\\;\\:\\?\\>\\<\\[\\]\\{\\}\\|\\`\\# {\n    \
+                color: black\n\
+            }\n\
+            ";
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                &CSS,
+                &options,
+                0,
+            ),
+            CSS
+        );
+    }
+
+    #[test]
+    fn exclude_fonts() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let mut options = Options::default();
+        options.no_fonts = true;
+        options.silent = true;
+
+        const CSS: &str = "\
+            @font-face {\n    \
+                font-family: 'My Font';\n    \
+                src: url(my_font.woff);\n\
+            }\n\
+            \n\
+            #identifier {\n    \
+                font-family: 'My Font' Arial\n\
+            }\n\
+            \n\
+            @font-face {\n    \
+                font-family: 'My Font';\n    \
+                src: url(my_font.woff);\n\
+            }\n\
+            \n\
+            div {\n    \
+                font-family: 'My Font' Verdana\n\
+            }\n\
+            ";
+
+        const CSS_OUT: &str = " \
+            \n\
+            \n\
+            #identifier {\n    \
+                font-family: 'My Font' Arial\n\
+            }\n\
+            \n \
+            \n\
+            \n\
+            div {\n    \
+                font-family: 'My Font' Verdana\n\
+            }\n\
+            ";
+
+        assert_eq!(
+            css::embed_css(
+                cache,
+                &client,
+                "https://doesntmatter.local/",
+                &CSS,
+                &options,
+                0,
+            ),
+            CSS_OUT
+        );
+    }
+}

src/tests/css/enquote.rs

@@ -0,0 +1,53 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::css;
+
+    #[test]
+    fn empty_input_single_quotes() {
+        assert_eq!(css::enquote(str!(""), false), "''");
+    }
+
+    #[test]
+    fn empty_input_double_quotes() {
+        assert_eq!(css::enquote(str!(""), true), "\"\"");
+    }
+
+    #[test]
+    fn apostrophes_single_quotes() {
+        assert_eq!(
+            css::enquote(str!("It's a lovely day, don't you think?"), false),
+            "'It\\'s a lovely day, don\\'t you think?'"
+        );
+    }
+
+    #[test]
+    fn apostrophes_double_quotes() {
+        assert_eq!(
+            css::enquote(str!("It's a lovely day, don't you think?"), true),
+            "\"It's a lovely day, don't you think?\""
+        );
+    }
+
+    #[test]
+    fn feet_and_inches_single_quotes() {
+        assert_eq!(
+            css::enquote(str!("5'2\", 6'5\""), false),
+            "'5\\'2\", 6\\'5\"'"
+        );
+    }
+
+    #[test]
+    fn feet_and_inches_double_quotes() {
+        assert_eq!(
+            css::enquote(str!("5'2\", 6'5\""), true),
+            "\"5'2\\\", 6'5\\\"\""
+        );
+    }
+}

src/tests/css/is_image_url_prop.rs

@@ -0,0 +1,88 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::css;
+
+    #[test]
+    fn backrgound() {
+        assert!(css::is_image_url_prop("background"));
+    }
+
+    #[test]
+    fn backrgound_image() {
+        assert!(css::is_image_url_prop("background-image"));
+    }
+
+    #[test]
+    fn backrgound_image_uppercase() {
+        assert!(css::is_image_url_prop("BACKGROUND-IMAGE"));
+    }
+
+    #[test]
+    fn border_image() {
+        assert!(css::is_image_url_prop("border-image"));
+    }
+
+    #[test]
+    fn content() {
+        assert!(css::is_image_url_prop("content"));
+    }
+
+    #[test]
+    fn cursor() {
+        assert!(css::is_image_url_prop("cursor"));
+    }
+
+    #[test]
+    fn list_style() {
+        assert!(css::is_image_url_prop("list-style"));
+    }
+
+    #[test]
+    fn list_style_image() {
+        assert!(css::is_image_url_prop("list-style-image"));
+    }
+
+    #[test]
+    fn mask_image() {
+        assert!(css::is_image_url_prop("mask-image"));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::css;
+
+    #[test]
+    fn empty() {
+        assert!(!css::is_image_url_prop(""));
+    }
+
+    #[test]
+    fn width() {
+        assert!(!css::is_image_url_prop("width"));
+    }
+
+    #[test]
+    fn color() {
+        assert!(!css::is_image_url_prop("color"));
+    }
+
+    #[test]
+    fn z_index() {
+        assert!(!css::is_image_url_prop("z-index"));
+    }
+}

src/tests/css/mod.rs

@@ -0,0 +1,3 @@
+mod embed_css;
+mod enquote;
+mod is_image_url_prop;

src/tests/data/basic/local-file.html

@@ -0,0 +1,19 @@
+<!doctype html>
+
+<html lang="en">
+
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <title>Local HTML file</title>
+  <link href="local-style.css" rel="stylesheet" type="text/css" />
+  <link href="local-style-does-not-exist.css" rel="stylesheet" type="text/css" />
+</head>
+
+<body>
+  <img src="monolith.png" alt="" />
+  <a href="//local-file.html">Tricky href</a>
+  <a href="https://github.com/Y2Z/monolith">Remote URL</a>
+  <script src="local-script.js"></script>
+</body>
+
+</html>

src/tests/data/basic/local-script.js

@@ -0,0 +1,2 @@
+document.body.style.backgroundColor = "green";
+document.body.style.color = "red";

src/tests/data/basic/local-style.css

@@ -0,0 +1,4 @@
+body {
+    background-color: #000;
+    color: #fff;
+}

src/tests/data/import-css-via-data-url/index.html

@@ -0,0 +1,23 @@
+<!doctype html>
+
+<html lang="en">
+
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <title>Attempt to import CSS via data URL asset</title>
+  <style>
+
+body {
+  background-color: white;
+  color: black;
+}
+
+  </style>
+  <link href="data:text/css;base64,QGltcG9ydCAic3R5bGUuY3NzIjsK" rel="stylesheet" type="text/css" />
+</head>
+
+<body>
+  <p>If you see pink background with white foreground then we’re in trouble</p>
+</body>
+
+</html>

src/tests/data/import-css-via-data-url/style.css

@@ -0,0 +1,4 @@
+body {
+  background-color: pink;
+  color: white;
+}

src/tests/data/integrity/index.html

@@ -0,0 +1,19 @@
+<!doctype html>
+
+<html lang="en">
+
+<head>
+  <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+  <meta http-equiv="Content-Security-Policy" content="default-src 'unsafe-inline' file:;" />
+  <title>Local HTML file</title>
+  <link href="style.css" rel="stylesheet" type="text/css" integrity="sha512-IWaCTORHkRhOWzcZeILSVmV6V6gPTHgNem6o6rsFAyaKTieDFkeeMrWjtO0DuWrX3bqZY46CVTZXUu0mia0qXQ==" crossorigin="anonymous" />
+  <link href="style.css" rel="stylesheet" type="text/css" integrity="sha512-vWBzl4NE9oIg8NFOPAyOZbaam0UXWr6aDHPaY2kodSzAFl+mKoj/RMNc6C31NDqK4mE2i68IWxYWqWJPLCgPOw==" crossorigin="anonymous" />
+</head>
+
+<body>
+  <p>This page should have black background and white foreground, but only when served via http: (not via file:)</p>
+  <script src="script.js" integrity="sha256-ecrEsYh3+ICCX8BCrNSotXgI5534282JwJjx8Q9ZWLc="></script>
+  <script src="script.js" integrity="sha256-6idk9dK0bOkVdG7Oz4/0YLXSJya8xZHqbRZKMhYrt6o="></script>
+</body>
+
+</html>

src/tests/data/integrity/script.js

@@ -0,0 +1,3 @@
+function noop() {
+    console.log("monolith");
+}

src/tests/data/integrity/style.css

@@ -0,0 +1,4 @@
+body {
+    background-color: #000;
+    color: #FFF;
+}

src/tests/html.rs

@@ -1,520 +0,0 @@
-use crate::html::{
-    get_node_name, get_parent_node, html_to_dom, is_icon, stringify_document, walk_and_embed_assets,
-};
-use html5ever::rcdom::{Handle, NodeData};
-use html5ever::serialize::{serialize, SerializeOpts};
-use reqwest::blocking::Client;
-use std::collections::HashMap;
-
-#[test]
-fn test_is_icon() {
-    assert_eq!(is_icon("icon"), true);
-    assert_eq!(is_icon("Shortcut Icon"), true);
-    assert_eq!(is_icon("ICON"), true);
-    assert_eq!(is_icon("mask-icon"), true);
-    assert_eq!(is_icon("fluid-icon"), true);
-    assert_eq!(is_icon("stylesheet"), false);
-    assert_eq!(is_icon(""), false);
-}
-
-#[test]
-fn test_get_parent_node_name() {
-    let html = "<!doctype html><html><HEAD></HEAD><body><div><P></P></div></body></html>";
-    let dom = html_to_dom(&html);
-    let mut count = 0;
-
-    fn test_walk(node: &Handle, i: &mut i8) {
-        *i += 1;
-
-        match &node.data {
-            NodeData::Document => {
-                for child in node.children.borrow().iter() {
-                    test_walk(child, &mut *i);
-                }
-            }
-            NodeData::Element { ref name, .. } => {
-                let node_name = name.local.as_ref().to_string();
-                let parent = get_parent_node(node);
-                let parent_node_name = get_node_name(&parent);
-                if node_name == "head" || node_name == "body" {
-                    assert_eq!(parent_node_name, "html");
-                } else if node_name == "div" {
-                    assert_eq!(parent_node_name, "body");
-                } else if node_name == "p" {
-                    assert_eq!(parent_node_name, "div");
-                }
-
-                println!("{}", node_name);
-
-                for child in node.children.borrow().iter() {
-                    test_walk(child, &mut *i);
-                }
-            }
-            _ => (),
-        };
-    }
-
-    test_walk(&dom.document, &mut count);
-
-    assert_eq!(count, 7);
-}
-
-#[test]
-fn test_walk_and_embed_assets() {
-    let cache = &mut HashMap::new();
-
-    let html = "<div><P></P></div>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_silent = true;
-
-    let client = Client::new();
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html><head></head><body><div><p></p></div></body></html>"
-    );
-}
-
-#[test]
-fn test_walk_and_embed_assets_ensure_no_recursive_iframe() {
-    let html = "<div><P></P><iframe src=\"\"></iframe></div>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-    let cache = &mut HashMap::new();
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_silent = true;
-
-    let client = Client::new();
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html><head></head><body><div><p></p><iframe src=\"\"></iframe></div></body></html>"
-    );
-}
-
-#[test]
-fn test_walk_and_embed_assets_no_css() {
-    let html = "<link rel=\"stylesheet\" href=\"main.css\">\
-                <style>html{background-color: #000;}</style>\
-                <div style=\"display: none;\"></div>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-    let cache = &mut HashMap::new();
-
-    let opt_no_css: bool = true;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_silent = true;
-    let client = Client::new();
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html>\
-         <head>\
-         <link rel=\"stylesheet\" href=\"\">\
-         <style></style>\
-         </head>\
-         <body>\
-         <div></div>\
-         </body>\
-         </html>"
-    );
-}
-
-#[test]
-fn test_walk_and_embed_assets_no_images() {
-    let html = "<link rel=\"icon\" href=\"favicon.ico\">\
-                <div><img src=\"http://localhost/assets/mono_lisa.png\" /></div>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-    let cache = &mut HashMap::new();
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = true;
-    let opt_silent = true;
-
-    let client = Client::new();
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html>\
-         <head>\
-         <link rel=\"icon\" href=\"\">\
-         </head>\
-         <body>\
-         <div>\
-         <img src=\"data:image/png;base64,\
-         iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0\
-         lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=\">\
-         </div>\
-         </body>\
-         </html>"
-    );
-}
-
-#[test]
-fn test_walk_and_embed_assets_no_frames() {
-    let html = "<iframe src=\"http://trackbook.com\"></iframe>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-    let cache = &mut HashMap::new();
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = true;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_silent = true;
-    let client = Client::new();
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html><head></head><body><iframe src=\"\"></iframe></body></html>"
-    );
-}
-
-#[test]
-fn test_walk_and_embed_assets_no_js() {
-    let html = "<div onClick=\"void(0)\">\
-                <script src=\"http://localhost/assets/some.js\"></script>\
-                <script>alert(1)</script>\
-                </div>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-    let cache = &mut HashMap::new();
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = true;
-    let opt_no_images: bool = false;
-    let opt_silent = true;
-
-    let client = Client::new();
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html><head></head><body><div><script src=\"\"></script>\
-         <script></script></div></body></html>"
-    );
-}
-
-#[test]
-fn test_walk_and_embed_with_no_integrity() {
-    let html = "<title>No integrity</title>\
-                <link integrity=\"sha384-...\" rel=\"something\"/>\
-                <script integrity=\"sha384-...\" src=\"some.js\"></script>";
-    let dom = html_to_dom(&html);
-    let url = "http://localhost";
-    let cache = &mut HashMap::new();
-    let client = Client::new();
-    let opt_no_css: bool = true;
-    let opt_no_frames: bool = true;
-    let opt_no_js: bool = true;
-    let opt_no_images: bool = true;
-    let opt_silent = true;
-
-    walk_and_embed_assets(
-        cache,
-        &client,
-        &url,
-        &dom.document,
-        opt_no_css,
-        opt_no_js,
-        opt_no_images,
-        opt_silent,
-        opt_no_frames,
-    );
-
-    let mut buf: Vec<u8> = Vec::new();
-    serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
-
-    assert_eq!(
-        buf.iter().map(|&c| c as char).collect::<String>(),
-        "<html>\
-         <head><title>No integrity</title><link rel=\"something\"><script src=\"\"></script></head>\
-         <body></body>\
-         </html>"
-    );
-}
-
-#[test]
-fn test_stringify_document() {
-    let html = "<div><script src=\"some.js\"></script></div>";
-    let dom = html_to_dom(&html);
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_isolate: bool = false;
-
-    assert_eq!(
-        stringify_document(
-            &dom.document,
-            opt_no_css,
-            opt_no_frames,
-            opt_no_js,
-            opt_no_images,
-            opt_isolate,
-        ),
-        "<html><head></head><body><div><script src=\"some.js\"></script></div></body></html>"
-    );
-}
-
-#[test]
-fn test_stringify_document_isolate() {
-    let html = "<title>Isolated document</title>\
-                <link rel=\"something\" href=\"some.css\" />\
-                <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
-                <div><script src=\"some.js\"></script></div>";
-    let dom = html_to_dom(&html);
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_isolate: bool = true;
-
-    assert_eq!(
-        stringify_document(
-            &dom.document,
-            opt_no_css,
-            opt_no_frames,
-            opt_no_js,
-            opt_no_images,
-            opt_isolate,
-        ),
-        "<html>\
-            <head>\
-                <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:;\"></meta>\
-                <title>Isolated document</title>\
-                <link rel=\"something\" href=\"some.css\">\
-                <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
-            </head>\
-            <body>\
-                <div>\
-                    <script src=\"some.js\"></script>\
-                </div>\
-            </body>\
-         </html>"
-    );
-}
-
-#[test]
-fn test_stringify_document_no_css() {
-    let html = "<!doctype html>\
-                <title>Unstyled document</title>\
-                <link rel=\"stylesheet\" href=\"main.css\"/>\
-                <div style=\"display: none;\"></div>";
-    let dom = html_to_dom(&html);
-
-    let opt_no_css: bool = true;
-    let opt_no_frames: bool = false;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_isolate: bool = false;
-
-    assert_eq!(
-        stringify_document(
-            &dom.document,
-            opt_no_css,
-            opt_no_frames,
-            opt_no_js,
-            opt_no_images,
-            opt_isolate,
-        ),
-        "<!DOCTYPE html>\
-         <html>\
-         <head>\
-         <meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none';\"></meta>\
-         <title>Unstyled document</title>\
-         <link rel=\"stylesheet\" href=\"main.css\">\
-         </head>\
-         <body><div style=\"display: none;\"></div></body>\
-         </html>"
-    );
-}
-
-#[test]
-fn test_stringify_document_no_frames() {
-    let html = "<!doctype html>\
-                <title>Frameless document</title>\
-                <link rel=\"something\"/>\
-                <div><script src=\"some.js\"></script></div>";
-    let dom = html_to_dom(&html);
-
-    let opt_no_css: bool = false;
-    let opt_no_frames: bool = true;
-    let opt_no_js: bool = false;
-    let opt_no_images: bool = false;
-    let opt_isolate: bool = false;
-
-    assert_eq!(
-        stringify_document(
-            &dom.document,
-            opt_no_css,
-            opt_no_frames,
-            opt_no_js,
-            opt_no_images,
-            opt_isolate,
-        ),
-        "<!DOCTYPE html>\
-            <html>\
-            <head>\
-            <meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none';child-src 'none';\"></meta>\
-            <title>Frameless document</title>\
-            <link rel=\"something\">\
-            </head>\
-            <body><div><script src=\"some.js\"></script></div></body>\
-            </html>"
-    );
-}
-
-#[test]
-fn test_stringify_document_isolate_no_frames_no_js_no_css_no_images() {
-    let html = "<!doctype html>\
-                <title>no-frame no-css no-js no-image isolated document</title>\
-                <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
-                <link rel=\"stylesheet\" href=\"some.css\">\
-                <div>\
-                <script src=\"some.js\"></script>\
-                <img style=\"width: 100%;\" src=\"some.png\" />\
-                <iframe src=\"some.html\"></iframe>\
-                </div>";
-    let dom = html_to_dom(&html);
-
-    let opt_isolate: bool = true;
-    let opt_no_css: bool = true;
-    let opt_no_frames: bool = true;
-    let opt_no_js: bool = true;
-    let opt_no_images: bool = true;
-
-    assert_eq!(
-        stringify_document(
-            &dom.document,
-            opt_no_css,
-            opt_no_frames,
-            opt_no_js,
-            opt_no_images,
-            opt_isolate,
-        ),
-        "<!DOCTYPE html>\
-            <html>\
-                <head>\
-                    <meta http-equiv=\"Content-Security-Policy\" content=\"default-src \'unsafe-inline\' data:; style-src \'none\'; frame-src \'none\';child-src \'none\'; script-src \'none\'; img-src data:;\"></meta>\
-                    <title>no-frame no-css no-js no-image isolated document</title>\
-                    <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
-                    <link rel=\"stylesheet\" href=\"some.css\">\
-                </head>\
-                <body>\
-                    <div>\
-                        <script src=\"some.js\"></script>\
-                        <img style=\"width: 100%;\" src=\"some.png\">\
-                        <iframe src=\"some.html\"></iframe>\
-                    </div>\
-                </body>\
-            </html>"
-    );
-}

src/tests/html/add_favicon.rs

@@ -0,0 +1,29 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::serialize::{serialize, SerializeOpts};
+
+    use crate::html;
+
+    #[test]
+    fn basic() {
+        let html = "<div>text</div>";
+        let mut dom = html::html_to_dom(&html);
+
+        dom = html::add_favicon(&dom.document, "I_AM_A_FAVICON_DATA_URL".to_string());
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head><link rel=\"icon\" href=\"I_AM_A_FAVICON_DATA_URL\"></link></head><body><div>text</div></body></html>"
+        );
+    }
+}

src/tests/html/check_integrity.rs

@@ -0,0 +1,89 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+
+    #[test]
+    fn empty_input_sha256() {
+        assert!(html::check_integrity(
+            "".as_bytes(),
+            "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
+        ));
+    }
+
+    #[test]
+    fn sha256() {
+        assert!(html::check_integrity(
+            "abcdef0123456789".as_bytes(),
+            "sha256-9EWAHgy4mSYsm54hmDaIDXPKLRsLnBX7lZyQ6xISNOM="
+        ));
+    }
+
+    #[test]
+    fn sha384() {
+        assert!(html::check_integrity(
+            "abcdef0123456789".as_bytes(),
+            "sha384-gc9l7omltke8C33bedgh15E12M7RrAQa5t63Yb8APlpe7ZhiqV23+oqiulSJl3Kw"
+        ));
+    }
+
+    #[test]
+    fn sha512() {
+        assert!(html::check_integrity(
+            "abcdef0123456789".as_bytes(),
+            "sha512-zG5B88cYMqcdiMi9gz0XkOFYw2BpjeYdn5V6+oFrMgSNjRpqL7EF8JEwl17ztZbK3N7I/tTwp3kxQbN1RgFBww=="
+        ));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn empty_hash() {
+        assert!(!html::check_integrity("abcdef0123456789".as_bytes(), ""));
+    }
+
+    #[test]
+    fn empty_input_empty_hash() {
+        assert!(!html::check_integrity("".as_bytes(), ""));
+    }
+
+    #[test]
+    fn sha256() {
+        assert!(!html::check_integrity(
+            "abcdef0123456789".as_bytes(),
+            "sha256-badhash"
+        ));
+    }
+
+    #[test]
+    fn sha384() {
+        assert!(!html::check_integrity(
+            "abcdef0123456789".as_bytes(),
+            "sha384-badhash"
+        ));
+    }
+
+    #[test]
+    fn sha512() {
+        assert!(!html::check_integrity(
+            "abcdef0123456789".as_bytes(),
+            "sha512-badhash"
+        ));
+    }
+}

src/tests/html/compose_csp.rs

@@ -0,0 +1,80 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn isolated() {
+        let mut options = Options::default();
+        options.isolate = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "default-src 'unsafe-inline' data:;");
+    }
+
+    #[test]
+    fn no_css() {
+        let mut options = Options::default();
+        options.no_css = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "style-src 'none';");
+    }
+
+    #[test]
+    fn no_fonts() {
+        let mut options = Options::default();
+        options.no_fonts = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "font-src 'none';");
+    }
+
+    #[test]
+    fn no_frames() {
+        let mut options = Options::default();
+        options.no_frames = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "frame-src 'none'; child-src 'none';");
+    }
+
+    #[test]
+    fn no_js() {
+        let mut options = Options::default();
+        options.no_js = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "script-src 'none';");
+    }
+
+    #[test]
+    fn no_images() {
+        let mut options = Options::default();
+        options.no_images = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "img-src data:;");
+    }
+
+    #[test]
+    fn all() {
+        let mut options = Options::default();
+        options.isolate = true;
+        options.no_css = true;
+        options.no_fonts = true;
+        options.no_frames = true;
+        options.no_js = true;
+        options.no_images = true;
+        let csp_content = html::compose_csp(&options);
+
+        assert_eq!(csp_content, "default-src 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;");
+    }
+}

src/tests/html/create_metadata_tag.rs

@@ -0,0 +1,82 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use chrono::prelude::*;
+
+    use crate::html;
+
+    #[test]
+    fn http_url() {
+        let url = "http://192.168.1.1/";
+        let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
+        let metadata_comment: String = html::create_metadata_tag(url);
+
+        assert_eq!(
+            metadata_comment,
+            format!(
+                "<!-- Saved from {} at {} using {} v{} -->",
+                &url,
+                timestamp,
+                env!("CARGO_PKG_NAME"),
+                env!("CARGO_PKG_VERSION"),
+            )
+        );
+    }
+
+    #[test]
+    fn file_url() {
+        let url = "file:///home/monolith/index.html";
+        let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
+        let metadata_comment: String = html::create_metadata_tag(url);
+
+        assert_eq!(
+            metadata_comment,
+            format!(
+                "<!-- Saved from local source at {} using {} v{} -->",
+                timestamp,
+                env!("CARGO_PKG_NAME"),
+                env!("CARGO_PKG_VERSION"),
+            )
+        );
+    }
+
+    #[test]
+    fn data_url() {
+        let url = "data:text/html,Hello%2C%20World!";
+        let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
+        let metadata_comment: String = html::create_metadata_tag(url);
+
+        assert_eq!(
+            metadata_comment,
+            format!(
+                "<!-- Saved from local source at {} using {} v{} -->",
+                timestamp,
+                env!("CARGO_PKG_NAME"),
+                env!("CARGO_PKG_VERSION"),
+            )
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn empty_string() {
+        assert_eq!(html::create_metadata_tag(""), "");
+    }
+}

src/tests/html/embed_srcset.rs

@@ -0,0 +1,100 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn replace_with_empty_images() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small.png 1x, large.png 2x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{} 1x, {} 2x", empty_image!(), empty_image!()),
+        );
+    }
+
+    #[test]
+    fn commas_within_file_names() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small,s.png 1x, large,l.png 2x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{} 1x, {} 2x", empty_image!(), empty_image!()),
+        );
+    }
+
+    #[test]
+    fn tabs_and_newlines_after_commas() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small,s.png 1x,\nmedium,m.png 2x,\nlarge,l.png 3x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!(
+                "{} 1x, {} 2x, {} 3x",
+                empty_image!(),
+                empty_image!(),
+                empty_image!()
+            ),
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn trailing_comma() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small.png 1x, large.png 2x,";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{} 1x, {} 2x,", empty_image!(), empty_image!()),
+        );
+    }
+}

src/tests/html/get_base_url.rs

@@ -0,0 +1,104 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+
+    #[test]
+    fn present() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"https://musicbrainz.org\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(
+            html::get_base_url(&dom.document),
+            Some(str!("https://musicbrainz.org"))
+        );
+    }
+
+    #[test]
+    fn multiple_tags() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"https://www.discogs.com/\" />
+        <base href=\"https://musicbrainz.org\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(
+            html::get_base_url(&dom.document),
+            Some(str!("https://www.discogs.com/"))
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn absent() {
+        let html = "<!doctype html>
+<html>
+    <head>
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), None);
+    }
+
+    #[test]
+    fn no_href() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), None);
+    }
+
+    #[test]
+    fn empty_href() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), Some(str!()));
+    }
+}

src/tests/html/get_node_attr.rs

@@ -0,0 +1,54 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn div_two_style_attributes() {
+        let html = "<!doctype html><html><head></head><body><DIV STYLE=\"color: blue;\" style=\"display: none;\"></div></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "class"), None);
+                    } else if node_name == "div" {
+                        assert_eq!(
+                            html::get_node_attr(node, "style"),
+                            Some(str!("color: blue;"))
+                        );
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 6);
+    }
+}

src/tests/html/get_node_name.rs

@@ -0,0 +1,53 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn parent_node_names() {
+        let html = "<!doctype html><html><HEAD></HEAD><body><div><P></P></div></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+                    let parent = html::get_parent_node(node);
+                    let parent_node_name = html::get_node_name(&parent);
+                    if node_name == "head" || node_name == "body" {
+                        assert_eq!(parent_node_name, Some("html"));
+                    } else if node_name == "div" {
+                        assert_eq!(parent_node_name, Some("body"));
+                    } else if node_name == "p" {
+                        assert_eq!(parent_node_name, Some("div"));
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 7);
+    }
+}

src/tests/html/has_favicon.rs

@@ -0,0 +1,50 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+
+    #[test]
+    fn icon() {
+        let html = "<link rel=\"icon\" href=\"\" /><div>text</div>";
+        let dom = html::html_to_dom(&html);
+        let res: bool = html::has_favicon(&dom.document);
+
+        assert!(res);
+    }
+
+    #[test]
+    fn shortcut_icon() {
+        let html = "<link rel=\"shortcut icon\" href=\"\" /><div>text</div>";
+        let dom = html::html_to_dom(&html);
+        let res: bool = html::has_favicon(&dom.document);
+
+        assert!(res);
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn absent() {
+        let html = "<div>text</div>";
+        let dom = html::html_to_dom(&html);
+        let res: bool = html::has_favicon(&dom.document);
+
+        assert!(!res);
+    }
+}

src/tests/html/is_icon.rs

@@ -0,0 +1,58 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+
+    #[test]
+    fn icon() {
+        assert!(html::is_icon("icon"));
+    }
+
+    #[test]
+    fn shortcut_icon_capitalized() {
+        assert!(html::is_icon("Shortcut Icon"));
+    }
+
+    #[test]
+    fn icon_uppercase() {
+        assert!(html::is_icon("ICON"));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn mask_icon() {
+        assert!(!html::is_icon("mask-icon"));
+    }
+
+    #[test]
+    fn fluid_icon() {
+        assert!(!html::is_icon("fluid-icon"));
+    }
+
+    #[test]
+    fn stylesheet() {
+        assert!(!html::is_icon("stylesheet"));
+    }
+
+    #[test]
+    fn empty_string() {
+        assert!(!html::is_icon(""));
+    }
+}

src/tests/html/mod.rs

@@ -0,0 +1,13 @@
+mod add_favicon;
+mod check_integrity;
+mod compose_csp;
+mod create_metadata_tag;
+mod embed_srcset;
+mod get_base_url;
+mod get_node_attr;
+mod get_node_name;
+mod has_favicon;
+mod is_icon;
+mod set_node_attr;
+mod stringify_document;
+mod walk_and_embed_assets;

src/tests/html/set_node_attr.rs

@@ -0,0 +1,105 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn html_lang_and_body_style() {
+        let html = "<!doctype html><html lang=\"en\"><head></head><body></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "html" {
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("en")));
+
+                        html::set_node_attr(node, "lang", Some(str!("de")));
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("de")));
+
+                        html::set_node_attr(node, "lang", None);
+                        assert_eq!(html::get_node_attr(node, "lang"), None);
+
+                        html::set_node_attr(node, "lang", Some(str!("")));
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("")));
+                    } else if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "style"), None);
+
+                        html::set_node_attr(node, "style", Some(str!("display: none;")));
+                        assert_eq!(
+                            html::get_node_attr(node, "style"),
+                            Some(str!("display: none;"))
+                        );
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 5);
+    }
+
+    #[test]
+    fn body_background() {
+        let html = "<!doctype html><html lang=\"en\"><head></head><body background=\"1\" background=\"2\"></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "background"), Some(str!("1")));
+
+                        html::set_node_attr(node, "background", None);
+                        assert_eq!(html::get_node_attr(node, "background"), None);
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 5);
+    }
+}

src/tests/html/stringify_document.rs

@@ -0,0 +1,150 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn div_as_root_element() {
+        let html = "<div><script src=\"some.js\"></script></div>";
+        let dom = html::html_to_dom(&html);
+        let options = Options::default();
+
+        assert_eq!(
+            html::stringify_document(&dom.document, &options),
+            "<html><head></head><body><div><script src=\"some.js\"></script></div></body></html>"
+        );
+    }
+
+    #[test]
+    fn full_page_with_no_html_head_or_body() {
+        let html = "<title>Isolated document</title>\
+                    <link rel=\"something\" href=\"some.css\" />\
+                    <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
+                    <div><script src=\"some.js\"></script></div>";
+        let dom = html::html_to_dom(&html);
+        let mut options = Options::default();
+        options.isolate = true;
+
+        assert_eq!(
+            html::stringify_document(
+                &dom.document,
+                &options
+            ),
+            "<html>\
+                <head>\
+                    <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:;\"></meta>\
+                    <title>Isolated document</title>\
+                    <link rel=\"something\" href=\"some.css\">\
+                    <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
+                </head>\
+                <body>\
+                    <div>\
+                        <script src=\"some.js\"></script>\
+                    </div>\
+                </body>\
+            </html>"
+        );
+    }
+
+    #[test]
+    fn doctype_and_the_rest_no_html_head_or_body() {
+        let html = "<!doctype html>\
+                    <title>Unstyled document</title>\
+                    <link rel=\"stylesheet\" href=\"main.css\"/>\
+                    <div style=\"display: none;\"></div>";
+        let dom = html::html_to_dom(&html);
+        let mut options = Options::default();
+        options.no_css = true;
+
+        assert_eq!(
+            html::stringify_document(&dom.document, &options),
+            "<!DOCTYPE html>\
+            <html>\
+            <head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none';\"></meta>\
+            <title>Unstyled document</title>\
+            <link rel=\"stylesheet\" href=\"main.css\">\
+            </head>\
+            <body><div style=\"display: none;\"></div></body>\
+            </html>"
+        );
+    }
+
+    #[test]
+    fn doctype_and_the_rest_no_html_head_or_body_forbid_frames() {
+        let html = "<!doctype html>\
+                    <title>Frameless document</title>\
+                    <link rel=\"something\"/>\
+                    <div><script src=\"some.js\"></script></div>";
+        let dom = html::html_to_dom(&html);
+        let mut options = Options::default();
+        options.no_frames = true;
+
+        assert_eq!(
+            html::stringify_document(
+                &dom.document,
+                &options
+            ),
+            "<!DOCTYPE html>\
+                <html>\
+                <head>\
+                <meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none'; child-src 'none';\"></meta>\
+                <title>Frameless document</title>\
+                <link rel=\"something\">\
+                </head>\
+                <body><div><script src=\"some.js\"></script></div></body>\
+                </html>"
+        );
+    }
+
+    #[test]
+    fn doctype_and_the_rest_all_forbidden() {
+        let html = "<!doctype html>\
+                    <title>no-frame no-css no-js no-image isolated document</title>\
+                    <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
+                    <link rel=\"stylesheet\" href=\"some.css\">\
+                    <div>\
+                        <script src=\"some.js\"></script>\
+                        <img style=\"width: 100%;\" src=\"some.png\" />\
+                        <iframe src=\"some.html\"></iframe>\
+                    </div>";
+        let dom = html::html_to_dom(&html);
+        let mut options = Options::default();
+        options.isolate = true;
+        options.no_css = true;
+        options.no_fonts = true;
+        options.no_frames = true;
+        options.no_js = true;
+        options.no_images = true;
+
+        assert_eq!(
+            html::stringify_document(
+                &dom.document,
+                &options
+            ),
+            "<!DOCTYPE html>\
+                <html>\
+                    <head>\
+                        <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;\"></meta>\
+                        <title>no-frame no-css no-js no-image isolated document</title>\
+                        <meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
+                        <link rel=\"stylesheet\" href=\"some.css\">\
+                    </head>\
+                    <body>\
+                        <div>\
+                            <script src=\"some.js\"></script>\
+                            <img style=\"width: 100%;\" src=\"some.png\">\
+                            <iframe src=\"some.html\"></iframe>\
+                        </div>\
+                    </body>\
+                </html>"
+        );
+    }
+}

src/tests/html/walk_and_embed_assets.rs

@@ -0,0 +1,329 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::serialize::{serialize, SerializeOpts};
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn basic() {
+        let cache = &mut HashMap::new();
+
+        let html = "<div><P></P></div>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><body><div><p></p></div></body></html>"
+        );
+    }
+
+    #[test]
+    fn ensure_no_recursive_iframe() {
+        let html = "<div><P></P><iframe src=\"\"></iframe></div>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><body><div><p></p><iframe src=\"\"></iframe></div></body></html>"
+        );
+    }
+
+    #[test]
+    fn ensure_no_recursive_frame() {
+        let html = "<frameset><frame src=\"\"></frameset>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><frameset><frame src=\"\"></frameset></html>"
+        );
+    }
+
+    #[test]
+    fn no_css() {
+        let html = "<link rel=\"stylesheet\" href=\"main.css\">\
+                    <style>html{background-color: #000;}</style>\
+                    <div style=\"display: none;\"></div>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_css = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html>\
+            <head>\
+            <link rel=\"stylesheet\">\
+            <style></style>\
+            </head>\
+            <body>\
+            <div></div>\
+            </body>\
+            </html>"
+        );
+    }
+
+    #[test]
+    fn no_images() {
+        let html = "<link rel=\"icon\" href=\"favicon.ico\">\
+                    <div><img src=\"http://localhost/assets/mono_lisa.png\" /></div>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            format!(
+                "<html>\
+                    <head>\
+                        <link rel=\"icon\">\
+                    </head>\
+                    <body>\
+                        <div>\
+                            <img src=\"{empty_image}\">\
+                        </div>\
+                    </body>\
+                </html>",
+                empty_image = empty_image!()
+            )
+        );
+    }
+
+    #[test]
+    fn no_body_background_images() {
+        let html =
+            "<body background=\"no/such/image.png\" background=\"no/such/image2.png\"></body>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><body></body></html>"
+        );
+    }
+
+    #[test]
+    fn no_frames() {
+        let html = "<frameset><frame src=\"http://trackbook.com\"></frameset>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_frames = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><frameset><frame src=\"\"></frameset></html>"
+        );
+    }
+
+    #[test]
+    fn no_iframes() {
+        let html = "<iframe src=\"http://trackbook.com\"></iframe>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_frames = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><body><iframe src=\"\"></iframe></body></html>"
+        );
+    }
+
+    #[test]
+    fn no_js() {
+        let html = "<div onClick=\"void(0)\">\
+                        <script src=\"http://localhost/assets/some.js\"></script>\
+                        <script>alert(1)</script>\
+                    </div>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_js = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html><head></head><body><div><script></script>\
+            <script></script></div></body></html>"
+        );
+    }
+
+    #[test]
+    fn discards_integrity() {
+        let html = "<title>No integrity</title>\
+                    <link integrity=\"sha384-...\" rel=\"something\"/>\
+                    <script integrity=\"sha384-...\" src=\"some.js\"></script>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_css = true;
+        options.no_frames = true;
+        options.no_js = true;
+        options.no_images = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html>\
+                <head><title>No integrity</title><link rel=\"something\"><script></script></head>\
+                <body></body>\
+            </html>"
+        );
+    }
+
+    #[test]
+    fn removes_unwanted_meta_tags() {
+        let html = "<html>\
+            <head>\
+                <meta http-equiv=\"Refresh\" value=\"20\"/>\
+                <meta http-equiv=\"Location\" value=\"https://freebsd.org\"/>\
+            </head>\
+            <body></body>\
+        </html>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_css = true;
+        options.no_frames = true;
+        options.no_js = true;
+        options.no_images = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            "<html>\
+                <head>\
+                    <meta http-equiv=\"disabled by monolith (Refresh)\" value=\"20\">\
+                    <meta http-equiv=\"disabled by monolith (Location)\" value=\"https://freebsd.org\">\
+                </head>\
+                <body></body>\
+            </html>"
+        );
+    }
+}

src/tests/http.rs

@@ -1,25 +0,0 @@
-use crate::http::retrieve_asset;
-use reqwest::blocking::Client;
-use std::collections::HashMap;
-
-#[test]
-fn test_retrieve_asset() {
-    let cache = &mut HashMap::new();
-    let client = Client::new();
-    let (data, final_url) =
-        retrieve_asset(cache, &client, "data:text/html;base64,...", true, "", false).unwrap();
-    assert_eq!(&data, "data:text/html;base64,...");
-    assert_eq!(&final_url, "data:text/html;base64,...");
-
-    let (data, final_url) = retrieve_asset(
-        cache,
-        &client,
-        "data:text/html;base64,...",
-        true,
-        "image/png",
-        false,
-    )
-    .unwrap();
-    assert_eq!(&data, "data:text/html;base64,...");
-    assert_eq!(&final_url, "data:text/html;base64,...");
-}

src/tests/js.rs

@@ -1,13 +0,0 @@
-use crate::js::attr_is_event_handler;
-
-#[test]
-fn test_attr_is_event_handler() {
-    // succeeding
-    assert!(attr_is_event_handler("onBlur"));
-    assert!(attr_is_event_handler("onclick"));
-    assert!(attr_is_event_handler("onClick"));
-    // failing
-    assert!(!attr_is_event_handler("href"));
-    assert!(!attr_is_event_handler(""));
-    assert!(!attr_is_event_handler("class"));
-}

src/tests/js/attr_is_event_handler.rs

@@ -0,0 +1,53 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::js;
+
+    #[test]
+    fn onblur_camelcase() {
+        assert!(js::attr_is_event_handler("onBlur"));
+    }
+
+    #[test]
+    fn onclick_lowercase() {
+        assert!(js::attr_is_event_handler("onclick"));
+    }
+
+    #[test]
+    fn onclick_camelcase() {
+        assert!(js::attr_is_event_handler("onClick"));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::js;
+
+    #[test]
+    fn href() {
+        assert!(!js::attr_is_event_handler("href"));
+    }
+
+    #[test]
+    fn empty_string() {
+        assert!(!js::attr_is_event_handler(""));
+    }
+
+    #[test]
+    fn class() {
+        assert!(!js::attr_is_event_handler("class"));
+    }
+}

src/tests/js/mod.rs

@@ -0,0 +1 @@
+mod attr_is_event_handler;

src/tests/macros/empty_image.rs

@@ -0,0 +1,14 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    #[test]
+    fn contains_correct_image_data() {
+        assert_eq!(empty_image!(), "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAA0AAAANCAQAAADY4iz3AAAAEUlEQVR42mNkwAkYR6UolgIACvgADsuK6xYAAAAASUVORK5CYII=");
+    }
+}

src/tests/macros/mod.rs

@@ -0,0 +1,2 @@
+mod empty_image;
+mod str;

src/tests/macros/str.rs

@@ -0,0 +1,24 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    #[test]
+    fn returns_empty_string() {
+        assert_eq!(str!(), "");
+    }
+
+    #[test]
+    fn converts_integer_into_string() {
+        assert_eq!(str!(123), "123");
+    }
+
+    #[test]
+    fn converts_str_into_string() {
+        assert_eq!(str!("abc"), "abc");
+    }
+}

src/tests/mod.rs

@@ -1,4 +1,8 @@
+mod cli;
+mod css;
 mod html;
-mod http;
 mod js;
+mod macros;
+mod opts;
+mod url;
 mod utils;

src/tests/opts.rs

@@ -0,0 +1,32 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::opts::Options;
+
+    #[test]
+    fn defaults() {
+        let options: Options = Options::default();
+
+        assert_eq!(options.target, str!());
+        assert_eq!(options.no_audio, false);
+        assert_eq!(options.no_css, false);
+        assert_eq!(options.no_frames, false);
+        assert_eq!(options.no_fonts, false);
+        assert_eq!(options.no_images, false);
+        assert_eq!(options.isolate, false);
+        assert_eq!(options.no_js, false);
+        assert_eq!(options.insecure, false);
+        assert_eq!(options.no_metadata, false);
+        assert_eq!(options.output, str!());
+        assert_eq!(options.silent, false);
+        assert_eq!(options.timeout, 0);
+        assert_eq!(options.user_agent, "");
+        assert_eq!(options.no_video, false);
+    }
+}

src/tests/url/clean_url.rs

@@ -0,0 +1,51 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn removes_fragment() {
+        assert_eq!(
+            url::clean_url("https://somewhere.com/font.eot#iefix"),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment() {
+        assert_eq!(
+            url::clean_url("https://somewhere.com/font.eot#"),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_query_and_empty_fragment() {
+        assert_eq!(
+            url::clean_url("https://somewhere.com/font.eot?#"),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_query_amp_and_empty_fragment() {
+        assert_eq!(
+            url::clean_url("https://somewhere.com/font.eot?a=b&#"),
+            "https://somewhere.com/font.eot?a=b"
+        );
+    }
+
+    #[test]
+    fn keeps_credentials() {
+        assert_eq!(
+            url::clean_url("https://cookie:monster@gibson.internet/"),
+            "https://cookie:monster@gibson.internet/"
+        );
+    }
+}

src/tests/url/data_to_data_url.rs

@@ -0,0 +1,31 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn encode_string_with_specific_media_type() {
+        let mime = "application/javascript";
+        let data = "var word = 'hello';\nalert(word);\n";
+        let data_url = url::data_to_data_url(mime, data.as_bytes(), "");
+
+        assert_eq!(
+            &data_url,
+            "data:application/javascript;base64,dmFyIHdvcmQgPSAnaGVsbG8nOwphbGVydCh3b3JkKTsK"
+        );
+    }
+
+    #[test]
+    fn encode_append_fragment() {
+        let data = "<svg></svg>\n";
+        let data_url = url::data_to_data_url("image/svg+xml", data.as_bytes(), "");
+
+        assert_eq!(&data_url, "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K");
+    }
+}

src/tests/url/decode_url.rs

@@ -0,0 +1,39 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn decode_unicode_characters() {
+        assert_eq!(
+            url::decode_url(str!(
+                "%E6%A4%9C%E3%83%92%E3%83%A0%E8%A7%A3%E5%A1%97%E3%82%83%E3%83%83%20%3D%20%E3%82%B5"
+            )),
+            "検ヒム解塗ゃッ = サ"
+        );
+    }
+
+    #[test]
+    fn decode_file_url() {
+        assert_eq!(
+            url::decode_url(str!("file:///tmp/space%20here/test%231.html")),
+            "file:///tmp/space here/test#1.html"
+        );
+    }
+
+    #[test]
+    fn plus_sign() {
+        assert_eq!(
+            url::decode_url(str!(
+                "fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
+            )),
+            "fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
+        );
+    }
+}

src/tests/url/file_url_to_fs_path.rs

@@ -0,0 +1,41 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn remove_protocl_and_fragment() {
+        if cfg!(windows) {
+            assert_eq!(
+                url::file_url_to_fs_path("file:///C:/documents/some-path/some-file.svg#fragment"),
+                "C:\\documents\\some-path\\some-file.svg"
+            );
+        } else {
+            assert_eq!(
+                url::file_url_to_fs_path("file:///tmp/some-path/some-file.svg#fragment"),
+                "/tmp/some-path/some-file.svg"
+            );
+        }
+    }
+
+    #[test]
+    fn decodes_urls() {
+        if cfg!(windows) {
+            assert_eq!(
+                url::file_url_to_fs_path("file:///C:/Documents%20and%20Settings/some-file.html"),
+                "C:\\Documents and Settings\\some-file.html"
+            );
+        } else {
+            assert_eq!(
+                url::file_url_to_fs_path("file:///home/user/My%20Documents"),
+                "/home/user/My Documents"
+            );
+        }
+    }
+}

src/tests/url/get_url_fragment.rs

@@ -0,0 +1,48 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn data_url() {
+        assert_eq!(
+            url::get_url_fragment(
+                "data:image/svg+xml;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h#test"
+            ),
+            "test"
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn https_empty() {
+        assert_eq!(url::get_url_fragment("https://kernel.org#"), "");
+    }
+
+    #[test]
+    fn no_fragment() {
+        assert_eq!(url::get_url_fragment("https://kernel.org"), "");
+    }
+
+    #[test]
+    fn dummy_data_url() {
+        assert_eq!(url::get_url_fragment("data:text/html,"), "");
+    }
+}

src/tests/url/is_data_url.rs

@@ -0,0 +1,52 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn data_url_text_html() {
+        assert!(url::is_data_url(
+            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
+        ));
+    }
+
+    #[test]
+    fn data_url_no_media_type() {
+        assert!(url::is_data_url(
+            "data:;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
+        ));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn https_url() {
+        assert!(!url::is_data_url("https://kernel.org"));
+    }
+
+    #[test]
+    fn no_protocol_url() {
+        assert!(!url::is_data_url("//kernel.org"));
+    }
+
+    #[test]
+    fn empty_string() {
+        assert!(!url::is_data_url(""));
+    }
+}

src/tests/url/is_file_url.rs

@@ -0,0 +1,83 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn unix_file_url() {
+        assert!(url::is_file_url(
+            "file:///home/user/Websites/my-website/index.html"
+        ));
+    }
+
+    #[test]
+    fn windows_file_url() {
+        assert!(url::is_file_url(
+            "file:///C:/Documents%20and%20Settings/user/Websites/my-website/assets/images/logo.png"
+        ));
+    }
+
+    #[test]
+    fn unix_url_with_backslashes() {
+        assert!(url::is_file_url(
+            "file:\\\\\\home\\user\\Websites\\my-website\\index.html"
+        ));
+    }
+
+    #[test]
+    fn windows_file_url_with_backslashes() {
+        assert!(url::is_file_url(
+            "file:\\\\\\C:\\Documents%20and%20Settings\\user\\Websites\\my-website\\assets\\images\\logo.png"
+        ));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn url_with_no_protocl() {
+        assert!(!url::is_file_url("//kernel.org"));
+    }
+
+    #[test]
+    fn dot_slash_filename() {
+        assert!(!url::is_file_url("./index.html"));
+    }
+
+    #[test]
+    fn just_filename() {
+        assert!(!url::is_file_url("some-local-page.htm"));
+    }
+
+    #[test]
+    fn https_ip_port_url() {
+        assert!(!url::is_file_url("https://1.2.3.4:80/www/index.html"));
+    }
+
+    #[test]
+    fn data_url() {
+        assert!(!url::is_file_url(
+            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
+        ));
+    }
+
+    #[test]
+    fn just_word_file() {
+        assert!(!url::is_file_url("file"));
+    }
+}

src/tests/url/is_http_url.rs

@@ -0,0 +1,65 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn http_url() {
+        assert!(url::is_http_url("http://kernel.org"));
+    }
+
+    #[test]
+    fn https_url() {
+        assert!(url::is_http_url("https://www.rust-lang.org/"));
+    }
+
+    #[test]
+    fn http_url_with_backslashes() {
+        assert!(url::is_http_url("http:\\\\freebsd.org\\"));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn url_with_no_protocol() {
+        assert!(!url::is_http_url("//kernel.org"));
+    }
+
+    #[test]
+    fn dot_slash_filename() {
+        assert!(!url::is_http_url("./index.html"));
+    }
+
+    #[test]
+    fn just_filename() {
+        assert!(!url::is_http_url("some-local-page.htm"));
+    }
+
+    #[test]
+    fn https_ip_port_url() {
+        assert!(!url::is_http_url("ftp://1.2.3.4/www/index.html"));
+    }
+
+    #[test]
+    fn data_url() {
+        assert!(!url::is_http_url(
+            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
+        ));
+    }
+}

src/tests/url/mod.rs

@@ -0,0 +1,12 @@
+mod clean_url;
+mod data_to_data_url;
+mod decode_url;
+mod file_url_to_fs_path;
+mod get_url_fragment;
+mod is_data_url;
+mod is_file_url;
+mod is_http_url;
+mod parse_data_url;
+mod resolve_url;
+mod url_has_protocol;
+mod url_with_fragment;

src/tests/url/parse_data_url.rs

@@ -0,0 +1,103 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn parse_text_html_base64() {
+        let (media_type, data) = url::parse_data_url("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
+
+        assert_eq!(media_type, "text/html");
+        assert_eq!(
+            String::from_utf8_lossy(&data),
+            "Work expands so as to fill the time available for its completion"
+        );
+    }
+
+    #[test]
+    fn parse_text_html_utf8() {
+        let (media_type, data) = url::parse_data_url(
+            "data:text/html;utf8,Work expands so as to fill the time available for its completion",
+        );
+
+        assert_eq!(media_type, "text/html");
+        assert_eq!(
+            String::from_utf8_lossy(&data),
+            "Work expands so as to fill the time available for its completion"
+        );
+    }
+
+    #[test]
+    fn parse_text_html_plaintext() {
+        let (media_type, data) = url::parse_data_url(
+            "data:text/html,Work expands so as to fill the time available for its completion",
+        );
+
+        assert_eq!(media_type, "text/html");
+        assert_eq!(
+            String::from_utf8_lossy(&data),
+            "Work expands so as to fill the time available for its completion"
+        );
+    }
+
+    #[test]
+    fn parse_text_html_charset_utf_8_between_two_whitespaces() {
+        let (media_type, data) = url::parse_data_url(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
+
+        assert_eq!(media_type, "text/html");
+        assert_eq!(
+            String::from_utf8_lossy(&data),
+            "Work expands so as to fill the time available for its completion"
+        );
+    }
+
+    #[test]
+    fn parse_text_css_url_encoded() {
+        let (media_type, data) = url::parse_data_url("data:text/css,div{background-color:%23000}");
+
+        assert_eq!(media_type, "text/css");
+        assert_eq!(String::from_utf8_lossy(&data), "div{background-color:#000}");
+    }
+
+    #[test]
+    fn parse_no_media_type_base64() {
+        let (media_type, data) = url::parse_data_url("data:;base64,dGVzdA==");
+
+        assert_eq!(media_type, "");
+        assert_eq!(String::from_utf8_lossy(&data), "test");
+    }
+
+    #[test]
+    fn parse_no_media_type_no_encoding() {
+        let (media_type, data) = url::parse_data_url("data:;,test%20test");
+
+        assert_eq!(media_type, "");
+        assert_eq!(String::from_utf8_lossy(&data), "test test");
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn just_word_data() {
+        let (media_type, data) = url::parse_data_url("data");
+
+        assert_eq!(media_type, "");
+        assert_eq!(String::from_utf8_lossy(&data), "");
+    }
+}

src/tests/url/resolve_url.rs

@@ -0,0 +1,194 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn from_https_to_level_up_relative() {
+        assert_eq!(
+            url::resolve_url("https://www.kernel.org", "../category/signatures.html")
+                .unwrap_or_default(),
+            "https://www.kernel.org/category/signatures.html"
+        );
+    }
+
+    #[test]
+    fn from_just_filename_to_full_https_url() {
+        assert_eq!(
+            url::resolve_url(
+                "saved_page.htm",
+                "https://www.kernel.org/category/signatures.html",
+            )
+            .unwrap_or_default(),
+            "https://www.kernel.org/category/signatures.html"
+        );
+    }
+
+    #[test]
+    fn from_https_url_to_url_with_no_protocol() {
+        assert_eq!(
+            url::resolve_url(
+                "https://www.kernel.org",
+                "//www.kernel.org/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
+            "https://www.kernel.org/theme/images/logos/tux.png"
+        );
+    }
+
+    #[test]
+    fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() {
+        assert_eq!(
+            url::resolve_url(
+                "https://www.kernel.org",
+                "//another-host.org/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
+            "https://another-host.org/theme/images/logos/tux.png"
+        );
+    }
+
+    #[test]
+    fn from_https_url_to_relative_root_path() {
+        assert_eq!(
+            url::resolve_url(
+                "https://www.kernel.org/category/signatures.html",
+                "/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
+            "https://www.kernel.org/theme/images/logos/tux.png"
+        );
+    }
+
+    #[test]
+    fn from_https_to_just_filename() {
+        assert_eq!(
+            url::resolve_url(
+                "https://www.w3schools.com/html/html_iframe.asp",
+                "default.asp",
+            )
+            .unwrap_or_default(),
+            "https://www.w3schools.com/html/default.asp"
+        );
+    }
+
+    #[test]
+    fn from_data_url_to_https() {
+        assert_eq!(
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "https://www.kernel.org/category/signatures.html",
+            )
+            .unwrap_or_default(),
+            "https://www.kernel.org/category/signatures.html"
+        );
+    }
+
+    #[test]
+    fn from_data_url_to_data_url() {
+        assert_eq!(
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K",
+            )
+            .unwrap_or_default(),
+            "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K"
+        );
+    }
+
+    #[test]
+    fn from_file_url_to_relative_path() {
+        assert_eq!(
+            url::resolve_url(
+                "file:///home/user/Websites/my-website/index.html",
+                "assets/images/logo.png",
+            )
+            .unwrap_or_default(),
+            "file:///home/user/Websites/my-website/assets/images/logo.png"
+        );
+    }
+
+    #[test]
+    fn from_file_url_to_relative_path_with_backslashes() {
+        assert_eq!(
+            url::resolve_url(
+                "file:\\\\\\home\\user\\Websites\\my-website\\index.html",
+                "assets\\images\\logo.png",
+            )
+            .unwrap_or_default(),
+            "file:///home/user/Websites/my-website/assets/images/logo.png"
+        );
+    }
+
+    #[test]
+    fn from_data_url_to_file_url() {
+        assert_eq!(
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "file:///etc/passwd",
+            )
+            .unwrap_or_default(),
+            "file:///etc/passwd"
+        );
+    }
+
+    #[test]
+    fn preserve_fragment() {
+        assert_eq!(
+            url::resolve_url(
+                "http://doesnt-matter.local/",
+                "css/fonts/fontmarvelous.svg#fontmarvelous",
+            )
+            .unwrap_or_default(),
+            "http://doesnt-matter.local/css/fonts/fontmarvelous.svg#fontmarvelous"
+        );
+    }
+
+    #[test]
+    fn resolve_from_file_url_to_file_url() {
+        assert_eq!(
+            if cfg!(windows) {
+                url::resolve_url("file:///c:/index.html", "file:///c:/image.png")
+                    .unwrap_or_default()
+            } else {
+                url::resolve_url("file:///tmp/index.html", "file:///tmp/image.png")
+                    .unwrap_or_default()
+            },
+            if cfg!(windows) {
+                "file:///c:/image.png"
+            } else {
+                "file:///tmp/image.png"
+            }
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn from_data_url_to_url_with_no_protocol() {
+        assert_eq!(
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "//www.w3schools.com/html/html_iframe.asp",
+            )
+            .unwrap_or_default(),
+            ""
+        );
+    }
+}

src/tests/url/url_has_protocol.rs

@@ -0,0 +1,94 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn mailto() {
+        assert!(url::url_has_protocol(
+            "mailto:somebody@somewhere.com?subject=hello"
+        ));
+    }
+
+    #[test]
+    fn tel() {
+        assert!(url::url_has_protocol("tel:5551234567"));
+    }
+
+    #[test]
+    fn ftp_no_slashes() {
+        assert!(url::url_has_protocol("ftp:some-ftp-server.com"));
+    }
+
+    #[test]
+    fn ftp_with_credentials() {
+        assert!(url::url_has_protocol(
+            "ftp://user:password@some-ftp-server.com"
+        ));
+    }
+
+    #[test]
+    fn javascript() {
+        assert!(url::url_has_protocol("javascript:void(0)"));
+    }
+
+    #[test]
+    fn http() {
+        assert!(url::url_has_protocol("http://news.ycombinator.com"));
+    }
+
+    #[test]
+    fn https() {
+        assert!(url::url_has_protocol("https://github.com"));
+    }
+
+    #[test]
+    fn mailto_uppercase() {
+        assert!(url::url_has_protocol(
+            "MAILTO:somebody@somewhere.com?subject=hello"
+        ));
+    }
+
+    #[test]
+    fn empty_data_url() {
+        assert!(url::url_has_protocol("data:text/html,"));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
+
+    #[test]
+    fn url_with_no_protocol() {
+        assert!(!url::url_has_protocol("//some-hostname.com/some-file.html"));
+    }
+
+    #[test]
+    fn relative_path() {
+        assert!(!url::url_has_protocol("some-hostname.com/some-file.html"));
+    }
+
+    #[test]
+    fn relative_to_root_path() {
+        assert!(!url::url_has_protocol("/some-file.html"));
+    }
+
+    #[test]
+    fn empty_string() {
+        assert!(!url::url_has_protocol(""));
+    }
+}

src/tests/url/url_with_fragment.rs

@@ -0,0 +1,40 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::url;
+
+    #[test]
+    fn url_with_fragment_url() {
+        let url = "https://localhost.localdomain/path/";
+        let fragment = "test";
+        let assembled_url = url::url_with_fragment(url, fragment);
+
+        assert_eq!(&assembled_url, "https://localhost.localdomain/path/#test");
+    }
+    #[test]
+    fn url_with_fragment_empty_url() {
+        let url = "https://localhost.localdomain/path/";
+        let fragment = "";
+        let assembled_url = url::url_with_fragment(url, fragment);
+
+        assert_eq!(&assembled_url, "https://localhost.localdomain/path/");
+    }
+
+    #[test]
+    fn url_with_fragment_data_url() {
+        let url = "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K";
+        let fragment = "fragment";
+        let assembled_url = url::url_with_fragment(url, fragment);
+
+        assert_eq!(
+            &assembled_url,
+            "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K#fragment"
+        );
+    }
+}

src/tests/utils.rs

@@ -1,177 +0,0 @@
-use crate::utils::{
-    clean_url, data_to_dataurl, detect_mimetype, is_data_url, is_valid_url, resolve_url,
-    url_has_protocol,
-};
-use url::ParseError;
-
-#[test]
-fn test_data_to_dataurl() {
-    let mime = "application/javascript";
-    let data = "var word = 'hello';\nalert(word);\n";
-    let datauri = data_to_dataurl(mime, data.as_bytes());
-    assert_eq!(
-        &datauri,
-        "data:application/javascript;base64,dmFyIHdvcmQgPSAnaGVsbG8nOwphbGVydCh3b3JkKTsK"
-    );
-}
-
-#[test]
-fn test_detect_mimetype() {
-    // image
-    assert_eq!(detect_mimetype(b"GIF87a"), "image/gif");
-    assert_eq!(detect_mimetype(b"GIF89a"), "image/gif");
-    assert_eq!(detect_mimetype(b"\xFF\xD8\xFF"), "image/jpeg");
-    assert_eq!(detect_mimetype(b"\x89PNG\x0D\x0A\x1A\x0A"), "image/png");
-    assert_eq!(detect_mimetype(b"<?xml "), "image/svg+xml");
-    assert_eq!(detect_mimetype(b"<svg "), "image/svg+xml");
-    assert_eq!(detect_mimetype(b"RIFF....WEBPVP8 "), "image/webp");
-    assert_eq!(detect_mimetype(b"\x00\x00\x01\x00"), "image/x-icon");
-    // audio
-    assert_eq!(detect_mimetype(b"ID3"), "audio/mpeg");
-    assert_eq!(detect_mimetype(b"\xFF\x0E"), "audio/mpeg");
-    assert_eq!(detect_mimetype(b"\xFF\x0F"), "audio/mpeg");
-    assert_eq!(detect_mimetype(b"OggS"), "audio/ogg");
-    assert_eq!(detect_mimetype(b"RIFF....WAVEfmt "), "audio/wav");
-    assert_eq!(detect_mimetype(b"fLaC"), "audio/x-flac");
-    // video
-    assert_eq!(detect_mimetype(b"RIFF....AVI LIST"), "video/avi");
-    assert_eq!(detect_mimetype(b"....ftyp"), "video/mp4");
-    assert_eq!(detect_mimetype(b"\x00\x00\x01\x0B"), "video/mpeg");
-    assert_eq!(detect_mimetype(b"....moov"), "video/quicktime");
-    assert_eq!(detect_mimetype(b"\x1A\x45\xDF\xA3"), "video/webm");
-}
-
-#[test]
-fn test_url_has_protocol() {
-    // succeeding
-    assert_eq!(
-        url_has_protocol("mailto:somebody@somewhere.com?subject=hello"),
-        true
-    );
-    assert_eq!(url_has_protocol("tel:5551234567"), true);
-    assert_eq!(
-        url_has_protocol("ftp:user:password@some-ftp-server.com"),
-        true
-    );
-    assert_eq!(url_has_protocol("javascript:void(0)"), true);
-    assert_eq!(url_has_protocol("http://news.ycombinator.com"), true);
-    assert_eq!(url_has_protocol("https://github.com"), true);
-    assert_eq!(
-        url_has_protocol("MAILTO:somebody@somewhere.com?subject=hello"),
-        true
-    );
-    // failing
-    assert_eq!(
-        url_has_protocol("//some-hostname.com/some-file.html"),
-        false
-    );
-    assert_eq!(url_has_protocol("some-hostname.com/some-file.html"), false);
-    assert_eq!(url_has_protocol("/some-file.html"), false);
-    assert_eq!(url_has_protocol(""), false);
-}
-
-#[test]
-fn test_is_valid_url() {
-    // succeeding
-    assert!(is_valid_url("https://www.rust-lang.org/"));
-    assert!(is_valid_url("http://kernel.org"));
-    // failing
-    assert!(!is_valid_url("//kernel.org"));
-    assert!(!is_valid_url("./index.html"));
-    assert!(!is_valid_url("some-local-page.htm"));
-    assert!(!is_valid_url("ftp://1.2.3.4/www/index.html"));
-    assert!(!is_valid_url(
-        "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
-    ));
-}
-
-#[test]
-fn test_resolve_url() -> Result<(), ParseError> {
-    let resolved_url = resolve_url("https://www.kernel.org", "../category/signatures.html")?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://www.kernel.org/category/signatures.html"
-    );
-
-    let resolved_url = resolve_url("https://www.kernel.org", "category/signatures.html")?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://www.kernel.org/category/signatures.html"
-    );
-
-    let resolved_url = resolve_url(
-        "saved_page.htm",
-        "https://www.kernel.org/category/signatures.html",
-    )?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://www.kernel.org/category/signatures.html"
-    );
-
-    let resolved_url = resolve_url(
-        "https://www.kernel.org",
-        "//www.kernel.org/theme/images/logos/tux.png",
-    )?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://www.kernel.org/theme/images/logos/tux.png"
-    );
-
-    let resolved_url = resolve_url(
-        "https://www.kernel.org",
-        "//another-host.org/theme/images/logos/tux.png",
-    )?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://another-host.org/theme/images/logos/tux.png"
-    );
-
-    let resolved_url = resolve_url(
-        "https://www.kernel.org/category/signatures.html",
-        "/theme/images/logos/tux.png",
-    )?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://www.kernel.org/theme/images/logos/tux.png"
-    );
-
-    let resolved_url = resolve_url(
-        "https://www.w3schools.com/html/html_iframe.asp",
-        "default.asp",
-    )?;
-    assert_eq!(
-        resolved_url.as_str(),
-        "https://www.w3schools.com/html/default.asp"
-    );
-
-    Ok(())
-}
-
-#[test]
-fn test_is_data_url() {
-    // succeeding
-    assert!(
-        is_data_url("data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h")
-            .unwrap_or(false)
-    );
-    // failing
-    assert!(!is_data_url("https://kernel.org").unwrap_or(false));
-    assert!(!is_data_url("//kernel.org").unwrap_or(false));
-    assert!(!is_data_url("").unwrap_or(false));
-}
-
-#[test]
-fn test_clean_url() {
-    assert_eq!(
-        clean_url("https://somewhere.com/font.eot#iefix"),
-        "https://somewhere.com/font.eot"
-    );
-    assert_eq!(
-        clean_url("https://somewhere.com/font.eot#"),
-        "https://somewhere.com/font.eot"
-    );
-    assert_eq!(
-        clean_url("https://somewhere.com/font.eot?#"),
-        "https://somewhere.com/font.eot"
-    );
-}

src/tests/utils/detect_media_type.rs

@@ -0,0 +1,155 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::utils;
+
+    #[test]
+    fn image_gif87() {
+        assert_eq!(utils::detect_media_type(b"GIF87a", ""), "image/gif");
+    }
+
+    #[test]
+    fn image_gif89() {
+        assert_eq!(utils::detect_media_type(b"GIF89a", ""), "image/gif");
+    }
+
+    #[test]
+    fn image_jpeg() {
+        assert_eq!(utils::detect_media_type(b"\xFF\xD8\xFF", ""), "image/jpeg");
+    }
+
+    #[test]
+    fn image_png() {
+        assert_eq!(
+            utils::detect_media_type(b"\x89PNG\x0D\x0A\x1A\x0A", ""),
+            "image/png"
+        );
+    }
+
+    #[test]
+    fn image_svg() {
+        assert_eq!(utils::detect_media_type(b"<svg ", ""), "image/svg+xml");
+    }
+
+    #[test]
+    fn image_webp() {
+        assert_eq!(
+            utils::detect_media_type(b"RIFF....WEBPVP8 ", ""),
+            "image/webp"
+        );
+    }
+
+    #[test]
+    fn image_icon() {
+        assert_eq!(
+            utils::detect_media_type(b"\x00\x00\x01\x00", ""),
+            "image/x-icon"
+        );
+    }
+
+    #[test]
+    fn image_svg_filename() {
+        assert_eq!(
+            utils::detect_media_type(b"<?xml ", "local-file.svg"),
+            "image/svg+xml"
+        );
+    }
+
+    #[test]
+    fn image_svg_url_uppercase() {
+        assert_eq!(
+            utils::detect_media_type(b"", "https://some-site.com/images/local-file.SVG"),
+            "image/svg+xml"
+        );
+    }
+
+    #[test]
+    fn audio_mpeg() {
+        assert_eq!(utils::detect_media_type(b"ID3", ""), "audio/mpeg");
+    }
+
+    #[test]
+    fn audio_mpeg_2() {
+        assert_eq!(utils::detect_media_type(b"\xFF\x0E", ""), "audio/mpeg");
+    }
+
+    #[test]
+    fn audio_mpeg_3() {
+        assert_eq!(utils::detect_media_type(b"\xFF\x0F", ""), "audio/mpeg");
+    }
+
+    #[test]
+    fn audio_ogg() {
+        assert_eq!(utils::detect_media_type(b"OggS", ""), "audio/ogg");
+    }
+
+    #[test]
+    fn audio_wav() {
+        assert_eq!(
+            utils::detect_media_type(b"RIFF....WAVEfmt ", ""),
+            "audio/wav"
+        );
+    }
+
+    #[test]
+    fn audio_flac() {
+        assert_eq!(utils::detect_media_type(b"fLaC", ""), "audio/x-flac");
+    }
+
+    #[test]
+    fn video_avi() {
+        assert_eq!(
+            utils::detect_media_type(b"RIFF....AVI LIST", ""),
+            "video/avi"
+        );
+    }
+
+    #[test]
+    fn video_mp4() {
+        assert_eq!(utils::detect_media_type(b"....ftyp", ""), "video/mp4");
+    }
+
+    #[test]
+    fn video_mpeg() {
+        assert_eq!(
+            utils::detect_media_type(b"\x00\x00\x01\x0B", ""),
+            "video/mpeg"
+        );
+    }
+
+    #[test]
+    fn video_quicktime() {
+        assert_eq!(utils::detect_media_type(b"....moov", ""), "video/quicktime");
+    }
+
+    #[test]
+    fn video_webm() {
+        assert_eq!(
+            utils::detect_media_type(b"\x1A\x45\xDF\xA3", ""),
+            "video/webm"
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::utils;
+
+    #[test]
+    fn unknown_media_type() {
+        assert_eq!(utils::detect_media_type(b"abcdef0123456789", ""), "");
+    }
+}

src/tests/utils/indent.rs

@@ -0,0 +1,31 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::utils;
+
+    #[test]
+    fn zero() {
+        assert_eq!(utils::indent(0), "");
+    }
+
+    #[test]
+    fn one() {
+        assert_eq!(utils::indent(1), " ");
+    }
+
+    #[test]
+    fn two() {
+        assert_eq!(utils::indent(2), "  ");
+    }
+
+    #[test]
+    fn three() {
+        assert_eq!(utils::indent(3), "   ");
+    }
+}

src/tests/utils/mod.rs

@@ -0,0 +1,3 @@
+mod detect_media_type;
+mod indent;
+mod retrieve_asset;

src/tests/utils/retrieve_asset.rs

@@ -0,0 +1,155 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+    use std::env;
+
+    use crate::opts::Options;
+    use crate::url;
+    use crate::utils;
+
+    #[test]
+    fn read_data_url() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        // If both source and target are data URLs,
+        // ensure the result contains target data URL
+        let (data, final_url, media_type) = utils::retrieve_asset(
+            cache,
+            &client,
+            "data:text/html;base64,c291cmNl",
+            "data:text/html;base64,dGFyZ2V0",
+            &options,
+            0,
+        )
+        .unwrap();
+        assert_eq!(
+            url::data_to_data_url(&media_type, &data, &final_url),
+            url::data_to_data_url("text/html", "target".as_bytes(), "")
+        );
+        assert_eq!(
+            final_url,
+            url::data_to_data_url("text/html", "target".as_bytes(), "")
+        );
+        assert_eq!(&media_type, "text/html");
+    }
+
+    #[test]
+    fn read_local_file_with_file_url_parent() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
+
+        // Inclusion of local assets from local sources should be allowed
+        let cwd = env::current_dir().unwrap();
+        let (data, final_url, _media_type) = utils::retrieve_asset(
+            cache,
+            &client,
+            &format!(
+                "{file}{cwd}/src/tests/data/basic/local-file.html",
+                file = file_url_protocol,
+                cwd = cwd.to_str().unwrap()
+            ),
+            &format!(
+                "{file}{cwd}/src/tests/data/basic/local-script.js",
+                file = file_url_protocol,
+                cwd = cwd.to_str().unwrap()
+            ),
+            &options,
+            0,
+        )
+        .unwrap();
+        assert_eq!(url::data_to_data_url("application/javascript", &data, &final_url), "data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==");
+        assert_eq!(
+            &final_url,
+            &format!(
+                "{file}{cwd}/src/tests/data/basic/local-script.js",
+                file = file_url_protocol,
+                cwd = cwd.to_str().unwrap()
+            )
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::opts::Options;
+    use crate::utils;
+
+    #[test]
+    fn read_local_file_with_data_url_parent() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        // Inclusion of local assets from data URL sources should not be allowed
+        match utils::retrieve_asset(
+            cache,
+            &client,
+            "data:text/html;base64,SoUrCe",
+            "file:///etc/passwd",
+            &options,
+            0,
+        ) {
+            Ok((..)) => {
+                assert!(false);
+            }
+            Err(_) => {
+                assert!(true);
+            }
+        }
+    }
+
+    #[test]
+    fn read_local_file_with_https_parent() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+
+        let mut options = Options::default();
+        options.silent = true;
+
+        // Inclusion of local assets from remote sources should not be allowed
+        match utils::retrieve_asset(
+            cache,
+            &client,
+            "https://kernel.org/",
+            "file:///etc/passwd",
+            &options,
+            0,
+        ) {
+            Ok((..)) => {
+                assert!(false);
+            }
+            Err(_) => {
+                assert!(true);
+            }
+        }
+    }
+}

src/url.rs

@@ -0,0 +1,167 @@
+use base64;
+use url::{form_urlencoded, ParseError, Url};
+
+use crate::utils::detect_media_type;
+
+pub fn clean_url<T: AsRef<str>>(input: T) -> String {
+    let mut url = Url::parse(input.as_ref()).unwrap();
+
+    // Clear fragment
+    url.set_fragment(None);
+
+    // Get rid of stray question mark
+    if url.query() == Some("") {
+        url.set_query(None);
+    }
+
+    // Remove empty trailing ampersand(s)
+    let mut result: String = url.to_string();
+    while result.ends_with("&") {
+        result.pop();
+    }
+
+    result
+}
+
+pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str) -> String {
+    let media_type: String = if media_type.is_empty() {
+        detect_media_type(data, &url)
+    } else {
+        media_type.to_string()
+    };
+
+    format!("data:{};base64,{}", media_type, base64::encode(data))
+}
+
+pub fn decode_url(input: String) -> String {
+    let input: String = input.replace("+", "%2B");
+
+    form_urlencoded::parse(input.as_bytes())
+        .map(|(key, val)| {
+            [
+                key.to_string(),
+                if val.to_string().len() == 0 {
+                    str!()
+                } else {
+                    str!('=')
+                },
+                val.to_string(),
+            ]
+            .concat()
+        })
+        .collect()
+}
+
+pub fn file_url_to_fs_path(url: &str) -> String {
+    if !is_file_url(url) {
+        return str!();
+    }
+
+    let cutoff_l = if cfg!(windows) { 8 } else { 7 };
+    let mut fs_file_path: String = decode_url(url.to_string()[cutoff_l..].to_string());
+    let url_fragment = get_url_fragment(url);
+    if url_fragment != "" {
+        let max_len = fs_file_path.len() - 1 - url_fragment.len();
+        fs_file_path = fs_file_path[0..max_len].to_string();
+    }
+
+    if cfg!(windows) {
+        fs_file_path = fs_file_path.replace("/", "\\");
+    }
+
+    // File paths should not be %-encoded
+    decode_url(fs_file_path)
+}
+
+pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
+    match Url::parse(url.as_ref()) {
+        Ok(parsed_url) => parsed_url.fragment().unwrap_or("").to_string(),
+        Err(_err) => str!(),
+    }
+}
+
+pub fn is_data_url<T: AsRef<str>>(url: T) -> bool {
+    Url::parse(url.as_ref())
+        .and_then(|u| Ok(u.scheme() == "data"))
+        .unwrap_or(false)
+}
+
+pub fn is_file_url<T: AsRef<str>>(url: T) -> bool {
+    Url::parse(url.as_ref())
+        .and_then(|u| Ok(u.scheme() == "file"))
+        .unwrap_or(false)
+}
+
+pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
+    Url::parse(url.as_ref())
+        .and_then(|u| Ok(u.scheme() == "http" || u.scheme() == "https"))
+        .unwrap_or(false)
+}
+
+pub fn parse_data_url<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
+    let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
+    let path: String = parsed_url.path().to_string();
+    let comma_loc: usize = path.find(',').unwrap_or(path.len());
+
+    let meta_data: String = path.chars().take(comma_loc).collect();
+    let raw_data: String = path.chars().skip(comma_loc + 1).collect();
+
+    let text: String = decode_url(raw_data);
+
+    let meta_data_items: Vec<&str> = meta_data.split(';').collect();
+    let mut media_type: String = str!();
+    let mut encoding: &str = "";
+
+    let mut i: i8 = 0;
+    for item in &meta_data_items {
+        if i == 0 {
+            media_type = str!(item);
+        } else {
+            if item.eq_ignore_ascii_case("base64")
+                || item.eq_ignore_ascii_case("utf8")
+                || item.eq_ignore_ascii_case("charset=UTF-8")
+            {
+                encoding = item;
+            }
+        }
+
+        i = i + 1;
+    }
+
+    let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
+        base64::decode(&text).unwrap_or(vec![])
+    } else {
+        text.as_bytes().to_vec()
+    };
+
+    (media_type, data)
+}
+
+pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
+    let result = if is_http_url(to.as_ref()) {
+        to.as_ref().to_string()
+    } else {
+        Url::parse(from.as_ref())?
+            .join(to.as_ref())?
+            .as_ref()
+            .to_string()
+    };
+    Ok(result)
+}
+
+pub fn url_has_protocol<T: AsRef<str>>(url: T) -> bool {
+    Url::parse(url.as_ref())
+        .and_then(|u| Ok(u.scheme().len() > 0))
+        .unwrap_or(false)
+}
+
+pub fn url_with_fragment(url: &str, fragment: &str) -> String {
+    let mut result = str!(&url);
+
+    if !fragment.is_empty() {
+        result += "#";
+        result += fragment;
+    }
+
+    result
+}

src/utils.rs

@@ -1,54 +1,20 @@
-use crate::http::retrieve_asset;
-use base64::encode;
-use regex::Regex;
 use reqwest::blocking::Client;
+use reqwest::header::CONTENT_TYPE;
 use std::collections::HashMap;
-use url::{ParseError, Url};
+use std::fs;
+use std::path::Path;
 
-/// This monster of a regex is used to match any kind of URL found in CSS.
-///
-/// There  are roughly three different categories that a found URL could fit
-/// into:
-///    - Font       [found after a src: property in an @font-family rule]
-///    - Stylesheet [denoted by an @import before the url
-///    - Image      [covers all other uses of the url() function]
-///
-/// This regex aims to extract the following information:
-///    - What type of URL is it (font/image/css)
-///    - Where is the part that needs to be replaced (incl any wrapping quotes)
-///    - What is the URL (excl any wrapping quotes)
-///
-/// Essentially, the regex can be broken down into two parts:
-///
-/// `(?:(?P<import>@import)|(?P<font>src\s*:)\s+)?`
-/// This matches the precursor to a font or CSS URL, and fills in a match under
-/// either `<import>` (if it's a CSS URL) or `<font>` (if it's a font).
-/// Determining whether or not it's an image can be done by the negation of both
-/// of these. Either zero or one of these can match.
-///
-/// `url\((?P<to_repl>['"]?(?P<url>[^"'\)]+)['"]?)\)`
-/// This matches the actual URL part of the url(), and must always match. It also
-/// sets `<to_repl>` and `<url>` which correspond to everything within
-/// `url(...)` and a usable URL, respectively.
-///
-/// Note, however, that this does not perform any validation of the found URL.
-/// Malformed CSS could lead to an invalid URL being present. It is therefore
-/// recomended that the URL gets manually validated.
-const CSS_URL_REGEX_STR: &str = r###"(?:(?:(?P<stylesheet>@import)|(?P<font>src\s*:))\s+)?url\((?P<to_repl>['"]?(?P<url>[^"'\)]+)['"]?)\)"###;
+use crate::opts::Options;
+use crate::url::{clean_url, file_url_to_fs_path, is_data_url, is_file_url, parse_data_url};
 
-lazy_static! {
-    static ref HAS_PROTOCOL: Regex = Regex::new(r"^[a-z0-9]+:").unwrap();
-    static ref REGEX_URL: Regex = Regex::new(r"^https?://").unwrap();
-    static ref REGEX_CSS_URL: Regex = Regex::new(CSS_URL_REGEX_STR).unwrap();
-}
+const INDENT: &str = " ";
 
-const MAGIC: [[&[u8]; 2]; 19] = [
+const MAGIC: [[&[u8]; 2]; 18] = [
     // Image
     [b"GIF87a", b"image/gif"],
     [b"GIF89a", b"image/gif"],
     [b"\xFF\xD8\xFF", b"image/jpeg"],
     [b"\x89PNG\x0D\x0A\x1A\x0A", b"image/png"],
-    [b"<?xml ", b"image/svg+xml"],
     [b"<svg ", b"image/svg+xml"],
     [b"RIFF....WEBPVP8 ", b"image/webp"],
     [b"\x00\x00\x01\x00", b"image/x-icon"],
@@ -67,141 +33,134 @@ const MAGIC: [[&[u8]; 2]; 19] = [
     [b"\x1A\x45\xDF\xA3", b"video/webm"],
 ];
 
-pub fn data_to_dataurl(mime: &str, data: &[u8]) -> String {
-    let mimetype = if mime.is_empty() {
-        detect_mimetype(data)
-    } else {
-        mime.to_string()
-    };
-    format!("data:{};base64,{}", mimetype, encode(data))
-}
+const PLAINTEXT_MEDIA_TYPES: &[&str] = &[
+    "image/svg+xml",
+    "text/css",
+    "text/html",
+    "text/javascript",
+    "text/plain",
+];
 
-pub fn detect_mimetype(data: &[u8]) -> String {
+pub fn detect_media_type(data: &[u8], url: &str) -> String {
     for item in MAGIC.iter() {
         if data.starts_with(item[0]) {
             return String::from_utf8(item[1].to_vec()).unwrap();
         }
     }
-    "".to_owned()
+
+    if url.to_lowercase().ends_with(".svg") {
+        return str!("image/svg+xml");
+    }
+
+    str!()
 }
 
-pub fn url_has_protocol<T: AsRef<str>>(url: T) -> bool {
-    HAS_PROTOCOL.is_match(url.as_ref().to_lowercase().as_str())
+pub fn is_plaintext_media_type(media_type: &str) -> bool {
+    PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
 }
 
-pub fn is_data_url<T: AsRef<str>>(url: T) -> Result<bool, ParseError> {
-    Url::parse(url.as_ref()).and_then(|u| Ok(u.scheme() == "data"))
+pub fn indent(level: u32) -> String {
+    let mut result = str!();
+    let mut l: u32 = level;
+    while l > 0 {
+        result += INDENT;
+        l -= 1;
+    }
+    result
 }
 
-pub fn is_valid_url<T: AsRef<str>>(path: T) -> bool {
-    REGEX_URL.is_match(path.as_ref())
-}
-
-pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
-    let result = if is_valid_url(to.as_ref()) {
-        to.as_ref().to_string()
-    } else {
-        Url::parse(from.as_ref())?
-            .join(to.as_ref())?
-            .as_ref()
-            .to_string()
-    };
-    Ok(result)
-}
-
-pub fn resolve_css_imports(
-    cache: &mut HashMap<String, String>,
+pub fn retrieve_asset(
+    cache: &mut HashMap<String, Vec<u8>>,
     client: &Client,
-    css_string: &str,
-    as_dataurl: bool,
-    href: &str,
-    opt_no_images: bool,
-    opt_silent: bool,
-) -> String {
-    let mut resolved_css = String::from(css_string);
+    parent_url: &str,
+    url: &str,
+    options: &Options,
+    depth: u32,
+) -> Result<(Vec<u8>, String, String), reqwest::Error> {
+    if url.len() == 0 {
+        // Provoke error
+        client.get("").send()?;
+    }
 
-    for link in REGEX_CSS_URL.captures_iter(&css_string) {
-        let target_link = link.name("url").unwrap().as_str();
-
-        // Determine the type of link
-        let is_stylesheet = link.name("stylesheet").is_some();
-        let is_font = link.name("font").is_some();
-        let is_image = !is_stylesheet && !is_font;
-
-        // Generate absolute URL for content
-        let embedded_url = match resolve_url(href, target_link) {
-            Ok(url) => url,
-            Err(_) => continue, // Malformed URL
-        };
-
-        // Download the asset. If it's more CSS, resolve that too
-        let content = if is_stylesheet {
-            // The link is an @import link
-            retrieve_asset(
-                cache,
-                client,
-                &embedded_url,
-                false,      // Formating as data URL will be done later
-                "text/css", // Expect CSS
-                opt_silent,
-            )
-            .map(|(content, _)| {
-                resolve_css_imports(
-                    cache,
-                    client,
-                    &content,
-                    true, // Finally, convert to a dataurl
-                    &embedded_url,
-                    opt_no_images,
-                    opt_silent,
-                )
-            })
-        } else if (is_image && !opt_no_images) || is_font {
-            // The link is some other, non-@import link
-            retrieve_asset(
-                cache,
-                client,
-                &embedded_url,
-                true, // Format as data URL
-                "",   // Unknown MIME type
-                opt_silent,
-            )
-            .map(|(a, _)| a)
-        } else {
-            // If it's a datatype that has been opt_no'd out of, replace with
-            // absolute URL
-
-            Ok(embedded_url.clone())
+    if is_data_url(&url) {
+        let (media_type, data) = parse_data_url(url);
+        Ok((data, url.to_string(), media_type))
+    } else if is_file_url(&url) {
+        // Check if parent_url is also file:///
+        // (if not, then we don't embed the asset)
+        if !is_file_url(&parent_url) {
+            // Provoke error
+            client.get("").send()?;
         }
-        .unwrap_or_else(|e| {
-            eprintln!("Warning: {}", e);
 
-            // If failed to resolve, replace with absolute URL
-            embedded_url
-        });
+        let fs_file_path: String = file_url_to_fs_path(url);
+        let path = Path::new(&fs_file_path);
+        if path.exists() {
+            if !options.silent {
+                eprintln!("{}{}", indent(depth).as_str(), &url);
+            }
 
-        let replacement = format!("\"{}\"", &content);
-        let dest = link.name("to_repl").unwrap();
-        let offset = resolved_css.len() - css_string.len();
-        let target_range = (dest.start() + offset)..(dest.end() + offset);
-
-        resolved_css.replace_range(target_range, &replacement);
-    }
-
-    if as_dataurl {
-        data_to_dataurl("text/css", resolved_css.as_bytes())
+            Ok((fs::read(&fs_file_path).expect(""), url.to_string(), str!()))
+        } else {
+            // Provoke error
+            Err(client.get("").send().unwrap_err())
+        }
     } else {
-        resolved_css
-    }
-}
+        let cache_key: String = clean_url(&url);
 
-pub fn clean_url<T: AsRef<str>>(url: T) -> String {
-    let mut result = Url::parse(url.as_ref()).unwrap();
-    // Clear fragment
-    result.set_fragment(None);
-    // Get rid of stray question mark
-    if result.query() == Some("") {
-        result.set_query(None);
+        if cache.contains_key(&cache_key) {
+            // URL is in cache, we get and return it
+            if !options.silent {
+                eprintln!("{}{} (from cache)", indent(depth).as_str(), &url);
+            }
+
+            Ok((
+                cache.get(&cache_key).unwrap().to_vec(),
+                url.to_string(),
+                str!(),
+            ))
+        } else {
+            // URL not in cache, we retrieve the file
+            match client.get(url).send() {
+                Ok(mut response) => {
+                    if !options.ignore_errors && response.status() != 200 {
+                        if !options.silent {
+                            eprintln!("Unable to retrieve {} ({})", &url, response.status());
+                        }
+                        // Provoke error
+                        return Err(client.get("").send().unwrap_err());
+                    }
+
+                    let res_url = response.url().to_string();
+
+                    if !options.silent {
+                        if url == res_url {
+                            eprintln!("{}{}", indent(depth).as_str(), &url);
+                        } else {
+                            eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &res_url);
+                        }
+                    }
+
+                    let new_cache_key: String = clean_url(&res_url);
+
+                    // Convert response into a byte array
+                    let mut data: Vec<u8> = vec![];
+                    response.copy_to(&mut data)?;
+
+                    // Attempt to obtain media type by reading the Content-Type header
+                    let media_type = response
+                        .headers()
+                        .get(CONTENT_TYPE)
+                        .and_then(|header| header.to_str().ok())
+                        .unwrap_or("");
+
+                    // Add retrieved resource to cache
+                    cache.insert(new_cache_key, data.clone());
+
+                    Ok((data, res_url, media_type.to_string()))
+                }
+                Err(error) => Err(error),
+            }
+        }
     }
-    result.to_string()
 }