Merge pull request #167 from snshn/version-bump

version bump

.github/workflows/cd.yml

@@ -10,7 +10,7 @@ on:
 jobs:
 
   windows:
-    runs-on: windows-latest
+    runs-on: windows-2019
     steps:
     - run: git config --global core.autocrlf false
     - name: Checkout the repository
@@ -24,19 +24,30 @@ jobs:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
 
   gnu_linux_armhf:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-18.04
     steps:
     - name: Checkout the repository
       uses: actions/checkout@v2
     - name: Prepare cross-platform environment
       run: |
+        sudo mkdir -p /cross-build-arm
+        sudo echo "deb [arch=armhf] http://ports.ubuntu.com/ubuntu-ports/ bionic main" >> /etc/apt/sources.list
+        sudo apt-get update
+        sudo apt-get install -y gcc-arm-linux-gnueabihf libc6-armhf-cross libc6-dev-armhf-cross
+        sudo apt-get download libssl1.1:armhf libssl-dev:armhf
+        sudo dpkg -x libssl1.1*.deb /cross-build-arm
+        sudo dpkg -x libssl-dev*.deb /cross-build-arm
         rustup target add arm-unknown-linux-gnueabihf
-        git clone https://github.com/raspberrypi/tools.git rpi_tools
-    - name: Build and install the executable
+        echo "::set-env name=C_INCLUDE_PATH::/cross-build-arm/usr/include"
+        echo "::set-env name=OPENSSL_INCLUDE_DIR::/cross-build-arm/usr/include/arm-linux-gnueabihf"
+        echo "::set-env name=OPENSSL_LIB_DIR::/cross-build-arm/usr/lib/arm-linux-gnueabihf"
+        echo "::set-env name=PKG_CONFIG_ALLOW_CROSS::1"
+        echo "::set-env name=RUSTFLAGS::-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build-arm/usr/lib/arm-linux-gnueabihf -L/cross-build-arm/lib/arm-linux-gnueabihf"
+    - name: Build the executable
       run: |
-        export RUSTFLAGS="-C linker=rpi_tools/arm-bcm2708/arm-rpi-4.9.3-linux-gnueabihf/bin/arm-linux-gnueabihf-gcc"
         cargo build --release --target=arm-unknown-linux-gnueabihf
-    - uses: Shopify/upload-to-release@1.0.0
+    - name: Attach artifact to the release
+      uses: Shopify/upload-to-release@1.0.0
       with:
         name: monolith-gnu-linux-armhf
         path: target/arm-unknown-linux-gnueabihf/release/monolith

Cargo.lock

@@ -573,7 +573,7 @@ dependencies = [
 
 [[package]]
 name = "monolith"
-version = "2.2.3"
+version = "2.2.4"
 dependencies = [
  "assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
  "base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "monolith"
-version = "2.2.3"
+version = "2.2.4"
 edition = "2018"
 authors = [
     "Sunshine <sunshine@uberspace.net>",

README.md

@@ -29,6 +29,9 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
     $ snap install monolith
 
 #### From source
+
+    Dependency: `libssl-dev`
+
     $ git clone https://github.com/Y2Z/monolith.git
     $ cd monolith
     $ make install
@@ -46,6 +49,7 @@ The guide can be found [here](docs/containers.md)
 ## Options
  - `-c`: Ignore styles
  - `-f`: Exclude frames and iframes
+ - `-F`: Omit web fonts
  - `-i`: Remove images
  - `-I`: Isolate the document
  - `-j`: Exclude JavaScript

docs/arch/0002-noscript-nodes.md

@@ -0,0 +1,19 @@
+# 2. NOSCRIPT nodes
+
+Date: 2020-04-16
+
+## Status
+
+Accepted
+
+## Context
+
+HTML pages sometimes contain NOSCRIPT nodes, which reveal their contents only in case when JavaScript is not available. Most of the time they contain hidden messages that inform about certain JavaScript-dependent features not being operational, however sometimes can also feature media assets or even iframes.
+
+## Decision
+
+When the document is being saved with or without JavaScript, each NOSCRIPT node should be preserved while its children need to be processed exactly the same way as the rest of the document. This approach will ensure that even hidden remote assets are embedded — since those hidden elements may have to be displayed later in a browser that has JavaScript turned off. An option should be available to "unwrap" all NOSCRIPT nodes in order to make their contents always visible in the document, complimenting the "disable JS" function of the program.
+
+## Consequences
+
+Saved documents will have contents of all NOSCRIPT nodes processed as if they are part of the document's DOM, therefore properly display images encapsulated within NOSCRIPT nodes when being viewed in browsers that have JavaScript turned off (or have no JavaScript support in the first place). The new option to "unwrap" NOSCRIPT elements will help the user ensure that the resulting document always represents what the original web page looked like in a browser that had JavaScript turned off.

docs/arch/0005-asset-minimization.md

@@ -21,5 +21,5 @@ saved by monolith, if needed.
 
 ## Consequences
 
-Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only a minimal
+Monolith will not support modification of original document assets for the purpose of reducing their size, sticking to performing only minimal
 amount of modifications to the original web page — whatever is needed to provide security or exclude unwanted asset types.

src/args.rs

@@ -4,6 +4,7 @@ use clap::{App, Arg};
 pub struct AppArgs {
     pub target: String,
     pub no_css: bool,
+    pub no_fonts: bool,
     pub no_frames: bool,
     pub no_images: bool,
     pub no_js: bool,
@@ -35,6 +36,7 @@ impl AppArgs {
             // .args_from_usage("-a, --include-audio 'Removes audio sources'")
             .args_from_usage("-c, --no-css 'Removes CSS'")
             .args_from_usage("-f, --no-frames 'Removes frames and iframes'")
+            .args_from_usage("-F, --no-fonts 'Removes fonts'")
             .args_from_usage("-i, --no-images 'Removes images'")
             .args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
             .args_from_usage("-j, --no-js 'Removes JavaScript'")
@@ -52,6 +54,7 @@ impl AppArgs {
             .expect("please set target")
             .to_string();
         app_args.no_css = app.is_present("no-css");
+        app_args.no_fonts = app.is_present("no-fonts");
         app_args.no_frames = app.is_present("no-frames");
         app_args.no_images = app.is_present("no-images");
         app_args.no_js = app.is_present("no-js");

src/css.rs

@@ -2,12 +2,12 @@ use cssparser::{ParseError, Parser, ParserInput, SourcePosition, Token};
 use reqwest::blocking::Client;
 use std::collections::HashMap;
 
-use crate::utils::{data_to_data_url, decode_url, get_url_fragment, resolve_url, retrieve_asset};
+use crate::utils::{data_to_data_url, get_url_fragment, resolve_url, retrieve_asset};
 
 const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
+    // Universal
     "background",
     "background-image",
-    "border",
     "border-image",
     "border-image-source",
     "content",
@@ -16,7 +16,15 @@ const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
     "list-style-image",
     "mask",
     "mask-image",
+    // Specific to @counter-style
+    "additive-symbols",
+    "negative",
+    "pad",
+    "prefix",
+    "suffix",
+    "symbols",
 ];
+const CSS_SPECIAL_CHARS: &str = "~!@$%^&*()+=,./'\";:?><[]{}|`#";
 
 pub fn is_image_url_prop(prop_name: &str) -> bool {
     CSS_PROPS_WITH_IMAGE_URLS
@@ -33,14 +41,27 @@ pub fn enquote(input: String, double: bool) -> String {
     }
 }
 
+pub fn escape(value: &str) -> String {
+    let mut res = str!(&value);
+
+    res = res.replace("\\", "\\\\");
+
+    for c in CSS_SPECIAL_CHARS.chars() {
+        res = res.replace(c, format!("\\{}", c).as_str());
+    }
+
+    res
+}
+
 pub fn process_css<'a>(
-    cache: &mut HashMap<String, String>,
+    cache: &mut HashMap<String, Vec<u8>>,
     client: &Client,
     parent_url: &str,
     parser: &mut Parser,
     rule_name: &str,
     prop_name: &str,
     func_name: &str,
+    opt_no_fonts: bool,
     opt_no_images: bool,
     opt_silent: bool,
 ) -> Result<String, ParseError<'a, String>> {
@@ -69,6 +90,10 @@ pub fn process_css<'a>(
             Token::Colon => result.push_str(":"),
             Token::Comma => result.push_str(","),
             Token::ParenthesisBlock | Token::SquareBracketBlock | Token::CurlyBracketBlock => {
+                if opt_no_fonts && curr_rule == "font-face" {
+                    continue;
+                }
+
                 let closure: &str;
                 if token == &Token::ParenthesisBlock {
                     result.push_str("(");
@@ -91,6 +116,7 @@ pub fn process_css<'a>(
                             rule_name,
                             curr_prop.as_str(),
                             func_name,
+                            opt_no_fonts,
                             opt_no_images,
                             opt_silent,
                         )
@@ -113,12 +139,18 @@ pub fn process_css<'a>(
             Token::WhiteSpace(ref value) => {
                 result.push_str(value);
             }
+            // div...
             Token::Ident(ref value) => {
+                curr_rule = str!();
                 curr_prop = str!(value);
-                result.push_str(value);
+                result.push_str(&escape(value));
             }
+            // @import, @font-face, @charset, @media...
             Token::AtKeyword(ref value) => {
                 curr_rule = str!(value);
+                if opt_no_fonts && curr_rule == "font-face" {
+                    continue;
+                }
                 result.push_str("@");
                 result.push_str(value);
             }
@@ -127,13 +159,10 @@ pub fn process_css<'a>(
                 result.push_str(value);
             }
             Token::QuotedString(ref value) => {
-                let is_import: bool = curr_rule == "import";
-                if is_import {
+                if curr_rule == "import" {
                     // Reset current at-rule value
                     curr_rule = str!();
-                }
 
-                if is_import {
                     // Skip empty import values
                     if value.len() < 1 {
                         result.push_str("''");
@@ -142,12 +171,11 @@ pub fn process_css<'a>(
 
                     let full_url = resolve_url(&parent_url, value).unwrap_or_default();
                     let url_fragment = get_url_fragment(full_url.clone());
-                    let full_url_decoded = decode_url(full_url);
                     let (css, final_url) = retrieve_asset(
                         cache,
                         client,
                         &parent_url,
-                        &full_url_decoded,
+                        &full_url,
                         false,
                         "",
                         opt_silent,
@@ -163,6 +191,7 @@ pub fn process_css<'a>(
                                     client,
                                     final_url.as_str(),
                                     &css,
+                                    opt_no_fonts,
                                     opt_no_images,
                                     opt_silent,
                                 )
@@ -235,12 +264,15 @@ pub fn process_css<'a>(
                 result.push_str(str!(value).as_str());
                 result.push_str(str!(unit).as_str());
             }
+            // #selector, #id...
             Token::IDHash(ref value) => {
+                curr_rule = str!();
                 result.push_str("#");
-                result.push_str(value);
+                result.push_str(&escape(value));
             }
             Token::UnquotedUrl(ref value) => {
                 let is_import: bool = curr_rule == "import";
+
                 if is_import {
                     // Reset current at-rule value
                     curr_rule = str!();
@@ -261,12 +293,11 @@ pub fn process_css<'a>(
                 if is_import {
                     let full_url = resolve_url(&parent_url, value).unwrap_or_default();
                     let url_fragment = get_url_fragment(full_url.clone());
-                    let full_url_decoded = decode_url(full_url);
                     let (css, final_url) = retrieve_asset(
                         cache,
                         client,
                         &parent_url,
-                        &full_url_decoded,
+                        &full_url,
                         false,
                         "",
                         opt_silent,
@@ -282,6 +313,7 @@ pub fn process_css<'a>(
                                     client,
                                     final_url.as_str(),
                                     &css,
+                                    opt_no_fonts,
                                     opt_no_images,
                                     opt_silent,
                                 )
@@ -329,6 +361,7 @@ pub fn process_css<'a>(
                             curr_rule.as_str(),
                             curr_prop.as_str(),
                             function_name,
+                            opt_no_fonts,
                             opt_no_images,
                             opt_silent,
                         )
@@ -346,10 +379,11 @@ pub fn process_css<'a>(
 }
 
 pub fn embed_css(
-    cache: &mut HashMap<String, String>,
+    cache: &mut HashMap<String, Vec<u8>>,
     client: &Client,
     parent_url: &str,
     css: &str,
+    opt_no_fonts: bool,
     opt_no_images: bool,
     opt_silent: bool,
 ) -> String {
@@ -364,6 +398,7 @@ pub fn embed_css(
         "",
         "",
         "",
+        opt_no_fonts,
         opt_no_images,
         opt_silent,
     )

src/html.rs

@@ -37,15 +37,16 @@ pub fn is_icon(attr_value: &str) -> bool {
 }
 
 pub fn walk_and_embed_assets(
-    cache: &mut HashMap<String, String>,
+    cache: &mut HashMap<String, Vec<u8>>,
     client: &Client,
     url: &str,
     node: &Handle,
     opt_no_css: bool,
+    opt_no_fonts: bool,
+    opt_no_frames: bool,
     opt_no_js: bool,
     opt_no_images: bool,
     opt_silent: bool,
-    opt_no_frames: bool,
 ) {
     match node.data {
         NodeData::Document => {
@@ -57,10 +58,11 @@ pub fn walk_and_embed_assets(
                     &url,
                     child,
                     opt_no_css,
+                    opt_no_fonts,
+                    opt_no_frames,
                     opt_no_js,
                     opt_no_images,
                     opt_silent,
-                    opt_no_frames,
                 );
             }
         }
@@ -157,17 +159,18 @@ pub fn walk_and_embed_assets(
                                         ) {
                                             // On successful retrieval, traverse CSS
                                             Ok((css_data, final_url)) => {
-                                                let x: String = embed_css(
+                                                let css: String = embed_css(
                                                     cache,
                                                     client,
                                                     &final_url,
                                                     &css_data,
+                                                    opt_no_fonts,
                                                     opt_no_images,
                                                     opt_silent,
                                                 );
                                                 data_to_data_url(
                                                     "text/css",
-                                                    x.as_bytes(),
+                                                    css.as_bytes(),
                                                     &final_url,
                                                     "",
                                                 )
@@ -462,6 +465,7 @@ pub fn walk_and_embed_assets(
                                     client,
                                     &url,
                                     tendril.as_ref(),
+                                    opt_no_fonts,
                                     opt_no_images,
                                     opt_silent,
                                 );
@@ -519,10 +523,11 @@ pub fn walk_and_embed_assets(
                                 &frame_final_url,
                                 &dom.document,
                                 opt_no_css,
+                                opt_no_fonts,
+                                opt_no_frames,
                                 opt_no_js,
                                 opt_no_images,
                                 opt_silent,
-                                opt_no_frames,
                             );
                             let mut buf: Vec<u8> = Vec::new();
                             serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -590,6 +595,7 @@ pub fn walk_and_embed_assets(
                         client,
                         &url,
                         attribute.value.as_ref(),
+                        opt_no_fonts,
                         opt_no_images,
                         opt_silent,
                     );
@@ -621,10 +627,11 @@ pub fn walk_and_embed_assets(
                     &url,
                     child,
                     opt_no_css,
+                    opt_no_fonts,
+                    opt_no_frames,
                     opt_no_js,
                     opt_no_images,
                     opt_silent,
-                    opt_no_frames,
                 );
             }
         }

src/main.rs

@@ -122,9 +122,9 @@ fn main() {
         base_url = final_url;
         dom = html_to_dom(&data);
     } else if is_data_url(target_url) {
-        let text: String = data_url_to_text(target_url);
-        if text.len() == 0 {
-            eprintln!("Unsupported data URL input");
+        let (media_type, text): (String, String) = data_url_to_text(target_url);
+        if !media_type.eq_ignore_ascii_case("text/html") {
+            eprintln!("Unsupported data URL media type");
             process::exit(1);
         }
         base_url = str!(target_url);
@@ -139,10 +139,11 @@ fn main() {
         &base_url,
         &dom.document,
         app_args.no_css,
+        app_args.no_fonts,
+        app_args.no_frames,
         app_args.no_js,
         app_args.no_images,
         app_args.silent,
-        app_args.no_frames,
     );
 
     let html: String = stringify_document(

src/tests/cli.rs

@@ -62,7 +62,7 @@ fn passing_bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
     // STDERR should contain error description
     assert_eq!(
         std::str::from_utf8(&out.stderr).unwrap(),
-        "Unsupported data URL input\n"
+        "Unsupported data URL media type\n"
     );
 
     // The exit code should be 1

src/tests/css/embed_css.rs

@@ -15,7 +15,10 @@ fn passing_empty_input() {
     let cache = &mut HashMap::new();
     let client = Client::new();
 
-    assert_eq!(css::embed_css(cache, &client, "", "", false, false,), "");
+    assert_eq!(
+        css::embed_css(cache, &client, "", "", false, false, false,),
+        ""
+    );
 }
 
 #[test]
@@ -37,6 +40,7 @@ height: calc(100vh - 10pt)";
             &client,
             "https://doesntmatter.local/",
             &STYLE,
+            false,
             true,
             true,
         ),
@@ -67,7 +71,7 @@ line-height: -1; \
 height: calc(100vh - 10pt)";
 
     assert_eq!(
-        css::embed_css(cache, &client, "", &STYLE, true, true,),
+        css::embed_css(cache, &client, "", &STYLE, false, true, true,),
         format!(
             "/* border: none;*/\
 background-image: url('{empty_image}'); \
@@ -95,7 +99,7 @@ fn passing_style_block() {
 html > body {}";
 
     assert_eq!(
-        css::embed_css(cache, &client, "file:///", &CSS, false, true,),
+        css::embed_css(cache, &client, "file:///", &CSS, false, false, true,),
         CSS
     );
 }
@@ -135,7 +139,10 @@ fn passing_attribute_selectors() {
 }
 ";
 
-    assert_eq!(css::embed_css(cache, &client, "", &CSS, false, false,), CSS);
+    assert_eq!(
+        css::embed_css(cache, &client, "", &CSS, false, false, false,),
+        CSS
+    );
 }
 
 #[test]
@@ -158,14 +165,15 @@ fn passing_import_string() {
             "https://doesntmatter.local/",
             &CSS,
             false,
+            false,
             true,
         ),
         "\
 @charset 'UTF-8';\n\
 \n\
-@import 'data:text/css;base64,ZGF0YTp0ZXh0L2NzcyxodG1se2JhY2tncm91bmQtY29sb3I6IzAwMH0=';\n\
+@import 'data:text/css;base64,aHRtbHtiYWNrZ3JvdW5kLWNvbG9yOiMwMDB9';\n\
 \n\
-@import url('data:text/css;base64,ZGF0YTp0ZXh0L2NzcyxodG1se2NvbG9yOiNmZmZ9')\n\
+@import url('data:text/css;base64,aHRtbHtjb2xvcjojZmZmfQ==')\n\
 "
     );
 }
@@ -192,6 +200,7 @@ body {\n    \
             "https://doesntmatter.local/",
             &CSS,
             false,
+            false,
             true,
         ),
         CSS
@@ -209,7 +218,6 @@ div {\n    \
     transform: translate(50%, 50%) rotate(45deg);\n\
     transform: translate(+50%, +50%) rotate(+45deg);\n\
 }\n\
-\n\
 ";
 
     assert_eq!(
@@ -219,8 +227,91 @@ div {\n    \
             "https://doesntmatter.local/",
             &CSS,
             false,
+            false,
             true,
         ),
         CSS
     );
 }
+
+#[test]
+fn passing_unusual_indents() {
+    let cache = &mut HashMap::new();
+    let client = Client::new();
+
+    const CSS: &str = "\
+.is\\:good:hover {\n    \
+    color: green\n\
+}\n\
+\n\
+#\\~\\!\\@\\$\\%\\^\\&\\*\\(\\)\\+\\=\\,\\.\\/\\\\\\'\\\"\\;\\:\\?\\>\\<\\[\\]\\{\\}\\|\\`\\# {\n    \
+    color: black\n\
+}\n\
+";
+
+    assert_eq!(
+        css::embed_css(
+            cache,
+            &client,
+            "https://doesntmatter.local/",
+            &CSS,
+            false,
+            false,
+            true,
+        ),
+        CSS
+    );
+}
+
+#[test]
+fn passing_exclude_fonts() {
+    let cache = &mut HashMap::new();
+    let client = Client::new();
+
+    const CSS: &str = "\
+@font-face {\n    \
+    font-family: 'My Font';\n    \
+    src: url(my_font.woff);\n\
+}\n\
+\n\
+#identifier {\n    \
+    font-family: 'My Font' Arial\n\
+}\n\
+\n\
+@font-face {\n    \
+    font-family: 'My Font';\n    \
+    src: url(my_font.woff);\n\
+}\n\
+\n\
+div {\n    \
+    font-family: 'My Font' Verdana\n\
+}\n\
+";
+
+    const CSS_OUT: &str = " \
+\n\
+\n\
+#identifier {\n    \
+    font-family: 'My Font' Arial\n\
+}\n\
+\n \
+\n\
+\n\
+div {\n    \
+    font-family: 'My Font' Verdana\n\
+}\n\
+";
+
+    assert_eq!(
+        css::embed_css(
+            cache,
+            &client,
+            "https://doesntmatter.local/",
+            &CSS,
+            true,
+            false,
+            true,
+        ),
+        CSS_OUT
+    );
+}

src/tests/html/walk_and_embed_assets.rs

@@ -19,6 +19,7 @@ fn passing_basic() {
     let url = "http://localhost";
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
@@ -32,10 +33,11 @@ fn passing_basic() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -55,6 +57,7 @@ fn passing_ensure_no_recursive_iframe() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
@@ -68,10 +71,11 @@ fn passing_ensure_no_recursive_iframe() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -91,6 +95,7 @@ fn passing_ensure_no_recursive_frame() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
@@ -104,10 +109,11 @@ fn passing_ensure_no_recursive_frame() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -129,6 +135,7 @@ fn passing_no_css() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = true;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
@@ -141,10 +148,11 @@ fn passing_no_css() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -173,6 +181,7 @@ fn passing_no_images() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = false;
     let opt_no_images: bool = true;
@@ -186,10 +195,11 @@ fn passing_no_images() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -221,6 +231,7 @@ fn passing_no_body_background_images() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = false;
     let opt_no_images: bool = true;
@@ -234,10 +245,11 @@ fn passing_no_body_background_images() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -257,6 +269,7 @@ fn passing_no_frames() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = true;
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
@@ -269,10 +282,11 @@ fn passing_no_frames() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -292,6 +306,7 @@ fn passing_no_iframes() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = true;
     let opt_no_js: bool = false;
     let opt_no_images: bool = false;
@@ -304,10 +319,11 @@ fn passing_no_iframes() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -330,6 +346,7 @@ fn passing_no_js() {
     let cache = &mut HashMap::new();
 
     let opt_no_css: bool = false;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = false;
     let opt_no_js: bool = true;
     let opt_no_images: bool = false;
@@ -343,10 +360,11 @@ fn passing_no_js() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();
@@ -369,6 +387,7 @@ fn passing_with_no_integrity() {
     let cache = &mut HashMap::new();
     let client = Client::new();
     let opt_no_css: bool = true;
+    let opt_no_fonts: bool = false;
     let opt_no_frames: bool = true;
     let opt_no_js: bool = true;
     let opt_no_images: bool = true;
@@ -380,10 +399,11 @@ fn passing_with_no_integrity() {
         &url,
         &dom.document,
         opt_no_css,
+        opt_no_fonts,
+        opt_no_frames,
         opt_no_js,
         opt_no_images,
         opt_silent,
-        opt_no_frames,
     );
 
     let mut buf: Vec<u8> = Vec::new();

src/tests/utils/clean_url.rs

@@ -30,3 +30,11 @@ fn passing_removes_empty_query_and_empty_fragment() {
         "https://somewhere.com/font.eot"
     );
 }
+
+#[test]
+fn passing_removes_empty_query_amp_and_empty_fragment() {
+    assert_eq!(
+        utils::clean_url("https://somewhere.com/font.eot?a=b&#"),
+        "https://somewhere.com/font.eot?a=b"
+    );
+}

src/tests/utils/data_url_to_text.rs

@@ -9,42 +9,76 @@ use crate::utils;
 
 #[test]
 fn passing_parse_text_html_base64() {
+    let (media_type, text) = utils::data_url_to_text("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
+
+    assert_eq!(media_type, "text/html");
     assert_eq!(
-        utils::data_url_to_text("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg=="),
+        text,
         "Work expands so as to fill the time available for its completion"
     );
 }
 
 #[test]
 fn passing_parse_text_html_utf8() {
+    let (media_type, text) = utils::data_url_to_text(
+        "data:text/html;utf8,Work expands so as to fill the time available for its completion",
+    );
+
+    assert_eq!(media_type, "text/html");
     assert_eq!(
-        utils::data_url_to_text(
-            "data:text/html;utf8,Work expands so as to fill the time available for its completion"
-        ),
+        text,
         "Work expands so as to fill the time available for its completion"
     );
 }
 
 #[test]
 fn passing_parse_text_html_plaintext() {
+    let (media_type, text) = utils::data_url_to_text(
+        "data:text/html,Work expands so as to fill the time available for its completion",
+    );
+
+    assert_eq!(media_type, "text/html");
     assert_eq!(
-        utils::data_url_to_text(
-            "data:text/html,Work expands so as to fill the time available for its completion"
-        ),
+        text,
         "Work expands so as to fill the time available for its completion"
     );
 }
 
 #[test]
 fn passing_parse_text_html_charset_utf_8_between_two_whitespaces() {
+    let (media_type, text) = utils::data_url_to_text(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
+
+    assert_eq!(media_type, "text/html");
     assert_eq!(
-        utils::data_url_to_text(
-            " data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion "
-        ),
+        text,
         "Work expands so as to fill the time available for its completion"
     );
 }
 
+#[test]
+fn passing_parse_text_css_url_encoded() {
+    let (media_type, text) = utils::data_url_to_text("data:text/css,div{background-color:%23000}");
+
+    assert_eq!(media_type, "text/css");
+    assert_eq!(text, "div{background-color:#000}");
+}
+
+#[test]
+fn passing_parse_no_media_type_base64() {
+    let (media_type, text) = utils::data_url_to_text("data:;base64,dGVzdA==");
+
+    assert_eq!(media_type, "");
+    assert_eq!(text, "test");
+}
+
+#[test]
+fn passing_parse_no_media_type_no_encoding() {
+    let (media_type, text) = utils::data_url_to_text("data:;,test%20test");
+
+    assert_eq!(media_type, "");
+    assert_eq!(text, "test test");
+}
+
 //  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
 //  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
 //  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
@@ -54,5 +88,8 @@ fn passing_parse_text_html_charset_utf_8_between_two_whitespaces() {
 
 #[test]
 fn failing_just_word_data() {
-    assert_eq!(utils::data_url_to_text("data"), "");
+    let (media_type, text) = utils::data_url_to_text("data");
+
+    assert_eq!(media_type, "");
+    assert_eq!(text, "");
 }

src/tests/utils/decode_url.rs

@@ -24,3 +24,13 @@ fn passing_decode_file_url() {
         "file:///tmp/space here/test#1.html"
     );
 }
+
+#[test]
+fn passing_plus_sign() {
+    assert_eq!(
+        utils::decode_url(str!(
+            "fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
+        )),
+        "fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
+    );
+}

src/tests/utils/file_url_to_fs_path.rs

@@ -21,3 +21,18 @@ fn passing_remove_protocl_and_fragment() {
         );
     }
 }
+
+#[test]
+fn passing_decodes_urls() {
+    if cfg!(windows) {
+        assert_eq!(
+            utils::file_url_to_fs_path("file:///C:/Documents%20and%20Settings/some-file.html"),
+            "C:\\Documents and Settings\\some-file.html"
+        );
+    } else {
+        assert_eq!(
+            utils::file_url_to_fs_path("file:///home/user/My%20Documents"),
+            "/home/user/My Documents"
+        );
+    }
+}

src/utils.rs

@@ -30,6 +30,14 @@ const MAGIC: [[&[u8]; 2]; 18] = [
     [b"\x1A\x45\xDF\xA3", b"video/webm"],
 ];
 
+const PLAINTEXT_MEDIA_TYPES: &[&str] = &[
+    "image/svg+xml",
+    "text/css",
+    "text/html",
+    "text/javascript",
+    "text/plain",
+];
+
 pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str, fragment: &str) -> String {
     let media_type: String = if media_type.is_empty() {
         detect_media_type(data, &url)
@@ -88,6 +96,10 @@ pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
         .unwrap_or(false)
 }
 
+pub fn is_plaintext_media_type(media_type: &str) -> bool {
+    PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
+}
+
 pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
     let result = if is_http_url(to.as_ref()) {
         to.as_ref().to_string()
@@ -108,65 +120,72 @@ pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
     }
 }
 
-pub fn clean_url<T: AsRef<str>>(url: T) -> String {
-    let mut result = Url::parse(url.as_ref()).unwrap();
+pub fn clean_url<T: AsRef<str>>(input: T) -> String {
+    let mut url = Url::parse(input.as_ref()).unwrap();
 
     // Clear fragment
-    result.set_fragment(None);
+    url.set_fragment(None);
 
     // Get rid of stray question mark
-    if result.query() == Some("") {
-        result.set_query(None);
+    if url.query() == Some("") {
+        url.set_query(None);
     }
-    result.to_string()
+
+    // Remove empty trailing ampersand(s)
+    let mut result: String = url.to_string();
+    while result.ends_with("&") {
+        result.pop();
+    }
+
+    result
 }
 
-pub fn data_url_to_text<T: AsRef<str>>(url: T) -> String {
-    let parsed_url = Url::parse(url.as_ref()).unwrap_or(Url::parse("http://[::1]").unwrap());
+pub fn data_url_to_text<T: AsRef<str>>(url: T) -> (String, String) {
+    let parsed_url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
     let path: String = parsed_url.path().to_string();
     let comma_loc: usize = path.find(',').unwrap_or(path.len());
 
-    if comma_loc == path.len() {
-        return str!();
-    }
-
     let meta_data: String = path.chars().take(comma_loc).collect();
     let raw_data: String = path.chars().skip(comma_loc + 1).collect();
 
     let data: String = decode_url(raw_data);
 
     let meta_data_items: Vec<&str> = meta_data.split(';').collect();
-    let mut media_type: &str = "";
     let mut encoding: &str = "";
 
+    let mut media_type: String = str!();
+    let mut text: String = str!();
+
     let mut i: i8 = 0;
     for item in &meta_data_items {
         if i == 0 {
-            if item.eq_ignore_ascii_case("text/html") {
-                media_type = item;
-                continue;
+            media_type = str!(item);
+        } else {
+            if item.eq_ignore_ascii_case("base64")
+                || item.eq_ignore_ascii_case("utf8")
+                || item.eq_ignore_ascii_case("charset=UTF-8")
+            {
+                encoding = item;
             }
         }
 
-        if item.eq_ignore_ascii_case("base64") || item.eq_ignore_ascii_case("utf8") {
-            encoding = item;
-        }
-
         i = i + 1;
     }
 
-    if media_type.eq_ignore_ascii_case("text/html") {
+    if is_plaintext_media_type(&media_type) || media_type.is_empty() {
         if encoding.eq_ignore_ascii_case("base64") {
-            String::from_utf8(base64::decode(&data).unwrap_or(vec![])).unwrap_or(str!())
+            text = String::from_utf8(base64::decode(&data).unwrap_or(vec![])).unwrap_or(str!())
         } else {
-            data
+            text = data
         }
-    } else {
-        str!()
     }
+
+    (media_type, text)
 }
 
 pub fn decode_url(input: String) -> String {
+    let input: String = input.replace("+", "%2B");
+
     form_urlencoded::parse(input.as_bytes())
         .map(|(key, val)| {
             [
@@ -200,11 +219,12 @@ pub fn file_url_to_fs_path(url: &str) -> String {
         fs_file_path = fs_file_path.replace("/", "\\");
     }
 
-    fs_file_path
+    // File paths should not be %-encoded
+    decode_url(fs_file_path)
 }
 
 pub fn retrieve_asset(
-    cache: &mut HashMap<String, String>,
+    cache: &mut HashMap<String, Vec<u8>>,
     client: &Client,
     parent_url: &str,
     url: &str,
@@ -216,10 +236,14 @@ pub fn retrieve_asset(
         return Ok((str!(), str!()));
     }
 
-    let cache_key = clean_url(&url);
-
     if is_data_url(&url) {
-        Ok((url.to_string(), url.to_string()))
+        if as_data_url {
+            Ok((url.to_string(), url.to_string()))
+        } else {
+            let (_media_type, text) = data_url_to_text(url);
+
+            Ok((text, url.to_string()))
+        }
     } else if is_file_url(&url) {
         // Check if parent_url is also file:///
         // (if not, then we don't embed the asset)
@@ -251,13 +275,25 @@ pub fn retrieve_asset(
             Ok((str!(), url.to_string()))
         }
     } else {
+        let cache_key: String = clean_url(&url);
+
         if cache.contains_key(&cache_key) {
-            // URL is in cache
+            // URL is in cache, we retrieve it
+            let data = cache.get(&cache_key).unwrap();
+
             if !opt_silent {
                 eprintln!("{} (from cache)", &url);
             }
-            let data = cache.get(&cache_key).unwrap();
-            Ok((data.to_string(), url.to_string()))
+
+            if as_data_url {
+                let url_fragment = get_url_fragment(url);
+                Ok((
+                    data_to_data_url(media_type, data, url, &url_fragment),
+                    url.to_string(),
+                ))
+            } else {
+                Ok((String::from_utf8_lossy(data).to_string(), url.to_string()))
+            }
         } else {
             // URL not in cache, we request it
             let mut response = client.get(url).send()?;
@@ -271,7 +307,7 @@ pub fn retrieve_asset(
                 }
             }
 
-            let new_cache_key = clean_url(&res_url);
+            let new_cache_key: String = clean_url(&res_url);
 
             if as_data_url {
                 // Convert response into a byte array
@@ -290,13 +326,17 @@ pub fn retrieve_asset(
                 };
                 let url_fragment = get_url_fragment(url);
                 let data_url = data_to_data_url(&media_type, &data, url, &url_fragment);
+
                 // Add to cache
-                cache.insert(new_cache_key, data_url.clone());
+                cache.insert(new_cache_key, data);
+
                 Ok((data_url, res_url))
             } else {
                 let content = response.text().unwrap();
+
                 // Add to cache
-                cache.insert(new_cache_key, content.clone());
+                cache.insert(new_cache_key, content.as_bytes().to_vec());
+
                 Ok((content, res_url))
             }
         }