Merge pull request #233 from snshn/cargo-fix-license-identifier

Fix license identifier in Cargo.toml

Cargo.toml

@@ -1,7 +1,6 @@
 [package]
 name = "monolith"
-version = "2.3.0"
-edition = "2018"
+version = "2.4.0"
 authors = [
     "Sunshine <sunshine@uberspace.net>",
     "Mahdi Robatipoor <mahdi.robatipoor@gmail.com>",
@@ -9,19 +8,32 @@ authors = [
     "Emi Simpson <emi@alchemi.dev>",
     "rhysd <lin90162@yahoo.co.jp>",
 ]
+edition = "2018"
 description = "CLI tool for saving web pages as a single HTML file"
+homepage = "https://github.com/Y2Z/monolith"
+repository = "https://github.com/Y2Z/monolith"
+readme = "README.md"
+keywords = ["web", "http", "html", "download", "command-line"]
+categories = ["command-line-utilities", "web-programming"]
+include = [
+    "src/*.rs",
+    "Cargo.toml",
+]
+license = "Unlicense"
+license-file = "LICENSE"
 
 [dependencies]
-base64 = "0.12.2"
-chrono = "0.4.11" # Used to render comments indicating the time the page was saved
-clap = "2.33.1"
+base64 = "0.13.0"
+chrono = "0.4.19" # Used for formatting creation timestamp
+clap = "2.33.3"
 cssparser = "0.27.2"
 html5ever = "0.24.1"
-sha2 = "0.9.0" # Used in calculating checksums during integrity checks
-url = "2.1.1"
+regex = "1.4.2" # Used for parsing srcset
+sha2 = "0.9.2" # Used for calculating checksums during integrity checks
+url = "2.2.0"
 
 [dependencies.reqwest]
-version = "0.10.6"
+version = "0.10.9"
 default-features = false
 features = ["default-tls", "blocking", "gzip"]
 

README.md

@@ -22,6 +22,9 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
 
 ## Installation
 
+### Using Cargo
+    $ cargo install monolith
+
 #### Via Homebrew (on macOS and GNU/Linux)
     $ brew install monolith
 
@@ -54,21 +57,26 @@ The guide can be found [here](docs/containers.md)
 ---------------------------------------------------
 
 ## Options
- - `-c`: Ignore styles
- - `-f`: Exclude frames
- - `-F`: Omit web fonts
+ - `-a`: Exclude audio sources
+ - `-b`: Use custom base URL
+ - `-c`: Exclude CSS
+ - `-e`: Ignore network errors
+ - `-f`: Omit frames
+ - `-F`: Exclude web fonts
  - `-i`: Remove images
  - `-I`: Isolate the document
  - `-j`: Exclude JavaScript
  - `-k`: Accept invalid X.509 (TLS) certificates
+ - `-M`: Don't add timestamp and URL information
  - `-o`: Write output to file
  - `-s`: Be quiet
  - `-t`: Adjust network request timeout
  - `-u`: Provide custom User-Agent
+ - `-v`: Exclude videos
 
 ---------------------------------------------------
 
-## HTTPS and HTTP proxies
+## Proxies
 Please set `https_proxy`, `http_proxy`, and `no_proxy` environment variables.
 
 ---------------------------------------------------
@@ -82,6 +90,7 @@ Please open an issue if something is wrong, that helps make this project better.
  - `Monolith Chrome Extension`: https://github.com/rhysd/monolith-of-web
  - `Pagesaver`: https://github.com/distributed-mind/pagesaver
  - `Personal WayBack Machine`: https://github.com/popey/pwbm
+ - `Hako`: https://github.com/dmpop/hako
 
 ---------------------------------------------------
 

docs/arch/0003-network-request-timeout.md

@@ -1,4 +1,4 @@
-# 2. Network request timeout
+# 3. Network request timeout
 
 Date: 2020-02-15
 

docs/arch/0005-asset-minimization.md

@@ -1,4 +1,4 @@
-# 4. Asset Minimization
+# 5. Asset Minimization
 
 Date: 2020-03-14
 

docs/arch/0006-reload-and-location-meta-tags.md

@@ -1,4 +1,4 @@
-# 4. Reload and location `meta` tags
+# 6. Reload and location `meta` tags
 
 Date: 2020-06-25
 

docs/arch/0007-network-errors.md

@@ -0,0 +1,19 @@
+# 7. Network errors
+
+Date: 2020-11-22
+
+## Status
+
+Accepted
+
+## Context
+
+Servers may return information with HTTP response codes other than `200`, however those responses may still contain useful data.
+
+## Decision
+
+Fail by default, notifying of the network error. Add option to continue retrieving assets by treating all response codes as `200`.
+
+## Consequences
+
+Monolith will fail to obtain resources with status other than `200`, unless told to ignore network errors.

docs/arch/0008-base-tag.md

@@ -0,0 +1,40 @@
+# 8. Base Tag
+
+Date: 2020-12-25
+
+## Status
+
+Accepted
+
+## Context
+
+HTML documents may contain `base` tag, which influences resolution of anchor links and relative URLs as well as dynamically loaded resources.
+
+Sometimes, in order to make certain saved documents function closer to how they operate while being served from a remote server, the `base` tag specifying the source page's URL may need to be added to the document.
+
+There can be only one such tag. If multiple `base` tags are present, only the first encountered tag ends up being used.
+
+## Decision
+
+Adding the `base` tag should be optional — saved documents should not contain the `base` tag unless it was specified by the user, or the document originally had the `base` tag in it.
+
+Existing `href` attribute's value of the original `base` tag should be used for resolving the document's relative links instead of document's own URL (precisely the way browsers do it).
+
+## Consequences
+
+#### If the base tag does not exist in the source document
+
+- If the base tag does not exist in the source document
+  - With base URL option provided
+    - use the specified base URL value to retrieve assets, keep original base URL value in the document
+  - Without base URL option provided
+    - download document as usual, do not add base tag
+-  If the base tag already exists in the source document
+   - With base URL option provided
+     - we overwrite the original base URL before retrieving assets, keep new base URL value in the document
+   - Without base URL option provided:
+     - use the base URL from the original document to retrieve assets, keep original base URL value in the document
+
+The program will obtain ability to retrieve remote assets for non-remote sources (such as data URLs and local files).
+
+The program will obatin ability to get rid of existing base tag values (by provind an empty one).

src/css.rs

@@ -176,7 +176,7 @@ pub fn process_css<'a>(
                         client,
                         &parent_url,
                         &import_full_url,
-                        options.silent,
+                        options,
                         depth + 1,
                     ) {
                         Ok((import_contents, import_final_url, _import_media_type)) => {
@@ -227,7 +227,7 @@ pub fn process_css<'a>(
                                 client,
                                 &parent_url,
                                 &resolved_url,
-                                options.silent,
+                                options,
                                 depth + 1,
                             ) {
                                 Ok((data, final_url, media_type)) => {
@@ -315,14 +315,8 @@ pub fn process_css<'a>(
                 if is_import {
                     let full_url = resolve_url(&parent_url, value).unwrap_or_default();
                     let url_fragment = get_url_fragment(full_url.clone());
-                    match retrieve_asset(
-                        cache,
-                        client,
-                        &parent_url,
-                        &full_url,
-                        options.silent,
-                        depth + 1,
-                    ) {
+                    match retrieve_asset(cache, client, &parent_url, &full_url, options, depth + 1)
+                    {
                         Ok((css, final_url, _media_type)) => {
                             let data_url = data_to_data_url(
                                 "text/css",
@@ -361,7 +355,7 @@ pub fn process_css<'a>(
                             client,
                             &parent_url,
                             &full_url,
-                            options.silent,
+                            options,
                             depth + 1,
                         ) {
                             Ok((data, final_url, media_type)) => {

src/main.rs

@@ -3,17 +3,18 @@ use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
 use std::collections::HashMap;
 use std::env;
 use std::fs;
-use std::io::{self, Error, Write};
+use std::io::{self, prelude::*, Error, Write};
 use std::path::Path;
 use std::process;
 use std::time::Duration;
 
 use monolith::html::{
-    add_favicon, has_favicon, html_to_dom, metadata_tag, stringify_document, walk_and_embed_assets,
+    add_favicon, create_metadata_tag, get_base_url, has_favicon, html_to_dom, set_base_url,
+    stringify_document, walk_and_embed_assets,
 };
 use monolith::opts::Options;
 use monolith::url::{
-    data_to_data_url, data_url_to_data, is_data_url, is_file_url, is_http_url, resolve_url,
+    data_to_data_url, is_data_url, is_file_url, is_http_url, parse_data_url, resolve_url,
 };
 use monolith::utils::retrieve_asset;
 
@@ -47,12 +48,22 @@ impl Output {
     }
 }
 
+pub fn read_stdin() -> String {
+    let mut buffer = String::new();
+    for line in io::stdin().lock().lines() {
+        buffer += line.unwrap_or_default().as_str();
+        buffer += "\n";
+    }
+    buffer
+}
+
 fn main() {
     let options = Options::from_args();
     let original_target: &str = &options.target;
     let target_url: &str;
-    let base_url;
+    let mut base_url: String;
     let mut dom;
+    let mut use_stdin: bool = false;
 
     // Pre-process the input
     let cwd_normalized: String =
@@ -63,15 +74,24 @@ fn main() {
 
     // Determine exact target URL
     if target.clone().len() == 0 {
-        eprintln!("No target specified");
+        if !options.silent {
+            eprintln!("No target specified");
+        }
         process::exit(1);
+    } else if target.clone() == "-" {
+        // Read from pipe (stdin)
+        use_stdin = true;
+        // Default target URL to empty data URL; the user can control it via --base-url
+        target_url = "data:text/html,"
     } else if is_http_url(target.clone()) || is_data_url(target.clone()) {
         target_url = target.as_str();
     } else if is_file_url(target.clone()) {
         target_url = target.as_str();
     } else if path.exists() {
         if !path.is_file() {
-            eprintln!("Local target is not a file: {}", original_target);
+            if !options.silent {
+                eprintln!("Local target is not a file: {}", original_target);
+            }
             process::exit(1);
         }
         target.insert_str(0, if cfg!(windows) { "file:///" } else { "file://" });
@@ -110,37 +130,64 @@ fn main() {
         .build()
         .expect("Failed to initialize HTTP client");
 
+    // At this stage we assume that the base URL is the same as the target URL
+    base_url = str!(target_url);
+
     // Retrieve target document
-    if is_file_url(target_url) || is_http_url(target_url) {
-        match retrieve_asset(
-            &mut cache,
-            &client,
-            target_url,
-            target_url,
-            options.silent,
-            0,
-        ) {
+    if use_stdin {
+        dom = html_to_dom(&read_stdin());
+    } else if is_file_url(target_url) || is_http_url(target_url) {
+        match retrieve_asset(&mut cache, &client, target_url, target_url, &options, 0) {
             Ok((data, final_url, _media_type)) => {
-                base_url = final_url;
+                if options.base_url.clone().unwrap_or(str!()).is_empty() {
+                    base_url = final_url
+                }
                 dom = html_to_dom(&String::from_utf8_lossy(&data));
             }
             Err(_) => {
-                eprintln!("Could not retrieve target document");
+                if !options.silent {
+                    eprintln!("Could not retrieve target document");
+                }
                 process::exit(1);
             }
         }
     } else if is_data_url(target_url) {
-        let (media_type, data): (String, Vec<u8>) = data_url_to_data(target_url);
+        let (media_type, data): (String, Vec<u8>) = parse_data_url(target_url);
         if !media_type.eq_ignore_ascii_case("text/html") {
-            eprintln!("Unsupported data URL media type");
+            if !options.silent {
+                eprintln!("Unsupported data URL media type");
+            }
             process::exit(1);
         }
-        base_url = str!(target_url);
         dom = html_to_dom(&String::from_utf8_lossy(&data));
     } else {
         process::exit(1);
     }
 
+    // Use custom base URL if specified, read and use what's in the DOM otherwise
+    if !options.base_url.clone().unwrap_or(str!()).is_empty() {
+        if is_data_url(options.base_url.clone().unwrap()) {
+            if !options.silent {
+                eprintln!("Data URLs cannot be used as base URL");
+            }
+            process::exit(1);
+        } else {
+            base_url = options.base_url.clone().unwrap();
+        }
+    } else {
+        if let Some(existing_base_url) = get_base_url(&dom.document) {
+            base_url = resolve_url(target_url, existing_base_url).unwrap();
+        }
+    }
+
+    // Embed remote assets
+    walk_and_embed_assets(&mut cache, &client, &base_url, &dom.document, &options, 0);
+
+    // Update or add new BASE tag to reroute network requests and hash-links in the final document
+    if let Some(new_base_url) = options.base_url.clone() {
+        dom = set_base_url(&dom.document, new_base_url);
+    }
+
     // Request and embed /favicon.ico (unless it's already linked in the document)
     if !options.no_images && is_http_url(target_url) && !has_favicon(&dom.document) {
         let favicon_ico_url: String = resolve_url(&base_url, "/favicon.ico").unwrap();
@@ -150,7 +197,7 @@ fn main() {
             &client,
             &base_url,
             &favicon_ico_url,
-            options.silent,
+            &options,
             0,
         ) {
             Ok((data, final_url, media_type)) => {
@@ -163,15 +210,12 @@ fn main() {
         }
     }
 
-    // Embed remote assets
-    walk_and_embed_assets(&mut cache, &client, &base_url, &dom.document, &options, 0);
-
     // Serialize DOM tree
     let mut result: String = stringify_document(&dom.document, &options);
 
     // Add metadata tag
     if !options.no_metadata {
-        let metadata_comment = metadata_tag(&base_url);
+        let metadata_comment: String = create_metadata_tag(&target_url);
         result.insert_str(0, &metadata_comment);
         if metadata_comment.len() > 0 {
             result.insert_str(metadata_comment.len(), "\n");

src/opts.rs

@@ -2,19 +2,23 @@ use clap::{App, Arg};
 
 #[derive(Default)]
 pub struct Options {
-    pub target: String,
+    pub no_audio: bool,
+    pub base_url: Option<String>,
     pub no_css: bool,
-    pub no_fonts: bool,
+    pub ignore_errors: bool,
     pub no_frames: bool,
+    pub no_fonts: bool,
     pub no_images: bool,
+    pub isolate: bool,
     pub no_js: bool,
     pub insecure: bool,
-    pub isolate: bool,
+    pub no_metadata: bool,
     pub output: String,
     pub silent: bool,
     pub timeout: u64,
     pub user_agent: String,
-    pub no_metadata: bool,
+    pub no_video: bool,
+    pub target: String,
 }
 
 const ASCII: &str = " \
@@ -36,6 +40,22 @@ impl Options {
             .version(crate_version!())
             .author(crate_authors!("\n"))
             .about(format!("{}\n{}", ASCII, crate_description!()).as_str())
+            .args_from_usage("-a, --no-audio 'Removes audio sources'")
+            .args_from_usage("-b, --base-url=[http://localhost/] 'Sets custom base URL'")
+            .args_from_usage("-c, --no-css 'Removes CSS'")
+            .args_from_usage("-e, --ignore-errors 'Ignore network errors'")
+            .args_from_usage("-f, --no-frames 'Removes frames and iframes'")
+            .args_from_usage("-F, --no-fonts 'Removes fonts'")
+            .args_from_usage("-i, --no-images 'Removes images'")
+            .args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
+            .args_from_usage("-j, --no-js 'Removes JavaScript'")
+            .args_from_usage("-k, --insecure 'Allows invalid X.509 (TLS) certificates'")
+            .args_from_usage("-M, --no-metadata 'Excludes timestamp and source information'")
+            .args_from_usage("-o, --output=[document.html] 'Writes output to <file>'")
+            .args_from_usage("-s, --silent 'Suppresses verbosity'")
+            .args_from_usage("-t, --timeout=[60] 'Adjusts network request timeout'")
+            .args_from_usage("-u, --user-agent=[Firefox] 'Sets custom User-Agent string'")
+            .args_from_usage("-v, --no-video 'Removes video sources'")
             .arg(
                 Arg::with_name("target")
                     .required(true)
@@ -43,20 +63,6 @@ impl Options {
                     .index(1)
                     .help("URL or file path"),
             )
-            // .args_from_usage("-a, --no-audio 'Removes audio sources'")
-            .args_from_usage("-c, --no-css 'Removes CSS'")
-            .args_from_usage("-f, --no-frames 'Removes frames and iframes'")
-            .args_from_usage("-F, --no-fonts 'Removes fonts'")
-            .args_from_usage("-i, --no-images 'Removes images'")
-            .args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
-            .args_from_usage("-j, --no-js 'Removes JavaScript'")
-            .args_from_usage("-k, --insecure 'Allows invalid X.509 (TLS) certificates'")
-            .args_from_usage("-M, --no-metadata 'Excludes metadata information from the document'")
-            .args_from_usage("-o, --output=[document.html] 'Write output to <file>'")
-            .args_from_usage("-s, --silent 'Suppresses verbosity'")
-            .args_from_usage("-t, --timeout=[60] 'Adjust network request timeout'")
-            .args_from_usage("-u, --user-agent=[Firefox] 'Set custom User-Agent string'")
-            // .args_from_usage("-v, --no-video 'Removes video sources'")
             .get_matches();
         let mut options: Options = Options::default();
 
@@ -65,7 +71,12 @@ impl Options {
             .value_of("target")
             .expect("please set target")
             .to_string();
+        options.no_audio = app.is_present("no-audio");
+        if let Some(base_url) = app.value_of("base-url") {
+            options.base_url = Some(str!(base_url));
+        }
         options.no_css = app.is_present("no-css");
+        options.ignore_errors = app.is_present("ignore-errors");
         options.no_frames = app.is_present("no-frames");
         options.no_fonts = app.is_present("no-fonts");
         options.no_images = app.is_present("no-images");
@@ -84,6 +95,7 @@ impl Options {
             .value_of("user-agent")
             .unwrap_or(DEFAULT_USER_AGENT)
             .to_string();
+        options.no_video = app.is_present("no-video");
 
         options
     }

src/tests/cli/base_url.rs

@@ -0,0 +1,123 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn add_new_when_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("http://localhost:8000/")
+            .arg("data:text/html,Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost:8000/\"></base>\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn keep_existing_when_none_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost:8000/\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn override_existing_when_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("http://localhost/")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost/\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_existing_when_empty_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}

src/tests/cli/basic.rs

@@ -337,7 +337,7 @@ mod passing {
                 <title>Local HTML file</title>\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
-                <img alt=\"\" src=\"{empty_image}\">\n  \
+                <img src=\"{empty_image}\" alt=\"\">\n  \
                 <a href=\"file://local-file.html/\">Tricky href</a>\n  \
                 <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
                 <script></script>\n\n\n\n\
@@ -399,7 +399,7 @@ mod passing {
                 <title>Local HTML file</title>\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
-                <img alt=\"\" src=\"{empty_image}\">\n  \
+                <img src=\"{empty_image}\" alt=\"\">\n  \
                 <a href=\"file://local-file.html/\">Tricky href</a>\n  \
                 <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
                 <script></script>\n\n\n\n\

src/tests/cli/mod.rs

@@ -0,0 +1,2 @@
+mod base_url;
+mod basic;

src/tests/html/check_integrity.rs

@@ -11,7 +11,7 @@ mod passing {
 
     #[test]
     fn empty_input_sha256() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "".as_bytes(),
             "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
         ));
@@ -19,7 +19,7 @@ mod passing {
 
     #[test]
     fn sha256() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha256-9EWAHgy4mSYsm54hmDaIDXPKLRsLnBX7lZyQ6xISNOM="
         ));
@@ -27,7 +27,7 @@ mod passing {
 
     #[test]
     fn sha384() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha384-gc9l7omltke8C33bedgh15E12M7RrAQa5t63Yb8APlpe7ZhiqV23+oqiulSJl3Kw"
         ));
@@ -35,7 +35,7 @@ mod passing {
 
     #[test]
     fn sha512() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha512-zG5B88cYMqcdiMi9gz0XkOFYw2BpjeYdn5V6+oFrMgSNjRpqL7EF8JEwl17ztZbK3N7I/tTwp3kxQbN1RgFBww=="
         ));
@@ -55,20 +55,17 @@ mod failing {
 
     #[test]
     fn empty_hash() {
-        assert!(!html::has_proper_integrity(
-            "abcdef0123456789".as_bytes(),
-            ""
-        ));
+        assert!(!html::check_integrity("abcdef0123456789".as_bytes(), ""));
     }
 
     #[test]
     fn empty_input_empty_hash() {
-        assert!(!html::has_proper_integrity("".as_bytes(), ""));
+        assert!(!html::check_integrity("".as_bytes(), ""));
     }
 
     #[test]
     fn sha256() {
-        assert!(!html::has_proper_integrity(
+        assert!(!html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha256-badhash"
         ));
@@ -76,7 +73,7 @@ mod failing {
 
     #[test]
     fn sha384() {
-        assert!(!html::has_proper_integrity(
+        assert!(!html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha384-badhash"
         ));
@@ -84,7 +81,7 @@ mod failing {
 
     #[test]
     fn sha512() {
-        assert!(!html::has_proper_integrity(
+        assert!(!html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha512-badhash"
         ));

src/tests/html/compose_csp.rs

@@ -14,7 +14,7 @@ mod passing {
     fn isolated() {
         let mut options = Options::default();
         options.isolate = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "default-src 'unsafe-inline' data:;");
     }
@@ -23,7 +23,7 @@ mod passing {
     fn no_css() {
         let mut options = Options::default();
         options.no_css = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "style-src 'none';");
     }
@@ -32,7 +32,7 @@ mod passing {
     fn no_fonts() {
         let mut options = Options::default();
         options.no_fonts = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "font-src 'none';");
     }
@@ -41,7 +41,7 @@ mod passing {
     fn no_frames() {
         let mut options = Options::default();
         options.no_frames = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "frame-src 'none'; child-src 'none';");
     }
@@ -50,7 +50,7 @@ mod passing {
     fn no_js() {
         let mut options = Options::default();
         options.no_js = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "script-src 'none';");
     }
@@ -59,7 +59,7 @@ mod passing {
     fn no_images() {
         let mut options = Options::default();
         options.no_images = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "img-src data:;");
     }
@@ -73,7 +73,7 @@ mod passing {
         options.no_frames = true;
         options.no_js = true;
         options.no_images = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "default-src 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;");
     }

src/tests/html/create_metadata_tag.rs

@@ -15,7 +15,7 @@ mod passing {
     fn http_url() {
         let url = "http://192.168.1.1/";
         let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
-        let metadata_comment: String = html::metadata_tag(url);
+        let metadata_comment: String = html::create_metadata_tag(url);
 
         assert_eq!(
             metadata_comment,
@@ -33,7 +33,7 @@ mod passing {
     fn file_url() {
         let url = "file:///home/monolith/index.html";
         let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
-        let metadata_comment: String = html::metadata_tag(url);
+        let metadata_comment: String = html::create_metadata_tag(url);
 
         assert_eq!(
             metadata_comment,
@@ -50,7 +50,7 @@ mod passing {
     fn data_url() {
         let url = "data:text/html,Hello%2C%20World!";
         let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
-        let metadata_comment: String = html::metadata_tag(url);
+        let metadata_comment: String = html::create_metadata_tag(url);
 
         assert_eq!(
             metadata_comment,
@@ -77,6 +77,6 @@ mod failing {
 
     #[test]
     fn empty_string() {
-        assert_eq!(html::metadata_tag(""), "");
+        assert_eq!(html::create_metadata_tag(""), "");
     }
 }

src/tests/html/embed_srcset.rs

@@ -24,8 +24,77 @@ mod passing {
         let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
 
         assert_eq!(
+            embedded_css,
             format!("{} 1x, {} 2x", empty_image!(), empty_image!()),
-            embedded_css
+        );
+    }
+
+    #[test]
+    fn commas_within_file_names() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small,s.png 1x, large,l.png 2x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{} 1x, {} 2x", empty_image!(), empty_image!()),
+        );
+    }
+
+    #[test]
+    fn tabs_and_newlines_after_commas() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small,s.png 1x,\nmedium,m.png 2x,\nlarge,l.png 3x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!(
+                "{} 1x, {} 2x, {} 3x",
+                empty_image!(),
+                empty_image!(),
+                empty_image!()
+            ),
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn trailing_comma() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small.png 1x, large.png 2x,";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{} 1x, {} 2x,", empty_image!(), empty_image!()),
         );
     }
 }

src/tests/html/get_base_url.rs

@@ -0,0 +1,104 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+
+    #[test]
+    fn present() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"https://musicbrainz.org\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(
+            html::get_base_url(&dom.document),
+            Some(str!("https://musicbrainz.org"))
+        );
+    }
+
+    #[test]
+    fn multiple_tags() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"https://www.discogs.com/\" />
+        <base href=\"https://musicbrainz.org\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(
+            html::get_base_url(&dom.document),
+            Some(str!("https://www.discogs.com/"))
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn absent() {
+        let html = "<!doctype html>
+<html>
+    <head>
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), None);
+    }
+
+    #[test]
+    fn no_href() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), None);
+    }
+
+    #[test]
+    fn empty_href() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), Some(str!()));
+    }
+}

src/tests/html/get_node_attr.rs

@@ -0,0 +1,54 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn div_two_style_attributes() {
+        let html = "<!doctype html><html><head></head><body><DIV STYLE=\"color: blue;\" style=\"display: none;\"></div></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "class"), None);
+                    } else if node_name == "div" {
+                        assert_eq!(
+                            html::get_node_attr(node, "style"),
+                            Some(str!("color: blue;"))
+                        );
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 6);
+    }
+}

src/tests/html/get_node_name.rs

@@ -12,7 +12,7 @@ mod passing {
     use crate::html;
 
     #[test]
-    fn get_node_name() {
+    fn parent_node_names() {
         let html = "<!doctype html><html><HEAD></HEAD><body><div><P></P></div></body></html>";
         let dom = html::html_to_dom(&html);
         let mut count = 0;

src/tests/html/has_favicon.rs

@@ -8,7 +8,6 @@
 #[cfg(test)]
 mod passing {
     use crate::html;
-    use crate::opts::Options;
 
     #[test]
     fn icon() {
@@ -39,7 +38,6 @@ mod passing {
 #[cfg(test)]
 mod failing {
     use crate::html;
-    use crate::opts::Options;
 
     #[test]
     fn absent() {

src/tests/html/mod.rs

@@ -1,10 +1,13 @@
 mod add_favicon;
-mod csp;
+mod check_integrity;
+mod compose_csp;
+mod create_metadata_tag;
 mod embed_srcset;
+mod get_base_url;
+mod get_node_attr;
 mod get_node_name;
 mod has_favicon;
-mod has_proper_integrity;
 mod is_icon;
-mod metadata_tag;
+mod set_node_attr;
 mod stringify_document;
 mod walk_and_embed_assets;

src/tests/html/set_node_attr.rs

@@ -0,0 +1,105 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn html_lang_and_body_style() {
+        let html = "<!doctype html><html lang=\"en\"><head></head><body></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "html" {
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("en")));
+
+                        html::set_node_attr(node, "lang", Some(str!("de")));
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("de")));
+
+                        html::set_node_attr(node, "lang", None);
+                        assert_eq!(html::get_node_attr(node, "lang"), None);
+
+                        html::set_node_attr(node, "lang", Some(str!("")));
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("")));
+                    } else if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "style"), None);
+
+                        html::set_node_attr(node, "style", Some(str!("display: none;")));
+                        assert_eq!(
+                            html::get_node_attr(node, "style"),
+                            Some(str!("display: none;"))
+                        );
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 5);
+    }
+
+    #[test]
+    fn body_background() {
+        let html = "<!doctype html><html lang=\"en\"><head></head><body background=\"1\" background=\"2\"></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "background"), Some(str!("1")));
+
+                        html::set_node_attr(node, "background", None);
+                        assert_eq!(html::get_node_attr(node, "background"), None);
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 5);
+    }
+}

src/tests/html/walk_and_embed_assets.rs

@@ -319,8 +319,8 @@ mod passing {
             buf.iter().map(|&c| c as char).collect::<String>(),
             "<html>\
                 <head>\
-                    <meta>\
-                    <meta>\
+                    <meta http-equiv=\"disabled by monolith (Refresh)\" value=\"20\">\
+                    <meta http-equiv=\"disabled by monolith (Location)\" value=\"https://freebsd.org\">\
                 </head>\
                 <body></body>\
             </html>"

src/tests/mod.rs

@@ -4,4 +4,5 @@ mod html;
 mod js;
 mod macros;
 mod opts;
+mod url;
 mod utils;

src/tests/opts.rs

@@ -14,6 +14,7 @@ mod passing {
         let options: Options = Options::default();
 
         assert_eq!(options.target, str!());
+        assert_eq!(options.no_audio, false);
         assert_eq!(options.no_css, false);
         assert_eq!(options.no_frames, false);
         assert_eq!(options.no_fonts, false);
@@ -26,5 +27,6 @@ mod passing {
         assert_eq!(options.silent, false);
         assert_eq!(options.timeout, 0);
         assert_eq!(options.user_agent, "");
+        assert_eq!(options.no_video, false);
     }
 }

src/tests/url/get_url_fragment.rs

@@ -18,9 +18,31 @@ mod passing {
             "test"
         );
     }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
 
     #[test]
     fn https_empty() {
         assert_eq!(url::get_url_fragment("https://kernel.org#"), "");
     }
+
+    #[test]
+    fn no_fragment() {
+        assert_eq!(url::get_url_fragment("https://kernel.org"), "");
+    }
+
+    #[test]
+    fn dummy_data_url() {
+        assert_eq!(url::get_url_fragment("data:text/html,"), "");
+    }
 }

src/tests/url/mod.rs

@@ -1,12 +1,12 @@
 mod clean_url;
 mod data_to_data_url;
-mod data_url_to_data;
 mod decode_url;
 mod file_url_to_fs_path;
 mod get_url_fragment;
 mod is_data_url;
 mod is_file_url;
 mod is_http_url;
+mod parse_data_url;
 mod resolve_url;
 mod url_has_protocol;
 mod url_with_fragment;

src/tests/url/parse_data_url.rs

@@ -11,7 +11,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_base64() {
-        let (media_type, data) = url::data_url_to_data("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
+        let (media_type, data) = url::parse_data_url("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
 
         assert_eq!(media_type, "text/html");
         assert_eq!(
@@ -22,7 +22,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_utf8() {
-        let (media_type, data) = url::data_url_to_data(
+        let (media_type, data) = url::parse_data_url(
             "data:text/html;utf8,Work expands so as to fill the time available for its completion",
         );
 
@@ -35,7 +35,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_plaintext() {
-        let (media_type, data) = url::data_url_to_data(
+        let (media_type, data) = url::parse_data_url(
             "data:text/html,Work expands so as to fill the time available for its completion",
         );
 
@@ -48,7 +48,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_charset_utf_8_between_two_whitespaces() {
-        let (media_type, data) = url::data_url_to_data(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
+        let (media_type, data) = url::parse_data_url(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
 
         assert_eq!(media_type, "text/html");
         assert_eq!(
@@ -59,8 +59,7 @@ mod passing {
 
     #[test]
     fn parse_text_css_url_encoded() {
-        let (media_type, data) =
-            url::data_url_to_data("data:text/css,div{background-color:%23000}");
+        let (media_type, data) = url::parse_data_url("data:text/css,div{background-color:%23000}");
 
         assert_eq!(media_type, "text/css");
         assert_eq!(String::from_utf8_lossy(&data), "div{background-color:#000}");
@@ -68,7 +67,7 @@ mod passing {
 
     #[test]
     fn parse_no_media_type_base64() {
-        let (media_type, data) = url::data_url_to_data("data:;base64,dGVzdA==");
+        let (media_type, data) = url::parse_data_url("data:;base64,dGVzdA==");
 
         assert_eq!(media_type, "");
         assert_eq!(String::from_utf8_lossy(&data), "test");
@@ -76,7 +75,7 @@ mod passing {
 
     #[test]
     fn parse_no_media_type_no_encoding() {
-        let (media_type, data) = url::data_url_to_data("data:;,test%20test");
+        let (media_type, data) = url::parse_data_url("data:;,test%20test");
 
         assert_eq!(media_type, "");
         assert_eq!(String::from_utf8_lossy(&data), "test test");
@@ -96,7 +95,7 @@ mod failing {
 
     #[test]
     fn just_word_data() {
-        let (media_type, data) = url::data_url_to_data("data");
+        let (media_type, data) = url::parse_data_url("data");
 
         assert_eq!(media_type, "");
         assert_eq!(String::from_utf8_lossy(&data), "");

src/tests/url/resolve_url.rs

@@ -7,208 +7,165 @@
 
 #[cfg(test)]
 mod passing {
-    use url::ParseError;
-
     use crate::url;
 
     #[test]
-    fn from_https_to_level_up_relative() -> Result<(), ParseError> {
-        let resolved_url =
-            url::resolve_url("https://www.kernel.org", "../category/signatures.html")?;
-
+    fn from_https_to_level_up_relative() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url("https://www.kernel.org", "../category/signatures.html")
+                .unwrap_or_default(),
             "https://www.kernel.org/category/signatures.html"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_just_filename_to_full_https_url() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "saved_page.htm",
-            "https://www.kernel.org/category/signatures.html",
-        )?;
-
+    fn from_just_filename_to_full_https_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "saved_page.htm",
+                "https://www.kernel.org/category/signatures.html",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/category/signatures.html"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_url_to_url_with_no_protocol() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "https://www.kernel.org",
-            "//www.kernel.org/theme/images/logos/tux.png",
-        )?;
-
+    fn from_https_url_to_url_with_no_protocol() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.kernel.org",
+                "//www.kernel.org/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/theme/images/logos/tux.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() -> Result<(), ParseError>
-    {
-        let resolved_url = url::resolve_url(
-            "https://www.kernel.org",
-            "//another-host.org/theme/images/logos/tux.png",
-        )?;
-
+    fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.kernel.org",
+                "//another-host.org/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
             "https://another-host.org/theme/images/logos/tux.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_url_to_relative_root_path() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "https://www.kernel.org/category/signatures.html",
-            "/theme/images/logos/tux.png",
-        )?;
-
+    fn from_https_url_to_relative_root_path() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.kernel.org/category/signatures.html",
+                "/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/theme/images/logos/tux.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_to_just_filename() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "https://www.w3schools.com/html/html_iframe.asp",
-            "default.asp",
-        )?;
-
+    fn from_https_to_just_filename() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.w3schools.com/html/html_iframe.asp",
+                "default.asp",
+            )
+            .unwrap_or_default(),
             "https://www.w3schools.com/html/default.asp"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_data_url_to_https() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "https://www.kernel.org/category/signatures.html",
-        )?;
-
+    fn from_data_url_to_https() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "https://www.kernel.org/category/signatures.html",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/category/signatures.html"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_data_url_to_data_url() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K",
-        )?;
-
+    fn from_data_url_to_data_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K",
+            )
+            .unwrap_or_default(),
             "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_file_url_to_relative_path() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "file:///home/user/Websites/my-website/index.html",
-            "assets/images/logo.png",
-        )
-        .unwrap_or(str!());
-
+    fn from_file_url_to_relative_path() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "file:///home/user/Websites/my-website/index.html",
+                "assets/images/logo.png",
+            )
+            .unwrap_or_default(),
             "file:///home/user/Websites/my-website/assets/images/logo.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_file_url_to_relative_path_with_backslashes() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "file:\\\\\\home\\user\\Websites\\my-website\\index.html",
-            "assets\\images\\logo.png",
-        )
-        .unwrap_or(str!());
-
+    fn from_file_url_to_relative_path_with_backslashes() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "file:\\\\\\home\\user\\Websites\\my-website\\index.html",
+                "assets\\images\\logo.png",
+            )
+            .unwrap_or_default(),
             "file:///home/user/Websites/my-website/assets/images/logo.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_data_url_to_file_url() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "file:///etc/passwd",
-        )
-        .unwrap_or(str!());
-
-        assert_eq!(resolved_url.as_str(), "file:///etc/passwd");
-
-        Ok(())
-    }
-
-    #[test]
-    fn preserve_fragment() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "http://doesnt-matter.local/",
-            "css/fonts/fontmarvelous.svg#fontmarvelous",
-        )
-        .unwrap_or(str!());
-
+    fn from_data_url_to_file_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "file:///etc/passwd",
+            )
+            .unwrap_or_default(),
+            "file:///etc/passwd"
+        );
+    }
+
+    #[test]
+    fn preserve_fragment() {
+        assert_eq!(
+            url::resolve_url(
+                "http://doesnt-matter.local/",
+                "css/fonts/fontmarvelous.svg#fontmarvelous",
+            )
+            .unwrap_or_default(),
             "http://doesnt-matter.local/css/fonts/fontmarvelous.svg#fontmarvelous"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn resolve_from_file_url_to_file_url() -> Result<(), ParseError> {
-        let resolved_url = if cfg!(windows) {
-            url::resolve_url("file:///c:/index.html", "file:///c:/image.png").unwrap_or(str!())
-        } else {
-            url::resolve_url("file:///tmp/index.html", "file:///tmp/image.png").unwrap_or(str!())
-        };
-
+    fn resolve_from_file_url_to_file_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            if cfg!(windows) {
+                url::resolve_url("file:///c:/index.html", "file:///c:/image.png")
+                    .unwrap_or_default()
+            } else {
+                url::resolve_url("file:///tmp/index.html", "file:///tmp/image.png")
+                    .unwrap_or_default()
+            },
             if cfg!(windows) {
                 "file:///c:/image.png"
             } else {
                 "file:///tmp/image.png"
             }
         );
-
-        Ok(())
     }
 }
 
@@ -222,18 +179,16 @@ mod passing {
 #[cfg(test)]
 mod failing {
     use crate::url;
-    use url::ParseError;
 
     #[test]
-    fn from_data_url_to_url_with_no_protocol() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "//www.w3schools.com/html/html_iframe.asp",
-        )
-        .unwrap_or(str!());
-
-        assert_eq!(resolved_url.as_str(), "");
-
-        Ok(())
+    fn from_data_url_to_url_with_no_protocol() {
+        assert_eq!(
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "//www.w3schools.com/html/html_iframe.asp",
+            )
+            .unwrap_or_default(),
+            ""
+        );
     }
 }

src/tests/url/url_has_protocol.rs

@@ -54,6 +54,11 @@ mod passing {
             "MAILTO:somebody@somewhere.com?subject=hello"
         ));
     }
+
+    #[test]
+    fn empty_data_url() {
+        assert!(url::url_has_protocol("data:text/html,"));
+    }
 }
 
 //  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
@@ -65,13 +70,11 @@ mod passing {
 
 #[cfg(test)]
 mod failing {
-    use crate::utils;
+    use crate::url;
 
     #[test]
     fn url_with_no_protocol() {
-        assert!(!url::url_has_protocol(
-            "//some-hostname.com/some-file.html"
-        ));
+        assert!(!url::url_has_protocol("//some-hostname.com/some-file.html"));
     }
 
     #[test]

src/tests/utils/retrieve_asset.rs

@@ -11,6 +11,7 @@ mod passing {
     use std::collections::HashMap;
     use std::env;
 
+    use crate::opts::Options;
     use crate::url;
     use crate::utils;
 
@@ -19,6 +20,9 @@ mod passing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         // If both source and target are data URLs,
         // ensure the result contains target data URL
         let (data, final_url, media_type) = utils::retrieve_asset(
@@ -26,7 +30,7 @@ mod passing {
             &client,
             "data:text/html;base64,c291cmNl",
             "data:text/html;base64,dGFyZ2V0",
-            false,
+            &options,
             0,
         )
         .unwrap();
@@ -46,6 +50,9 @@ mod passing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
 
         // Inclusion of local assets from local sources should be allowed
@@ -63,7 +70,7 @@ mod passing {
                 file = file_url_protocol,
                 cwd = cwd.to_str().unwrap()
             ),
-            false,
+            &options,
             0,
         )
         .unwrap();
@@ -91,6 +98,7 @@ mod failing {
     use reqwest::blocking::Client;
     use std::collections::HashMap;
 
+    use crate::opts::Options;
     use crate::utils;
 
     #[test]
@@ -98,13 +106,16 @@ mod failing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         // Inclusion of local assets from data URL sources should not be allowed
         match utils::retrieve_asset(
             cache,
             &client,
             "data:text/html;base64,SoUrCe",
             "file:///etc/passwd",
-            false,
+            &options,
             0,
         ) {
             Ok((..)) => {
@@ -121,13 +132,16 @@ mod failing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         // Inclusion of local assets from remote sources should not be allowed
         match utils::retrieve_asset(
             cache,
             &client,
             "https://kernel.org/",
             "file:///etc/passwd",
-            false,
+            &options,
             0,
         ) {
             Ok((..)) => {

src/url.rs

@@ -33,45 +33,6 @@ pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str) -> String {
     format!("data:{};base64,{}", media_type, base64::encode(data))
 }
 
-pub fn data_url_to_data<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
-    let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
-    let path: String = parsed_url.path().to_string();
-    let comma_loc: usize = path.find(',').unwrap_or(path.len());
-
-    let meta_data: String = path.chars().take(comma_loc).collect();
-    let raw_data: String = path.chars().skip(comma_loc + 1).collect();
-
-    let text: String = decode_url(raw_data);
-
-    let meta_data_items: Vec<&str> = meta_data.split(';').collect();
-    let mut media_type: String = str!();
-    let mut encoding: &str = "";
-
-    let mut i: i8 = 0;
-    for item in &meta_data_items {
-        if i == 0 {
-            media_type = str!(item);
-        } else {
-            if item.eq_ignore_ascii_case("base64")
-                || item.eq_ignore_ascii_case("utf8")
-                || item.eq_ignore_ascii_case("charset=UTF-8")
-            {
-                encoding = item;
-            }
-        }
-
-        i = i + 1;
-    }
-
-    let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
-        base64::decode(&text).unwrap_or(vec![])
-    } else {
-        text.as_bytes().to_vec()
-    };
-
-    (media_type, data)
-}
-
 pub fn decode_url(input: String) -> String {
     let input: String = input.replace("+", "%2B");
 
@@ -113,10 +74,9 @@ pub fn file_url_to_fs_path(url: &str) -> String {
 }
 
 pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
-    if Url::parse(url.as_ref()).unwrap().fragment() == None {
-        str!()
-    } else {
-        str!(Url::parse(url.as_ref()).unwrap().fragment().unwrap())
+    match Url::parse(url.as_ref()) {
+        Ok(parsed_url) => parsed_url.fragment().unwrap_or("").to_string(),
+        Err(_err) => str!(),
     }
 }
 
@@ -138,6 +98,45 @@ pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
         .unwrap_or(false)
 }
 
+pub fn parse_data_url<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
+    let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
+    let path: String = parsed_url.path().to_string();
+    let comma_loc: usize = path.find(',').unwrap_or(path.len());
+
+    let meta_data: String = path.chars().take(comma_loc).collect();
+    let raw_data: String = path.chars().skip(comma_loc + 1).collect();
+
+    let text: String = decode_url(raw_data);
+
+    let meta_data_items: Vec<&str> = meta_data.split(';').collect();
+    let mut media_type: String = str!();
+    let mut encoding: &str = "";
+
+    let mut i: i8 = 0;
+    for item in &meta_data_items {
+        if i == 0 {
+            media_type = str!(item);
+        } else {
+            if item.eq_ignore_ascii_case("base64")
+                || item.eq_ignore_ascii_case("utf8")
+                || item.eq_ignore_ascii_case("charset=UTF-8")
+            {
+                encoding = item;
+            }
+        }
+
+        i = i + 1;
+    }
+
+    let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
+        base64::decode(&text).unwrap_or(vec![])
+    } else {
+        text.as_bytes().to_vec()
+    };
+
+    (media_type, data)
+}
+
 pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
     let result = if is_http_url(to.as_ref()) {
         to.as_ref().to_string()

src/utils.rs

@@ -4,7 +4,8 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
-use crate::url::{clean_url, data_url_to_data, file_url_to_fs_path, is_data_url, is_file_url};
+use crate::opts::Options;
+use crate::url::{clean_url, file_url_to_fs_path, is_data_url, is_file_url, parse_data_url};
 
 const INDENT: &str = " ";
 
@@ -73,7 +74,7 @@ pub fn retrieve_asset(
     client: &Client,
     parent_url: &str,
     url: &str,
-    opt_silent: bool,
+    options: &Options,
     depth: u32,
 ) -> Result<(Vec<u8>, String, String), reqwest::Error> {
     if url.len() == 0 {
@@ -82,7 +83,7 @@ pub fn retrieve_asset(
     }
 
     if is_data_url(&url) {
-        let (media_type, data) = data_url_to_data(url);
+        let (media_type, data) = parse_data_url(url);
         Ok((data, url.to_string(), media_type))
     } else if is_file_url(&url) {
         // Check if parent_url is also file:///
@@ -95,7 +96,7 @@ pub fn retrieve_asset(
         let fs_file_path: String = file_url_to_fs_path(url);
         let path = Path::new(&fs_file_path);
         if path.exists() {
-            if !opt_silent {
+            if !options.silent {
                 eprintln!("{}{}", indent(depth).as_str(), &url);
             }
 
@@ -109,7 +110,7 @@ pub fn retrieve_asset(
 
         if cache.contains_key(&cache_key) {
             // URL is in cache, we get and return it
-            if !opt_silent {
+            if !options.silent {
                 eprintln!("{}{} (from cache)", indent(depth).as_str(), &url);
             }
 
@@ -120,34 +121,46 @@ pub fn retrieve_asset(
             ))
         } else {
             // URL not in cache, we retrieve the file
-            let mut response = client.get(url).send()?;
-            let res_url = response.url().to_string();
+            match client.get(url).send() {
+                Ok(mut response) => {
+                    if !options.ignore_errors && response.status() != 200 {
+                        if !options.silent {
+                            eprintln!("Unable to retrieve {} ({})", &url, response.status());
+                        }
+                        // Provoke error
+                        return Err(client.get("").send().unwrap_err());
+                    }
 
-            if !opt_silent {
-                if url == res_url {
-                    eprintln!("{}{}", indent(depth).as_str(), &url);
-                } else {
-                    eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &res_url);
+                    let res_url = response.url().to_string();
+
+                    if !options.silent {
+                        if url == res_url {
+                            eprintln!("{}{}", indent(depth).as_str(), &url);
+                        } else {
+                            eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &res_url);
+                        }
+                    }
+
+                    let new_cache_key: String = clean_url(&res_url);
+
+                    // Convert response into a byte array
+                    let mut data: Vec<u8> = vec![];
+                    response.copy_to(&mut data)?;
+
+                    // Attempt to obtain media type by reading the Content-Type header
+                    let media_type = response
+                        .headers()
+                        .get(CONTENT_TYPE)
+                        .and_then(|header| header.to_str().ok())
+                        .unwrap_or("");
+
+                    // Add retrieved resource to cache
+                    cache.insert(new_cache_key, data.clone());
+
+                    Ok((data, res_url, media_type.to_string()))
                 }
+                Err(error) => Err(error),
             }
-
-            let new_cache_key: String = clean_url(&res_url);
-
-            // Convert response into a byte array
-            let mut data: Vec<u8> = vec![];
-            response.copy_to(&mut data)?;
-
-            // Attempt to obtain media type by reading the Content-Type header
-            let media_type = response
-                .headers()
-                .get(CONTENT_TYPE)
-                .and_then(|header| header.to_str().ok())
-                .unwrap_or("");
-
-            // Add to cache
-            cache.insert(new_cache_key, data.clone());
-
-            Ok((data, res_url, media_type.to_string()))
         }
     }
 }