Merge pull request #251 from snshn/bump-version-again

Bump version number to 2.4.1

.github/workflows/cd.yml

@@ -39,11 +39,11 @@ jobs:
         sudo dpkg -x libssl1.1*.deb /cross-build-arm
         sudo dpkg -x libssl-dev*.deb /cross-build-arm
         rustup target add arm-unknown-linux-gnueabihf
-        echo "::set-env name=C_INCLUDE_PATH::/cross-build-arm/usr/include"
-        echo "::set-env name=OPENSSL_INCLUDE_DIR::/cross-build-arm/usr/include/arm-linux-gnueabihf"
-        echo "::set-env name=OPENSSL_LIB_DIR::/cross-build-arm/usr/lib/arm-linux-gnueabihf"
-        echo "::set-env name=PKG_CONFIG_ALLOW_CROSS::1"
-        echo "::set-env name=RUSTFLAGS::-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build-arm/usr/lib/arm-linux-gnueabihf -L/cross-build-arm/lib/arm-linux-gnueabihf"
+        echo "C_INCLUDE_PATH=/cross-build-arm/usr/include" >> $GITHUB_ENV
+        echo "OPENSSL_INCLUDE_DIR=/cross-build-arm/usr/include/arm-linux-gnueabihf" >> $GITHUB_ENV
+        echo "OPENSSL_LIB_DIR=/cross-build-arm/usr/lib/arm-linux-gnueabihf" >> $GITHUB_ENV
+        echo "PKG_CONFIG_ALLOW_CROSS=1" >> $GITHUB_ENV
+        echo "RUSTFLAGS=-C linker=arm-linux-gnueabihf-gcc -L/usr/arm-linux-gnueabihf/lib -L/cross-build-arm/usr/lib/arm-linux-gnueabihf -L/cross-build-arm/lib/arm-linux-gnueabihf" >> $GITHUB_ENV
     - name: Build the executable
       run: cargo build --release --target=arm-unknown-linux-gnueabihf
     - name: Attach artifact to the release

Cargo.toml

@@ -1,7 +1,6 @@
 [package]
 name = "monolith"
-version = "2.3.0"
-edition = "2018"
+version = "2.4.1"
 authors = [
     "Sunshine <sunshine@uberspace.net>",
     "Mahdi Robatipoor <mahdi.robatipoor@gmail.com>",
@@ -9,22 +8,35 @@ authors = [
     "Emi Simpson <emi@alchemi.dev>",
     "rhysd <lin90162@yahoo.co.jp>",
 ]
+edition = "2018"
 description = "CLI tool for saving web pages as a single HTML file"
+homepage = "https://github.com/Y2Z/monolith"
+repository = "https://github.com/Y2Z/monolith"
+readme = "README.md"
+keywords = ["web", "http", "html", "download", "command-line"]
+categories = ["command-line-utilities", "web-programming"]
+include = [
+    "src/*.rs",
+    "Cargo.toml",
+]
+license = "CC0-1.0"
 
 [dependencies]
-base64 = "0.12.2"
-chrono = "0.4.11" # Used to render comments indicating the time the page was saved
-clap = "2.33.1"
-cssparser = "0.27.2"
+atty = "0.2" # Used for highlighting network errors
+base64 = "0.13.0"
+chrono = "0.4.19" # Used for formatting creation timestamp
+clap = "2.33.3"
+cssparser = "0.28.1"
 html5ever = "0.24.1"
-sha2 = "0.9.0" # Used in calculating checksums during integrity checks
-url = "2.1.1"
+regex = "1.4.3" # Used for parsing srcset
+sha2 = "0.9.2" # Used for calculating checksums during integrity checks
+url = "2.2.0"
 
 [dependencies.reqwest]
-version = "0.10.6"
+version = "0.11.0"
 default-features = false
 features = ["default-tls", "blocking", "gzip"]
 
 [dev-dependencies]
-assert_cmd = "1.0.1"
-tempfile = "3.1.0"
+assert_cmd = "1.0.2"
+tempfile = "3.2.0"

Dockerfile

@@ -2,13 +2,13 @@ FROM rust
 
 WORKDIR /usr/local/src/
 RUN curl -s https://api.github.com/repos/y2z/monolith/releases/latest \
-| grep "tarball_url.*\"," \
-| cut -d '"' -f 4 \
-| wget -qi - -O monolith.tar.gz
+    | grep "tarball_url.*\"," \
+    | cut -d '"' -f 4 \
+    | wget -qi - -O monolith.tar.gz
 
 RUN tar xfz monolith.tar.gz \
-&& mv Y2Z-monolith-* monolith \
-&& rm monolith.tar.gz
+    && mv Y2Z-monolith-* monolith \
+    && rm monolith.tar.gz
 
 WORKDIR /usr/local/src/monolith
 RUN ls -a

LICENSE

@@ -1,24 +1,121 @@
-This is free and unencumbered software released into the public domain.
+Creative Commons Legal Code
 
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
+CC0 1.0 Universal
 
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
+    CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE
+    LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN
+    ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS
+    INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES
+    REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS
+    PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM
+    THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED
+    HEREUNDER.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
+Statement of Purpose
 
-For more information, please refer to <http://unlicense.org>
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator
+and subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for
+the purpose of contributing to a commons of creative, cultural and
+scientific works ("Commons") that the public can reliably and without fear
+of later claims of infringement build upon, modify, incorporate in other
+works, reuse and redistribute as freely as possible in any form whatsoever
+and for any purposes, including without limitation commercial purposes.
+These owners may contribute to the Commons to promote the ideal of a free
+culture and the further production of creative, cultural and scientific
+works, or to gain reputation or greater distribution for their Work in
+part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any
+expectation of additional consideration or compensation, the person
+associating CC0 with a Work (the "Affirmer"), to the extent that he or she
+is an owner of Copyright and Related Rights in the Work, voluntarily
+elects to apply CC0 to the Work and publicly distribute the Work under its
+terms, with knowledge of his or her Copyright and Related Rights in the
+Work and the meaning and intended legal effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not
+limited to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display,
+     communicate, and translate a Work;
+ ii. moral rights retained by the original author(s) and/or performer(s);
+iii. publicity and privacy rights pertaining to a person's image or
+     likeness depicted in a Work;
+ iv. rights protecting against unfair competition in regards to a Work,
+     subject to the limitations in paragraph 4(a), below;
+  v. rights protecting the extraction, dissemination, use and reuse of data
+     in a Work;
+ vi. database rights (such as those arising under Directive 96/9/EC of the
+     European Parliament and of the Council of 11 March 1996 on the legal
+     protection of databases, and under any national implementation
+     thereof, including any amended or successor version of such
+     directive); and
+vii. other similar, equivalent or corresponding rights throughout the
+     world based on applicable law or treaty, and any national
+     implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention
+of, applicable law, Affirmer hereby overtly, fully, permanently,
+irrevocably and unconditionally waives, abandons, and surrenders all of
+Affirmer's Copyright and Related Rights and associated claims and causes
+of action, whether now known or unknown (including existing as well as
+future claims and causes of action), in the Work (i) in all territories
+worldwide, (ii) for the maximum duration provided by applicable law or
+treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional
+purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each
+member of the public at large and to the detriment of Affirmer's heirs and
+successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public
+as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason
+be judged legally invalid or ineffective under applicable law, then the
+Waiver shall be preserved to the maximum extent permitted taking into
+account Affirmer's express Statement of Purpose. In addition, to the
+extent the Waiver is so judged Affirmer hereby grants to each affected
+person a royalty-free, non transferable, non sublicensable, non exclusive,
+irrevocable and unconditional license to exercise Affirmer's Copyright and
+Related Rights in the Work (i) in all territories worldwide, (ii) for the
+maximum duration provided by applicable law or treaty (including future
+time extensions), (iii) in any current or future medium and for any number
+of copies, and (iv) for any purpose whatsoever, including without
+limitation commercial, advertising or promotional purposes (the
+"License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any
+reason be judged legally invalid or ineffective under applicable law, such
+partial invalidity or ineffectiveness shall not invalidate the remainder
+of the License, and in such case Affirmer hereby affirms that he or she
+will not (i) exercise any of his or her remaining Copyright and Related
+Rights in the Work or (ii) assert any associated claims and causes of
+action with respect to the Work, in either case contrary to Affirmer's
+express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+ a. No trademark or patent rights held by Affirmer are waived, abandoned,
+    surrendered, licensed or otherwise affected by this document.
+ b. Affirmer offers the Work as-is and makes no representations or
+    warranties of any kind concerning the Work, express, implied,
+    statutory or otherwise, including without limitation warranties of
+    title, merchantability, fitness for a particular purpose, non
+    infringement, or the absence of latent or other defects, accuracy, or
+    the present or absence of errors, whether or not discoverable, all to
+    the greatest extent permissible under applicable law.
+ c. Affirmer disclaims responsibility for clearing rights of other persons
+    that may apply to the Work or any use thereof, including without
+    limitation any person's Copyright and Related Rights in the Work.
+    Further, Affirmer disclaims responsibility for obtaining any necessary
+    consents, permissions or other rights required for any use of the
+    Work.
+ d. Affirmer understands and acknowledges that Creative Commons is not a
+    party to this document and has no duty or obligation with respect to
+    this CC0 or use of the Work.

Makefile

@@ -10,7 +10,7 @@ build:
 test: build
 	@cargo test --locked
 	@cargo fmt --all -- --check
-.PHONY: test_code_formatting
+.PHONY: test
 
 lint:
 	@cargo fmt --all --

README.md

@@ -22,6 +22,9 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
 
 ## Installation
 
+### Using Cargo
+    $ cargo install monolith
+
 #### Via Homebrew (on macOS and GNU/Linux)
     $ brew install monolith
 
@@ -44,31 +47,41 @@ Dependency: `libssl-dev`
     $ make install
 
 #### Using Containers
-The guide can be found [here](docs/containers.md)
+
+    $ docker build -t Y2Z/monolith .
+    $ sudo install -b utils/run-in-container.sh /usr/local/bin/monolith
 
 ---------------------------------------------------
 
 ## Usage
     $ monolith https://lyrics.github.io/db/P/Portishead/Dummy/Roads/ -o portishead-roads-lyrics.html
+or
+
+    $ cat index.html | monolith -aIiFfcMv - > index-processed.html
 
 ---------------------------------------------------
 
 ## Options
- - `-c`: Ignore styles
- - `-f`: Exclude frames
- - `-F`: Omit web fonts
+ - `-a`: Exclude audio sources
+ - `-b`: Use custom `base URL`
+ - `-c`: Exclude CSS
+ - `-e`: Ignore network errors
+ - `-f`: Omit frames
+ - `-F`: Exclude web fonts
  - `-i`: Remove images
  - `-I`: Isolate the document
  - `-j`: Exclude JavaScript
  - `-k`: Accept invalid X.509 (TLS) certificates
- - `-o`: Write output to file
+ - `-M`: Don't add timestamp and URL information
+ - `-o`: Write output to `file`
  - `-s`: Be quiet
- - `-t`: Adjust network request timeout
- - `-u`: Provide custom User-Agent
+ - `-t`: Adjust `network request timeout`
+ - `-u`: Provide `custom User-Agent`
+ - `-v`: Exclude videos
 
 ---------------------------------------------------
 
-## HTTPS and HTTP proxies
+## Proxies
 Please set `https_proxy`, `http_proxy`, and `no_proxy` environment variables.
 
 ---------------------------------------------------
@@ -82,11 +95,18 @@ Please open an issue if something is wrong, that helps make this project better.
  - `Monolith Chrome Extension`: https://github.com/rhysd/monolith-of-web
  - `Pagesaver`: https://github.com/distributed-mind/pagesaver
  - `Personal WayBack Machine`: https://github.com/popey/pwbm
+ - `Hako`: https://github.com/dmpop/hako
 
 ---------------------------------------------------
 
 ## License
-The Unlicense
+
+<a href="http://creativecommons.org/publicdomain/zero/1.0/">
+    <img src="http://i.creativecommons.org/p/zero/1.0/88x31.png" alt="CC0-1.0" />
+</a>
+<br />
+To the extent possible under law, the author(s) have dedicated all copyright related and neighboring rights to this software to the public domain worldwide.
+This software is distributed without any warranty.
 
 ---------------------------------------------------
 

docs/arch/0003-network-request-timeout.md

@@ -1,4 +1,4 @@
-# 2. Network request timeout
+# 3. Network request timeout
 
 Date: 2020-02-15
 

docs/arch/0005-asset-minimization.md

@@ -1,4 +1,4 @@
-# 4. Asset Minimization
+# 5. Asset Minimization
 
 Date: 2020-03-14
 

docs/arch/0006-reload-and-location-meta-tags.md

@@ -1,4 +1,4 @@
-# 4. Reload and location `meta` tags
+# 6. Reload and location `meta` tags
 
 Date: 2020-06-25
 

docs/arch/0007-network-errors.md

@@ -0,0 +1,19 @@
+# 7. Network errors
+
+Date: 2020-11-22
+
+## Status
+
+Accepted
+
+## Context
+
+Servers may return information with HTTP response codes other than `200`, however those responses may still contain useful data.
+
+## Decision
+
+Fail by default, notifying of the network error. Add option to continue retrieving assets by treating all response codes as `200`.
+
+## Consequences
+
+Monolith will fail to obtain resources with status other than `200`, unless told to ignore network errors.

docs/arch/0008-base-tag.md

@@ -0,0 +1,40 @@
+# 8. Base Tag
+
+Date: 2020-12-25
+
+## Status
+
+Accepted
+
+## Context
+
+HTML documents may contain `base` tag, which influences resolution of anchor links and relative URLs as well as dynamically loaded resources.
+
+Sometimes, in order to make certain saved documents function closer to how they operate while being served from a remote server, the `base` tag specifying the source page's URL may need to be added to the document.
+
+There can be only one such tag. If multiple `base` tags are present, only the first encountered tag ends up being used.
+
+## Decision
+
+Adding the `base` tag should be optional — saved documents should not contain the `base` tag unless it was specified by the user, or the document originally had the `base` tag in it.
+
+Existing `href` attribute's value of the original `base` tag should be used for resolving the document's relative links instead of document's own URL (precisely the way browsers do it).
+
+## Consequences
+
+#### If the base tag does not exist in the source document
+
+- If the base tag does not exist in the source document
+  - With base URL option provided
+    - use the specified base URL value to retrieve assets, keep original base URL value in the document
+  - Without base URL option provided
+    - download document as usual, do not add base tag
+-  If the base tag already exists in the source document
+   - With base URL option provided
+     - we overwrite the original base URL before retrieving assets, keep new base URL value in the document
+   - Without base URL option provided:
+     - use the base URL from the original document to retrieve assets, keep original base URL value in the document
+
+The program will obtain ability to retrieve remote assets for non-remote sources (such as data URLs and local files).
+
+The program will obatin ability to get rid of existing base tag values (by provind an empty one).

monolith.nuspec

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?>
+<package xmlns="http://schemas.microsoft.com/packaging/2015/06/nuspec.xsd">
+  <metadata>
+    <id>monolith</id>
+    <version>2.4.0</version>
+    <title>Monolith</title>
+    <authors>Sunshine, Mahdi Robatipoor, Emmanuel Delaborde, Emi Simpson, rhysd</authors>
+    <projectUrl>https://github.com/Y2Z/monolith</projectUrl>
+    <iconUrl>https://raw.githubusercontent.com/Y2Z/monolith/master/assets/icon/icon.png</iconUrl>
+    <licenseUrl>https://raw.githubusercontent.com/Y2Z/monolith/master/LICENSE</licenseUrl>
+    <requireLicenseAcceptance>false</requireLicenseAcceptance>
+    <description>CLI tool for saving complete web pages as a single HTML file
+
+A data hoarder’s dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
+
+Unlike the conventional “Save page as”, monolith not only saves the target document, it embeds CSS, image, and JavaScript assets all at once, producing a single HTML5 document that is a joy to store and share.
+
+If compared to saving websites using wget, this tool embeds all assets as data URLs and therefore lets browsers render the saved page exactly the way it was on the Internet, even when no network connection is available.
+    </description>
+    <copyright>Public Domain</copyright>
+    <language>en-US</language>
+    <tags>scraping archiving</tags>
+    <docsUrl>https://github.com/Y2Z/monolith/blob/master/README.md</docsUrl>
+  </metadata>
+</package>

src/css.rs

@@ -26,7 +26,7 @@ const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
     "suffix",
     "symbols",
 ];
-const CSS_SPECIAL_CHARS: &str = "~!@$%^&*()+=,./'\";:?><[]{}|`#";
+const CSS_SPECIAL_CHARS: &'static str = "~!@$%^&*()+=,./'\";:?><[]{}|`#";
 
 pub fn is_image_url_prop(prop_name: &str) -> bool {
     CSS_PROPS_WITH_IMAGE_URLS
@@ -176,7 +176,7 @@ pub fn process_css<'a>(
                         client,
                         &parent_url,
                         &import_full_url,
-                        options.silent,
+                        options,
                         depth + 1,
                     ) {
                         Ok((import_contents, import_final_url, _import_media_type)) => {
@@ -227,7 +227,7 @@ pub fn process_css<'a>(
                                 client,
                                 &parent_url,
                                 &resolved_url,
-                                options.silent,
+                                options,
                                 depth + 1,
                             ) {
                                 Ok((data, final_url, media_type)) => {
@@ -315,14 +315,8 @@ pub fn process_css<'a>(
                 if is_import {
                     let full_url = resolve_url(&parent_url, value).unwrap_or_default();
                     let url_fragment = get_url_fragment(full_url.clone());
-                    match retrieve_asset(
-                        cache,
-                        client,
-                        &parent_url,
-                        &full_url,
-                        options.silent,
-                        depth + 1,
-                    ) {
+                    match retrieve_asset(cache, client, &parent_url, &full_url, options, depth + 1)
+                    {
                         Ok((css, final_url, _media_type)) => {
                             let data_url = data_to_data_url(
                                 "text/css",
@@ -361,7 +355,7 @@ pub fn process_css<'a>(
                             client,
                             &parent_url,
                             &full_url,
-                            options.silent,
+                            options,
                             depth + 1,
                         ) {
                             Ok((data, final_url, media_type)) => {

src/js.rs

@@ -1,4 +1,4 @@
-const JS_DOM_EVENT_ATTRS: &[&str] = &[
+const JS_DOM_EVENT_ATTRS: &'static [&str] = &[
     // From WHATWG HTML spec 8.1.5.2 "Event handlers on elements, Document objects, and Window objects":
     //   https://html.spec.whatwg.org/#event-handlers-on-elements,-document-objects,-and-window-objects
     //   https://html.spec.whatwg.org/#attributes-3 (table "List of event handler content attributes")

src/main.rs

@@ -3,17 +3,18 @@ use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
 use std::collections::HashMap;
 use std::env;
 use std::fs;
-use std::io::{self, Error, Write};
+use std::io::{self, prelude::*, Error, Write};
 use std::path::Path;
 use std::process;
 use std::time::Duration;
 
 use monolith::html::{
-    add_favicon, has_favicon, html_to_dom, metadata_tag, stringify_document, walk_and_embed_assets,
+    add_favicon, create_metadata_tag, get_base_url, has_favicon, html_to_dom, set_base_url,
+    stringify_document, walk_and_embed_assets,
 };
 use monolith::opts::Options;
 use monolith::url::{
-    data_to_data_url, data_url_to_data, is_data_url, is_file_url, is_http_url, resolve_url,
+    data_to_data_url, is_data_url, is_file_url, is_http_url, parse_data_url, resolve_url,
 };
 use monolith::utils::retrieve_asset;
 
@@ -47,12 +48,22 @@ impl Output {
     }
 }
 
+pub fn read_stdin() -> String {
+    let mut buffer = String::new();
+    for line in io::stdin().lock().lines() {
+        buffer += line.unwrap_or_default().as_str();
+        buffer += "\n";
+    }
+    buffer
+}
+
 fn main() {
     let options = Options::from_args();
     let original_target: &str = &options.target;
     let target_url: &str;
-    let base_url;
+    let mut base_url: String;
     let mut dom;
+    let mut use_stdin: bool = false;
 
     // Pre-process the input
     let cwd_normalized: String =
@@ -63,15 +74,24 @@ fn main() {
 
     // Determine exact target URL
     if target.clone().len() == 0 {
-        eprintln!("No target specified");
+        if !options.silent {
+            eprintln!("No target specified");
+        }
         process::exit(1);
+    } else if target.clone() == "-" {
+        // Read from pipe (stdin)
+        use_stdin = true;
+        // Default target URL to empty data URL; the user can control it via --base-url
+        target_url = "data:text/html,"
     } else if is_http_url(target.clone()) || is_data_url(target.clone()) {
         target_url = target.as_str();
     } else if is_file_url(target.clone()) {
         target_url = target.as_str();
     } else if path.exists() {
         if !path.is_file() {
-            eprintln!("Local target is not a file: {}", original_target);
+            if !options.silent {
+                eprintln!("Local target is not a file: {}", original_target);
+            }
             process::exit(1);
         }
         target.insert_str(0, if cfg!(windows) { "file:///" } else { "file://" });
@@ -94,10 +114,12 @@ fn main() {
     // Initialize client
     let mut cache = HashMap::new();
     let mut header_map = HeaderMap::new();
-    header_map.insert(
-        USER_AGENT,
-        HeaderValue::from_str(&options.user_agent).expect("Invalid User-Agent header specified"),
-    );
+    if let Some(user_agent) = &options.user_agent {
+        header_map.insert(
+            USER_AGENT,
+            HeaderValue::from_str(&user_agent).expect("Invalid User-Agent header specified"),
+        );
+    }
     let timeout: u64 = if options.timeout > 0 {
         options.timeout
     } else {
@@ -110,37 +132,64 @@ fn main() {
         .build()
         .expect("Failed to initialize HTTP client");
 
+    // At this stage we assume that the base URL is the same as the target URL
+    base_url = str!(target_url);
+
     // Retrieve target document
-    if is_file_url(target_url) || is_http_url(target_url) {
-        match retrieve_asset(
-            &mut cache,
-            &client,
-            target_url,
-            target_url,
-            options.silent,
-            0,
-        ) {
+    if use_stdin {
+        dom = html_to_dom(&read_stdin());
+    } else if is_file_url(target_url) || is_http_url(target_url) {
+        match retrieve_asset(&mut cache, &client, target_url, target_url, &options, 0) {
             Ok((data, final_url, _media_type)) => {
-                base_url = final_url;
+                if options.base_url.clone().unwrap_or(str!()).is_empty() {
+                    base_url = final_url
+                }
                 dom = html_to_dom(&String::from_utf8_lossy(&data));
             }
             Err(_) => {
-                eprintln!("Could not retrieve target document");
+                if !options.silent {
+                    eprintln!("Could not retrieve target document");
+                }
                 process::exit(1);
             }
         }
     } else if is_data_url(target_url) {
-        let (media_type, data): (String, Vec<u8>) = data_url_to_data(target_url);
+        let (media_type, data): (String, Vec<u8>) = parse_data_url(target_url);
         if !media_type.eq_ignore_ascii_case("text/html") {
-            eprintln!("Unsupported data URL media type");
+            if !options.silent {
+                eprintln!("Unsupported data URL media type");
+            }
             process::exit(1);
         }
-        base_url = str!(target_url);
         dom = html_to_dom(&String::from_utf8_lossy(&data));
     } else {
         process::exit(1);
     }
 
+    // Use custom base URL if specified, read and use what's in the DOM otherwise
+    if !options.base_url.clone().unwrap_or(str!()).is_empty() {
+        if is_data_url(options.base_url.clone().unwrap()) {
+            if !options.silent {
+                eprintln!("Data URLs cannot be used as base URL");
+            }
+            process::exit(1);
+        } else {
+            base_url = options.base_url.clone().unwrap();
+        }
+    } else {
+        if let Some(existing_base_url) = get_base_url(&dom.document) {
+            base_url = resolve_url(target_url, existing_base_url).unwrap();
+        }
+    }
+
+    // Embed remote assets
+    walk_and_embed_assets(&mut cache, &client, &base_url, &dom.document, &options, 0);
+
+    // Update or add new BASE tag to reroute network requests and hash-links in the final document
+    if let Some(new_base_url) = options.base_url.clone() {
+        dom = set_base_url(&dom.document, new_base_url);
+    }
+
     // Request and embed /favicon.ico (unless it's already linked in the document)
     if !options.no_images && is_http_url(target_url) && !has_favicon(&dom.document) {
         let favicon_ico_url: String = resolve_url(&base_url, "/favicon.ico").unwrap();
@@ -150,7 +199,7 @@ fn main() {
             &client,
             &base_url,
             &favicon_ico_url,
-            options.silent,
+            &options,
             0,
         ) {
             Ok((data, final_url, media_type)) => {
@@ -163,15 +212,12 @@ fn main() {
         }
     }
 
-    // Embed remote assets
-    walk_and_embed_assets(&mut cache, &client, &base_url, &dom.document, &options, 0);
-
     // Serialize DOM tree
     let mut result: String = stringify_document(&dom.document, &options);
 
     // Add metadata tag
     if !options.no_metadata {
-        let metadata_comment = metadata_tag(&base_url);
+        let metadata_comment: String = create_metadata_tag(&target_url);
         result.insert_str(0, &metadata_comment);
         if metadata_comment.len() > 0 {
             result.insert_str(metadata_comment.len(), "\n");

src/opts.rs

@@ -1,23 +1,29 @@
 use clap::{App, Arg};
+use std::env;
 
 #[derive(Default)]
 pub struct Options {
-    pub target: String,
+    pub no_audio: bool,
+    pub base_url: Option<String>,
     pub no_css: bool,
-    pub no_fonts: bool,
+    pub ignore_errors: bool,
     pub no_frames: bool,
+    pub no_fonts: bool,
     pub no_images: bool,
+    pub isolate: bool,
     pub no_js: bool,
     pub insecure: bool,
-    pub isolate: bool,
+    pub no_metadata: bool,
     pub output: String,
     pub silent: bool,
     pub timeout: u64,
-    pub user_agent: String,
-    pub no_metadata: bool,
+    pub user_agent: Option<String>,
+    pub no_video: bool,
+    pub target: String,
+    pub no_color: bool,
 }
 
-const ASCII: &str = " \
+const ASCII: &'static str = " \
  _____     ______________    __________      ___________________    ___
 |     \\   /              \\  |          |    |                   |  |   |
 |      \\_/       __       \\_|    __    |    |    ___     ___    |__|   |
@@ -27,36 +33,40 @@ const ASCII: &str = " \
 |___|       |__________|  \\_____________________|   |___|   |___|  |___|
 ";
 const DEFAULT_NETWORK_TIMEOUT: u64 = 120;
-const DEFAULT_USER_AGENT: &str =
+const DEFAULT_USER_AGENT: &'static str =
     "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0";
+const ENV_VAR_NO_COLOR: &str = "NO_COLOR";
+const ENV_VAR_TERM: &str = "TERM";
 
 impl Options {
     pub fn from_args() -> Options {
         let app = App::new(env!("CARGO_PKG_NAME"))
             .version(crate_version!())
-            .author(crate_authors!("\n"))
+            .author(format!("\n{}", crate_authors!("\n")).as_str())
             .about(format!("{}\n{}", ASCII, crate_description!()).as_str())
-            .arg(
-                Arg::with_name("target")
-                    .required(true)
-                    .takes_value(true)
-                    .index(1)
-                    .help("URL or file path"),
-            )
-            // .args_from_usage("-a, --no-audio 'Removes audio sources'")
+            .args_from_usage("-a, --no-audio 'Removes audio sources'")
+            .args_from_usage("-b, --base-url=[http://localhost/] 'Sets custom base URL'")
             .args_from_usage("-c, --no-css 'Removes CSS'")
+            .args_from_usage("-e, --ignore-errors 'Ignore network errors'")
             .args_from_usage("-f, --no-frames 'Removes frames and iframes'")
             .args_from_usage("-F, --no-fonts 'Removes fonts'")
             .args_from_usage("-i, --no-images 'Removes images'")
             .args_from_usage("-I, --isolate 'Cuts off document from the Internet'")
             .args_from_usage("-j, --no-js 'Removes JavaScript'")
             .args_from_usage("-k, --insecure 'Allows invalid X.509 (TLS) certificates'")
-            .args_from_usage("-M, --no-metadata 'Excludes metadata information from the document'")
-            .args_from_usage("-o, --output=[document.html] 'Write output to <file>'")
+            .args_from_usage("-M, --no-metadata 'Excludes timestamp and source information'")
+            .args_from_usage("-o, --output=[document.html] 'Writes output to <file>'")
             .args_from_usage("-s, --silent 'Suppresses verbosity'")
-            .args_from_usage("-t, --timeout=[60] 'Adjust network request timeout'")
-            .args_from_usage("-u, --user-agent=[Firefox] 'Set custom User-Agent string'")
-            // .args_from_usage("-v, --no-video 'Removes video sources'")
+            .args_from_usage("-t, --timeout=[60] 'Adjusts network request timeout'")
+            .args_from_usage("-u, --user-agent=[Firefox] 'Sets custom User-Agent string'")
+            .args_from_usage("-v, --no-video 'Removes video sources'")
+            .arg(
+                Arg::with_name("target")
+                    .required(true)
+                    .takes_value(true)
+                    .index(1)
+                    .help("URL or file path, use - for stdin"),
+            )
             .get_matches();
         let mut options: Options = Options::default();
 
@@ -65,7 +75,12 @@ impl Options {
             .value_of("target")
             .expect("please set target")
             .to_string();
+        options.no_audio = app.is_present("no-audio");
+        if let Some(base_url) = app.value_of("base-url") {
+            options.base_url = Some(str!(base_url));
+        }
         options.no_css = app.is_present("no-css");
+        options.ignore_errors = app.is_present("ignore-errors");
         options.no_frames = app.is_present("no-frames");
         options.no_fonts = app.is_present("no-fonts");
         options.no_images = app.is_present("no-images");
@@ -80,10 +95,20 @@ impl Options {
             .unwrap_or(&DEFAULT_NETWORK_TIMEOUT.to_string())
             .parse::<u64>()
             .unwrap();
-        options.user_agent = app
-            .value_of("user-agent")
-            .unwrap_or(DEFAULT_USER_AGENT)
-            .to_string();
+        if let Some(user_agent) = app.value_of("user-agent") {
+            options.user_agent = Some(str!(user_agent));
+        } else {
+            options.user_agent = Some(DEFAULT_USER_AGENT.to_string());
+        }
+        options.no_video = app.is_present("no-video");
+
+        options.no_color =
+            env::var_os(ENV_VAR_NO_COLOR).is_some() || atty::isnt(atty::Stream::Stderr);
+        if let Some(term) = env::var_os(ENV_VAR_TERM) {
+            if term == "dumb" {
+                options.no_color = true;
+            }
+        }
 
         options
     }

src/tests/cli/base_url.rs

@@ -0,0 +1,123 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn add_new_when_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("http://localhost:8000/")
+            .arg("data:text/html,Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost:8000/\"></base>\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn keep_existing_when_none_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost:8000/\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn override_existing_when_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("http://localhost/")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"http://localhost/\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_existing_when_empty_provided() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-b")
+            .arg("")
+            .arg("data:text/html,<base href=\"http://localhost:8000/\" />Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <base href=\"\">\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}

src/tests/cli/basic.rs

@@ -0,0 +1,145 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::io::Write;
+    use std::process::{Command, Stdio};
+    use tempfile::NamedTempFile;
+
+    #[test]
+    fn print_version() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd.arg("-V").output().unwrap();
+
+        // STDOUT should contain program name and version
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            format!("{} {}\n", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"))
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn stdin_target_input() -> Result<(), Box<dyn std::error::Error>> {
+        let mut echo = Command::new("echo")
+            .arg("Hello from STDIN")
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+        let echo_out = echo.stdout.take().unwrap();
+        echo.wait().unwrap();
+
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        cmd.stdin(echo_out);
+        let out = cmd.arg("-M").arg("-").output().unwrap();
+
+        // STDOUT should contain HTML from STDIN
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head></head><body>Hello from STDIN\n</body></html>\n"
+        );
+
+        Ok(())
+    }
+
+    #[test]
+    fn css_import_string() -> Result<(), Box<dyn std::error::Error>> {
+        let file_url_prefix: &str = if cfg!(windows) { "file:///" } else { "file://" };
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let mut file_css = NamedTempFile::new()?;
+        writeln!(file_css, "body{{background-color:#000;color:#fff}}")?;
+        let mut file_html = NamedTempFile::new()?;
+        writeln!(
+            file_html,
+            "\
+            <style>\n\
+            @charset 'UTF-8';\n\
+            \n\
+            @import '{file}{css_path}';\n\
+            \n\
+            @import url({file}{css_path});\n\
+            \n\
+            @import url('{file}{css_path}')\n\
+            </style>\n\
+            ",
+            file = file_url_prefix,
+            css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
+        )?;
+        let out = cmd.arg("-M").arg(file_html.path()).output().unwrap();
+
+        // STDOUT should contain embedded CSS url()'s
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head><style>\n@charset 'UTF-8';\n\n@import 'data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K';\n\n@import url('data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K');\n\n@import url('data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K')\n</style>\n\n</head><body></body></html>\n"
+        );
+
+        // STDERR should list temporary files that got retrieved
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            format!(
+                "\
+                {file}{html_path}\n \
+                {file}{css_path}\n \
+                {file}{css_path}\n \
+                {file}{css_path}\n\
+                ",
+                file = file_url_prefix,
+                html_path = str!(file_html.path().to_str().unwrap()).replace("\\", "/"),
+                css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
+            )
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn bad_input_empty_target() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd.arg("").output().unwrap();
+
+        // STDOUT should be empty
+        assert_eq!(std::str::from_utf8(&out.stdout).unwrap(), "");
+
+        // STDERR should contain error description
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            "No target specified\n"
+        );
+
+        // The exit code should be 1
+        out.assert().code(1);
+
+        Ok(())
+    }
+}

src/tests/cli/data_url.rs

@@ -0,0 +1,234 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd.arg("data:,Hello%2C%20World!").output().unwrap();
+
+        // STDOUT should contain HTML
+        assert_eq!(std::str::from_utf8(&out.stdout).unwrap(), "");
+
+        // STDERR should contain error description
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            "Unsupported data URL media type\n"
+        );
+
+        // The exit code should be 1
+        out.assert().code(1);
+
+        Ok(())
+    }
+
+    #[test]
+    fn isolate_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-I")
+            .arg("data:text/html,Hello%2C%20World!")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain isolated HTML
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:;\"></meta>\
+            </head><body>Hello, World!</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_css_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-c")
+            .arg("data:text/html,<style>body{background-color:pink}</style>Hello")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no CSS
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none';\"></meta>\
+            <style></style>\
+            </head><body>Hello</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_fonts_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-F")
+            .arg("data:text/html,<style>@font-face { font-family: myFont; src: url(font.woff); }</style>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no web fonts
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"font-src 'none';\"></meta>\
+            <style></style>\
+            </head><body>Hi</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_frames_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-f")
+            .arg("data:text/html,<iframe src=\"https://duckduckgo.com\"></iframe>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no iframes
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none'; child-src 'none';\"></meta>\
+            </head><body><iframe src=\"\"></iframe>Hi</body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_images_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-i")
+            .arg("data:text/html,<img src=\"https://google.com\"/>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no images
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            format!(
+                "<html>\
+                <head>\
+                <meta http-equiv=\"Content-Security-Policy\" content=\"img-src data:;\"></meta>\
+                </head>\
+                <body>\
+                <img src=\"{empty_image}\">\
+                Hi\
+                </body>\
+                </html>\n",
+                empty_image = empty_image!()
+            )
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn remove_js_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("-j")
+            .arg("data:text/html,<script>alert(2)</script>Hi")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no JS
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html>\
+            <head>\
+            <meta http-equiv=\"Content-Security-Policy\" content=\"script-src 'none';\"></meta>\
+            <script></script></head>\
+            <body>Hi</body>\
+            </html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+
+    #[test]
+    fn security_disallow_local_assets_within_data_url_targets(
+    ) -> Result<(), Box<dyn std::error::Error>> {
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg("data:text/html,%3Cscript%20src=\"src/tests/data/basic/local-script.js\"%3E%3C/script%3E")
+            .output()
+            .unwrap();
+
+        // STDOUT should contain HTML with no JS in it
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head><script></script></head><body></body></html>\n"
+        );
+
+        // STDERR should be empty
+        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}

src/tests/cli/local_files.rs

@@ -13,242 +13,6 @@ mod passing {
     use std::process::Command;
     use tempfile::NamedTempFile;
 
-    #[test]
-    fn print_version() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd.arg("-V").output().unwrap();
-
-        // STDOUT should contain program name and version
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            format!("{} {}\n", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"))
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn bad_input_empty_target() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd.arg("").output().unwrap();
-
-        // STDOUT should be empty
-        assert_eq!(std::str::from_utf8(&out.stdout).unwrap(), "");
-
-        // STDERR should contain error description
-        assert_eq!(
-            std::str::from_utf8(&out.stderr).unwrap(),
-            "No target specified\n"
-        );
-
-        // The exit code should be 1
-        out.assert().code(1);
-
-        Ok(())
-    }
-
-    #[test]
-    fn bad_input_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd.arg("data:,Hello%2C%20World!").output().unwrap();
-
-        // STDOUT should contain HTML
-        assert_eq!(std::str::from_utf8(&out.stdout).unwrap(), "");
-
-        // STDERR should contain error description
-        assert_eq!(
-            std::str::from_utf8(&out.stderr).unwrap(),
-            "Unsupported data URL media type\n"
-        );
-
-        // The exit code should be 1
-        out.assert().code(1);
-
-        Ok(())
-    }
-
-    #[test]
-    fn isolate_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("-I")
-            .arg("data:text/html,Hello%2C%20World!")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain isolated HTML
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html><head>\
-            <meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:;\"></meta>\
-            </head><body>Hello, World!</body></html>\n"
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn remove_css_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("-c")
-            .arg("data:text/html,<style>body{background-color:pink}</style>Hello")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain HTML with no CSS
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html><head>\
-            <meta http-equiv=\"Content-Security-Policy\" content=\"style-src 'none';\"></meta>\
-            <style></style>\
-            </head><body>Hello</body></html>\n"
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn remove_fonts_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("-F")
-            .arg("data:text/html,<style>@font-face { font-family: myFont; src: url(font.woff); }</style>Hi")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain HTML with no web fonts
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html><head>\
-            <meta http-equiv=\"Content-Security-Policy\" content=\"font-src 'none';\"></meta>\
-            <style></style>\
-            </head><body>Hi</body></html>\n"
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn remove_frames_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("-f")
-            .arg("data:text/html,<iframe src=\"https://duckduckgo.com\"></iframe>Hi")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain HTML with no iframes
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html><head>\
-            <meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none'; child-src 'none';\"></meta>\
-            </head><body><iframe src=\"\"></iframe>Hi</body></html>\n"
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn remove_images_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("-i")
-            .arg("data:text/html,<img src=\"https://google.com\"/>Hi")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain HTML with no images
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            format!(
-                "<html>\
-                <head>\
-                <meta http-equiv=\"Content-Security-Policy\" content=\"img-src data:;\"></meta>\
-                </head>\
-                <body>\
-                <img src=\"{empty_image}\">\
-                Hi\
-                </body>\
-                </html>\n",
-                empty_image = empty_image!()
-            )
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
-    #[test]
-    fn remove_js_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("-j")
-            .arg("data:text/html,<script>alert(2)</script>Hi")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain HTML with no JS
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html>\
-            <head>\
-            <meta http-equiv=\"Content-Security-Policy\" content=\"script-src 'none';\"></meta>\
-            <script></script></head>\
-            <body>Hi</body>\
-            </html>\n"
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
     #[test]
     fn local_file_target_input() -> Result<(), Box<dyn std::error::Error>> {
         let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
@@ -305,8 +69,7 @@ mod passing {
     #[test]
     fn local_file_target_input_absolute_target_path() -> Result<(), Box<dyn std::error::Error>> {
         let cwd = env::current_dir().unwrap();
-        let cwd_normalized: String =
-            str!(env::current_dir().unwrap().to_str().unwrap()).replace("\\", "/");
+        let cwd_normalized: String = str!(cwd.to_str().unwrap()).replace("\\", "/");
         let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
         let out = cmd
             .arg("-M")
@@ -337,7 +100,7 @@ mod passing {
                 <title>Local HTML file</title>\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
-                <img alt=\"\" src=\"{empty_image}\">\n  \
+                <img src=\"{empty_image}\" alt=\"\">\n  \
                 <a href=\"file://local-file.html/\">Tricky href</a>\n  \
                 <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
                 <script></script>\n\n\n\n\
@@ -399,7 +162,7 @@ mod passing {
                 <title>Local HTML file</title>\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n  \
                 <link rel=\"stylesheet\" type=\"text/css\">\n</head>\n\n<body>\n  \
-                <img alt=\"\" src=\"{empty_image}\">\n  \
+                <img src=\"{empty_image}\" alt=\"\">\n  \
                 <a href=\"file://local-file.html/\">Tricky href</a>\n  \
                 <a href=\"https://github.com/Y2Z/monolith\">Remote URL</a>\n  \
                 <script></script>\n\n\n\n\
@@ -433,31 +196,6 @@ mod passing {
         Ok(())
     }
 
-    #[test]
-    fn security_disallow_local_assets_within_data_url_targets(
-    ) -> Result<(), Box<dyn std::error::Error>> {
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let out = cmd
-            .arg("-M")
-            .arg("data:text/html,%3Cscript%20src=\"src/tests/data/basic/local-script.js\"%3E%3C/script%3E")
-            .output()
-            .unwrap();
-
-        // STDOUT should contain HTML with no JS in it
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html><head><script></script></head><body></body></html>\n"
-        );
-
-        // STDERR should be empty
-        assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
-
     #[test]
     fn embed_file_url_local_asset_within_style_attribute() -> Result<(), Box<dyn std::error::Error>>
     {
@@ -503,57 +241,4 @@ mod passing {
 
         Ok(())
     }
-
-    #[test]
-    fn css_import_string() -> Result<(), Box<dyn std::error::Error>> {
-        let file_url_prefix: &str = if cfg!(windows) { "file:///" } else { "file://" };
-        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
-        let mut file_css = NamedTempFile::new()?;
-        writeln!(file_css, "body{{background-color:#000;color:#fff}}")?;
-        let mut file_html = NamedTempFile::new()?;
-        writeln!(
-            file_html,
-            "\
-            <style>\n\
-            @charset 'UTF-8';\n\
-            \n\
-            @import '{file}{css_path}';\n\
-            \n\
-            @import url({file}{css_path});\n\
-            \n\
-            @import url('{file}{css_path}')\n\
-            </style>\n\
-            ",
-            file = file_url_prefix,
-            css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
-        )?;
-        let out = cmd.arg("-M").arg(file_html.path()).output().unwrap();
-
-        // STDOUT should contain embedded CSS url()'s
-        assert_eq!(
-            std::str::from_utf8(&out.stdout).unwrap(),
-            "<html><head><style>\n@charset 'UTF-8';\n\n@import 'data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K';\n\n@import url('data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K');\n\n@import url('data:text/css;base64,Ym9keXtiYWNrZ3JvdW5kLWNvbG9yOiMwMDA7Y29sb3I6I2ZmZn0K')\n</style>\n\n</head><body></body></html>\n"
-        );
-
-        // STDERR should list temporary files that got retrieved
-        assert_eq!(
-            std::str::from_utf8(&out.stderr).unwrap(),
-            format!(
-                "\
-                {file}{html_path}\n \
-                {file}{css_path}\n \
-                {file}{css_path}\n \
-                {file}{css_path}\n\
-                ",
-                file = file_url_prefix,
-                html_path = str!(file_html.path().to_str().unwrap()).replace("\\", "/"),
-                css_path = str!(file_css.path().to_str().unwrap()).replace("\\", "/"),
-            )
-        );
-
-        // The exit code should be 0
-        out.assert().code(0);
-
-        Ok(())
-    }
 }

src/tests/cli/mod.rs

@@ -0,0 +1,5 @@
+mod base_url;
+mod basic;
+mod data_url;
+mod local_files;
+mod unusual_encodings;

src/tests/cli/unusual_encodings.rs

@@ -0,0 +1,51 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use assert_cmd::prelude::*;
+    use std::env;
+    use std::process::Command;
+
+    #[test]
+    fn change_encoding_to_utf_8() -> Result<(), Box<dyn std::error::Error>> {
+        let cwd = env::current_dir().unwrap();
+        let cwd_normalized: String = str!(cwd.to_str().unwrap()).replace("\\", "/");
+        let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
+        let out = cmd
+            .arg("-M")
+            .arg(if cfg!(windows) {
+                "src\\tests\\data\\unusual_encodings\\iso-8859-1.html"
+            } else {
+                "src/tests/data/unusual_encodings/iso-8859-1.html"
+            })
+            .output()
+            .unwrap();
+        let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
+
+        // STDOUT should contain newly added base URL
+        assert_eq!(
+            std::str::from_utf8(&out.stdout).unwrap(),
+            "<html><head>\n        <meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">\n    </head>\n    <body>\n        © Some Company\n    \n\n</body></html>\n"
+        );
+
+        // STDERR should contain only the target file
+        assert_eq!(
+            std::str::from_utf8(&out.stderr).unwrap(),
+            format!(
+                "{file}{cwd}/src/tests/data/unusual_encodings/iso-8859-1.html\n",
+                file = file_url_protocol,
+                cwd = cwd_normalized,
+            )
+        );
+
+        // The exit code should be 0
+        out.assert().code(0);
+
+        Ok(())
+    }
+}

src/tests/data/unusual_encodings/iso-8859-1.html

@@ -0,0 +1,8 @@
+<html>
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+    </head>
+    <body>
+        &copy; Some Company
+    </body>
+</html>

src/tests/html/check_integrity.rs

@@ -11,7 +11,7 @@ mod passing {
 
     #[test]
     fn empty_input_sha256() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "".as_bytes(),
             "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
         ));
@@ -19,7 +19,7 @@ mod passing {
 
     #[test]
     fn sha256() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha256-9EWAHgy4mSYsm54hmDaIDXPKLRsLnBX7lZyQ6xISNOM="
         ));
@@ -27,7 +27,7 @@ mod passing {
 
     #[test]
     fn sha384() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha384-gc9l7omltke8C33bedgh15E12M7RrAQa5t63Yb8APlpe7ZhiqV23+oqiulSJl3Kw"
         ));
@@ -35,7 +35,7 @@ mod passing {
 
     #[test]
     fn sha512() {
-        assert!(html::has_proper_integrity(
+        assert!(html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha512-zG5B88cYMqcdiMi9gz0XkOFYw2BpjeYdn5V6+oFrMgSNjRpqL7EF8JEwl17ztZbK3N7I/tTwp3kxQbN1RgFBww=="
         ));
@@ -55,20 +55,17 @@ mod failing {
 
     #[test]
     fn empty_hash() {
-        assert!(!html::has_proper_integrity(
-            "abcdef0123456789".as_bytes(),
-            ""
-        ));
+        assert!(!html::check_integrity("abcdef0123456789".as_bytes(), ""));
     }
 
     #[test]
     fn empty_input_empty_hash() {
-        assert!(!html::has_proper_integrity("".as_bytes(), ""));
+        assert!(!html::check_integrity("".as_bytes(), ""));
     }
 
     #[test]
     fn sha256() {
-        assert!(!html::has_proper_integrity(
+        assert!(!html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha256-badhash"
         ));
@@ -76,7 +73,7 @@ mod failing {
 
     #[test]
     fn sha384() {
-        assert!(!html::has_proper_integrity(
+        assert!(!html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha384-badhash"
         ));
@@ -84,7 +81,7 @@ mod failing {
 
     #[test]
     fn sha512() {
-        assert!(!html::has_proper_integrity(
+        assert!(!html::check_integrity(
             "abcdef0123456789".as_bytes(),
             "sha512-badhash"
         ));

src/tests/html/compose_csp.rs

@@ -14,7 +14,7 @@ mod passing {
     fn isolated() {
         let mut options = Options::default();
         options.isolate = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "default-src 'unsafe-inline' data:;");
     }
@@ -23,7 +23,7 @@ mod passing {
     fn no_css() {
         let mut options = Options::default();
         options.no_css = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "style-src 'none';");
     }
@@ -32,7 +32,7 @@ mod passing {
     fn no_fonts() {
         let mut options = Options::default();
         options.no_fonts = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "font-src 'none';");
     }
@@ -41,7 +41,7 @@ mod passing {
     fn no_frames() {
         let mut options = Options::default();
         options.no_frames = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "frame-src 'none'; child-src 'none';");
     }
@@ -50,7 +50,7 @@ mod passing {
     fn no_js() {
         let mut options = Options::default();
         options.no_js = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "script-src 'none';");
     }
@@ -59,7 +59,7 @@ mod passing {
     fn no_images() {
         let mut options = Options::default();
         options.no_images = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "img-src data:;");
     }
@@ -73,7 +73,7 @@ mod passing {
         options.no_frames = true;
         options.no_js = true;
         options.no_images = true;
-        let csp_content = html::csp(&options);
+        let csp_content = html::compose_csp(&options);
 
         assert_eq!(csp_content, "default-src 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;");
     }

src/tests/html/create_metadata_tag.rs

@@ -15,7 +15,7 @@ mod passing {
     fn http_url() {
         let url = "http://192.168.1.1/";
         let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
-        let metadata_comment: String = html::metadata_tag(url);
+        let metadata_comment: String = html::create_metadata_tag(url);
 
         assert_eq!(
             metadata_comment,
@@ -33,7 +33,7 @@ mod passing {
     fn file_url() {
         let url = "file:///home/monolith/index.html";
         let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
-        let metadata_comment: String = html::metadata_tag(url);
+        let metadata_comment: String = html::create_metadata_tag(url);
 
         assert_eq!(
             metadata_comment,
@@ -50,7 +50,7 @@ mod passing {
     fn data_url() {
         let url = "data:text/html,Hello%2C%20World!";
         let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
-        let metadata_comment: String = html::metadata_tag(url);
+        let metadata_comment: String = html::create_metadata_tag(url);
 
         assert_eq!(
             metadata_comment,
@@ -77,6 +77,6 @@ mod failing {
 
     #[test]
     fn empty_string() {
-        assert_eq!(html::metadata_tag(""), "");
+        assert_eq!(html::create_metadata_tag(""), "");
     }
 }

src/tests/html/embed_srcset.rs

@@ -14,18 +14,108 @@ mod passing {
     use crate::opts::Options;
 
     #[test]
-    fn replace_with_empty_images() {
+    fn small_medium_large() {
         let cache = &mut HashMap::new();
         let client = Client::new();
-        let srcset_value = "small.png 1x, large.png 2x";
+        let srcset_value = "small.png 1x, medium.png 1.5x, large.png 2x";
         let mut options = Options::default();
         options.no_images = true;
         options.silent = true;
         let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
 
         assert_eq!(
+            embedded_css,
+            format!(
+                "{} 1x, {} 1.5x, {} 2x",
+                empty_image!(),
+                empty_image!(),
+                empty_image!(),
+            ),
+        );
+    }
+
+    #[test]
+    fn small_medium_only_medium_has_scale() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small.png, medium.png 1.5x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{}, {} 1.5x", empty_image!(), empty_image!()),
+        );
+    }
+
+    #[test]
+    fn commas_within_file_names() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small,s.png 1x, large,l.png 2x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
             format!("{} 1x, {} 2x", empty_image!(), empty_image!()),
-            embedded_css
+        );
+    }
+
+    #[test]
+    fn tabs_and_newlines_after_commas() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small,s.png 1x,\nmedium,m.png 2x,\nlarge,l.png 3x";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!(
+                "{} 1x, {} 2x, {} 3x",
+                empty_image!(),
+                empty_image!(),
+                empty_image!()
+            ),
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use reqwest::blocking::Client;
+    use std::collections::HashMap;
+
+    use crate::html;
+    use crate::opts::Options;
+
+    #[test]
+    fn trailing_comma() {
+        let cache = &mut HashMap::new();
+        let client = Client::new();
+        let srcset_value = "small.png 1x, large.png 2x,";
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+        let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
+
+        assert_eq!(
+            embedded_css,
+            format!("{} 1x, {} 2x,", empty_image!(), empty_image!()),
         );
     }
 }

src/tests/html/get_base_url.rs

@@ -0,0 +1,104 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use crate::html;
+
+    #[test]
+    fn present() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"https://musicbrainz.org\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(
+            html::get_base_url(&dom.document),
+            Some(str!("https://musicbrainz.org"))
+        );
+    }
+
+    #[test]
+    fn multiple_tags() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"https://www.discogs.com/\" />
+        <base href=\"https://musicbrainz.org\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(
+            html::get_base_url(&dom.document),
+            Some(str!("https://www.discogs.com/"))
+        );
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::html;
+
+    #[test]
+    fn absent() {
+        let html = "<!doctype html>
+<html>
+    <head>
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), None);
+    }
+
+    #[test]
+    fn no_href() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), None);
+    }
+
+    #[test]
+    fn empty_href() {
+        let html = "<!doctype html>
+<html>
+    <head>
+        <base href=\"\" />
+    </head>
+    <body>
+    </body>
+</html>";
+        let dom = html::html_to_dom(&html);
+
+        assert_eq!(html::get_base_url(&dom.document), Some(str!()));
+    }
+}

src/tests/html/get_node_attr.rs

@@ -0,0 +1,54 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn div_two_style_attributes() {
+        let html = "<!doctype html><html><head></head><body><DIV STYLE=\"color: blue;\" style=\"display: none;\"></div></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "class"), None);
+                    } else if node_name == "div" {
+                        assert_eq!(
+                            html::get_node_attr(node, "style"),
+                            Some(str!("color: blue;"))
+                        );
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 6);
+    }
+}

src/tests/html/get_node_name.rs

@@ -12,7 +12,7 @@ mod passing {
     use crate::html;
 
     #[test]
-    fn get_node_name() {
+    fn parent_node_names() {
         let html = "<!doctype html><html><HEAD></HEAD><body><div><P></P></div></body></html>";
         let dom = html::html_to_dom(&html);
         let mut count = 0;

src/tests/html/has_favicon.rs

@@ -8,7 +8,6 @@
 #[cfg(test)]
 mod passing {
     use crate::html;
-    use crate::opts::Options;
 
     #[test]
     fn icon() {
@@ -39,7 +38,6 @@ mod passing {
 #[cfg(test)]
 mod failing {
     use crate::html;
-    use crate::opts::Options;
 
     #[test]
     fn absent() {

src/tests/html/mod.rs

@@ -1,10 +1,13 @@
 mod add_favicon;
-mod csp;
+mod check_integrity;
+mod compose_csp;
+mod create_metadata_tag;
 mod embed_srcset;
+mod get_base_url;
+mod get_node_attr;
 mod get_node_name;
 mod has_favicon;
-mod has_proper_integrity;
 mod is_icon;
-mod metadata_tag;
+mod set_node_attr;
 mod stringify_document;
 mod walk_and_embed_assets;

src/tests/html/set_node_attr.rs

@@ -0,0 +1,105 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use html5ever::rcdom::{Handle, NodeData};
+
+    use crate::html;
+
+    #[test]
+    fn html_lang_and_body_style() {
+        let html = "<!doctype html><html lang=\"en\"><head></head><body></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "html" {
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("en")));
+
+                        html::set_node_attr(node, "lang", Some(str!("de")));
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("de")));
+
+                        html::set_node_attr(node, "lang", None);
+                        assert_eq!(html::get_node_attr(node, "lang"), None);
+
+                        html::set_node_attr(node, "lang", Some(str!("")));
+                        assert_eq!(html::get_node_attr(node, "lang"), Some(str!("")));
+                    } else if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "style"), None);
+
+                        html::set_node_attr(node, "style", Some(str!("display: none;")));
+                        assert_eq!(
+                            html::get_node_attr(node, "style"),
+                            Some(str!("display: none;"))
+                        );
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 5);
+    }
+
+    #[test]
+    fn body_background() {
+        let html = "<!doctype html><html lang=\"en\"><head></head><body background=\"1\" background=\"2\"></body></html>";
+        let dom = html::html_to_dom(&html);
+        let mut count = 0;
+
+        fn test_walk(node: &Handle, i: &mut i8) {
+            *i += 1;
+
+            match &node.data {
+                NodeData::Document => {
+                    // Dig deeper
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                NodeData::Element { ref name, .. } => {
+                    let node_name = name.local.as_ref().to_string();
+
+                    if node_name == "body" {
+                        assert_eq!(html::get_node_attr(node, "background"), Some(str!("1")));
+
+                        html::set_node_attr(node, "background", None);
+                        assert_eq!(html::get_node_attr(node, "background"), None);
+                    }
+
+                    for child in node.children.borrow().iter() {
+                        test_walk(child, &mut *i);
+                    }
+                }
+                _ => (),
+            };
+        }
+
+        test_walk(&dom.document, &mut count);
+
+        assert_eq!(count, 5);
+    }
+}

src/tests/html/walk_and_embed_assets.rs

@@ -87,6 +87,7 @@ mod passing {
     #[test]
     fn no_css() {
         let html = "<link rel=\"stylesheet\" href=\"main.css\">\
+                    <link rel=\"alternate stylesheet\" href=\"main.css\">\
                     <style>html{background-color: #000;}</style>\
                     <div style=\"display: none;\"></div>";
         let dom = html::html_to_dom(&html);
@@ -109,6 +110,7 @@ mod passing {
             "<html>\
             <head>\
             <link rel=\"stylesheet\">\
+            <link rel=\"alternate stylesheet\">\
             <style></style>\
             </head>\
             <body>\
@@ -319,11 +321,52 @@ mod passing {
             buf.iter().map(|&c| c as char).collect::<String>(),
             "<html>\
                 <head>\
-                    <meta>\
-                    <meta>\
+                    <meta http-equiv=\"disabled by monolith (Refresh)\" value=\"20\">\
+                    <meta http-equiv=\"disabled by monolith (Location)\" value=\"https://freebsd.org\">\
                 </head>\
                 <body></body>\
             </html>"
         );
     }
+
+    #[test]
+    fn processes_noscript_tags() {
+        let html = "<html>\
+            <body>\
+                <noscript>\
+                    <img src=\"image.png\" />\
+                </noscript>\
+            </body>\
+        </html>";
+        let dom = html::html_to_dom(&html);
+        let url = "http://localhost";
+        let cache = &mut HashMap::new();
+
+        let mut options = Options::default();
+        options.no_images = true;
+        options.silent = true;
+
+        let client = Client::new();
+
+        html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
+
+        let mut buf: Vec<u8> = Vec::new();
+        serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
+
+        assert_eq!(
+            buf.iter().map(|&c| c as char).collect::<String>(),
+            format!(
+                "<html>\
+                    <head>\
+                    </head>\
+                    <body>\
+                        <noscript>\
+                            <img src=\"{}\">\
+                        </noscript>\
+                    </body>\
+                </html>",
+                empty_image!(),
+            )
+        );
+    }
 }

src/tests/mod.rs

@@ -4,4 +4,5 @@ mod html;
 mod js;
 mod macros;
 mod opts;
+mod url;
 mod utils;

src/tests/opts.rs

@@ -13,7 +13,8 @@ mod passing {
     fn defaults() {
         let options: Options = Options::default();
 
-        assert_eq!(options.target, str!());
+        assert_eq!(options.no_audio, false);
+        assert_eq!(options.base_url, None);
         assert_eq!(options.no_css, false);
         assert_eq!(options.no_frames, false);
         assert_eq!(options.no_fonts, false);
@@ -25,6 +26,9 @@ mod passing {
         assert_eq!(options.output, str!());
         assert_eq!(options.silent, false);
         assert_eq!(options.timeout, 0);
-        assert_eq!(options.user_agent, "");
+        assert_eq!(options.user_agent, None);
+        assert_eq!(options.no_video, false);
+
+        assert_eq!(options.target, str!());
     }
 }

src/tests/url/get_url_fragment.rs

@@ -18,9 +18,31 @@ mod passing {
             "test"
         );
     }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use crate::url;
 
     #[test]
     fn https_empty() {
         assert_eq!(url::get_url_fragment("https://kernel.org#"), "");
     }
+
+    #[test]
+    fn no_fragment() {
+        assert_eq!(url::get_url_fragment("https://kernel.org"), "");
+    }
+
+    #[test]
+    fn dummy_data_url() {
+        assert_eq!(url::get_url_fragment("data:text/html,"), "");
+    }
 }

src/tests/url/mod.rs

@@ -1,12 +1,12 @@
 mod clean_url;
 mod data_to_data_url;
-mod data_url_to_data;
 mod decode_url;
 mod file_url_to_fs_path;
 mod get_url_fragment;
 mod is_data_url;
 mod is_file_url;
 mod is_http_url;
+mod parse_data_url;
 mod resolve_url;
 mod url_has_protocol;
 mod url_with_fragment;

src/tests/url/parse_data_url.rs

@@ -11,7 +11,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_base64() {
-        let (media_type, data) = url::data_url_to_data("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
+        let (media_type, data) = url::parse_data_url("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
 
         assert_eq!(media_type, "text/html");
         assert_eq!(
@@ -22,7 +22,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_utf8() {
-        let (media_type, data) = url::data_url_to_data(
+        let (media_type, data) = url::parse_data_url(
             "data:text/html;utf8,Work expands so as to fill the time available for its completion",
         );
 
@@ -35,7 +35,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_plaintext() {
-        let (media_type, data) = url::data_url_to_data(
+        let (media_type, data) = url::parse_data_url(
             "data:text/html,Work expands so as to fill the time available for its completion",
         );
 
@@ -48,7 +48,7 @@ mod passing {
 
     #[test]
     fn parse_text_html_charset_utf_8_between_two_whitespaces() {
-        let (media_type, data) = url::data_url_to_data(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
+        let (media_type, data) = url::parse_data_url(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
 
         assert_eq!(media_type, "text/html");
         assert_eq!(
@@ -59,8 +59,7 @@ mod passing {
 
     #[test]
     fn parse_text_css_url_encoded() {
-        let (media_type, data) =
-            url::data_url_to_data("data:text/css,div{background-color:%23000}");
+        let (media_type, data) = url::parse_data_url("data:text/css,div{background-color:%23000}");
 
         assert_eq!(media_type, "text/css");
         assert_eq!(String::from_utf8_lossy(&data), "div{background-color:#000}");
@@ -68,7 +67,7 @@ mod passing {
 
     #[test]
     fn parse_no_media_type_base64() {
-        let (media_type, data) = url::data_url_to_data("data:;base64,dGVzdA==");
+        let (media_type, data) = url::parse_data_url("data:;base64,dGVzdA==");
 
         assert_eq!(media_type, "");
         assert_eq!(String::from_utf8_lossy(&data), "test");
@@ -76,7 +75,7 @@ mod passing {
 
     #[test]
     fn parse_no_media_type_no_encoding() {
-        let (media_type, data) = url::data_url_to_data("data:;,test%20test");
+        let (media_type, data) = url::parse_data_url("data:;,test%20test");
 
         assert_eq!(media_type, "");
         assert_eq!(String::from_utf8_lossy(&data), "test test");
@@ -96,7 +95,7 @@ mod failing {
 
     #[test]
     fn just_word_data() {
-        let (media_type, data) = url::data_url_to_data("data");
+        let (media_type, data) = url::parse_data_url("data");
 
         assert_eq!(media_type, "");
         assert_eq!(String::from_utf8_lossy(&data), "");

src/tests/url/resolve_url.rs

@@ -7,208 +7,165 @@
 
 #[cfg(test)]
 mod passing {
-    use url::ParseError;
-
     use crate::url;
 
     #[test]
-    fn from_https_to_level_up_relative() -> Result<(), ParseError> {
-        let resolved_url =
-            url::resolve_url("https://www.kernel.org", "../category/signatures.html")?;
-
+    fn from_https_to_level_up_relative() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url("https://www.kernel.org", "../category/signatures.html")
+                .unwrap_or_default(),
             "https://www.kernel.org/category/signatures.html"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_just_filename_to_full_https_url() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "saved_page.htm",
-            "https://www.kernel.org/category/signatures.html",
-        )?;
-
+    fn from_just_filename_to_full_https_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "saved_page.htm",
+                "https://www.kernel.org/category/signatures.html",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/category/signatures.html"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_url_to_url_with_no_protocol() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "https://www.kernel.org",
-            "//www.kernel.org/theme/images/logos/tux.png",
-        )?;
-
+    fn from_https_url_to_url_with_no_protocol() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.kernel.org",
+                "//www.kernel.org/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/theme/images/logos/tux.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() -> Result<(), ParseError>
-    {
-        let resolved_url = url::resolve_url(
-            "https://www.kernel.org",
-            "//another-host.org/theme/images/logos/tux.png",
-        )?;
-
+    fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.kernel.org",
+                "//another-host.org/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
             "https://another-host.org/theme/images/logos/tux.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_url_to_relative_root_path() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "https://www.kernel.org/category/signatures.html",
-            "/theme/images/logos/tux.png",
-        )?;
-
+    fn from_https_url_to_relative_root_path() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.kernel.org/category/signatures.html",
+                "/theme/images/logos/tux.png",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/theme/images/logos/tux.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_https_to_just_filename() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "https://www.w3schools.com/html/html_iframe.asp",
-            "default.asp",
-        )?;
-
+    fn from_https_to_just_filename() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "https://www.w3schools.com/html/html_iframe.asp",
+                "default.asp",
+            )
+            .unwrap_or_default(),
             "https://www.w3schools.com/html/default.asp"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_data_url_to_https() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "https://www.kernel.org/category/signatures.html",
-        )?;
-
+    fn from_data_url_to_https() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "https://www.kernel.org/category/signatures.html",
+            )
+            .unwrap_or_default(),
             "https://www.kernel.org/category/signatures.html"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_data_url_to_data_url() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K",
-        )?;
-
+    fn from_data_url_to_data_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K",
+            )
+            .unwrap_or_default(),
             "data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_file_url_to_relative_path() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "file:///home/user/Websites/my-website/index.html",
-            "assets/images/logo.png",
-        )
-        .unwrap_or(str!());
-
+    fn from_file_url_to_relative_path() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "file:///home/user/Websites/my-website/index.html",
+                "assets/images/logo.png",
+            )
+            .unwrap_or_default(),
             "file:///home/user/Websites/my-website/assets/images/logo.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_file_url_to_relative_path_with_backslashes() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "file:\\\\\\home\\user\\Websites\\my-website\\index.html",
-            "assets\\images\\logo.png",
-        )
-        .unwrap_or(str!());
-
+    fn from_file_url_to_relative_path_with_backslashes() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "file:\\\\\\home\\user\\Websites\\my-website\\index.html",
+                "assets\\images\\logo.png",
+            )
+            .unwrap_or_default(),
             "file:///home/user/Websites/my-website/assets/images/logo.png"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn from_data_url_to_file_url() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "file:///etc/passwd",
-        )
-        .unwrap_or(str!());
-
-        assert_eq!(resolved_url.as_str(), "file:///etc/passwd");
-
-        Ok(())
-    }
-
-    #[test]
-    fn preserve_fragment() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "http://doesnt-matter.local/",
-            "css/fonts/fontmarvelous.svg#fontmarvelous",
-        )
-        .unwrap_or(str!());
-
+    fn from_data_url_to_file_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "file:///etc/passwd",
+            )
+            .unwrap_or_default(),
+            "file:///etc/passwd"
+        );
+    }
+
+    #[test]
+    fn preserve_fragment() {
+        assert_eq!(
+            url::resolve_url(
+                "http://doesnt-matter.local/",
+                "css/fonts/fontmarvelous.svg#fontmarvelous",
+            )
+            .unwrap_or_default(),
             "http://doesnt-matter.local/css/fonts/fontmarvelous.svg#fontmarvelous"
         );
-
-        Ok(())
     }
 
     #[test]
-    fn resolve_from_file_url_to_file_url() -> Result<(), ParseError> {
-        let resolved_url = if cfg!(windows) {
-            url::resolve_url("file:///c:/index.html", "file:///c:/image.png").unwrap_or(str!())
-        } else {
-            url::resolve_url("file:///tmp/index.html", "file:///tmp/image.png").unwrap_or(str!())
-        };
-
+    fn resolve_from_file_url_to_file_url() {
         assert_eq!(
-            resolved_url.as_str(),
+            if cfg!(windows) {
+                url::resolve_url("file:///c:/index.html", "file:///c:/image.png")
+                    .unwrap_or_default()
+            } else {
+                url::resolve_url("file:///tmp/index.html", "file:///tmp/image.png")
+                    .unwrap_or_default()
+            },
             if cfg!(windows) {
                 "file:///c:/image.png"
             } else {
                 "file:///tmp/image.png"
             }
         );
-
-        Ok(())
     }
 }
 
@@ -222,18 +179,16 @@ mod passing {
 #[cfg(test)]
 mod failing {
     use crate::url;
-    use url::ParseError;
 
     #[test]
-    fn from_data_url_to_url_with_no_protocol() -> Result<(), ParseError> {
-        let resolved_url = url::resolve_url(
-            "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
-            "//www.w3schools.com/html/html_iframe.asp",
-        )
-        .unwrap_or(str!());
-
-        assert_eq!(resolved_url.as_str(), "");
-
-        Ok(())
+    fn from_data_url_to_url_with_no_protocol() {
+        assert_eq!(
+            url::resolve_url(
+                "data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
+                "//www.w3schools.com/html/html_iframe.asp",
+            )
+            .unwrap_or_default(),
+            ""
+        );
     }
 }

src/tests/url/url_has_protocol.rs

@@ -54,6 +54,11 @@ mod passing {
             "MAILTO:somebody@somewhere.com?subject=hello"
         ));
     }
+
+    #[test]
+    fn empty_data_url() {
+        assert!(url::url_has_protocol("data:text/html,"));
+    }
 }
 
 //  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
@@ -65,13 +70,11 @@ mod passing {
 
 #[cfg(test)]
 mod failing {
-    use crate::utils;
+    use crate::url;
 
     #[test]
     fn url_with_no_protocol() {
-        assert!(!url::url_has_protocol(
-            "//some-hostname.com/some-file.html"
-        ));
+        assert!(!url::url_has_protocol("//some-hostname.com/some-file.html"));
     }
 
     #[test]

src/tests/utils/retrieve_asset.rs

@@ -11,6 +11,7 @@ mod passing {
     use std::collections::HashMap;
     use std::env;
 
+    use crate::opts::Options;
     use crate::url;
     use crate::utils;
 
@@ -19,6 +20,9 @@ mod passing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         // If both source and target are data URLs,
         // ensure the result contains target data URL
         let (data, final_url, media_type) = utils::retrieve_asset(
@@ -26,7 +30,7 @@ mod passing {
             &client,
             "data:text/html;base64,c291cmNl",
             "data:text/html;base64,dGFyZ2V0",
-            false,
+            &options,
             0,
         )
         .unwrap();
@@ -46,6 +50,9 @@ mod passing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
 
         // Inclusion of local assets from local sources should be allowed
@@ -63,7 +70,7 @@ mod passing {
                 file = file_url_protocol,
                 cwd = cwd.to_str().unwrap()
             ),
-            false,
+            &options,
             0,
         )
         .unwrap();
@@ -91,6 +98,7 @@ mod failing {
     use reqwest::blocking::Client;
     use std::collections::HashMap;
 
+    use crate::opts::Options;
     use crate::utils;
 
     #[test]
@@ -98,13 +106,16 @@ mod failing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         // Inclusion of local assets from data URL sources should not be allowed
         match utils::retrieve_asset(
             cache,
             &client,
             "data:text/html;base64,SoUrCe",
             "file:///etc/passwd",
-            false,
+            &options,
             0,
         ) {
             Ok((..)) => {
@@ -121,13 +132,16 @@ mod failing {
         let cache = &mut HashMap::new();
         let client = Client::new();
 
+        let mut options = Options::default();
+        options.silent = true;
+
         // Inclusion of local assets from remote sources should not be allowed
         match utils::retrieve_asset(
             cache,
             &client,
             "https://kernel.org/",
             "file:///etc/passwd",
-            false,
+            &options,
             0,
         ) {
             Ok((..)) => {

src/url.rs

@@ -33,45 +33,6 @@ pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str) -> String {
     format!("data:{};base64,{}", media_type, base64::encode(data))
 }
 
-pub fn data_url_to_data<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
-    let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
-    let path: String = parsed_url.path().to_string();
-    let comma_loc: usize = path.find(',').unwrap_or(path.len());
-
-    let meta_data: String = path.chars().take(comma_loc).collect();
-    let raw_data: String = path.chars().skip(comma_loc + 1).collect();
-
-    let text: String = decode_url(raw_data);
-
-    let meta_data_items: Vec<&str> = meta_data.split(';').collect();
-    let mut media_type: String = str!();
-    let mut encoding: &str = "";
-
-    let mut i: i8 = 0;
-    for item in &meta_data_items {
-        if i == 0 {
-            media_type = str!(item);
-        } else {
-            if item.eq_ignore_ascii_case("base64")
-                || item.eq_ignore_ascii_case("utf8")
-                || item.eq_ignore_ascii_case("charset=UTF-8")
-            {
-                encoding = item;
-            }
-        }
-
-        i = i + 1;
-    }
-
-    let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
-        base64::decode(&text).unwrap_or(vec![])
-    } else {
-        text.as_bytes().to_vec()
-    };
-
-    (media_type, data)
-}
-
 pub fn decode_url(input: String) -> String {
     let input: String = input.replace("+", "%2B");
 
@@ -113,10 +74,9 @@ pub fn file_url_to_fs_path(url: &str) -> String {
 }
 
 pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
-    if Url::parse(url.as_ref()).unwrap().fragment() == None {
-        str!()
-    } else {
-        str!(Url::parse(url.as_ref()).unwrap().fragment().unwrap())
+    match Url::parse(url.as_ref()) {
+        Ok(parsed_url) => parsed_url.fragment().unwrap_or("").to_string(),
+        Err(_err) => str!(),
     }
 }
 
@@ -138,6 +98,45 @@ pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
         .unwrap_or(false)
 }
 
+pub fn parse_data_url<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
+    let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
+    let path: String = parsed_url.path().to_string();
+    let comma_loc: usize = path.find(',').unwrap_or(path.len());
+
+    let meta_data: String = path.chars().take(comma_loc).collect();
+    let raw_data: String = path.chars().skip(comma_loc + 1).collect();
+
+    let text: String = decode_url(raw_data);
+
+    let meta_data_items: Vec<&str> = meta_data.split(';').collect();
+    let mut media_type: String = str!();
+    let mut encoding: &str = "";
+
+    let mut i: i8 = 0;
+    for item in &meta_data_items {
+        if i == 0 {
+            media_type = str!(item);
+        } else {
+            if item.eq_ignore_ascii_case("base64")
+                || item.eq_ignore_ascii_case("utf8")
+                || item.eq_ignore_ascii_case("charset=UTF-8")
+            {
+                encoding = item;
+            }
+        }
+
+        i = i + 1;
+    }
+
+    let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
+        base64::decode(&text).unwrap_or(vec![])
+    } else {
+        text.as_bytes().to_vec()
+    };
+
+    (media_type, data)
+}
+
 pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
     let result = if is_http_url(to.as_ref()) {
         to.as_ref().to_string()

src/utils.rs

@@ -4,9 +4,12 @@ use std::collections::HashMap;
 use std::fs;
 use std::path::Path;
 
-use crate::url::{clean_url, data_url_to_data, file_url_to_fs_path, is_data_url, is_file_url};
+use crate::opts::Options;
+use crate::url::{clean_url, file_url_to_fs_path, is_data_url, is_file_url, parse_data_url};
 
-const INDENT: &str = " ";
+const ANSI_COLOR_RED: &'static str = "\x1b[31m";
+const ANSI_COLOR_RESET: &'static str = "\x1b[0m";
+const INDENT: &'static str = " ";
 
 const MAGIC: [[&[u8]; 2]; 18] = [
     // Image
@@ -31,13 +34,14 @@ const MAGIC: [[&[u8]; 2]; 18] = [
     [b"....moov", b"video/quicktime"],
     [b"\x1A\x45\xDF\xA3", b"video/webm"],
 ];
-
 const PLAINTEXT_MEDIA_TYPES: &[&str] = &[
+    "application/javascript",
     "image/svg+xml",
-    "text/css",
-    "text/html",
-    "text/javascript",
-    "text/plain",
+    // "text/css",
+    // "text/csv",
+    // "text/html",
+    // "text/javascript",
+    // "text/plain",
 ];
 
 pub fn detect_media_type(data: &[u8], url: &str) -> String {
@@ -55,7 +59,8 @@ pub fn detect_media_type(data: &[u8], url: &str) -> String {
 }
 
 pub fn is_plaintext_media_type(media_type: &str) -> bool {
-    PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
+    media_type.to_lowercase().as_str().starts_with("text/")
+        || PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
 }
 
 pub fn indent(level: u32) -> String {
@@ -73,7 +78,7 @@ pub fn retrieve_asset(
     client: &Client,
     parent_url: &str,
     url: &str,
-    opt_silent: bool,
+    options: &Options,
     depth: u32,
 ) -> Result<(Vec<u8>, String, String), reqwest::Error> {
     if url.len() == 0 {
@@ -82,7 +87,7 @@ pub fn retrieve_asset(
     }
 
     if is_data_url(&url) {
-        let (media_type, data) = data_url_to_data(url);
+        let (media_type, data) = parse_data_url(url);
         Ok((data, url.to_string(), media_type))
     } else if is_file_url(&url) {
         // Check if parent_url is also file:///
@@ -95,7 +100,7 @@ pub fn retrieve_asset(
         let fs_file_path: String = file_url_to_fs_path(url);
         let path = Path::new(&fs_file_path);
         if path.exists() {
-            if !opt_silent {
+            if !options.silent {
                 eprintln!("{}{}", indent(depth).as_str(), &url);
             }
 
@@ -109,7 +114,7 @@ pub fn retrieve_asset(
 
         if cache.contains_key(&cache_key) {
             // URL is in cache, we get and return it
-            if !opt_silent {
+            if !options.silent {
                 eprintln!("{}{} (from cache)", indent(depth).as_str(), &url);
             }
 
@@ -120,34 +125,57 @@ pub fn retrieve_asset(
             ))
         } else {
             // URL not in cache, we retrieve the file
-            let mut response = client.get(url).send()?;
-            let res_url = response.url().to_string();
+            match client.get(url).send() {
+                Ok(mut response) => {
+                    if !options.ignore_errors && response.status() != 200 {
+                        if !options.silent {
+                            eprintln!(
+                                "{}{}{} ({}){}",
+                                indent(depth).as_str(),
+                                if options.no_color { "" } else { ANSI_COLOR_RED },
+                                &url,
+                                response.status(),
+                                if options.no_color {
+                                    ""
+                                } else {
+                                    ANSI_COLOR_RESET
+                                },
+                            );
+                        }
+                        // Provoke error
+                        return Err(client.get("").send().unwrap_err());
+                    }
 
-            if !opt_silent {
-                if url == res_url {
-                    eprintln!("{}{}", indent(depth).as_str(), &url);
-                } else {
-                    eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &res_url);
+                    let res_url = response.url().to_string();
+
+                    if !options.silent {
+                        if url == res_url {
+                            eprintln!("{}{}", indent(depth).as_str(), &url);
+                        } else {
+                            eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &res_url);
+                        }
+                    }
+
+                    let new_cache_key: String = clean_url(&res_url);
+
+                    // Convert response into a byte array
+                    let mut data: Vec<u8> = vec![];
+                    response.copy_to(&mut data)?;
+
+                    // Attempt to obtain media type by reading the Content-Type header
+                    let media_type = response
+                        .headers()
+                        .get(CONTENT_TYPE)
+                        .and_then(|header| header.to_str().ok())
+                        .unwrap_or("");
+
+                    // Add retrieved resource to cache
+                    cache.insert(new_cache_key, data.clone());
+
+                    Ok((data, res_url, media_type.to_string()))
                 }
+                Err(error) => Err(error),
             }
-
-            let new_cache_key: String = clean_url(&res_url);
-
-            // Convert response into a byte array
-            let mut data: Vec<u8> = vec![];
-            response.copy_to(&mut data)?;
-
-            // Attempt to obtain media type by reading the Content-Type header
-            let media_type = response
-                .headers()
-                .get(CONTENT_TYPE)
-                .and_then(|header| header.to_str().ok())
-                .unwrap_or("");
-
-            // Add to cache
-            cache.insert(new_cache_key, data.clone());
-
-            Ok((data, res_url, media_type.to_string()))
         }
     }
 }

utils/run-in-container.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+DOCKER=docker
+PROG_NAME=monolith
+
+if which podman 2>&1 > /dev/null; then
+    DOCKER=podman
+fi
+
+$DOCKER run --rm Y2Z/$PROG_NAME $PROG_NAME "$@"