mirror
/
monolith
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

make tests work again

This commit is contained in:
Sunshine 2024-05-19 13:28:29 +00:00
parent c85220bb61
commit 40acdd5a2a
No known key found for this signature in database
GPG Key ID: 1C850498C410353C
9 changed files with 157 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main.rs
View File

@ -169,12 +169,10 @@ fn main() {
// Initialize client
let mut cache = HashMap::new();
let mut header_map = HeaderMap::new();
if let Some(user_agent) = &options.user_agent {
header_map.insert(
USER_AGENT,
HeaderValue::from_str(&user_agent).expect("Invalid User-Agent header specified"),
);
}
header_map.insert(
USER_AGENT,
HeaderValue::from_str(&options.user_agent).expect("Invalid User-Agent header specified"),
);
let client = if options.timeout > 0 {
Client::builder().timeout(Duration::from_secs(options.timeout))
} else {

14
src/opts.rs
View File

@ -1,4 +1,4 @@
use clap::Parser;
use clap::{Args, Parser, Subcommand, ValueEnum};
use std::env;
// #[derive(Default)]
@ -38,9 +38,9 @@ const ASCII: &'static str = " \
| | \\___/ | | \\ | | | | | | |
|___| |__________| \\_____________________| |___| |___| |___|
";
const DEFAULT_NETWORK_TIMEOUT: u64 = 120;
const DEFAULT_USER_AGENT: &'static str =
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0";
// const DEFAULT_NETWORK_TIMEOUT: u64 = 60;
// const DEFAULT_USER_AGENT: &'static str =
// "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0";
#[derive(Parser, Debug)]
#[command(
@ -114,7 +114,7 @@ pub struct Options {
pub unwrap_noscript: bool,
/// Write output to <file>, use - for STDOUT
#[arg(short = 'o', long = "output")]
#[arg(short = 'o', long = "output", default_value = "-")]
pub output: String,
/// Suppress verbosity
@ -127,8 +127,8 @@ pub struct Options {
pub timeout: u64,
/// Set custom User-Agent string
#[arg(short = 'u', long = "user-agent")]
pub user_agent: Option<String>,
#[arg(short = 'u', long = "user-agent", default_value = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0")]
pub user_agent: String,
/// Remove video sources
#[arg(short = 'v', long = "no-video")]

67
tests/css/embed_css.rs
View File

@ -7,10 +7,12 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use reqwest::blocking::Client;
use reqwest::Url;
use std::collections::HashMap;
use monolith::cookies::Cookie;
use monolith::css;
use monolith::opts::Options;
use monolith::url::EMPTY_IMAGE_DATA_URL;
@ -19,11 +21,12 @@ mod passing {
fn empty_input() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("data:,").unwrap();
let options = Options::default();
let options = Options::parse();
assert_eq!(
css::embed_css(cache, &client, &document_url, "", &options),
css::embed_css(cache, &client, &document_url, "", &options, &cookies),
""
);
}
@ -32,11 +35,12 @@ mod passing {
fn trim_if_empty() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let options = Options::default();
let options = Options::parse();
assert_eq!(
css::embed_css(cache, &client, &document_url, "\t \t ", &options),
css::embed_css(cache, &client, &document_url, "\t \t ", &options, &cookies),
""
);
}
@ -45,8 +49,9 @@ mod passing {
fn style_exclude_unquoted_images() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
@ -59,7 +64,7 @@ mod passing {
height: calc(100vh - 10pt)";
assert_eq!(
css::embed_css(cache, &client, &document_url, &STYLE, &options),
css::embed_css(cache, &client, &document_url, &STYLE, &options, &cookies),
format!(
"/* border: none;*/\
background-image: url(\"{empty_image}\"); \
@ -77,8 +82,9 @@ mod passing {
fn style_exclude_single_quoted_images() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("data:,").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
@ -91,7 +97,7 @@ mod passing {
height: calc(100vh - 10pt)";
assert_eq!(
css::embed_css(cache, &client, &document_url, &STYLE, &options),
css::embed_css(cache, &client, &document_url, &STYLE, &options, &cookies),
format!(
"/* border: none;*/\
background-image: url(\"{empty_image}\"); \
@ -109,8 +115,9 @@ mod passing {
fn style_block() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("file:///").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -122,7 +129,7 @@ mod passing {
html > body {}";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS
);
}
@ -131,8 +138,9 @@ mod passing {
fn attribute_selectors() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -166,7 +174,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS
);
}
@ -175,8 +183,9 @@ mod passing {
fn import_string() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -188,7 +197,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
"\
@charset \"UTF-8\";\n\
\n\
@ -203,8 +212,9 @@ mod passing {
fn hash_urls() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -218,7 +228,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS
);
}
@ -227,8 +237,9 @@ mod passing {
fn transform_percentages_and_degrees() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -240,7 +251,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS
);
}
@ -249,8 +260,9 @@ mod passing {
fn unusual_indents() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -264,7 +276,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS
);
}
@ -273,8 +285,9 @@ mod passing {
fn exclude_fonts() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("https://doesntmatter.local/").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.no_fonts = true;
options.silent = true;
@ -312,7 +325,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS_OUT
);
}
@ -321,8 +334,9 @@ mod passing {
fn content() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("data:,").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -337,7 +351,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS_OUT
);
}
@ -346,8 +360,9 @@ mod passing {
fn ie_css_hack() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let document_url: Url = Url::parse("data:,").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
const CSS: &str = "\
@ -364,7 +379,7 @@ mod passing {
";
assert_eq!(
css::embed_css(cache, &client, &document_url, &CSS, &options),
css::embed_css(cache, &client, &document_url, &CSS, &options, &cookies),
CSS_OUT
);
}

16
tests/html/compose_csp.rs
View File

@ -7,12 +7,14 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use monolith::html;
use monolith::opts::Options;
#[test]
fn isolated() {
let mut options = Options::default();
let mut options = Options::parse();
options.isolate = true;
let csp_content = html::compose_csp(&options);
@ -24,7 +26,7 @@ mod passing {
#[test]
fn no_css() {
let mut options = Options::default();
let mut options = Options::parse();
options.no_css = true;
let csp_content = html::compose_csp(&options);
@ -33,7 +35,7 @@ mod passing {
#[test]
fn no_fonts() {
let mut options = Options::default();
let mut options = Options::parse();
options.no_fonts = true;
let csp_content = html::compose_csp(&options);
@ -42,7 +44,7 @@ mod passing {
#[test]
fn no_frames() {
let mut options = Options::default();
let mut options = Options::parse();
options.no_frames = true;
let csp_content = html::compose_csp(&options);
@ -51,7 +53,7 @@ mod passing {
#[test]
fn no_js() {
let mut options = Options::default();
let mut options = Options::parse();
options.no_js = true;
let csp_content = html::compose_csp(&options);
@ -60,7 +62,7 @@ mod passing {
#[test]
fn no_images() {
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
let csp_content = html::compose_csp(&options);
@ -69,7 +71,7 @@ mod passing {
#[test]
fn all() {
let mut options = Options::default();
let mut options = Options::parse();
options.isolate = true;
options.no_css = true;
options.no_fonts = true;

24
tests/html/embed_srcset.rs
View File

@ -7,10 +7,12 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use reqwest::blocking::Client;
use reqwest::Url;
use std::collections::HashMap;
use monolith::cookies::Cookie;
use monolith::html;
use monolith::opts::Options;
use monolith::url::EMPTY_IMAGE_DATA_URL;
@ -19,8 +21,9 @@ mod passing {
fn small_medium_large() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let srcset_value = "small.png 1x, medium.png 1.5x, large.png 2x";
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let embedded_css = html::embed_srcset(
@ -29,6 +32,7 @@ mod passing {
&Url::parse("data:,").unwrap(),
&srcset_value,
&options,
&cookies,
);
assert_eq!(
@ -44,8 +48,9 @@ mod passing {
fn small_medium_only_medium_has_scale() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let srcset_value = "small.png, medium.png 1.5x";
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let embedded_css = html::embed_srcset(
@ -54,6 +59,7 @@ mod passing {
&Url::parse("data:,").unwrap(),
&srcset_value,
&options,
&cookies,
);
assert_eq!(
@ -66,8 +72,9 @@ mod passing {
fn commas_within_file_names() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let srcset_value = "small,s.png 1x, large,l.png 2x";
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let embedded_css = html::embed_srcset(
@ -76,6 +83,7 @@ mod passing {
&Url::parse("data:,").unwrap(),
&srcset_value,
&options,
&cookies,
);
assert_eq!(
@ -88,8 +96,9 @@ mod passing {
fn tabs_and_newlines_after_commas() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let srcset_value = "small,s.png 1x,\nmedium,m.png 2x,\nlarge,l.png 3x";
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let embedded_css = html::embed_srcset(
@ -98,6 +107,7 @@ mod passing {
&Url::parse("data:,").unwrap(),
&srcset_value,
&options,
&cookies,
);
assert_eq!(
@ -119,10 +129,12 @@ mod passing {
#[cfg(test)]
mod failing {
use clap::Parser;
use reqwest::blocking::Client;
use reqwest::Url;
use std::collections::HashMap;
use monolith::cookies::Cookie;
use monolith::html;
use monolith::opts::Options;
use monolith::url::EMPTY_IMAGE_DATA_URL;
@ -131,8 +143,9 @@ mod failing {
fn trailing_comma() {
let cache = &mut HashMap::new();
let client = Client::new();
let cookies: Vec<Cookie> = vec![];
let srcset_value = "small.png 1x, large.png 2x,";
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let embedded_css = html::embed_srcset(
@ -141,6 +154,7 @@ mod failing {
&Url::parse("data:,").unwrap(),
&srcset_value,
&options,
&cookies,
);
assert_eq!(

12
tests/html/serialize_document.rs
View File

@ -7,6 +7,8 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use monolith::html;
use monolith::opts::Options;
@ -14,7 +16,7 @@ mod passing {
fn div_as_root_element() {
let html = "<div><script src=\"some.js\"></script></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let options = Options::default();
let options = Options::parse();
assert_eq!(
String::from_utf8_lossy(&html::serialize_document(dom, "".to_string(), &options)),
@ -29,7 +31,7 @@ mod passing {
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
<div><script src=\"some.js\"></script></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let mut options = Options::default();
let mut options = Options::parse();
options.isolate = true;
assert_eq!(
@ -61,7 +63,7 @@ mod passing {
<link rel=\"stylesheet\" href=\"main.css\"/>\
<div style=\"display: none;\"></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let mut options = Options::default();
let mut options = Options::parse();
options.no_css = true;
assert_eq!(
@ -85,7 +87,7 @@ mod passing {
<link rel=\"something\"/>\
<div><script src=\"some.js\"></script></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let mut options = Options::default();
let mut options = Options::parse();
options.no_frames = true;
assert_eq!(
@ -118,7 +120,7 @@ mod passing {
<iframe src=\"some.html\"></iframe>\
</div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let mut options = Options::default();
let mut options = Options::parse();
options.isolate = true;
options.no_css = true;
options.no_fonts = true;

107
tests/html/walk_and_embed_assets.rs
View File

@ -7,12 +7,14 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use html5ever::serialize::{serialize, SerializeOpts};
use markup5ever_rcdom::SerializableHandle;
use reqwest::blocking::Client;
use std::collections::HashMap;
use url::Url;
use monolith::cookies::Cookie;
use monolith::html;
use monolith::opts::Options;
use monolith::url::EMPTY_IMAGE_DATA_URL;
@ -20,17 +22,16 @@ mod passing {
#[test]
fn basic() {
let cache = &mut HashMap::new();
let cookies: Vec<Cookie> = vec![];
let html: &str = "<div><P></P></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -48,17 +49,16 @@ mod passing {
#[test]
fn ensure_no_recursive_iframe() {
let cookies: Vec<Cookie> = vec![];
let html = "<div><P></P><iframe src=\"\"></iframe></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -76,17 +76,16 @@ mod passing {
#[test]
fn ensure_no_recursive_frame() {
let cookies: Vec<Cookie> = vec![];
let html = "<frameset><frame src=\"\"></frameset>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -104,6 +103,7 @@ mod passing {
#[test]
fn no_css() {
let cookies: Vec<Cookie> = vec![];
let html = "\
<link rel=\"stylesheet\" href=\"main.css\">\
<link rel=\"alternate stylesheet\" href=\"main.css\">\
@ -113,14 +113,12 @@ mod passing {
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_css = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -149,19 +147,18 @@ mod passing {
#[test]
fn no_images() {
let cookies: Vec<Cookie> = vec![];
let html = "<link rel=\"icon\" href=\"favicon.ico\">\
<div><img src=\"http://localhost/assets/mono_lisa.png\" /></div>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -191,19 +188,18 @@ mod passing {
#[test]
fn no_body_background_images() {
let cookies: Vec<Cookie> = vec![];
let html =
"<body background=\"no/such/image.png\" background=\"no/such/image2.png\"></body>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -221,18 +217,17 @@ mod passing {
#[test]
fn no_frames() {
let cookies: Vec<Cookie> = vec![];
let html = "<frameset><frame src=\"http://trackbook.com\"></frameset>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_frames = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -258,18 +253,17 @@ mod passing {
#[test]
fn no_iframes() {
let cookies: Vec<Cookie> = vec![];
let html = "<iframe src=\"http://trackbook.com\"></iframe>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_frames = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -294,6 +288,7 @@ mod passing {
#[test]
fn no_js() {
let cookies: Vec<Cookie> = vec![];
let html = "\
<div onClick=\"void(0)\">\
<script src=\"http://localhost/assets/some.js\"></script>\
@ -303,14 +298,12 @@ mod passing {
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_js = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -338,18 +331,17 @@ mod passing {
#[test]
fn keeps_integrity_for_unfamiliar_links() {
let cookies: Vec<Cookie> = vec![];
let html = "<title>Has integrity</title>\
<link integrity=\"sha384-12345\" rel=\"something\" href=\"https://some-site.com/some-file.ext\" />";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -375,6 +367,7 @@ mod passing {
#[test]
fn discards_integrity_for_known_links_nojs_nocss() {
let cookies: Vec<Cookie> = vec![];
let html = "\
<title>No integrity</title>\
<link integrity=\"\" rel=\"stylesheet\" href=\"data:;\"/>\
@ -383,15 +376,13 @@ mod passing {
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_css = true;
options.no_js = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -418,6 +409,7 @@ mod passing {
#[test]
fn discards_integrity_for_embedded_assets() {
let cookies: Vec<Cookie> = vec![];
let html = "\
<title>No integrity</title>\
<link integrity=\"sha384-123\" rel=\"something\" href=\"data:;\"/>\
@ -426,15 +418,13 @@ mod passing {
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_css = true;
options.no_js = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -462,6 +452,7 @@ mod passing {
#[test]
fn removes_unwanted_meta_tags() {
let cookies: Vec<Cookie> = vec![];
let html = "\
<html>\
<head>\
@ -475,17 +466,15 @@ mod passing {
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_css = true;
options.no_frames = true;
options.no_js = true;
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -511,6 +500,7 @@ mod passing {
#[test]
fn processes_noscript_tags() {
let cookies: Vec<Cookie> = vec![];
let html = "\
<html>\
<body>\
@ -522,14 +512,12 @@ mod passing {
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(
@ -559,17 +547,16 @@ mod passing {
#[test]
fn preserves_script_type_json() {
let cookies: Vec<Cookie> = vec![];
let html = "<script id=\"data\" type=\"application/json\">{\"mono\":\"lith\"}</script>";
let dom = html::html_to_dom(&html.as_bytes().to_vec(), "".to_string());
let url: Url = Url::parse("http://localhost").unwrap();
let cache = &mut HashMap::new();
let mut options = Options::default();
let mut options = Options::parse();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, &cookies);
let mut buf: Vec<u8> = Vec::new();
serialize(

12
tests/opts.rs
View File

@ -7,11 +7,13 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use monolith::opts::Options;
#[test]
fn defaults() {
let options: Options = Options::default();
let options: Options = Options::parse();
assert_eq!(options.no_audio, false);
assert_eq!(options.base_url, None);
@ -25,12 +27,12 @@ mod passing {
assert_eq!(options.no_js, false);
assert_eq!(options.insecure, false);
assert_eq!(options.no_metadata, false);
assert_eq!(options.output, "".to_string());
assert_eq!(options.output, "-".to_string());
assert_eq!(options.silent, false);
assert_eq!(options.timeout, 0);
assert_eq!(options.user_agent, None);
assert_eq!(options.timeout, 60);
assert_eq!(options.user_agent, "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0");
assert_eq!(options.no_video, false);
assert_eq!(options.target, "".to_string());
assert_eq!(options.target, "-".to_string());
}
}

24
tests/utils/retrieve_asset.rs
View File

@ -7,11 +7,13 @@
#[cfg(test)]
mod passing {
use clap::Parser;
use reqwest::blocking::Client;
use reqwest::Url;
use std::collections::HashMap;
use std::env;
use monolith::cookies::Cookie;
use monolith::opts::Options;
use monolith::url;
use monolith::utils;
@ -20,8 +22,8 @@ mod passing {
fn read_data_url() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
let cookies: Vec<Cookie> = vec![];
let mut options = Options::parse();
options.silent = true;
// If both source and target are data URLs,
@ -32,6 +34,7 @@ mod passing {
&Url::parse("data:text/html;base64,c291cmNl").unwrap(),
&Url::parse("data:text/html;base64,dGFyZ2V0").unwrap(),
&options,
&cookies,
)
.unwrap();
assert_eq!(&media_type, "text/html");
@ -50,8 +53,8 @@ mod passing {
fn read_local_file_with_file_url_parent() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
let cookies: Vec<Cookie> = vec![];
let mut options = Options::parse();
options.silent = true;
let file_url_protocol: &str = if cfg!(windows) { "file:///" } else { "file://" };
@ -74,6 +77,7 @@ mod passing {
))
.unwrap(),
&options,
&cookies,
)
.unwrap();
assert_eq!(&media_type, "application/javascript");
@ -100,10 +104,12 @@ mod passing {
#[cfg(test)]
mod failing {
use clap::Parser;
use reqwest::blocking::Client;
use reqwest::Url;
use std::collections::HashMap;
use monolith::cookies::Cookie;
use monolith::opts::Options;
use monolith::utils;
@ -111,8 +117,8 @@ mod failing {
fn read_local_file_with_data_url_parent() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
let cookies: Vec<Cookie> = vec![];
let mut options = Options::parse();
options.silent = true;
// Inclusion of local assets from data URL sources should not be allowed
@ -122,6 +128,7 @@ mod failing {
&Url::parse("data:text/html;base64,SoUrCe").unwrap(),
&Url::parse("file:///etc/passwd").unwrap(),
&options,
&cookies,
) {
Ok((..)) => {
assert!(false);
@ -136,8 +143,8 @@ mod failing {
fn read_local_file_with_https_parent() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
let cookies: Vec<Cookie> = vec![];
let mut options = Options::parse();
options.silent = true;
// Inclusion of local assets from remote sources should not be allowed
@ -147,6 +154,7 @@ mod failing {
&Url::parse("https://kernel.org/").unwrap(),
&Url::parse("file:///etc/passwd").unwrap(),
&options,
&cookies,
) {
Ok((..)) => {
assert!(false);