52 Commits

Author SHA1 Message Date
Sunshine
03cdc0e0b2 Merge pull request #201 from snshn/refactor-and-version-bump
Refactor and version bump
2020-07-14 03:51:31 -04:00
Sunshine
b98b7af0b4 Merge pull request #202 from snshn/minus-stdout
Treat - for stdout
2020-07-14 03:51:18 -04:00
Sunshine
73c35eaccb treat minus for output target file path as stdout 2020-07-14 03:35:59 -04:00
Sunshine
2c5d1e930b bump version (2.2.7 -> 2.3.0) 2020-07-14 03:29:08 -04:00
Sunshine
90f7c3a0d0 alphabetical order for function names 2020-07-14 03:27:52 -04:00
Sunshine
c1fec5967d Merge pull request #200 from snshn/favicon
Automatically obtain favicon.ico
2020-07-14 03:24:10 -04:00
Sunshine
09d41d2cf1 automatically obtain favicon.ico 2020-07-14 02:58:29 -04:00
Sunshine
8f1da3c792 Update cd.yml 2020-07-13 19:09:01 -04:00
Sunshine
a8449a2b32 Update README.md 2020-07-13 01:16:38 -04:00
Sunshine
164e728ad3 Merge pull request #197 from snshn/addetional-black-box-test-data
Additional black box test data
2020-07-06 16:51:49 -04:00
Sunshine
8883bd6aca add more black box test data 2020-07-06 16:15:57 -04:00
Sunshine
eae5d4dc6b Merge pull request #196 from snshn/help-message-update
Update help message
2020-07-01 06:41:32 -04:00
Sunshine
ec85121d28 update help message 2020-07-01 06:29:56 -04:00
Sunshine
a8a85a4191 Merge pull request #195 from snshn/logo
Logo
2020-07-01 06:24:28 -04:00
Sunshine
decd5b2119 add ASCII logo atop of help message 2020-07-01 06:13:58 -04:00
Sunshine
bef6d848e9 add raster icon along with its Blender scene 2020-07-01 05:54:48 -04:00
Sunshine
4263e42cd1 Merge pull request #194 from snshn/indented-tree
Indented tree
2020-06-28 16:37:10 -04:00
Sunshine
23de5ced21 add tests for utils::indent() 2020-06-28 16:15:42 -04:00
Sunshine
bc98aca2a2 indent items in retrieval log to form a tree-like structure 2020-06-28 16:11:15 -04:00
Sunshine
438ebd520a Merge pull request #193 from snshn/options-struct
Pass options object instead of using separate parameters
2020-06-28 01:51:05 -04:00
Sunshine
ddb97009e9 pass options object instead of using separate parameters 2020-06-28 01:36:41 -04:00
Sunshine
6e67545b92 Merge pull request #192 from snshn/more-test-data
Add more sample data for blackbox tests
2020-06-27 14:57:07 -04:00
Sunshine
9e5d8ec691 add more sample data for blackbox tests 2020-06-27 14:55:10 -04:00
Sunshine
fb835fae28 Merge pull request #191 from snshn/trim-style
Trim CSS if it contains nothing but whitespaces
2020-06-26 23:41:41 -04:00
Sunshine
29bf042da0 trim CSS if it contains nothing but whitespaces 2020-06-26 23:26:55 -04:00
Sunshine
d67483cf8e Merge pull request #190 from snshn/refactor-csp
Refactor CSP code
2020-06-26 21:42:19 -04:00
Sunshine
4140d8ebad Create references.md 2020-06-26 18:16:18 -04:00
Sunshine
2ac964fae5 include font-src into CSP 2020-06-26 18:14:46 -04:00
Sunshine
ae5d6d2df4 refactor CSP code 2020-06-26 16:19:44 -04:00
Sunshine
2ed151d883 Update web-apps.md 2020-06-26 15:05:47 -04:00
Sunshine
3cdfdc45d3 Update snapcraft.yaml 2020-06-26 14:57:52 -04:00
Sunshine
ac04af2cfc Update ADR-0006 2020-06-26 14:44:54 -04:00
Sunshine
769953d7bd Merge pull request #187 from snshn/arm-snapcraft
Add armhf target to snapcraft.yaml
2020-06-26 14:40:46 -04:00
Sunshine
136dcc31cf Merge pull request #189 from snshn/remove-unwanted-meta-tags
Automatically remove "Refresh" and "Location" META tags
2020-06-26 01:29:41 -04:00
Sunshine
44cac65a83 automatically remove "Refresh" and "Location" META tags 2020-06-26 01:18:52 -04:00
Sunshine
c3ca2ad1d5 Merge pull request #188 from snshn/metadata-tag-function
Move metadata tag code into a function
2020-06-25 18:31:50 -04:00
Sunshine
0347fd3985 move metadata tag code into a function 2020-06-25 18:23:56 -04:00
Sunshine
95d0083b3c Update README.md 2020-06-25 17:38:39 -04:00
Sunshine
3ce26b5fdd Merge pull request #186 from snshn/code-improvements
Code improvements
2020-06-24 03:31:11 -04:00
Sunshine
7f9458adfe add armhf target to snapcraft.yaml 2020-06-24 03:20:36 -04:00
Sunshine
5c229c51da move functions related to URL manipulation into url.rs 2020-06-24 03:16:40 -04:00
Sunshine
f6ea16b3ad create a separate function for appending URL fragments 2020-06-24 02:26:05 -04:00
Sunshine
877b11d52c Merge pull request #185 from snshn/upd-crates
Update crates
2020-06-20 03:41:00 -04:00
Sunshine
f9aac6f41b update crates 2020-06-20 01:05:39 -04:00
Sunshine
0a30c286fe add x86_64 GNU/Linux target to CD 2020-06-19 07:44:57 -04:00
Sunshine
ea56b9b4c1 Update README.md 2020-06-19 03:51:19 -04:00
Sunshine
e821591efe Merge pull request #183 from snshn/update-readme-freebsd-instructions
Add FreeBSD installation instructions to README.md
2020-06-18 22:26:56 -04:00
Sunshine
4e5d2fdc8d Merge pull request #184 from snshn/update-readme-ascii
Update README.md
2020-06-18 00:25:55 -04:00
Sunshine
7c2ed2c9ca Update README.md 2020-06-18 00:19:16 -04:00
Sunshine
60d21ae071 Update README.md 2020-06-17 07:42:10 -04:00
Sunshine
bfdcd459e1 Update web-apps.md 2020-06-04 02:32:01 -04:00
Sunshine
6c020dfa88 Create web-apps.md 2020-06-04 02:31:41 -04:00
53 changed files with 1782 additions and 1085 deletions

View File

@@ -15,7 +15,7 @@ jobs:
- run: git config --global core.autocrlf false
- name: Checkout the repository
uses: actions/checkout@v2
- name: Build and install the executable
- name: Build the executable
run: cargo build --release
- uses: Shopify/upload-to-release@1.0.0
with:
@@ -52,3 +52,16 @@ jobs:
name: monolith-gnu-linux-armhf
path: target/arm-unknown-linux-gnueabihf/release/monolith
repo-token: ${{ secrets.GITHUB_TOKEN }}
gnu_linux_x86_64:
runs-on: ubuntu-18.04
steps:
- name: Checkout the repository
uses: actions/checkout@v2
- name: Build the executable
run: cargo build --release
- uses: Shopify/upload-to-release@1.0.0
with:
name: monolith-gnu-linux-x86_64
path: target/release/monolith
repo-token: ${{ secrets.GITHUB_TOKEN }}

179
Cargo.lock generated
View File

@@ -20,26 +20,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "assert_cmd"
version = "0.12.0"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"escargot 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)",
"predicates 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)",
"predicates-core 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"predicates-tree 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "async-compression"
version = "0.2.0"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -57,13 +57,13 @@ version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "base64"
version = "0.11.0"
name = "autocfg"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "base64"
version = "0.12.0"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@@ -73,13 +73,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "block-buffer"
version = "0.7.3"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
"byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
"generic-array 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -128,9 +128,19 @@ name = "cfg-if"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "chrono"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "clap"
version = "2.33.0"
version = "2.33.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -204,10 +214,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "digest"
version = "0.8.1"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)",
"generic-array 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -236,17 +246,6 @@ dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "escargot"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "fake-simd"
version = "0.1.2"
@@ -352,10 +351,11 @@ dependencies = [
[[package]]
name = "generic-array"
version = "0.12.3"
version = "0.14.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -370,7 +370,7 @@ dependencies = [
[[package]]
name = "h2"
version = "0.2.1"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -383,7 +383,7 @@ dependencies = [
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-util 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -441,21 +441,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "hyper"
version = "0.13.1"
version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-channel 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"h2 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"itoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)",
"socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -467,7 +467,7 @@ name = "hyper-tls"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"hyper 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)",
"native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -623,17 +623,17 @@ dependencies = [
[[package]]
name = "monolith"
version = "2.2.7"
version = "2.3.0"
dependencies = [
"assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)",
"assert_cmd 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
"chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)",
"clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)",
"cssparser 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)",
"html5ever 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)",
"sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)",
"sha2 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -678,6 +678,23 @@ dependencies = [
"version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-integer"
version = "0.1.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
"num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num-traits"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "num_cpus"
version = "1.11.1"
@@ -821,7 +838,7 @@ dependencies = [
[[package]]
name = "pin-project-lite"
version = "0.1.2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
@@ -1053,18 +1070,18 @@ dependencies = [
[[package]]
name = "reqwest"
version = "0.10.0"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)",
"async-compression 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)",
"base64 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)",
"http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper-tls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"js-sys 0.3.34 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1073,17 +1090,16 @@ dependencies = [
"mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)",
"native-tls 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
"percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)",
"time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)",
"url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
"wasm-bindgen 0.2.57 (registry+https://github.com/rust-lang/crates.io-index)",
"wasm-bindgen-futures 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)",
"web-sys 0.3.34 (registry+https://github.com/rust-lang/crates.io-index)",
"winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)",
"winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
@@ -1123,9 +1139,6 @@ dependencies = [
name = "serde"
version = "1.0.103"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "serde_derive"
@@ -1160,11 +1173,11 @@ dependencies = [
[[package]]
name = "sha2"
version = "0.8.1"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"block-buffer 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)",
"digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)",
"fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -1189,6 +1202,17 @@ name = "smallvec"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "socket2"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
"redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)",
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "sourcefile"
version = "0.1.4"
@@ -1288,12 +1312,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)",
"lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)",
"memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)",
"num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -1308,14 +1333,14 @@ dependencies = [
[[package]]
name = "tokio-util"
version = "0.2.0"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)",
"pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
@@ -1413,6 +1438,14 @@ name = "version_check"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "wait-timeout"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "want"
version = "0.3.0"
@@ -1555,7 +1588,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "winreg"
version = "0.6.2"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)",
@@ -1574,14 +1607,14 @@ dependencies = [
"checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2"
"checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
"checksum anyhow 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c"
"checksum assert_cmd 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6283bac8dd7226470d491bc4737816fea4ca1fba7a2847f2e9097fd6bfb4624c"
"checksum async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c5c52622726d68ec35fec88edfb4ccb862d4f3b3bfa4af2f45142e69ef9b220"
"checksum assert_cmd 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c88b9ca26f9c16ec830350d309397e74ee9abdfd8eb1f71cb6ecc71a3fc818da"
"checksum async-compression 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ae84766bab9f774e32979583ba56d6af8c701288c6dc99144819d5d2ee0b170f"
"checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90"
"checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2"
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
"checksum base64 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7d5ca2cd0adc3f48f9e9ea5a6bbdf9ccc0bfade884847e484d452414c7ccffb3"
"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
"checksum base64 0.12.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e223af0dc48c96d4f8342ec01a4974f139df863896b316681efd36742f22cc67"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
"checksum block-buffer 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dbcf92448676f82bb7a334c58bbce8b0d43580fb5362a9d608b18879d12a3d31"
"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
"checksum bumpalo 3.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8fe2567a8d8a3aedb4e39aa39e186d5673acfd56393c6ac83b2bc5bd82f4369c"
"checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
@@ -1590,7 +1623,8 @@ dependencies = [
"checksum c2-chacha 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "214238caa1bf3a496ec3392968969cab8549f96ff30652c9e56885329315f6bb"
"checksum cc 1.0.47 (registry+https://github.com/rust-lang/crates.io-index)" = "aa87058dce70a3ff5621797f1506cb837edd02ac4c0ae642b4542dce802908b8"
"checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
"checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9"
"checksum chrono 0.4.11 (registry+https://github.com/rust-lang/crates.io-index)" = "80094f509cf8b5ae86a4966a39b3ff66cd7e2a3e594accec3743ff3fabeab5b2"
"checksum clap 2.33.1 (registry+https://github.com/rust-lang/crates.io-index)" = "bdfa80d47f954d53a35a64987ca1422f495b8d6483c0fe9f7117b36c2a792129"
"checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"
"checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d"
"checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
@@ -1598,12 +1632,11 @@ dependencies = [
"checksum cssparser 0.27.2 (registry+https://github.com/rust-lang/crates.io-index)" = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
"checksum cssparser-macros 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
"checksum difference 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "524cbf6897b527295dff137cec09ecf3a05f4fddffd7dfcd1585403449e74198"
"checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
"checksum digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
"checksum doc-comment 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "923dea538cea0aa3025e8685b20d6ee21ef99c4f77e954a30febbaac5ec73a97"
"checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e"
"checksum dtoa-short 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "59020b8513b76630c49d918c33db9f4c91638e7d3404a28084083b87e33f76f2"
"checksum encoding_rs 0.8.20 (registry+https://github.com/rust-lang/crates.io-index)" = "87240518927716f79692c2ed85bfe6e98196d18c6401ec75355760233a7e12e9"
"checksum escargot 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "74cf96bec282dcdb07099f7e31d9fed323bca9435a09aba7b6d99b7617bca96d"
"checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f"
"checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3"
@@ -1619,16 +1652,16 @@ dependencies = [
"checksum futures-sink 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "171be33efae63c2d59e6dbba34186fe0d6394fb378069a76dfd80fdcffd43c16"
"checksum futures-task 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0bae52d6b29cf440e298856fec3965ee6fa71b06aa7495178615953fd669e5f9"
"checksum futures-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d66274fb76985d3c62c886d1da7ac4c0903a8c9f754e8fe0f35a6a6cc39e76"
"checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
"checksum generic-array 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ac746a5f3bbfdadd6106868134545e684693d54d9d44f6e9588a7d54af0bf980"
"checksum getrandom 0.1.13 (registry+https://github.com/rust-lang/crates.io-index)" = "e7db7ca94ed4cd01190ceee0d8a8052f08a247aa1b469a7f68c6a3b71afcf407"
"checksum h2 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b9433d71e471c1736fd5a61b671fc0b148d7a2992f666c958d03cd8feb3b88d1"
"checksum h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff"
"checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
"checksum hermit-abi 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "eff2656d88f158ce120947499e971d743c05dbcbed62e5bd2f38f1698bbc3772"
"checksum html5ever 0.24.1 (registry+https://github.com/rust-lang/crates.io-index)" = "025483b0a1e4577bb28578318c886ee5f817dda6eb62473269349044406644cb"
"checksum http 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b708cc7f06493459026f53b9a61a7a121a5d1ec6238dee58ea4941132b30156b"
"checksum http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
"checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9"
"checksum hyper 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8bf49cfb32edee45d890537d9057d1b02ed55f53b7b6a30bae83a38c9231749e"
"checksum hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7655b9594024ad0ee439f3b5a7299369dc2a3f459b47c696f9ff676f9aa1f"
"checksum hyper-tls 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ab58a31960b2f78c5c24cf255216789863754438a1e48849a956846f899e762e"
"checksum idna 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9"
"checksum indexmap 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712d7b3ea5827fcb9d4fda14bf4da5f136f0db2ae9c8f4bd4e2d1c6fde4e6db2"
@@ -1652,6 +1685,8 @@ dependencies = [
"checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88"
"checksum new_debug_unreachable 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "f40f005c60db6e03bae699e414c58bf9aa7ea02a2d0b9bfbcf19286cc4c82b30"
"checksum nom 4.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
"checksum num-integer 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "8d59457e662d541ba17869cf51cf177c0b5f0cbf476c66bdc90bf1edac4f875b"
"checksum num-traits 0.2.12 (registry+https://github.com/rust-lang/crates.io-index)" = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
"checksum num_cpus 1.11.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76dac5ed2a876980778b8b85f75a71b6cbf0db0b1232ee12f826bccb00d09d72"
"checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
"checksum openssl 0.10.26 (registry+https://github.com/rust-lang/crates.io-index)" = "3a3cc5799d98e1088141b8e01ff760112bbd9f19d850c124500566ca6901a585"
@@ -1668,7 +1703,7 @@ dependencies = [
"checksum phf_shared 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
"checksum pin-project 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "94b90146c7216e4cb534069fb91366de4ea0ea353105ee45ed297e2d1619e469"
"checksum pin-project-internal 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "44ca92f893f0656d3cba8158dd0f2b99b94de256a4a54e870bd6922fcc6c8355"
"checksum pin-project-lite 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e8822eb8bb72452f038ebf6048efa02c3fe22bf83f76519c9583e47fc194a422"
"checksum pin-project-lite 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "282adbf10f2698a7a77f8e983a74b2d18176c19a7fd32a45446139ae7b02b715"
"checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587"
"checksum pkg-config 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
"checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b"
@@ -1697,7 +1732,7 @@ dependencies = [
"checksum rdrand 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
"checksum redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)" = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84"
"checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
"checksum reqwest 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "03c6cbd2bc1c1cb7052dbe30f4a70cf65811967c800f2dfbb2e6036dc9ee2553"
"checksum reqwest 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)" = "3b82c9238b305f26f53443e3a4bc8528d64b8d0bee408ec949eb7bf5635ec680"
"checksum ryu 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bfa8506c1de11c9c4e4c38863ccbe02a305c8188e85a05a784c9e11e1c3910c8"
"checksum schannel 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "87f550b06b6cba9c8b8be3ee73f391990116bf527450d2556e9b9ce263b9a021"
"checksum security-framework 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8ef2429d7cefe5fd28bd1d2ed41c944547d4ff84776f5935b456da44593a16df"
@@ -1706,11 +1741,12 @@ dependencies = [
"checksum serde_derive 1.0.103 (registry+https://github.com/rust-lang/crates.io-index)" = "a8c6faef9a2e64b0064f48570289b4bf8823b7581f1d6157c1b52152306651d0"
"checksum serde_json 1.0.42 (registry+https://github.com/rust-lang/crates.io-index)" = "1a3351dcbc1f067e2c92ab7c3c1f288ad1a4cffc470b5aaddb4c2e0a3ae80043"
"checksum serde_urlencoded 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97"
"checksum sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0"
"checksum sha2 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "72377440080fd008550fe9b441e854e43318db116f90181eef92e9ae9aedab48"
"checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac"
"checksum siphasher 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "8e88f89a550c01e4cd809f3df4f52dc9e939f3273a2017eabd5c6d12fd98bb23"
"checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
"checksum smallvec 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecf3b85f68e8abaa7555aa5abdb1153079387e60b718283d732f03897fcfc86"
"checksum socket2 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)" = "03088793f677dce356f3ccc2edb1b314ad191ab702a5de3faf49304f7e104918"
"checksum sourcefile 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4bf77cb82ba8453b42b6ae1d692e4cdc92f9a47beaf89a847c8be83f4e328ad3"
"checksum string_cache 0.7.5 (registry+https://github.com/rust-lang/crates.io-index)" = "89c058a82f9fd69b1becf8c274f412281038877c553182f1d02eb027045a2d67"
"checksum string_cache_codegen 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "f0f45ed1b65bf9a4bf2f7b7dc59212d1926e9eaf00fa998988e420fd124467c6"
@@ -1723,7 +1759,7 @@ dependencies = [
"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
"checksum tokio 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0e1bef565a52394086ecac0a6fa3b8ace4cb3a138ee1d96bd2b93283b56824e3"
"checksum tokio-tls 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7bde02a3a5291395f59b06ec6945a3077602fac2b07eeeaf0dee2122f3619828"
"checksum tokio-util 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "571da51182ec208780505a32528fc5512a8fe1443ab960b3f2f3ef093cd16930"
"checksum tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
"checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860"
"checksum treeline 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f741b240f1a48843f9b8e0444fb55fb2a4ff67293b50a9179dfd5ea67f8d41"
"checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382"
@@ -1740,6 +1776,7 @@ dependencies = [
"checksum vec_map 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a"
"checksum version_check 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
"checksum version_check 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "078775d0255232fb988e6fccf26ddc9d1ac274299aaedcedce21c6f72cc533ce"
"checksum wait-timeout 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
"checksum want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
"checksum wasi 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b89c3ce4ce14bdc6fb6beaf9ec7928ca331de5df7e5ea278375642a2f478570d"
"checksum wasm-bindgen 0.2.57 (registry+https://github.com/rust-lang/crates.io-index)" = "701bc20794a7f9e8dcd85984a848f951ef6c5083322b6dd17fe880c99390f7cd"
@@ -1756,5 +1793,5 @@ dependencies = [
"checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
"checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
"checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
"checksum winreg 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b2986deb581c4fe11b621998a5e53361efe6b48a151178d0cd9eeffa4dc6acc9"
"checksum winreg 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
"checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"

View File

@@ -1,6 +1,6 @@
[package]
name = "monolith"
version = "2.2.7"
version = "2.3.0"
edition = "2018"
authors = [
"Sunshine <sunshine@uberspace.net>",
@@ -12,19 +12,19 @@ authors = [
description = "CLI tool for saving web pages as a single HTML file"
[dependencies]
base64 = "0.12.0"
clap = "2.33.0"
base64 = "0.12.2"
chrono = "0.4.11" # Used to render comments indicating the time the page was saved
clap = "2.33.1"
cssparser = "0.27.2"
html5ever = "0.24.1"
sha2 = "0.8.1" # Used in calculating checksums during integrity checks
time = "0.1.42" # Used to render comments indicating the time the page was saved
sha2 = "0.9.0" # Used in calculating checksums during integrity checks
url = "2.1.1"
[dependencies.reqwest]
version = "0.10.*"
version = "0.10.6"
default-features = false
features = ["default-tls", "blocking", "gzip"]
[dev-dependencies]
assert_cmd = "0.12.0"
assert_cmd = "1.0.1"
tempfile = "3.1.0"

View File

@@ -23,3 +23,7 @@ install:
uninstall:
@cargo uninstall
.PHONY: uninstall
clean:
@cargo clean
.PHONY: clean

View File

@@ -3,13 +3,13 @@
[![Monolith Build Status for Windows](https://github.com/Y2Z/monolith/workflows/Windows/badge.svg)](https://github.com/Y2Z/monolith/actions?query=workflow%3AWindows)
```
___ ___________ __________ ___________________ ___
| \ / \ | | | | | |
| \_/ __ \_| __ | | ___ ___ |__| |
| | | | | | | | | | | |
| |__| _ |__| |____| | | | | __ |
| |\_/| | \ | | | | | | |
|___| |__________| \____________________| |___| |___| |___|
_____ ______________ __________ ___________________ ___
| \ / \ | | | | | |
| \_/ __ \_| __ | | ___ ___ |__| |
| | | | | | | | | | | |
| |\ /| |__| _ |__| |____| | | | | __ |
| | \___/ | | \ | | | | | | |
|___| |__________| \_____________________| |___| |___| |___|
```
A data hoarders dream come true: bundle any web page into a single HTML file. You can finally replace that gazillion of open tabs with a gazillion of .html files stored somewhere on your precious little drive.
@@ -28,6 +28,13 @@ If compared to saving websites with `wget -mpk`, this tool embeds all assets as
#### Using Snapcraft (on GNU/Linux)
$ snap install monolith
#### Using Ports collection (on FreeBSD and TrueOS)
$ cd /usr/ports/www/monolith/
$ make install clean
#### Using pre-built binaries (Windows, ARM-based devices, etc)
Every [release](https://github.com/Y2Z/monolith/releases) contains pre-built binaries for Windows, GNU/Linux, as well as platforms with non-standart CPU architecture.
#### From source
Dependency: `libssl-dev`
@@ -36,7 +43,7 @@ Dependency: `libssl-dev`
$ cd monolith
$ make install
#### With Docker
#### Using Containers
The guide can be found [here](docs/containers.md)
---------------------------------------------------
@@ -48,14 +55,14 @@ The guide can be found [here](docs/containers.md)
## Options
- `-c`: Ignore styles
- `-f`: Exclude frames and iframes
- `-f`: Exclude frames
- `-F`: Omit web fonts
- `-i`: Remove images
- `-I`: Isolate the document
- `-j`: Exclude JavaScript
- `-k`: Accept invalid X.509 (TLS) certificates
- `-o`: Write output to file
- `-s`: Enable silent mode
- `-s`: Be quiet
- `-t`: Adjust network request timeout
- `-u`: Provide custom User-Agent

BIN
assets/icon/icon.blend Normal file

Binary file not shown.

BIN
assets/icon/icon.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.2 MiB

View File

@@ -0,0 +1,19 @@
# 4. Reload and location `meta` tags
Date: 2020-06-25
## Status
Accepted
## Context
HTML documents may contain `meta` tags capable of automatically refreshing the page or redirecting to another location.
## Decision
Since the resulting document is saved to disk and generally not intended to be served over the network, it only makes sense to remove `meta` tags that have `http-equiv` attribute equal to "Refresh" or "Location", in order to prevent them from reloading the page or redirecting to another location.
## Consequences
Monolith will ensure that saved documents do not contain `meta` tags capable of changing location or reloading the page.

3
docs/references.md Normal file
View File

@@ -0,0 +1,3 @@
# References
- https://content-security-policy.com/

10
docs/web-apps.md Normal file
View File

@@ -0,0 +1,10 @@
# Web apps that can be saved with Monolith
These apps retain most or all of their functionality when saved with Monolith
|Converse|https://conversejs.org|
|:--|:--|
|Description|An XMPP client built using web technologies|
|Functionality retained|**full**|
|Command to use|`monolith https://conversejs.org/fullscreen.html > conversejs.html`|
|Monolith version used|2.2.7|

View File

@@ -18,11 +18,11 @@ description: |
confinement: strict
# Building on armhf fails, so we specify all supported non-armhf architectures
architectures:
- build-on: amd64
- build-on: i386
- build-on: arm64
- build-on: armhf
- build-on: i386
- build-on: ppc64el
- build-on: s390x

View File

@@ -2,7 +2,9 @@ use cssparser::{ParseError, Parser, ParserInput, SourcePosition, Token};
use reqwest::blocking::Client;
use std::collections::HashMap;
use crate::utils::{data_to_data_url, get_url_fragment, is_http_url, resolve_url, retrieve_asset};
use crate::opts::Options;
use crate::url::{data_to_data_url, get_url_fragment, is_http_url, resolve_url, url_with_fragment};
use crate::utils::retrieve_asset;
const CSS_PROPS_WITH_IMAGE_URLS: &[&str] = &[
// Universal
@@ -58,12 +60,11 @@ pub fn process_css<'a>(
client: &Client,
parent_url: &str,
parser: &mut Parser,
options: &Options,
depth: u32,
rule_name: &str,
prop_name: &str,
func_name: &str,
opt_no_fonts: bool,
opt_no_images: bool,
opt_silent: bool,
) -> Result<String, ParseError<'a, String>> {
let mut result: String = str!();
@@ -90,7 +91,7 @@ pub fn process_css<'a>(
Token::Colon => result.push_str(":"),
Token::Comma => result.push_str(","),
Token::ParenthesisBlock | Token::SquareBracketBlock | Token::CurlyBracketBlock => {
if opt_no_fonts && curr_rule == "font-face" {
if options.no_fonts && curr_rule == "font-face" {
continue;
}
@@ -113,12 +114,11 @@ pub fn process_css<'a>(
client,
parent_url,
parser,
options,
depth,
rule_name,
curr_prop.as_str(),
func_name,
opt_no_fonts,
opt_no_images,
opt_silent,
)
})
.unwrap();
@@ -148,7 +148,7 @@ pub fn process_css<'a>(
// @import, @font-face, @charset, @media...
Token::AtKeyword(ref value) => {
curr_rule = str!(value);
if opt_no_fonts && curr_rule == "font-face" {
if options.no_fonts && curr_rule == "font-face" {
continue;
}
result.push_str("@");
@@ -171,34 +171,42 @@ pub fn process_css<'a>(
let import_full_url = resolve_url(&parent_url, value).unwrap_or_default();
let import_url_fragment = get_url_fragment(import_full_url.clone());
match retrieve_asset(cache, client, &parent_url, &import_full_url, opt_silent) {
match retrieve_asset(
cache,
client,
&parent_url,
&import_full_url,
options.silent,
depth + 1,
) {
Ok((import_contents, import_final_url, _import_media_type)) => {
result.push_str(
enquote(
data_to_data_url(
"text/css",
embed_css(
cache,
client,
&import_final_url,
&String::from_utf8_lossy(&import_contents),
opt_no_fonts,
opt_no_images,
opt_silent,
)
.as_bytes(),
&import_final_url,
&import_url_fragment,
),
false,
let import_data_url = data_to_data_url(
"text/css",
embed_css(
cache,
client,
&import_final_url,
&String::from_utf8_lossy(&import_contents),
options,
depth + 1,
)
.as_str(),
.as_bytes(),
&import_final_url,
);
let assembled_url: String = url_with_fragment(
import_data_url.as_str(),
import_url_fragment.as_str(),
);
result.push_str(enquote(assembled_url, false).as_str());
}
Err(_) => {
// Keep remote reference if unable to retrieve the asset
if is_http_url(import_full_url.clone()) {
result.push_str(enquote(import_full_url, false).as_str());
let assembled_url: String = url_with_fragment(
import_full_url.as_str(),
import_url_fragment.as_str(),
);
result.push_str(enquote(assembled_url, false).as_str());
}
}
}
@@ -209,7 +217,7 @@ pub fn process_css<'a>(
continue;
}
if opt_no_images && is_image_url_prop(curr_prop.as_str()) {
if options.no_images && is_image_url_prop(curr_prop.as_str()) {
result.push_str(enquote(str!(empty_image!()), false).as_str());
} else {
let resolved_url = resolve_url(&parent_url, value).unwrap_or_default();
@@ -219,21 +227,23 @@ pub fn process_css<'a>(
client,
&parent_url,
&resolved_url,
opt_silent,
options.silent,
depth + 1,
) {
Ok((data, final_url, media_type)) => {
let data_url = data_to_data_url(
&media_type,
&data,
&final_url,
&url_fragment,
);
result.push_str(enquote(data_url, false).as_str());
let data_url = data_to_data_url(&media_type, &data, &final_url);
let assembled_url: String =
url_with_fragment(data_url.as_str(), url_fragment.as_str());
result.push_str(enquote(assembled_url, false).as_str());
}
Err(_) => {
// Keep remote reference if unable to retrieve the asset
if is_http_url(resolved_url.clone()) {
result.push_str(enquote(resolved_url, false).as_str());
let assembled_url: String = url_with_fragment(
resolved_url.as_str(),
url_fragment.as_str(),
);
result.push_str(enquote(assembled_url, false).as_str());
}
}
}
@@ -261,7 +271,7 @@ pub fn process_css<'a>(
if *has_sign && *unit_value >= 0. {
result.push_str("+");
}
result.push_str(str!(unit_value * 100.).as_str());
result.push_str(str!(unit_value * 100.0).as_str());
result.push_str("%");
}
Token::Dimension {
@@ -305,7 +315,14 @@ pub fn process_css<'a>(
if is_import {
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
let url_fragment = get_url_fragment(full_url.clone());
match retrieve_asset(cache, client, &parent_url, &full_url, opt_silent) {
match retrieve_asset(
cache,
client,
&parent_url,
&full_url,
options.silent,
depth + 1,
) {
Ok((css, final_url, _media_type)) => {
let data_url = data_to_data_url(
"text/css",
@@ -314,39 +331,51 @@ pub fn process_css<'a>(
client,
&final_url,
&String::from_utf8_lossy(&css),
opt_no_fonts,
opt_no_images,
opt_silent,
options,
depth + 1,
)
.as_bytes(),
&final_url,
&url_fragment,
);
result.push_str(enquote(data_url, false).as_str());
let assembled_url: String =
url_with_fragment(data_url.as_str(), url_fragment.as_str());
result.push_str(enquote(assembled_url, false).as_str());
}
Err(_) => {
// Keep remote reference if unable to retrieve the asset
if is_http_url(full_url.clone()) {
result.push_str(enquote(full_url, false).as_str());
let assembled_url: String =
url_with_fragment(full_url.as_str(), url_fragment.as_str());
result.push_str(enquote(assembled_url, false).as_str());
}
}
}
} else {
if opt_no_images && is_image_url_prop(curr_prop.as_str()) {
if is_image_url_prop(curr_prop.as_str()) && options.no_images {
result.push_str(enquote(str!(empty_image!()), false).as_str());
} else {
let full_url = resolve_url(&parent_url, value).unwrap_or_default();
let url_fragment = get_url_fragment(full_url.clone());
match retrieve_asset(cache, client, &parent_url, &full_url, opt_silent) {
match retrieve_asset(
cache,
client,
&parent_url,
&full_url,
options.silent,
depth + 1,
) {
Ok((data, final_url, media_type)) => {
let data_url =
data_to_data_url(&media_type, &data, &final_url, &url_fragment);
result.push_str(enquote(data_url, false).as_str());
let data_url = data_to_data_url(&media_type, &data, &final_url);
let assembled_url: String =
url_with_fragment(data_url.as_str(), url_fragment.as_str());
result.push_str(enquote(assembled_url, false).as_str());
}
Err(_) => {
// Keep remote reference if unable to retrieve the asset
if is_http_url(full_url.clone()) {
result.push_str(enquote(full_url, false).as_str());
let assembled_url: String =
url_with_fragment(full_url.as_str(), url_fragment.as_str());
result.push_str(enquote(assembled_url, false).as_str());
}
}
}
@@ -367,12 +396,11 @@ pub fn process_css<'a>(
client,
parent_url,
parser,
options,
depth,
curr_rule.as_str(),
curr_prop.as_str(),
function_name,
opt_no_fonts,
opt_no_images,
opt_silent,
)
})
.unwrap();
@@ -384,6 +412,11 @@ pub fn process_css<'a>(
}
}
// Ensure empty CSS is really empty
if result.len() > 0 && result.trim().len() == 0 {
result = result.trim().to_string()
}
Ok(result)
}
@@ -392,9 +425,8 @@ pub fn embed_css(
client: &Client,
parent_url: &str,
css: &str,
opt_no_fonts: bool,
opt_no_images: bool,
opt_silent: bool,
options: &Options,
depth: u32,
) -> String {
let mut input = ParserInput::new(&css);
let mut parser = Parser::new(&mut input);
@@ -404,12 +436,11 @@ pub fn embed_css(
client,
parent_url,
&mut parser,
options,
depth,
"",
"",
"",
opt_no_fonts,
opt_no_images,
opt_silent,
)
.unwrap()
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,9 +1,14 @@
#[macro_use]
extern crate clap;
#[macro_use]
mod macros;
pub mod css;
pub mod html;
pub mod js;
pub mod opts;
pub mod url;
pub mod utils;
#[cfg(test)]

View File

@@ -1,8 +1,5 @@
use monolith::html::{html_to_dom, stringify_document, walk_and_embed_assets};
use monolith::utils::{data_url_to_data, is_data_url, is_file_url, is_http_url, retrieve_asset};
use reqwest::blocking::Client;
use reqwest::header::{HeaderMap, HeaderValue, USER_AGENT};
use reqwest::Url;
use std::collections::HashMap;
use std::env;
use std::fs;
@@ -11,12 +8,16 @@ use std::path::Path;
use std::process;
use std::time::Duration;
mod args;
mod macros;
use monolith::html::{
add_favicon, has_favicon, html_to_dom, metadata_tag, stringify_document, walk_and_embed_assets,
};
use monolith::opts::Options;
use monolith::url::{
data_to_data_url, data_url_to_data, is_data_url, is_file_url, is_http_url, resolve_url,
};
use monolith::utils::retrieve_asset;
#[macro_use]
extern crate clap;
use crate::args::AppArgs;
mod macros;
enum Output {
Stdout(io::Stdout),
@@ -25,7 +26,7 @@ enum Output {
impl Output {
fn new(file_path: &str) -> Result<Output, Error> {
if file_path.is_empty() {
if file_path.is_empty() || file_path.eq("-") {
Ok(Output::Stdout(io::stdout()))
} else {
Ok(Output::File(fs::File::create(file_path)?))
@@ -47,11 +48,11 @@ impl Output {
}
fn main() {
let app_args = AppArgs::get();
let original_target: &str = &app_args.target;
let options = Options::from_args();
let original_target: &str = &options.target;
let target_url: &str;
let base_url;
let dom;
let mut dom;
// Pre-process the input
let cwd_normalized: String =
@@ -60,6 +61,7 @@ fn main() {
let mut target: String = str!(original_target.clone()).replace("\\", "/");
let path_is_relative: bool = path.is_relative();
// Determine exact target URL
if target.clone().len() == 0 {
eprintln!("No target specified");
process::exit(1);
@@ -86,31 +88,38 @@ fn main() {
target_url = target.as_str();
}
let mut output = Output::new(&app_args.output).expect("Could not prepare output");
// Define output
let mut output = Output::new(&options.output).expect("Could not prepare output");
// Initialize client
let mut cache = HashMap::new();
let mut header_map = HeaderMap::new();
header_map.insert(
USER_AGENT,
HeaderValue::from_str(&app_args.user_agent).expect("Invalid User-Agent header specified"),
HeaderValue::from_str(&options.user_agent).expect("Invalid User-Agent header specified"),
);
let timeout: u64 = if app_args.timeout > 0 {
app_args.timeout
let timeout: u64 = if options.timeout > 0 {
options.timeout
} else {
std::u64::MAX / 4
};
let client = Client::builder()
.timeout(Duration::from_secs(timeout))
.danger_accept_invalid_certs(app_args.insecure)
.danger_accept_invalid_certs(options.insecure)
.default_headers(header_map)
.build()
.expect("Failed to initialize HTTP client");
// Retrieve root document
// Retrieve target document
if is_file_url(target_url) || is_http_url(target_url) {
match retrieve_asset(&mut cache, &client, target_url, target_url, app_args.silent) {
match retrieve_asset(
&mut cache,
&client,
target_url,
target_url,
options.silent,
0,
) {
Ok((data, final_url, _media_type)) => {
base_url = final_url;
dom = html_to_dom(&String::from_utf8_lossy(&data));
@@ -132,60 +141,45 @@ fn main() {
process::exit(1);
}
let time_saved = time::now_utc();
// Request and embed /favicon.ico (unless it's already linked in the document)
if !options.no_images && is_http_url(target_url) && !has_favicon(&dom.document) {
let favicon_ico_url: String = resolve_url(&base_url, "/favicon.ico").unwrap();
walk_and_embed_assets(
&mut cache,
&client,
&base_url,
&dom.document,
app_args.no_css,
app_args.no_fonts,
app_args.no_frames,
app_args.no_js,
app_args.no_images,
app_args.silent,
);
let mut html: String = stringify_document(
&dom.document,
app_args.no_css,
app_args.no_frames,
app_args.no_js,
app_args.no_images,
app_args.isolate,
);
if !app_args.no_metadata {
// Safe to unwrap (we just put this through an HTTP request)
let mut clean_url = Url::parse(&base_url).unwrap();
clean_url.set_fragment(None);
// Prevent credentials from getting into metadata
if is_http_url(&base_url) {
// Only HTTP(S) URLs may feature credentials
clean_url.set_username("").unwrap();
clean_url.set_password(None).unwrap();
match retrieve_asset(
&mut cache,
&client,
&base_url,
&favicon_ico_url,
options.silent,
0,
) {
Ok((data, final_url, media_type)) => {
let favicon_data_url: String = data_to_data_url(&media_type, &data, &final_url);
dom = add_favicon(&dom.document, favicon_data_url);
}
Err(_) => {
// Failed to retrieve favicon.ico
}
}
let metadata_comment = if is_http_url(&base_url) {
format!(
"<!-- Saved from {} at {} using {} v{} -->\n",
&clean_url,
time_saved.rfc3339(),
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION"),
)
} else {
format!(
"<!-- Saved from local source at {} using {} v{} -->\n",
time_saved.rfc3339(),
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION"),
)
};
html.insert_str(0, &metadata_comment);
}
// Embed remote assets
walk_and_embed_assets(&mut cache, &client, &base_url, &dom.document, &options, 0);
// Serialize DOM tree
let mut result: String = stringify_document(&dom.document, &options);
// Add metadata tag
if !options.no_metadata {
let metadata_comment = metadata_tag(&base_url);
result.insert_str(0, &metadata_comment);
if metadata_comment.len() > 0 {
result.insert_str(metadata_comment.len(), "\n");
}
}
// Write result into stdout or file
output
.writeln_str(&html)
.writeln_str(&result)
.expect("Could not write HTML output");
}

View File

@@ -1,7 +1,7 @@
use clap::{App, Arg};
#[derive(Default)]
pub struct AppArgs {
pub struct Options {
pub target: String,
pub no_css: bool,
pub no_fonts: bool,
@@ -17,16 +17,25 @@ pub struct AppArgs {
pub no_metadata: bool,
}
const ASCII: &str = " \
_____ ______________ __________ ___________________ ___
| \\ / \\ | | | | | |
| \\_/ __ \\_| __ | | ___ ___ |__| |
| | | | | | | | | | | |
| |\\ /| |__| _ |__| |____| | | | | __ |
| | \\___/ | | \\ | | | | | | |
|___| |__________| \\_____________________| |___| |___| |___|
";
const DEFAULT_NETWORK_TIMEOUT: u64 = 120;
const DEFAULT_USER_AGENT: &str =
"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:73.0) Gecko/20100101 Firefox/73.0";
impl AppArgs {
pub fn get() -> AppArgs {
impl Options {
pub fn from_args() -> Options {
let app = App::new(env!("CARGO_PKG_NAME"))
.version(crate_version!())
.author(crate_authors!("\n"))
.about(crate_description!())
.about(format!("{}\n{}", ASCII, crate_description!()).as_str())
.arg(
Arg::with_name("target")
.required(true)
@@ -34,7 +43,7 @@ impl AppArgs {
.index(1)
.help("URL or file path"),
)
// .args_from_usage("-a, --include-audio 'Removes audio sources'")
// .args_from_usage("-a, --no-audio 'Removes audio sources'")
.args_from_usage("-c, --no-css 'Removes CSS'")
.args_from_usage("-f, --no-frames 'Removes frames and iframes'")
.args_from_usage("-F, --no-fonts 'Removes fonts'")
@@ -43,37 +52,39 @@ impl AppArgs {
.args_from_usage("-j, --no-js 'Removes JavaScript'")
.args_from_usage("-k, --insecure 'Allows invalid X.509 (TLS) certificates'")
.args_from_usage("-M, --no-metadata 'Excludes metadata information from the document'")
.args_from_usage("-o, --output=[document.html] 'Writes output to <file>'")
.args_from_usage("-o, --output=[document.html] 'Write output to <file>'")
.args_from_usage("-s, --silent 'Suppresses verbosity'")
.args_from_usage("-t, --timeout=[60] 'Adjusts network request timeout'")
.args_from_usage("-u, --user-agent=[Firefox] 'Sets custom User-Agent string'")
// .args_from_usage("-v, --include-video 'Removes video sources'")
.args_from_usage("-t, --timeout=[60] 'Adjust network request timeout'")
.args_from_usage("-u, --user-agent=[Firefox] 'Set custom User-Agent string'")
// .args_from_usage("-v, --no-video 'Removes video sources'")
.get_matches();
let mut app_args = AppArgs::default();
let mut options: Options = Options::default();
// Process the command
app_args.target = app
options.target = app
.value_of("target")
.expect("please set target")
.to_string();
app_args.no_css = app.is_present("no-css");
app_args.no_fonts = app.is_present("no-fonts");
app_args.no_frames = app.is_present("no-frames");
app_args.no_images = app.is_present("no-images");
app_args.no_js = app.is_present("no-js");
app_args.insecure = app.is_present("insecure");
app_args.no_metadata = app.is_present("no-metadata");
app_args.isolate = app.is_present("isolate");
app_args.silent = app.is_present("silent");
app_args.timeout = app
options.no_css = app.is_present("no-css");
options.no_frames = app.is_present("no-frames");
options.no_fonts = app.is_present("no-fonts");
options.no_images = app.is_present("no-images");
options.isolate = app.is_present("isolate");
options.no_js = app.is_present("no-js");
options.insecure = app.is_present("insecure");
options.no_metadata = app.is_present("no-metadata");
options.output = app.value_of("output").unwrap_or("").to_string();
options.silent = app.is_present("silent");
options.timeout = app
.value_of("timeout")
.unwrap_or(&DEFAULT_NETWORK_TIMEOUT.to_string())
.parse::<u64>()
.unwrap();
app_args.output = app.value_of("output").unwrap_or("").to_string();
app_args.user_agent = app
options.user_agent = app
.value_of("user-agent")
.unwrap_or(DEFAULT_USER_AGENT)
.to_string();
app_args
options
}
}

View File

@@ -128,13 +128,41 @@ mod passing {
Ok(())
}
#[test]
fn remove_fonts_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
let out = cmd
.arg("-M")
.arg("-F")
.arg("data:text/html,<style>@font-face { font-family: myFont; src: url(font.woff); }</style>Hi")
.output()
.unwrap();
// STDOUT should contain HTML with no web fonts
assert_eq!(
std::str::from_utf8(&out.stdout).unwrap(),
"<html><head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"font-src 'none';\"></meta>\
<style></style>\
</head><body>Hi</body></html>\n"
);
// STDERR should be empty
assert_eq!(std::str::from_utf8(&out.stderr).unwrap(), "");
// The exit code should be 0
out.assert().code(0);
Ok(())
}
#[test]
fn remove_frames_from_data_url() -> Result<(), Box<dyn std::error::Error>> {
let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME"))?;
let out = cmd
.arg("-M")
.arg("-f")
.arg("data:text/html,<iframe src=\"https://google.com\"></iframe>Hi")
.arg("data:text/html,<iframe src=\"https://duckduckgo.com\"></iframe>Hi")
.output()
.unwrap();
@@ -142,7 +170,7 @@ mod passing {
assert_eq!(
std::str::from_utf8(&out.stdout).unwrap(),
"<html><head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none';child-src 'none';\"></meta>\
<meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none'; child-src 'none';\"></meta>\
</head><body><iframe src=\"\"></iframe>Hi</body></html>\n"
);
@@ -259,8 +287,8 @@ mod passing {
std::str::from_utf8(&out.stderr).unwrap(),
format!(
"\
{file}{cwd}/src/tests/data/basic/local-file.html\n\
{file}{cwd}/src/tests/data/basic/local-style.css\n\
{file}{cwd}/src/tests/data/basic/local-file.html\n \
{file}{cwd}/src/tests/data/basic/local-style.css\n \
{file}{cwd}/src/tests/data/basic/local-script.js\n\
",
file = file_url_protocol,
@@ -461,7 +489,7 @@ mod passing {
std::str::from_utf8(&out.stderr).unwrap(),
format!(
"\
{file}{html_path}\n\
{file}{html_path}\n \
{file}{svg_path}\n\
",
file = file_url_prefix,
@@ -512,9 +540,9 @@ mod passing {
std::str::from_utf8(&out.stderr).unwrap(),
format!(
"\
{file}{html_path}\n\
{file}{css_path}\n\
{file}{css_path}\n\
{file}{html_path}\n \
{file}{css_path}\n \
{file}{css_path}\n \
{file}{css_path}\n\
",
file = file_url_prefix,

View File

@@ -7,17 +7,36 @@
#[cfg(test)]
mod passing {
use crate::css;
use reqwest::blocking::Client;
use std::collections::HashMap;
use crate::css;
use crate::opts::Options;
#[test]
fn empty_input() {
let cache = &mut HashMap::new();
let client = Client::new();
let options = Options::default();
assert_eq!(css::embed_css(cache, &client, "", "", &options, 0), "");
}
#[test]
fn trim_if_empty() {
let cache = &mut HashMap::new();
let client = Client::new();
let options = Options::default();
assert_eq!(
css::embed_css(cache, &client, "", "", false, false, false,),
css::embed_css(
cache,
&client,
"https://doesntmatter.local/",
"\t \t ",
&options,
0,
),
""
);
}
@@ -26,6 +45,9 @@ mod passing {
fn style_exclude_unquoted_images() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.no_images = true;
options.silent = true;
const STYLE: &str = "/* border: none;*/\
background-image: url(https://somewhere.com/bg.png); \
@@ -41,9 +63,8 @@ mod passing {
&client,
"https://doesntmatter.local/",
&STYLE,
false,
true,
true,
&options,
0,
),
format!(
"/* border: none;*/\
@@ -62,6 +83,9 @@ mod passing {
fn style_exclude_single_quoted_images() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.no_images = true;
options.silent = true;
const STYLE: &str = "/* border: none;*/\
background-image: url('https://somewhere.com/bg.png'); \
@@ -72,7 +96,7 @@ mod passing {
height: calc(100vh - 10pt)";
assert_eq!(
css::embed_css(cache, &client, "", &STYLE, false, true, true,),
css::embed_css(cache, &client, "", &STYLE, &options, 0),
format!(
"/* border: none;*/\
background-image: url('{empty_image}'); \
@@ -90,6 +114,8 @@ mod passing {
fn style_block() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.silent = true;
const CSS: &str = "\
#id.class-name:not(:nth-child(3n+0)) {\n \
@@ -100,7 +126,7 @@ mod passing {
html > body {}";
assert_eq!(
css::embed_css(cache, &client, "file:///", &CSS, false, false, true,),
css::embed_css(cache, &client, "file:///", &CSS, &options, 0),
CSS
);
}
@@ -109,6 +135,8 @@ mod passing {
fn attribute_selectors() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.silent = true;
const CSS: &str = "\
[data-value] {
@@ -140,16 +168,15 @@ mod passing {
}
";
assert_eq!(
css::embed_css(cache, &client, "", &CSS, false, false, false,),
CSS
);
assert_eq!(css::embed_css(cache, &client, "", &CSS, &options, 0), CSS);
}
#[test]
fn import_string() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.silent = true;
const CSS: &str = "\
@charset 'UTF-8';\n\
@@ -165,9 +192,8 @@ mod passing {
&client,
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
&options,
0,
),
"\
@charset 'UTF-8';\n\
@@ -183,6 +209,8 @@ mod passing {
fn hash_urls() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.silent = true;
const CSS: &str = "\
body {\n \
@@ -200,9 +228,8 @@ mod passing {
&client,
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
&options,
0,
),
CSS
);
@@ -212,6 +239,8 @@ mod passing {
fn transform_percentages_and_degrees() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.silent = true;
const CSS: &str = "\
div {\n \
@@ -227,9 +256,8 @@ mod passing {
&client,
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
&options,
0,
),
CSS
);
@@ -239,6 +267,8 @@ mod passing {
fn unusual_indents() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.silent = true;
const CSS: &str = "\
.is\\:good:hover {\n \
@@ -256,9 +286,8 @@ mod passing {
&client,
"https://doesntmatter.local/",
&CSS,
false,
false,
true,
&options,
0,
),
CSS
);
@@ -268,6 +297,9 @@ mod passing {
fn exclude_fonts() {
let cache = &mut HashMap::new();
let client = Client::new();
let mut options = Options::default();
options.no_fonts = true;
options.silent = true;
const CSS: &str = "\
@font-face {\n \
@@ -309,9 +341,8 @@ mod passing {
&client,
"https://doesntmatter.local/",
&CSS,
true,
false,
true,
&options,
0,
),
CSS_OUT
);

View File

@@ -0,0 +1,23 @@
<!doctype html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>Attempt to import CSS via data URL asset</title>
<style>
body {
background-color: white;
color: black;
}
</style>
<link href="data:text/css;base64,QGltcG9ydCAic3R5bGUuY3NzIjsK" rel="stylesheet" type="text/css" />
</head>
<body>
<p>If you see pink background with white foreground then were in trouble</p>
</body>
</html>

View File

@@ -0,0 +1,4 @@
body {
background-color: pink;
color: white;
}

View File

@@ -0,0 +1,19 @@
<!doctype html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta http-equiv="Content-Security-Policy" content="default-src 'unsafe-inline' file:;" />
<title>Local HTML file</title>
<link href="style.css" rel="stylesheet" type="text/css" integrity="sha512-IWaCTORHkRhOWzcZeILSVmV6V6gPTHgNem6o6rsFAyaKTieDFkeeMrWjtO0DuWrX3bqZY46CVTZXUu0mia0qXQ==" crossorigin="anonymous" />
<link href="style.css" rel="stylesheet" type="text/css" integrity="sha512-vWBzl4NE9oIg8NFOPAyOZbaam0UXWr6aDHPaY2kodSzAFl+mKoj/RMNc6C31NDqK4mE2i68IWxYWqWJPLCgPOw==" crossorigin="anonymous" />
</head>
<body>
<p>This page should have black background and white foreground, but only when served via http: (not via file:)</p>
<script src="script.js" integrity="sha256-ecrEsYh3+ICCX8BCrNSotXgI5534282JwJjx8Q9ZWLc="></script>
<script src="script.js" integrity="sha256-6idk9dK0bOkVdG7Oz4/0YLXSJya8xZHqbRZKMhYrt6o="></script>
</body>
</html>

View File

@@ -0,0 +1,3 @@
function noop() {
console.log("monolith");
}

View File

@@ -0,0 +1,4 @@
body {
background-color: #000;
color: #FFF;
}

View File

@@ -0,0 +1,29 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use html5ever::serialize::{serialize, SerializeOpts};
use crate::html;
#[test]
fn basic() {
let html = "<div>text</div>";
let mut dom = html::html_to_dom(&html);
dom = html::add_favicon(&dom.document, "I_AM_A_FAVICON_DATA_URL".to_string());
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
assert_eq!(
buf.iter().map(|&c| c as char).collect::<String>(),
"<html><head><link rel=\"icon\" href=\"I_AM_A_FAVICON_DATA_URL\"></link></head><body><div>text</div></body></html>"
);
}
}

80
src/tests/html/csp.rs Normal file
View File

@@ -0,0 +1,80 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use crate::html;
use crate::opts::Options;
#[test]
fn isolated() {
let mut options = Options::default();
options.isolate = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "default-src 'unsafe-inline' data:;");
}
#[test]
fn no_css() {
let mut options = Options::default();
options.no_css = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "style-src 'none';");
}
#[test]
fn no_fonts() {
let mut options = Options::default();
options.no_fonts = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "font-src 'none';");
}
#[test]
fn no_frames() {
let mut options = Options::default();
options.no_frames = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "frame-src 'none'; child-src 'none';");
}
#[test]
fn no_js() {
let mut options = Options::default();
options.no_js = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "script-src 'none';");
}
#[test]
fn no_images() {
let mut options = Options::default();
options.no_images = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "img-src data:;");
}
#[test]
fn all() {
let mut options = Options::default();
options.isolate = true;
options.no_css = true;
options.no_fonts = true;
options.no_frames = true;
options.no_js = true;
options.no_images = true;
let csp_content = html::csp(&options);
assert_eq!(csp_content, "default-src 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;");
}
}

View File

@@ -7,16 +7,21 @@
#[cfg(test)]
mod passing {
use crate::html;
use reqwest::blocking::Client;
use std::collections::HashMap;
use crate::html;
use crate::opts::Options;
#[test]
fn replace_with_empty_images() {
let cache = &mut HashMap::new();
let client = Client::new();
let srcset_value = "small.png 1x, large.png 2x";
let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, true, true);
let mut options = Options::default();
options.no_images = true;
options.silent = true;
let embedded_css = html::embed_srcset(cache, &client, "", &srcset_value, &options, 0);
assert_eq!(
format!("{} 1x, {} 2x", empty_image!(), empty_image!()),

View File

@@ -7,9 +7,10 @@
#[cfg(test)]
mod passing {
use crate::html;
use html5ever::rcdom::{Handle, NodeData};
use crate::html;
#[test]
fn get_node_name() {
let html = "<!doctype html><html><HEAD></HEAD><body><div><P></P></div></body></html>";

View File

@@ -0,0 +1,52 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use crate::html;
use crate::opts::Options;
#[test]
fn icon() {
let html = "<link rel=\"icon\" href=\"\" /><div>text</div>";
let dom = html::html_to_dom(&html);
let res: bool = html::has_favicon(&dom.document);
assert!(res);
}
#[test]
fn shortcut_icon() {
let html = "<link rel=\"shortcut icon\" href=\"\" /><div>text</div>";
let dom = html::html_to_dom(&html);
let res: bool = html::has_favicon(&dom.document);
assert!(res);
}
}
// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗
// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝
// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗
// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod failing {
use crate::html;
use crate::opts::Options;
#[test]
fn absent() {
let html = "<div>text</div>";
let dom = html::html_to_dom(&html);
let res: bool = html::has_favicon(&dom.document);
assert!(!res);
}
}

View File

@@ -23,16 +23,6 @@ mod passing {
fn icon_uppercase() {
assert!(html::is_icon("ICON"));
}
#[test]
fn mask_icon() {
assert!(html::is_icon("mask-icon"));
}
#[test]
fn fluid_icon() {
assert!(html::is_icon("fluid-icon"));
}
}
// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗
@@ -46,6 +36,16 @@ mod passing {
mod failing {
use crate::html;
#[test]
fn mask_icon() {
assert!(!html::is_icon("mask-icon"));
}
#[test]
fn fluid_icon() {
assert!(!html::is_icon("fluid-icon"));
}
#[test]
fn stylesheet() {
assert!(!html::is_icon("stylesheet"));

View File

@@ -0,0 +1,82 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use chrono::prelude::*;
use crate::html;
#[test]
fn http_url() {
let url = "http://192.168.1.1/";
let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
let metadata_comment: String = html::metadata_tag(url);
assert_eq!(
metadata_comment,
format!(
"<!-- Saved from {} at {} using {} v{} -->",
&url,
timestamp,
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION"),
)
);
}
#[test]
fn file_url() {
let url = "file:///home/monolith/index.html";
let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
let metadata_comment: String = html::metadata_tag(url);
assert_eq!(
metadata_comment,
format!(
"<!-- Saved from local source at {} using {} v{} -->",
timestamp,
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION"),
)
);
}
#[test]
fn data_url() {
let url = "data:text/html,Hello%2C%20World!";
let timestamp = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true);
let metadata_comment: String = html::metadata_tag(url);
assert_eq!(
metadata_comment,
format!(
"<!-- Saved from local source at {} using {} v{} -->",
timestamp,
env!("CARGO_PKG_NAME"),
env!("CARGO_PKG_VERSION"),
)
);
}
}
// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗
// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝
// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗
// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod failing {
use crate::html;
#[test]
fn empty_string() {
assert_eq!(html::metadata_tag(""), "");
}
}

View File

@@ -1,6 +1,10 @@
mod add_favicon;
mod csp;
mod embed_srcset;
mod get_node_name;
mod has_favicon;
mod has_proper_integrity;
mod is_icon;
mod metadata_tag;
mod stringify_document;
mod walk_and_embed_assets;

View File

@@ -8,27 +8,16 @@
#[cfg(test)]
mod passing {
use crate::html;
use crate::opts::Options;
#[test]
fn div_as_root_element() {
let html = "<div><script src=\"some.js\"></script></div>";
let dom = html::html_to_dom(&html);
let opt_no_css: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_isolate: bool = false;
let options = Options::default();
assert_eq!(
html::stringify_document(
&dom.document,
opt_no_css,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_isolate,
),
html::stringify_document(&dom.document, &options),
"<html><head></head><body><div><script src=\"some.js\"></script></div></body></html>"
);
}
@@ -40,21 +29,13 @@ mod passing {
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
<div><script src=\"some.js\"></script></div>";
let dom = html::html_to_dom(&html);
let opt_no_css: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_isolate: bool = true;
let mut options = Options::default();
options.isolate = true;
assert_eq!(
html::stringify_document(
&dom.document,
opt_no_css,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_isolate,
&options
),
"<html>\
<head>\
@@ -79,22 +60,11 @@ mod passing {
<link rel=\"stylesheet\" href=\"main.css\"/>\
<div style=\"display: none;\"></div>";
let dom = html::html_to_dom(&html);
let opt_no_css: bool = true;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_isolate: bool = false;
let mut options = Options::default();
options.no_css = true;
assert_eq!(
html::stringify_document(
&dom.document,
opt_no_css,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_isolate,
),
html::stringify_document(&dom.document, &options),
"<!DOCTYPE html>\
<html>\
<head>\
@@ -114,26 +84,18 @@ mod passing {
<link rel=\"something\"/>\
<div><script src=\"some.js\"></script></div>";
let dom = html::html_to_dom(&html);
let opt_no_css: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_isolate: bool = false;
let mut options = Options::default();
options.no_frames = true;
assert_eq!(
html::stringify_document(
&dom.document,
opt_no_css,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_isolate,
&options
),
"<!DOCTYPE html>\
<html>\
<head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none';child-src 'none';\"></meta>\
<meta http-equiv=\"Content-Security-Policy\" content=\"frame-src 'none'; child-src 'none';\"></meta>\
<title>Frameless document</title>\
<link rel=\"something\">\
</head>\
@@ -149,31 +111,28 @@ mod passing {
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
<link rel=\"stylesheet\" href=\"some.css\">\
<div>\
<script src=\"some.js\"></script>\
<img style=\"width: 100%;\" src=\"some.png\" />\
<iframe src=\"some.html\"></iframe>\
<script src=\"some.js\"></script>\
<img style=\"width: 100%;\" src=\"some.png\" />\
<iframe src=\"some.html\"></iframe>\
</div>";
let dom = html::html_to_dom(&html);
let opt_isolate: bool = true;
let opt_no_css: bool = true;
let opt_no_frames: bool = true;
let opt_no_js: bool = true;
let opt_no_images: bool = true;
let mut options = Options::default();
options.isolate = true;
options.no_css = true;
options.no_fonts = true;
options.no_frames = true;
options.no_js = true;
options.no_images = true;
assert_eq!(
html::stringify_document(
&dom.document,
opt_no_css,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_isolate,
&options
),
"<!DOCTYPE html>\
<html>\
<head>\
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:; style-src 'none'; frame-src 'none';child-src 'none'; script-src 'none'; img-src data:;\"></meta>\
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src 'unsafe-inline' data:; style-src 'none'; font-src 'none'; frame-src 'none'; child-src 'none'; script-src 'none'; img-src data:;\"></meta>\
<title>no-frame no-css no-js no-image isolated document</title>\
<meta http-equiv=\"Content-Security-Policy\" content=\"default-src https:\">\
<link rel=\"stylesheet\" href=\"some.css\">\

View File

@@ -7,11 +7,13 @@
#[cfg(test)]
mod passing {
use crate::html;
use html5ever::serialize::{serialize, SerializeOpts};
use reqwest::blocking::Client;
use std::collections::HashMap;
use crate::html;
use crate::opts::Options;
#[test]
fn basic() {
let cache = &mut HashMap::new();
@@ -20,27 +22,12 @@ mod passing {
let dom = html::html_to_dom(&html);
let url = "http://localhost";
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -58,27 +45,12 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -96,27 +68,12 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -136,26 +93,13 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = true;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.no_css = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -182,27 +126,13 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = true;
let opt_silent = true;
let mut options = Options::default();
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -211,15 +141,15 @@ mod passing {
buf.iter().map(|&c| c as char).collect::<String>(),
format!(
"<html>\
<head>\
<link rel=\"icon\">\
</head>\
<body>\
<div>\
<img src=\"{empty_image}\">\
</div>\
</body>\
</html>",
<head>\
<link rel=\"icon\">\
</head>\
<body>\
<div>\
<img src=\"{empty_image}\">\
</div>\
</body>\
</html>",
empty_image = empty_image!()
)
);
@@ -233,27 +163,13 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = false;
let opt_no_images: bool = true;
let opt_silent = true;
let mut options = Options::default();
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -271,26 +187,13 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.no_frames = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -308,26 +211,13 @@ mod passing {
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = false;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.no_frames = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -341,34 +231,20 @@ mod passing {
#[test]
fn no_js() {
let html = "<div onClick=\"void(0)\">\
<script src=\"http://localhost/assets/some.js\"></script>\
<script>alert(1)</script>\
<script src=\"http://localhost/assets/some.js\"></script>\
<script>alert(1)</script>\
</div>";
let dom = html::html_to_dom(&html);
let url = "http://localhost";
let cache = &mut HashMap::new();
let opt_no_css: bool = false;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = false;
let opt_no_js: bool = true;
let opt_no_images: bool = false;
let opt_silent = true;
let mut options = Options::default();
options.no_js = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -381,33 +257,24 @@ mod passing {
}
#[test]
fn with_no_integrity() {
fn discards_integrity() {
let html = "<title>No integrity</title>\
<link integrity=\"sha384-...\" rel=\"something\"/>\
<script integrity=\"sha384-...\" src=\"some.js\"></script>";
let dom = html::html_to_dom(&html);
let url = "http://localhost";
let cache = &mut HashMap::new();
let client = Client::new();
let opt_no_css: bool = true;
let opt_no_fonts: bool = false;
let opt_no_frames: bool = true;
let opt_no_js: bool = true;
let opt_no_images: bool = true;
let opt_silent = true;
html::walk_and_embed_assets(
cache,
&client,
&url,
&dom.document,
opt_no_css,
opt_no_fonts,
opt_no_frames,
opt_no_js,
opt_no_images,
opt_silent,
);
let mut options = Options::default();
options.no_css = true;
options.no_frames = true;
options.no_js = true;
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
@@ -415,8 +282,47 @@ mod passing {
assert_eq!(
buf.iter().map(|&c| c as char).collect::<String>(),
"<html>\
<head><title>No integrity</title><link rel=\"something\"><script></script></head>\
<head><title>No integrity</title><link rel=\"something\"><script></script></head>\
<body></body>\
</html>"
);
}
#[test]
fn removes_unwanted_meta_tags() {
let html = "<html>\
<head>\
<meta http-equiv=\"Refresh\" value=\"20\"/>\
<meta http-equiv=\"Location\" value=\"https://freebsd.org\"/>\
</head>\
<body></body>\
</html>";
let dom = html::html_to_dom(&html);
let url = "http://localhost";
let cache = &mut HashMap::new();
let mut options = Options::default();
options.no_css = true;
options.no_frames = true;
options.no_js = true;
options.no_images = true;
options.silent = true;
let client = Client::new();
html::walk_and_embed_assets(cache, &client, &url, &dom.document, &options, 0);
let mut buf: Vec<u8> = Vec::new();
serialize(&mut buf, &dom.document, SerializeOpts::default()).unwrap();
assert_eq!(
buf.iter().map(|&c| c as char).collect::<String>(),
"<html>\
<head>\
<meta>\
<meta>\
</head>\
<body></body>\
</html>"
);
}

View File

@@ -3,4 +3,5 @@ mod css;
mod html;
mod js;
mod macros;
mod opts;
mod utils;

30
src/tests/opts.rs Normal file
View File

@@ -0,0 +1,30 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use crate::opts::Options;
#[test]
fn defaults() {
let options: Options = Options::default();
assert_eq!(options.target, str!());
assert_eq!(options.no_css, false);
assert_eq!(options.no_frames, false);
assert_eq!(options.no_fonts, false);
assert_eq!(options.no_images, false);
assert_eq!(options.isolate, false);
assert_eq!(options.no_js, false);
assert_eq!(options.insecure, false);
assert_eq!(options.no_metadata, false);
assert_eq!(options.output, str!());
assert_eq!(options.silent, false);
assert_eq!(options.timeout, 0);
assert_eq!(options.user_agent, "");
}
}

View File

@@ -7,12 +7,12 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn removes_fragment() {
assert_eq!(
utils::clean_url("https://somewhere.com/font.eot#iefix"),
url::clean_url("https://somewhere.com/font.eot#iefix"),
"https://somewhere.com/font.eot"
);
}
@@ -20,7 +20,7 @@ mod passing {
#[test]
fn removes_empty_fragment() {
assert_eq!(
utils::clean_url("https://somewhere.com/font.eot#"),
url::clean_url("https://somewhere.com/font.eot#"),
"https://somewhere.com/font.eot"
);
}
@@ -28,7 +28,7 @@ mod passing {
#[test]
fn removes_empty_query_and_empty_fragment() {
assert_eq!(
utils::clean_url("https://somewhere.com/font.eot?#"),
url::clean_url("https://somewhere.com/font.eot?#"),
"https://somewhere.com/font.eot"
);
}
@@ -36,7 +36,7 @@ mod passing {
#[test]
fn removes_empty_query_amp_and_empty_fragment() {
assert_eq!(
utils::clean_url("https://somewhere.com/font.eot?a=b&#"),
url::clean_url("https://somewhere.com/font.eot?a=b&#"),
"https://somewhere.com/font.eot?a=b"
);
}
@@ -44,7 +44,7 @@ mod passing {
#[test]
fn keeps_credentials() {
assert_eq!(
utils::clean_url("https://cookie:monster@gibson.internet/"),
url::clean_url("https://cookie:monster@gibson.internet/"),
"https://cookie:monster@gibson.internet/"
);
}

View File

@@ -7,13 +7,13 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn encode_string_with_specific_media_type() {
let mime = "application/javascript";
let data = "var word = 'hello';\nalert(word);\n";
let data_url = utils::data_to_data_url(mime, data.as_bytes(), "", "");
let data_url = url::data_to_data_url(mime, data.as_bytes(), "");
assert_eq!(
&data_url,
@@ -24,8 +24,8 @@ mod passing {
#[test]
fn encode_append_fragment() {
let data = "<svg></svg>\n";
let data_url = utils::data_to_data_url("text/css", data.as_bytes(), "", "fragment");
let data_url = url::data_to_data_url("image/svg+xml", data.as_bytes(), "");
assert_eq!(&data_url, "data:text/css;base64,PHN2Zz48L3N2Zz4K#fragment");
assert_eq!(&data_url, "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K");
}
}

View File

@@ -7,11 +7,11 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn parse_text_html_base64() {
let (media_type, data) = utils::data_url_to_data("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
let (media_type, data) = url::data_url_to_data("data:text/html;base64,V29yayBleHBhbmRzIHNvIGFzIHRvIGZpbGwgdGhlIHRpbWUgYXZhaWxhYmxlIGZvciBpdHMgY29tcGxldGlvbg==");
assert_eq!(media_type, "text/html");
assert_eq!(
@@ -22,7 +22,7 @@ mod passing {
#[test]
fn parse_text_html_utf8() {
let (media_type, data) = utils::data_url_to_data(
let (media_type, data) = url::data_url_to_data(
"data:text/html;utf8,Work expands so as to fill the time available for its completion",
);
@@ -35,7 +35,7 @@ mod passing {
#[test]
fn parse_text_html_plaintext() {
let (media_type, data) = utils::data_url_to_data(
let (media_type, data) = url::data_url_to_data(
"data:text/html,Work expands so as to fill the time available for its completion",
);
@@ -48,7 +48,7 @@ mod passing {
#[test]
fn parse_text_html_charset_utf_8_between_two_whitespaces() {
let (media_type, data) = utils::data_url_to_data(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
let (media_type, data) = url::data_url_to_data(" data:text/html;charset=utf-8,Work expands so as to fill the time available for its completion ");
assert_eq!(media_type, "text/html");
assert_eq!(
@@ -60,7 +60,7 @@ mod passing {
#[test]
fn parse_text_css_url_encoded() {
let (media_type, data) =
utils::data_url_to_data("data:text/css,div{background-color:%23000}");
url::data_url_to_data("data:text/css,div{background-color:%23000}");
assert_eq!(media_type, "text/css");
assert_eq!(String::from_utf8_lossy(&data), "div{background-color:#000}");
@@ -68,7 +68,7 @@ mod passing {
#[test]
fn parse_no_media_type_base64() {
let (media_type, data) = utils::data_url_to_data("data:;base64,dGVzdA==");
let (media_type, data) = url::data_url_to_data("data:;base64,dGVzdA==");
assert_eq!(media_type, "");
assert_eq!(String::from_utf8_lossy(&data), "test");
@@ -76,7 +76,7 @@ mod passing {
#[test]
fn parse_no_media_type_no_encoding() {
let (media_type, data) = utils::data_url_to_data("data:;,test%20test");
let (media_type, data) = url::data_url_to_data("data:;,test%20test");
assert_eq!(media_type, "");
assert_eq!(String::from_utf8_lossy(&data), "test test");
@@ -92,11 +92,11 @@ mod passing {
#[cfg(test)]
mod failing {
use crate::utils;
use crate::url;
#[test]
fn just_word_data() {
let (media_type, data) = utils::data_url_to_data("data");
let (media_type, data) = url::data_url_to_data("data");
assert_eq!(media_type, "");
assert_eq!(String::from_utf8_lossy(&data), "");

View File

@@ -7,12 +7,12 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn decode_unicode_characters() {
assert_eq!(
utils::decode_url(str!(
url::decode_url(str!(
"%E6%A4%9C%E3%83%92%E3%83%A0%E8%A7%A3%E5%A1%97%E3%82%83%E3%83%83%20%3D%20%E3%82%B5"
)),
"検ヒム解塗ゃッ = サ"
@@ -22,7 +22,7 @@ mod passing {
#[test]
fn decode_file_url() {
assert_eq!(
utils::decode_url(str!("file:///tmp/space%20here/test%231.html")),
url::decode_url(str!("file:///tmp/space%20here/test%231.html")),
"file:///tmp/space here/test#1.html"
);
}
@@ -30,7 +30,7 @@ mod passing {
#[test]
fn plus_sign() {
assert_eq!(
utils::decode_url(str!(
url::decode_url(str!(
"fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"
)),
"fonts.somewhere.com/css?family=Open+Sans:300,400,400italic,600,600italic"

View File

@@ -7,18 +7,18 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn remove_protocl_and_fragment() {
if cfg!(windows) {
assert_eq!(
utils::file_url_to_fs_path("file:///C:/documents/some-path/some-file.svg#fragment"),
url::file_url_to_fs_path("file:///C:/documents/some-path/some-file.svg#fragment"),
"C:\\documents\\some-path\\some-file.svg"
);
} else {
assert_eq!(
utils::file_url_to_fs_path("file:///tmp/some-path/some-file.svg#fragment"),
url::file_url_to_fs_path("file:///tmp/some-path/some-file.svg#fragment"),
"/tmp/some-path/some-file.svg"
);
}
@@ -28,12 +28,12 @@ mod passing {
fn decodes_urls() {
if cfg!(windows) {
assert_eq!(
utils::file_url_to_fs_path("file:///C:/Documents%20and%20Settings/some-file.html"),
url::file_url_to_fs_path("file:///C:/Documents%20and%20Settings/some-file.html"),
"C:\\Documents and Settings\\some-file.html"
);
} else {
assert_eq!(
utils::file_url_to_fs_path("file:///home/user/My%20Documents"),
url::file_url_to_fs_path("file:///home/user/My%20Documents"),
"/home/user/My Documents"
);
}

View File

@@ -7,12 +7,12 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn data_url() {
assert_eq!(
utils::get_url_fragment(
url::get_url_fragment(
"data:image/svg+xml;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h#test"
),
"test"
@@ -21,6 +21,6 @@ mod passing {
#[test]
fn https_empty() {
assert_eq!(utils::get_url_fragment("https://kernel.org#"), "");
assert_eq!(url::get_url_fragment("https://kernel.org#"), "");
}
}

View File

@@ -7,18 +7,18 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn data_url_text_html() {
assert!(utils::is_data_url(
assert!(url::is_data_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
));
}
#[test]
fn data_url_no_media_type() {
assert!(utils::is_data_url(
assert!(url::is_data_url(
"data:;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
));
}
@@ -33,20 +33,20 @@ mod passing {
#[cfg(test)]
mod failing {
use crate::utils;
use crate::url;
#[test]
fn https_url() {
assert!(!utils::is_data_url("https://kernel.org"));
assert!(!url::is_data_url("https://kernel.org"));
}
#[test]
fn no_protocol_url() {
assert!(!utils::is_data_url("//kernel.org"));
assert!(!url::is_data_url("//kernel.org"));
}
#[test]
fn empty_string() {
assert!(!utils::is_data_url(""));
assert!(!url::is_data_url(""));
}
}

View File

@@ -7,32 +7,32 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn unix_file_url() {
assert!(utils::is_file_url(
assert!(url::is_file_url(
"file:///home/user/Websites/my-website/index.html"
));
}
#[test]
fn windows_file_url() {
assert!(utils::is_file_url(
assert!(url::is_file_url(
"file:///C:/Documents%20and%20Settings/user/Websites/my-website/assets/images/logo.png"
));
}
#[test]
fn unix_url_with_backslashes() {
assert!(utils::is_file_url(
assert!(url::is_file_url(
"file:\\\\\\home\\user\\Websites\\my-website\\index.html"
));
}
#[test]
fn windows_file_url_with_backslashes() {
assert!(utils::is_file_url(
assert!(url::is_file_url(
"file:\\\\\\C:\\Documents%20and%20Settings\\user\\Websites\\my-website\\assets\\images\\logo.png"
));
}
@@ -47,37 +47,37 @@ mod passing {
#[cfg(test)]
mod failing {
use crate::utils;
use crate::url;
#[test]
fn url_with_no_protocl() {
assert!(!utils::is_file_url("//kernel.org"));
assert!(!url::is_file_url("//kernel.org"));
}
#[test]
fn dot_slash_filename() {
assert!(!utils::is_file_url("./index.html"));
assert!(!url::is_file_url("./index.html"));
}
#[test]
fn just_filename() {
assert!(!utils::is_file_url("some-local-page.htm"));
assert!(!url::is_file_url("some-local-page.htm"));
}
#[test]
fn https_ip_port_url() {
assert!(!utils::is_file_url("https://1.2.3.4:80/www/index.html"));
assert!(!url::is_file_url("https://1.2.3.4:80/www/index.html"));
}
#[test]
fn data_url() {
assert!(!utils::is_file_url(
assert!(!url::is_file_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
));
}
#[test]
fn just_word_file() {
assert!(!utils::is_file_url("file"));
assert!(!url::is_file_url("file"));
}
}

View File

@@ -7,21 +7,21 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn http_url() {
assert!(utils::is_http_url("http://kernel.org"));
assert!(url::is_http_url("http://kernel.org"));
}
#[test]
fn https_url() {
assert!(utils::is_http_url("https://www.rust-lang.org/"));
assert!(url::is_http_url("https://www.rust-lang.org/"));
}
#[test]
fn http_url_with_backslashes() {
assert!(utils::is_http_url("http:\\\\freebsd.org\\"));
assert!(url::is_http_url("http:\\\\freebsd.org\\"));
}
}
@@ -34,31 +34,31 @@ mod passing {
#[cfg(test)]
mod failing {
use crate::utils;
use crate::url;
#[test]
fn url_with_no_protocol() {
assert!(!utils::is_http_url("//kernel.org"));
assert!(!url::is_http_url("//kernel.org"));
}
#[test]
fn dot_slash_filename() {
assert!(!utils::is_http_url("./index.html"));
assert!(!url::is_http_url("./index.html"));
}
#[test]
fn just_filename() {
assert!(!utils::is_http_url("some-local-page.htm"));
assert!(!url::is_http_url("some-local-page.htm"));
}
#[test]
fn https_ip_port_url() {
assert!(!utils::is_http_url("ftp://1.2.3.4/www/index.html"));
assert!(!url::is_http_url("ftp://1.2.3.4/www/index.html"));
}
#[test]
fn data_url() {
assert!(!utils::is_http_url(
assert!(!url::is_http_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h"
));
}

12
src/tests/url/mod.rs Normal file
View File

@@ -0,0 +1,12 @@
mod clean_url;
mod data_to_data_url;
mod data_url_to_data;
mod decode_url;
mod file_url_to_fs_path;
mod get_url_fragment;
mod is_data_url;
mod is_file_url;
mod is_http_url;
mod resolve_url;
mod url_has_protocol;
mod url_with_fragment;

View File

@@ -7,13 +7,14 @@
#[cfg(test)]
mod passing {
use crate::utils;
use url::ParseError;
use crate::url;
#[test]
fn from_https_to_level_up_relative() -> Result<(), ParseError> {
let resolved_url =
utils::resolve_url("https://www.kernel.org", "../category/signatures.html")?;
url::resolve_url("https://www.kernel.org", "../category/signatures.html")?;
assert_eq!(
resolved_url.as_str(),
@@ -25,7 +26,7 @@ mod passing {
#[test]
fn from_just_filename_to_full_https_url() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"saved_page.htm",
"https://www.kernel.org/category/signatures.html",
)?;
@@ -40,7 +41,7 @@ mod passing {
#[test]
fn from_https_url_to_url_with_no_protocol() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"https://www.kernel.org",
"//www.kernel.org/theme/images/logos/tux.png",
)?;
@@ -56,7 +57,7 @@ mod passing {
#[test]
fn from_https_url_to_url_with_no_protocol_and_on_different_hostname() -> Result<(), ParseError>
{
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"https://www.kernel.org",
"//another-host.org/theme/images/logos/tux.png",
)?;
@@ -71,7 +72,7 @@ mod passing {
#[test]
fn from_https_url_to_relative_root_path() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"https://www.kernel.org/category/signatures.html",
"/theme/images/logos/tux.png",
)?;
@@ -86,7 +87,7 @@ mod passing {
#[test]
fn from_https_to_just_filename() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"https://www.w3schools.com/html/html_iframe.asp",
"default.asp",
)?;
@@ -101,7 +102,7 @@ mod passing {
#[test]
fn from_data_url_to_https() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
"https://www.kernel.org/category/signatures.html",
)?;
@@ -116,7 +117,7 @@ mod passing {
#[test]
fn from_data_url_to_data_url() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
"data:text/html;base64,PGEgaHJlZj0iaW5kZXguaHRtbCI+SG9tZTwvYT4K",
)?;
@@ -131,7 +132,7 @@ mod passing {
#[test]
fn from_file_url_to_relative_path() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"file:///home/user/Websites/my-website/index.html",
"assets/images/logo.png",
)
@@ -147,7 +148,7 @@ mod passing {
#[test]
fn from_file_url_to_relative_path_with_backslashes() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"file:\\\\\\home\\user\\Websites\\my-website\\index.html",
"assets\\images\\logo.png",
)
@@ -163,7 +164,7 @@ mod passing {
#[test]
fn from_data_url_to_file_url() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
"file:///etc/passwd",
)
@@ -176,7 +177,7 @@ mod passing {
#[test]
fn preserve_fragment() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"http://doesnt-matter.local/",
"css/fonts/fontmarvelous.svg#fontmarvelous",
)
@@ -193,9 +194,9 @@ mod passing {
#[test]
fn resolve_from_file_url_to_file_url() -> Result<(), ParseError> {
let resolved_url = if cfg!(windows) {
utils::resolve_url("file:///c:/index.html", "file:///c:/image.png").unwrap_or(str!())
url::resolve_url("file:///c:/index.html", "file:///c:/image.png").unwrap_or(str!())
} else {
utils::resolve_url("file:///tmp/index.html", "file:///tmp/image.png").unwrap_or(str!())
url::resolve_url("file:///tmp/index.html", "file:///tmp/image.png").unwrap_or(str!())
};
assert_eq!(
@@ -220,12 +221,12 @@ mod passing {
#[cfg(test)]
mod failing {
use crate::utils;
use crate::url;
use url::ParseError;
#[test]
fn from_data_url_to_url_with_no_protocol() -> Result<(), ParseError> {
let resolved_url = utils::resolve_url(
let resolved_url = url::resolve_url(
"data:text/html;base64,V2VsY29tZSBUbyBUaGUgUGFydHksIDxiPlBhbDwvYj4h",
"//www.w3schools.com/html/html_iframe.asp",
)

View File

@@ -7,50 +7,50 @@
#[cfg(test)]
mod passing {
use crate::utils;
use crate::url;
#[test]
fn mailto() {
assert!(utils::url_has_protocol(
assert!(url::url_has_protocol(
"mailto:somebody@somewhere.com?subject=hello"
));
}
#[test]
fn tel() {
assert!(utils::url_has_protocol("tel:5551234567"));
assert!(url::url_has_protocol("tel:5551234567"));
}
#[test]
fn ftp_no_slashes() {
assert!(utils::url_has_protocol("ftp:some-ftp-server.com"));
assert!(url::url_has_protocol("ftp:some-ftp-server.com"));
}
#[test]
fn ftp_with_credentials() {
assert!(utils::url_has_protocol(
assert!(url::url_has_protocol(
"ftp://user:password@some-ftp-server.com"
));
}
#[test]
fn javascript() {
assert!(utils::url_has_protocol("javascript:void(0)"));
assert!(url::url_has_protocol("javascript:void(0)"));
}
#[test]
fn http() {
assert!(utils::url_has_protocol("http://news.ycombinator.com"));
assert!(url::url_has_protocol("http://news.ycombinator.com"));
}
#[test]
fn https() {
assert!(utils::url_has_protocol("https://github.com"));
assert!(url::url_has_protocol("https://github.com"));
}
#[test]
fn mailto_uppercase() {
assert!(utils::url_has_protocol(
assert!(url::url_has_protocol(
"MAILTO:somebody@somewhere.com?subject=hello"
));
}
@@ -69,23 +69,23 @@ mod failing {
#[test]
fn url_with_no_protocol() {
assert!(!utils::url_has_protocol(
assert!(!url::url_has_protocol(
"//some-hostname.com/some-file.html"
));
}
#[test]
fn relative_path() {
assert!(!utils::url_has_protocol("some-hostname.com/some-file.html"));
assert!(!url::url_has_protocol("some-hostname.com/some-file.html"));
}
#[test]
fn relative_to_root_path() {
assert!(!utils::url_has_protocol("/some-file.html"));
assert!(!url::url_has_protocol("/some-file.html"));
}
#[test]
fn empty_string() {
assert!(!utils::url_has_protocol(""));
assert!(!url::url_has_protocol(""));
}
}

View File

@@ -0,0 +1,40 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use crate::url;
#[test]
fn url_with_fragment_url() {
let url = "https://localhost.localdomain/path/";
let fragment = "test";
let assembled_url = url::url_with_fragment(url, fragment);
assert_eq!(&assembled_url, "https://localhost.localdomain/path/#test");
}
#[test]
fn url_with_fragment_empty_url() {
let url = "https://localhost.localdomain/path/";
let fragment = "";
let assembled_url = url::url_with_fragment(url, fragment);
assert_eq!(&assembled_url, "https://localhost.localdomain/path/");
}
#[test]
fn url_with_fragment_data_url() {
let url = "data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K";
let fragment = "fragment";
let assembled_url = url::url_with_fragment(url, fragment);
assert_eq!(
&assembled_url,
"data:image/svg+xml;base64,PHN2Zz48L3N2Zz4K#fragment"
);
}
}

31
src/tests/utils/indent.rs Normal file
View File

@@ -0,0 +1,31 @@
// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗
// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝
// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗
// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║
// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝
// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝
#[cfg(test)]
mod passing {
use crate::utils;
#[test]
fn zero() {
assert_eq!(utils::indent(0), "");
}
#[test]
fn one() {
assert_eq!(utils::indent(1), " ");
}
#[test]
fn two() {
assert_eq!(utils::indent(2), " ");
}
#[test]
fn three() {
assert_eq!(utils::indent(3), " ");
}
}

View File

@@ -1,13 +1,3 @@
mod clean_url;
mod data_to_data_url;
mod data_url_to_data;
mod decode_url;
mod detect_media_type;
mod file_url_to_fs_path;
mod get_url_fragment;
mod is_data_url;
mod is_file_url;
mod is_http_url;
mod resolve_url;
mod indent;
mod retrieve_asset;
mod url_has_protocol;

View File

@@ -7,11 +7,13 @@
#[cfg(test)]
mod passing {
use crate::utils;
use reqwest::blocking::Client;
use std::collections::HashMap;
use std::env;
use crate::url;
use crate::utils;
#[test]
fn read_data_url() {
let cache = &mut HashMap::new();
@@ -25,15 +27,16 @@ mod passing {
"data:text/html;base64,c291cmNl",
"data:text/html;base64,dGFyZ2V0",
false,
0,
)
.unwrap();
assert_eq!(
utils::data_to_data_url(&media_type, &data, &final_url, ""),
utils::data_to_data_url("text/html", "target".as_bytes(), "", "")
url::data_to_data_url(&media_type, &data, &final_url),
url::data_to_data_url("text/html", "target".as_bytes(), "")
);
assert_eq!(
final_url,
utils::data_to_data_url("text/html", "target".as_bytes(), "", "")
url::data_to_data_url("text/html", "target".as_bytes(), "")
);
assert_eq!(&media_type, "text/html");
}
@@ -61,9 +64,10 @@ mod passing {
cwd = cwd.to_str().unwrap()
),
false,
0,
)
.unwrap();
assert_eq!(utils::data_to_data_url("application/javascript", &data, &final_url, ""), "data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==");
assert_eq!(url::data_to_data_url("application/javascript", &data, &final_url), "data:application/javascript;base64,ZG9jdW1lbnQuYm9keS5zdHlsZS5iYWNrZ3JvdW5kQ29sb3IgPSAiZ3JlZW4iOwpkb2N1bWVudC5ib2R5LnN0eWxlLmNvbG9yID0gInJlZCI7Cg==");
assert_eq!(
&final_url,
&format!(
@@ -84,10 +88,11 @@ mod passing {
#[cfg(test)]
mod failing {
use crate::utils;
use reqwest::blocking::Client;
use std::collections::HashMap;
use crate::utils;
#[test]
fn read_local_file_with_data_url_parent() {
let cache = &mut HashMap::new();
@@ -100,6 +105,7 @@ mod failing {
"data:text/html;base64,SoUrCe",
"file:///etc/passwd",
false,
0,
) {
Ok((..)) => {
assert!(false);
@@ -122,6 +128,7 @@ mod failing {
"https://kernel.org/",
"file:///etc/passwd",
false,
0,
) {
Ok((..)) => {
assert!(false);

168
src/url.rs Normal file
View File

@@ -0,0 +1,168 @@
use base64;
use url::{form_urlencoded, ParseError, Url};
use crate::utils::detect_media_type;
pub fn clean_url<T: AsRef<str>>(input: T) -> String {
let mut url = Url::parse(input.as_ref()).unwrap();
// Clear fragment
url.set_fragment(None);
// Get rid of stray question mark
if url.query() == Some("") {
url.set_query(None);
}
// Remove empty trailing ampersand(s)
let mut result: String = url.to_string();
while result.ends_with("&") {
result.pop();
}
result
}
pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str) -> String {
let media_type: String = if media_type.is_empty() {
detect_media_type(data, &url)
} else {
media_type.to_string()
};
format!("data:{};base64,{}", media_type, base64::encode(data))
}
pub fn data_url_to_data<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
let path: String = parsed_url.path().to_string();
let comma_loc: usize = path.find(',').unwrap_or(path.len());
let meta_data: String = path.chars().take(comma_loc).collect();
let raw_data: String = path.chars().skip(comma_loc + 1).collect();
let text: String = decode_url(raw_data);
let meta_data_items: Vec<&str> = meta_data.split(';').collect();
let mut media_type: String = str!();
let mut encoding: &str = "";
let mut i: i8 = 0;
for item in &meta_data_items {
if i == 0 {
media_type = str!(item);
} else {
if item.eq_ignore_ascii_case("base64")
|| item.eq_ignore_ascii_case("utf8")
|| item.eq_ignore_ascii_case("charset=UTF-8")
{
encoding = item;
}
}
i = i + 1;
}
let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
base64::decode(&text).unwrap_or(vec![])
} else {
text.as_bytes().to_vec()
};
(media_type, data)
}
pub fn decode_url(input: String) -> String {
let input: String = input.replace("+", "%2B");
form_urlencoded::parse(input.as_bytes())
.map(|(key, val)| {
[
key.to_string(),
if val.to_string().len() == 0 {
str!()
} else {
str!('=')
},
val.to_string(),
]
.concat()
})
.collect()
}
pub fn file_url_to_fs_path(url: &str) -> String {
if !is_file_url(url) {
return str!();
}
let cutoff_l = if cfg!(windows) { 8 } else { 7 };
let mut fs_file_path: String = decode_url(url.to_string()[cutoff_l..].to_string());
let url_fragment = get_url_fragment(url);
if url_fragment != "" {
let max_len = fs_file_path.len() - 1 - url_fragment.len();
fs_file_path = fs_file_path[0..max_len].to_string();
}
if cfg!(windows) {
fs_file_path = fs_file_path.replace("/", "\\");
}
// File paths should not be %-encoded
decode_url(fs_file_path)
}
pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
if Url::parse(url.as_ref()).unwrap().fragment() == None {
str!()
} else {
str!(Url::parse(url.as_ref()).unwrap().fragment().unwrap())
}
}
pub fn is_data_url<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme() == "data"))
.unwrap_or(false)
}
pub fn is_file_url<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme() == "file"))
.unwrap_or(false)
}
pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme() == "http" || u.scheme() == "https"))
.unwrap_or(false)
}
pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
let result = if is_http_url(to.as_ref()) {
to.as_ref().to_string()
} else {
Url::parse(from.as_ref())?
.join(to.as_ref())?
.as_ref()
.to_string()
};
Ok(result)
}
pub fn url_has_protocol<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme().len() > 0))
.unwrap_or(false)
}
pub fn url_with_fragment(url: &str, fragment: &str) -> String {
let mut result = str!(&url);
if !fragment.is_empty() {
result += "#";
result += fragment;
}
result
}

View File

@@ -1,10 +1,12 @@
use base64;
use reqwest::blocking::Client;
use reqwest::header::CONTENT_TYPE;
use std::collections::HashMap;
use std::fs;
use std::path::Path;
use url::{form_urlencoded, ParseError, Url};
use crate::url::{clean_url, data_url_to_data, file_url_to_fs_path, is_data_url, is_file_url};
const INDENT: &str = " ";
const MAGIC: [[&[u8]; 2]; 18] = [
// Image
@@ -38,26 +40,6 @@ const PLAINTEXT_MEDIA_TYPES: &[&str] = &[
"text/plain",
];
pub fn data_to_data_url(media_type: &str, data: &[u8], url: &str, fragment: &str) -> String {
let media_type: String = if media_type.is_empty() {
detect_media_type(data, &url)
} else {
media_type.to_string()
};
let hash: String = if fragment != "" {
format!("#{}", fragment)
} else {
str!()
};
format!(
"data:{};base64,{}{}",
media_type,
base64::encode(data),
hash
)
}
pub fn detect_media_type(data: &[u8], url: &str) -> String {
for item in MAGIC.iter() {
if data.starts_with(item[0]) {
@@ -72,159 +54,27 @@ pub fn detect_media_type(data: &[u8], url: &str) -> String {
str!()
}
pub fn url_has_protocol<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme().len() > 0))
.unwrap_or(false)
}
pub fn is_data_url<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme() == "data"))
.unwrap_or(false)
}
pub fn is_file_url<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme() == "file"))
.unwrap_or(false)
}
pub fn is_http_url<T: AsRef<str>>(url: T) -> bool {
Url::parse(url.as_ref())
.and_then(|u| Ok(u.scheme() == "http" || u.scheme() == "https"))
.unwrap_or(false)
}
pub fn is_plaintext_media_type(media_type: &str) -> bool {
PLAINTEXT_MEDIA_TYPES.contains(&media_type.to_lowercase().as_str())
}
pub fn resolve_url<T: AsRef<str>, U: AsRef<str>>(from: T, to: U) -> Result<String, ParseError> {
let result = if is_http_url(to.as_ref()) {
to.as_ref().to_string()
} else {
Url::parse(from.as_ref())?
.join(to.as_ref())?
.as_ref()
.to_string()
};
Ok(result)
}
pub fn get_url_fragment<T: AsRef<str>>(url: T) -> String {
if Url::parse(url.as_ref()).unwrap().fragment() == None {
str!()
} else {
str!(Url::parse(url.as_ref()).unwrap().fragment().unwrap())
pub fn indent(level: u32) -> String {
let mut result = str!();
let mut l: u32 = level;
while l > 0 {
result += INDENT;
l -= 1;
}
}
pub fn clean_url<T: AsRef<str>>(input: T) -> String {
let mut url = Url::parse(input.as_ref()).unwrap();
// Clear fragment
url.set_fragment(None);
// Get rid of stray question mark
if url.query() == Some("") {
url.set_query(None);
}
// Remove empty trailing ampersand(s)
let mut result: String = url.to_string();
while result.ends_with("&") {
result.pop();
}
result
}
pub fn data_url_to_data<T: AsRef<str>>(url: T) -> (String, Vec<u8>) {
let parsed_url: Url = Url::parse(url.as_ref()).unwrap_or(Url::parse("data:,").unwrap());
let path: String = parsed_url.path().to_string();
let comma_loc: usize = path.find(',').unwrap_or(path.len());
let meta_data: String = path.chars().take(comma_loc).collect();
let raw_data: String = path.chars().skip(comma_loc + 1).collect();
let text: String = decode_url(raw_data);
let meta_data_items: Vec<&str> = meta_data.split(';').collect();
let mut media_type: String = str!();
let mut encoding: &str = "";
let mut i: i8 = 0;
for item in &meta_data_items {
if i == 0 {
media_type = str!(item);
} else {
if item.eq_ignore_ascii_case("base64")
|| item.eq_ignore_ascii_case("utf8")
|| item.eq_ignore_ascii_case("charset=UTF-8")
{
encoding = item;
}
}
i = i + 1;
}
let data: Vec<u8> = if encoding.eq_ignore_ascii_case("base64") {
base64::decode(&text).unwrap_or(vec![])
} else {
text.as_bytes().to_vec()
};
(media_type, data)
}
pub fn decode_url(input: String) -> String {
let input: String = input.replace("+", "%2B");
form_urlencoded::parse(input.as_bytes())
.map(|(key, val)| {
[
key.to_string(),
if val.to_string().len() == 0 {
str!()
} else {
str!('=')
},
val.to_string(),
]
.concat()
})
.collect()
}
pub fn file_url_to_fs_path(url: &str) -> String {
if !is_file_url(url) {
return str!();
}
let cutoff_l = if cfg!(windows) { 8 } else { 7 };
let mut fs_file_path: String = decode_url(url.to_string()[cutoff_l..].to_string());
let url_fragment = get_url_fragment(url);
if url_fragment != "" {
let max_len = fs_file_path.len() - 1 - url_fragment.len();
fs_file_path = fs_file_path[0..max_len].to_string();
}
if cfg!(windows) {
fs_file_path = fs_file_path.replace("/", "\\");
}
// File paths should not be %-encoded
decode_url(fs_file_path)
}
pub fn retrieve_asset(
cache: &mut HashMap<String, Vec<u8>>,
client: &Client,
parent_url: &str,
url: &str,
opt_silent: bool,
depth: u32,
) -> Result<(Vec<u8>, String, String), reqwest::Error> {
if url.len() == 0 {
// Provoke error
@@ -246,7 +96,7 @@ pub fn retrieve_asset(
let path = Path::new(&fs_file_path);
if path.exists() {
if !opt_silent {
eprintln!("{}", &url);
eprintln!("{}{}", indent(depth).as_str(), &url);
}
Ok((fs::read(&fs_file_path).expect(""), url.to_string(), str!()))
@@ -260,7 +110,7 @@ pub fn retrieve_asset(
if cache.contains_key(&cache_key) {
// URL is in cache, we get and return it
if !opt_silent {
eprintln!("{} (from cache)", &url);
eprintln!("{}{} (from cache)", indent(depth).as_str(), &url);
}
Ok((
@@ -275,9 +125,9 @@ pub fn retrieve_asset(
if !opt_silent {
if url == res_url {
eprintln!("{}", &url);
eprintln!("{}{}", indent(depth).as_str(), &url);
} else {
eprintln!("{} -> {}", &url, &res_url);
eprintln!("{}{} -> {}", indent(depth).as_str(), &url, &res_url);
}
}