Skip to content

Various improvements and additions #434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
snshn merged 3 commits into from
Mar 10, 2025
Merged

Various improvements and additions #434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e7b6bbe
implement stub for supporting various output formats
snshn Mar 10, 2025
e2f0bf8
add functions that handle error and info output
snshn Mar 10, 2025
c3429ec
make it possible to lint staged files
snshn Mar 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ install:
.PHONY: install

lint:
@cargo clippy --fix --allow-dirty
@cargo clippy --fix --allow-dirty --allow-staged
.PHONY: lint

lint_check:
Expand Down
170 changes: 74 additions & 96 deletions src/core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,16 @@ impl Error for MonolithError {
}
}

#[derive(Debug, PartialEq, Eq, Default)]
pub enum MonolithOutputFormat {
#[default]
HTML,
// MHT,
// WARC,
// ZIM,
// HAR,
}

#[derive(Default)]
pub struct Options {
pub base_url: Option<String>,
Expand All @@ -63,6 +73,7 @@ pub struct Options {
pub no_js: bool,
pub no_metadata: bool,
pub no_video: bool,
pub output_format: MonolithOutputFormat,
pub silent: bool,
pub timeout: u64,
pub unwrap_noscript: bool,
Expand Down Expand Up @@ -333,17 +344,21 @@ pub fn create_monolithic_document(
dom = set_charset(dom, document_encoding.clone());
}

// Serialize DOM tree
let mut result: Vec<u8> = serialize_document(dom, document_encoding, options);
if options.output_format == MonolithOutputFormat::HTML {
// Serialize DOM tree
let mut result: Vec<u8> = serialize_document(dom, document_encoding, options);

// Prepend metadata comment tag
if !options.no_metadata {
let mut metadata_comment: String = create_metadata_tag(&target_url);
metadata_comment += "\n";
result.splice(0..0, metadata_comment.as_bytes().to_vec());
}
// Prepend metadata comment tag
if !options.no_metadata {
let mut metadata_comment: String = create_metadata_tag(&target_url);
metadata_comment += "\n";
result.splice(0..0, metadata_comment.as_bytes().to_vec());
}

Ok(result)
Ok(result)
} else {
Ok(vec![])
}
}

pub fn detect_media_type(data: &[u8], url: &Url) -> String {
Expand Down Expand Up @@ -498,18 +513,8 @@ pub fn retrieve_asset(
} else if url.scheme() == "file" {
// Check if parent_url is also a file: URL (if not, then we don't embed the asset)
if parent_url.scheme() != "file" {
if !options.silent {
eprintln!(
"{}{} (Security Error){}",
if options.no_color { "" } else { ANSI_COLOR_RED },
&url,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
print_error_message(&format!("{} (security error)", &url), options);

// Provoke error
client.get("").send()?;
}
Expand All @@ -518,27 +523,14 @@ pub fn retrieve_asset(
let path: &Path = path_buf.as_path();
if path.exists() {
if path.is_dir() {
if !options.silent {
eprintln!(
"{}{} (is a directory){}",
if options.no_color { "" } else { ANSI_COLOR_RED },
&url,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
print_error_message(&format!("{} (is a directory)", &url), options);

// Provoke error
Err(client.get("").send().unwrap_err())
} else {
if !options.silent {
eprintln!("{}", &url);
}
print_info_message(&format!("{}", &url), options);

let file_blob: Vec<u8> = fs::read(path).expect("Unable to read file");
let file_blob: Vec<u8> = fs::read(path).expect("unable to read file");

Ok((
file_blob.clone(),
Expand All @@ -548,18 +540,7 @@ pub fn retrieve_asset(
))
}
} else {
if !options.silent {
eprintln!(
"{}{} (not found){}",
if options.no_color { "" } else { ANSI_COLOR_RED },
&url,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
print_error_message(&format!("{} (not found)", &url), options);

// Provoke error
Err(client.get("").send().unwrap_err())
Expand All @@ -569,9 +550,7 @@ pub fn retrieve_asset(

if cache.is_some() && cache.as_ref().unwrap().contains_key(&cache_key) {
// URL is in cache, we get and return it
if !options.silent {
eprintln!("{} (from cache)", &url);
}
print_info_message(&format!("{} (from cache)", &url), options);

Ok((
cache.as_ref().unwrap().get(&cache_key).unwrap().0.to_vec(),
Expand Down Expand Up @@ -612,31 +591,18 @@ pub fn retrieve_asset(
match client.get(url.as_str()).headers(headers).send() {
Ok(response) => {
if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {
if !options.silent {
eprintln!(
"{}{} ({}){}",
if options.no_color { "" } else { ANSI_COLOR_RED },
&url,
response.status(),
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
print_error_message(&format!("{} ({})", &url, response.status()), options);

// Provoke error
return Err(client.get("").send().unwrap_err());
}

let response_url: Url = response.url().clone();

if !options.silent {
if url.as_str() == response_url.as_str() {
eprintln!("{}", &url);
} else {
eprintln!("{} -> {}", &url, &response_url);
}
if url.as_str() == response_url.as_str() {
print_info_message(&format!("{}", &url), options);
} else {
print_info_message(&format!("{} -> {}", &url, &response_url), options);
}

let new_cache_key: String = clean_url(response_url.clone()).to_string();
Expand All @@ -657,18 +623,7 @@ pub fn retrieve_asset(
data = b.to_vec();
}
Err(error) => {
if !options.silent {
eprintln!(
"{}{}{}",
if options.no_color { "" } else { ANSI_COLOR_RED },
error,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
print_error_message(&format!("{}", error), options);
}
}

Expand All @@ -686,19 +641,7 @@ pub fn retrieve_asset(
Ok((data, response_url, media_type, charset))
}
Err(error) => {
if !options.silent {
eprintln!(
"{}{} ({}){}",
if options.no_color { "" } else { ANSI_COLOR_RED },
&url,
error,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
);
}
print_error_message(&format!("{} ({})", &url, error), options);

Err(client.get("").send().unwrap_err())
}
Expand All @@ -715,3 +658,38 @@ pub fn read_stdin() -> Vec<u8> {
Err(_) => buffer,
}
}

use std::io::Write;

pub fn print_error_message(text: &str, options: &Options) {
if !options.silent {
let stderr = io::stderr();
let mut handle = stderr.lock();

if handle
.write_all(
format!(
"{}{}{}\n",
if options.no_color { "" } else { ANSI_COLOR_RED },
&text,
if options.no_color {
""
} else {
ANSI_COLOR_RESET
},
)
.as_bytes(),
)
.is_ok()
{}
}
}

pub fn print_info_message(text: &str, options: &Options) {
if !options.silent {
let stderr = io::stderr();
let mut handle = stderr.lock();

if handle.write_all(format!("{}\n", &text).as_bytes()).is_ok() {}
}
}
28 changes: 19 additions & 9 deletions src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ use tempfile::Builder;

use monolith::cache::Cache;
use monolith::cookies::parse_cookie_file_contents;
use monolith::core::{create_monolithic_document, Options};
use monolith::core::{create_monolithic_document, print_error_message, Options};

enum Output {
Stdout(io::Stdout),
Expand Down Expand Up @@ -193,18 +193,30 @@ fn main() {

// Read and parse cookie file
if let Some(opt_cookie_file) = cookie_file_path.clone() {
match fs::read_to_string(opt_cookie_file) {
match fs::read_to_string(&opt_cookie_file) {
Ok(str) => match parse_cookie_file_contents(&str) {
Ok(parsed_cookies_from_file) => {
options.cookies = parsed_cookies_from_file;
}
Err(_) => {
eprintln!("Could not parse specified cookie file");
print_error_message(
&format!(
"could not parse specified cookie file \"{}\"",
opt_cookie_file
),
&options,
);
process::exit(1);
}
},
Err(_) => {
eprintln!("Could not read specified cookie file");
print_error_message(
&format!(
"could not read specified cookie file \"{}\"",
opt_cookie_file
),
&options,
);
process::exit(1);
}
}
Expand All @@ -213,15 +225,13 @@ fn main() {
match create_monolithic_document(source, &options, &mut Some(cache)) {
Ok(result) => {
// Define output
let mut output = Output::new(&destination).expect("Could not prepare output");
let mut output = Output::new(&destination).expect("could not prepare output");

// Write result into STDOUT or file
output.write(&result).expect("Could not write output");
output.write(&result).expect("could not write output");
}
Err(error) => {
if !options.silent {
eprintln!("Error: {}", error);
}
print_error_message(&format!("Error: {}", error), &options);

process::exit(1);
}
Expand Down