Download many pages blocking http-reqwest

• Using the reqwest crate in blocking mode.
• 7 seconds

[package]
name = "download-many-pages"
version = "0.1.0"
edition = "2024"

[dependencies]
reqwest = { version = "0.12.23", features = ["blocking"] }

use std::fs::File;
use std::io::{BufRead, BufReader};

use reqwest::Error;

fn download_page(url: &str) -> Result<String, Error> {
    let response = reqwest::blocking::get(url)?;
    let content = response.text()?;
    Ok(content)
}

fn read_urls_from_file(path: &str) -> Vec<String> {
    let file = File::open(path).expect("Failed to open file");
    let reader = BufReader::new(file);
    reader
        .lines()
        .filter_map(|line| {
            let line = line.ok()?.trim().to_string();
            if line.is_empty() || line.starts_with('#') {
                None
            } else {
                Some(line)
            }
        })
        .collect()
}


fn main() {
    let start = std::time::Instant::now();
    let args = std::env::args().collect::<Vec<String>>();
    if args.len() < 2 {
        eprintln!("Usage: {} <urls_file>", args[0]);
        std::process::exit(1);
    }
    let urls_file = &args[1];
    let urls = read_urls_from_file(urls_file);
    println!("URLs: {:?}", urls);

    for url in &urls {
        match download_page(url) {
            Ok(content) => {
                println!("Downloaded {} bytes from {}", content.len(), url);
            }
            Err(e) => println!("Error downloading {}: {}", url, e),
        }
    }
    let elapsed = start.elapsed();
    println!("Elapsed time: {:.2?}", elapsed);
}