Keyboard shortcuts

Press or to navigate between chapters

Press S or / to search in the book

Press ? to show this help

Press Esc to hide this help

Download many pages bad async

I think this code was generated by some AI, and while it uses the async constructs in reality it sends the requests sequentially.


use std::fs::File;
use std::io::{BufRead, BufReader};

use reqwest::Error;

async fn download_page(url: &str) -> Result<String, Error> {
    let response = reqwest::get(url).await?;
    let content = response.text().await?;
    Ok(content)
}

fn read_urls_from_file(path: &str) -> Vec<String> {
    let file = File::open(path).expect("Failed to open file");
    let reader = BufReader::new(file);
    reader
        .lines()
        .filter_map(|line| {
            let line = line.ok()?.trim().to_string();
            if line.is_empty() || line.starts_with('#') {
                None
            } else {
                Some(line)
            }
        })
        .collect()
}


#[tokio::main]
async fn main() {
    let start = std::time::Instant::now();
    let args = std::env::args().collect::<Vec<String>>();
    if args.len() < 2 {
        eprintln!("Usage: {} <urls_file>", args[0]);
        std::process::exit(1);
    }
    let urls_file = &args[1];
    let urls = read_urls_from_file(urls_file);
    println!("URLs: {:?}", urls);

    for url in &urls {
        match download_page(url).await {
            Ok(content) => {
                println!("Downloaded {} bytes from {}", content.len(), url);
            }
            Err(e) => println!("Error downloading {}: {}", url, e),
        }
    }
    let elapsed = start.elapsed();
    println!("Elapsed time: {:.2?}", elapsed);
}
[package]
name = "download-many-pages"
version = "0.1.0"
edition = "2024"

[dependencies]
reqwest = "0.12.23"
tokio = { version = "1.47.1", features = ["full"] }