diff --git a/src/rust/lektor_repo/src/download.rs b/src/rust/lektor_repo/src/download.rs index 7008c955c2ace05e64d4afbf1525a5ffc65903bb..c792110463bf5a686b029ba2bf6b1b868a2a9288 100644 --- a/src/rust/lektor_repo/src/download.rs +++ b/src/rust/lektor_repo/src/download.rs @@ -128,6 +128,7 @@ impl Download { pub async fn download(self) { let db = &self.db; let uri = &self.uri; + let dry = self.dry; let repo_id = { let mut db = self.db.lock().expect("failed to lock the database..."); let repo_id = @@ -167,8 +168,15 @@ impl Download { log::error!("can't find uri {uri} in repo {name}"); return; }; + + if dry { + log::info!("fetch metadata for karas from {name}: {karas_id:?}"); + return; + } + let other_urls = urls.iter().filter_map(|url| (url != base).then_some(url.as_str())); let search_urls: Vec<_> = Some(base).into_iter().chain(other_urls).collect(); + let search_urls = &search_urls; let (karas_id, errors): (Vec<_>, Vec<_>) = karas_id .into_iter() .map(|repo_kara_id| Download::build_destination_file_path(db.clone(), repo_id, repo_kara_id)) @@ -176,7 +184,8 @@ impl Download { for error in errors.into_iter().map(Result::unwrap_err) { log::error!(target: "REPO", "failed to build a file path for kara from repo {name}: {error}"); } - for (repo_kara_id, destination) in karas_id.into_iter().map(Result::unwrap) { + + stream::iter(karas_id.into_iter().map(Result::unwrap)).for_each_concurrent(2, |(repo_kara_id, destination)| async move { if let Err(err) = Download::download_kara(db.clone(), *api, &search_urls[..], repo_id, repo_kara_id, &destination).await { log::error!(target: "REPO", "failed to download file `{}` for kara {repo_kara_id}: {err}", destination.to_string_lossy()); return; @@ -186,7 +195,8 @@ impl Download { log::error!(target: "REPO", "failed to make kara `{repo_kara_id}` from {name} available: {err}"); return; } - } + log::info!("downloaded kara {repo_kara_id} from {name} at location {}", destination.to_string_lossy()); + }).await; }) .await; }