diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock index 022fa35291003306b4ed3b97a9212ffd457d49df..5314aaf704d5fbe48d8b6b86764dcc54391a8dfe 100644 --- a/src/rust/Cargo.lock +++ b/src/rust/Cargo.lock @@ -1757,9 +1757,11 @@ name = "lektor_repo" version = "0.1.0" dependencies = [ "commons", + "futures", "hashbrown 0.13.2", "kurisu_api", "lektor_c_compat", + "lektor_config", "lektor_db", "reqwest", "serde", diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml index 3a8cce81ed21e4e9dcf785e3aeed21db893e3f17..5c914887aef9e6b5fd033ed2cd2795066a412fd1 100644 --- a/src/rust/Cargo.toml +++ b/src/rust/Cargo.toml @@ -68,6 +68,7 @@ reqwest = { version = "0.11", default-features = false, features = [ # Async stuff async-trait = "^0.1" +futures = "^0.3" tokio = { version = "1", features = [ "rt", "rt-multi-thread", diff --git a/src/rust/lektor_config/src/config.rs b/src/rust/lektor_config/src/config.rs index 5eea48befcbadf83caae8e8844ff221e22242f75..9dff4f9636653f459f7e090b53efa0d00cdaf402 100644 --- a/src/rust/lektor_config/src/config.rs +++ b/src/rust/lektor_config/src/config.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use std::path::PathBuf; -#[derive(Debug, Serialize, Deserialize, Default)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, Default)] pub enum RepoApiVersion { #[default] V1, @@ -64,7 +64,7 @@ pub struct LektorRepoConfig { pub server: Vec<RepoConfig>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize, Clone)] pub struct RepoConfig { pub name: String, pub api: RepoApiVersion, diff --git a/src/rust/lektor_repo/Cargo.toml b/src/rust/lektor_repo/Cargo.toml index 00d4fe96f6c215c45a670fe35e0c36319fe28617..1d11b3ab08c9e2fbb8437ff8885f1c801124f5f3 100644 --- a/src/rust/lektor_repo/Cargo.toml +++ b/src/rust/lektor_repo/Cargo.toml @@ -11,11 +11,14 @@ doctest = false [dependencies] serde.workspace = true tokio.workspace = true +futures.workspace = true reqwest.workspace = true hashbrown.workspace = true lektor_c_compat = { path = "../lektor_c_compat" } +lektor_config = { path = "../lektor_config" } +lektor_db = { path = "../lektor_db" } + smallstring = { path = "../smallstring" } kurisu_api = { path = "../kurisu_api" } -lektor_db = { path = "../lektor_db" } commons = { path = "../commons" } diff --git a/src/rust/lektor_repo/src/download.rs b/src/rust/lektor_repo/src/download.rs index fc8eeb498cefcfa37b4910cd15b049ced8298c59..20a01f616b4b141d58a04db054020215b73d8291 100644 --- a/src/rust/lektor_repo/src/download.rs +++ b/src/rust/lektor_repo/src/download.rs @@ -1,7 +1,10 @@ +use std::path::PathBuf; + use crate::*; +use futures::stream::{self, StreamExt}; pub struct DownloadBuilder { - hosts: Arc<HashMap<SmallString, Vec<SmallString>>>, + hosts: Vec<RepoConfig>, queue: LktCQueue, uri: Option<LktUri>, dry: bool, @@ -9,19 +12,59 @@ pub struct DownloadBuilder { } pub struct Download { - hosts: Arc<HashMap<SmallString, Vec<SmallString>>>, + hosts: Vec<RepoConfig>, queue: LktCQueue, uri: LktUri, dry: bool, db: LktLockDbPtr, } +/// TODO: Add an uri type in the kurisu API and add a converter +fn formatter_for_api_version(version: RepoApiVersion) -> fn(&LktUri) -> Result<String, String> { + use LktUri::*; + match version { + RepoApiVersion::V1 => |uri| { + let mut ret = match uri { + Id(id) => format!("id={id}"), + KaraMaker(author) => format!("author={author}"), + Origin(cat) => format!("cat={cat}"), + Type(ty) => format!("type={ty}"), + Language(lang) => format!("lang={lang}"), + FuzzySearch(string) | Search(string) => { + format!("search={}", string.join(" ")) + } + Playlist(plt) => { + return Err(format!( + "can't filter playlist `{plt}`, playlist uris are not supported by kurisu V1" + )) + } + Any => String::new(), + }; + Ok(either!(ret.is_empty() => ret; { + ret.insert_str(0, "?"); + ret + })) + }, + RepoApiVersion::V2 => |_uri| todo!(), + } +} + +/// Download the metadata of the kara, should be a json thing. Deserialize it latter depending on +/// the API version. +fn download_metadata<'a>( + _api: RepoApiVersion, + _base: &'a str, + _url: &str, +) -> Result<(&'a str, String), String> { + todo!() +} + +fn download_kara(search_urls: &[String], destination: PathBuf) -> Result<(), String> { + todo!() +} + impl Download { - pub fn builder( - queue: LktCQueue, - hosts: Arc<HashMap<SmallString, Vec<SmallString>>>, - db: LktLockDbPtr, - ) -> DownloadBuilder { + pub fn builder(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> DownloadBuilder { DownloadBuilder { uri: None, dry: false, @@ -32,8 +75,31 @@ impl Download { } pub async fn download(self) { - log::info!("do some work to dl (dry? -> {}): {:?}", self.dry, self.uri); - todo!() + let uri = &self.uri; + stream::iter(&self.hosts) + .for_each_concurrent(2, |RepoConfig { name, api, urls }| async move { + let uri = match formatter_for_api_version(*api)(uri) { + Ok(uri) => uri, + Err(err) => { + log::error!(target: "REPO", "{err}"); + return; + } + }; + let ok_urls = stream::iter(urls.iter().map(|base| (base, format!("{base}/{uri}")))) + .filter_map(|(base, url)| async move { + download_metadata(*api, base, &url).map_err(|err| log::error!(target: "REPO", "failed to download metadata with url `{url}`: {err}")).ok() + }); + let Some((base, content)) = Box::pin(ok_urls).next().await else { + log::error!("can't find uri {uri} in repo {name}"); + return; + }; + + let search_urls: Vec<_> = Some(base).into_iter().chain(urls.iter().filter_map(|url| (url != base).then_some(url.as_str()))).collect(); + + // Find the first url with an Ok status, then get the id and dl the kara if not dry. + // If the dl was not successfull, try the next url + }) + .await; } } diff --git a/src/rust/lektor_repo/src/lib.rs b/src/rust/lektor_repo/src/lib.rs index 3609a12fb8b505ab333a75bf3a32392d352623bf..4b05c0f5b7faefcd898b6abdfd0af090d60b5062 100644 --- a/src/rust/lektor_repo/src/lib.rs +++ b/src/rust/lektor_repo/src/lib.rs @@ -1,8 +1,9 @@ //! The crate responsible of downloading karas from kurisu. -pub(crate) use commons::log; +pub(crate) use commons::*; pub(crate) use hashbrown::HashMap; pub(crate) use lektor_c_compat::rs::*; +pub(crate) use lektor_config::*; pub(crate) use lektor_db::{connexion::LktDatabaseConnection, uri::LktUri}; pub(crate) use smallstring::SmallString; pub(crate) use std::sync::{Arc, Mutex}; diff --git a/src/rust/lektor_repo/src/repo.rs b/src/rust/lektor_repo/src/repo.rs index d739e1abe4c44642abca3b6248fff70ac16aeca1..b236b20321e1302a7b7e2a9335ffd68371889e0e 100644 --- a/src/rust/lektor_repo/src/repo.rs +++ b/src/rust/lektor_repo/src/repo.rs @@ -7,24 +7,20 @@ use crate::*; pub struct LktModuleRepoRs { queue: LktCQueue, handlers: Vec<JoinHandle<()>>, - hosts: Arc<HashMap<SmallString, Vec<SmallString>>>, + hosts: Vec<RepoConfig>, db: LktLockDbPtr, rt: Runtime, } impl LktModuleRepoRs { - pub fn new( - queue: LktCQueue, - hosts: HashMap<SmallString, Vec<SmallString>>, - db: LktLockDbPtr, - ) -> Self { + pub fn new(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> Self { let par = std::thread::available_parallelism() .expect("failed to get an estimate of the default amount of parallelism"); log::info!(target: "REPO", "detected parallelism {par} for repo module, min value is always 2"); Self { db, queue, - hosts: Arc::new(hosts), + hosts, handlers: Default::default(), rt: tokio::runtime::Builder::new_multi_thread() .worker_threads(std::cmp::min(2_usize, par.into()))