REPO: Begin to write the fetch logic of the new repo

src/rust/Cargo.lock

@@ -1757,9 +1757,11 @@ name = "lektor_repo"
 version = "0.1.0"
 dependencies = [
  "commons",
+ "futures",
  "hashbrown 0.13.2",
  "kurisu_api",
  "lektor_c_compat",
+ "lektor_config",
  "lektor_db",
  "reqwest",
  "serde",

src/rust/Cargo.toml

@@ -68,6 +68,7 @@ reqwest = { version = "0.11", default-features = false, features = [
 
 # Async stuff
 async-trait = "^0.1"
+futures = "^0.3"
 tokio = { version = "1", features = [
     "rt",
     "rt-multi-thread",

src/rust/lektor_config/src/config.rs

 use serde::{Deserialize, Serialize};
 use std::path::PathBuf;
 
-#[derive(Debug, Serialize, Deserialize, Default)]
+#[derive(Debug, Serialize, Deserialize, Clone, Copy, Default)]
 pub enum RepoApiVersion {
     #[default]
     V1,
@@ -64,7 +64,7 @@ pub struct LektorRepoConfig {
     pub server: Vec<RepoConfig>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct RepoConfig {
     pub name: String,
     pub api: RepoApiVersion,

src/rust/lektor_repo/Cargo.toml

@@ -11,11 +11,14 @@ doctest = false
 [dependencies]
 serde.workspace = true
 tokio.workspace = true
+futures.workspace = true
 reqwest.workspace = true
 hashbrown.workspace = true
 
 lektor_c_compat = { path = "../lektor_c_compat" }
+lektor_config = { path = "../lektor_config" }
+lektor_db = { path = "../lektor_db" }
+
 smallstring = { path = "../smallstring" }
 kurisu_api = { path = "../kurisu_api" }
-lektor_db = { path = "../lektor_db" }
 commons = { path = "../commons" }

src/rust/lektor_repo/src/download.rs

+use std::path::PathBuf;
+
 use crate::*;
+use futures::stream::{self, StreamExt};
 
 pub struct DownloadBuilder {
-    hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
+    hosts: Vec<RepoConfig>,
     queue: LktCQueue,
     uri: Option<LktUri>,
     dry: bool,
@@ -9,19 +12,59 @@ pub struct DownloadBuilder {
 }
 
 pub struct Download {
-    hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
+    hosts: Vec<RepoConfig>,
     queue: LktCQueue,
     uri: LktUri,
     dry: bool,
     db: LktLockDbPtr,
 }
 
+/// TODO: Add an uri type in the kurisu API and add a converter
+fn formatter_for_api_version(version: RepoApiVersion) -> fn(&LktUri) -> Result<String, String> {
+    use LktUri::*;
+    match version {
+        RepoApiVersion::V1 => |uri| {
+            let mut ret = match uri {
+                Id(id) => format!("id={id}"),
+                KaraMaker(author) => format!("author={author}"),
+                Origin(cat) => format!("cat={cat}"),
+                Type(ty) => format!("type={ty}"),
+                Language(lang) => format!("lang={lang}"),
+                FuzzySearch(string) | Search(string) => {
+                    format!("search={}", string.join(" "))
+                }
+                Playlist(plt) => {
+                    return Err(format!(
+                    "can't filter playlist `{plt}`, playlist uris are not supported by kurisu V1"
+                ))
+                }
+                Any => String::new(),
+            };
+            Ok(either!(ret.is_empty() => ret; {
+                ret.insert_str(0, "?");
+                ret
+            }))
+        },
+        RepoApiVersion::V2 => |_uri| todo!(),
+    }
+}
+
+/// Download the metadata of the kara, should be a json thing. Deserialize it latter depending on
+/// the API version.
+fn download_metadata<'a>(
+    _api: RepoApiVersion,
+    _base: &'a str,
+    _url: &str,
+) -> Result<(&'a str, String), String> {
+    todo!()
+}
+
+fn download_kara(search_urls: &[String], destination: PathBuf) -> Result<(), String> {
+    todo!()
+}
+
 impl Download {
-    pub fn builder(
-        queue: LktCQueue,
-        hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
-        db: LktLockDbPtr,
-    ) -> DownloadBuilder {
+    pub fn builder(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> DownloadBuilder {
         DownloadBuilder {
             uri: None,
             dry: false,
@@ -32,8 +75,31 @@ impl Download {
     }
 
     pub async fn download(self) {
-        log::info!("do some work to dl (dry? -> {}): {:?}", self.dry, self.uri);
-        todo!()
+        let uri = &self.uri;
+        stream::iter(&self.hosts)
+            .for_each_concurrent(2, |RepoConfig { name, api, urls }| async move {
+                let uri = match formatter_for_api_version(*api)(uri) {
+                    Ok(uri) => uri,
+                    Err(err) => {
+                        log::error!(target: "REPO", "{err}");
+                        return;
+                    }
+                };
+                let ok_urls = stream::iter(urls.iter().map(|base| (base, format!("{base}/{uri}"))))
+                    .filter_map(|(base, url)| async move {
+                        download_metadata(*api, base, &url).map_err(|err| log::error!(target: "REPO", "failed to download metadata with url `{url}`: {err}")).ok()
+                    });
+                let Some((base, content)) = Box::pin(ok_urls).next().await else {
+                    log::error!("can't find uri {uri} in repo {name}");
+                    return;
+                };
+
+                let search_urls: Vec<_> = Some(base).into_iter().chain(urls.iter().filter_map(|url| (url != base).then_some(url.as_str()))).collect();
+
+                // Find the first url with an Ok status, then get the id and dl the kara if not dry.
+                // If the dl was not successfull, try the next url
+            })
+            .await;
     }
 }
 

src/rust/lektor_repo/src/lib.rs

 //! The crate responsible of downloading karas from kurisu.
 
-pub(crate) use commons::log;
+pub(crate) use commons::*;
 pub(crate) use hashbrown::HashMap;
 pub(crate) use lektor_c_compat::rs::*;
+pub(crate) use lektor_config::*;
 pub(crate) use lektor_db::{connexion::LktDatabaseConnection, uri::LktUri};
 pub(crate) use smallstring::SmallString;
 pub(crate) use std::sync::{Arc, Mutex};

src/rust/lektor_repo/src/repo.rs

@@ -7,24 +7,20 @@ use crate::*;
 pub struct LktModuleRepoRs {
     queue: LktCQueue,
     handlers: Vec<JoinHandle<()>>,
-    hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
+    hosts: Vec<RepoConfig>,
     db: LktLockDbPtr,
     rt: Runtime,
 }
 
 impl LktModuleRepoRs {
-    pub fn new(
-        queue: LktCQueue,
-        hosts: HashMap<SmallString, Vec<SmallString>>,
-        db: LktLockDbPtr,
-    ) -> Self {
+    pub fn new(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> Self {
         let par = std::thread::available_parallelism()
             .expect("failed to get an estimate of the default amount of parallelism");
         log::info!(target: "REPO", "detected parallelism {par} for repo module, min value is always 2");
         Self {
             db,
             queue,
-            hosts: Arc::new(hosts),
+            hosts,
             handlers: Default::default(),
             rt: tokio::runtime::Builder::new_multi_thread()
                 .worker_threads(std::cmp::min(2_usize, par.into()))