From a12627735f5a79ed6424ff8acd7e5b1881a404d0 Mon Sep 17 00:00:00 2001
From: Kubat <mael.martin31@gmail.com>
Date: Wed, 1 Mar 2023 14:00:09 +0100
Subject: [PATCH] REPO: Begin to write the fetch logic of the new repo
---
src/rust/Cargo.lock | 2 +
src/rust/Cargo.toml | 1 +
src/rust/lektor_config/src/config.rs | 4 +-
src/rust/lektor_repo/Cargo.toml | 5 +-
src/rust/lektor_repo/src/download.rs | 84 +++++++++++++++++++++++++---
src/rust/lektor_repo/src/lib.rs | 3 +-
src/rust/lektor_repo/src/repo.rs | 10 +---
7 files changed, 89 insertions(+), 20 deletions(-)
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 022fa352..5314aaf7 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -1757,9 +1757,11 @@ name = "lektor_repo"
version = "0.1.0"
dependencies = [
"commons",
+ "futures",
"hashbrown 0.13.2",
"kurisu_api",
"lektor_c_compat",
+ "lektor_config",
"lektor_db",
"reqwest",
"serde",
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
index 3a8cce81..5c914887 100644
--- a/src/rust/Cargo.toml
+++ b/src/rust/Cargo.toml
@@ -68,6 +68,7 @@ reqwest = { version = "0.11", default-features = false, features = [
# Async stuff
async-trait = "^0.1"
+futures = "^0.3"
tokio = { version = "1", features = [
"rt",
"rt-multi-thread",
diff --git a/src/rust/lektor_config/src/config.rs b/src/rust/lektor_config/src/config.rs
index 5eea48be..9dff4f96 100644
--- a/src/rust/lektor_config/src/config.rs
+++ b/src/rust/lektor_config/src/config.rs
@@ -1,7 +1,7 @@
use serde::{Deserialize, Serialize};
use std::path::PathBuf;
-#[derive(Debug, Serialize, Deserialize, Default)]
+#[derive(Debug, Serialize, Deserialize, Clone, Copy, Default)]
pub enum RepoApiVersion {
#[default]
V1,
@@ -64,7 +64,7 @@ pub struct LektorRepoConfig {
pub server: Vec<RepoConfig>,
}
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct RepoConfig {
pub name: String,
pub api: RepoApiVersion,
diff --git a/src/rust/lektor_repo/Cargo.toml b/src/rust/lektor_repo/Cargo.toml
index 00d4fe96..1d11b3ab 100644
--- a/src/rust/lektor_repo/Cargo.toml
+++ b/src/rust/lektor_repo/Cargo.toml
@@ -11,11 +11,14 @@ doctest = false
[dependencies]
serde.workspace = true
tokio.workspace = true
+futures.workspace = true
reqwest.workspace = true
hashbrown.workspace = true
lektor_c_compat = { path = "../lektor_c_compat" }
+lektor_config = { path = "../lektor_config" }
+lektor_db = { path = "../lektor_db" }
+
smallstring = { path = "../smallstring" }
kurisu_api = { path = "../kurisu_api" }
-lektor_db = { path = "../lektor_db" }
commons = { path = "../commons" }
diff --git a/src/rust/lektor_repo/src/download.rs b/src/rust/lektor_repo/src/download.rs
index fc8eeb49..20a01f61 100644
--- a/src/rust/lektor_repo/src/download.rs
+++ b/src/rust/lektor_repo/src/download.rs
@@ -1,7 +1,10 @@
+use std::path::PathBuf;
+
use crate::*;
+use futures::stream::{self, StreamExt};
pub struct DownloadBuilder {
- hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
+ hosts: Vec<RepoConfig>,
queue: LktCQueue,
uri: Option<LktUri>,
dry: bool,
@@ -9,19 +12,59 @@ pub struct DownloadBuilder {
}
pub struct Download {
- hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
+ hosts: Vec<RepoConfig>,
queue: LktCQueue,
uri: LktUri,
dry: bool,
db: LktLockDbPtr,
}
+/// TODO: Add an uri type in the kurisu API and add a converter
+fn formatter_for_api_version(version: RepoApiVersion) -> fn(&LktUri) -> Result<String, String> {
+ use LktUri::*;
+ match version {
+ RepoApiVersion::V1 => |uri| {
+ let mut ret = match uri {
+ Id(id) => format!("id={id}"),
+ KaraMaker(author) => format!("author={author}"),
+ Origin(cat) => format!("cat={cat}"),
+ Type(ty) => format!("type={ty}"),
+ Language(lang) => format!("lang={lang}"),
+ FuzzySearch(string) | Search(string) => {
+ format!("search={}", string.join(" "))
+ }
+ Playlist(plt) => {
+ return Err(format!(
+ "can't filter playlist `{plt}`, playlist uris are not supported by kurisu V1"
+ ))
+ }
+ Any => String::new(),
+ };
+ Ok(either!(ret.is_empty() => ret; {
+ ret.insert_str(0, "?");
+ ret
+ }))
+ },
+ RepoApiVersion::V2 => |_uri| todo!(),
+ }
+}
+
+/// Download the metadata of the kara, should be a json thing. Deserialize it latter depending on
+/// the API version.
+fn download_metadata<'a>(
+ _api: RepoApiVersion,
+ _base: &'a str,
+ _url: &str,
+) -> Result<(&'a str, String), String> {
+ todo!()
+}
+
+fn download_kara(search_urls: &[String], destination: PathBuf) -> Result<(), String> {
+ todo!()
+}
+
impl Download {
- pub fn builder(
- queue: LktCQueue,
- hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
- db: LktLockDbPtr,
- ) -> DownloadBuilder {
+ pub fn builder(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> DownloadBuilder {
DownloadBuilder {
uri: None,
dry: false,
@@ -32,8 +75,31 @@ impl Download {
}
pub async fn download(self) {
- log::info!("do some work to dl (dry? -> {}): {:?}", self.dry, self.uri);
- todo!()
+ let uri = &self.uri;
+ stream::iter(&self.hosts)
+ .for_each_concurrent(2, |RepoConfig { name, api, urls }| async move {
+ let uri = match formatter_for_api_version(*api)(uri) {
+ Ok(uri) => uri,
+ Err(err) => {
+ log::error!(target: "REPO", "{err}");
+ return;
+ }
+ };
+ let ok_urls = stream::iter(urls.iter().map(|base| (base, format!("{base}/{uri}"))))
+ .filter_map(|(base, url)| async move {
+ download_metadata(*api, base, &url).map_err(|err| log::error!(target: "REPO", "failed to download metadata with url `{url}`: {err}")).ok()
+ });
+ let Some((base, content)) = Box::pin(ok_urls).next().await else {
+ log::error!("can't find uri {uri} in repo {name}");
+ return;
+ };
+
+ let search_urls: Vec<_> = Some(base).into_iter().chain(urls.iter().filter_map(|url| (url != base).then_some(url.as_str()))).collect();
+
+ // Find the first url with an Ok status, then get the id and dl the kara if not dry.
+ // If the dl was not successfull, try the next url
+ })
+ .await;
}
}
diff --git a/src/rust/lektor_repo/src/lib.rs b/src/rust/lektor_repo/src/lib.rs
index 3609a12f..4b05c0f5 100644
--- a/src/rust/lektor_repo/src/lib.rs
+++ b/src/rust/lektor_repo/src/lib.rs
@@ -1,8 +1,9 @@
//! The crate responsible of downloading karas from kurisu.
-pub(crate) use commons::log;
+pub(crate) use commons::*;
pub(crate) use hashbrown::HashMap;
pub(crate) use lektor_c_compat::rs::*;
+pub(crate) use lektor_config::*;
pub(crate) use lektor_db::{connexion::LktDatabaseConnection, uri::LktUri};
pub(crate) use smallstring::SmallString;
pub(crate) use std::sync::{Arc, Mutex};
diff --git a/src/rust/lektor_repo/src/repo.rs b/src/rust/lektor_repo/src/repo.rs
index d739e1ab..b236b203 100644
--- a/src/rust/lektor_repo/src/repo.rs
+++ b/src/rust/lektor_repo/src/repo.rs
@@ -7,24 +7,20 @@ use crate::*;
pub struct LktModuleRepoRs {
queue: LktCQueue,
handlers: Vec<JoinHandle<()>>,
- hosts: Arc<HashMap<SmallString, Vec<SmallString>>>,
+ hosts: Vec<RepoConfig>,
db: LktLockDbPtr,
rt: Runtime,
}
impl LktModuleRepoRs {
- pub fn new(
- queue: LktCQueue,
- hosts: HashMap<SmallString, Vec<SmallString>>,
- db: LktLockDbPtr,
- ) -> Self {
+ pub fn new(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> Self {
let par = std::thread::available_parallelism()
.expect("failed to get an estimate of the default amount of parallelism");
log::info!(target: "REPO", "detected parallelism {par} for repo module, min value is always 2");
Self {
db,
queue,
- hosts: Arc::new(hosts),
+ hosts,
handlers: Default::default(),
rt: tokio::runtime::Builder::new_multi_thread()
.worker_threads(std::cmp::min(2_usize, par.into()))
--
GitLab