From 8c31926125321c258bf6f8eb4f01994ef8845bd6 Mon Sep 17 00:00:00 2001
From: Kubat <mael.martin31@gmail.com>
Date: Wed, 1 Mar 2023 17:53:07 +0100
Subject: [PATCH] REPO: Continue to implement the update logic for the new
repo, only working with V1 of the API
---
src/rust/Cargo.lock | 1 +
src/rust/Cargo.toml | 1 -
src/rust/kurisu_api/src/route.rs | 6 ++
src/rust/kurisu_api/src/v1.rs | 16 +++
src/rust/kurisu_api/src/v2.rs | 33 ++++++-
src/rust/lektor_db/src/connexion.rs | 15 ++-
src/rust/lektor_db/src/error.rs | 3 +
src/rust/lektor_db/src/lib.rs | 3 +-
src/rust/lektor_db/src/models.rs | 9 +-
src/rust/lektor_repo/Cargo.toml | 1 +
src/rust/lektor_repo/src/download.rs | 141 ++++++++++++++++++++++-----
11 files changed, 198 insertions(+), 31 deletions(-)
diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock
index 5314aaf7..79624569 100644
--- a/src/rust/Cargo.lock
+++ b/src/rust/Cargo.lock
@@ -1765,6 +1765,7 @@ dependencies = [
"lektor_db",
"reqwest",
"serde",
+ "serde_json",
"smallstring",
"tokio",
]
diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml
index 5c914887..a2765b89 100644
--- a/src/rust/Cargo.toml
+++ b/src/rust/Cargo.toml
@@ -63,7 +63,6 @@ serde = { version = "^1", default-features = false, features = [
reqwest = { version = "0.11", default-features = false, features = [
"rustls-tls",
"rustls-tls-native-roots",
- "json",
] }
# Async stuff
diff --git a/src/rust/kurisu_api/src/route.rs b/src/rust/kurisu_api/src/route.rs
index c497855d..a6666012 100644
--- a/src/rust/kurisu_api/src/route.rs
+++ b/src/rust/kurisu_api/src/route.rs
@@ -39,6 +39,12 @@ macro_rules! route {
const COUNT: usize = $crate::route!(count_args $($expr),*);
$crate::route::build_request::<$route, COUNT>([$($expr),*])
}};
+
+ ($route: ident -> $base: expr ; $($expr: expr),*) => {{
+ let mut route = $crate::route!($route -> $($expr),*);
+ route.insert_str(0, &$base);
+ route
+ }};
}
/// Create a new route.
diff --git a/src/rust/kurisu_api/src/v1.rs b/src/rust/kurisu_api/src/v1.rs
index 31af9317..f28ca2d0 100644
--- a/src/rust/kurisu_api/src/v1.rs
+++ b/src/rust/kurisu_api/src/v1.rs
@@ -18,6 +18,9 @@ pub struct Kara<'a> {
pub popularity: i64,
pub unix_timestamp: i64,
pub size: i64,
+
+ /// Still need to be present on kurisu...
+ pub file_hash: &'a str,
}
#[derive(Debug, Deserialize)]
@@ -41,6 +44,19 @@ pub enum MaybeKaraList<'a> {
Multi(Vec<Kara<'a>>),
}
+impl<'a> IntoIterator for MaybeKaraList<'a> {
+ type Item = Kara<'a>;
+ type IntoIter = std::vec::IntoIter<Kara<'a>>;
+
+ fn into_iter(self) -> Self::IntoIter {
+ match self {
+ MaybeKaraList::Empty => vec![].into_iter(),
+ MaybeKaraList::Single(kara) => vec![kara].into_iter(),
+ MaybeKaraList::Multi(karas) => karas.into_iter(),
+ }
+ }
+}
+
impl<'a, 'de> Deserialize<'de> for MaybeKaraList<'a>
where
'de: 'a,
diff --git a/src/rust/kurisu_api/src/v2.rs b/src/rust/kurisu_api/src/v2.rs
index 0ba6f985..d6d5bb44 100644
--- a/src/rust/kurisu_api/src/v2.rs
+++ b/src/rust/kurisu_api/src/v2.rs
@@ -42,8 +42,34 @@ pub enum SongOrigin {
Autre,
}
+impl SongType {
+ pub fn as_str(&self) -> &str {
+ match self {
+ SongType::OP => "OP",
+ SongType::ED => "ED",
+ SongType::IS => "IS",
+ SongType::MV => "MV",
+ SongType::OT => "OT",
+ }
+ }
+}
+
+impl SongOrigin {
+ pub fn as_str(&self) -> &str {
+ match self {
+ SongOrigin::Anime => "anime",
+ SongOrigin::VN => "vn",
+ SongOrigin::Game => "game",
+ SongOrigin::Music => "music",
+ SongOrigin::Autre => "autre",
+ }
+ }
+}
+
#[derive(Debug, Deserialize, PartialEq, Eq)]
pub struct Kara<'a> {
+ pub id: i64,
+
#[serde(rename = "title")]
pub song_title: &'a str,
@@ -66,6 +92,8 @@ pub struct Kara<'a> {
pub last_modified_epoch: i64,
pub creation_epoch: i64,
+ pub file_hash: &'a str,
+
#[serde(flatten)]
pub tags: HashMap<&'a str, Vec<&'a str>>,
}
@@ -99,7 +127,8 @@ pub struct PlaylistList<'a> {
new_route! { GetRepo :: "/" #0 -> Repo }
-new_route! { GetKaras :: "/kara" #0 -> KaraList }
+new_route! { GetAllKaras :: "/kara" #0 -> KaraList }
+new_route! { GetKaras :: "/kara/@" #1 -> KaraList }
new_route! { GetKara :: "/kara/@" #1 -> Kara }
new_route! { GetKaraDl :: "/kara/dl/@" #1 -> KaraDl }
@@ -199,6 +228,7 @@ mod test {
, "karamakers": [ "Viieux", "Totoro" ]
, "is_virtual": false
, "type": "op"
+ , "id": 42
, "origin": "anime"
, "languages": [ "jp" ]
, "last_modified_epoch": 1
@@ -216,6 +246,7 @@ mod test {
, "is_virtual": false
, "tag_1": [ "1", "2" ]
, "type": "op"
+ , "id": 42
, "origin": "anime"
, "languages": [ "jp" ]
, "tag_2": []
diff --git a/src/rust/lektor_db/src/connexion.rs b/src/rust/lektor_db/src/connexion.rs
index 65551092..11af90b9 100644
--- a/src/rust/lektor_db/src/connexion.rs
+++ b/src/rust/lektor_db/src/connexion.rs
@@ -107,6 +107,15 @@ impl LktDatabaseConnection {
Ok(Self { sqlite })
}
+ pub fn get_repo_id(&mut self, repo_name: impl AsRef<str>) -> LktDatabaseResult<i64> {
+ with_dsl!(repo => repo
+ .select(id)
+ .filter(name.is(repo_name.as_ref()))
+ .first::<i64>(&mut self.sqlite)
+ .map_err(LktDatabaseError::DieselResult)
+ )
+ }
+
/// Get a tag id by its name. If the tag doesn't exists, create it.
fn get_tag_id_by_name(&mut self, tag_name: impl AsRef<str>) -> LktDatabaseResult<i64> {
with_dsl!(tag => self.sqlite.exclusive_transaction(|conn| {
@@ -203,6 +212,10 @@ impl LktDatabaseConnection {
})
}
+ pub fn make_kara_available<'a>(&mut self, remote_id: RemoteKaraId) -> LktDatabaseResult<()> {
+ todo!()
+ }
+
/// Create a series of models from a kara signature from Kurisu's V1 API.
pub fn new_kara_v1<'a>(
&mut self,
@@ -237,7 +250,7 @@ impl LktDatabaseConnection {
song_type: kara.song_type,
song_origin: kara.category,
source_name: kara.source_name,
- file_hash: format!("{}", kara.unix_timestamp),
+ file_hash: kara.file_hash,
};
Ok((id, kara, kara_maker, vec![lang], tags))
}
diff --git a/src/rust/lektor_db/src/error.rs b/src/rust/lektor_db/src/error.rs
index 05e562c9..bbbff5e8 100644
--- a/src/rust/lektor_db/src/error.rs
+++ b/src/rust/lektor_db/src/error.rs
@@ -14,6 +14,9 @@ pub enum LktDatabaseError {
#[error("database error: {0}")]
Str(&'static str),
+
+ #[error("database error: not found!")]
+ NotFound,
}
#[derive(Debug, thiserror::Error)]
diff --git a/src/rust/lektor_db/src/lib.rs b/src/rust/lektor_db/src/lib.rs
index add72f28..be141247 100644
--- a/src/rust/lektor_db/src/lib.rs
+++ b/src/rust/lektor_db/src/lib.rs
@@ -11,8 +11,9 @@ pub mod uri;
pub(self) mod schema;
+pub use error::*;
+
pub(self) use diesel::prelude::*;
-pub(self) use error::*;
pub(self) use std::{collections::VecDeque, ops::Range, path::Path};
/// All the information needed to add a kara recieved from a repo!
diff --git a/src/rust/lektor_db/src/models.rs b/src/rust/lektor_db/src/models.rs
index b42f25ed..f07ed820 100644
--- a/src/rust/lektor_db/src/models.rs
+++ b/src/rust/lektor_db/src/models.rs
@@ -18,6 +18,13 @@ pub struct KaraId {
pub local_kara_id: i64,
}
+#[derive(Debug, Queryable, Selectable)]
+#[diesel(table_name = repo_kara)]
+pub struct RemoteKaraId {
+ pub repo_id: i64,
+ pub repo_kara_id: i64,
+}
+
#[derive(Debug, Insertable, Queryable, Selectable)]
#[diesel(table_name = history)]
pub struct HistoryRecord {
@@ -43,7 +50,7 @@ pub struct NewKara<'a> {
pub song_type: &'a str,
pub song_origin: &'a str,
pub source_name: &'a str,
- pub file_hash: String,
+ pub file_hash: &'a str,
}
#[derive(Debug, Insertable)]
diff --git a/src/rust/lektor_repo/Cargo.toml b/src/rust/lektor_repo/Cargo.toml
index 1d11b3ab..fd087212 100644
--- a/src/rust/lektor_repo/Cargo.toml
+++ b/src/rust/lektor_repo/Cargo.toml
@@ -14,6 +14,7 @@ tokio.workspace = true
futures.workspace = true
reqwest.workspace = true
hashbrown.workspace = true
+serde_json.workspace = true
lektor_c_compat = { path = "../lektor_c_compat" }
lektor_config = { path = "../lektor_config" }
diff --git a/src/rust/lektor_repo/src/download.rs b/src/rust/lektor_repo/src/download.rs
index 20a01f61..7008c955 100644
--- a/src/rust/lektor_repo/src/download.rs
+++ b/src/rust/lektor_repo/src/download.rs
@@ -1,7 +1,6 @@
-use std::path::PathBuf;
-
use crate::*;
use futures::stream::{self, StreamExt};
+use std::path::{Path, PathBuf};
pub struct DownloadBuilder {
hosts: Vec<RepoConfig>,
@@ -49,20 +48,6 @@ fn formatter_for_api_version(version: RepoApiVersion) -> fn(&LktUri) -> Result<S
}
}
-/// Download the metadata of the kara, should be a json thing. Deserialize it latter depending on
-/// the API version.
-fn download_metadata<'a>(
- _api: RepoApiVersion,
- _base: &'a str,
- _url: &str,
-) -> Result<(&'a str, String), String> {
- todo!()
-}
-
-fn download_kara(search_urls: &[String], destination: PathBuf) -> Result<(), String> {
- todo!()
-}
-
impl Download {
pub fn builder(queue: LktCQueue, hosts: Vec<RepoConfig>, db: LktLockDbPtr) -> DownloadBuilder {
DownloadBuilder {
@@ -74,30 +59,134 @@ impl Download {
}
}
+ /// Download the metadata of the kara. We update the database with the downloaded metadata. The
+ /// function only returns the karas' id viewed from the repo.
+ async fn download_metadata<'a>(
+ db: LktLockDbPtr,
+ api: RepoApiVersion,
+ base: &'a str,
+ repo_id: i64,
+ uri: &str,
+ ) -> Result<(&'a str, Vec<i64>), String> {
+ use kurisu_api::route;
+ match api {
+ RepoApiVersion::V1 => {
+ use kurisu_api::v1::*;
+ let content = reqwest::get(route!(GetKara -> base; uri))
+ .await
+ .map_err(|err| format!("{err}"))?
+ .text()
+ .await
+ .map_err(|err| format!("{err}"))?;
+ let (new_karas, errors): (Vec<_>, Vec<_>) =
+ serde_json::from_str::<MaybeKaraList>(&content)
+ .map_err(|err| format!("{err}"))?
+ .into_iter()
+ .map(|kara_v1| {
+ let mut db = db.lock().expect("failed to lock the database...");
+ let new_kara = db.new_kara_v1(repo_id, kara_v1)?;
+ let repo_kara_id = new_kara.0.repo_kara_id;
+ db.add_kara_from_request(new_kara)?;
+ Ok::<i64, lektor_db::LktDatabaseError>(repo_kara_id)
+ })
+ .partition(Result::is_ok);
+ for error in errors.into_iter().map(Result::unwrap_err) {
+ log::error!(target: "REPO", "failed to download metadata for kara from {base}: {error}");
+ }
+ either!(new_karas.is_empty()
+ => Err(format!("no kara found in {base} for uri: {uri}"))
+ ; Ok((base, new_karas.into_iter().map(Result::unwrap).collect())))
+ }
+
+ RepoApiVersion::V2 => {
+ todo!()
+ }
+ }
+ }
+
+ /// Build the destination file path for the kara.
+ fn build_destination_file_path(
+ _db: LktLockDbPtr,
+ _repo_id: i64,
+ _repo_kara_id: i64,
+ ) -> Result<(i64, PathBuf), String> {
+ todo!()
+ }
+
+ /// Download the kara file.
+ async fn download_kara(
+ _db: LktLockDbPtr,
+ _api: RepoApiVersion,
+ _search_urls: &[&str],
+ _repo_id: i64,
+ _repo_kara_id: i64,
+ _destination: impl AsRef<Path>,
+ ) -> Result<(), String> {
+ todo!()
+ }
+
pub async fn download(self) {
+ let db = &self.db;
let uri = &self.uri;
+ let repo_id = {
+ let mut db = self.db.lock().expect("failed to lock the database...");
+ let repo_id =
+ self.hosts
+ .iter()
+ .find_map(|RepoConfig { name, .. }| match db.get_repo_id(name) {
+ Ok(repo_id) => Some(repo_id),
+ Err(err) => {
+ log::error!(target: "REPO", "no id found for repo: {err}");
+ None
+ }
+ });
+ match repo_id {
+ Some(repo_id) => repo_id,
+ None => return,
+ }
+ };
+
stream::iter(&self.hosts)
.for_each_concurrent(2, |RepoConfig { name, api, urls }| async move {
- let uri = match formatter_for_api_version(*api)(uri) {
+ let uri = match formatter_for_api_version(*api)(&uri) {
Ok(uri) => uri,
Err(err) => {
log::error!(target: "REPO", "{err}");
return;
}
};
- let ok_urls = stream::iter(urls.iter().map(|base| (base, format!("{base}/{uri}"))))
- .filter_map(|(base, url)| async move {
- download_metadata(*api, base, &url).map_err(|err| log::error!(target: "REPO", "failed to download metadata with url `{url}`: {err}")).ok()
+ let uri = &uri;
+ let ok_urls = stream::iter(urls.iter())
+ .filter_map(|base | async move {
+ Download::download_metadata(db.clone(), *api, base, repo_id, &uri)
+ .await
+ .map_err(|err| log::error!(target: "REPO", "failed to download metadata with uri `{uri}`: {err}"))
+ .ok()
});
- let Some((base, content)) = Box::pin(ok_urls).next().await else {
+ let Some((base, karas_id)) = Box::pin(ok_urls).next().await else {
log::error!("can't find uri {uri} in repo {name}");
return;
};
-
- let search_urls: Vec<_> = Some(base).into_iter().chain(urls.iter().filter_map(|url| (url != base).then_some(url.as_str()))).collect();
-
- // Find the first url with an Ok status, then get the id and dl the kara if not dry.
- // If the dl was not successfull, try the next url
+ let other_urls = urls.iter().filter_map(|url| (url != base).then_some(url.as_str()));
+ let search_urls: Vec<_> = Some(base).into_iter().chain(other_urls).collect();
+ let (karas_id, errors): (Vec<_>, Vec<_>) = karas_id
+ .into_iter()
+ .map(|repo_kara_id| Download::build_destination_file_path(db.clone(), repo_id, repo_kara_id))
+ .partition(Result::is_ok);
+ for error in errors.into_iter().map(Result::unwrap_err) {
+ log::error!(target: "REPO", "failed to build a file path for kara from repo {name}: {error}");
+ }
+ for (repo_kara_id, destination) in karas_id.into_iter().map(Result::unwrap) {
+ if let Err(err) = Download::download_kara(db.clone(), *api, &search_urls[..], repo_id, repo_kara_id, &destination).await {
+ log::error!(target: "REPO", "failed to download file `{}` for kara {repo_kara_id}: {err}", destination.to_string_lossy());
+ return;
+ }
+ if let Err(err) = db.lock().expect("failed to lock the database...")
+ .make_kara_available(lektor_db::models::RemoteKaraId { repo_id, repo_kara_id }) {
+ log::error!(target: "REPO", "failed to make kara `{repo_kara_id}` from {name} available: {err}");
+ return;
+ }
+ }
})
.await;
}
--
GitLab