From 2d5c3a52cafa5ca9f7aef63024ce21d31f93a2e3 Mon Sep 17 00:00:00 2001
From: Kubat <mael.martin31@gmail.com>
Date: Mon, 23 Oct 2023 20:35:43 +0200
Subject: [PATCH] DATABASE+CLIENT-LIB: Add the num of the last epoch + search
by multiple queries
---
amadeus/src/components/config/mod.rs | 13 +++++-
lektor_lib/src/requests.rs | 6 ++-
lektor_nkdb/src/database/epoch.rs | 13 ++++--
lektor_nkdb/src/lib.rs | 18 +++++--
lektor_nkdb/src/search/mod.rs | 28 ++---------
lektor_nkdb/src/storage/disk_storage.rs | 4 +-
lektor_nkdb/src/storage/mod.rs | 2 +-
lektor_nkdb/src/storage/test_storage.rs | 4 +-
lektor_payloads/src/lib.rs | 62 ++-----------------------
lektor_payloads/src/search.rs | 60 ++++++++++++++++++++++++
lektord/src/routes.rs | 3 +-
lkt/src/main.rs | 13 ++++--
12 files changed, 125 insertions(+), 101 deletions(-)
create mode 100644 lektor_payloads/src/search.rs
diff --git a/amadeus/src/components/config/mod.rs b/amadeus/src/components/config/mod.rs
index 331b2114..9b21e81b 100644
--- a/amadeus/src/components/config/mod.rs
+++ b/amadeus/src/components/config/mod.rs
@@ -22,7 +22,7 @@ use lektor_utils::{
logger,
};
use serde::{Deserialize, Serialize};
-use std::{net::SocketAddr, ops::Deref, sync::Arc};
+use std::{borrow::Cow, net::SocketAddr, ops::Deref, sync::Arc};
pub struct State {
config: Arc<AmadeusConfig>,
@@ -473,7 +473,16 @@ impl State {
]],
match &self.remote_infos {
Some(Some(infos)) => section![ "Remote Infos" => [
- text(format!("Server version {}", infos.version)).size(SIZE_FONT_NORMAL),
+ row![
+ text("Server version").size(SIZE_FONT_NORMAL),
+ horizontal_space(Length::Fill),
+ text(&infos.version).size(SIZE_FONT_NORMAL),
+ ],
+ row![
+ text("Database epoch").size(SIZE_FONT_NORMAL),
+ horizontal_space(Length::Fill),
+ text(infos.last_epoch.map(|x| x.to_string().into()).unwrap_or(Cow::Borrowed("None"))).size(SIZE_FONT_NORMAL),
+ ],
]],
_ => None,
},
diff --git a/lektor_lib/src/requests.rs b/lektor_lib/src/requests.rs
index e7c65408..f49ff348 100644
--- a/lektor_lib/src/requests.rs
+++ b/lektor_lib/src/requests.rs
@@ -279,7 +279,8 @@ pub async fn search_karas(
from: SearchFrom,
by: KaraBy,
) -> Result<Vec<KId>> {
- let (len, obj) = encode_base64_value(SearchData { from, regex: by })?;
+ let regex = vec![by];
+ let (len, obj) = encode_base64_value(SearchData { from, regex })?;
let get = std::str::from_utf8(&obj[..len])?;
Ok(request!(config; GET @ "/search/{get}" => Vec<KId>))
}
@@ -289,7 +290,8 @@ pub async fn count_karas(
from: SearchFrom,
by: KaraBy,
) -> Result<usize> {
- let (len, obj) = encode_base64_value(SearchData { from, regex: by })?;
+ let regex = vec![by];
+ let (len, obj) = encode_base64_value(SearchData { from, regex })?;
let get = std::str::from_utf8(&obj[..len])?;
Ok(request!(config; GET @ "/count/{get}" => usize))
}
diff --git a/lektor_nkdb/src/database/epoch.rs b/lektor_nkdb/src/database/epoch.rs
index dc9044f2..c09db611 100644
--- a/lektor_nkdb/src/database/epoch.rs
+++ b/lektor_nkdb/src/database/epoch.rs
@@ -6,14 +6,14 @@ use crate::*;
/// The epoch contains all available karas at a certain point in time. It can be submitted and
/// available to all readers of the database or unsubmited and only one writter can edit it.
#[derive(Debug, Default)]
-pub(crate) struct Epoch(EpochData);
+pub(crate) struct Epoch(EpochData, u64);
/// Represent the data contained in an epoch.
pub type EpochData = HashMap<KId, Kara>;
-impl From<EpochData> for Epoch {
- fn from(value: EpochData) -> Self {
- Self(value)
+impl From<(EpochData, u64)> for Epoch {
+ fn from((data, num): (EpochData, u64)) -> Self {
+ Self(data, num)
}
}
@@ -54,4 +54,9 @@ impl Epoch {
pub fn data_mut(&mut self) -> &mut EpochData {
&mut self.0
}
+
+ /// Get the number of the epoch
+ pub fn epoch_num(&self) -> u64 {
+ self.1
+ }
}
diff --git a/lektor_nkdb/src/lib.rs b/lektor_nkdb/src/lib.rs
index 7ad89a4e..3e3aec64 100644
--- a/lektor_nkdb/src/lib.rs
+++ b/lektor_nkdb/src/lib.rs
@@ -142,8 +142,15 @@ impl<Storage: DatabaseStorage> Database<Storage> {
}
/// Search the database with a specific regex.
- pub async fn search(&self, from: SearchFrom, regex: KaraBy) -> Result<Vec<KId>> {
- let regex = Search::new(regex)?;
+ pub async fn search(&self, from: SearchFrom, regex: Vec<KaraBy>) -> Result<Vec<KId>> {
+ let (regex, error): (Vec<_>, Vec<_>) = regex
+ .into_iter()
+ .map(SearchBy::new)
+ .partition(Result::is_ok);
+ if let Some(Err(err)) = error.into_iter().next() {
+ bail!("failed to search karas: {err}");
+ }
+ let regex = SearchBy::from_iter(regex.into_iter().map(Result::unwrap));
let kids = match from {
SearchFrom::Playlist(plt) => self.playlists.get_content(plt).await.unwrap_or_default(),
SearchFrom::History => self.history(RangeFull).await,
@@ -180,7 +187,7 @@ impl<Storage: DatabaseStorage> Database<Storage> {
}
/// Returns the kara count from the search set.
- pub async fn count(&self, from: SearchFrom, regex: KaraBy) -> usize {
+ pub async fn count(&self, from: SearchFrom, regex: Vec<KaraBy>) -> usize {
log::error!("find a way to not allocate the search result buffer...");
self.search(from, regex)
.await
@@ -193,6 +200,11 @@ impl<Storage: DatabaseStorage> Database<Storage> {
self.epochs.last().await
}
+ /// Get the last epoch from the database.
+ pub async fn last_epoch_num(&self) -> Option<u64> {
+ self.epochs.last().await.map(|epoch| epoch.epoch_num())
+ }
+
/// Get the current kara with the play state.
///
/// We have to do shenanigans. Because between karas we have an idle peridod and the playback
diff --git a/lektor_nkdb/src/search/mod.rs b/lektor_nkdb/src/search/mod.rs
index cce5b95d..d93ecd57 100644
--- a/lektor_nkdb/src/search/mod.rs
+++ b/lektor_nkdb/src/search/mod.rs
@@ -6,14 +6,11 @@ mod kara_by;
pub use kara_by::*;
use crate::{playlist::PlaylistName, Kara};
+use anyhow::Result;
use kurisu_api::v2::{SongOrigin, SongType};
use regex::Regex;
use serde::{Deserialize, Serialize};
-/// Structure wrapping a regex to fuzzy search into the database or queue for matching karas.
-#[derive(Debug)]
-pub(crate) struct Search(SearchBy);
-
/// Structure to tell from which KId set we are searching.
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
pub enum SearchFrom {
@@ -44,6 +41,10 @@ impl FromIterator<SearchBy> for SearchBy {
}
impl SearchBy {
+ pub(crate) fn new(regex: KaraBy) -> Result<Self> {
+ regex.try_into()
+ }
+
/// Get the list of playlist that are needed for the kara to match. This is the only
/// informations that is not present in the epoch and thus need to be handled differently...
pub(crate) fn into_needed_playlists(self) -> Vec<String> {
@@ -80,22 +81,3 @@ impl SearchBy {
}
}
}
-
-impl Search {
- /// Create a new search regex. We only accept alphanumeric stuff to avoid vulnerabilities
- /// around regex from untrusted sources. We will do the fuzzy stuff latter.
- pub(crate) fn new(regex: KaraBy) -> anyhow::Result<Self> {
- Ok(Self(SearchBy::try_from(regex)?))
- }
-
- /// Get the list of playlist that are needed for the kara to match. This is the only
- /// informations that is not present in the epoch and thus need to be handled differently...
- pub(crate) fn into_needed_playlists(self) -> Vec<String> {
- self.0.into_needed_playlists()
- }
-
- /// A match function.
- pub(crate) fn matches(&self, kara: &Kara) -> bool {
- self.0.matches(kara)
- }
-}
diff --git a/lektor_nkdb/src/storage/disk_storage.rs b/lektor_nkdb/src/storage/disk_storage.rs
index 6dc408fd..4de66f2b 100644
--- a/lektor_nkdb/src/storage/disk_storage.rs
+++ b/lektor_nkdb/src/storage/disk_storage.rs
@@ -319,7 +319,7 @@ impl DatabaseStorage for DatabaseDiskStorage {
})
}
- async fn read_last_epoch(&self) -> Result<Option<EpochData>> {
+ async fn read_last_epoch(&self) -> Result<Option<(EpochData, u64)>> {
read_json_folder! {
(self) "epoch" [get_regex_epoch_ok(), get_regex_epoch_json()] => u64;
valid_epochs => { match valid_epochs.into_iter().max() {
@@ -342,7 +342,7 @@ impl DatabaseStorage for DatabaseDiskStorage {
data.remove(&kid);
});
log::info!("load epoch {id} with {} karas from path: {}", data.len(), path.to_string_lossy());
- Ok(Some(data))
+ Ok(Some((data, *id)))
}
}}
}
diff --git a/lektor_nkdb/src/storage/mod.rs b/lektor_nkdb/src/storage/mod.rs
index 974ba4cc..e4348092 100644
--- a/lektor_nkdb/src/storage/mod.rs
+++ b/lektor_nkdb/src/storage/mod.rs
@@ -34,7 +34,7 @@ pub trait DatabaseStorage: Sized + std::fmt::Debug {
// =========== //
/// Read the last valid epoch from disk/memory/network/like-the-implementation-does.
- async fn read_last_epoch(&self) -> Result<Option<EpochData>>;
+ async fn read_last_epoch(&self) -> Result<Option<(EpochData, u64)>>;
/// Write the data from an epoch to the disk.
async fn write_epoch(&self, epoch: &EpochData) -> Result<()>;
diff --git a/lektor_nkdb/src/storage/test_storage.rs b/lektor_nkdb/src/storage/test_storage.rs
index e75c5b3e..49b41847 100644
--- a/lektor_nkdb/src/storage/test_storage.rs
+++ b/lektor_nkdb/src/storage/test_storage.rs
@@ -23,8 +23,8 @@ impl DatabaseStorage for DatabaseTestStorage {
Ok(Self(Default::default()))
}
- async fn read_last_epoch(&self) -> Result<Option<EpochData>> {
- Ok(Some(self.0.read().await.clone()))
+ async fn read_last_epoch(&self) -> Result<Option<(EpochData, u64)>> {
+ Ok(Some((self.0.read().await.clone(), 1)))
}
async fn write_epoch(&self, data: &EpochData) -> Result<()> {
diff --git a/lektor_payloads/src/lib.rs b/lektor_payloads/src/lib.rs
index b05cd017..33964b83 100644
--- a/lektor_payloads/src/lib.rs
+++ b/lektor_payloads/src/lib.rs
@@ -3,26 +3,22 @@
mod playlist_name;
mod range;
+mod search;
mod userid;
-pub use crate::{playlist_name::*, range::*, userid::LektorUser};
-use anyhow::{anyhow, bail};
+pub use crate::{playlist_name::*, range::*, search::*, userid::LektorUser};
pub use lektor_nkdb::{
KId, Kara, KaraBy, KaraStatus, KaraTimeStamps, PlayState, Playlist, PlaylistInfo, Priority,
RemoteKId, SearchFrom, SongOrigin, SongType, PRIORITY_LENGTH, PRIORITY_VALUES,
};
+use anyhow::{anyhow, bail};
use serde::{Deserialize, Serialize};
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
pub struct Infos {
pub version: String,
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
-pub struct SearchData {
- pub from: SearchFrom,
- pub regex: KaraBy,
+ pub last_epoch: Option<u64>,
}
#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
@@ -59,20 +55,6 @@ pub struct AddArguments {
pub query: KaraBy,
}
-/// Add to something (playlist/queue/...), or remove something. Some times we can decide to shuffle
-/// the set of kara/the playlist before adding it. For the removing the shuffle flag is ignored.
-#[derive(Debug, Serialize, Deserialize)]
-pub enum KaraFilter {
- /// A single kara.
- KId(KId),
-
- /// A set of karas.
- List(bool, Vec<KId>),
-
- /// The content of a playlist.
- Playlist(bool, PlaylistName),
-}
-
/// Add to the queue. We add at a certain priority, some times we can decide to shuffle the set of
/// kara/the playlist before adding it to the queue.
#[derive(Debug, Serialize, Deserialize)]
@@ -123,39 +105,3 @@ impl std::str::FromStr for AddArguments {
})
}
}
-
-#[cfg(test)]
-mod test {
- use super::*;
- use anyhow::Result;
- use lektor_utils::assert_ok;
- use serde::{Deserialize, Serialize};
- use serde_json::json;
-
- /// Test the json serialization/deserialization to check if we obtains the same object back.
- fn assert_serde<T: Serialize + for<'de> Deserialize<'de> + std::fmt::Debug + std::cmp::Eq>(
- obj: T,
- ) -> Result<()> {
- let res = serde_json::from_str(&serde_json::to_string(&obj)?)?;
- assert_eq!(obj, res);
- Ok(())
- }
-
- #[test]
- fn json_search_data() -> Result<()> {
- assert_serde(SearchData {
- from: SearchFrom::Database,
- regex: KaraBy::Id(42),
- })?;
- assert_serde(SearchData {
- from: SearchFrom::Playlist("jibun".parse().unwrap()),
- regex: KaraBy::Query("Chicka".to_string()),
- })?;
-
- assert_ok!(serde_json::from_value::<SearchData>(json!(
- {"from":"Database","regex":{"Query":"chicka"}}
- )));
-
- Ok(())
- }
-}
diff --git a/lektor_payloads/src/search.rs b/lektor_payloads/src/search.rs
new file mode 100644
index 00000000..bc4567b5
--- /dev/null
+++ b/lektor_payloads/src/search.rs
@@ -0,0 +1,60 @@
+use crate::*;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Eq)]
+pub struct SearchData {
+ pub from: SearchFrom,
+ pub regex: Vec<KaraBy>,
+}
+
+/// Add to something (playlist/queue/...), or remove something. Some times we can decide to shuffle
+/// the set of kara/the playlist before adding it. For the removing the shuffle flag is ignored.
+#[derive(Debug, Serialize, Deserialize)]
+pub enum KaraFilter {
+ /// A single kara.
+ KId(KId),
+
+ /// A set of karas.
+ List(bool, Vec<KId>),
+
+ /// The content of a playlist.
+ Playlist(bool, PlaylistName),
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+ use anyhow::Result;
+ use lektor_utils::assert_ok;
+ use serde::{Deserialize, Serialize};
+ use serde_json::json;
+
+ /// Test the json serialization/deserialization to check if we obtains the same object back.
+ fn assert_serde<T: Serialize + for<'de> Deserialize<'de> + std::fmt::Debug + std::cmp::Eq>(
+ obj: T,
+ ) -> Result<()> {
+ let res = serde_json::from_str(&serde_json::to_string(&obj)?)?;
+ assert_eq!(obj, res);
+ Ok(())
+ }
+
+ #[test]
+ fn json_search_data() -> Result<()> {
+ assert_serde(SearchData {
+ from: SearchFrom::Database,
+ regex: vec![KaraBy::Id(42)],
+ })?;
+ assert_serde(SearchData {
+ from: SearchFrom::Playlist("jibun".parse().unwrap()),
+ regex: vec![KaraBy::Query("Chicka".to_string())],
+ })?;
+
+ assert_ok!(serde_json::from_value::<SearchData>(json!(
+ { "from": "Database"
+ , "regex": [ {"Query":"chicka"} ]
+ }
+ )));
+
+ Ok(())
+ }
+}
diff --git a/lektord/src/routes.rs b/lektord/src/routes.rs
index b128ce81..fca68aef 100644
--- a/lektord/src/routes.rs
+++ b/lektord/src/routes.rs
@@ -20,9 +20,10 @@ use std::{ops::RangeBounds, str::FromStr};
use tokio::task::LocalSet;
/// Get informations abount the lektord server.
-pub(crate) async fn root() -> Json<Infos> {
+pub(crate) async fn root(State(state): State<LektorStatePtr>) -> Json<Infos> {
Json(Infos {
version: lektor_utils::version().to_string(),
+ last_epoch: state.database.last_epoch_num().await,
})
}
diff --git a/lkt/src/main.rs b/lkt/src/main.rs
index 55c519fb..1742cf74 100644
--- a/lkt/src/main.rs
+++ b/lkt/src/main.rs
@@ -4,8 +4,6 @@ mod args;
mod config;
mod manpage;
-use std::ops::RangeBounds;
-
use crate::{args::*, config::*};
use anyhow::{anyhow, Result};
use chrono::TimeZone;
@@ -13,6 +11,7 @@ use futures::{stream, StreamExt};
use lektor_lib::*;
use lektor_payloads::*;
use lektor_utils::*;
+use std::{borrow::Cow, ops::RangeBounds};
fn main() -> Result<()> {
logger::init(Some(log::Level::Trace)).expect("failed to install logger");
@@ -91,7 +90,10 @@ async fn exec_lkt(config: LktConfig, cmd: SubCommand) -> Result<()> {
// Display the status of lektord
Playback { status: true, .. } => {
- let Infos { version } = requests::get_infos(config.clone()).await?;
+ let Infos {
+ version,
+ last_epoch,
+ } = requests::get_infos(config.clone()).await?;
let PlayStateWithCurrent { state, current } =
requests::get_status(config.clone()).await?;
let current = match current {
@@ -104,6 +106,10 @@ async fn exec_lkt(config: LktConfig, cmd: SubCommand) -> Result<()> {
};
let queue_counts = requests::get_queue_count(config.clone()).await?;
let history_count = requests::get_history_count(config).await?;
+ let last_epoch = match last_epoch {
+ Some(num) => num.to_string().into(),
+ None => Cow::Borrowed("None"),
+ };
println!("Version: {version}");
println!("Playback State: {state:?}");
@@ -114,6 +120,7 @@ async fn exec_lkt(config: LktConfig, cmd: SubCommand) -> Result<()> {
}
println!("Karas in Queue: {queue_counts:?}");
println!("Karas in Hustory: {history_count}");
+ println!("Database epoch: {last_epoch}");
Ok(())
}
--
GitLab