From bf6a15fac11ebf83f2249661aa1d52b2a769812c Mon Sep 17 00:00:00 2001 From: Kubat <mael.martin31@gmail.com> Date: Sun, 2 May 2021 15:57:35 +0200 Subject: [PATCH] DB: Cache the 'mkv_magic_error' into memory for each kara --- inc/lektor/database.h | 5 +++++ src/base/launch.c | 49 +++++++++++++++++++++++++++++++++++------ src/database/cache.c | 29 ++++++++++++++++++++++++ src/database/find.c | 31 +++++++++++++++++++++----- src/database/memory.sql | 3 ++- src/database/open.c | 4 ++-- src/database/update.c | 9 ++++---- 7 files changed, 109 insertions(+), 21 deletions(-) diff --git a/inc/lektor/database.h b/inc/lektor/database.h index 4f4ef9cc..71474060 100644 --- a/inc/lektor/database.h +++ b/inc/lektor/database.h @@ -64,9 +64,14 @@ bool database_get_kara_duration_id (volatile sqlite3 *db, int id, uint64_t *dur bool database_get_kara_mtime_path (volatile sqlite3 *db, char filepath[PATH_MAX], uint64_t *mtime); bool database_get_kara_duration_path(volatile sqlite3 *db, char filepath[PATH_MAX], uint64_t *duration); void database_cache_kara (volatile sqlite3 *db, int id); +void database_cache_kara_magic (volatile sqlite3 *db, int id); +/* Get a list of all non-cached karas from the disk */ void database_get_all_non_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len); +/* Get a list of all cached karas from the memory */ +void database_get_all_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len); + /* Update the database. */ bool database_update (volatile sqlite3 *db, const char *kara_dir, int check_timestamp); bool database_update_add (volatile sqlite3 *db, const char *kara_path, struct kara_metadata *mdt, uint64_t id, bool avail); diff --git a/src/base/launch.c b/src/base/launch.c index 08bb9206..c86d73da 100644 --- a/src/base/launch.c +++ b/src/base/launch.c @@ -184,29 +184,64 @@ ___sleep(void) nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */ } -PRIVATE_FUNCTION void * -___caching_walker(void *args) +/* Cache karas into disk */ +PRIVATE_FUNCTION void +___cache_kara(volatile sqlite3 *db) { - volatile sqlite3 *db = (volatile sqlite3 *)args; - size_t ids_count = 0; - int *ids = NULL; + size_t ids_count = 0; + int *ids = NULL; database_get_all_non_cached_kara(db, &ids, &ids_count); if (ids == NULL || ids_count == 0) { - LOG_ERROR("CACHING", "No kara present in the database, exit caching thread"); - return NULL; + LOG_INFO("CACHING", "No kara present in the database to cache"); + return; } LOG_INFO("CACHE", "Begin to cache %ld karas", ids_count); for (size_t index = 0; index < ids_count; ++index) { database_cache_kara(db, ids[index]); + /* Don't burn the disk */ if ((index % 10) == 0) ___sleep(); } + free(ids); LOG_INFO("CACHE", "Finished caching %ld karas", ids_count); +} + +/* Check magic number */ +PRIVATE_FUNCTION void +___check_kara(volatile sqlite3 *db) +{ + size_t ids_count = 0; + int *ids = NULL; + + database_get_all_cached_kara(db, &ids, &ids_count); + if (ids == NULL || ids_count == 0) { + LOG_ERROR("CACHING", "No kara present in the database cache"); + return; + } + + LOG_INFO("CACHE", "Begin to check mkv magic number for %ld karas", ids_count); + + for (size_t index = 0; index < ids_count; ++index) { + database_cache_kara_magic(db, ids[index]); + /* Don't burn the disk */ + if ((index % 10) == 0) + ___sleep(); + } + free(ids); + LOG_INFO("CACHE", "Finished checking mkv magic number for %ld karas", ids_count); +} + +PRIVATE_FUNCTION void * +___caching_walker(void *args) +{ + volatile sqlite3 *db = (volatile sqlite3 *)args; + ___cache_kara(db); + ___check_kara(db); return NULL; } diff --git a/src/database/cache.c b/src/database/cache.c index 0aa526b8..719a4735 100644 --- a/src/database/cache.c +++ b/src/database/cache.c @@ -35,6 +35,35 @@ get_mtime(const char *path) return (stat(path, &statbuf) == -1) ? 0 : statbuf.st_mtime; } +void +database_cache_kara_magic(volatile sqlite3 *db, int id) +{ + /* Get the filepath */ + char filename[PATH_MAX]; + memset(filename, 0, sizeof(filename)); + database_get_kara_path(db, id, filename); + if (filename[0] == 0) { + LOG_ERROR("CACHE", "Failed to get filename for kara %ld", id); + return; + } + + /* Update the mkv magic error */ + int magic_error = mkv_check_magic(filename); + sqlite3_stmt *stmt = NULL; + static const char *SQL = "UPDATE kara_cache SET magic_error = ? WHERE kara_id = ?;"; + SQLITE_PREPARE(db, stmt, SQL, error); + SQLITE_BIND_INT(db, stmt, 1, magic_error, error); + SQLITE_BIND_INT(db, stmt, 2, id, error); + SQLITE_STEP_OK(db, stmt, error); + sqlite3_finalize(stmt); + return; + +error: + LOG_ERROR("CACHE", "Failed to set mkv magic error to %d for kara %ld", magic_error, id); + sqlite3_finalize(stmt); + return; +} + /* Request to cache a kara */ void database_cache_kara(volatile sqlite3 *db, int id) diff --git a/src/database/find.c b/src/database/find.c index 7bea8d7f..1e1808e4 100644 --- a/src/database/find.c +++ b/src/database/find.c @@ -398,18 +398,21 @@ error: return ret; } -void -database_get_all_non_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len) +PRIVATE_FUNCTION void +___get_all_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len, const char *SQL_DATA, const char *SQL_COUNT) { - static const char *SQL_DATA = "SELECT id FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;"; - static const char *SQL_COUNT = "SELECT count(id) FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;"; - sqlite3_stmt *stmt = NULL; - size_t index = 0; + sqlite3_stmt *stmt = NULL; + size_t index = 0; /* Get storage for all IDs */ SQLITE_PREPARE(db, stmt, SQL_COUNT, error); SQLITE_STEP_ROW(db, stmt, error); *ret_ids_len = sqlite3_column_int64(stmt, 0); + if (*ret_ids_len == 0) { + LOG_WARN("DB", "Got no kara for the specific 'all' query: %s", SQL_COUNT); + sqlite3_finalize(stmt); + return; + } sqlite3_finalize(stmt); stmt = NULL; *ret_ids = (int *)safe_malloc((*ret_ids_len) * sizeof(int)); @@ -437,3 +440,19 @@ error: *ret_ids_len = 0; return; } + +void +database_get_all_non_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len) +{ + static const char *SQL_DATA = "SELECT id FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;"; + static const char *SQL_COUNT = "SELECT count(id) FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;"; + ___get_all_kara(db, ret_ids, ret_ids_len, SQL_DATA, SQL_COUNT); +} + +void +database_get_all_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len) +{ + static const char *SQL_DATA = "SELECT kara_id FROM kara_cache;"; + static const char *SQL_COUNT = "SELECT count(kara_id) FROM kara_cache;"; + ___get_all_kara(db, ret_ids, ret_ids_len, SQL_DATA, SQL_COUNT); +} diff --git a/src/database/memory.sql b/src/database/memory.sql index 61843fdd..8b98cfef 100644 --- a/src/database/memory.sql +++ b/src/database/memory.sql @@ -34,10 +34,11 @@ CREATE TABLE IF NOT EXISTS queue_tmp -- Memory replica of the kara cache CREATE TABLE IF NOT EXISTS kara_cache - ( kara_id INTEGER -- Valid kara id, can't foreign key as the disk is not loaded + ( kara_id INTEGER NOT NULL -- Valid kara id, can't foreign key as the disk is not loaded , mtime INTEGER NOT NULL -- Last modification date , duration INTEGER NOT NULL -- Duration of the kara , file_path TEXT NOT NULL -- Cached filepath + , magic_error INTEGER NOT NULL -- 1 if the mkv is valid, 0 otherwise ); -- Used to store the content of the ini configuration file. diff --git a/src/database/open.c b/src/database/open.c index ba91f34d..3a0bde42 100644 --- a/src/database/open.c +++ b/src/database/open.c @@ -175,8 +175,8 @@ error: PRIVATE_FUNCTION void ___flush_cache_from_disk(volatile sqlite3 *db) { - static const char *SQL = "INSERT INTO kara_cache (kara_id, mtime, duration, file_path)" - " SELECT id, cached_mtime, cached_duration, file_path" + static const char *SQL = "INSERT INTO kara_cache (kara_id, mtime, duration, file_path, magic_error)" + " SELECT id, cached_mtime, cached_duration, file_path, 2" " FROM kara;"; SQLITE_EXEC(db, SQL, error); LOG_INFO("DB", "Kara cache was flushed from disk"); diff --git a/src/database/update.c b/src/database/update.c index 88c75cd9..16848bcd 100644 --- a/src/database/update.c +++ b/src/database/update.c @@ -45,11 +45,10 @@ error: PRIVATE_FUNCTION void ___flush_cache_from_disk(volatile sqlite3 *db, const char *filename) { - static const char *SQL = - "INSERT OR REPLACE INTO kara_cache (kara_id, file_path)" - " SELECT (id, file_path)" - " FROM kara" - " WHERE file_path = ?"; + static const char *SQL = "INSERT OR REPLACE INTO kara_cache (kara_id, file_path)" + " SELECT (id, file_path)" + " FROM kara" + " WHERE file_path = ?"; sqlite3_stmt *stmt = NULL; SQLITE_PREPARE(db, stmt, SQL, error); -- GitLab