From bf6a15fac11ebf83f2249661aa1d52b2a769812c Mon Sep 17 00:00:00 2001
From: Kubat <mael.martin31@gmail.com>
Date: Sun, 2 May 2021 15:57:35 +0200
Subject: [PATCH] DB: Cache the 'mkv_magic_error' into memory for each kara

---
 inc/lektor/database.h   |  5 +++++
 src/base/launch.c       | 49 +++++++++++++++++++++++++++++++++++------
 src/database/cache.c    | 29 ++++++++++++++++++++++++
 src/database/find.c     | 31 +++++++++++++++++++++-----
 src/database/memory.sql |  3 ++-
 src/database/open.c     |  4 ++--
 src/database/update.c   |  9 ++++----
 7 files changed, 109 insertions(+), 21 deletions(-)

diff --git a/inc/lektor/database.h b/inc/lektor/database.h
index 4f4ef9cc..71474060 100644
--- a/inc/lektor/database.h
+++ b/inc/lektor/database.h
@@ -64,9 +64,14 @@ bool database_get_kara_duration_id  (volatile sqlite3 *db, int id, uint64_t *dur
 bool database_get_kara_mtime_path   (volatile sqlite3 *db, char filepath[PATH_MAX], uint64_t *mtime);
 bool database_get_kara_duration_path(volatile sqlite3 *db, char filepath[PATH_MAX], uint64_t *duration);
 void database_cache_kara            (volatile sqlite3 *db, int id);
+void database_cache_kara_magic      (volatile sqlite3 *db, int id);
 
+/* Get a list of all non-cached karas from the disk */
 void database_get_all_non_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len);
 
+/* Get a list of all cached karas from the memory */
+void database_get_all_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len);
+
 /* Update the database. */
 bool database_update              (volatile sqlite3 *db, const char *kara_dir, int check_timestamp);
 bool database_update_add          (volatile sqlite3 *db, const char *kara_path, struct kara_metadata *mdt, uint64_t id, bool avail);
diff --git a/src/base/launch.c b/src/base/launch.c
index 08bb9206..c86d73da 100644
--- a/src/base/launch.c
+++ b/src/base/launch.c
@@ -184,29 +184,64 @@ ___sleep(void)
     nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */
 }
 
-PRIVATE_FUNCTION void *
-___caching_walker(void *args)
+/* Cache karas into disk */
+PRIVATE_FUNCTION void
+___cache_kara(volatile sqlite3 *db)
 {
-    volatile sqlite3 *db = (volatile sqlite3 *)args;
-    size_t ids_count     = 0;
-    int *ids             = NULL;
+    size_t ids_count = 0;
+    int *ids         = NULL;
 
     database_get_all_non_cached_kara(db, &ids, &ids_count);
     if (ids == NULL || ids_count == 0) {
-        LOG_ERROR("CACHING", "No kara present in the database, exit caching thread");
-        return NULL;
+        LOG_INFO("CACHING", "No kara present in the database to cache");
+        return;
     }
 
     LOG_INFO("CACHE", "Begin to cache %ld karas", ids_count);
 
     for (size_t index = 0; index < ids_count; ++index) {
         database_cache_kara(db, ids[index]);
+        /* Don't burn the disk */
         if ((index % 10) == 0)
             ___sleep();
     }
 
+    free(ids);
     LOG_INFO("CACHE", "Finished caching %ld karas", ids_count);
+}
+
+/* Check magic number */
+PRIVATE_FUNCTION void
+___check_kara(volatile sqlite3 *db)
+{
+    size_t ids_count = 0;
+    int *ids         = NULL;
+
+    database_get_all_cached_kara(db, &ids, &ids_count);
+    if (ids == NULL || ids_count == 0) {
+        LOG_ERROR("CACHING", "No kara present in the database cache");
+        return;
+    }
+
+    LOG_INFO("CACHE", "Begin to check mkv magic number for %ld karas", ids_count);
+
+    for (size_t index = 0; index < ids_count; ++index) {
+        database_cache_kara_magic(db, ids[index]);
+        /* Don't burn the disk */
+        if ((index % 10) == 0)
+            ___sleep();
+    }
+
     free(ids);
+    LOG_INFO("CACHE", "Finished checking mkv magic number for %ld karas", ids_count);
+}
+
+PRIVATE_FUNCTION void *
+___caching_walker(void *args)
+{
+    volatile sqlite3 *db = (volatile sqlite3 *)args;
+    ___cache_kara(db);
+    ___check_kara(db);
     return NULL;
 }
 
diff --git a/src/database/cache.c b/src/database/cache.c
index 0aa526b8..719a4735 100644
--- a/src/database/cache.c
+++ b/src/database/cache.c
@@ -35,6 +35,35 @@ get_mtime(const char *path)
     return (stat(path, &statbuf) == -1) ? 0 : statbuf.st_mtime;
 }
 
+void
+database_cache_kara_magic(volatile sqlite3 *db, int id)
+{
+    /* Get the filepath */
+    char filename[PATH_MAX];
+    memset(filename, 0, sizeof(filename));
+    database_get_kara_path(db, id, filename);
+    if (filename[0] == 0) {
+        LOG_ERROR("CACHE", "Failed to get filename for kara %ld", id);
+        return;
+    }
+
+    /* Update the mkv magic error */
+    int magic_error        = mkv_check_magic(filename);
+    sqlite3_stmt *stmt     = NULL;
+    static const char *SQL = "UPDATE kara_cache SET magic_error = ? WHERE kara_id = ?;";
+    SQLITE_PREPARE(db, stmt, SQL, error);
+    SQLITE_BIND_INT(db, stmt, 1, magic_error, error);
+    SQLITE_BIND_INT(db, stmt, 2, id, error);
+    SQLITE_STEP_OK(db, stmt, error);
+    sqlite3_finalize(stmt);
+    return;
+
+error:
+    LOG_ERROR("CACHE", "Failed to set mkv magic error to %d for kara %ld", magic_error, id);
+    sqlite3_finalize(stmt);
+    return;
+}
+
 /* Request to cache a kara */
 void
 database_cache_kara(volatile sqlite3 *db, int id)
diff --git a/src/database/find.c b/src/database/find.c
index 7bea8d7f..1e1808e4 100644
--- a/src/database/find.c
+++ b/src/database/find.c
@@ -398,18 +398,21 @@ error:
     return ret;
 }
 
-void
-database_get_all_non_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len)
+PRIVATE_FUNCTION void
+___get_all_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len, const char *SQL_DATA, const char *SQL_COUNT)
 {
-    static const char *SQL_DATA  = "SELECT id FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;";
-    static const char *SQL_COUNT = "SELECT count(id) FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;";
-    sqlite3_stmt *stmt           = NULL;
-    size_t index                 = 0;
+    sqlite3_stmt *stmt = NULL;
+    size_t index       = 0;
 
     /* Get storage for all IDs */
     SQLITE_PREPARE(db, stmt, SQL_COUNT, error);
     SQLITE_STEP_ROW(db, stmt, error);
     *ret_ids_len = sqlite3_column_int64(stmt, 0);
+    if (*ret_ids_len == 0) {
+        LOG_WARN("DB", "Got no kara for the specific 'all' query: %s", SQL_COUNT);
+        sqlite3_finalize(stmt);
+        return;
+    }
     sqlite3_finalize(stmt);
     stmt     = NULL;
     *ret_ids = (int *)safe_malloc((*ret_ids_len) * sizeof(int));
@@ -437,3 +440,19 @@ error:
     *ret_ids_len = 0;
     return;
 }
+
+void
+database_get_all_non_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len)
+{
+    static const char *SQL_DATA  = "SELECT id FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;";
+    static const char *SQL_COUNT = "SELECT count(id) FROM kara WHERE cached_mtime = 0 OR cached_duration = 0;";
+    ___get_all_kara(db, ret_ids, ret_ids_len, SQL_DATA, SQL_COUNT);
+}
+
+void
+database_get_all_cached_kara(volatile sqlite3 *db, int **ret_ids, size_t *ret_ids_len)
+{
+    static const char *SQL_DATA  = "SELECT kara_id FROM kara_cache;";
+    static const char *SQL_COUNT = "SELECT count(kara_id) FROM kara_cache;";
+    ___get_all_kara(db, ret_ids, ret_ids_len, SQL_DATA, SQL_COUNT);
+}
diff --git a/src/database/memory.sql b/src/database/memory.sql
index 61843fdd..8b98cfef 100644
--- a/src/database/memory.sql
+++ b/src/database/memory.sql
@@ -34,10 +34,11 @@ CREATE TABLE IF NOT EXISTS queue_tmp
 
 -- Memory replica of the kara cache
 CREATE TABLE IF NOT EXISTS kara_cache
-  ( kara_id     INTEGER             -- Valid kara id, can't foreign key as the disk is not loaded
+  ( kara_id     INTEGER NOT NULL    -- Valid kara id, can't foreign key as the disk is not loaded
   , mtime       INTEGER NOT NULL    -- Last modification date
   , duration    INTEGER NOT NULL    -- Duration of the kara
   , file_path   TEXT    NOT NULL    -- Cached filepath
+  , magic_error INTEGER NOT NULL    -- 1 if the mkv is valid, 0 otherwise
   );
 
 -- Used to store the content of the ini configuration file.
diff --git a/src/database/open.c b/src/database/open.c
index ba91f34d..3a0bde42 100644
--- a/src/database/open.c
+++ b/src/database/open.c
@@ -175,8 +175,8 @@ error:
 PRIVATE_FUNCTION void
 ___flush_cache_from_disk(volatile sqlite3 *db)
 {
-    static const char *SQL = "INSERT INTO kara_cache (kara_id, mtime, duration, file_path)"
-                             " SELECT id, cached_mtime, cached_duration, file_path"
+    static const char *SQL = "INSERT INTO kara_cache (kara_id, mtime, duration, file_path, magic_error)"
+                             " SELECT id, cached_mtime, cached_duration, file_path, 2"
                              " FROM kara;";
     SQLITE_EXEC(db, SQL, error);
     LOG_INFO("DB", "Kara cache was flushed from disk");
diff --git a/src/database/update.c b/src/database/update.c
index 88c75cd9..16848bcd 100644
--- a/src/database/update.c
+++ b/src/database/update.c
@@ -45,11 +45,10 @@ error:
 PRIVATE_FUNCTION void
 ___flush_cache_from_disk(volatile sqlite3 *db, const char *filename)
 {
-    static const char *SQL =
-        "INSERT OR REPLACE INTO kara_cache (kara_id, file_path)"
-        " SELECT (id, file_path)"
-        " FROM kara"
-        " WHERE file_path = ?";
+    static const char *SQL = "INSERT OR REPLACE INTO kara_cache (kara_id, file_path)"
+                             " SELECT (id, file_path)"
+                             " FROM kara"
+                             " WHERE file_path = ?";
     sqlite3_stmt *stmt = NULL;
 
     SQLITE_PREPARE(db, stmt, SQL, error);
-- 
GitLab