Add update jobs

inc/lektor/database.h

@@ -22,7 +22,7 @@ struct lkt_queue_state {
     int length;
 };
 
-long database_get_timestamp(volatile sqlite3 *db);
+void database_get_update(volatile sqlite3 *db, long *timestamp, long *job, int *current);
 void database_stamp     (volatile sqlite3 *db);
 void database_updated   (volatile sqlite3 *db);
 

scripts/init.sql

@@ -119,6 +119,18 @@ CREATE TABLE IF NOT EXISTS 'stickers.plt'
   ) WITHOUT ROWID;
 
 
+-- Update jobs history
+-- Here are stored karas ids present during a certain update job. May be
+-- usefull to keep tracks of kara. Now used to handle the 'deleted' case, i.e.
+-- when a kara was here, but was deleted on kurisu.
+
+CREATE TABLE IF NOT EXISTS updates
+  ( job     INTEGER NOT NULL CHECK(job > 0)
+  , kara_id INTEGER NOT NULL CHECK(kara_id > 0)
+  , PRIMARY KEY (job, kara_id)
+  ) WITHOUT ROWID;
+
+
 -- Some useful values:
 -- last_update is the timestamp of the last time the table of kara has been
 -- updated. This is so lektor doesn't have to read all kara in the filesystem,
@@ -131,6 +143,7 @@ CREATE TABLE IF NOT EXISTS misc
   ( id              INTEGER PRIMARY KEY DEFAULT 42 CHECK(id = 42)
   , last_update     INTEGER
   , last_end_update INTEGER
+  , update_job      INTEGER NOT NULL CHECK(update_job >= 0)
   );
 
-INSERT OR REPLACE INTO misc (id) VALUES (42);
+INSERT OR REPLACE INTO misc (id, update_job) VALUES (42, 0);

src/database/open.c

@@ -190,18 +190,22 @@ error:
     return false;
 }
 
-long
-database_get_timestamp(volatile sqlite3 *db)
+void
+database_get_update(volatile sqlite3 *db, long *timestamp, long *job, int *current)
 {
-    long ret = 0;
-    static const char *SQL = "SELECT last_update FROM misc WHERE id = 42;";
+    static const char *SQL = "SELECT last_update, update_job, last_update > last_end_update FROM misc WHERE id = 42;";
     sqlite3_stmt *stmt;
     SQLITE_PREPARE(db, stmt, SQL, error);
     SQLITE_STEP_ROW(db, stmt, error);
-    ret = sqlite3_column_int(stmt, 0);
+    if (timestamp)
+        *timestamp = sqlite3_column_int(stmt, 0);
+    if (job)
+        *job = sqlite3_column_int(stmt, 1);
+    if (current)
+        *current = sqlite3_column_int(stmt, 2);
+    return;
 error:
-    sqlite3_finalize(stmt);
-    return ret;
+    LOG_WARN_SCT("DB", "Failed to get informations about the last update: %s", sqlite3_errmsg((sqlite3 *) db));
 }
 
 void
@@ -215,7 +219,7 @@ error:
 void
 database_updated(volatile sqlite3 *db)
 {
-    SQLITE_EXEC(db, "UPDATE misc SET last_end_update = strftime('%s','now');", error);
+    SQLITE_EXEC(db, "UPDATE misc SET last_end_update = strftime('%s','now'), update_job = update_job + 1;", error);
 error:
     return;
 }

src/database/update.c

@@ -148,6 +148,7 @@ database_update(volatile sqlite3 *db, const char *kara_dir, int check_timestamp)
     DIR *d;
     struct dirent *dir;
     char path[PATH_MAX];
+    long db_timestamp = 0;
     memset(path, 0, PATH_MAX * sizeof(char));
 
     if (!(d = opendir(kara_dir))) {
@@ -161,7 +162,8 @@ database_update(volatile sqlite3 *db, const char *kara_dir, int check_timestamp)
         strncat(path, dir->d_name, PATH_MAX - 1);
 
         if (dir->d_type == DT_REG) {
-            if (check_timestamp && get_mtime(path) < database_get_timestamp(db)) {
+            database_get_update(db, &db_timestamp, NULL, NULL);
+            if (check_timestamp && get_mtime(path) < db_timestamp) {
                 LOG_INFO_SCT("DB", "Skip update of kara '%s' be cause of timestamps", path);
                 continue;
             }

src/net/downloader.c

@@ -324,7 +324,7 @@ __handle_got_json(volatile sqlite3 *db, struct lkt_repo *repo, struct json_objec
     int32_t integer, err;
     struct kara kara;
     char url[URL_MAX_LEN];
-    long filestamp, timestamp;
+    long filestamp = 0, timestamp = 0, db_timestamp = 0;
     char mkvpropedit[PATH_MAX];
     RETURN_UNLESS(len > 0 && json_object_get_array(json), "Json invalid or array empty", NOTHING);
     RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit", mkvpropedit, PATH_MAX - 1),
@@ -365,7 +365,8 @@ __handle_got_json(volatile sqlite3 *db, struct lkt_repo *repo, struct json_objec
         if (safe_json_get_long(kara_json, "unix_timestamp", &timestamp))
             continue;
         filestamp = get_mtime(kara.filename);
-        if (database_get_timestamp(db) >= filestamp && filestamp > timestamp &&
+        database_get_update(db, &db_timestamp, NULL, NULL);
+        if (db_timestamp >= filestamp && filestamp > timestamp &&
             ! kara_metadata_equals(&kara.mdt, kara.filename)) {
             LOG_INFO_SCT("REPO", "Ignore kara '%ld' with path '%s'", kara.id, kara.filename);
             continue;