From 25e81392efede46b15cb371e7ceff378196eda29 Mon Sep 17 00:00:00 2001
From: Kubat <mael.martin31@gmail.com>
Date: Thu, 7 May 2020 10:59:27 +0200
Subject: [PATCH] Add update jobs
---
inc/lektor/database.h | 6 +++---
scripts/init.sql | 15 ++++++++++++++-
src/database/open.c | 20 ++++++++++++--------
src/database/update.c | 4 +++-
src/net/downloader.c | 5 +++--
5 files changed, 35 insertions(+), 15 deletions(-)
diff --git a/inc/lektor/database.h b/inc/lektor/database.h
index aa2b4aab..d1e31ddf 100644
--- a/inc/lektor/database.h
+++ b/inc/lektor/database.h
@@ -22,9 +22,9 @@ struct lkt_queue_state {
int length;
};
-long database_get_timestamp(volatile sqlite3 *db);
-void database_stamp (volatile sqlite3 *db);
-void database_updated (volatile sqlite3 *db);
+void database_get_update(volatile sqlite3 *db, long *timestamp, long *job, int *current);
+void database_stamp (volatile sqlite3 *db);
+void database_updated (volatile sqlite3 *db);
/* Open correctly a database for lektor. */
bool database_new (volatile sqlite3 **db);
diff --git a/scripts/init.sql b/scripts/init.sql
index d4e3e1c3..99029bd9 100644
--- a/scripts/init.sql
+++ b/scripts/init.sql
@@ -119,6 +119,18 @@ CREATE TABLE IF NOT EXISTS 'stickers.plt'
) WITHOUT ROWID;
+-- Update jobs history
+-- Here are stored karas ids present during a certain update job. May be
+-- usefull to keep tracks of kara. Now used to handle the 'deleted' case, i.e.
+-- when a kara was here, but was deleted on kurisu.
+
+CREATE TABLE IF NOT EXISTS updates
+ ( job INTEGER NOT NULL CHECK(job > 0)
+ , kara_id INTEGER NOT NULL CHECK(kara_id > 0)
+ , PRIMARY KEY (job, kara_id)
+ ) WITHOUT ROWID;
+
+
-- Some useful values:
-- last_update is the timestamp of the last time the table of kara has been
-- updated. This is so lektor doesn't have to read all kara in the filesystem,
@@ -131,6 +143,7 @@ CREATE TABLE IF NOT EXISTS misc
( id INTEGER PRIMARY KEY DEFAULT 42 CHECK(id = 42)
, last_update INTEGER
, last_end_update INTEGER
+ , update_job INTEGER NOT NULL CHECK(update_job >= 0)
);
-INSERT OR REPLACE INTO misc (id) VALUES (42);
+INSERT OR REPLACE INTO misc (id, update_job) VALUES (42, 0);
diff --git a/src/database/open.c b/src/database/open.c
index 9c288868..c6d7a4e6 100644
--- a/src/database/open.c
+++ b/src/database/open.c
@@ -190,18 +190,22 @@ error:
return false;
}
-long
-database_get_timestamp(volatile sqlite3 *db)
+void
+database_get_update(volatile sqlite3 *db, long *timestamp, long *job, int *current)
{
- long ret = 0;
- static const char *SQL = "SELECT last_update FROM misc WHERE id = 42;";
+ static const char *SQL = "SELECT last_update, update_job, last_update > last_end_update FROM misc WHERE id = 42;";
sqlite3_stmt *stmt;
SQLITE_PREPARE(db, stmt, SQL, error);
SQLITE_STEP_ROW(db, stmt, error);
- ret = sqlite3_column_int(stmt, 0);
+ if (timestamp)
+ *timestamp = sqlite3_column_int(stmt, 0);
+ if (job)
+ *job = sqlite3_column_int(stmt, 1);
+ if (current)
+ *current = sqlite3_column_int(stmt, 2);
+ return;
error:
- sqlite3_finalize(stmt);
- return ret;
+ LOG_WARN_SCT("DB", "Failed to get informations about the last update: %s", sqlite3_errmsg((sqlite3 *) db));
}
void
@@ -215,7 +219,7 @@ error:
void
database_updated(volatile sqlite3 *db)
{
- SQLITE_EXEC(db, "UPDATE misc SET last_end_update = strftime('%s','now');", error);
+ SQLITE_EXEC(db, "UPDATE misc SET last_end_update = strftime('%s','now'), update_job = update_job + 1;", error);
error:
return;
}
diff --git a/src/database/update.c b/src/database/update.c
index 369df055..d5ad26f6 100644
--- a/src/database/update.c
+++ b/src/database/update.c
@@ -148,6 +148,7 @@ database_update(volatile sqlite3 *db, const char *kara_dir, int check_timestamp)
DIR *d;
struct dirent *dir;
char path[PATH_MAX];
+ long db_timestamp = 0;
memset(path, 0, PATH_MAX * sizeof(char));
if (!(d = opendir(kara_dir))) {
@@ -161,7 +162,8 @@ database_update(volatile sqlite3 *db, const char *kara_dir, int check_timestamp)
strncat(path, dir->d_name, PATH_MAX - 1);
if (dir->d_type == DT_REG) {
- if (check_timestamp && get_mtime(path) < database_get_timestamp(db)) {
+ database_get_update(db, &db_timestamp, NULL, NULL);
+ if (check_timestamp && get_mtime(path) < db_timestamp) {
LOG_INFO_SCT("DB", "Skip update of kara '%s' be cause of timestamps", path);
continue;
}
diff --git a/src/net/downloader.c b/src/net/downloader.c
index a076deef..2bda911e 100644
--- a/src/net/downloader.c
+++ b/src/net/downloader.c
@@ -324,7 +324,7 @@ __handle_got_json(volatile sqlite3 *db, struct lkt_repo *repo, struct json_objec
int32_t integer, err;
struct kara kara;
char url[URL_MAX_LEN];
- long filestamp, timestamp;
+ long filestamp = 0, timestamp = 0, db_timestamp = 0;
char mkvpropedit[PATH_MAX];
RETURN_UNLESS(len > 0 && json_object_get_array(json), "Json invalid or array empty", NOTHING);
RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit", mkvpropedit, PATH_MAX - 1),
@@ -365,7 +365,8 @@ __handle_got_json(volatile sqlite3 *db, struct lkt_repo *repo, struct json_objec
if (safe_json_get_long(kara_json, "unix_timestamp", ×tamp))
continue;
filestamp = get_mtime(kara.filename);
- if (database_get_timestamp(db) >= filestamp && filestamp > timestamp &&
+ database_get_update(db, &db_timestamp, NULL, NULL);
+ if (db_timestamp >= filestamp && filestamp > timestamp &&
! kara_metadata_equals(&kara.mdt, kara.filename)) {
LOG_INFO_SCT("REPO", "Ignore kara '%ld' with path '%s'", kara.id, kara.filename);
continue;
--
GitLab