diff --git a/inc/lektor/queue.h b/inc/lektor/queue.h
index 326e3324a119de3277ac27e54cab3c598c50c6b8..f62bad11b520f91ee329bf484e84a631c4a6faa7 100644
--- a/inc/lektor/queue.h
+++ b/inc/lektor/queue.h
@@ -11,22 +11,31 @@ enum {
__LKT_PLAY_TOGGLE = 3,
};
+enum {
+ __LKT_DB_UPDATING_FINISHED = 0, /* Not updating */
+ __LKT_DB_UPDATING_PROGRESS = 1, /* Update in progress */
+};
+
#define LKT_PLAY_STOP ((void *) (size_t) __LKT_PLAY_STOP)
#define LKT_PLAY_PLAY ((void *) (size_t) __LKT_PLAY_PLAY)
#define LKT_PLAY_PAUSE ((void *) (size_t) __LKT_PLAY_PAUSE)
#define LKT_PLAY_TOGGLE ((void *) (size_t) __LKT_PLAY_TOGGLE)
+#define LKT_DB_UPDATING_PROGRESS ((void *) (size_t) __LKT_DB_UPDATING_PROGRESS)
+#define LKT_DB_UPDATING_FINISHED ((void *) (size_t) __LKT_DB_UPDATING_FINISHED)
+
enum lkt_event_type {
- lkt_event_null = 0, // NULL
- lkt_event_play_pos = (1 << 1), // size_t
- lkt_event_play_file = (1 << 2), // XXX: UNUSED //
- lkt_event_play_next = (1 << 3), // NULL
- lkt_event_play_prev = (1 << 4), // NULL
- lkt_event_play_toggle = (1 << 5), // size_t, `LKT_PLAY_.*`
- lkt_event_prop_vol = (1 << 6), // size_t
- lkt_event_prop_dur = (1 << 7), // size_t
- lkt_event_prop_time = (1 << 8), // size_t
- lkt_event_skip_current = (1 << 9), // NULL
+ lkt_event_null = 0, // NULL
+ lkt_event_play_pos = (1 << 1), // size_t
+ lkt_event_play_file = (1 << 2), // XXX: UNUSED //
+ lkt_event_play_next = (1 << 3), // NULL
+ lkt_event_play_prev = (1 << 4), // NULL
+ lkt_event_play_toggle = (1 << 5), // size_t, `LKT_PLAY_.*`
+ lkt_event_prop_vol = (1 << 6), // size_t
+ lkt_event_prop_dur = (1 << 7), // size_t
+ lkt_event_prop_time = (1 << 8), // size_t
+ lkt_event_skip_current = (1 << 9), // NULL
+ lkt_event_db_updating = (1 << 10), // size_t, `LKT_DB_UPDATING_*`
};
#define lkt_event_play ( lkt_event_play_pos | lkt_event_play_file \
diff --git a/src/module/module_repo.c b/src/module/module_repo.c
index a368648b494aa931c2cd7e36d15dd84c9cfbad7c..54c0c6fc5ad19810dfb669b46698cf2eddeb7d60 100644
--- a/src/module/module_repo.c
+++ b/src/module/module_repo.c
@@ -67,7 +67,7 @@ struct __file {
* Private functions *
*********************/
-__attribute__((unused)) static inline void
+static inline void
__clean_file(struct __file *f)
{
if (f->fd) {
@@ -100,7 +100,7 @@ __write_mem(char *data, size_t size, size_t nmem, void *user)
return realsize;
}
-__attribute__((unused)) static size_t
+static size_t
__write_disk(char *data, size_t size, size_t nmem, void *user)
{
ssize_t realsize = size * nmem;
@@ -124,7 +124,7 @@ __safe_json_get_string(struct json_object *jobj, const char *key,
return 0;
}
-__attribute__((unused)) static int
+static int
__safe_json_get_long(struct json_object *json, const char *key, long *ret)
{
const int len = long_length(LONG_MAX);
@@ -135,7 +135,7 @@ __safe_json_get_long(struct json_object *json, const char *key, long *ret)
return err;
}
-__attribute__((unused)) static int
+static int
__json_sync(struct module_repo_internal *repo, struct json_object **json)
{
RETURN_UNLESS(json, "Invalid argument", 1);
@@ -167,22 +167,227 @@ err:
return ret;
}
+static inline int
+__download_kara(const char *url, const char *path, int override)
+{
+ CURL *curl_handle;
+ char ret = 1;
+ errno = 0;
+ int fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR);
+
+retest:
+ if (fd < 0) {
+ if (errno == EEXIST && ! override) {
+ LOG_ERROR("REPO", "File '%s' already exists", path);
+ return 1;
+ }
+
+ else if (errno == EEXIST && override) {
+ if (unlink(path)) {
+ LOG_ERROR("REPO", "Failed to unlink file '%s'", path);
+ return 1;
+ }
+
+ override = false;
+ fd = open(path, O_WRONLY | O_CREAT | O_NOFOLLOW, S_IRUSR | S_IWUSR);
+ goto retest;
+ }
+
+ else {
+ LOG_ERROR("REPO", "Could not open file '%s'", path);
+ return 1;
+ }
+ }
+
+ struct __file file = {
+ .path = path,
+ .fd = fd,
+ };
+
+ curl_handle = curl_easy_init();
+ curl_easy_setopt(curl_handle, CURLOPT_URL, url);
+ curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, __write_disk);
+ curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
+ curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
+
+ if (CURLE_OK != (ret = curl_easy_perform(curl_handle))) {
+ LOG_ERROR("CURL", "curl_easy_perform failed: %s", curl_easy_strerror(ret));
+ goto err;
+ }
+
+ ret = 0;
+err:
+ __clean_file(&file);
+ curl_easy_cleanup(curl_handle);
+ return ret;
+}
+
+static inline void
+__handle_got_json(volatile sqlite3 *db, struct module_repo_internal *repo,
+ struct json_object *json)
+{
+ size_t i, ignored_count = 0, update_count = 0,
+ len = json_object_array_length(json);
+ struct json_object *kara_json;
+ struct kara kara;
+ long filestamp = 0, timestamp = 0, download_id;
+ char *mkvpropedit = safe_zero_malloc(sizeof(char) * PATH_MAX);
+ char *url = safe_zero_malloc(sizeof(char) * LKT_LINE_MAX);
+ int current_id, err;
+ struct timespec time_sleep = {
+ .tv_sec = 0,
+ .tv_nsec = 100000000L,
+ }; /* Sleep for 0.1s */
+
+ RETURN_UNLESS(len > 0 && json_object_get_array(json),
+ "Json invalid or array empty", NOTHING);
+ RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit",
+ mkvpropedit, PATH_MAX - 1),
+ "Can't get the mkvpropedit executable path", NOTHING);
+
+ LOG_INFO("REPO", "Starting to process json for repo %s", repo->name);
+ for (i = 0; i < len; ++i) {
+ nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */
+ kara_json = json_object_array_get_idx(json, i);
+ err = 0;
+
+ /* Get the id of the kara. */
+ if (__safe_json_get_long(kara_json, "id", &download_id))
+ continue;
+ kara.id = download_id;
+
+ /* Craft a fake filepath here, it will be used later. */
+ size_t kara_dir_len = strlen(repo->kara_dir);
+ memcpy(kara.filename, repo->kara_dir, sizeof(char) * (kara_dir_len + 1));
+ if (kara.filename[kara_dir_len - 1] != '/') {
+ strncat(kara.filename, "/", PATH_MAX - 1);
+ kara.filename[++kara_dir_len] = 0;
+ }
+ safe_snprintf(kara.filename + kara_dir_len, PATH_MAX - kara_dir_len, "%ld.mkv", download_id);
+
+ /* Timestamp and presence verification */
+ if (!database_get_kara_path(db, kara.id, NULL))
+ goto do_it;
+ if (__safe_json_get_long(kara_json, "unix_timestamp", ×tamp))
+ continue;
+ filestamp = get_mtime(kara.filename);
+ if (!(filestamp > timestamp))
+ goto do_it;
+ else {
+ ++ignored_count;
+ database_update_touch(db, kara.id);
+ database_update_set_available(db, kara.id);
+ LOG_DEBUG("REPO", "Ignore kara '%ld' with path '%s'",
+ kara.id, kara.filename);
+ continue;
+ }
+ do_it:
+
+ /* Reads the json */
+#define __get_string(field, json_field) \
+ err |= __safe_json_get_string(kara_json, #field, kara.mdt.json_field, LEKTOR_TAG_MAX)
+ __get_string(song_name, song_name);
+ __get_string(source_name, source_name);
+ __get_string(category, category);
+ __get_string(language, language);
+ __get_string(author_name, author_name);
+ __get_string(song_type, song_type);
+#undef __get_string
+ if (err || __safe_json_get_long(kara_json, "song_number", &download_id)) {
+ LOG_WARN("REPO", "Json is invalid for kara '%ld', skip it", kara.id);
+ continue;
+ }
+ kara.mdt.song_number = download_id;
+
+ current_id = 0;
+ database_queue_current_kara(db, NULL, ¤t_id);
+ if (current_id == (int) kara.id) {
+ LOG_WARN("REPO", "Update currently playing kara %d, skip it",
+ current_id);
+ lkt_queue_send(repo->queue, lkt_event_skip_current, NULL);
+ }
+
+ if (!database_update_add(db, kara.filename, &kara.mdt, kara.id, false)) {
+ LOG_ERROR("REPO", "Could not add unavailable kara %ld to db",
+ kara.id);
+ continue;
+ }
+
+ safe_snprintf(url, LKT_LINE_MAX, repo->get_id_file, kara.id);
+
+ if (__download_kara(url, kara.filename, true)) {
+ LOG_WARN("REPO", "Could not download kara %ld at path '%s'",
+ kara.id, kara.filename);
+ continue;
+ }
+
+ if (kara_metadata_write(&kara.mdt, kara.filename, mkvpropedit)) {
+ LOG_WARN("REPO", "Could not write metadata to kara '%ld' with "
+ "path '%s'", kara.id, kara.filename);
+ continue;
+ }
+
+ if (!database_update_set_available(db, kara.id)) {
+ LOG_WARN("REPO", "Could not set kara %ld available", kara.id);
+ continue;
+ }
+
+ database_stamp(db);
+ ++update_count;
+ LOG_INFO("REPO", "Added kara %ld from repo %s, filepath is %s",
+ kara.id, repo->name, kara.filename);
+ }
+ LOG_INFO("REPO", "Updated %ld karas and ignored %ld karas, total is %ld",
+ update_count, ignored_count, len);
+ free(mkvpropedit);
+ free(url);
+}
+
+static inline void
+__handle_deleted_kara(volatile sqlite3 *db)
+{
+ size_t len, i;
+ int *kara_ids;
+ char filepath[PATH_MAX];
+ database_deleted_kara(db, &kara_ids, &len);
+ for (i = 0; i < len; ++i) {
+ if (!database_get_kara_path(db, kara_ids[i], filepath))
+ continue;
+ database_update_del(db, kara_ids[i]);
+ }
+ free(kara_ids);
+}
+
static void *
__worker_update(void *__repo)
{
- /* TODO: Notify update in progress */
struct module_repo_internal *repo = __repo;
+ lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_PROGRESS);
repo->updating = 1;
+
+ struct json_object *json;
+ LOG_INFO("REPO", "Download kara list from %s (%s), directory is %s",
+ repo->name, repo->get_all_json, repo->kara_dir);
+ __json_sync(repo, &json);
+ __handle_got_json(repo->db, repo, json);
+ LOG_INFO("REPO", "Finished to download and insert kara list");
+ json_object_put(json);
+ __handle_deleted_kara(repo->db);
+ LOG_INFO("REPO", "Finished to deal with deleted kara");
+ database_updated(repo->db);
+
+ lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_FINISHED);
pthread_exit(NULL);
}
static void *
__worker_rescan(void *__repo)
{
- /* TODO: Notify rescan in progress */
struct module_repo_internal *repo = __repo;
+ lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_PROGRESS);
repo->updating = 1;
/* Use the database_update(db, prefix, forced) function here ! */
+ lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_FINISHED);
pthread_exit(NULL);
}
@@ -255,14 +460,15 @@ mod_new(va_list *va)
struct queue *queue = va_arg(copy, struct queue *);
volatile sqlite3 *db = va_arg(copy, volatile sqlite3 *);
- if (!*repo)
+ if (NULL == *repo)
*repo = malloc(sizeof(struct module_repo_internal));
- if (!*repo) {
+ if (NULL == *repo) {
LOG_ERROR("REPO", "Out of memory");
return 1;
}
bool ret = module_repo_new(*repo, queue, db);
+ lkt_queue_make_available(queue, lkt_event_db_updating);
va_end(copy);
return ! ret;
}
@@ -315,7 +521,6 @@ mod_update(va_list *va)
return 0;
}
(*repo)->updating = 1;
- /* TODO: Notify update pending */
if (worker_pool_push(&(*repo)->workers, __worker_update, (void *) *repo)) {
LOG_ERROR("REPO", "Out of memory");
va_end(copy);
@@ -341,7 +546,6 @@ mod_rescan(va_list *va)
return 0;
}
(*repo)->updating = 1;
- /* TODO: Notify update pending */
if (worker_pool_push(&(*repo)->workers, __worker_rescan, (void *) *repo)) {
LOG_ERROR("REPO", "Out of memory");
va_end(copy);