MODULE: Migrate content of repo.c file to module_repo.c

inc/lektor/queue.h

@@ -11,22 +11,31 @@ enum {
     __LKT_PLAY_TOGGLE = 3,
 };
 
+enum {
+    __LKT_DB_UPDATING_FINISHED = 0, /* Not updating       */
+    __LKT_DB_UPDATING_PROGRESS = 1, /* Update in progress */
+};
+
 #define LKT_PLAY_STOP   ((void *) (size_t) __LKT_PLAY_STOP)
 #define LKT_PLAY_PLAY   ((void *) (size_t) __LKT_PLAY_PLAY)
 #define LKT_PLAY_PAUSE  ((void *) (size_t) __LKT_PLAY_PAUSE)
 #define LKT_PLAY_TOGGLE ((void *) (size_t) __LKT_PLAY_TOGGLE)
 
+#define LKT_DB_UPDATING_PROGRESS ((void *) (size_t) __LKT_DB_UPDATING_PROGRESS)
+#define LKT_DB_UPDATING_FINISHED ((void *) (size_t) __LKT_DB_UPDATING_FINISHED)
+
 enum lkt_event_type {
-    lkt_event_null          = 0,        // NULL
-    lkt_event_play_pos      = (1 << 1), // size_t
-    lkt_event_play_file     = (1 << 2), // XXX: UNUSED //
-    lkt_event_play_next     = (1 << 3), // NULL
-    lkt_event_play_prev     = (1 << 4), // NULL
-    lkt_event_play_toggle   = (1 << 5), // size_t, `LKT_PLAY_.*`
-    lkt_event_prop_vol      = (1 << 6), // size_t
-    lkt_event_prop_dur      = (1 << 7), // size_t
-    lkt_event_prop_time     = (1 << 8), // size_t
-    lkt_event_skip_current  = (1 << 9), // NULL
+    lkt_event_null          = 0,         // NULL
+    lkt_event_play_pos      = (1 <<  1), // size_t
+    lkt_event_play_file     = (1 <<  2), // XXX: UNUSED //
+    lkt_event_play_next     = (1 <<  3), // NULL
+    lkt_event_play_prev     = (1 <<  4), // NULL
+    lkt_event_play_toggle   = (1 <<  5), // size_t, `LKT_PLAY_.*`
+    lkt_event_prop_vol      = (1 <<  6), // size_t
+    lkt_event_prop_dur      = (1 <<  7), // size_t
+    lkt_event_prop_time     = (1 <<  8), // size_t
+    lkt_event_skip_current  = (1 <<  9), // NULL
+    lkt_event_db_updating   = (1 << 10), // size_t, `LKT_DB_UPDATING_*`
 };
 
 #define lkt_event_play ( lkt_event_play_pos    | lkt_event_play_file    \

src/module/module_repo.c

@@ -67,7 +67,7 @@ struct __file {
  * Private functions *
  *********************/
 
-__attribute__((unused)) static inline void
+static inline void
 __clean_file(struct __file *f)
 {
     if (f->fd) {
@@ -100,7 +100,7 @@ __write_mem(char *data, size_t size, size_t nmem, void *user)
     return realsize;
 }
 
-__attribute__((unused)) static size_t
+static size_t
 __write_disk(char *data, size_t size, size_t nmem, void *user)
 {
     ssize_t realsize = size * nmem;
@@ -124,7 +124,7 @@ __safe_json_get_string(struct json_object *jobj, const char *key,
     return 0;
 }
 
-__attribute__((unused)) static int
+static int
 __safe_json_get_long(struct json_object *json, const char *key, long *ret)
 {
     const int len = long_length(LONG_MAX);
@@ -135,7 +135,7 @@ __safe_json_get_long(struct json_object *json, const char *key, long *ret)
     return err;
 }
 
-__attribute__((unused)) static int
+static int
 __json_sync(struct module_repo_internal *repo, struct json_object **json)
 {
     RETURN_UNLESS(json, "Invalid argument", 1);
@@ -167,22 +167,227 @@ err:
     return ret;
 }
 
+static inline int
+__download_kara(const char *url, const char *path, int override)
+{
+    CURL *curl_handle;
+    char ret = 1;
+    errno    = 0;
+    int fd   = open(path, O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_NOFOLLOW, S_IRUSR | S_IWUSR);
+
+retest:
+    if (fd < 0) {
+        if (errno == EEXIST && ! override) {
+            LOG_ERROR("REPO", "File '%s' already exists", path);
+            return 1;
+        }
+
+        else if (errno == EEXIST && override) {
+            if (unlink(path)) {
+                LOG_ERROR("REPO", "Failed to unlink file '%s'", path);
+                return 1;
+            }
+
+            override = false;
+            fd       = open(path, O_WRONLY | O_CREAT | O_NOFOLLOW, S_IRUSR | S_IWUSR);
+            goto retest;
+        }
+
+        else {
+            LOG_ERROR("REPO", "Could not open file '%s'", path);
+            return 1;
+        }
+    }
+
+    struct __file file = {
+        .path = path,
+        .fd   = fd,
+    };
+
+    curl_handle = curl_easy_init();
+    curl_easy_setopt(curl_handle, CURLOPT_URL, url);
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, __write_disk);
+    curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
+    curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
+
+    if (CURLE_OK != (ret = curl_easy_perform(curl_handle))) {
+        LOG_ERROR("CURL", "curl_easy_perform failed: %s", curl_easy_strerror(ret));
+        goto err;
+    }
+
+    ret = 0;
+err:
+    __clean_file(&file);
+    curl_easy_cleanup(curl_handle);
+    return ret;
+}
+
+static inline void
+__handle_got_json(volatile sqlite3 *db, struct module_repo_internal *repo,
+                  struct json_object *json)
+{
+    size_t i, ignored_count = 0, update_count = 0,
+           len = json_object_array_length(json);
+    struct json_object *kara_json;
+    struct kara kara;
+    long filestamp = 0, timestamp = 0, download_id;
+    char *mkvpropedit = safe_zero_malloc(sizeof(char) * PATH_MAX);
+    char *url         = safe_zero_malloc(sizeof(char) * LKT_LINE_MAX);
+    int current_id, err;
+    struct timespec time_sleep = {
+        .tv_sec  = 0,
+        .tv_nsec = 100000000L,
+    };  /* Sleep for 0.1s */
+
+    RETURN_UNLESS(len > 0 && json_object_get_array(json),
+                  "Json invalid or array empty", NOTHING);
+    RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit",
+                                           mkvpropedit, PATH_MAX - 1),
+                  "Can't get the mkvpropedit executable path", NOTHING);
+
+    LOG_INFO("REPO", "Starting to process json for repo %s", repo->name);
+    for (i = 0; i < len; ++i) {
+        nanosleep(&time_sleep, NULL);   /* Sleep a bit, better for Hard drive */
+        kara_json = json_object_array_get_idx(json, i);
+        err = 0;
+
+        /* Get the id of the kara. */
+        if (__safe_json_get_long(kara_json, "id", &download_id))
+            continue;
+        kara.id = download_id;
+
+        /* Craft a fake filepath here, it will be used later. */
+        size_t kara_dir_len = strlen(repo->kara_dir);
+        memcpy(kara.filename, repo->kara_dir, sizeof(char) * (kara_dir_len + 1));
+        if (kara.filename[kara_dir_len - 1] != '/') {
+            strncat(kara.filename, "/", PATH_MAX - 1);
+            kara.filename[++kara_dir_len] = 0;
+        }
+        safe_snprintf(kara.filename + kara_dir_len, PATH_MAX - kara_dir_len, "%ld.mkv", download_id);
+
+        /* Timestamp and presence verification */
+        if (!database_get_kara_path(db, kara.id, NULL))
+            goto do_it;
+        if (__safe_json_get_long(kara_json, "unix_timestamp", &timestamp))
+            continue;
+        filestamp = get_mtime(kara.filename);
+        if (!(filestamp > timestamp))
+            goto do_it;
+        else {
+            ++ignored_count;
+            database_update_touch(db, kara.id);
+            database_update_set_available(db, kara.id);
+            LOG_DEBUG("REPO", "Ignore kara '%ld' with path '%s'",
+                      kara.id, kara.filename);
+            continue;
+        }
+    do_it:
+
+        /* Reads the json */
+#define __get_string(field, json_field) \
+        err |= __safe_json_get_string(kara_json, #field, kara.mdt.json_field, LEKTOR_TAG_MAX)
+        __get_string(song_name,     song_name);
+        __get_string(source_name,   source_name);
+        __get_string(category,      category);
+        __get_string(language,      language);
+        __get_string(author_name,   author_name);
+        __get_string(song_type,     song_type);
+#undef __get_string
+        if (err || __safe_json_get_long(kara_json, "song_number", &download_id)) {
+            LOG_WARN("REPO", "Json is invalid for kara '%ld', skip it", kara.id);
+            continue;
+        }
+        kara.mdt.song_number = download_id;
+
+        current_id = 0;
+        database_queue_current_kara(db, NULL, &current_id);
+        if (current_id == (int) kara.id) {
+            LOG_WARN("REPO", "Update currently playing kara %d, skip it",
+                     current_id);
+            lkt_queue_send(repo->queue, lkt_event_skip_current, NULL);
+        }
+
+        if (!database_update_add(db, kara.filename, &kara.mdt, kara.id, false)) {
+            LOG_ERROR("REPO", "Could not add unavailable kara %ld to db",
+                      kara.id);
+            continue;
+        }
+
+        safe_snprintf(url, LKT_LINE_MAX, repo->get_id_file, kara.id);
+
+        if (__download_kara(url, kara.filename, true)) {
+            LOG_WARN("REPO", "Could not download kara %ld at path '%s'",
+                     kara.id, kara.filename);
+            continue;
+        }
+
+        if (kara_metadata_write(&kara.mdt, kara.filename, mkvpropedit)) {
+            LOG_WARN("REPO", "Could not write metadata to kara '%ld' with "
+                     "path '%s'", kara.id, kara.filename);
+            continue;
+        }
+
+        if (!database_update_set_available(db, kara.id)) {
+            LOG_WARN("REPO", "Could not set kara %ld available", kara.id);
+            continue;
+        }
+
+        database_stamp(db);
+        ++update_count;
+        LOG_INFO("REPO", "Added kara %ld from repo %s, filepath is %s",
+                 kara.id, repo->name, kara.filename);
+    }
+    LOG_INFO("REPO", "Updated %ld karas and ignored %ld karas, total is %ld",
+             update_count, ignored_count, len);
+    free(mkvpropedit);
+    free(url);
+}
+
+static inline void
+__handle_deleted_kara(volatile sqlite3 *db)
+{
+    size_t len, i;
+    int *kara_ids;
+    char filepath[PATH_MAX];
+    database_deleted_kara(db, &kara_ids, &len);
+    for (i = 0; i < len; ++i) {
+        if (!database_get_kara_path(db, kara_ids[i], filepath))
+            continue;
+        database_update_del(db, kara_ids[i]);
+    }
+    free(kara_ids);
+}
+
 static void *
 __worker_update(void *__repo)
 {
-    /* TODO: Notify update in progress */
     struct module_repo_internal *repo = __repo;
+    lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_PROGRESS);
     repo->updating = 1;
+
+    struct json_object *json;
+    LOG_INFO("REPO", "Download kara list from %s (%s), directory is %s",
+             repo->name, repo->get_all_json, repo->kara_dir);
+    __json_sync(repo, &json);
+    __handle_got_json(repo->db, repo, json);
+    LOG_INFO("REPO", "Finished to download and insert kara list");
+    json_object_put(json);
+    __handle_deleted_kara(repo->db);
+    LOG_INFO("REPO", "Finished to deal with deleted kara");
+    database_updated(repo->db);
+
+    lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_FINISHED);
     pthread_exit(NULL);
 }
 
 static void *
 __worker_rescan(void *__repo)
 {
-    /* TODO: Notify rescan in progress */
     struct module_repo_internal *repo = __repo;
+    lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_PROGRESS);
     repo->updating = 1;
     /* Use the database_update(db, prefix, forced) function here ! */
+    lkt_queue_send(repo->queue, lkt_event_db_updating, LKT_DB_UPDATING_FINISHED);
     pthread_exit(NULL);
 }
 
@@ -255,14 +460,15 @@ mod_new(va_list *va)
     struct queue *queue  = va_arg(copy, struct queue *);
     volatile sqlite3 *db = va_arg(copy, volatile sqlite3 *);
 
-    if (!*repo)
+    if (NULL == *repo)
         *repo = malloc(sizeof(struct module_repo_internal));
-    if (!*repo) {
+    if (NULL == *repo) {
         LOG_ERROR("REPO", "Out of memory");
         return 1;
     }
 
     bool ret = module_repo_new(*repo, queue, db);
+    lkt_queue_make_available(queue, lkt_event_db_updating);
     va_end(copy);
     return ! ret;
 }
@@ -315,7 +521,6 @@ mod_update(va_list *va)
         return 0;
     }
     (*repo)->updating = 1;
-    /* TODO: Notify update pending */
     if (worker_pool_push(&(*repo)->workers, __worker_update, (void *) *repo)) {
         LOG_ERROR("REPO", "Out of memory");
         va_end(copy);
@@ -341,7 +546,6 @@ mod_rescan(va_list *va)
         return 0;
     }
     (*repo)->updating = 1;
-    /* TODO: Notify update pending */
     if (worker_pool_push(&(*repo)->workers, __worker_rescan, (void *) *repo)) {
         LOG_ERROR("REPO", "Out of memory");
         va_end(copy);