diff --git a/src/module/repo.c b/src/module/repo.c
deleted file mode 100644
index 326d070680842689627b2151193687167f64c0e0..0000000000000000000000000000000000000000
--- a/src/module/repo.c
+++ /dev/null
@@ -1,507 +0,0 @@
-#define _POSIX_C_SOURCE 200809L
-
-#include <errno.h>
-#include <stdio.h>
-#include <unistd.h>
-#include <string.h>
-#include <strings.h>
-#include <limits.h>
-#include <stdlib.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <time.h>
-#include <stdarg.h>
-
-#include <common/common.h>
-#include <lektor/module/mthread.h>
-#include <lektor/database.h>
-#include <lektor/net.h>
-#include <lektor/reg.h>
-
-static volatile unsigned int curl_init = false;
-
-struct memory {
- void *mem;
- size_t size;
-};
-
-struct file {
- const char *path;
- int fd;
-};
-
-size_t
-write_mem__(char *data, size_t size, size_t nmem, void *user)
-{
- size_t realsize = size * nmem;
- struct memory *mem = (struct memory *) user;
-
- void *ptr = realloc(mem->mem, mem->size + realsize);
- RETURN_UNLESS(ptr, "Out of memory", 0);
-
- mem->mem = ptr;
- memcpy(((uint8_t *) mem->mem) + mem->size, data, realsize);
- mem->size += realsize;
- return realsize;
-}
-
-size_t
-write_disk__(char *data, size_t size, size_t nmem, void *user)
-{
- ssize_t realsize = size * nmem;
- struct file *file = (struct file *) user;
- RETURN_IF(write(file->fd, data, realsize) != realsize, "Failed to write", 0);
- return realsize;
-}
-
-int
-repo_new(struct lkt_repo *const repo_, struct lkt_state *srv)
-{
- RETURN_UNLESS(srv && srv->db, "Invalid argument", 1);
- if (!curl_init) {
- curl_global_init(CURL_GLOBAL_ALL);
- curl_init = 1;
- } else
- ++curl_init;
-
- volatile sqlite3 *db = srv->db;
- struct lkt_repo repo = {
- .version = 1,
- .srv = srv,
- .db = srv->db,
- .name = safe_zero_malloc(INI_MAX_LINE_LEN * sizeof(char)),
- .kara_dir = safe_zero_malloc(PATH_MAX * sizeof(char)),
- .get_id_json = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
- .get_id_file = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
- .get_all_json = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
- .base_url = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
- };
-
- /* Copies */
- if (!database_config_get_text(db, "database", "kara_dir", repo.kara_dir, PATH_MAX - 1) ||
- !database_config_get_text(db, "repo", "name", repo.name, INI_MAX_LINE_LEN) ||
- !database_config_get_text(db, "repo", "url", repo.base_url, URL_MAX_LEN) ||
- !database_config_get_text(db, "repo", "id_json", repo.get_id_json, URL_MAX_LEN) ||
- !database_config_get_text(db, "repo", "id_kara", repo.get_id_file, URL_MAX_LEN) ||
- !database_config_get_text(db, "repo", "json", repo.get_all_json, URL_MAX_LEN))
- return 1;
- memcpy(repo_, &repo, sizeof(struct lkt_repo));
- return 0;
-}
-
-int
-load_repo_https(va_list *va)
-{
- va_list copy;
- va_copy(copy, *va);
- struct lkt_repo **repo = (struct lkt_repo **) va_arg(copy, void **);
- struct lkt_state *srv = va_arg(copy, struct lkt_state *);
- if (!*repo)
- *repo = malloc(sizeof(struct lkt_repo));
- if (!*repo)
- return 1;
- int ret = repo_new(*repo, srv);
- va_end(copy);
- return ret;
-}
-
-void
-repo_free(struct lkt_repo *const repo)
-{
- UNUSED(repo);
- --curl_init;
- if (!curl_init)
- curl_global_cleanup();
- free(repo->kara_dir);
- free(repo->get_id_json);
- free(repo->get_id_file);
- free(repo->base_url);
- free(repo->get_all_json);
- worker_pool_free(&repo->workers);
-}
-
-int
-safe_json_get_string(struct json_object *jobj, const char *key,
- char *content, const size_t len)
-{
- const char *got;
- struct json_object *field;
- RETURN_UNLESS(json_object_object_get_ex(jobj, key, &field),
- "Key not found in json", 1);
- got = json_object_get_string(field);
- RETURN_UNLESS(got, "Got a NULL for the key, may be an error", 1);
- strncpy(content, got, len - 1);
- content[len - 1] = 0;
- return 0;
-}
-
-int
-safe_json_get_int32(struct json_object *json, const char *key, int32_t *ret)
-{
- struct json_object *field;
- RETURN_UNLESS(json_object_object_get_ex(json, key, &field),
- "Key not found", 1);
- errno = 0;
- *ret = json_object_get_int(field);
- RETURN_IF(*ret == INT32_MAX || *ret == INT32_MIN,
- "Out of bound 32bit integer", 1);
- return 0;
-}
-
-int
-safe_json_get_long(struct json_object *json, const char *key, long *ret)
-{
- const int len = long_length(LONG_MAX);
- char content[len], *endptr, err;
- if (safe_json_get_string(json, key, content, len))
- return 1;
- STRTOL(*ret, content, endptr, err);
- return err;
-}
-
-int
-__json_sync(struct lkt_repo *const repo, struct json_object **json)
-{
- RETURN_UNLESS(json, "Invalid argument", 1);
- CURL *curl_handle;
- CURLcode res;
- int ret = 1;
- struct memory file = {
- .mem = NULL,
- .size = 0.
- };
-
- curl_handle = curl_easy_init();
- curl_easy_setopt(curl_handle, CURLOPT_URL, repo->get_all_json);
- curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_mem__);
- curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
- curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
- res = curl_easy_perform(curl_handle);
-
- if (res != CURLE_OK) {
- LOG_ERROR("CURL", "curl_easy_perform failed: %s",
- curl_easy_strerror(res));
- goto err;
- }
-
- *json = json_tokener_parse(file.mem);
- ret = 0;
-err:
- curl_easy_cleanup(curl_handle);
- free(file.mem);
- return ret;
-}
-
-int
-repo_get_id(struct lkt_repo *const repo, const uint64_t id,
- struct kara_metadata *mdt)
-{
- RETURN_UNLESS(mdt, "Invalid argument", 1);
- CURL *curl_handle;
- CURLcode res;
- struct json_object *jobj, *field;
- int ret = 1, err = 0;
- char url[URL_MAX_LEN];
- struct memory file = {
- .mem = NULL,
- .size = 0,
- };
-
- memset(url, 0, URL_MAX_LEN * sizeof(char));
- safe_snprintf(url, URL_MAX_LEN, repo->get_id_json, id);
- curl_handle = curl_easy_init();
- curl_easy_setopt(curl_handle, CURLOPT_URL, url);
- curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_mem__);
- curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
- curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
-
- if (CURLE_OK != (res = curl_easy_perform(curl_handle))) {
- LOG_ERROR("CURL", "curl_easy_perform failed: %s",
- curl_easy_strerror(res));
- free(file.mem);
- curl_easy_cleanup(curl_handle);
- return 1;
- }
-
- curl_easy_cleanup(curl_handle);
-
- /* Read the json. */
-
- jobj = json_tokener_parse(file.mem);
- if (json_object_object_get_ex(jobj, "message", &field)) {
- LOG_ERROR("REPO", "Kara with id %lu not found, message is: %s", id,
- json_object_get_string(field));
- goto err;
- }
-
- LOG_INFO("REPO", "Got kara with id %lu", id);
- err |= safe_json_get_string(jobj, "song_name", mdt->song_name, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(jobj, "source_name", mdt->source_name, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(jobj, "category", mdt->category, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(jobj, "language", mdt->language, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(jobj, "author_name", mdt->author_name, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(jobj, "song_type", mdt->song_type, LEKTOR_TAG_MAX);
- GOTO_IF(err || safe_json_get_int32(jobj, "song_number", &mdt->song_number),
- "Invalid json", err);
- ret = 0;
-err:
- json_object_put(jobj); /* Delete object. */
- return ret;
-}
-
-int
-__download_kara(const char *url, const char *path, int override)
-{
- CURL *curl_handle;
- errno = 0;
- char ret = 1;
- int fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_NOFOLLOW,
- S_IRUSR | S_IWUSR);
-
-retest:
- if (fd < 0) {
- if (errno == EEXIST && ! override) {
- LOG_ERROR("REPO", "File '%s' already exists", path);
- return 1;
- }
-
- else if (errno == EEXIST && override) {
- if (unlink(path)) {
- LOG_ERROR("REPO", "Failed to unlink file '%s'", path);
- return 1;
- }
-
- fd = open(path, O_WRONLY | O_CREAT | O_NOFOLLOW, S_IRUSR | S_IWUSR);
- override = false;
- goto retest;
- }
-
- else {
- LOG_ERROR("REPO", "Could not open file '%s'", path);
- return 1;
- }
- }
-
- struct file file = {
- .path = path,
- .fd = fd,
- };
-
- curl_handle = curl_easy_init();
- curl_easy_setopt(curl_handle, CURLOPT_URL, url);
- curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_disk__);
- curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
- curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
-
- if (CURLE_OK != (ret = curl_easy_perform(curl_handle))) {
- LOG_ERROR("CURL", "curl_easy_perform failed: %s",
- curl_easy_strerror(ret));
- goto err;
- }
-
- ret = 0;
-err:
- if (fd >= 0)
- close(fd);
- curl_easy_cleanup(curl_handle);
- return ret;
-}
-
-int
-repo_download_id_sync(struct lkt_repo *const repo, const uint64_t id,
- const char *kara_path, struct kara_metadata *mdt_ret)
-{
- RETURN_UNLESS(kara_path, "Invalid argument", 1);
- struct kara_metadata mdt;
- char url[URL_MAX_LEN];
-
- if (repo_get_id(repo, id, mdt_ret ? mdt_ret : &mdt)) {
- LOG_ERROR("REPO", "Failed to get kara metadata from kurisu");
- return 1;
- }
-
- memset(url, 0, URL_MAX_LEN * sizeof(char));
- safe_snprintf(url, URL_MAX_LEN, repo->get_id_file, id);
-
- if (__download_kara(url, kara_path, false)) {
- LOG_ERROR("REPO", "Failed to download kara '%s' with url '%s'",
- kara_path, url);
- return 1;
- }
-
- if (repo->db && !database_update_add((sqlite3 *) repo->db, kara_path,
- mdt_ret ? mdt_ret : &mdt, id, true)) {
- LOG_ERROR("REPO", "Failed to add kara to database");
- return 1;
- }
-
- return 0;
-}
-
-/* Get all the kara, make them unavailable */
-
-void
-__handle_got_json(volatile sqlite3 *db, struct lkt_repo *repo,
- struct json_object *json)
-{
- size_t i, len = json_object_array_length(json), update_count = 0,
- ignored_count = 0;
- struct json_object *kara_json;
- int32_t integer, err;
- struct kara kara;
- long filestamp = 0, timestamp = 0;
- char *mkvpropedit = safe_zero_malloc(sizeof(char) * PATH_MAX);
- char *url = safe_zero_malloc(sizeof(char) * URL_MAX_LEN);
- int current_id;
- struct lkt_state *srv = repo->srv;
- struct timespec time_sleep = {
- .tv_sec = 0,
- .tv_nsec = 100000000L,
- }; /* Sleep for 0.1s */
-
- RETURN_UNLESS(len > 0 && json_object_get_array(json),
- "Json invalid or array empty", NOTHING);
- RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit",
- mkvpropedit, PATH_MAX - 1),
- "Can't get the mkvpropedit executable path", NOTHING);
-
- LOG_INFO("REPO", "Starting to process json for repo %s", repo->name);
- for (i = 0; i < len; ++i) {
- nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */
- kara_json = json_object_array_get_idx(json, i);
- err = 0;
-
- /* Get the id of the kara. */
- if (safe_json_get_int32(kara_json, "id", &integer))
- continue;
- kara.id = integer;
-
- /* Craft a fake filepath here, it will be used later. */
- size_t kara_dir_len = strlen(repo->kara_dir);
- memcpy(kara.filename, repo->kara_dir,
- sizeof(char) * (kara_dir_len + 1));
- if (kara.filename[kara_dir_len - 1] != '/') {
- strncat(kara.filename, "/", PATH_MAX - 1);
- kara.filename[++kara_dir_len] = 0;
- }
- safe_snprintf(kara.filename + kara_dir_len, PATH_MAX - kara_dir_len,
- "%d.mkv", integer);
-
- /* Timestamp and presence verification */
- if (!database_get_kara_path(db, kara.id, NULL))
- goto do_it;
- if (safe_json_get_long(kara_json, "unix_timestamp", ×tamp))
- continue;
- filestamp = get_mtime(kara.filename);
- if (!(filestamp > timestamp))
- goto do_it;
- else {
- ++ignored_count;
- database_update_touch(db, kara.id);
- database_update_set_available(db, kara.id);
- LOG_DEBUG("REPO", "Ignore kara '%ld' with path '%s'",
- kara.id, kara.filename);
- continue;
- }
- do_it:
-
- /* Reads the json */
- err |= safe_json_get_string(kara_json, "song_name", kara.mdt.song_name, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(kara_json, "source_name", kara.mdt.source_name, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(kara_json, "category", kara.mdt.category, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(kara_json, "language", kara.mdt.language, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(kara_json, "author_name", kara.mdt.author_name, LEKTOR_TAG_MAX);
- err |= safe_json_get_string(kara_json, "song_type", kara.mdt.song_type, LEKTOR_TAG_MAX);
- if (err || safe_json_get_int32(kara_json, "song_number", &kara.mdt.song_number)) {
- LOG_WARN("REPO", "Json is invalid for kara '%ld', skip it", kara.id);
- continue;
- }
-
- current_id = 0;
- database_queue_current_kara(db, NULL, ¤t_id);
- if (current_id == (int) kara.id) {
- LOG_WARN("REPO", "Update currently playing kara %d, skip it",
- current_id);
- lkt_queue_send(&srv->queue, lkt_event_skip_current, NULL);
- }
-
- if (!database_update_add(db, kara.filename, &kara.mdt, kara.id, false)) {
- LOG_ERROR("REPO", "Could not add unavailable kara %ld to db",
- kara.id);
- continue;
- }
-
- safe_snprintf(url, URL_MAX_LEN, repo->get_id_file, kara.id);
-
- if (__download_kara(url, kara.filename, true)) {
- LOG_WARN("REPO", "Could not download kara %ld at path '%s'",
- kara.id, kara.filename);
- continue;
- }
-
- if (kara_metadata_write(&kara.mdt, kara.filename, mkvpropedit)) {
- LOG_WARN("REPO", "Could not write metadata to kara '%ld' with "
- "path '%s'", kara.id, kara.filename);
- continue;
- }
-
- if (!database_update_set_available(db, kara.id)) {
- LOG_WARN("REPO", "Could not set kara %ld available", kara.id);
- continue;
- }
-
- database_stamp(db);
- ++update_count;
- LOG_INFO("REPO", "Added kara %ld from repo %s, filepath is %s",
- kara.id, repo->name, kara.filename);
- }
- LOG_INFO("REPO", "Updated %ld karas and ignored %ld karas, total is %ld",
- update_count, ignored_count, len);
- free(mkvpropedit);
- free(url);
-}
-
-static inline void
-__handle_deleted_kara(volatile sqlite3 *db)
-{
- size_t len, i;
- int *kara_ids;
- char filepath[PATH_MAX];
- database_deleted_kara(db, &kara_ids, &len);
- for (i = 0; i < len; ++i) {
- if (!database_get_kara_path(db, kara_ids[i], filepath))
- continue;
- database_update_del(db, kara_ids[i]);
- }
- free(kara_ids);
-}
-
-void *
-__repo_get_all_id_async(void *arg)
-{
- struct json_object *json;
- struct lkt_repo *const repo = arg;
- LOG_INFO("REPO", "Download kara list from %s (%s), directory is %s",
- repo->name, repo->get_all_json, repo->kara_dir);
- __json_sync(repo, &json);
- __handle_got_json(repo->db, repo, json);
- LOG_INFO("REPO", "Finished to download and insert kara list");
- json_object_put(json);
- __handle_deleted_kara(repo->db);
- LOG_INFO("REPO", "Finished to deal with deleted kara");
- database_updated(repo->db);
- return NULL;
-}
-
-int
-repo_update(struct lkt_repo *const repo)
-{
- return mthread_create(NULL, ATTR_DETACHED, __repo_get_all_id_async, repo);
-}
-
-REG_BEGIN(repo_reg)
-REG_ADD_NAMED("new", load_repo_https)
-REG_END()
-// REG_EXPORT(repo_reg)