diff --git a/inc/lektor/json.h b/inc/lektor/json.h
index cbfa1ebb16a445d71fce6e9d4877b2ccb2a7171f..37025fc58b7e6c61897b47598ace92d1bd991198 100644
--- a/inc/lektor/json.h
+++ b/inc/lektor/json.h
@@ -25,4 +25,7 @@ typedef void(*json_parse_callback)(
* 'completed' argument at 1, 'key' and 'value' will be NULL. */
int json_parse(const char *str, int level, json_parse_callback, void *user);
+/* Get the object count at the specified level of the json. */
+int json_parse_get_count(const char *str, int level);
+
#endif /* __LKT_JSON_H__ */
diff --git a/inc/lektor/lktmodule.h b/inc/lektor/lktmodule.h
index b7db5dc10c3a934301cd69ffd5fb06fafe84dac3..3a9402280a44cf2b1cef41835d5fa158d653a668 100644
--- a/inc/lektor/lktmodule.h
+++ b/inc/lektor/lktmodule.h
@@ -12,6 +12,7 @@
#include <lektor/net.h>
#include <lektor/reg.h>
#include <lektor/stack.h>
+#include <lektor/json.h>
/* Include source files, only do it once per module ! */
#if defined(__LKT_MODULE_MAIN_SOURCE__) && ! defined(LKT_STATIC_MODULE)
@@ -19,6 +20,7 @@
#include "../../src/base/stack.c"
#include "../../src/database/queue.c"
#include "../../src/module/thread.c"
+ #include "../../src/module/json.c"
#endif /* __LKT_MODULE_MAIN_SOURCE__ && ! LKT_STATIC_MODULE*/
#endif /* __LKT_LKTMODULE_H__ */
diff --git a/inc/lektor/mkv.h b/inc/lektor/mkv.h
index 07e531899e960d7147d188c65ac5b4a79add8d2a..8bb30f4dab5a7c4b9a6da960507e82713d993466 100644
--- a/inc/lektor/mkv.h
+++ b/inc/lektor/mkv.h
@@ -38,12 +38,6 @@ struct kara_metadata {
int song_number;
};
-struct kara {
- size_t id; /* Should never be NULL. NEVER!! */
- struct kara_metadata mdt; /* The metadata of the kara. */
- char filename[PATH_MAX]; /* The path to the matroska file. */
-};
-
/* Reads the .mkv file at `filename` and stores its metadata in `dst`.
Returns 0 on success and -1 on error. */
int kara_metadata_read(struct kara_metadata *dst, const char *filename);
diff --git a/src/base/json.c b/src/base/json.c
index faae37fbd095b108cffbf06e2d78d481c5329cc9..64dc327250787ce40a4396018a8e2a3a2fba6bab 100644
--- a/src/base/json.c
+++ b/src/base/json.c
@@ -12,28 +12,11 @@
#define __JSON_END "}]"
#define __JSON_SEP '"'
-static inline char *
-strip(char *s)
-{
- char *p = s + strlen(s);
- while (p > s && isspace(*(--p)))
- *p = 0;
- return s;
-}
-
-static inline char *
-skip(char *s)
-{
- while (*s && isspace(*s))
- s++;
- return s;
-}
-
#define __SKIP_JSON(str) (str = &str[strspn(str, __JSON_SPACE)])
-#define __NEXT_JSON(str, len, is_paren, dest) \
+#define __NEXT_JSON(str, len, dest) \
{ \
- is_paren = 0; \
+ int is_paren = 0; \
if (str[0] == __JSON_SEP) { \
++str; \
const char *begin = str, *end; \
@@ -60,6 +43,27 @@ skip(char *s)
} \
}
+#define __SKIP_NEXT_JSON(str, len) \
+{ \
+ int is_paren = 0; \
+ if (str[0] == __JSON_SEP) { \
+ ++str; \
+ const char *begin = str, *end; \
+ for (;;) { \
+ end = strchr(begin, __JSON_SEP); \
+ if (*(end - 1) == '\\') { \
+ begin = end + 1; \
+ continue; \
+ } \
+ break; \
+ } \
+ len = (end - str); \
+ is_paren = 1; \
+ } else \
+ len = strcspn(str, __JSON_SPACE __JSON_BEGIN __JSON_END); \
+ str += len; \
+}
+
/* WARN: strlen(from) >= strlen(to) */
static inline int
__replace(char *string, const char *from, const char *to)
@@ -93,8 +97,7 @@ __replace(char *string, const char *from, const char *to)
int
json_parse(const char *str, int asked_level, json_parse_callback call, void *user)
{
- int level = 0;
- int is_paren = 0;
+ int level = 0;
char key[LKT_LINE_MAX];
char val[LKT_LINE_MAX];
@@ -122,9 +125,9 @@ json_parse(const char *str, int asked_level, json_parse_callback call, void *use
memset(key, 0, sizeof(key));
memset(val, 0, sizeof(val));
- __NEXT_JSON(str, len, is_paren, key);
+ __NEXT_JSON(str, len, key);
__SKIP_JSON(str);
- __NEXT_JSON(str, len, is_paren, val);
+ __NEXT_JSON(str, len, val);
if (asked_level == level)
call(key, val, 0, user);
@@ -136,3 +139,43 @@ json_parse(const char *str, int asked_level, json_parse_callback call, void *use
}
return 1;
}
+
+int
+json_parse_get_count(const char *str, int level)
+{
+ int level = 0;
+ int ret_len = 0;
+
+ for (;;) {
+ size_t len = 0;
+ __SKIP_JSON(str);
+
+ /* Begin of a block */
+ if ((len = strspn(str, __JSON_BEGIN))) {
+ level += len;
+ str += len;
+ }
+
+ /* End of a block */
+ else if ((len = strspn(str, __JSON_END))) {
+ ret_len += (level == asked_level);
+ str += len;
+ level -= len;
+ }
+
+ /* Key Value */
+ else {
+ __SKIP_NEXT_JSON(str, len);
+ __SKIP_JSON(str);
+ __SKIP_NEXT_JSON(str, len);
+
+ if (asked_level == level)
+ call(key, val, 0, user);
+ }
+
+ if (level <= 0) {
+ return len;
+ }
+ }
+ return -1;
+}
diff --git a/src/module/module_repo.c b/src/module/module_repo.c
index c24f09a53780dec8fd5f2d605ee2a5cc9d85f7e0..1a038ebe622c95ffbbca03f7e385862f7901be5b 100644
--- a/src/module/module_repo.c
+++ b/src/module/module_repo.c
@@ -2,7 +2,6 @@
#define __LKT_MODULE_MAIN_SOURCE__
#include <lektor/lktmodule.h>
-#include <lektor/json.h>
#include "worker.h"
@@ -36,6 +35,22 @@ static volatile unsigned int __curl_init = false;
* Private structure *
*********************/
+struct module_repo_internal;
+
+struct kara {
+ int ignored_count;
+ int update_count;
+ volatile sqlite3 *db;
+ struct module_repo_internal *repo;
+ long id;
+ long unix_timestamp;
+ struct kara_metadata mdt;
+ char mkvpropedit[LKT_LINE_MAX];
+ char url[LKT_LINE_MAX];
+ char database_filepath[PATH_MAX];
+ char filename[PATH_MAX];
+};
+
struct module_repo_internal {
/* Just the repo */
char *name;
@@ -142,7 +157,8 @@ __clean_memory(struct __memory *m)
{
if (m->mem) {
free(m->mem);
- m->mem = NULL;
+ m->mem = NULL;
+ m->size = 0;
}
}
@@ -171,32 +187,7 @@ __write_disk(char *data, size_t size, size_t nmem, void *user)
}
static int
-__safe_json_get_string(struct json_object *jobj, const char *key,
- char *content, const size_t len)
-{
- const char *got;
- struct json_object *field;
- RETURN_UNLESS(json_object_object_get_ex(jobj, key, &field), "Key not found in json", 1);
- got = json_object_get_string(field);
- RETURN_UNLESS(got, "Got a NULL for the key, may be an error", 1);
- strncpy(content, got, len - 1);
- content[len - 1] = 0;
- return 0;
-}
-
-static int
-__safe_json_get_long(struct json_object *json, const char *key, long *ret)
-{
- const int len = long_length(LONG_MAX);
- char content[len], *endptr, err;
- if (__safe_json_get_string(json, key, content, len))
- return 1;
- STRTOL(*ret, content, endptr, err);
- return err;
-}
-
-static int
-__json_dl(const char *url, struct json_object **json)
+__json_dl(const char *url, char **json)
{
RETURN_UNLESS(json, "Invalid argument", 1);
CURL *curl_handle;
@@ -231,10 +222,11 @@ __json_dl(const char *url, struct json_object **json)
goto err;
}
- *json = json_tokener_parse(file.mem);
+ *json = file.mem;
ret = 0;
err:
- __clean_memory(&file);
+ if (ret != 0)
+ __clean_memory(&file);
curl_easy_cleanup(curl_handle);
curl_slist_free_all(headers);
return ret;
@@ -272,6 +264,7 @@ retest:
}
}
+ /* TODO: Buffered writes */
struct __file file = {
.path = path,
.fd = fd,
@@ -295,125 +288,151 @@ err:
return ret;
}
-static inline void
-__handle_got_json(volatile sqlite3 *db, struct module_repo_internal *repo,
- struct json_object *json)
+static void
+__handle_got_json_dl(struct kara *kara, int current_id)
{
- size_t ignored_count = 0, update_count = 0,
- len = json_object_array_length(json), i;
- struct json_object *kara_json = NULL;
- struct kara kara;
- long filestamp = 0, timestamp = 0, long_integer;
- char *mkvpropedit = safe_zero_malloc(sizeof(char) * PATH_MAX);
- char *url = safe_zero_malloc(sizeof(char) * LKT_LINE_MAX);
- char *database_filepath = safe_zero_malloc(sizeof(char) * PATH_MAX);
- int current_id, err;
- struct timespec time_sleep = {
- .tv_sec = 0,
- .tv_nsec = 100000000L,
- }; /* Sleep for 0.1s */
-
- RETURN_UNLESS(len > 0 && json_object_get_array(json),
- "Json invalid or array empty", NOTHING);
- RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit", mkvpropedit, PATH_MAX),
- "Can't get the mkvpropedit executable path", NOTHING);
+ /* Download the kara */
+ database_queue_current_kara(kara->db, NULL, ¤t_id);
+ if (current_id == (int) kara->id) {
+ LOG_WARN("REPO", "Update currently playing kara %d, skip it", current_id);
+ lkt_queue_send(kara->repo->queue, lkt_event_skip_current, NULL);
+ }
- /* Handle the json */
- LOG_INFO("REPO", "Starting to process json for repo %s", repo->name);
- for (i = 0; i < len; ++i) {
- nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */
- kara_json = json_object_array_get_idx(json, i);
- err = 0;
+ if (!database_update_add(kara->db, kara->filename, &kara->mdt, kara->id, false)) {
+ LOG_ERROR("REPO", "Could not add unavailable kara %ld to db", kara->id);
+ return;
+ }
- if (__safe_json_get_long(kara_json, "id", &long_integer))
- continue;
- kara.id = long_integer;
-
- /* Craft a fake filepath here, it will be used later. */
- size_t kara_dir_len = strlen(repo->kara_dir);
- memcpy(kara.filename, repo->kara_dir, sizeof(char) * (kara_dir_len + 1));
- if (kara.filename[kara_dir_len - 1] != '/') {
- strncat(kara.filename, "/", PATH_MAX - 1);
- kara.filename[++kara_dir_len] = 0;
- }
+ safe_snprintf(kara->url, LKT_LINE_MAX, kara->repo->get_id_file, kara->id);
- /* Reads the json, needed in case of download without obfuscation */
-#define __get_string(field, json_field) \
- err |= __safe_json_get_string(kara_json, #field, kara.mdt.json_field, LEKTOR_TAG_MAX)
- __get_string(song_name, song_name);
- __get_string(source_name, source_name);
- __get_string(category, category);
- __get_string(language, language);
- __get_string(author_name, author_name);
- __get_string(song_type, song_type);
-#undef __get_string
- if (err || __safe_json_get_long(kara_json, "song_number", &long_integer)) {
- LOG_WARN("REPO", "Json is invalid for kara '%ld', skip it", kara.id);
- continue;
- }
- kara.mdt.song_number = long_integer;
- repo->craft_filename(kara.filename, kara_dir_len, &kara);
+ if (__download_kara(kara->url, kara->filename, true)) {
+ LOG_WARN("REPO", "Could not download kara %ld at path '%s'",
+ kara->id, kara->filename);
+ return;
+ }
+
+ if (kara_metadata_write(&kara->mdt, kara->filename, kara->mkvpropedit)) {
+ LOG_WARN("REPO", "Could not write metadata to kara '%ld' with path '%s'",
+ kara->id, kara->filename);
+ return;
+ }
+
+ if (!database_update_set_available(kara->db, kara->id)) {
+ LOG_WARN("REPO", "Could not set kara %ld available", kara->id);
+ return;
+ }
+
+ database_stamp(kara->db);
+ ++(kara->update_count);
+ LOG_INFO("REPO", "Added kara %ld from repo %s, filepath is %s",
+ kara->id, kara->repo->name, kara->filename);
+}
+
+static void
+__handle_got_json_internal_callback(const char *key, const char *val, int comp, void *user)
+{
+ struct kara *kara = (struct kara *) user;
+
+ /* Get the fields */
+ if (!comp && key && val) {
+#define __get_field_string(field) { \
+ if (STR_MATCH(#field, key)) { \
+ safe_strncpy(kara->mdt.field, val, LEKTOR_TAG_MAX); \
+ }}
+#define __get_field_long_ex(field, json) { \
+ if (STR_MATCH(#json, key)) { \
+ kara->field = strtol(val, NULL, 0); \
+ }}
+#define __get_field_long_mdt(field) __get_field_long_ex(mdt.field, field)
+#define __get_field_long_kara(field) __get_field_long_ex(field, field)
+
+ __get_field_long_kara(id);
+ __get_field_long_kara(unix_timestamp);
+
+ __get_field_long_mdt(song_number);
+
+ __get_field_string(song_name);
+ __get_field_string(source_name);
+ __get_field_string(category);
+ __get_field_string(language);
+ __get_field_string(author_name);
+ __get_field_string(song_type);
+
+#undef __get_field_long_mdt
+#undef __get_field_long_kara
+#undef __get_field_long_ex
+#undef __get_field_string
+ }
+
+ /* The `void *user` is complete */
+ else if (comp) {
+ struct timespec time_sleep = {
+ .tv_sec = 0,
+ .tv_nsec = 100000000L,
+ }; /* Sleep for 0.1s */
+ nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */
+
+ long filestamp = 0;
+ int current_id = 0;
/* Timestamp and presence verification */
- if (!database_get_kara_path(db, kara.id, database_filepath))
+ if (!database_get_kara_path(kara->db, kara->id, kara->database_filepath))
goto do_it;
+
/* Override calculated filename if it exists */
- memcpy(kara.filename, database_filepath, (strlen(database_filepath) + 1) * sizeof(char));
- if (__safe_json_get_long(kara_json, "unix_timestamp", ×tamp))
- continue;
- filestamp = get_mtime(kara.filename);
- if (!(filestamp > timestamp))
+ size_t db_fp_size = (strlen(kara->database_filepath) + 1) * sizeof(char);
+ memcpy(kara->filename, kara->database_filepath, db_fp_size);
+
+ filestamp = get_mtime(kara->filename);
+ if (!(filestamp > kara->unix_timestamp))
goto do_it;
else {
- ++ignored_count;
- database_update_touch(db, kara.id);
- database_update_set_available(db, kara.id);
- LOG_DEBUG("REPO", "Ignore kara '%ld' with path '%s'", kara.id, kara.filename);
- continue;
+ ++(kara->ignored_count);
+ database_update_touch(kara->db, kara->id);
+ database_update_set_available(kara->db, kara->id);
+ LOG_DEBUG("REPO", "Ignore kara '%ld' with path '%s'", kara->id, kara->filename);
+ return;
}
- do_it:
- current_id = 0;
- database_queue_current_kara(db, NULL, ¤t_id);
- if (current_id == (int) kara.id) {
- LOG_WARN("REPO", "Update currently playing kara %d, skip it", current_id);
- lkt_queue_send(repo->queue, lkt_event_skip_current, NULL);
- }
-
- if (!database_update_add(db, kara.filename, &kara.mdt, kara.id, false)) {
- LOG_ERROR("REPO", "Could not add unavailable kara %ld to db", kara.id);
- continue;
- }
-
- safe_snprintf(url, LKT_LINE_MAX, repo->get_id_file, kara.id);
+ do_it:
+ __handle_got_json_dl(kara, current_id);
+ }
- if (__download_kara(url, kara.filename, true)) {
- LOG_WARN("REPO", "Could not download kara %ld at path '%s'",
- kara.id, kara.filename);
- continue;
- }
+ else {
+ LOG_ERROR("REPO", "Invalid call to this function, 'comp', 'key' and 'val' are null...");
+ }
+}
- if (kara_metadata_write(&kara.mdt, kara.filename, mkvpropedit)) {
- LOG_WARN("REPO", "Could not write metadata to kara '%ld' with path '%s'",
- kara.id, kara.filename);
- continue;
- }
+static inline void
+__handle_got_json(volatile sqlite3 *db, struct module_repo_internal *repo, const char *json)
+{
+ size_t len = json_parse_get_count(json, 1);
+ struct kara kara = {
+ .repo = repo,
+ .db = db,
+ .ignored_count = 0,
+ .update_count = 0,
+ };
- if (!database_update_set_available(db, kara.id)) {
- LOG_WARN("REPO", "Could not set kara %ld available", kara.id);
- continue;
- }
+ RETURN_UNLESS(len > 0, "Json invalid or array empty", NOTHING);
+ RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit",
+ kara.mkvpropedit, PATH_MAX),
+ "Can't get the mkvpropedit executable path", NOTHING);
- database_stamp(db);
- ++update_count;
- LOG_INFO("REPO", "Added kara %ld from repo %s, filepath is %s",
- kara.id, repo->name, kara.filename);
+ /* Craft a folder path here, it will be used later */
+ size_t kara_dir_len = strlen(repo->kara_dir);
+ kara_dir_len = sizeof(char) * (kara_dir_len + 1);
+ memcpy(kara.database_filepath, repo->kara_dir, kara_dir_len);
+ if (kara.database_filepath[kara_dir_len - 1] != '/') {
+ strncat(kara.database_filepath, "/", PATH_MAX - 1);
+ kara.database_filepath[++kara_dir_len] = 0;
}
+
+ /* Handle the json */
+ LOG_INFO("REPO", "Starting to process json for repo %s", repo->name);
+ json_parse(json, 1, __handle_got_json_internal_callback, (void *) &kara);
LOG_INFO("REPO", "Updated %ld karas and ignored %ld karas, total is %ld",
- update_count, ignored_count, len);
- free(mkvpropedit);
- free(url);
- free(database_filepath);
+ kara.update_count, kara.ignored_count, len);
}
static inline void
@@ -441,7 +460,7 @@ __worker_update(void *__repo)
repo->updating &= REPO_UPDATE_KARA;
GOTO_IF(pthread_mutex_unlock(&(repo->mtx)), "Failed to unlock", end_no_lock);
- struct json_object *json;
+ char *json;
LOG_INFO("REPO", "Download kara list from %s (%s), directory is %s",
repo->name, repo->get_all_json, repo->kara_dir);
if (__json_dl(repo->get_all_json, &json)) {
@@ -450,7 +469,7 @@ __worker_update(void *__repo)
}
__handle_got_json(repo->db, repo, json);
LOG_INFO("REPO", "Finished to download and insert kara list");
- json_object_put(json);
+ free(json);
__handle_deleted_kara(repo->db);
LOG_INFO("REPO", "Finished to deal with deleted kara");
database_updated(repo->db);