Skip to content
Extraits de code Groupes Projets
Vérifiée Valider 0bf57823 rédigé par Kubat's avatar Kubat
Parcourir les fichiers

MODULE: Delete the no more used repo.c file

parent 237ce330
Aucune branche associée trouvée
Aucune étiquette associée trouvée
1 requête de fusion!105Refactor and more
#define _POSIX_C_SOURCE 200809L
#include <errno.h>
#include <stdio.h>
#include <unistd.h>
#include <string.h>
#include <strings.h>
#include <limits.h>
#include <stdlib.h>
#include <stdlib.h>
#include <unistd.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <time.h>
#include <stdarg.h>
#include <common/common.h>
#include <lektor/module/mthread.h>
#include <lektor/database.h>
#include <lektor/net.h>
#include <lektor/reg.h>
static volatile unsigned int curl_init = false;
struct memory {
void *mem;
size_t size;
};
struct file {
const char *path;
int fd;
};
size_t
write_mem__(char *data, size_t size, size_t nmem, void *user)
{
size_t realsize = size * nmem;
struct memory *mem = (struct memory *) user;
void *ptr = realloc(mem->mem, mem->size + realsize);
RETURN_UNLESS(ptr, "Out of memory", 0);
mem->mem = ptr;
memcpy(((uint8_t *) mem->mem) + mem->size, data, realsize);
mem->size += realsize;
return realsize;
}
size_t
write_disk__(char *data, size_t size, size_t nmem, void *user)
{
ssize_t realsize = size * nmem;
struct file *file = (struct file *) user;
RETURN_IF(write(file->fd, data, realsize) != realsize, "Failed to write", 0);
return realsize;
}
int
repo_new(struct lkt_repo *const repo_, struct lkt_state *srv)
{
RETURN_UNLESS(srv && srv->db, "Invalid argument", 1);
if (!curl_init) {
curl_global_init(CURL_GLOBAL_ALL);
curl_init = 1;
} else
++curl_init;
volatile sqlite3 *db = srv->db;
struct lkt_repo repo = {
.version = 1,
.srv = srv,
.db = srv->db,
.name = safe_zero_malloc(INI_MAX_LINE_LEN * sizeof(char)),
.kara_dir = safe_zero_malloc(PATH_MAX * sizeof(char)),
.get_id_json = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
.get_id_file = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
.get_all_json = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
.base_url = safe_zero_malloc(URL_MAX_LEN * sizeof(char)),
};
/* Copies */
if (!database_config_get_text(db, "database", "kara_dir", repo.kara_dir, PATH_MAX - 1) ||
!database_config_get_text(db, "repo", "name", repo.name, INI_MAX_LINE_LEN) ||
!database_config_get_text(db, "repo", "url", repo.base_url, URL_MAX_LEN) ||
!database_config_get_text(db, "repo", "id_json", repo.get_id_json, URL_MAX_LEN) ||
!database_config_get_text(db, "repo", "id_kara", repo.get_id_file, URL_MAX_LEN) ||
!database_config_get_text(db, "repo", "json", repo.get_all_json, URL_MAX_LEN))
return 1;
memcpy(repo_, &repo, sizeof(struct lkt_repo));
return 0;
}
int
load_repo_https(va_list *va)
{
va_list copy;
va_copy(copy, *va);
struct lkt_repo **repo = (struct lkt_repo **) va_arg(copy, void **);
struct lkt_state *srv = va_arg(copy, struct lkt_state *);
if (!*repo)
*repo = malloc(sizeof(struct lkt_repo));
if (!*repo)
return 1;
int ret = repo_new(*repo, srv);
va_end(copy);
return ret;
}
void
repo_free(struct lkt_repo *const repo)
{
UNUSED(repo);
--curl_init;
if (!curl_init)
curl_global_cleanup();
free(repo->kara_dir);
free(repo->get_id_json);
free(repo->get_id_file);
free(repo->base_url);
free(repo->get_all_json);
worker_pool_free(&repo->workers);
}
int
safe_json_get_string(struct json_object *jobj, const char *key,
char *content, const size_t len)
{
const char *got;
struct json_object *field;
RETURN_UNLESS(json_object_object_get_ex(jobj, key, &field),
"Key not found in json", 1);
got = json_object_get_string(field);
RETURN_UNLESS(got, "Got a NULL for the key, may be an error", 1);
strncpy(content, got, len - 1);
content[len - 1] = 0;
return 0;
}
int
safe_json_get_int32(struct json_object *json, const char *key, int32_t *ret)
{
struct json_object *field;
RETURN_UNLESS(json_object_object_get_ex(json, key, &field),
"Key not found", 1);
errno = 0;
*ret = json_object_get_int(field);
RETURN_IF(*ret == INT32_MAX || *ret == INT32_MIN,
"Out of bound 32bit integer", 1);
return 0;
}
int
safe_json_get_long(struct json_object *json, const char *key, long *ret)
{
const int len = long_length(LONG_MAX);
char content[len], *endptr, err;
if (safe_json_get_string(json, key, content, len))
return 1;
STRTOL(*ret, content, endptr, err);
return err;
}
int
__json_sync(struct lkt_repo *const repo, struct json_object **json)
{
RETURN_UNLESS(json, "Invalid argument", 1);
CURL *curl_handle;
CURLcode res;
int ret = 1;
struct memory file = {
.mem = NULL,
.size = 0.
};
curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL, repo->get_all_json);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_mem__);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
res = curl_easy_perform(curl_handle);
if (res != CURLE_OK) {
LOG_ERROR("CURL", "curl_easy_perform failed: %s",
curl_easy_strerror(res));
goto err;
}
*json = json_tokener_parse(file.mem);
ret = 0;
err:
curl_easy_cleanup(curl_handle);
free(file.mem);
return ret;
}
int
repo_get_id(struct lkt_repo *const repo, const uint64_t id,
struct kara_metadata *mdt)
{
RETURN_UNLESS(mdt, "Invalid argument", 1);
CURL *curl_handle;
CURLcode res;
struct json_object *jobj, *field;
int ret = 1, err = 0;
char url[URL_MAX_LEN];
struct memory file = {
.mem = NULL,
.size = 0,
};
memset(url, 0, URL_MAX_LEN * sizeof(char));
safe_snprintf(url, URL_MAX_LEN, repo->get_id_json, id);
curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_mem__);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
if (CURLE_OK != (res = curl_easy_perform(curl_handle))) {
LOG_ERROR("CURL", "curl_easy_perform failed: %s",
curl_easy_strerror(res));
free(file.mem);
curl_easy_cleanup(curl_handle);
return 1;
}
curl_easy_cleanup(curl_handle);
/* Read the json. */
jobj = json_tokener_parse(file.mem);
if (json_object_object_get_ex(jobj, "message", &field)) {
LOG_ERROR("REPO", "Kara with id %lu not found, message is: %s", id,
json_object_get_string(field));
goto err;
}
LOG_INFO("REPO", "Got kara with id %lu", id);
err |= safe_json_get_string(jobj, "song_name", mdt->song_name, LEKTOR_TAG_MAX);
err |= safe_json_get_string(jobj, "source_name", mdt->source_name, LEKTOR_TAG_MAX);
err |= safe_json_get_string(jobj, "category", mdt->category, LEKTOR_TAG_MAX);
err |= safe_json_get_string(jobj, "language", mdt->language, LEKTOR_TAG_MAX);
err |= safe_json_get_string(jobj, "author_name", mdt->author_name, LEKTOR_TAG_MAX);
err |= safe_json_get_string(jobj, "song_type", mdt->song_type, LEKTOR_TAG_MAX);
GOTO_IF(err || safe_json_get_int32(jobj, "song_number", &mdt->song_number),
"Invalid json", err);
ret = 0;
err:
json_object_put(jobj); /* Delete object. */
return ret;
}
int
__download_kara(const char *url, const char *path, int override)
{
CURL *curl_handle;
errno = 0;
char ret = 1;
int fd = open(path, O_WRONLY | O_APPEND | O_CREAT | O_EXCL | O_NOFOLLOW,
S_IRUSR | S_IWUSR);
retest:
if (fd < 0) {
if (errno == EEXIST && ! override) {
LOG_ERROR("REPO", "File '%s' already exists", path);
return 1;
}
else if (errno == EEXIST && override) {
if (unlink(path)) {
LOG_ERROR("REPO", "Failed to unlink file '%s'", path);
return 1;
}
fd = open(path, O_WRONLY | O_CREAT | O_NOFOLLOW, S_IRUSR | S_IWUSR);
override = false;
goto retest;
}
else {
LOG_ERROR("REPO", "Could not open file '%s'", path);
return 1;
}
}
struct file file = {
.path = path,
.fd = fd,
};
curl_handle = curl_easy_init();
curl_easy_setopt(curl_handle, CURLOPT_URL, url);
curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, write_disk__);
curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *) &file);
curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0");
if (CURLE_OK != (ret = curl_easy_perform(curl_handle))) {
LOG_ERROR("CURL", "curl_easy_perform failed: %s",
curl_easy_strerror(ret));
goto err;
}
ret = 0;
err:
if (fd >= 0)
close(fd);
curl_easy_cleanup(curl_handle);
return ret;
}
int
repo_download_id_sync(struct lkt_repo *const repo, const uint64_t id,
const char *kara_path, struct kara_metadata *mdt_ret)
{
RETURN_UNLESS(kara_path, "Invalid argument", 1);
struct kara_metadata mdt;
char url[URL_MAX_LEN];
if (repo_get_id(repo, id, mdt_ret ? mdt_ret : &mdt)) {
LOG_ERROR("REPO", "Failed to get kara metadata from kurisu");
return 1;
}
memset(url, 0, URL_MAX_LEN * sizeof(char));
safe_snprintf(url, URL_MAX_LEN, repo->get_id_file, id);
if (__download_kara(url, kara_path, false)) {
LOG_ERROR("REPO", "Failed to download kara '%s' with url '%s'",
kara_path, url);
return 1;
}
if (repo->db && !database_update_add((sqlite3 *) repo->db, kara_path,
mdt_ret ? mdt_ret : &mdt, id, true)) {
LOG_ERROR("REPO", "Failed to add kara to database");
return 1;
}
return 0;
}
/* Get all the kara, make them unavailable */
void
__handle_got_json(volatile sqlite3 *db, struct lkt_repo *repo,
struct json_object *json)
{
size_t i, len = json_object_array_length(json), update_count = 0,
ignored_count = 0;
struct json_object *kara_json;
int32_t integer, err;
struct kara kara;
long filestamp = 0, timestamp = 0;
char *mkvpropedit = safe_zero_malloc(sizeof(char) * PATH_MAX);
char *url = safe_zero_malloc(sizeof(char) * URL_MAX_LEN);
int current_id;
struct lkt_state *srv = repo->srv;
struct timespec time_sleep = {
.tv_sec = 0,
.tv_nsec = 100000000L,
}; /* Sleep for 0.1s */
RETURN_UNLESS(len > 0 && json_object_get_array(json),
"Json invalid or array empty", NOTHING);
RETURN_UNLESS(database_config_get_text(db, "externals", "mkvpropedit",
mkvpropedit, PATH_MAX - 1),
"Can't get the mkvpropedit executable path", NOTHING);
LOG_INFO("REPO", "Starting to process json for repo %s", repo->name);
for (i = 0; i < len; ++i) {
nanosleep(&time_sleep, NULL); /* Sleep a bit, better for Hard drive */
kara_json = json_object_array_get_idx(json, i);
err = 0;
/* Get the id of the kara. */
if (safe_json_get_int32(kara_json, "id", &integer))
continue;
kara.id = integer;
/* Craft a fake filepath here, it will be used later. */
size_t kara_dir_len = strlen(repo->kara_dir);
memcpy(kara.filename, repo->kara_dir,
sizeof(char) * (kara_dir_len + 1));
if (kara.filename[kara_dir_len - 1] != '/') {
strncat(kara.filename, "/", PATH_MAX - 1);
kara.filename[++kara_dir_len] = 0;
}
safe_snprintf(kara.filename + kara_dir_len, PATH_MAX - kara_dir_len,
"%d.mkv", integer);
/* Timestamp and presence verification */
if (!database_get_kara_path(db, kara.id, NULL))
goto do_it;
if (safe_json_get_long(kara_json, "unix_timestamp", &timestamp))
continue;
filestamp = get_mtime(kara.filename);
if (!(filestamp > timestamp))
goto do_it;
else {
++ignored_count;
database_update_touch(db, kara.id);
database_update_set_available(db, kara.id);
LOG_DEBUG("REPO", "Ignore kara '%ld' with path '%s'",
kara.id, kara.filename);
continue;
}
do_it:
/* Reads the json */
err |= safe_json_get_string(kara_json, "song_name", kara.mdt.song_name, LEKTOR_TAG_MAX);
err |= safe_json_get_string(kara_json, "source_name", kara.mdt.source_name, LEKTOR_TAG_MAX);
err |= safe_json_get_string(kara_json, "category", kara.mdt.category, LEKTOR_TAG_MAX);
err |= safe_json_get_string(kara_json, "language", kara.mdt.language, LEKTOR_TAG_MAX);
err |= safe_json_get_string(kara_json, "author_name", kara.mdt.author_name, LEKTOR_TAG_MAX);
err |= safe_json_get_string(kara_json, "song_type", kara.mdt.song_type, LEKTOR_TAG_MAX);
if (err || safe_json_get_int32(kara_json, "song_number", &kara.mdt.song_number)) {
LOG_WARN("REPO", "Json is invalid for kara '%ld', skip it", kara.id);
continue;
}
current_id = 0;
database_queue_current_kara(db, NULL, &current_id);
if (current_id == (int) kara.id) {
LOG_WARN("REPO", "Update currently playing kara %d, skip it",
current_id);
lkt_queue_send(&srv->queue, lkt_event_skip_current, NULL);
}
if (!database_update_add(db, kara.filename, &kara.mdt, kara.id, false)) {
LOG_ERROR("REPO", "Could not add unavailable kara %ld to db",
kara.id);
continue;
}
safe_snprintf(url, URL_MAX_LEN, repo->get_id_file, kara.id);
if (__download_kara(url, kara.filename, true)) {
LOG_WARN("REPO", "Could not download kara %ld at path '%s'",
kara.id, kara.filename);
continue;
}
if (kara_metadata_write(&kara.mdt, kara.filename, mkvpropedit)) {
LOG_WARN("REPO", "Could not write metadata to kara '%ld' with "
"path '%s'", kara.id, kara.filename);
continue;
}
if (!database_update_set_available(db, kara.id)) {
LOG_WARN("REPO", "Could not set kara %ld available", kara.id);
continue;
}
database_stamp(db);
++update_count;
LOG_INFO("REPO", "Added kara %ld from repo %s, filepath is %s",
kara.id, repo->name, kara.filename);
}
LOG_INFO("REPO", "Updated %ld karas and ignored %ld karas, total is %ld",
update_count, ignored_count, len);
free(mkvpropedit);
free(url);
}
static inline void
__handle_deleted_kara(volatile sqlite3 *db)
{
size_t len, i;
int *kara_ids;
char filepath[PATH_MAX];
database_deleted_kara(db, &kara_ids, &len);
for (i = 0; i < len; ++i) {
if (!database_get_kara_path(db, kara_ids[i], filepath))
continue;
database_update_del(db, kara_ids[i]);
}
free(kara_ids);
}
void *
__repo_get_all_id_async(void *arg)
{
struct json_object *json;
struct lkt_repo *const repo = arg;
LOG_INFO("REPO", "Download kara list from %s (%s), directory is %s",
repo->name, repo->get_all_json, repo->kara_dir);
__json_sync(repo, &json);
__handle_got_json(repo->db, repo, json);
LOG_INFO("REPO", "Finished to download and insert kara list");
json_object_put(json);
__handle_deleted_kara(repo->db);
LOG_INFO("REPO", "Finished to deal with deleted kara");
database_updated(repo->db);
return NULL;
}
int
repo_update(struct lkt_repo *const repo)
{
return mthread_create(NULL, ATTR_DETACHED, __repo_get_all_id_async, repo);
}
REG_BEGIN(repo_reg)
REG_ADD_NAMED("new", load_repo_https)
REG_END()
// REG_EXPORT(repo_reg)
0% Chargement en cours ou .
You are about to add 0 people to the discussion. Proceed with caution.
Veuillez vous inscrire ou vous pour commenter