DB: Add an upgrade field rule with a fallback thing to help upgrading the database

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Add: the folder permission is not settable at the creation of it, the value is modifiable in the config file at `database/dir_permission`
 - Add: the regex used to populate the database from disk is now modifiable from the config file, the keys are `database/{types,categories,languages}`. Those fields must only contain alpha characters and commas
 - Add: re-use more informations from databases when upgrading the database because minimal version where added to the different fields in the table descriptions
+- Add: new field `repo_timestamp` which is the timestamp of the kara from the repo. This field is defaulted to the `cached_mtime` at upgrade time if needed
 - Fix: fix incorrect parsing of lists of commands with multiple arguments: the parser could overrun the correct command buffer and insert a '\0' between the next command and its furst argument
 - Fix: fix issue #100 -> the `database_queue_add_id` won't add ids that doesn't exist
 - Fix: be explicit about truncated commands, when a command is truncated, all the bytes after the end of the new string and before the end of the old string are not set to '\0' for safety

src/database/upgrade.c

@@ -5,8 +5,9 @@
 #include <lektor/internal/dbmacro.h>
 
 struct sql_field_description {
-    const char *name;
     const LKT_DATABASE_VERSION minimal_version;
+    const char *name;
+    const char *fallback_field;
 };
 
 struct sql_table_description {
@@ -21,10 +22,17 @@ struct sql_table_description {
 
 #define DCL_SQL_FIELD(field_name, version)                                                      \
     ((const struct sql_field_description){ .name            = field_name,                       \
-                                           .minimal_version = (LKT_DATABASE_VERSION_##version) })
+                                           .minimal_version = (LKT_DATABASE_VERSION_##version), \
+                                           .fallback_field  = NULL })
+
+#define DCL_SQL_FIELD_WITH_FALLBACK(field_name, fallback_field_name, version)                   \
+    ((const struct sql_field_description){ .name            = field_name,                       \
+                                           .minimal_version = (LKT_DATABASE_VERSION_##version), \
+                                           .fallback_field  = fallback_field_name })
 
 #define DCL_SQL_FIELD_NULL                 \
-    ((const struct sql_field_description){ .name = NULL, .minimal_version = 0 })
+    ((const struct sql_field_description){ \
+        .name = NULL, .minimal_version = 0, .fallback_field = NULL })
 
 #define DCL_SQL_TABLE(tbl_name, ...)                                      \
     {                                                                     \
@@ -53,7 +61,7 @@ static const struct sql_table_description sql_tables_list[] = {
                   DCL_SQL_FIELD("available", ALPHA),                                      //
                   DCL_SQL_FIELD("cached_mtime", MK_7_1),                                  //
                   DCL_SQL_FIELD("cached_duration", MK_7_1),                               //
-                  DCL_SQL_FIELD("repo_timestamp", MK_7_3)), //
+                  DCL_SQL_FIELD_WITH_FALLBACK("repo_timestamp", "cached_mtime", MK_7_3)), //
 
     DCL_SQL_TABLE(kara_type,                     //
                   DCL_SQL_FIELD("id", ALPHA),    //
@@ -149,13 +157,29 @@ ___compute_upgrade_sql_query(LKT_DATABASE_VERSION base_version,
         "  SELECT %s FROM " LKT_PROTECTED_DATABASE ".%s;";
 
     /* Create the comma separated list for the table's fields */
-    size_t field_list_length = 0;
+    size_t field_list_new_length = 0;
+    size_t field_list_old_length = 0;
     size_t added_fields          = 0;
     for (size_t i = 0; table->fields[i].name != NULL; ++i) {
         if (table->fields[i].minimal_version <= base_version) {
-            field_list_length += 1;                             /* The comma */
-            field_list_length += strlen(table->fields[i].name); /* The field name */
             added_fields += 1;
+
+            /* New table field list, add the comma to the total size */
+            field_list_new_length += 1;
+            field_list_new_length += strlen(table->fields[i].name);
+
+            /*
+            ** Old table field list, add the comma to the total size and take
+            ** into account the copy fallback fields when upgrading
+            */
+
+            if (table->fields[i].fallback_field != NULL) {
+                field_list_old_length += 1;
+                field_list_new_length += strlen(table->fields[i].fallback_field);
+            } else {
+                field_list_old_length += 1;
+                field_list_old_length += strlen(table->fields[i].name);
+            }
         }
     }
     if (added_fields == 0) {
@@ -163,7 +187,8 @@ ___compute_upgrade_sql_query(LKT_DATABASE_VERSION base_version,
         return false;
     }
 
-    if (field_list_length >= LKT_MAX_SQLITE_STATEMENT) {
+    if ((field_list_new_length >= LKT_MAX_SQLITE_STATEMENT) ||
+        (field_list_old_length >= LKT_MAX_SQLITE_STATEMENT)) {
         LOG_ERROR("DB-UPGRADE",
                   "The comma separated list for all the fields "
                   "in the table '%s' will overflow, failed to "
@@ -172,26 +197,31 @@ ___compute_upgrade_sql_query(LKT_DATABASE_VERSION base_version,
         return false;
     }
 
-    char field_list[LKT_MAX_SQLITE_STATEMENT];
-    memset(field_list, 0, sizeof(field_list));
-    added_fields = 0;
+    char field_list_new[LKT_MAX_SQLITE_STATEMENT];
+    char field_list_old[LKT_MAX_SQLITE_STATEMENT];
+    memset(field_list_new, 0, sizeof(field_list_new));
+    memset(field_list_old, 0, sizeof(field_list_old));
+
     for (size_t i = 0; table->fields[i].name; ++i) {
         if (table->fields[i].minimal_version <= base_version) {
-            strcat(field_list, table->fields[i].name);
-            strcat(field_list, ",");
-            added_fields += 1;
+            strcat(field_list_new, table->fields[i].name);
+            strcat(field_list_new, ",");
+            if (table->fields[i].fallback_field != NULL) {
+                strcat(field_list_old, table->fields[i].fallback_field);
+                strcat(field_list_old, ",");
+            } else {
+                strcat(field_list_old, table->fields[i].name);
+                strcat(field_list_old, ",");
             }
         }
-    if (added_fields == 0) {
-        LOG_ERROR("DB-UPGRADE", "No fields where found for the database table %s", table->name);
-        return false;
     }
-    field_list[strlen(field_list) - 1] = '\0'; /* Crop the last "," */
+    field_list_new[strlen(field_list_new) - 1] = '\0'; /* Crop the last "," */
+    field_list_old[strlen(field_list_old) - 1] = '\0'; /* Crop the last "," */
 
     /* Generate the SQL query */
     memset(buffer, 0, buffer_length * sizeof(char));
-    ssize_t length = safe_snprintf(buffer, buffer_length, SQL_UPGRADE_QUERY_TEMPLATE, table->name,
-                                   field_list, field_list, table->name);
+    const ssize_t length = safe_snprintf(buffer, buffer_length, SQL_UPGRADE_QUERY_TEMPLATE,
+                                         table->name, field_list_new, field_list_old, table->name);
 
     /* No overflow -> truncated query, we will have a problem... */
     return (length > 0) && ((buffer_length - 1) >= (size_t)length);