From f9bdcb088a9330f71b9100c3c25840f5ef3f3f5a Mon Sep 17 00:00:00 2001
From: Elliu <elliu@hashi.re>
Date: Mon, 15 Aug 2022 23:36:34 +0200
Subject: [PATCH] WIP: implement checksum usage
---
matrixpack.sh | 10 +++----
stpkg | 83 +++++++++++++++++++++++++++++++++++++++++++++++----
2 files changed, 83 insertions(+), 10 deletions(-)
diff --git a/matrixpack.sh b/matrixpack.sh
index 2b81610..84add5f 100644
--- a/matrixpack.sh
+++ b/matrixpack.sh
@@ -1,7 +1,7 @@
#!/bin/bash
# Usage
# matrixpack token homeserver directory...
-#
+#
# Pack name is the directory name
# Sticker name is the file name
# /!\ Token is a sensitive information
@@ -58,7 +58,7 @@ else
fi
# Checking if the token exist.
-# Save the error value on file "headers" and
+# Save the error value on file "headers" and
# evaluate it to verify the token
curl -s -X POST "https://$homeserver/_matrix/media/r0/upload?access_token=$token" | python3 -c "import sys, json; print(json.load(sys.stdin)['errcode'])" >> headers
while read line; do
@@ -132,10 +132,10 @@ do
echo -n "."
convert -background none -gravity center "$slug/tmp/$f.$type" -resize "${width}x$height" $opts"$slug/tmp/$f.$type"
echo -ne ". \033[92mdone\033[0m! "
-
+
# First item in array
echo -n "$first" >> "$slug/$slug.json"
-
+
# Uploading image
echo -n "uploading."
mxc=$(curl -s -X POST -H "Content-Type: image/$type" --data-binary "@$slug/tmp/$f.$type" "https://$homeserver/_matrix/media/r0/upload?access_token=$token" | python -c "import sys, json; print(json.load(sys.stdin)['content_uri'])")
@@ -152,7 +152,7 @@ do
echo -e ". \033[92msuccess\033[0m!"
fi
done
-
+
rm "$slug/tmp/size"
montage "$slug/tmp/*"[0] -background none "$slug/preview.png"
rm -r "$slug/tmp"
diff --git a/stpkg b/stpkg
index a074f17..9296b5d 100755
--- a/stpkg
+++ b/stpkg
@@ -282,6 +282,7 @@ STPKG_MAX_WIDTH=256
STPKG_GENERATE_PREVIEW="no"
STPKG_IMAGE_TARGET_FILETYPE="image/png"
STPKG_ANIMATED_TARGET_FILETYPE="image/gif"
+STPKG_REUPLOAD_SAME_FILE="no"
__do_mount() { :; }
@@ -582,7 +583,6 @@ __list() {
# Usage: process_sticker_file <filename> <&JSON>
process_sticker_file() {
FILE="$1"
-
local -n JSON_REF="$2"
# Get sizes
@@ -612,7 +612,6 @@ process_sticker_file() {
local INIT_TYPE=$(file_get_type "$FILE")
local INIT_EXT=$(file_get_ext_from_type "$INIT_TYPE")
local NAME=$(file_get_name "$FILE")
-
local TEMP=`mktemp --suffix=.stpkg`
if [ "x$TYPE" != "xgif" ] && [ "x$TYPE" != "xpng" ]; then
@@ -791,12 +790,48 @@ __pack() {
local TOTAL_FILES=`find . -maxdepth 1 -type f | wc -l`
local CURRENT_FILE=1
local FIRST_IN_ARRAY=""
- for FILE in *; do
+
+ # Check checksums
+ local SUM_FILE=".${PACK_NAME}.checksums"
+ if [ ! -f "$SUM_FILE" ]; then
+ touch $SUM_FILE
+ fi
+ local -a newFiles
+ local -A hashes unchangedFiles updatedFiles deletedFiles
+ local TEMP_CHECK=`mktemp --suffix=.stpkg`
+ md5sum -c $SUM_FILE &>/dev/null >$TEMP_CHECK
+ while read -r _hash _file; do
+ hashes["$_file"]="$_hash"
+ done < $SUM_FILE
+ newFiles=(*)
+
+ for i in "`cat $TEMP_CHECK | grep ": OK$" | sed "s/: OK$//g"`"; do
+ unchangedFiles["$i"]=("${hashes["$i"]}")
+ unset hashes["$i"]
+ newFiles="${newFiles[@]/$i}"
+ done
+ for i in "`cat $TEMP_CHECK | grep ": FAILED$" | sed "s/: FAILED$//g"`"; do
+ updatedFiles["$i"]=("${hashes["$i"]}")
+ unset hashes["$i"]
+ newFiles="${newFiles[@]/$i}"
+ done
+ for i in "`cat $TEMP_CHECK | grep ": FAILED open or read$" | sed "s/: FAILED open or read$//g"`"; do
+ deletedFiles["$i"]=("${hashes["$i"]}")
+ unset hashes["$i"]
+ newFiles="${newFiles[@]/$i}"
+ done
+
+ for FILE in "${!newFiles[@]}"; do
# Add a ',' only if it's not the first in the array
echo -n "$FIRST_IN_ARRAY" >> $DEST_INDEX
+ if [[ $(md5sum $FILE) = $(cat $SUM_FILE | grep ) ]]; then
+ else
+ md5sum $FILE >> $SUM_FILE
+ fi
+
local ST_JSON=""
- process_sticker_file $FILE ST_JSON
+ process_sticker_file $FILE ST_JSON $SUM_FILE
echo "$ST_JSON" >> $DEST_INDEX
# For the report
@@ -804,9 +839,41 @@ __pack() {
local FIRST_IN_ARRAY=","
echo -e "$ST_JSON" | jq -r '[.body, .url, .info.mimetype, .info.w, .info.h]|join(" ")'
+ # TODO: add hashes
+
progress_reset
done | column -t
+ for FILE in "${!unchangedFiles[@]}"; do
+ echo -n "$FIRST_IN_ARRAY" >> $DEST_INDEX
+ [ "x$FIRST_IN_ARRAY" = "x" ] && echo -e "StickerName MXC Type Width Height"
+ local FIRST_IN_ARRAY=","
+ #echo -e "$ST_JSON" | jq -r '[.body, .url, .info.mimetype, .info.w, .info.h]|join(" ")'
+ echo "$FILE unchanged"
+ done | column -t
+
+ for FILE in "${!updatedFiles[@]}"; do
+ echo -n "$FIRST_IN_ARRAY" >> $DEST_INDEX
+ [ "x$FIRST_IN_ARRAY" = "x" ] && echo -e "StickerName MXC Type Width Height"
+ local FIRST_IN_ARRAY=","
+ echo "$FILE updated"
+ done | column -t
+
+ for FILE in "${!deletedFiles[@]}"; do
+ echo -n "$FIRST_IN_ARRAY" >> $DEST_INDEX
+ [ "x$FIRST_IN_ARRAY" = "x" ] && echo -e "StickerName MXC Type Width Height"
+ local FIRST_IN_ARRAY=","
+ #echo -e "$ST_JSON" | jq -r '[.body, .url, .info.mimetype, .info.w, .info.h]|join(" ")'
+ echo "$FILE deleted, but kept in the pack"
+ done | column -t
+
+ # Update the sum file:
+ # Keep deleted but kept
+ # Remove deleted and deleted
+ # Update updated
+ # Keep unchanged
+ # Add new
+
# Create the preview
#montage "$DEST_FOLDER/*.{png,gif}" -background none "$DEST_FOLDER/preview.png"
#echo -ne "# $PACK_NAME\n" > "$DEST_FOLDER/README.md"
@@ -903,6 +970,12 @@ __do_command() {
shift
;;
+ # Do not reupload unchanged stickers
+ -reupload)
+ STPKG_REUPLOAD_SAME_FILE="yes"
+ shift
+ ;;
+
*)
local PARSE_FLAGS=0
;;
@@ -916,7 +989,7 @@ __do_command() {
STPKG_GENERATE_PREVIEW=${STPKG_GENERATE_PREVIEW:="no"}
STPKG_IMAGE_TARGET_FILETYPE=${STPKG_IMAGE_TARGET_FILETYPE:="image/png"}
STPKG_ANIMATED_TARGET_FILETYPE=${STPKG_ANIMATED_TARGET_FILETYPE:="image/gif"}
-
+ STPKG_REUPLOAD_SAME_FILE=${STPKG_REUPLOAD_SAME_FILE:="no"}
;;&
*)
--
GitLab