Merge branch 'tool_db-update-file-state'
This commit is contained in:
commit
10d8bb1a39
@ -7,7 +7,8 @@ CREATE TABLE File (
|
|||||||
MD5 VARCHAR(32),
|
MD5 VARCHAR(32),
|
||||||
FT_ID INTEGER DEFAULT 0,
|
FT_ID INTEGER DEFAULT 0,
|
||||||
Size INTEGER, -- KB (max 2TB anggebbar)
|
Size INTEGER, -- KB (max 2TB anggebbar)
|
||||||
TTSync INTEGER DEFAULT 0
|
TTSync INTEGER DEFAULT 0,
|
||||||
|
TTCheck DATE DEFAULT NULL
|
||||||
);
|
);
|
||||||
|
|
||||||
CREATE TABLE VideoFile (
|
CREATE TABLE VideoFile (
|
||||||
|
|||||||
244
tools/db/update-file-state
Normal file → Executable file
244
tools/db/update-file-state
Normal file → Executable file
@ -7,35 +7,48 @@
|
|||||||
# [$1]: <path to movie folder>
|
# [$1]: <path to movie folder>
|
||||||
|
|
||||||
{ #set enviroemnt
|
{ #set enviroemnt
|
||||||
readonly TRUE=1 #like <stdbool.h>
|
readonly TRUE=1 #like <stdbool.h>
|
||||||
readonly FALSE=0 #like <stdbool.h>
|
readonly FALSE=0 #like <stdbool.h>
|
||||||
readonly TTSYNC_EXIST=0
|
readonly TTSYNC_EXIST=0 #file exist local
|
||||||
readonly TTSYNC_BACKUP=1
|
readonly TTSYNC_BACKUP=1 #file exist local - backup exist
|
||||||
readonly TTSYNC_REMOTE=2
|
readonly TTSYNC_REMOTE=2 #file exists remote - not used jet
|
||||||
readonly TTSYNC_DEL=3
|
readonly TTSYNC_DEL=3 #local file deleted
|
||||||
readonly globalconf="/etc/tt/global"
|
readonly globalconf="/etc/tt/global"
|
||||||
|
readonly DEBUG=$TRUE
|
||||||
}
|
}
|
||||||
|
|
||||||
{ #get options & args
|
{ #get options & args
|
||||||
readonly mode=$(echo $1 | tr -d '-')
|
readonly mode=$(echo $1 | tr -d '-')
|
||||||
moviedir="$1"
|
moviedir="$1"
|
||||||
|
[ -z "$moviedir" ] && [ -f "$globalconf" ] && { # if (no argument) and (globalconf exist)
|
||||||
|
if [ -z "$(grep "moviedir=" "$globalconf" )" ]; then # and (moviedir is NOT in globalconf)
|
||||||
|
echo 'neither "movie folder" is set in global config nor given as argument'
|
||||||
|
exit 2
|
||||||
|
else # and (moviedir is in globalconf)
|
||||||
|
moviedir="$(grep "moviedir=" "$globalconf" | cut -d '=' -f 2)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
[ -d "$moviedir" ] || { echo "moviedir \"$moviedir\" do not exist"; exit 3; } # check moviedir
|
||||||
|
moviedir=""$(echo $moviedir | sed 's|/$||g')""
|
||||||
|
[ -z "$moviedir" ] && exit 255 #catch case moviedir="/"
|
||||||
|
readonly DATE=$(date '+%Y-%m-%d')
|
||||||
}
|
}
|
||||||
|
|
||||||
{ #db
|
{ #db
|
||||||
dbname="tt_test"
|
dbname="tt_test"
|
||||||
dbhost="localhost"
|
dbhost="localhost"
|
||||||
dbuser="root"
|
dbuser="root"
|
||||||
dbpwd=""
|
dbpwd=""
|
||||||
}
|
}
|
||||||
|
|
||||||
function help() {
|
function help() {
|
||||||
#help menue
|
#help menue
|
||||||
echo "$0 is part of the TerabyteTerminator Project"
|
echo "$0 is part of the TerabyteTerminator Project"
|
||||||
echo ""
|
echo ""
|
||||||
echo "update-file-state [moviedir]"
|
echo "update-file-state [moviedir]"
|
||||||
echo " moviedir: Is optional if it is already in the global config"
|
echo " moviedir: Is optional if it is already in the global config"
|
||||||
|
|
||||||
exit 0
|
exit 0
|
||||||
}
|
}
|
||||||
[ "$mode" == "help" ] && help
|
[ "$mode" == "help" ] && help
|
||||||
|
|
||||||
@ -43,41 +56,72 @@ function help() {
|
|||||||
## Programm Haupt Ablauf ##
|
## Programm Haupt Ablauf ##
|
||||||
###########################
|
###########################
|
||||||
function main() {
|
function main() {
|
||||||
#cd to moviedir
|
|
||||||
cd "$moviedir"
|
|
||||||
|
|
||||||
## dateiliste erstellen die mit DB abgeglichen wird
|
#db checks
|
||||||
|
[ "$(db_check)" == $FALSE ] && {
|
||||||
|
echo "DB ERROR!!" >&2
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
## alle die keien DB.PATH eintrag haben -> file_add
|
#clean DB from old stuff
|
||||||
|
{
|
||||||
|
db_query "DELETE From File WHERE Size=0 OR MD5='' OR MD5 IS NULL OR Name='';"
|
||||||
|
#find dublicates !
|
||||||
|
}
|
||||||
|
|
||||||
## Für alle existierenden Dateien:
|
#erselle zwei dateilisten
|
||||||
## * Path=DB.PATH && TTSync == TTSYNC_DEL
|
list_tmp=$(mktemp -d)
|
||||||
## -> DB.Size != 0 && Size == DB.Size -> TTSYNC_DEL -> TTSYNC_EXIST // >>file_undel<< konsider other state ?!?
|
db_query "SELECT CONCAT('$moviedir/', Folder, '/', Name) FROM File WHERE TTSync=$TTSYNC_EXIST OR TTSync=$TTSYNC_BACKUP;" > "$list_tmp/indb"
|
||||||
## -> DB.Size != 0 || Size != DB.Size -> compare if md5 is same
|
find "$moviedir" -type f > "$list_tmp/exist"
|
||||||
## -> Ja -> TTSYNC_DEL -> TTSYNC_EXIST // >>file_undel<<
|
|
||||||
## -> NA -> delete DB Entry
|
|
||||||
## * Path=DB.PATH && TTSync == TTSYNC_EXIST / TTSYNC_BACKUP
|
|
||||||
## -> DB.Size = 0 -> file_add_size
|
|
||||||
## -> Size != DB.Size -> compare if md5 is same
|
|
||||||
## -> Ja -> update file size (file_add_size)
|
|
||||||
## -> NA
|
|
||||||
## -> makr DB.FID = TTSYNC_DEL
|
|
||||||
## -> file_add // this is a new file
|
|
||||||
## * Path=DB.PATH && TTSync == TTSYNC_REMOTE
|
|
||||||
## -> DB.Size = 0 -> compare if md5 is same
|
|
||||||
## -> Ja
|
|
||||||
## -> update file size (file_add_size)
|
|
||||||
## -> TTSYNC_REMOTE to TTSYNC_BACKUP
|
|
||||||
## -> NA
|
|
||||||
## -> Size = DB.Size TTSYNC_REMOTE to TTSYNC_BACKUP
|
|
||||||
## -> Size != DB.Size // Error Remote CP incorect!!!
|
|
||||||
|
|
||||||
## alle nicht gecheckten path einträge in DB (fiels die nicht existieren)
|
#find new files
|
||||||
## A: HDD kaput
|
grep -vf "$list_tmp/indb" "$list_tmp/exist" > "$list_tmp/new"
|
||||||
## B: Absichtlich gelöscht
|
|
||||||
## hdd_check // ob eine HDD defekt ist bzw das mounten irgendwo fehlgeschlagen ist
|
#all IDs wich an old check date:
|
||||||
## -> ja -> error mesage!
|
db_query "SELECT ID FROM File Where TTCheck < '$DATE' OR TTCheck IS NULL;" | while read line; do
|
||||||
## -> na -> set DB.FID TTSYNC_DEL
|
#for each ID with an old check date ...
|
||||||
|
ID=$line
|
||||||
|
[ $DEBUG == $TRUE ] && echo "Progress ID: $ID" >&2
|
||||||
|
FILE=$(db_query "SELECT CONCAT('$moviedir/', Folder, '/', Name) FROM File WHERE ID=$ID;")
|
||||||
|
if [ -f "$FILE" ]; then
|
||||||
|
SIZE=$(db_query "SELECT Size FROM File WHERE ID=$ID;")
|
||||||
|
SIZE_new="$(file_size "$FILE")"
|
||||||
|
[ "$SIZE" != "$SIZE_new" ] && {
|
||||||
|
echo "FILE CHANGE: $FILE"
|
||||||
|
if [ "0" == "$SIZE_new" ]; then
|
||||||
|
#file exist but empty
|
||||||
|
#rm "$FILE" <- secure? 2check
|
||||||
|
db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE ID=$ID;"
|
||||||
|
else
|
||||||
|
#recalc md5 ... + check
|
||||||
|
#update DB
|
||||||
|
MD5_new=$(md5sum "$FILE" | cut -c -32)
|
||||||
|
MD5=$(db_query "SELECT MD5 FROM File WHERE ID=$ID;")
|
||||||
|
|
||||||
|
if [ "$MD5" == "$MD5_new" ]; then
|
||||||
|
db_query "UPDATE File SET Size=$SIZE_new,TTCheck='$DATE' WHERE ID=$ID;"
|
||||||
|
else
|
||||||
|
#new file/file changed!
|
||||||
|
db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE ID=$ID;"
|
||||||
|
file_add "$(db_query "SELECT CONCAT(Folder, '/', Name) FROM File WHERE ID=$ID;")" "$MD5_new"
|
||||||
|
fi
|
||||||
|
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
else
|
||||||
|
# file dont exist
|
||||||
|
db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE ID=$ID;"
|
||||||
|
fi
|
||||||
|
|
||||||
|
done
|
||||||
|
|
||||||
|
#add new files ....
|
||||||
|
cat "$list_tmp/new" | while read line; do
|
||||||
|
file_add "$line"
|
||||||
|
done
|
||||||
|
|
||||||
|
|
||||||
|
rm -r "$list_tmp/"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -86,48 +130,92 @@ function main() {
|
|||||||
###########################
|
###########################
|
||||||
|
|
||||||
##check & prepair args
|
##check & prepair args
|
||||||
[ -z "$moviedir" ] && [ -f "$globalconf" ] && { # if (no argument) and (globalconf exist)
|
[ -z "$moviedir" ] && [ -f "$globalconf" ] && { # if (no argument) and (globalconf exist)
|
||||||
if [ -z "$(grep "moviedir=" "$globalconf" )" ]; then # and (moviedir is NOT in globalconf)
|
if [ -z "$(grep "moviedir=" "$globalconf" )" ]; then # and (moviedir is NOT in globalconf)
|
||||||
echo 'neither "movie folder" is set in global config nor given as argument'
|
echo 'neither "movie folder" is set in global config nor given as argument'
|
||||||
exit 2
|
exit 2
|
||||||
else # and (moviedir is in globalconf)
|
else # and (moviedir is in globalconf)
|
||||||
moviedir="$(grep "moviedir=" "$globalconf" | cut -d '=' -f 2)"
|
moviedir="$(grep "moviedir=" "$globalconf" | cut -d '=' -f 2)"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
[ -d "$moviedir" ] || { echo "moviedir \"$moviedir\" do not exist"; exit 3; } # check moviedir
|
[ -d "$moviedir" ] || { echo "moviedir \"$moviedir\" do not exist"; exit 3; } # check moviedir
|
||||||
moviedir=""$(echo $moviedir | sed 's|/$||g')""
|
moviedir=""$(echo $moviedir | sed 's|/$||g')""
|
||||||
[ -z "$moviedir" ] && exit 255 #catch case moviedir="/"
|
[ -z "$moviedir" ] && exit 255 #catch case moviedir="/"
|
||||||
|
|
||||||
function md5_out_exist() {
|
|
||||||
db_query "SELECT CONCAT(MD5, ' ', ' ' ,'$moviedir/', Folder, '/', Name) FROM File WHERE TTSync=$TTSYNC_EXIST OR TTSync=$TTSYNC_BACKUP;" >> $md5list
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
function md5_out_del() {
|
|
||||||
db_query "SELECT CONCAT(MD5, ' ', ' ' ,'$moviedir/', Folder, '/', Name) FROM File WHERE TTSync=$TTSYNC_DEL;" >> $md5list
|
|
||||||
exit 0
|
|
||||||
}
|
|
||||||
|
|
||||||
# file_size <path> -> (du -b)/1024 [Kb]
|
# file_size <path> -> (du -b)/1024 [Kb]
|
||||||
function file_size() {
|
function file_size() {
|
||||||
echo $(($(du -b "$1" | cut -f 1)/1024))
|
if [ -f "$1" ]; then
|
||||||
|
echo $(($(du -b "$1" | cut -f 1)/1024))
|
||||||
|
else
|
||||||
|
echo 0
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# file_add [rel.path] (md5)
|
||||||
|
function file_add() {
|
||||||
|
local MD5="$2"
|
||||||
|
local relrevpath="$(echo "$1" | sed "s|^$moviedir||g" | sed 's|^/||g' | sed 's|/$||g' | rev | sed "s/'/''/g")" #last statement set ' as '' so it wont have an evect in the sql statement
|
||||||
|
local name="$(echo "$relrevpath" | cut -d '/' -f 1 | rev)"
|
||||||
|
local folder="$(echo "$relrevpath" | cut -d '/' -f 2- | rev)"
|
||||||
|
local FILE="$moviedir/$(echo "$relrevpath" | rev)"
|
||||||
|
[ -f "$FILE" ] || {
|
||||||
|
echo "ERROR: file_add: file not exist: $FILE"
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
[ $DEBUG == $TRUE ] && echo "add: $name" >&2
|
||||||
|
[ -z "$MD5" ] && MD5="$(md5sum "$FILE" | cut -c -32)"
|
||||||
|
local SIZE="$(file_size "$FILE")"
|
||||||
|
[ "$SIZE" -le 0 ] && {
|
||||||
|
[ $DEBUG == $TRUE ] && echo File to smal >&2
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
#wenn einmal existiert ...
|
||||||
|
{
|
||||||
|
# undelet files ...
|
||||||
|
local IDs_undel="$(db_query "SELECT ID FROM File WHERE TTSync='$TTSYNC_DEL' AND MD5='$MD5' AND Name='$name' AND Folder='$folder';")"
|
||||||
|
[ $DEBUG == $TRUE ] && echo "IDs_undel: $IDs_undel" >&2
|
||||||
|
db_query "UPDATE File SET TTSync=$TTSYNC_EXIST,TTCheck='$DATE',Size='$SIZE' WHERE TTSync='$TTSYNC_DEL' AND MD5='$MD5' AND Name='$name' AND Folder='$folder';"
|
||||||
|
|
||||||
|
#same path wrong md5
|
||||||
|
[ $DEBUG == $TRUE ] && echo "IDs_W5: $(db_query "SELECT ID FROM File WHERE TTSync!='$TTSYNC_REMOTE' AND TTSync!='$TTSYNC_DEL' AND MD5!='$MD5' AND Name='$name' AND Folder='$folder';")" >&2
|
||||||
|
db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE TTSync!='$TTSYNC_REMOTE' AND TTSync!='$TTSYNC_DEL' AND MD5!='$MD5' AND Name='$name' AND Folder='$folder';"
|
||||||
|
|
||||||
|
#if something MVs ...
|
||||||
|
local IDs_mv="$(db_query "SELECT ID FROM File WHERE MD5='$MD5' AND Size='$SIZE' AND (Name!='$name' OR Folder!='$folder');")"
|
||||||
|
[ $DEBUG == $TRUE ] && echo "IDs_mv: $IDs_mv" >&2
|
||||||
|
db_query "UPDATE File SET TTCheck='$DATE',Name='$name',Folder='$folder',TTSync='$TTSYNC_EXIST' WHERE MD5='$MD5' AND Size='$SIZE' AND (Name!='$name' OR Folder!='$folder');"
|
||||||
|
|
||||||
|
#es existiert genau so ein eintrag
|
||||||
|
local IDs_orig="$(db_query "SELECT ID FROM File WHERE MD5='$MD5' AND Name='$name' AND Folder='$folder';")"
|
||||||
|
[ $DEBUG == $TRUE ] && echo "IDs_orig: $IDs_orig" >&2
|
||||||
|
}
|
||||||
|
|
||||||
|
#wenn weder gelöschte wiederhergestellt noch verschobene dateien entdekt... -> neuer eintrag
|
||||||
|
[ "$((${#IDs_mv}+${#IDs_undel}+${#IDs_orig}))" == 0 ] && db_query "INSERT INTO File (Name,Folder,MD5,Size,TTSync,TTCheck) VALUES ('$name','$folder','$MD5','$SIZE',$TTSYNC_EXIST,'$DATE');"
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# db_query <query> -> result
|
# db_query <query> -> result
|
||||||
function db_query() {
|
function db_query() {
|
||||||
local args="-sN" #no headers, no grid
|
local args="-sN" #no headers, no grid
|
||||||
args+=" -h$dbhost"
|
args+=" -h$dbhost"
|
||||||
args+=" -u$dbuser"
|
args+=" -u$dbuser"
|
||||||
[ -n "$dbpwd" ] && args+=" -p$dbpwd"
|
[ -n "$dbpwd" ] && args+=" -p$dbpwd"
|
||||||
args+=" $dbname"
|
args+=" $dbname"
|
||||||
echo $@ | mysql $args
|
[ $DEBUG == $TRUE ] && echo $@ >&2
|
||||||
|
echo $@ | mysql $args
|
||||||
}
|
}
|
||||||
|
|
||||||
# db_F-ID_getPath <F-ID> -> <Path>
|
function db_check() {
|
||||||
function db_id_getPath() {
|
db_query "SHOW GRANTS FOR CURRENT_USER();" > /dev/null 2> /dev/null
|
||||||
db_query "SELECT CONCAT('$moviedir/', Folder, '/', Name) FROM File WHERE ID=$1;"
|
if [ $? == 0 ]; then
|
||||||
|
echo $TRUE
|
||||||
|
else
|
||||||
|
echo $FALSE
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
main #aufruf der main function (eig. programmstart)
|
main #aufruf der main function (eig. programmstart)
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user