Merge branch 'tool_db-update-file-state'

DB/shema.sql

@@ -7,7 +7,8 @@ CREATE TABLE File (
 	MD5 VARCHAR(32),
 	FT_ID INTEGER DEFAULT 0,
 	Size INTEGER,							-- KB (max 2TB anggebbar)
-	TTSync INTEGER DEFAULT 0
+	TTSync INTEGER DEFAULT 0,
+	TTCheck DATE DEFAULT NULL
  );
 
 CREATE TABLE VideoFile (

tools/db/update-file-state

@@ -9,16 +9,29 @@
 { #set enviroemnt
     readonly TRUE=1      #like <stdbool.h>
     readonly FALSE=0     #like <stdbool.h>
-  readonly TTSYNC_EXIST=0
+    readonly TTSYNC_EXIST=0       #file exist local
-  readonly TTSYNC_BACKUP=1
+    readonly TTSYNC_BACKUP=1      #file exist local - backup exist
-  readonly TTSYNC_REMOTE=2
+    readonly TTSYNC_REMOTE=2      #file exists remote - not used jet
-  readonly TTSYNC_DEL=3
+    readonly TTSYNC_DEL=3         #local file deleted
     readonly globalconf="/etc/tt/global"
+    readonly DEBUG=$TRUE
 }
 
 { #get options & args
     readonly mode=$(echo $1 | tr -d '-')
     moviedir="$1"
+    [ -z "$moviedir" ] && [ -f "$globalconf" ] && {          # if (no argument) and (globalconf exist)
+        if [ -z "$(grep "moviedir=" "$globalconf" )" ]; then      # and (moviedir is NOT in globalconf)
+            echo 'neither "movie folder" is set in global config nor given as argument'
+            exit 2
+        else                              # and (moviedir is in globalconf)
+            moviedir="$(grep "moviedir=" "$globalconf" | cut -d '=' -f 2)"
+        fi
+    }
+    [ -d "$moviedir" ] || { echo "moviedir \"$moviedir\" do not exist"; exit 3; } # check moviedir
+    moviedir=""$(echo $moviedir | sed 's|/$||g')""
+    [ -z "$moviedir" ] && exit 255 #catch case moviedir="/"
+    readonly DATE=$(date '+%Y-%m-%d')
 }
 
 { #db
@@ -43,41 +56,72 @@ function help() {
 ## Programm Haupt Ablauf ##
 ###########################
 function main() {
-  #cd to moviedir
-  cd "$moviedir"
 
-  ## dateiliste erstellen die mit DB abgeglichen wird
+    #db checks
+    [ "$(db_check)" == $FALSE ] && {
+        echo "DB ERROR!!" >&2
+        exit 1
+    }
 
-  ## alle die keien DB.PATH eintrag haben -> file_add
+    #clean DB from old stuff
+    {
+        db_query "DELETE From File WHERE Size=0 OR MD5='' OR MD5 IS NULL OR Name='';"
+        #find dublicates !
+    }
 
-  ## Für alle existierenden Dateien:
+    #erselle zwei dateilisten
-  ## * Path=DB.PATH && TTSync == TTSYNC_DEL
+    list_tmp=$(mktemp -d)
-  ##    -> DB.Size != 0 && Size == DB.Size -> TTSYNC_DEL -> TTSYNC_EXIST // >>file_undel<< konsider other state ?!?
+    db_query "SELECT CONCAT('$moviedir/', Folder, '/', Name) FROM File WHERE TTSync=$TTSYNC_EXIST OR TTSync=$TTSYNC_BACKUP;" > "$list_tmp/indb"
-  ##    -> DB.Size != 0 || Size != DB.Size -> compare if md5 is same
+    find "$moviedir" -type f > "$list_tmp/exist"
-  ##         -> Ja -> TTSYNC_DEL -> TTSYNC_EXIST // >>file_undel<<
-  ##         -> NA -> delete DB Entry
-  ## * Path=DB.PATH && TTSync == TTSYNC_EXIST / TTSYNC_BACKUP
-  ##    -> DB.Size = 0 -> file_add_size
-  ##    -> Size != DB.Size -> compare if md5 is same
-  ##         -> Ja -> update file size (file_add_size)
-  ##         -> NA
-  ##            -> makr DB.FID = TTSYNC_DEL
-  ##            -> file_add  // this is a new file
-  ## * Path=DB.PATH && TTSync == TTSYNC_REMOTE
-  ##    -> DB.Size = 0 ->  compare if md5 is same
-  ##         -> Ja
-  ##            -> update file size (file_add_size)
-  ##            -> TTSYNC_REMOTE to TTSYNC_BACKUP
-  ##         -> NA
-  ##    -> Size = DB.Size TTSYNC_REMOTE to TTSYNC_BACKUP
-  ##    -> Size != DB.Size  // Error Remote CP incorect!!!
 
- ## alle nicht gecheckten path einträge in DB (fiels die nicht existieren)
+    #find new files
- ## A: HDD kaput
+    grep -vf "$list_tmp/indb" "$list_tmp/exist" > "$list_tmp/new"
- ## B: Absichtlich gelöscht
+
- ## hdd_check // ob eine HDD defekt ist bzw das mounten irgendwo fehlgeschlagen ist
+    #all IDs wich an old check date:
- ## -> ja -> error mesage!
+    db_query "SELECT ID FROM File Where TTCheck < '$DATE' OR TTCheck IS NULL;" | while read line; do
- ## -> na -> set DB.FID TTSYNC_DEL
+        #for each ID with an old check date ...
+        ID=$line
+        [ $DEBUG == $TRUE ] && echo "Progress ID: $ID" >&2
+        FILE=$(db_query "SELECT CONCAT('$moviedir/', Folder, '/', Name) FROM File WHERE ID=$ID;")
+        if [ -f "$FILE" ]; then
+            SIZE=$(db_query "SELECT Size FROM File WHERE ID=$ID;")
+            SIZE_new="$(file_size "$FILE")"
+            [ "$SIZE" != "$SIZE_new" ] && {
+                echo "FILE CHANGE: $FILE"
+                if [ "0" ==  "$SIZE_new" ]; then
+                    #file exist but empty
+                    #rm "$FILE" <- secure? 2check
+                    db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE ID=$ID;"
+                else
+                    #recalc md5 ... + check
+                    #update DB
+                    MD5_new=$(md5sum "$FILE" | cut -c -32)
+                    MD5=$(db_query "SELECT MD5 FROM File WHERE ID=$ID;")
+
+                    if [ "$MD5" == "$MD5_new" ]; then
+                        db_query "UPDATE File SET Size=$SIZE_new,TTCheck='$DATE' WHERE ID=$ID;"
+                    else
+                        #new file/file changed!
+                        db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE ID=$ID;"
+                        file_add "$(db_query "SELECT CONCAT(Folder, '/', Name) FROM File WHERE ID=$ID;")" "$MD5_new"
+                    fi
+
+                fi
+            }
+        else
+            # file dont exist
+            db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE ID=$ID;"
+        fi
+
+    done
+
+    #add new files ....
+    cat "$list_tmp/new" | while read line; do
+        file_add "$line"
+    done
+
+
+    rm -r "$list_tmp/"
 
 }
 
@@ -98,21 +142,62 @@ function main() {
 moviedir=""$(echo $moviedir | sed 's|/$||g')""
 [ -z "$moviedir" ] && exit 255 #catch case moviedir="/"
 
-function md5_out_exist() {
-  db_query "SELECT CONCAT(MD5, ' ', ' ' ,'$moviedir/', Folder, '/', Name) FROM File WHERE TTSync=$TTSYNC_EXIST OR TTSync=$TTSYNC_BACKUP;" >> $md5list
-  exit 0
-}
-
-function md5_out_del() {
-  db_query "SELECT CONCAT(MD5, ' ', ' ' ,'$moviedir/', Folder, '/', Name) FROM File WHERE TTSync=$TTSYNC_DEL;" >> $md5list
-  exit 0
-}
 
 # file_size <path> -> (du -b)/1024  [Kb]
 function file_size() {
+    if [ -f "$1" ]; then
         echo $(($(du -b "$1" | cut -f 1)/1024))
+    else
+        echo 0
+    fi
 }
 
+# file_add [rel.path] (md5)
+function file_add() {
+    local MD5="$2"
+    local relrevpath="$(echo "$1" | sed "s|^$moviedir||g" | sed 's|^/||g' | sed 's|/$||g' | rev | sed "s/'/''/g")"   #last statement set ' as '' so it wont have an evect in the sql statement
+    local name="$(echo "$relrevpath" | cut -d '/' -f 1 | rev)"
+    local folder="$(echo "$relrevpath" | cut -d '/' -f 2- | rev)"
+    local FILE="$moviedir/$(echo "$relrevpath" | rev)"
+    [ -f "$FILE" ] || {
+        echo "ERROR: file_add: file not exist: $FILE"
+        return 1
+    }
+    [ $DEBUG == $TRUE ] && echo "add: $name" >&2
+    [ -z "$MD5" ] && MD5="$(md5sum "$FILE" | cut -c -32)"
+    local SIZE="$(file_size "$FILE")"
+    [ "$SIZE" -le 0 ] && {
+        [ $DEBUG == $TRUE ] && echo File to smal >&2
+        return 0
+    }
+
+    #wenn einmal existiert ...
+    {
+        # undelet files ...
+        local IDs_undel="$(db_query "SELECT ID FROM File WHERE TTSync='$TTSYNC_DEL' AND MD5='$MD5' AND Name='$name' AND Folder='$folder';")"
+        [ $DEBUG == $TRUE ] && echo "IDs_undel: $IDs_undel" >&2
+        db_query "UPDATE File SET TTSync=$TTSYNC_EXIST,TTCheck='$DATE',Size='$SIZE' WHERE TTSync='$TTSYNC_DEL' AND MD5='$MD5' AND Name='$name' AND Folder='$folder';"
+
+        #same path wrong md5
+        [ $DEBUG == $TRUE ] && echo "IDs_W5: $(db_query "SELECT ID FROM File WHERE TTSync!='$TTSYNC_REMOTE' AND TTSync!='$TTSYNC_DEL' AND MD5!='$MD5' AND Name='$name' AND Folder='$folder';")" >&2
+        db_query "UPDATE File SET TTSync=$TTSYNC_DEL,TTCheck='$DATE' WHERE TTSync!='$TTSYNC_REMOTE' AND TTSync!='$TTSYNC_DEL' AND MD5!='$MD5' AND Name='$name' AND Folder='$folder';"
+
+        #if something MVs ...
+        local IDs_mv="$(db_query "SELECT ID FROM File WHERE MD5='$MD5' AND Size='$SIZE' AND (Name!='$name' OR Folder!='$folder');")"
+        [ $DEBUG == $TRUE ] && echo "IDs_mv: $IDs_mv" >&2
+        db_query "UPDATE File SET TTCheck='$DATE',Name='$name',Folder='$folder',TTSync='$TTSYNC_EXIST' WHERE MD5='$MD5' AND Size='$SIZE' AND (Name!='$name' OR Folder!='$folder');"
+
+        #es existiert genau so ein eintrag
+        local IDs_orig="$(db_query "SELECT ID FROM File WHERE MD5='$MD5' AND Name='$name' AND Folder='$folder';")"
+        [ $DEBUG == $TRUE ] && echo "IDs_orig: $IDs_orig" >&2
+    }
+
+    #wenn weder gelöschte wiederhergestellt noch verschobene dateien entdekt... -> neuer eintrag
+    [ "$((${#IDs_mv}+${#IDs_undel}+${#IDs_orig}))" == 0 ] && db_query "INSERT INTO File (Name,Folder,MD5,Size,TTSync,TTCheck) VALUES ('$name','$folder','$MD5','$SIZE',$TTSYNC_EXIST,'$DATE');"
+
+}
+
+
 # db_query <query> -> result
 function db_query() {
     local args="-sN"  #no headers, no grid
@@ -120,14 +205,17 @@ function db_query() {
     args+=" -u$dbuser"
     [ -n "$dbpwd" ] && args+=" -p$dbpwd"
     args+=" $dbname"
+    [ $DEBUG == $TRUE ] && echo $@ >&2
     echo $@ | mysql $args
 }
 
-# db_F-ID_getPath <F-ID> -> <Path>
+function db_check()  {
-function db_id_getPath() {
+    db_query "SHOW GRANTS FOR CURRENT_USER();" > /dev/null 2> /dev/null
-  db_query "SELECT CONCAT('$moviedir/', Folder, '/', Name) FROM File WHERE ID=$1;"
+    if [ $? == 0 ]; then
+        echo $TRUE
+    else
+        echo $FALSE
+    fi
 }
 
-
-
 main #aufruf der main function (eig. programmstart)