Coding a Simple Dropbox with Bash

Posted: 29th May 2012 by hackengineer in networking, scripting
Tags: , , , ,

I wrote this bash script very recently and I would say its still in “beta”. USE AT YOUR OWN RISK!! It has been working great but I recommend backing up your synced data just in case.

With that said lets get into the code! I have two bash scripts, one to sync the data and one to properly delete files. In order to get two folders synced for the first time is enter the path to the local sync folder, the path to the remote sync folder, and the port. Remember that by default rsync does not have access to the remotes root folder only the user home directory that you log in as. The remote sync directory is relative to this directory.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#!/bin/sh

#Copyright (2012 hackengineer.com)
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

VAR_LOCAL_SYNC_DIR="/Users/myUserName/mySyncDir" #ADD root sync directory here without trailing /
VAR_REMOTE_SYNC_DIR="www.remoteHost.com:remoteSyncDir" #ADD remote sync directory relative to ssh directory here without trailing /
#VAR_REMOTE_SYNC_DIR="mac.local:remoteSyncDir" #local network example
VAR_PORT=22
VAR_RSYNC="/usr/bin/rsync" #specify where rsync lives if needed
#--------------------------------------------------

VAR_VERSION=0.1


function deleteFile {
    echo "removing $VAR_FULL_FILE"
    VAR_DATE=`date`
    echo "$VAR_FULL_FILE,$VAR_TIMESTAMP,$VAR_SIZE,$VAR_DATE" >> $VAR_LOCAL_SYNC_DIR/.rsync/rsyncLocalDeleteLog
    `rm -r $VAR_FULL_FILE`
}

#Will not delete from a cron
function proptDelete {
    if [ "$RUNNINGFROMCRON" != 1 ]; then
        echo "$VAR_FULL_FILE changed, Rem->Loc Size:$VAR_SIZE->$VAR_MY_SIZE ModTime:$VAR_TIMESTAMP->$VAR_MY_TIMESTAMP"
        echo "Delete?(y/n) "
        read word1
        if [ $word1 = "y" ]; then
            deleteFile
        fi
    else
        echo "$VAR_FULL_FILE changed!! Size:$VAR_SIZE->$VAR_MY_SIZE ModTime:$VAR_TIMESTAMP->$VAR_MY_TIMESTAMP" >> $VAR_LOCAL_SYNC_DIR/.rsync/rsyncErrorLog
    fi
}


#Verify local sync dir exists
if [ ! -e $VAR_LOCAL_SYNC_DIR/.rsync ]; then
    echo "<initializing local sync folder>"
   
    #make sure root dir exists
    if [ ! -e $VAR_LOCAL_SYNC_DIR ]; then
        mkdir $VAR_LOCAL_SYNC_DIR
    fi

    #create .rsync dir
    `mkdir $VAR_LOCAL_SYNC_DIR/.rsync`

    #add neccisary files.
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDelete
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteTemp
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncExcludeList
    echo 'clear' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLock
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp
    echo 'clear' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLock
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLockTemp
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersion
    touch $VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersionTemp
    touch > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncErrorLog
    echo 'Filename,ModifiedTime,Size,DeleteTime' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncLocalDeleteLog
    echo $VAR_VERSION > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersion
fi

#Verify remote sync dir exists suppress error if rsync fails
echo "none" > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersionTemp
$VAR_RSYNC -q -e "ssh -p $VAR_PORT" $VAR_REMOTE_SYNC_DIR/.rsync/rsyncVersion $VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersionTemp 2> /dev/null
#echo "1" >>$VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersionTemp
VAR_REMOTE_VERSION=`cat $VAR_LOCAL_SYNC_DIR/.rsync/rsyncVersionTemp`

if [ $VAR_REMOTE_VERSION != $VAR_VERSION ]; then # if versions DONT match
    echo "</initializing><initializing remote sync folder $VAR_REMOTE_VERSION->$VAR_VERSION>"
    # copy all files in .rsync
    $VAR_RSYNC -r -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync $VAR_REMOTE_SYNC_DIR/
    # init delete/exclude list
    echo "" > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp
    $VAR_RSYNC -r -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncDelete
    $VAR_RSYNC -r -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncExcludeList
    # init locks to clear
    echo "clear" > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp
    $VAR_RSYNC -r -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncDeleteLock    
    $VAR_RSYNC -r -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncSyncLock
fi

#Send Lock
echo "<setting Remote Delete Lock>\r"
echo 'lock' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp
$VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncDeleteLock

#Check Remote SyncLock Before Sync
echo  "<checking Remote Sync Lock>\r"
$VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_REMOTE_SYNC_DIR/.rsync/rsyncSyncLock $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLockTemp
VAR_LOCK=`cat $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLockTemp`
while [ $VAR_LOCK = 'lock' ];
do
    echo "remote Delete in progress.  please wait..."
    sleep 2;
    $VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_REMOTE_SYNC_DIR/.rsync/rsyncSyncLock $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLockTemp
    VAR_LOCK=`cat $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLockTemp`
done


#Get Delete List
echo  "</checking><checking for remote Deletions>\r"
$VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_REMOTE_SYNC_DIR/.rsync/rsyncDelete $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteTemp


#Delete Files that match name/size/mod date
for i in `cat $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteTemp`
do
    VAR_FILE=`echo $i | awk -F, '{print $1}'`
    VAR_TIMESTAMP=`echo $i | awk -F, '{print $2}'`
    VAR_SIZE=`echo $i | awk -F, '{print $3}'`

    VAR_FULL_FILE="$VAR_LOCAL_SYNC_DIR/$VAR_FILE"
    if [ -e $VAR_FULL_FILE ]; then  

        #Get size for directory or file
        if [ -d $VAR_FULL_FILE ]; then
            VAR_MY_SIZE=`du -s $VAR_FULL_FILE | awk {'print $1'}`
        else
            VAR_MY_SIZE=`ls -laT $VAR_FULL_FILE | awk {'print $5'}`
        fi
        VAR_MY_TIMESTAMP=`stat -f "%m" -t "%s"  $VAR_FULL_FILE`
       
        #Verify Local and romote files are identical.
        if [ $VAR_TIMESTAMP != $VAR_MY_TIMESTAMP ]; then
            proptDelete
        elif [ $VAR_SIZE != $VAR_MY_SIZE ]; then
            proptDelete
        else
            deleteFile
        fi
    else
        echo "$VAR_FULL_FILE < --File does not exist localy"
    fi
done

#Send Clear Delete List
echo  "<Deleted files in Sync>\r"
`echo > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteTemp`
$VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncDelete
$VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncExcludeList

#Send Lock release
echo  "<clearing Remote Delete Lock>\r"
echo 'clear' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp
$VAR_RSYNC -e "ssh -p $VAR_PORT" $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLockTemp $VAR_REMOTE_SYNC_DIR/.rsync/rsyncDeleteLock

#Bi Directional Sync
echo "<receiving>"
$VAR_RSYNC -auv --copy-links --keep-dirlinks -e "ssh -p $VAR_PORT" --exclude '.rsync' --exclude-from $VAR_LOCAL_SYNC_DIR/.rsync/rsyncExcludeList $VAR_REMOTE_SYNC_DIR/ $VAR_LOCAL_SYNC_DIR/

echo "\n\n<sending>"
$VAR_RSYNC -auv --copy-links --keep-dirlinks -e "ssh -p $VAR_PORT" --exclude '.rsync' $VAR_LOCAL_SYNC_DIR/ $VAR_REMOTE_SYNC_DIR/
</sending></receiving></clearing></checking></setting></initializing>

The code is well commented but here is a quick rundown. When the script is ran for the first time it will create the specified directorys if they dont exist and initializes a .rsync folder which holds the system files.

Next it sets the remote “deleteLock file” to prevent any remote files from being deleted while a sync is taking place. In case files were being deleted before the lock was set, a synclock file is checked prior to checking out the deleted file list.

Once cleared the delete file list is downloaded. The files are checked to make sure the modification date and size are identical before deleting the files. This helps ensure only identical files are deleted in case of the match had been modified after a long period without syncing. After the deleted files have been removed locally, the remote list is cleared.

Finally the remote modified files are downloaded using rsync. The –copy-links and –keep-dirlinks options have been enabled as I like to selectively chose which folders to sync while maintaining the original directory structure. My sync folder typically contains soft links (ln -s ) aka shortcuts to folders I would like to have synced. Also the -a options ensures that the permissions, modified date, for example are preserved. The -u option only transfers files that are newer.

Rsync has many options which can be customized to your situation. Be sure to be careful as it can delete files on both sides.

For the delete script all you need to do is again add the local sync directory. It take the files you would like to delete as command line arguments and DOES NOT PROMPT for each deletion, so be sure you have the right files.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#!/bin/sh

#Copyright (2012 hackengineer.com)
#Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
#The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
#THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

#ADD root sync directory Here
VAR_LOCAL_SYNC_DIR="/Users/userName/sync"
#--------------------------------------------------


#make sure sync directory has been inititalized
if [ ! -e $VAR_LOCAL_SYNC_DIR/.rsync ]; then
    echo "Please run 2 way sync to initialize folder prior to deleting files"
    exit
fi

#Set Sync Lock
echo 'lock' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLock

#Verify no remote sync is in progress
VAR_LOCK=`cat $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLock`
while [ $VAR_LOCK = "lock" ];
do
    echo "remote sync in progress.  please wait..."
    sleep 2;
    VAR_LOCK=`cat $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDeleteLock`
done

#Log relative file path, size and mod date prior to deletion
VAR_LOCAL_SYNC_DIR_E=`echo $VAR_LOCAL_SYNC_DIR | perl -pe 's|\/|\\\/|g'`
for i in $@
do
    #Verify file exists
    if [ -e $i ]; then

        #Check for directory
        if [ -d $i ]; then
            VAR_SIZE=`du -s $i | awk {'print $1'}`
        else
            VAR_SIZE=`ls -laT $i | awk {'print $5'}`
        fi
       
        #if in root dir
        if [ `pwd` = $VAR_LOCAL_SYNC_DIR ]; then
            VAR_RELATIVE_DIR=""
        else
            VAR_RELATIVE_DIR=`pwd | perl -pe "s/.*($VAR_LOCAL_SYNC_DIR_E\/?)(.*)/"'\$2'"/"`"/"
           
        fi

        VAR_TIMESTAMP=`stat -f "%m" -t "%s"  $i`
        #Log files for remote deletion
        echo "$VAR_RELATIVE_DIR$i,$VAR_TIMESTAMP,$VAR_SIZE" >> $VAR_LOCAL_SYNC_DIR/.rsync/rsyncDelete
        #Log blocked sync list until remote has deleted local copies
        echo "$VAR_RELATIVE_DIR$i" >> $VAR_LOCAL_SYNC_DIR/.rsync/rsyncExcludeList
        #Remove file
        echo "Removing $VAR_RELATIVE_DIR$i"
        `rm -r $i`
    else
        echo "$i does not exist"
    fi
done

#Clear Sync Lock
echo 'clear' > $VAR_LOCAL_SYNC_DIR/.rsync/rsyncSyncLock

It is worth noting that this script does use “rm -r”! Be sure of what your deleting when using this script

next page —>


Be Sociable, Share!