I’m doing a talk at Linuxfest Northwest on making Very Sexy Backup Scripts. This is because you are more empowered when you know your filesystem and a little bit of bash scripting on your sweet linux system.
I’ll start with an out-of-order post showing rsync and find. If you don’t know [[WTF]] I’m talking about, do comment, but stick with me. This could save you hundreds or thousands of dollars by avoiding purchasing a separate piece of backup software or proprietary solution.
1 #!/bin/bash 2 ## 3 ## backup script, (C) 2013, Jed Reynolds 4 ## 5 source ./bu-rsync.sh 6 export DEST="latitude" 7 EXCLUDES="XX.gvfs PP.cache PPCache PPTrash" 8 bu_steps home/jreynolds/ / $DEST/home-jreynolds JUST_FILES $EXCLUDES 9 bu_steps home/jreynolds/ / $DEST/home-jreynolds $EXCLUDES 10 bu_steps home/liam/ / $DEST/home-liam JUST_FILES $EXCLUDES 11 bu_steps home/liam/ / $DEST/home-liam $EXCLUDES 12 bu_steps home/Music / $MDEST SIZEONLY 13 bu_steps home/Pictures / $PDEST SIZEONLY 14 15 EXCLUDES="" 16 bu_steps home/candela/btbits/x64_btbits/client / $DEST/home-candela/client \ 17 'XX*.class' 18 bu_steps home/candela/btbits/x64_btbits/server / $DEST/home-candela/server \ 19 'XX*.o' 20 bu_steps home/candela/btbits/x64_btbits/tools / $DEST/home-candela/btbits/x64_btbits/tools \ 21 'XX*.o' 22 bu_steps home/candela/btbits/x64_btbits/3plibs / $DEST/home-candela/3plibs \ 23 'XX*.o' 24 bu_steps home/candela/btbits/x64_btbits/html / $DEST/home-candela/html 25 bu_steps etc / $DEST/etc 26 bu_steps usr/local / $DEST/usr-local \ 27 SIZE_ONLY 28 ## 29 ## Free for non-commercial use. No warrany or support offered. 30 ##
That was the config file. It wont do anything without the library of functions, below:
1 #!/bin/bash
2 ##
3 ## backup script library, (C) 2013, Jed Reynolds
4 ## Free for non-commercial use. No warrany or support offered.
5 ##
6 BU_HOST=beavertail
7 SSH_OP="-i /home/jreynolds/.ssh/beavertail_dsa"
8 LSYNC="rsync "
9 export REALM=tank
10 RHOST="backup@$BU_HOST"
11 RSYNC="rsync --progress -rlpt --copy-unsafe-links "
12 Y=`date +%Y`
13 DEST="latitude"
14 PDEST="pictures"
15 MDEST="music"
16 SDEST="softlib"
17 VDEST="VMs"
18 RSYNC_PASSWORD="mkcU6N84wJHT5u"
19 RSYNC_PASSWD="mkcU6N84wJHT5u"
20 FLIST="/tmp/bu-list"
21 MK_SNAP="000-mksnap-000"
22 export RSYNC_PASSWORD RSYNC_PASSWD
23
24 function fail() {
25 local msg=${1:-"unknown cause"}
26 echo " -- $msg --"
27 exit 1
28 }
29 function ping_gw() {
30 local default_gw=$(ip r | grep default | cut -d ' ' -f 3)
31 ping_host $default_gw
32 return $?
33 }
34 function ping_host() {
35 [ -z "$1" ] && echo "ping_host cannot ping no host, bye" && exit 1
36 ping -q -w 1 -c 1 $1 > /dev/null && return 0 || return 1
37 }
38 function bu_steps() {
39 ping_gw || fail "no gateway"
40 ping_host $BU_HOST || fail "no ping to $BU_HOST"
41 local f_others=""
42 local r_others=""
43 local dir_a=$1; shift
44 local sit_on=$1; shift
45 local dir_b=$1; shift
46 [ "${dir_b:0:1}" != "/" ] && dir_b="$dir_b"
47 local xcld=""
48 local prun=""
49 local minusv=""
50 local flist="/tmp/flist.txt"
51 local RECT=".recent"
52 while(( "$#" )); do
53 [ "$1" == "SIZEONLY" ] && r_others="$r_others --size-only "
54 [ "$1" == "JUST_FILES" ] && f_others="-maxdepth 1 -type f $f_others "
55 [ "$1" == "JUST_FILES" ] && RECT=".recent_files"
56 if [[ $1 == XX* ]] ; then
57 [ ! -z "$xlcd" ] && xcld="$xcld -o"
58 [ ! -z "$minusv" ] && minusv="${minusv}|"
59 xcld="$xcld ${1/XX/-name }"
60 minuxv="${minusv}${1/XX/}"
61 fi
62 if [[ $1 == PP* ]] ; then
63 [ ! -z "$prun" ] && prun="$prun -o"
64 [ ! -z "$minusv" ] && minusv="${minusv}|"
65 prun="$prun ${1/PP/-name } -prune"
66 minuxv="${minusv}${1/XX/}"
67 fi
68 shift;
69 done
70 [ ! -z "$prun" ] && prun="-a ( ! $prun )"
71 [ ! -z "$xcld" ] && xcld="-a ( ! $xcld )"
72 [ ! -z "$minusv" ] && minusv="| grep -v ($minusv)"
73
74 local mk_remot=0
75 $LSYNC ${RHOST}::${REALM}/${dir_b} || mk_remot=1
76 if [ $mk_remot -eq 1 ] ; then
77 ssh jreynolds@$BU_HOST "sudo mkdir -p /$REALM/$dir_b && sudo chmod 777 /$REALM/$dir_b " || fail "Could not make dir in /$REALM/$dir_b"
78 fi
79 cd $sit_on
80
81 # Find the .recent file, or create a recent file of Jan 1, 1990
82 if [ ! -f $dir_a/$RECT ] ; then
83 echo "creating ${sit_on}${dir_a}/$RECT that will pull Everything!"
84 touch -d "01 Jan 1990" ${sit_on}${dir_a}/$RECT || fail "permission denied in ${sit_on}${dir_a}, bye"
85 fi
86 cat /dev/null > $flist
87 if [ ! -d ${sit_on}${dir_a} ] ; then
88 echo "** ${sit_on}${dir_a} not found, skipping **"
89 return 0
90 else
91 FND_CMD="find $dir_a $f_others -type f -a -newer ${sit_on}${dir_a}/$RECT $xcld $prun"
92 echo "$FND_CMD $minusv > $flist"
93 $FND_CMD $minusv > $flist || fail "unable to complete find command"
94 fi
95 local fil_ct=`wc -l $flist | cut -d' ' -f1`
96 if [ $fil_ct -lt 1 ] ; then
97 echo "Skipping $dir_b, no files found"
98 else
99 echo "== $RSYNC $r_others --files-from=$flist $sit_on $RHOST::$REALM/$dir_b/ =="
100 $RSYNC $r_others -v --files-from=$flist $sit_on $RHOST::$REALM/$dir_b/ || fail "Rsync failed, bad paths?"
101 touch_mksnap $dir_b
102 touch $dir_a/$RECT
103 fi
104 }
105 function check_nas_ready() {
106 ssh $SSH_OP jreynolds@$BU_HOST "~/bin/mount_up"
107 }
108 function touch_mksnap() {
109 ssh $SSH_OP jreynolds@$BU_HOST "sudo touch /$REALM/$1/$MK_SNAP"
110 }
111
112 [ `id -u` == "0" ] || fail "Do not run but as root, bye."
113 ping_gw || fail "no gateway"
114 ping_host $BU_HOST || fail "no ping to $BU_HOST"
115 check_nas_ready || fail "NAS not prepared, cannot continue."
116
117 [ -f $DEST/NOT-MOUNTED ] && mount $DEST
118 [ -f $PDEST/NOT-MOUNTED ] && mount $PDEST
119 [ -f $MDEST/NOT-MOUNTED ] && mount $MDEST
120
121 [ -f $DEST/NOT-MOUNTED ] && fail "Remote system not mounted to $DEST, bye."
122 [ -f $MDEST/NOT-MOUNTED ] && fail "Remote system not mounted to $MDEST, bye."
123 [ -f $PDEST/NOT-MOUNTED ] && fail "Remote system not mounted to $PDEST, bye."
124 #
Filed under: computing Tagged: backups, Linux
