You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
245 lines
7.2 KiB
245 lines
7.2 KiB
#! /bin/bash
|
|
|
|
if [[ $# < 3 ]]; then
|
|
echo 'Usage: sync local_dir rsync_host rsync_dir [trace_host [trace_dir]]'
|
|
exit 1
|
|
fi
|
|
|
|
set -e
|
|
|
|
TOP_DIR=/mirror
|
|
TO=$TOP_DIR/root/$1
|
|
RSYNC_HOST=$2
|
|
RSYNC_DIR=$3
|
|
if [ $# = 4 ]; then
|
|
TRACE_HOST=$4
|
|
fi
|
|
TRACE_DIR=project/trace
|
|
if [ $# = 5 ]; then
|
|
TRACE_DIR=$5
|
|
fi
|
|
LOGDIR=/var/log/mirror/$1_$2
|
|
ADDRESS=$(cat ~/config/ADDRESS)
|
|
|
|
mkdir -p $LOGDIR
|
|
|
|
# This script originates from http://www.debian.org/mirror/anonftpsync
|
|
|
|
# CVS: cvs.debian.org:/cvs/webwml - webwml/english/mirror/anonftpsync
|
|
# Version: $Id: anonftpsync,v 1.33 2007/09/12 15:19:03 joy Exp $
|
|
|
|
# Note: You MUST have rsync 2.6.4 or newer, which is available in sarge
|
|
# and all newer Debian releases, or at http://rsync.samba.org/
|
|
|
|
# Don't forget:
|
|
# chmod u+x anonftpsync
|
|
|
|
# Set the variables below to fit your site. You can then use cron to have
|
|
# this script run daily to automatically update your copy of the archive.
|
|
|
|
# TO is the destination for the base of the Debian mirror directory
|
|
# (the dir that holds dists/ and ls-lR).
|
|
# (mandatory)
|
|
|
|
#TO=
|
|
|
|
# RSYNC_HOST is the site you have chosen from the mirrors file.
|
|
# (http://www.debian.org/mirror/list-full)
|
|
# (mandatory)
|
|
|
|
#RSYNC_HOST=
|
|
|
|
# RSYNC_DIR is the directory given in the "Packages over rsync:" line of
|
|
# the mirrors file for the site you have chosen to mirror.
|
|
# (mandatory)
|
|
|
|
#RSYNC_DIR=
|
|
|
|
# LOGDIR is the directory where the logs will be written to
|
|
# (mandatory)
|
|
|
|
#LOGDIR=
|
|
|
|
# ARCH_EXCLUDE can be used to exclude a complete architecture from
|
|
# mirrorring. Please use as space seperated list.
|
|
# Possible values are:
|
|
# alpha, amd64, arm, hppa, hurd-i386, i386, ia64, m68k, mipsel, mips, powerpc, s390, sh and sparc
|
|
#
|
|
# There is one special value: source
|
|
# This is not an architecture but will exclude all source code in /pool
|
|
#
|
|
# eg.
|
|
# ARCH_EXCLUDE="alpha arm hppa hurd-i386 ia64 m68k mipsel mips s390 sparc"
|
|
#
|
|
# With a blank ARCH_EXCLUDE you will mirror all available architectures
|
|
# (optional)
|
|
|
|
# EXCLUDE is a list of parameters listing patterns that rsync will exclude, in
|
|
# addition to the architectures excluded by ARCH_EXCLUDE.
|
|
#
|
|
# Use ARCH_EXCLUDE to exclude specific architectures or all sources
|
|
#
|
|
# --exclude stable, testing, unstable options DON'T remove the packages of
|
|
# the given distribution. If you want do so, use debmirror instead.
|
|
#
|
|
# The following example would exclude mostly everything:
|
|
#EXCLUDE="\
|
|
# --exclude stable/ --exclude testing/ --exclude unstable/ \
|
|
# --exclude source/ \
|
|
# --exclude *.orig.tar.gz --exclude *.diff.gz --exclude *.dsc \
|
|
# --exclude /contrib/ --exclude /non-free/ \
|
|
# "
|
|
|
|
# With a blank EXCLUDE you will mirror the entire archive, except the
|
|
# architectures excluded by ARCH_EXCLUDE.
|
|
# (optional)
|
|
|
|
EXCLUDE=
|
|
|
|
# LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
|
|
# This program creates a lock to ensure that only one copy
|
|
# of it is mirroring any one archive at any one time.
|
|
# Locks held for longer than the timeout are broken, unless
|
|
# a running rsync process appears to be connected to $RSYNC_HOST.
|
|
|
|
LOCK_TIMEOUT=360
|
|
|
|
# There should be no need to edit anything below this point, unless there
|
|
# are problems.
|
|
|
|
#-----------------------------------------------------------------------------#
|
|
|
|
# If you are accessing a rsync server/module which is password-protected,
|
|
# uncomment the following lines (and edit the other file).
|
|
|
|
# . ftpsync.conf
|
|
# export RSYNC_PASSWORD
|
|
# RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
|
|
|
#-----------------------------------------------------------------------------#
|
|
|
|
# Check for some environment variables
|
|
if [ -z $TO ] || [ -z $RSYNC_HOST ] || [ -z $RSYNC_DIR ] || [ -z $LOGDIR ]; then
|
|
echo "One of the following variables seems to be empty:"
|
|
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
|
|
exit 2
|
|
fi
|
|
|
|
if ! [ -d ${TO}/${TRACE_DIR} ]; then
|
|
# we are running mirror script for the first time
|
|
umask 002
|
|
mkdir -p ${TO}/${TRACE_DIR}
|
|
fi
|
|
|
|
# Note: on some non-Debian systems, hostname doesn't accept -f option.
|
|
# If that's the case on your system, make sure hostname prints the full
|
|
# hostname, and remove the -f option. If there's no hostname command,
|
|
# explicitly replace `hostname -f` with the hostname.
|
|
|
|
HOSTNAME=`hostname -f`
|
|
|
|
# The hostname must match the "Site" field written in the list of mirrors.
|
|
# If hostname doesn't returns the correct value, fill and uncomment below
|
|
HOSTNAME=mirror.csclub.uwaterloo.ca
|
|
|
|
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"
|
|
|
|
# The temp directory used by rsync --delay-updates is not
|
|
# world-readable remotely. It must be excluded to avoid errors.
|
|
TMP_EXCLUDE="--exclude .~tmp~/"
|
|
|
|
# Exclude architectures defined in $ARCH_EXCLUDE
|
|
for ARCH in $ARCH_EXCLUDE; do
|
|
EXCLUDE=$EXCLUDE"\
|
|
--exclude binary-$ARCH/ \
|
|
--exclude disks-$ARCH/ \
|
|
--exclude installer-$ARCH/ \
|
|
--exclude Contents-$ARCH.gz \
|
|
--exclude Contents-$ARCH.diff/ \
|
|
--exclude arch-$ARCH.files \
|
|
--exclude arch-$ARCH.list.gz \
|
|
--exclude *_$ARCH.deb \
|
|
--exclude *_$ARCH.udeb "
|
|
if [ "$ARCH" == "source" ]; then
|
|
SOURCE_EXCLUDE="\
|
|
--exclude source/ \
|
|
--exclude *.tar.gz \
|
|
--exclude *.diff.gz \
|
|
--exclude *.dsc "
|
|
fi
|
|
done
|
|
|
|
# Logfile
|
|
LOGFILE=$LOGDIR/mirror.log
|
|
|
|
# Get in the right directory and set the umask to be group writable
|
|
#
|
|
cd $HOME
|
|
umask 002
|
|
|
|
# Check to see if another sync is in progress
|
|
if [ -f "$LOCK" ]; then
|
|
# Note: this requires the findutils find; for other finds, adjust as necessary
|
|
if [ "`find $LOCK -maxdepth 1 -amin -$LOCK_TIMEOUT`" = "" ]; then
|
|
# Note: this requires the procps ps; for other ps', adjust as necessary
|
|
if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
|
|
echo "stale lock found, but a rsync is still running, aiee!" >&2
|
|
exit 1
|
|
else
|
|
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
|
|
rm -f $LOCK
|
|
fi
|
|
else
|
|
echo "current lock file exists, unable to start rsync!"
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
touch $LOCK
|
|
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
|
|
# specification. If that's the case on your system, try using "0".
|
|
trap "rm -f $LOCK" exit
|
|
|
|
set +e
|
|
|
|
# check if we need to sync
|
|
if [[ "$TRACE_HOST" != "" ]]; then
|
|
TRACE_OLD_TIME=`stat -c%Y $TO/$TRACE_DIR/$TRACE_HOST 2> /dev/null`
|
|
TRACE_NEW_FILE=/tmp/$RSYNC_HOST_$RSYNC_DIR_$RANDOM
|
|
nice rsync -tv -4 --address=$ADDRESS \
|
|
$RSYNC_HOST::$RSYNC_DIR/$TRACE_DIR/$TRACE_HOST \
|
|
$TRACE_NEW_FILE >> $LOGFILE 2>&1
|
|
TRACE_NEW_TIME=`stat -c%Y $TRACE_NEW_FILE`
|
|
rm -f $TRACE_NEW_FILE
|
|
if [ "$TRACE_OLD_TIME" = "$TRACE_NEW_TIME" ]; then
|
|
echo 'Trace file for' $RSYNC_HOST::$RSYNC_DIR \
|
|
'unchanged!' >> $LOGFILE
|
|
fi
|
|
fi
|
|
|
|
# First sync /pool
|
|
nice rsync -rlHtvp \
|
|
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
|
|
--timeout=3600 -4 --address=$ADDRESS \
|
|
$RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
|
|
result=$?
|
|
|
|
if [ 0 = $result ]; then
|
|
# Now sync the remaining stuff
|
|
nice rsync -rlHtvp --delay-updates --delete-after \
|
|
--exclude "Archive-Update-in-Progress-${HOSTNAME}" \
|
|
--exclude "${TRACE_DIR}/${HOSTNAME}" \
|
|
--timeout=3600 -4 --address=$ADDRESS \
|
|
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
|
|
$RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
|
|
|
mkdir -p ${TO}/${TRACE_DIR}
|
|
LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"
|
|
else
|
|
echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
|
|
echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
|
|
fi
|
|
|
|
savelog $LOGFILE >/dev/null
|
|
|
|
rm $LOCK
|
|
|