mirror/bin/csc-sync-debian

#! /bin/bash

if [[ $# < 3 ]]; then
  echo 'Usage: sync local_dir rsync_host rsync_dir [trace_host [trace_dir]]'
  exit 1
fi

set -e

TOP_DIR=/mirror
TO=$TOP_DIR/root/$1
RSYNC_HOST=$2
RSYNC_DIR=$3
if [ $# = 4 ]; then
    TRACE_HOST=$4
fi
TRACE_DIR=project/trace
if [ $# = 5 ]; then
    TRACE_DIR=$5
fi
LOGDIR=/var/log/mirror/$1_$2
ADDRESS=$(cat ~/config/ADDRESS)

mkdir -p $LOGDIR

# This script originates from http://www.debian.org/mirror/anonftpsync

# CVS: cvs.debian.org:/cvs/webwml - webwml/english/mirror/anonftpsync
# Version: $Id: anonftpsync,v 1.33 2007/09/12 15:19:03 joy Exp $ 

# Note: You MUST have rsync 2.6.4 or newer, which is available in sarge
# and all newer Debian releases, or at http://rsync.samba.org/

# Don't forget:
# chmod u+x anonftpsync

# Set the variables below to fit your site. You can then use cron to have
# this script run daily to automatically update your copy of the archive.

# TO is the destination for the base of the Debian mirror directory
# (the dir that holds dists/ and ls-lR).
# (mandatory)

#TO=

# RSYNC_HOST is the site you have chosen from the mirrors file.
# (http://www.debian.org/mirror/list-full)
# (mandatory)

#RSYNC_HOST=

# RSYNC_DIR is the directory given in the "Packages over rsync:" line of
# the mirrors file for the site you have chosen to mirror.
# (mandatory)

#RSYNC_DIR=

# LOGDIR is the directory where the logs will be written to
# (mandatory)

#LOGDIR=

# ARCH_EXCLUDE can be used to exclude a complete architecture from
# mirrorring. Please use as space seperated list.
# Possible values are:
# alpha, amd64, arm, hppa, hurd-i386, i386, ia64, m68k, mipsel, mips, powerpc, s390, sh and sparc
#
# There is one special value: source
# This is not an architecture but will exclude all source code in /pool
#
# eg.
# ARCH_EXCLUDE="alpha arm hppa hurd-i386 ia64 m68k mipsel mips s390 sparc"
# 
# With a blank ARCH_EXCLUDE you will mirror all available architectures
# (optional)

# EXCLUDE is a list of parameters listing patterns that rsync will exclude, in
# addition to the architectures excluded by ARCH_EXCLUDE.
#
# Use ARCH_EXCLUDE to exclude specific architectures or all sources
#
# --exclude stable, testing, unstable options DON'T remove the packages of
# the given distribution. If you want do so, use debmirror instead.
#
# The following example would exclude mostly everything:
#EXCLUDE="\
#  --exclude stable/ --exclude testing/ --exclude unstable/ \
#  --exclude source/ \
#  --exclude *.orig.tar.gz --exclude *.diff.gz --exclude *.dsc \
#  --exclude /contrib/ --exclude /non-free/ \
# "

# With a blank EXCLUDE you will mirror the entire archive, except the
# architectures excluded by ARCH_EXCLUDE.
# (optional)

EXCLUDE=

# LOCK_TIMEOUT is a timeout in minutes.  Defaults to 360 (6 hours).
# This program creates a lock to ensure that only one copy
# of it is mirroring any one archive at any one time.
# Locks held for longer than the timeout are broken, unless
# a running rsync process appears to be connected to $RSYNC_HOST.

LOCK_TIMEOUT=360

# There should be no need to edit anything below this point, unless there
# are problems.

#-----------------------------------------------------------------------------#

# If you are accessing a rsync server/module which is password-protected,
# uncomment the following lines (and edit the other file).

# . ftpsync.conf
# export RSYNC_PASSWORD
# RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST

#-----------------------------------------------------------------------------#

# Check for some environment variables
if [ -z $TO ] || [ -z $RSYNC_HOST ] || [ -z $RSYNC_DIR ] || [ -z $LOGDIR ]; then
	echo "One of the following variables seems to be empty:"
	echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
	exit 2
fi

if ! [ -d ${TO}/${TRACE_DIR} ]; then
	# we are running mirror script for the first time
	umask 002
	mkdir -p ${TO}/${TRACE_DIR}
fi

# Note: on some non-Debian systems, hostname doesn't accept -f option.
# If that's the case on your system, make sure hostname prints the full
# hostname, and remove the -f option. If there's no hostname command,
# explicitly replace `hostname -f` with the hostname.

HOSTNAME=`hostname -f`

# The hostname must match the "Site" field written in the list of mirrors.
# If hostname doesn't returns the correct value, fill and uncomment below 
HOSTNAME=mirror.csclub.uwaterloo.ca
 
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"

# The temp directory used by rsync --delay-updates is not
# world-readable remotely. It must be excluded to avoid errors. 
TMP_EXCLUDE="--exclude .~tmp~/"

# Exclude architectures defined in $ARCH_EXCLUDE
for ARCH in $ARCH_EXCLUDE; do
	EXCLUDE=$EXCLUDE"\
		--exclude binary-$ARCH/ \
		--exclude disks-$ARCH/ \
		--exclude installer-$ARCH/ \
		--exclude Contents-$ARCH.gz \
		--exclude Contents-$ARCH.diff/ \
		--exclude arch-$ARCH.files \
		--exclude arch-$ARCH.list.gz \
		--exclude *_$ARCH.deb \
		--exclude *_$ARCH.udeb "
	if [ "$ARCH" == "source" ]; then
		SOURCE_EXCLUDE="\
		--exclude source/ \
		--exclude *.tar.gz \
		--exclude *.diff.gz \
		--exclude *.dsc "
	fi
done

# Logfile
LOGFILE=$LOGDIR/mirror.log

# Get in the right directory and set the umask to be group writable
# 
cd $HOME
umask 002

# Check to see if another sync is in progress
if [ -f "$LOCK" ]; then
# Note: this requires the findutils find; for other finds, adjust as necessary
  if [ "`find $LOCK -maxdepth 1 -amin -$LOCK_TIMEOUT`" = "" ]; then
# Note: this requires the procps ps; for other ps', adjust as necessary
    if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
      echo "stale lock found, but a rsync is still running, aiee!" >&2
      exit 1
    else
      echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
      rm -f $LOCK
    fi
  else
    echo "current lock file exists, unable to start rsync!"
    exit 1
  fi
fi

touch $LOCK
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
# specification. If that's the case on your system, try using "0".
trap "rm -f $LOCK" exit

set +e

# check if we need to sync
if [[ "$TRACE_HOST" != "" ]]; then
    TRACE_OLD_TIME=`stat -c%Y $TO/$TRACE_DIR/$TRACE_HOST 2> /dev/null`
    TRACE_NEW_FILE=/tmp/$RSYNC_HOST_$RSYNC_DIR_$RANDOM
    nice rsync -tv -4 --address=$ADDRESS \
        $RSYNC_HOST::$RSYNC_DIR/$TRACE_DIR/$TRACE_HOST \
        $TRACE_NEW_FILE >> $LOGFILE 2>&1
    TRACE_NEW_TIME=`stat -c%Y $TRACE_NEW_FILE`
    rm -f $TRACE_NEW_FILE
    if [ "$TRACE_OLD_TIME" = "$TRACE_NEW_TIME" ]; then
        echo 'Trace file for' $RSYNC_HOST::$RSYNC_DIR \
            'unchanged!' >> $LOGFILE
    fi
fi

# First sync /pool
nice rsync -rlHtvp \
     $TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
     --timeout=3600 -4 --address=$ADDRESS \
     $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
result=$?

if [ 0 = $result ]; then
	# Now sync the remaining stuff
	nice rsync -rlHtvp --delay-updates --delete-after \
	     --exclude "Archive-Update-in-Progress-${HOSTNAME}" \
	     --exclude "${TRACE_DIR}/${HOSTNAME}" \
	     --timeout=3600 -4 --address=$ADDRESS \
	     $TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
	     $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1

	mkdir -p ${TO}/${TRACE_DIR}
	LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"
else
	echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
	echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
fi

savelog $LOGFILE >/dev/null

rm $LOCK