Initial checkin
This commit is contained in:
commit
3d1d2c392b
|
@ -0,0 +1,14 @@
|
|||
# m h dom mon dow command
|
||||
0 * * * * ~/bin/csc-sync-debian debian gulus.usherbrooke.ca debian ftp-master.debian.org
|
||||
0 * * * * ~/bin/csc-sync-debian debian-multimedia mirror.home-dn.net debian-multimedia roxane.home-dn.net
|
||||
0 * * * * ~/bin/csc-sync-debian debian-backports www.backports.org backports.org www.backports.org
|
||||
0 * * * * ~/bin/csc-sync-debian debian-volatile volatile.debian.org debian-volatile volatile-master.debian.net
|
||||
0 * * * * ~/bin/csc-sync-debian debian-security security.debian.org debian-security security-master.debian.org
|
||||
0 * * * * ~/bin/csc-sync-debian debian-unofficial ftp.debian-unofficial.org debian ftp-master.debian-unofficial.org
|
||||
0 0 * * * ~/bin/csc-sync-debian emdebian www.emdebian.org emdebian
|
||||
0 * * * * ~/bin/csc-sync-debian ubuntu archive.ubuntu.com ubuntu drescher.canonical.com
|
||||
0 */12 * * * ~/bin/csc-sync-standard gobuntu-releases cdimage.ubuntu.com cdimage/gobuntu/releases/
|
||||
0 */12 * * * ~/bin/csc-sync-standard ubuntu-releases releases.ubuntu.com releases
|
||||
0 */12 * * * ~/bin/csc-sync-standard xubuntu-releases cdimage.ubuntu.com cdimage/xubuntu/releases/
|
||||
0 */12 * * * ~/bin/csc-sync-standard CPAN rsync.nic.funet.fi CPAN
|
||||
0 */12 * * * ~/bin/csc-sync-standard openoffice ftp.ussg.iu.edu openoffice
|
|
@ -0,0 +1,247 @@
|
|||
#! /bin/sh
|
||||
|
||||
if [[ $# < 3 ]]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir [trace_host [trace_dir]]'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
TOP_DIR=/mirror
|
||||
TO=$TOP_DIR/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
if [ $# = 4 ]; then
|
||||
TRACE_HOST=$4
|
||||
fi
|
||||
TRACE_DIR=project/trace
|
||||
if [ $# = 5 ]; then
|
||||
TRACE_DIR=$5
|
||||
fi
|
||||
LOGDIR=/home/mirror/logs/$1_$2
|
||||
ADDRESS=129.97.134.71
|
||||
|
||||
mkdir -p $LOGDIR
|
||||
|
||||
# This script originates from http://www.debian.org/mirror/anonftpsync
|
||||
|
||||
# CVS: cvs.debian.org:/cvs/webwml - webwml/english/mirror/anonftpsync
|
||||
# Version: $Id: anonftpsync,v 1.33 2007/09/12 15:19:03 joy Exp $
|
||||
|
||||
# Note: You MUST have rsync 2.6.4 or newer, which is available in sarge
|
||||
# and all newer Debian releases, or at http://rsync.samba.org/
|
||||
|
||||
# Don't forget:
|
||||
# chmod u+x anonftpsync
|
||||
|
||||
# Set the variables below to fit your site. You can then use cron to have
|
||||
# this script run daily to automatically update your copy of the archive.
|
||||
|
||||
# TO is the destination for the base of the Debian mirror directory
|
||||
# (the dir that holds dists/ and ls-lR).
|
||||
# (mandatory)
|
||||
|
||||
#TO=
|
||||
|
||||
# RSYNC_HOST is the site you have chosen from the mirrors file.
|
||||
# (http://www.debian.org/mirror/list-full)
|
||||
# (mandatory)
|
||||
|
||||
#RSYNC_HOST=
|
||||
|
||||
# RSYNC_DIR is the directory given in the "Packages over rsync:" line of
|
||||
# the mirrors file for the site you have chosen to mirror.
|
||||
# (mandatory)
|
||||
|
||||
#RSYNC_DIR=
|
||||
|
||||
# LOGDIR is the directory where the logs will be written to
|
||||
# (mandatory)
|
||||
|
||||
#LOGDIR=
|
||||
|
||||
# ARCH_EXCLUDE can be used to exclude a complete architecture from
|
||||
# mirrorring. Please use as space seperated list.
|
||||
# Possible values are:
|
||||
# alpha, amd64, arm, hppa, hurd-i386, i386, ia64, m68k, mipsel, mips, powerpc, s390, sh and sparc
|
||||
#
|
||||
# There is one special value: source
|
||||
# This is not an architecture but will exclude all source code in /pool
|
||||
#
|
||||
# eg.
|
||||
# ARCH_EXCLUDE="alpha arm hppa hurd-i386 ia64 m68k mipsel mips s390 sparc"
|
||||
#
|
||||
# With a blank ARCH_EXCLUDE you will mirror all available architectures
|
||||
# (optional)
|
||||
|
||||
# EXCLUDE is a list of parameters listing patterns that rsync will exclude, in
|
||||
# addition to the architectures excluded by ARCH_EXCLUDE.
|
||||
#
|
||||
# Use ARCH_EXCLUDE to exclude specific architectures or all sources
|
||||
#
|
||||
# --exclude stable, testing, unstable options DON'T remove the packages of
|
||||
# the given distribution. If you want do so, use debmirror instead.
|
||||
#
|
||||
# The following example would exclude mostly everything:
|
||||
#EXCLUDE="\
|
||||
# --exclude stable/ --exclude testing/ --exclude unstable/ \
|
||||
# --exclude source/ \
|
||||
# --exclude *.orig.tar.gz --exclude *.diff.gz --exclude *.dsc \
|
||||
# --exclude /contrib/ --exclude /non-free/ \
|
||||
# "
|
||||
|
||||
# With a blank EXCLUDE you will mirror the entire archive, except the
|
||||
# architectures excluded by ARCH_EXCLUDE.
|
||||
# (optional)
|
||||
|
||||
EXCLUDE=
|
||||
|
||||
# LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
|
||||
# This program creates a lock to ensure that only one copy
|
||||
# of it is mirroring any one archive at any one time.
|
||||
# Locks held for longer than the timeout are broken, unless
|
||||
# a running rsync process appears to be connected to $RSYNC_HOST.
|
||||
|
||||
LOCK_TIMEOUT=360
|
||||
|
||||
# There should be no need to edit anything below this point, unless there
|
||||
# are problems.
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# If you are accessing a rsync server/module which is password-protected,
|
||||
# uncomment the following lines (and edit the other file).
|
||||
|
||||
# . ftpsync.conf
|
||||
# export RSYNC_PASSWORD
|
||||
# RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# Check for some environment variables
|
||||
if [ -z $TO ] || [ -z $RSYNC_HOST ] || [ -z $RSYNC_DIR ] || [ -z $LOGDIR ]; then
|
||||
echo "One of the following variables seems to be empty:"
|
||||
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! [ -d ${TO}/${TRACE_DIR} ]; then
|
||||
# we are running mirror script for the first time
|
||||
umask 002
|
||||
mkdir -p ${TO}/${TRACE_DIR}
|
||||
fi
|
||||
|
||||
# Note: on some non-Debian systems, hostname doesn't accept -f option.
|
||||
# If that's the case on your system, make sure hostname prints the full
|
||||
# hostname, and remove the -f option. If there's no hostname command,
|
||||
# explicitly replace `hostname -f` with the hostname.
|
||||
|
||||
HOSTNAME=`hostname -f`
|
||||
|
||||
# The hostname must match the "Site" field written in the list of mirrors.
|
||||
# If hostname doesn't returns the correct value, fill and uncomment below
|
||||
HOSTNAME=mirror.csclub.uwaterloo.ca
|
||||
|
||||
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"
|
||||
|
||||
# The temp directory used by rsync --delay-updates is not
|
||||
# world-readable remotely. It must be excluded to avoid errors.
|
||||
TMP_EXCLUDE="--exclude .~tmp~/"
|
||||
|
||||
# Exclude architectures defined in $ARCH_EXCLUDE
|
||||
for ARCH in $ARCH_EXCLUDE; do
|
||||
EXCLUDE=$EXCLUDE"\
|
||||
--exclude binary-$ARCH/ \
|
||||
--exclude disks-$ARCH/ \
|
||||
--exclude installer-$ARCH/ \
|
||||
--exclude Contents-$ARCH.gz \
|
||||
--exclude Contents-$ARCH.diff/ \
|
||||
--exclude arch-$ARCH.files \
|
||||
--exclude arch-$ARCH.list.gz \
|
||||
--exclude *_$ARCH.deb \
|
||||
--exclude *_$ARCH.udeb "
|
||||
if [ "$ARCH" == "source" ]; then
|
||||
SOURCE_EXCLUDE="\
|
||||
--exclude source/ \
|
||||
--exclude *.tar.gz \
|
||||
--exclude *.diff.gz \
|
||||
--exclude *.dsc "
|
||||
fi
|
||||
done
|
||||
|
||||
# Logfile
|
||||
LOGFILE=$LOGDIR/debian-mirror.log
|
||||
|
||||
# Get in the right directory and set the umask to be group writable
|
||||
#
|
||||
cd $HOME
|
||||
umask 002
|
||||
|
||||
# Check to see if another sync is in progress
|
||||
if [ -f "$LOCK" ]; then
|
||||
# Note: this requires the findutils find; for other finds, adjust as necessary
|
||||
if [ "`find $LOCK -maxdepth 1 -amin -$LOCK_TIMEOUT`" = "" ]; then
|
||||
# Note: this requires the procps ps; for other ps', adjust as necessary
|
||||
if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
|
||||
echo "stale lock found, but a rsync is still running, aiee!"
|
||||
exit 1
|
||||
else
|
||||
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
|
||||
rm -f $LOCK
|
||||
fi
|
||||
else
|
||||
echo "current lock file exists, unable to start rsync!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
touch $LOCK
|
||||
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
|
||||
# specification. If that's the case on your system, try using "0".
|
||||
trap "rm -f $LOCK" exit
|
||||
|
||||
set +e
|
||||
|
||||
# check if we need to sync
|
||||
if [[ "$TRACE_HOST" != "" ]]; then
|
||||
TRACE_FILE=$TO/${TRACE_DIR}/${TRACE_HOST}
|
||||
TRACE_PRE_DATE=`ls -al $TRACE_FILE`
|
||||
rsync --recursive --times --verbose \
|
||||
--address=$ADDRESS \
|
||||
$RSYNC_HOST::$RSYNC_DIR/${TRACE_DIR}/ \
|
||||
$TO/${TRACE_DIR}/ >> $LOGFILE 2>&1
|
||||
TRACE_POST_DATE=`ls -al $TRACE_FILE`
|
||||
if [ "$TRACE_PRE_DATE" = "$TRACE_POST_DATE" ]; then
|
||||
LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"
|
||||
echo 'Trace file for' $RSYNC_HOST::$RSYNC_DIR \
|
||||
'unchanged, not rsyncing.' >> $LOGFILE
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# First sync /pool
|
||||
rsync --recursive --links --hard-links --times --verbose \
|
||||
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
|
||||
--address=$ADDRESS \
|
||||
$RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
|
||||
result=$?
|
||||
|
||||
if [ 0 = $result ]; then
|
||||
# Now sync the remaining stuff
|
||||
rsync --recursive --links --hard-links --times --verbose --delay-updates --delete-after \
|
||||
--exclude "Archive-Update-in-Progress-${HOSTNAME}" \
|
||||
--exclude "${TRACE_DIR}/${HOSTNAME}" \
|
||||
--address=$ADDRESS \
|
||||
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||||
|
||||
mkdir -p ${TO}/${TRACE_DIR}
|
||||
LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"
|
||||
else
|
||||
echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
|
||||
echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
|
||||
fi
|
||||
|
||||
savelog $LOGFILE >/dev/null
|
||||
|
||||
rm $LOCK
|
|
@ -0,0 +1,106 @@
|
|||
#! /bin/sh
|
||||
|
||||
if [[ $# != 3 ]]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
TOP_DIR=/mirror
|
||||
TO=$TOP_DIR/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
LOGDIR=/home/mirror/logs/$1_$2
|
||||
ADDRESS=129.97.134.71
|
||||
|
||||
mkdir -p $LOGDIR
|
||||
|
||||
# LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
|
||||
# This program creates a lock to ensure that only one copy
|
||||
# of it is mirroring any one archive at any one time.
|
||||
# Locks held for longer than the timeout are broken, unless
|
||||
# a running rsync process appears to be connected to $RSYNC_HOST.
|
||||
|
||||
LOCK_TIMEOUT=360
|
||||
|
||||
# There should be no need to edit anything below this point, unless there
|
||||
# are problems.
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# If you are accessing a rsync server/module which is password-protected,
|
||||
# uncomment the following lines (and edit the other file).
|
||||
|
||||
# . ftpsync.conf
|
||||
# export RSYNC_PASSWORD
|
||||
# RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# Check for some environment variables
|
||||
if [ -z $TO ] || [ -z $RSYNC_HOST ] || [ -z $RSYNC_DIR ] || [ -z $LOGDIR ]; then
|
||||
echo "One of the following variables seems to be empty:"
|
||||
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Note: on some non-Debian systems, hostname doesn't accept -f option.
|
||||
# If that's the case on your system, make sure hostname prints the full
|
||||
# hostname, and remove the -f option. If there's no hostname command,
|
||||
# explicitly replace `hostname -f` with the hostname.
|
||||
|
||||
HOSTNAME=`hostname -f`
|
||||
|
||||
# The hostname must match the "Site" field written in the list of mirrors.
|
||||
# If hostname doesn't returns the correct value, fill and uncomment below
|
||||
HOSTNAME=mirror.csclub.uwaterloo.ca
|
||||
|
||||
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"
|
||||
|
||||
# The temp directory used by rsync --delay-updates is not
|
||||
# world-readable remotely. It must be excluded to avoid errors.
|
||||
TMP_EXCLUDE="--exclude .~tmp~/"
|
||||
|
||||
# Logfile
|
||||
LOGFILE=$LOGDIR/debian-mirror.log
|
||||
|
||||
# Get in the right directory and set the umask to be group writable
|
||||
#
|
||||
cd $HOME
|
||||
umask 002
|
||||
|
||||
# Check to see if another sync is in progress
|
||||
if [ -f "$LOCK" ]; then
|
||||
# Note: this requires the findutils find; for other finds, adjust as necessary
|
||||
if [ "`find $LOCK -maxdepth 1 -amin -$LOCK_TIMEOUT`" = "" ]; then
|
||||
# Note: this requires the procps ps; for other ps', adjust as necessary
|
||||
if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
|
||||
echo "stale lock found, but a rsync is still running, aiee!"
|
||||
exit 1
|
||||
else
|
||||
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
|
||||
rm -f $LOCK
|
||||
fi
|
||||
else
|
||||
echo "current lock file exists, unable to start rsync!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
touch $LOCK
|
||||
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
|
||||
# specification. If that's the case on your system, try using "0".
|
||||
trap "rm -f $LOCK" exit
|
||||
|
||||
set +e
|
||||
|
||||
# Now sync the remaining stuff
|
||||
rsync --recursive --links --hard-links --times --verbose --delay-updates --delete-after \
|
||||
--exclude "Archive-Update-in-Progress-${HOSTNAME}" \
|
||||
--address=$ADDRESS $TMP_EXCLUDE \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||||
|
||||
savelog $LOGFILE >/dev/null
|
||||
|
||||
rm $LOCK
|
Loading…
Reference in New Issue