make sure to add mirror repo
This commit is contained in:
parent
e8ab91403f
commit
9d33c1aa93
|
@ -66,6 +66,10 @@ The ubunutu autoinstall can only handle basic installation. We require a more po
|
|||
|
||||
First, install ansible and sshpass. Perform all the following commands in the `post-install/` directory.
|
||||
|
||||
Also install the extra roles
|
||||
```
|
||||
$ ansible-galaxy install -r requirements.yml
|
||||
```
|
||||
Check that ansible can talk to the vm:
|
||||
```
|
||||
$ ansible -m ping all
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
```
|
||||
- name: Create mirror Vm
|
||||
- name: Create mirror VM
|
||||
hosts: localhost
|
||||
connection: local
|
||||
tasks:
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
- src: ansible.posix
|
|
@ -0,0 +1 @@
|
|||
- name: Install
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
if test -n "$RSYNC_USER"; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
exec nice rsync -az --no-owner --no-group --delete --safe-links \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
--exclude .~tmp~/ \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO
|
|
@ -0,0 +1,93 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Source: https://git.server-speed.net/users/flo/bin/plain/syncrepo.sh
|
||||
# (from https://bugs.archlinux.org/task/52853)
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: $0 local_dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 022
|
||||
LOCAL_DIR=$1
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
# This is a simple mirroring script. To save bandwidth it first checks a
|
||||
# timestamp via HTTP and only runs rsync when the timestamp differs from the
|
||||
# local copy. As of 2016, a single rsync run without changes transfers roughly
|
||||
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
|
||||
# is run every minute. Performing a simple check via HTTP first can thus save a
|
||||
# lot of traffic.
|
||||
|
||||
# Directory where the repo is stored locally. Example: /srv/repo
|
||||
target="/mirror/root/$1"
|
||||
|
||||
# Directory where files are downloaded to before being moved in place.
|
||||
# This should be on the same filesystem as $target, but not a subdirectory of $target.
|
||||
# Example: /srv/tmp
|
||||
tmp="/home/mirror/tmp"
|
||||
|
||||
# Lockfile path
|
||||
lock="/home/mirror/tmp/mirrorsync-$1.lck"
|
||||
|
||||
# If you want to limit the bandwidth used by rsync set this.
|
||||
# Use 0 to disable the limit.
|
||||
# The default unit is KiB (see man rsync /--bwlimit for more)
|
||||
bwlimit=0
|
||||
|
||||
# The source URL of the mirror you want to sync from.
|
||||
# If you are a tier 1 mirror use rsync.archlinux.org, for example like this:
|
||||
# rsync://rsync.archlinux.org/ftp_tier1
|
||||
# Otherwise chose a tier 1 mirror from this list and use its rsync URL:
|
||||
# https://www.archlinux.org/mirrors/
|
||||
source_url='rsync://rsync.archlinux.org/ftp_tier1'
|
||||
|
||||
# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror.
|
||||
# If you are a tier 1 mirror use: http://rsync.archlinux.org/lastupdate
|
||||
# Otherwise use the HTTP(S) URL from your chosen mirror.
|
||||
lastupdate_url='http://rsync.archlinux.org/lastupdate'
|
||||
|
||||
#### END CONFIG
|
||||
|
||||
[ ! -d "${target}" ] && mkdir -p "${target}"
|
||||
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"
|
||||
|
||||
exec 9>"${lock}"
|
||||
flock -n 9 || exit
|
||||
|
||||
rsync_cmd() {
|
||||
local -a cmd=(rsync -rtlH --safe-links --delete-after ${VERBOSE} "--timeout=600" "--contimeout=60" -p \
|
||||
--delay-updates --no-motd "--temp-dir=${tmp}" \
|
||||
"--log-file=/home/mirror/merlin/logs/$LOCAL_DIR.log" \
|
||||
--address=$ADDRESS)
|
||||
|
||||
if stty &>/dev/null; then
|
||||
cmd+=(-h -v --progress)
|
||||
fi
|
||||
|
||||
if ((bwlimit>0)); then
|
||||
cmd+=("--bwlimit=$bwlimit")
|
||||
fi
|
||||
|
||||
"${cmd[@]}" "$@"
|
||||
}
|
||||
|
||||
|
||||
# if we are called without a tty (cronjob) only run when there are changes
|
||||
if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl --interface $ADDRESS -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then
|
||||
# keep lastsync file in sync for statistics generated by the Arch Linux website
|
||||
rsync_cmd "$source_url/lastsync" "$target/lastsync"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# optional
|
||||
# --exclude='*.links.tar.gz*' \
|
||||
# --exclude='/other' \
|
||||
# --exclude='/sources' \
|
||||
# --exclude='/iso' \
|
||||
|
||||
rsync_cmd \
|
||||
"${source_url}" \
|
||||
"${target}"
|
||||
|
||||
#echo "Last sync was $(date -d @$(cat ${target}/lastsync))"
|
|
@ -0,0 +1,66 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Source: https://git.server-speed.net/users/flo/bin/plain/syncrepo.sh
|
||||
# as recommended in our request to become Tier 1:
|
||||
# https://bugs.archlinux.org/task/52853
|
||||
|
||||
if [ $# -lt 1 ]; then
|
||||
echo "Usage: $0 local_dir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 022
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
# This is a simple mirroring script. To save bandwidth it first checks a
|
||||
# timestamp via HTTP and only runs rsync when the timestamp differs from the
|
||||
# local copy. As of 2016, a single rsync run without changes transfers roughly
|
||||
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync
|
||||
# is run every minute. Performing a simple check via HTTP first can thus save a
|
||||
# lot of traffic.
|
||||
|
||||
home="/home/mirror"
|
||||
target=/mirror/root/$1
|
||||
tmp="${home}/tmp"
|
||||
lock="${tmp}/mirrorsync-$1.lck"
|
||||
# NOTE: You'll probably want to change this or remove the --bwlimit setting in
|
||||
# the rsync call below
|
||||
bwlimit=4096
|
||||
# NOTE: most people reading this very likely need to change this since
|
||||
# rsync.archlinux.org requires you to be a tier 1 mirror
|
||||
source='rsync://rsync.archlinux.org/ftp_tier1'
|
||||
lastupdate_url="http://rsync.archlinux.org/lastupdate"
|
||||
|
||||
[ ! -d "${target}" ] && mkdir -p "${target}"
|
||||
[ ! -d "${tmp}" ] && mkdir -p "${tmp}"
|
||||
|
||||
exec 9>"${lock}"
|
||||
flock -n 9 || exit
|
||||
|
||||
# if we are called without a tty (cronjob) only run when there are changes
|
||||
if ! tty -s && diff -b <(curl --interface $ADDRESS -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then
|
||||
date +'%s' > "$target/lastsync"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! stty &>/dev/null; then
|
||||
QUIET="-q"
|
||||
fi
|
||||
|
||||
# this can be added into the rsync: --bwlimit=$bwlimit \
|
||||
# --exclude='/iso' \
|
||||
# --exclude='*.links.tar.gz*' \
|
||||
# --exclude='/other' \
|
||||
# --exclude='/sources' \
|
||||
|
||||
rsync -rtlvH --safe-links --delete-after --progress -h ${QUIET} --timeout=600 --contimeout=60 -p \
|
||||
--delay-updates --no-motd \
|
||||
--temp-dir="${tmp}" \
|
||||
--stats \
|
||||
--log-file=$home/merlin/logs/$1.log \
|
||||
--address=$ADDRESS \
|
||||
${source} \
|
||||
"${target}"
|
||||
|
||||
date +'%s' > "$target/lastsync"
|
||||
#echo "Last sync was $(date -d @$(cat ${target}/lastsync))"
|
|
@ -0,0 +1,23 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
if test -n "$RSYNC_USER"; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
exec nice rsync -aH --no-owner --no-group --chmod=o=rX --delete \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
--exclude .~tmp~/ \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
if test -n "$RSYNC_USER"; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
mkdir -p $TO
|
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
--exclude ".*/" \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO
|
|
@ -0,0 +1,105 @@
|
|||
#!/usr/bin/env bash
|
||||
set -e
|
||||
#
|
||||
# Script to mirror Ceph locally
|
||||
#
|
||||
# Please, choose a local source and do not sync in a shorter interval than
|
||||
# 3 hours.
|
||||
#
|
||||
SILENT=0
|
||||
|
||||
# All available source mirrors
|
||||
declare -A SOURCES
|
||||
SOURCES[eu]="eu.ceph.com"
|
||||
SOURCES[de]="de.ceph.com"
|
||||
SOURCES[se]="se.ceph.com"
|
||||
SOURCES[au]="au.ceph.com"
|
||||
SOURCES[us]="download.ceph.com"
|
||||
SOURCES[hk]="hk.ceph.com"
|
||||
SOURCES[fr]="fr.ceph.com"
|
||||
SOURCES[us-east]="us-east.ceph.com"
|
||||
SOURCES[us-west]="us-west.ceph.com"
|
||||
SOURCES[global]="download.ceph.com"
|
||||
|
||||
function print_usage() {
|
||||
echo "$0 [-q ] -s <source mirror> -t <target directory>"
|
||||
}
|
||||
|
||||
while getopts ":qhs:t:" opt; do
|
||||
case $opt in
|
||||
q)
|
||||
SILENT=1
|
||||
;;
|
||||
s)
|
||||
SOURCE=$OPTARG
|
||||
;;
|
||||
t)
|
||||
TARGET=/mirror/root/$OPTARG
|
||||
;;
|
||||
h)
|
||||
HELP=1
|
||||
;;
|
||||
\?)
|
||||
print_usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then
|
||||
print_usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$TARGET" ]; then
|
||||
echo "$TARGET is not a valid target directory"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for i in "${!SOURCES[@]}"; do
|
||||
if [ "$i" == "$SOURCE" ]; then
|
||||
SOURCE_HOST=${SOURCES[$i]}
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$SOURCE_HOST" ]; then
|
||||
echo -n "Please select one of the following sources:"
|
||||
for i in "${!SOURCES[@]}"; do
|
||||
echo -n " $i"
|
||||
done
|
||||
echo ""
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RSYNC_OPTS="--stats --progress"
|
||||
if [ $SILENT -eq 1 ]; then
|
||||
RSYNC_OPTS="--quiet"
|
||||
fi
|
||||
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
RSYNC_OPTS="$RSYNC_OPTS -4 --address=$ADDRESS"
|
||||
|
||||
# We start a two-stage sync here for DEB and RPM
|
||||
# Based on: https://www.debian.org/mirror/ftpmirror
|
||||
#
|
||||
# The idea is to prevent temporary situations where metadata points to files
|
||||
# which do not exist
|
||||
#
|
||||
|
||||
# Exclude all metadata files
|
||||
rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
|
||||
--hard-links \
|
||||
--exclude Packages* \
|
||||
--exclude Sources* \
|
||||
--exclude Release* \
|
||||
--exclude InRelease \
|
||||
--exclude i18n/* \
|
||||
--exclude ls-lR* \
|
||||
--exclude repodata/* \
|
||||
${TARGET}
|
||||
|
||||
# Now also transfer the metadata and delete afterwards
|
||||
rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
|
||||
--hard-links --delete-after \
|
||||
${TARGET}
|
||||
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
if test -n "$RSYNC_USER"; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
mkdir -p $TO
|
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
--exclude .~tmp~/ \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \
|
||||
--chmod=u=rwX,go=rX \
|
||||
rsync://$RSYNC_HOST/$RSYNC_DIR $TO
|
||||
#$RSYNC_HOST::$RSYNC_DIR $TO
|
|
@ -0,0 +1,245 @@
|
|||
#! /bin/bash
|
||||
|
||||
if [[ $# < 3 ]]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir [trace_host [trace_dir]]'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
TOP_DIR=/mirror
|
||||
TO=$TOP_DIR/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
if [ $# = 4 ]; then
|
||||
TRACE_HOST=$4
|
||||
fi
|
||||
TRACE_DIR=project/trace
|
||||
if [ $# = 5 ]; then
|
||||
TRACE_DIR=$5
|
||||
fi
|
||||
LOGDIR=/var/log/mirror/$1_$2
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
mkdir -p $LOGDIR
|
||||
|
||||
# This script originates from http://www.debian.org/mirror/anonftpsync
|
||||
|
||||
# CVS: cvs.debian.org:/cvs/webwml - webwml/english/mirror/anonftpsync
|
||||
# Version: $Id: anonftpsync,v 1.33 2007/09/12 15:19:03 joy Exp $
|
||||
|
||||
# Note: You MUST have rsync 2.6.4 or newer, which is available in sarge
|
||||
# and all newer Debian releases, or at http://rsync.samba.org/
|
||||
|
||||
# Don't forget:
|
||||
# chmod u+x anonftpsync
|
||||
|
||||
# Set the variables below to fit your site. You can then use cron to have
|
||||
# this script run daily to automatically update your copy of the archive.
|
||||
|
||||
# TO is the destination for the base of the Debian mirror directory
|
||||
# (the dir that holds dists/ and ls-lR).
|
||||
# (mandatory)
|
||||
|
||||
#TO=
|
||||
|
||||
# RSYNC_HOST is the site you have chosen from the mirrors file.
|
||||
# (http://www.debian.org/mirror/list-full)
|
||||
# (mandatory)
|
||||
|
||||
#RSYNC_HOST=
|
||||
|
||||
# RSYNC_DIR is the directory given in the "Packages over rsync:" line of
|
||||
# the mirrors file for the site you have chosen to mirror.
|
||||
# (mandatory)
|
||||
|
||||
#RSYNC_DIR=
|
||||
|
||||
# LOGDIR is the directory where the logs will be written to
|
||||
# (mandatory)
|
||||
|
||||
#LOGDIR=
|
||||
|
||||
# ARCH_EXCLUDE can be used to exclude a complete architecture from
|
||||
# mirrorring. Please use as space seperated list.
|
||||
# Possible values are:
|
||||
# alpha, amd64, arm, hppa, hurd-i386, i386, ia64, m68k, mipsel, mips, powerpc, s390, sh and sparc
|
||||
#
|
||||
# There is one special value: source
|
||||
# This is not an architecture but will exclude all source code in /pool
|
||||
#
|
||||
# eg.
|
||||
# ARCH_EXCLUDE="alpha arm hppa hurd-i386 ia64 m68k mipsel mips s390 sparc"
|
||||
#
|
||||
# With a blank ARCH_EXCLUDE you will mirror all available architectures
|
||||
# (optional)
|
||||
|
||||
# EXCLUDE is a list of parameters listing patterns that rsync will exclude, in
|
||||
# addition to the architectures excluded by ARCH_EXCLUDE.
|
||||
#
|
||||
# Use ARCH_EXCLUDE to exclude specific architectures or all sources
|
||||
#
|
||||
# --exclude stable, testing, unstable options DON'T remove the packages of
|
||||
# the given distribution. If you want do so, use debmirror instead.
|
||||
#
|
||||
# The following example would exclude mostly everything:
|
||||
#EXCLUDE="\
|
||||
# --exclude stable/ --exclude testing/ --exclude unstable/ \
|
||||
# --exclude source/ \
|
||||
# --exclude *.orig.tar.gz --exclude *.diff.gz --exclude *.dsc \
|
||||
# --exclude /contrib/ --exclude /non-free/ \
|
||||
# "
|
||||
|
||||
# With a blank EXCLUDE you will mirror the entire archive, except the
|
||||
# architectures excluded by ARCH_EXCLUDE.
|
||||
# (optional)
|
||||
|
||||
EXCLUDE=
|
||||
|
||||
# LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
|
||||
# This program creates a lock to ensure that only one copy
|
||||
# of it is mirroring any one archive at any one time.
|
||||
# Locks held for longer than the timeout are broken, unless
|
||||
# a running rsync process appears to be connected to $RSYNC_HOST.
|
||||
|
||||
LOCK_TIMEOUT=360
|
||||
|
||||
# There should be no need to edit anything below this point, unless there
|
||||
# are problems.
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# If you are accessing a rsync server/module which is password-protected,
|
||||
# uncomment the following lines (and edit the other file).
|
||||
|
||||
# . ftpsync.conf
|
||||
# export RSYNC_PASSWORD
|
||||
# RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# Check for some environment variables
|
||||
if [ -z $TO ] || [ -z $RSYNC_HOST ] || [ -z $RSYNC_DIR ] || [ -z $LOGDIR ]; then
|
||||
echo "One of the following variables seems to be empty:"
|
||||
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if ! [ -d ${TO}/${TRACE_DIR} ]; then
|
||||
# we are running mirror script for the first time
|
||||
umask 002
|
||||
mkdir -p ${TO}/${TRACE_DIR}
|
||||
fi
|
||||
|
||||
# Note: on some non-Debian systems, hostname doesn't accept -f option.
|
||||
# If that's the case on your system, make sure hostname prints the full
|
||||
# hostname, and remove the -f option. If there's no hostname command,
|
||||
# explicitly replace `hostname -f` with the hostname.
|
||||
|
||||
HOSTNAME=`hostname -f`
|
||||
|
||||
# The hostname must match the "Site" field written in the list of mirrors.
|
||||
# If hostname doesn't returns the correct value, fill and uncomment below
|
||||
HOSTNAME=mirror.csclub.uwaterloo.ca
|
||||
|
||||
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"
|
||||
|
||||
# The temp directory used by rsync --delay-updates is not
|
||||
# world-readable remotely. It must be excluded to avoid errors.
|
||||
TMP_EXCLUDE="--exclude .~tmp~/"
|
||||
|
||||
# Exclude architectures defined in $ARCH_EXCLUDE
|
||||
for ARCH in $ARCH_EXCLUDE; do
|
||||
EXCLUDE=$EXCLUDE"\
|
||||
--exclude binary-$ARCH/ \
|
||||
--exclude disks-$ARCH/ \
|
||||
--exclude installer-$ARCH/ \
|
||||
--exclude Contents-$ARCH.gz \
|
||||
--exclude Contents-$ARCH.diff/ \
|
||||
--exclude arch-$ARCH.files \
|
||||
--exclude arch-$ARCH.list.gz \
|
||||
--exclude *_$ARCH.deb \
|
||||
--exclude *_$ARCH.udeb "
|
||||
if [ "$ARCH" == "source" ]; then
|
||||
SOURCE_EXCLUDE="\
|
||||
--exclude source/ \
|
||||
--exclude *.tar.gz \
|
||||
--exclude *.diff.gz \
|
||||
--exclude *.dsc "
|
||||
fi
|
||||
done
|
||||
|
||||
# Logfile
|
||||
LOGFILE=$LOGDIR/mirror.log
|
||||
|
||||
# Get in the right directory and set the umask to be group writable
|
||||
#
|
||||
cd $HOME
|
||||
umask 002
|
||||
|
||||
# Check to see if another sync is in progress
|
||||
if [ -f "$LOCK" ]; then
|
||||
# Note: this requires the findutils find; for other finds, adjust as necessary
|
||||
if [ "`find $LOCK -maxdepth 1 -amin -$LOCK_TIMEOUT`" = "" ]; then
|
||||
# Note: this requires the procps ps; for other ps', adjust as necessary
|
||||
if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
|
||||
echo "stale lock found, but a rsync is still running, aiee!" >&2
|
||||
exit 1
|
||||
else
|
||||
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
|
||||
rm -f $LOCK
|
||||
fi
|
||||
else
|
||||
echo "current lock file exists, unable to start rsync!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
touch $LOCK
|
||||
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
|
||||
# specification. If that's the case on your system, try using "0".
|
||||
trap "rm -f $LOCK" exit
|
||||
|
||||
set +e
|
||||
|
||||
# check if we need to sync
|
||||
if [[ "$TRACE_HOST" != "" ]]; then
|
||||
TRACE_OLD_TIME=`stat -c%Y $TO/$TRACE_DIR/$TRACE_HOST 2> /dev/null`
|
||||
TRACE_NEW_FILE=/tmp/$RSYNC_HOST_$RSYNC_DIR_$RANDOM
|
||||
nice rsync -tv -4 --address=$ADDRESS \
|
||||
$RSYNC_HOST::$RSYNC_DIR/$TRACE_DIR/$TRACE_HOST \
|
||||
$TRACE_NEW_FILE >> $LOGFILE 2>&1
|
||||
TRACE_NEW_TIME=`stat -c%Y $TRACE_NEW_FILE`
|
||||
rm -f $TRACE_NEW_FILE
|
||||
if [ "$TRACE_OLD_TIME" = "$TRACE_NEW_TIME" ]; then
|
||||
echo 'Trace file for' $RSYNC_HOST::$RSYNC_DIR \
|
||||
'unchanged!' >> $LOGFILE
|
||||
fi
|
||||
fi
|
||||
|
||||
# First sync /pool
|
||||
nice rsync -rlHtvp \
|
||||
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
$RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
|
||||
result=$?
|
||||
|
||||
if [ 0 = $result ]; then
|
||||
# Now sync the remaining stuff
|
||||
nice rsync -rlHtvp --delay-updates --delete-after \
|
||||
--exclude "Archive-Update-in-Progress-${HOSTNAME}" \
|
||||
--exclude "${TRACE_DIR}/${HOSTNAME}" \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
$TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||||
|
||||
mkdir -p ${TO}/${TRACE_DIR}
|
||||
LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"
|
||||
else
|
||||
echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
|
||||
echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
|
||||
fi
|
||||
|
||||
savelog $LOGFILE >/dev/null
|
||||
|
||||
rm $LOCK
|
|
@ -0,0 +1,107 @@
|
|||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
TOP_DIR=/mirror
|
||||
TO=$TOP_DIR/root/debian-cd
|
||||
RSYNC_HOST=cdimage.debian.org
|
||||
RSYNC_DIR=debian-cd
|
||||
LOGDIR=/var/log/mirror/debian-cd_cdimage.debian.org
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
mkdir -p $LOGDIR
|
||||
|
||||
# LOCK_TIMEOUT is a timeout in minutes. Defaults to 360 (6 hours).
|
||||
# This program creates a lock to ensure that only one copy
|
||||
# of it is mirroring any one archive at any one time.
|
||||
# Locks held for longer than the timeout are broken, unless
|
||||
# a running rsync process appears to be connected to $RSYNC_HOST.
|
||||
|
||||
LOCK_TIMEOUT=360
|
||||
|
||||
# There should be no need to edit anything below this point, unless there
|
||||
# are problems.
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# If you are accessing a rsync server/module which is password-protected,
|
||||
# uncomment the following lines (and edit the other file).
|
||||
|
||||
# . ftpsync.conf
|
||||
export RSYNC_PASSWORD
|
||||
if [[ "$RSYNC_USER" != "" ]]; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
#-----------------------------------------------------------------------------#
|
||||
|
||||
# Check for some environment variables
|
||||
if [ -z $TO ] || [ -z $RSYNC_HOST ] || [ -z $RSYNC_DIR ] || [ -z $LOGDIR ]; then
|
||||
echo "One of the following variables seems to be empty:"
|
||||
echo "TO, RSYNC_HOST, RSYNC_DIR or LOGDIR"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Note: on some non-Debian systems, hostname doesn't accept -f option.
|
||||
# If that's the case on your system, make sure hostname prints the full
|
||||
# hostname, and remove the -f option. If there's no hostname command,
|
||||
# explicitly replace `hostname -f` with the hostname.
|
||||
|
||||
HOSTNAME=`hostname -f`
|
||||
|
||||
# The hostname must match the "Site" field written in the list of mirrors.
|
||||
# If hostname doesn't returns the correct value, fill and uncomment below
|
||||
HOSTNAME=mirror.csclub.uwaterloo.ca
|
||||
|
||||
LOCK="${TO}/Archive-Update-in-Progress-${HOSTNAME}"
|
||||
|
||||
# The temp directory used by rsync --delay-updates is not
|
||||
# world-readable remotely. It must be excluded to avoid errors.
|
||||
TMP_EXCLUDE="--exclude .~tmp~/"
|
||||
|
||||
# Logfile
|
||||
LOGFILE=$LOGDIR/mirror.log
|
||||
|
||||
# Get in the right directory and set the umask to be group writable
|
||||
#
|
||||
cd $HOME
|
||||
umask 002
|
||||
|
||||
# Check to see if another sync is in progress
|
||||
if [ -f "$LOCK" ]; then
|
||||
# Note: this requires the findutils find; for other finds, adjust as necessary
|
||||
if [ "`find $LOCK -maxdepth 1 -amin -$LOCK_TIMEOUT`" = "" ]; then
|
||||
# Note: this requires the procps ps; for other ps', adjust as necessary
|
||||
if ps ax | grep '[r]'sync | grep -q $RSYNC_HOST; then
|
||||
echo "stale lock found, but a rsync is still running, aiee!" >&2
|
||||
exit 1
|
||||
else
|
||||
echo "stale lock found (not accessed in the last $LOCK_TIMEOUT minutes), forcing update!"
|
||||
rm -f $LOCK
|
||||
fi
|
||||
else
|
||||
echo "current lock file exists, unable to start rsync!"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
touch $LOCK
|
||||
# Note: on some non-Debian systems, trap doesn't accept "exit" as signal
|
||||
# specification. If that's the case on your system, try using "0".
|
||||
trap "rm -f $LOCK" exit
|
||||
|
||||
set +e
|
||||
|
||||
# Now sync the remaining stuff
|
||||
nice rsync -rlHtvp --delete \
|
||||
--timeout=3600 -4 --address=$ADDRESS $TMP_EXCLUDE \
|
||||
--exclude "Archive-Update-in-Progress-${HOSTNAME}" \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||||
if [[ "$?" != "0" ]]; then
|
||||
echo "ERROR: Help, something weird happened" | tee -a $LOGFILE
|
||||
echo "mirroring /pool exited with exitcode" $result | tee -a $LOGFILE
|
||||
fi
|
||||
|
||||
savelog $LOGFILE >/dev/null
|
||||
|
||||
rm $LOCK
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/sh
|
||||
|
||||
RSYNC_USER=gentoo exec ~/bin/csc-sync-standard gentoo-distfiles masterdistfiles.gentoo.org gentoo gentoo-distfiles
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 2 ]; then
|
||||
echo 'Usage: sync local_dir endpoint'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
export RCLONE_CONFIG_S3_ENDPOINT=$2 RCLONE_CONFIG_S3_TYPE=s3 RCLONE_CONFIG_S3_PROVIDER=Other RCLONE_CONFIG_S3_ENV_AUTH=false
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
mkdir -p $TO
|
||||
cd $TO
|
||||
exec nice rclone sync --fast-list --use-server-modtime --bind $ADDRESS s3:s3/ $TO
|
|
@ -0,0 +1,24 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir ssh_user ssh_keyfile'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
SSH_USER=$4
|
||||
SSH_KEYFILE=$5
|
||||
RSYNC_HOST=$SSH_USER@$RSYNC_HOST
|
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete \
|
||||
--timeout=3600 -4 \
|
||||
--exclude .~tmp~/ \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \
|
||||
-e "ssh -i $SSH_KEYFILE" \
|
||||
$RSYNC_HOST:$RSYNC_DIR/ $TO
|
||||
|
||||
#134.71 needs to be used
|
|
@ -0,0 +1,31 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir [password file]'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
RSYNC_PASSWORD_ARGS=
|
||||
if [ $# -ge 4 ]; then
|
||||
RSYNC_PASSWORD_ARGS="--password-file $HOME/passwords/$4"
|
||||
fi
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
if test -n "$RSYNC_USER"; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
mkdir -p $TO
|
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \
|
||||
--timeout=3600 -4 --address=$ADDRESS \
|
||||
--exclude .~tmp~/ \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \
|
||||
$RSYNC_PASSWORD_ARGS \
|
||||
rsync://$RSYNC_HOST/$RSYNC_DIR $TO
|
||||
#$RSYNC_HOST::$RSYNC_DIR $TO
|
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir rsync_host rsync_dir'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
RSYNC_HOST=$2
|
||||
RSYNC_DIR=$3
|
||||
ADDRESS=$(cat ~/config/ADDRESS_V6)
|
||||
|
||||
if test -n "$RSYNC_USER"; then
|
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST
|
||||
fi
|
||||
|
||||
mkdir -p $TO
|
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \
|
||||
--timeout=3600 -6 --address=$ADDRESS \
|
||||
--exclude .~tmp~/ \
|
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \
|
||||
$RSYNC_HOST::$RSYNC_DIR $TO
|
|
@ -0,0 +1,17 @@
|
|||
#!/bin/bash -e
|
||||
|
||||
if [ $# -lt 3 ]; then
|
||||
echo 'Usage: sync local_dir path cut'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
umask 002
|
||||
|
||||
TO=/mirror/root/$1
|
||||
SOURCE_PATH=$2
|
||||
CUT=$3
|
||||
ADDRESS=$(cat ~/config/ADDRESS)
|
||||
|
||||
mkdir -p $TO
|
||||
cd $TO
|
||||
exec nice wget -q --bind-address=$ADDRESS --mirror --no-parent --no-host-directories --cut-dirs=$CUT --content-disposition --execute robots=off --recursive --reject "*\?*" $SOURCE_PATH
|
|
@ -0,0 +1,31 @@
|
|||
#!/usr/bin/python
|
||||
import os.path
|
||||
|
||||
base = '/mirror/root/csclub'
|
||||
htbase = 'http://csclub.uwaterloo.ca/media/files'
|
||||
announce = 'http://bittorrent.csclub.uwaterloo.ca/announce'
|
||||
scpto = 'bittorrent.csclub:/var/lib/bnbt/torrents/'
|
||||
comment = 'Produced by the University of Waterloo Computer Science Club (http://csclub.uwaterloo.ca/)'
|
||||
minsize = 10*1024*1024 # 10 MiB
|
||||
|
||||
btmake = '/usr/bin/btmakemetafile.bittornado'
|
||||
scp = '/usr/bin/scp'
|
||||
|
||||
mediafiles = [ file for file in os.listdir(base) if
|
||||
not file.endswith('.torrent') and
|
||||
not os.path.basename(file).startswith('.') and
|
||||
not os.path.isdir(base + '/' + file) and
|
||||
os.path.getsize(base + '/' + file) > minsize
|
||||
]
|
||||
|
||||
for file in mediafiles:
|
||||
path = base + '/' + file
|
||||
torrentpath = path + '.torrent'
|
||||
htpath = htbase + '/' + file
|
||||
if not os.path.exists(torrentpath):
|
||||
print "Making torrent for %s..." % torrentpath
|
||||
os.spawnl(os.P_WAIT, btmake, btmake, announce, path,
|
||||
'--comment', comment, '--target', torrentpath,
|
||||
'--httpseeds', htpath)
|
||||
os.spawnl(os.P_WAIT, scp, scp, torrentpath, scpto)
|
||||
print "The bittorrent tracker will begin tracking new torrents within five minutes."
|
|
@ -0,0 +1,272 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import os, sys
|
||||
from os.path import join
|
||||
import pickle
|
||||
import re
|
||||
import ConfigParser
|
||||
import pprint
|
||||
import time
|
||||
import xmlrpclib
|
||||
import base64
|
||||
import bz2
|
||||
import socket
|
||||
|
||||
class HostConfig(object):
|
||||
"""Holder for config info from the configuration file"""
|
||||
def __init__(self):
|
||||
self.config = { 'version' : 0,
|
||||
'global': {},
|
||||
'site': {},
|
||||
'host': {},
|
||||
'stats': {},
|
||||
}
|
||||
|
||||
_translation = [chr(_x) for _x in range(256)]
|
||||
def _translate(s, altchars):
|
||||
translation = _translation[:]
|
||||
for k, v in altchars.items():
|
||||
translation[ord(k)] = v
|
||||
return s.translate(''.join(translation))
|
||||
|
||||
def urlsafe_b64encode(s):
|
||||
import binascii
|
||||
altchars = '-_'
|
||||
encoded = binascii.b2a_base64(s)[:-1]
|
||||
if altchars is not None:
|
||||
return _translate(encoded, {'+': altchars[0], '/': altchars[1]})
|
||||
return encoded
|
||||
|
||||
def gen_dirtree(path):
|
||||
# structure here is:
|
||||
# dirtree is a dict
|
||||
# {
|
||||
# dirpath :
|
||||
# {
|
||||
# filename1 : size1,
|
||||
# filename2 : size2,
|
||||
# ...
|
||||
# },
|
||||
# ...
|
||||
# }
|
||||
#
|
||||
# 2009-03-09: MM's web app ignores the statfiles dict. So don't bother generating it.
|
||||
|
||||
dirtree = {}
|
||||
for dirpath, dirnames, filenames in os.walk(path):
|
||||
statfiles = {}
|
||||
if path.endswith('/'):
|
||||
short_path = dirpath[len(path):]
|
||||
else:
|
||||
short_path = dirpath[len(path)+1:]
|
||||
if len(short_path) > 0:
|
||||
dirtree[short_path] = statfiles
|
||||
else:
|
||||
dirtree[''] = statfiles
|
||||
|
||||
return dirtree
|
||||
|
||||
def errorprint(error):
|
||||
sys.stderr.write(error+'\n')
|
||||
|
||||
class MissingOption(Exception):
|
||||
pass
|
||||
|
||||
def check_required_options(conf, section, required_options):
|
||||
for o in required_options:
|
||||
if not conf.has_option(section, o):
|
||||
errorprint('missing required option %s in config [%s]' % (o, section))
|
||||
raise MissingOption()
|
||||
return True
|
||||
|
||||
def parse_value(value):
|
||||
"""Split multi-line values into a list"""
|
||||
if value.find('\n') > -1:
|
||||
return value.split()
|
||||
return value
|
||||
|
||||
def parse_section(conf, section, item, required_options, optional_options=[]):
|
||||
if conf.has_option(section, 'enabled'):
|
||||
if conf.get(section, 'enabled') != '1' and section.lower() in item.config:
|
||||
print 'removing disabled section %s' % (section)
|
||||
del item.config[section.lower()]
|
||||
return False
|
||||
|
||||
if not check_required_options(conf, section, required_options):
|
||||
return False
|
||||
|
||||
if not section.lower() in item.config:
|
||||
item.config[section.lower()] = {}
|
||||
|
||||
for o in required_options:
|
||||
item.config[section.lower()][o] = parse_value(conf.get(section, o))
|
||||
for o in optional_options:
|
||||
if conf.has_option(section, o):
|
||||
item.config[section.lower()][o] = parse_value(conf.get(section, o))
|
||||
|
||||
return True
|
||||
|
||||
def parse_global(conf, section, item):
|
||||
required_options = [ 'enabled', 'server' ]
|
||||
if not parse_section(conf, section, item, required_options):
|
||||
errorprint('missing required options (server AND enabled) in [%s] section' % (section))
|
||||
return False
|
||||
return True
|
||||
|
||||
def parse_site(conf, section, item):
|
||||
required_options = [ 'enabled', 'name', 'password' ]
|
||||
return parse_section(conf, section, item, required_options)
|
||||
|
||||
def parse_host(conf, section, item):
|
||||
required_options = [ 'enabled', 'name' ]
|
||||
optional_options = [ 'user_active' ]
|
||||
return parse_section(conf, section, item, required_options, optional_options=optional_options)
|
||||
|
||||
def get_stats(conf, section):
|
||||
if conf.has_option(section, 'enabled'):
|
||||
if conf.get(section, 'enabled') != '1':
|
||||
return None
|
||||
statsdata = {}
|
||||
for name, value in conf.items(section):
|
||||
if name == 'enabled':
|
||||
continue
|
||||
filenames = parse_value(conf.get(section, name))
|
||||
if type(filenames) != list:
|
||||
filenames = [ filenames ]
|
||||
for fn in filenames:
|
||||
try:
|
||||
f = open(fn, 'r')
|
||||
contents = contents + f.readlines()
|
||||
statsdata[name] = pickle.dumps(contents, -1)
|
||||
f.close()
|
||||
except:
|
||||
pass
|
||||
return statsdata
|
||||
|
||||
def parse_category(conf, section, item, crawl):
|
||||
required_options = [ 'enabled', 'path' ]
|
||||
if not parse_section(conf, section, item, required_options):
|
||||
return False
|
||||
|
||||
if crawl:
|
||||
dirtree = gen_dirtree(conf.get(section, 'path'))
|
||||
item.config[section.lower()]['dirtree'] = dirtree
|
||||
# database doesn't need to know the disk path
|
||||
del item.config[section.lower()]['path']
|
||||
|
||||
def config(cfg, item, crawl=True):
|
||||
broken = False
|
||||
conf = ConfigParser.ConfigParser()
|
||||
files = conf.read(cfg)
|
||||
if files == []:
|
||||
errorprint('Configuration file %s not found' % (cfg))
|
||||
return False
|
||||
conf.read(cfg)
|
||||
|
||||
try:
|
||||
# don't grab parse_stats here
|
||||
for section, parsefunc in [ ('global', parse_global), ('site', parse_site),
|
||||
('host', parse_host)]:
|
||||
if conf.has_section(section):
|
||||
if not parsefunc(conf, section, item):
|
||||
return False
|
||||
else:
|
||||
errorprint('Invalid configuration - missing section [%s]' % (section))
|
||||
sys.exit(1)
|
||||
|
||||
for section in conf.sections():
|
||||
if section in [ 'global', 'site', 'host', 'stats']:
|
||||
continue
|
||||
parse_category(conf, section, item, crawl)
|
||||
|
||||
except MissingOption:
|
||||
errorprint('Invalid configuration - Exiting')
|
||||
sys.exit(1)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
|
||||
def main():
|
||||
from optparse import OptionParser
|
||||
parser = OptionParser(usage= sys.argv[0] + " [options]")
|
||||
parser.add_option("-c", "--config",
|
||||
dest="config",
|
||||
default='/home/mirror/mirrormanager-client/report_mirror.conf',
|
||||
help='Configuration filename (required)')
|
||||
parser.add_option("-s", "--stats",
|
||||
action="store_true",
|
||||
dest="stats",
|
||||
default=False,
|
||||
help='Send stats')
|
||||
parser.add_option("-i", "--input",
|
||||
dest="input",
|
||||
default=None,
|
||||
help="Input filename (for debugging)")
|
||||
parser.add_option("-o", "--output",
|
||||
dest="output",
|
||||
default=None,
|
||||
help="Output filename (for debugging)")
|
||||
parser.add_option("-n", "--no-send",
|
||||
action="store_true",
|
||||
dest="no_send",
|
||||
default=False,
|
||||
help="Don't send data to the server.")
|
||||
parser.add_option("-d", "--debug",
|
||||
action="store_true",
|
||||
dest="debug",
|
||||
default=False,
|
||||
help='Enable debugging output')
|
||||
|
||||
|
||||
(options, args) = parser.parse_args()
|
||||
item = HostConfig()
|
||||
if options.input:
|
||||
infile = open(options.input, 'rb')
|
||||
item.config = pickle.load(infile)
|
||||
infile.close()
|
||||
if not config(options.config, item, crawl=False):
|
||||
sys.exit(1)
|
||||
else:
|
||||
if not config(options.config, item, crawl=True):
|
||||
sys.exit(1)
|
||||
|
||||
p = pickle.dumps(item.config, -1)
|
||||
|
||||
if options.debug:
|
||||
pp = pprint.PrettyPrinter(indent=4)
|
||||
pp.pprint(item.config)
|
||||
|
||||
if options.output is not None:
|
||||
outfile = open(options.output, 'wb')
|
||||
outfile.write(p)
|
||||
outfile.close()
|
||||
|
||||
if options.stats:
|
||||
statdata = get_stats(conf, 'stats')
|
||||
|
||||
# upload p and statsdata here
|
||||
if not item.config.has_key('global') or not item.config['global'].has_key('enabled') or item.config['global']['enabled'] != '1':
|
||||
sys.exit(1)
|
||||
|
||||
if not options.no_send:
|
||||
# print "Connecting to %s" % item.config['global']['server']
|
||||
server = xmlrpclib.ServerProxy(item.config['global']['server'])
|
||||
data = None
|
||||
try:
|
||||
data = base64.urlsafe_b64encode(bz2.compress(p))
|
||||
except AttributeError:
|
||||
data = urlsafe_b64encode(bz2.compress(p))
|
||||
|
||||
if data is not None:
|
||||
try:
|
||||
print server.checkin(data)
|
||||
except socket.error, m:
|
||||
print "Error checking in: %s. Please try again later." % (m[1])
|
||||
except xmlrpclib.Fault:
|
||||
print "Error checking in. Connection closed before checkin complete. Please try again later."
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/sh
|
||||
# This file is used for ubuntu push mirroring. Do not remove it!
|
||||
exec ~/bin/csc-sync-standard ubuntu-releases rsync.releases.ubuntu.com releases
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo 'Usage: project' 1>&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exec 200>/tmp/$1.lock
|
||||
flock 200 || exit 1
|
||||
sudo /usr/local/bin/zfssync.py $1
|
|
@ -0,0 +1 @@
|
|||
129.97.134.71
|
|
@ -0,0 +1 @@
|
|||
2620:101:f000:4901:c5c::f:1055
|
|
@ -0,0 +1,7 @@
|
|||
# ~/.bash_logout: executed by bash(1) when login shell exits.
|
||||
|
||||
# when leaving the console clear the screen to increase privacy
|
||||
|
||||
if [ "$SHLVL" = 1 ]; then
|
||||
[ -x /usr/bin/clear_console ] && /usr/bin/clear_console -q
|
||||
fi
|
|
@ -0,0 +1,11 @@
|
|||
# ~/.bash_profile: executed by bash(1) for login shells.
|
||||
|
||||
# include .bashrc if it exists
|
||||
if [ -f ~/.bashrc ]; then
|
||||
. ~/.bashrc
|
||||
fi
|
||||
|
||||
# set PATH so it includes user's private bin if it exists
|
||||
if [ -d ~/bin ] ; then
|
||||
PATH=~/bin:"${PATH}"
|
||||
fi
|
|
@ -0,0 +1,46 @@
|
|||
# ~/.bashrc: executed by bash(1) for non-login shells.
|
||||
|
||||
# If not running interactively, don't do anything
|
||||
[ -z "$PS1" ] && return
|
||||
|
||||
export HISTCONTROL=ignoreboth
|
||||
|
||||
# check the window size after each command and, if necessary,
|
||||
# update the values of LINES and COLUMNS.
|
||||
shopt -s checkwinsize
|
||||
|
||||
# make less more friendly for non-text input files, see lesspipe(1)
|
||||
[ -x /usr/bin/lesspipe ] && eval "$(lesspipe)"
|
||||
|
||||
# A little nice prompt.
|
||||
PS1='\[\033[01;33m\][`git branch 2>/dev/null|cut -f2 -d\* -s` ]\[\033[01;32m\]\u@\[\033[00;36m\]\h\[\033[01m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
|
||||
|
||||
# If this is an xterm set the title to user@host:dir
|
||||
case "$TERM" in
|
||||
xterm*|rxvt*)
|
||||
PROMPT_COMMAND='echo -ne "\033]0;${USER}@${HOSTNAME}: ${PWD/$HOME/~}\007"'
|
||||
;;
|
||||
*)
|
||||
;;
|
||||
esac
|
||||
|
||||
# Alias definitions.
|
||||
|
||||
# enable color support of ls and also add handy aliases
|
||||
eval "`dircolors -b`"
|
||||
alias ls='ls --color=auto'
|
||||
alias ll='ls -l'
|
||||
alias la='ls -A'
|
||||
alias l='ls -CF'
|
||||
|
||||
alias cp='cp -i'
|
||||
alias mv='mv -i'
|
||||
|
||||
alias ..='cd ..'
|
||||
|
||||
# enable programmable completion features (you don't need to enable
|
||||
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
|
||||
# sources /etc/bash.bashrc).
|
||||
if [ -f /etc/bash_completion ]; then
|
||||
. /etc/bash_completion
|
||||
fi
|
|
@ -0,0 +1,257 @@
|
|||
Archvsync
|
||||
=========
|
||||
|
||||
This is the central repository for the Debian mirror scripts. The scripts
|
||||
in this repository are written for the purposes of maintaining a Debian
|
||||
archive mirror (and shortly, a Debian bug mirror), but they should be
|
||||
easily generalizable.
|
||||
|
||||
|
||||
Currently the following scripts are available:
|
||||
|
||||
* ftpsync - Used to sync an archive using rsync
|
||||
* runmirrors - Used to notify leaf nodes of available updates
|
||||
* dircombine - Internal script to manage the mirror user's $HOME
|
||||
on debian.org machines
|
||||
* typicalsync - Generates a typical Debian mirror
|
||||
* udh - We are lazy, just a shorthand to avoid typing the
|
||||
commands, ignore... :)
|
||||
|
||||
Usage
|
||||
=====
|
||||
For impatient people, short usage instruction:
|
||||
|
||||
- Create a dedicated user for the whole mirror.
|
||||
- Create a seperate directory for the mirror, writeable by the new user.
|
||||
- Place the ftpsync script in the mirror user's $HOME/bin (or just $HOME)
|
||||
- Place the ftpsync.conf.sample into $HOME/etc as ftpsync.conf and edit
|
||||
it to suit your system. You should at the very least change the TO=
|
||||
and RSYNC_HOST lines.
|
||||
- Create $HOME/log (or wherever you point $LOGDIR to)
|
||||
- Setup the .ssh/authorized_keys for the mirror user and place the public key of
|
||||
your upstream mirror into it. Preface it with
|
||||
no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="~/bin/ftpsync",from="IPADDRESS"
|
||||
and replace $IPADDRESS with that of your upstream mirror.
|
||||
- You are finished
|
||||
|
||||
In order to receive different pushes or syncs from different archives,
|
||||
name the config file ftpsync-$ARCHIVE.conf and call the ftpsync script
|
||||
with the commandline "sync:archive:$ARCHIVE". Replace $ARCHIVE with a
|
||||
sensible value. If your upstream mirror pushes you using runmirrors
|
||||
bundled together with this sync script, you do not need to add the
|
||||
"sync:archive" parameter to the commandline, the scripts deal with it
|
||||
automatically.
|
||||
|
||||
|
||||
|
||||
Debian mirror script minimum requirements
|
||||
=========================================
|
||||
As always, you may use whatever scripts you want for your Debian mirror,
|
||||
but we *STRONGLY* recommend you to not invent your own. However, if you
|
||||
want to be listed as a mirror it *MUST* support the following minimal
|
||||
functionality:
|
||||
|
||||
- Must perform a 2-stage sync
|
||||
The archive mirroring must be done in 2 stages. The first rsync run
|
||||
must ignore the index files. The correct exclude options for the
|
||||
first rsync run are:
|
||||
--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*
|
||||
The first stage must not delete any files.
|
||||
|
||||
The second stage should then transfer the above excluded files and
|
||||
delete files that no longer belong on the mirror.
|
||||
|
||||
Rationale: If archive mirroring is done in a single stage, there will be
|
||||
periods of time during which the index files will reference files not
|
||||
yet mirrored.
|
||||
|
||||
- Must not ignore pushes whil(e|st) running.
|
||||
If a push is received during a run of the mirror sync, it MUST NOT
|
||||
be ignored. The whole synchronization process must be rerun.
|
||||
|
||||
Rationale: Most implementations of Debian mirror scripts will leave the
|
||||
mirror in an inconsistent state in the event of a second push being
|
||||
received while the first sync is still running. It is likely that in
|
||||
the near future, the frequency of pushes will increase.
|
||||
|
||||
- Should understand multi-stage pushes.
|
||||
The script should parse the arguments it gets via ssh, and if they
|
||||
contain a hint to only sync stage1 or stage2, then ONLY those steps
|
||||
SHOULD be performed.
|
||||
|
||||
Rationale: This enables us to coordinate the timing of the first
|
||||
and second stage pushes and minimize the time during which the
|
||||
archive is desynchronized. This is especially important for mirrors
|
||||
that are involved in a round robin or GeoDNS setup.
|
||||
|
||||
The minimum arguments the script has to understand are:
|
||||
sync:stage1 Only sync stage1
|
||||
sync:stage2 Only sync stage2
|
||||
sync:all Do everything. Default if none of stage1/2 are
|
||||
present.
|
||||
There are more possible arguments, for a complete list see the
|
||||
ftpsync script in our git repository.
|
||||
|
||||
|
||||
|
||||
ftpsync
|
||||
=======
|
||||
|
||||
This script is based on the old anonftpsync script. It has been rewritten
|
||||
to add flexibilty and fix a number of outstanding issues.
|
||||
|
||||
Some of the advantages of the new version are:
|
||||
- Nearly every aspect is configurable
|
||||
- Correct support for multiple pushes
|
||||
- Support for multi-stage archive synchronisations
|
||||
- Support for hook scripts at various points
|
||||
- Support for multiple archives, even if they are pushed using one ssh key
|
||||
- Support for multi-hop, multi-stage archive synchronisations
|
||||
|
||||
Correct support for multiple pushes
|
||||
-----------------------------------
|
||||
When the script receives a second push while it is running and syncing
|
||||
the archive it won't ignore it. Instead it will rerun the
|
||||
synchronisation step to ensure the archive is correctly synchronised.
|
||||
|
||||
Scripts that fail to do that risk ending up with an inconsistent archive.
|
||||
|
||||
|
||||
Can do multi-stage archive synchronisations
|
||||
-------------------------------------------
|
||||
The script can be told to only perform the first or second stage of the
|
||||
archive synchronisation.
|
||||
|
||||
This enables us to send all the binary packages and sources to a
|
||||
number of mirrors, and then tell all of them to sync the
|
||||
Packages/Release files at once. This will keep the timeframe in which
|
||||
the mirrors are out of sync very small and will greatly help things like
|
||||
DNS RR entries or even the planned GeoDNS setup.
|
||||
|
||||
|
||||
Multi-hop, multi-stage archive synchronisations
|
||||
-----------------------------------------------
|
||||
The script can be told to perform a multi-hop multi-stage archive
|
||||
synchronisation.
|
||||
|
||||
This is basically the same as the multi-stage synchronisation
|
||||
explained above, but enables the downstream mirror to push his own
|
||||
staged/multi-hop downstreams before returning. This has the same
|
||||
advantage than the multi-stage synchronisation but allows us to do
|
||||
this over multiple level of mirrors. (Imagine one push going from
|
||||
Europe to Australia, where then locally 3 others get updated before
|
||||
stage2 is sent out. Instead of 4times transferring data from Europe to
|
||||
Australia, just to have them all updated near instantly).
|
||||
|
||||
|
||||
Can run hook scripts
|
||||
--------------------
|
||||
ftpsync currently allows 5 hook scripts to run at various points of the
|
||||
mirror sync run.
|
||||
|
||||
Hook1: After lock is acquired, before first rsync
|
||||
Hook2: After first rsync, if successful
|
||||
Hook3: After second rsync, if successful
|
||||
Hook4: Right before leaf mirror triggering
|
||||
Hook5: After leaf mirror trigger (only if we have slave mirrors; HUB=true)
|
||||
|
||||
Note that Hook3 and Hook4 are likely to be called directly after each other.
|
||||
The difference is that Hook3 is called *every* time the second rsync
|
||||
succeeds even if the mirroring needs to re-run due to a second push.
|
||||
Hook4 is only executed if mirroring is completed.
|
||||
|
||||
|
||||
Support for multiple archives, even if they are pushed using one ssh key
|
||||
------------------------------------------------------------------------
|
||||
If you get multiple archives from your upstream mirror (say Debian,
|
||||
Debian-Backports and Volatile), previously you had to use 3 different ssh
|
||||
keys to be able to automagically synchronize them. This script can do it
|
||||
all with just one key, if your upstream mirror tells you which archive.
|
||||
See "Commandline/SSH options" below for further details.
|
||||
|
||||
|
||||
For details of all available options, please see the extensive documentation
|
||||
in the sample configuration file.
|
||||
|
||||
|
||||
Commandline/SSH options
|
||||
=======================
|
||||
Script options may be set either on the local command line, or passed by
|
||||
specifying an ssh "command". Local commandline options always have
|
||||
precedence over the SSH_ORIGINAL_COMMAND ones.
|
||||
|
||||
Currently this script understands the options listed below. To make them
|
||||
take effect they MUST be prepended by "sync:".
|
||||
|
||||
Option Behaviour
|
||||
stage1 Only do stage1 sync
|
||||
stage2 Only do stage2 sync
|
||||
all Do a complete sync (default)
|
||||
mhop Do a multi-hop sync
|
||||
archive:foo Sync archive foo (if the file $HOME/etc/ftpsync-foo.conf
|
||||
exists and is configured)
|
||||
callback Call back when done (needs proper ssh setup for this to
|
||||
work). It will always use the "command" callback:$HOSTNAME
|
||||
where $HOSTNAME is the one defined in config and
|
||||
will happen before slave mirrors are triggered.
|
||||
|
||||
So, to get the script to sync all of the archive behind bpo and call back when
|
||||
it is complete, use an upstream trigger of
|
||||
ssh $USER@$HOST sync:all sync:archive:bpo sync:callback
|
||||
|
||||
|
||||
Mirror trace files
|
||||
==================
|
||||
Every mirror needs to have a 'trace' file under project/trace.
|
||||
The file format is as follows:
|
||||
|
||||
The filename has to be the full hostname (eg. hostname -f), or in the
|
||||
case of a mirror participating in RR DNS (where users will never use
|
||||
the hostname) the name of the DNS RR entry, eg. security.debian.org
|
||||
for the security rotation)
|
||||
|
||||
The content has (no leading spaces):
|
||||
Sat Nov 8 13:20:22 UTC 2008
|
||||
Used ftpsync version: 42
|
||||
Running on host: steffani.debian.org
|
||||
|
||||
First line: Output of date -u
|
||||
Second line: Freeform text containing the program name and version
|
||||
Third line: Text "Running on host: " followed by hostname -f
|
||||
|
||||
The third line MUST NOT be the DNS RR name, even if the mirror is part
|
||||
of it. It MUST BE the hosts own name. This is in contrast to the filename,
|
||||
which SHOULD be the DNS RR name.
|
||||
|
||||
|
||||
runmirrors
|
||||
==========
|
||||
This script is used to tell leaf mirrors that it is time to synchronize
|
||||
their copy of the archive. This is done by parsing a mirror list and
|
||||
using ssh to "push" the leaf nodes. You can read much more about the
|
||||
principle behind the push at [1], essentially it tells the receiving
|
||||
end to run a pre-defined script. As the whole setup is extremely limited
|
||||
and the ssh key is not usable for anything else than the pre-defined
|
||||
script this is the most secure method for such an action.
|
||||
|
||||
This script supports two types of pushes: The normal single stage push,
|
||||
as well as the newer multi-stage push.
|
||||
|
||||
The normal push, as described above, will simply push the leaf node and
|
||||
then go on with the other nodes.
|
||||
|
||||
The multi-staged push first pushes a mirror and tells it to only do a
|
||||
stage1 sync run. Then it waits for the mirror (and all others being pushed
|
||||
in the same run) to finish that run, before it tells all of the staged
|
||||
mirrors to do the stage2 sync.
|
||||
|
||||
This way you can do a nearly-simultaneous update of multiple hosts.
|
||||
This is useful in situations where periods of desynchronization should
|
||||
be kept as small as possible. Examples of scenarios where this might be
|
||||
useful include multiple hosts in a DNS Round Robin entry.
|
||||
|
||||
For details on the mirror list please see the documented
|
||||
runmirrors.mirror.sample file.
|
||||
|
||||
|
||||
[1] http://blog.ganneff.de/blog/2007/12/29/ssh-triggers.html
|
|
@ -0,0 +1,62 @@
|
|||
#!/usr/bin/perl
|
||||
# Uses symlinks to merge the files contained in a set of vcs
|
||||
# checkouts to into a single directory. Keeps track of when files are
|
||||
# removed from the merged directories and removes the symlinks.
|
||||
#
|
||||
# Only merges files that match the specified pattern.
|
||||
#
|
||||
# Note that the directories given to merge should be paths that will work
|
||||
# for symlink targets from the destination directory (so either full paths,
|
||||
# or they should be right inside the destination directory).
|
||||
#
|
||||
# Note that other files in the destination directory will be left as-is.
|
||||
#
|
||||
# Copyright 2006 by Joey Hess, licensed under the GPL.
|
||||
|
||||
if (! @ARGV) {
|
||||
die "usage: dircombine include-pattern dest dir1 [dir2 ...]\n";
|
||||
}
|
||||
|
||||
my $pattern=shift;
|
||||
my $dest=shift;
|
||||
|
||||
foreach my $dir (@ARGV) {
|
||||
my %known;
|
||||
|
||||
# Link in each thing from the dir.
|
||||
opendir(DIR, $dir) || die "opendir: $!";
|
||||
while ($_=readdir(DIR)) {
|
||||
next if $_ eq '.' || $_ eq '..' || $_ eq 'known' || $_ eq '.svn' || $_ eq '.git' || $_ eq '.gitignore' || $_ eq '_darcs';
|
||||
next unless /$pattern/;
|
||||
|
||||
$known{$_}=1;
|
||||
|
||||
if (! -l "$dest/$_" && -e "$dest/$_") {
|
||||
print STDERR "$_ in $dir is also in $dest\n";
|
||||
}
|
||||
elsif (! -l "$dest/$_") {
|
||||
system("ln", "-svf", "$dir/$_", $dest);
|
||||
}
|
||||
}
|
||||
closedir(DIR);
|
||||
|
||||
# Remove anything that was previously linked in but is not in the
|
||||
# dir anymore.
|
||||
if (-e "$dir/known") {
|
||||
open(KNOWN, "$dir/known") || die "open $dir/known: $!";
|
||||
while (<KNOWN>) {
|
||||
chomp;
|
||||
if (! $known{$_}) {
|
||||
system("rm", "-vf", "$dest/$_");
|
||||
}
|
||||
}
|
||||
close KNOWN;
|
||||
}
|
||||
|
||||
# Save state for next time.
|
||||
open(KNOWN, ">$dir/known") || die "write $dir/known: $!";
|
||||
foreach my $file (sort keys %known) {
|
||||
print KNOWN "$file\n";
|
||||
}
|
||||
close KNOWN;
|
||||
}
|
|
@ -0,0 +1,585 @@
|
|||
#! /bin/bash
|
||||
# No, we can not deal with sh alone.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
# ERR traps should be inherited from functions too. (And command
|
||||
# substitutions and subshells and whatnot, but for us the function is
|
||||
# the important part here)
|
||||
set -E
|
||||
|
||||
# ftpsync script for Debian
|
||||
# Based losely on a number of existing scripts, written by an
|
||||
# unknown number of different people over the years.
|
||||
#
|
||||
# Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License as
|
||||
# published by the Free Software Foundation; version 2.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
||||
|
||||
# In case the admin somehow wants to have this script located someplace else,
|
||||
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
|
||||
# How the admin sets this isn't our place to deal with. One could use a wrapper
|
||||
# for that. Or pam_env. Or whatever fits in the local setup. :)
|
||||
BASEDIR=${BASEDIR:-"${HOME}"}
|
||||
|
||||
# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
|
||||
# by Joerg Jaspert and the Debian mirroradm group.
|
||||
# This is used to track which mirror is using which script version.
|
||||
VERSION="80387"
|
||||
|
||||
# Source our common functions
|
||||
. "${BASEDIR}/etc/common"
|
||||
|
||||
########################################################################
|
||||
########################################################################
|
||||
## functions ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
# We want to be able to get told what kind of sync we should do. This
|
||||
# might be anything, from the archive to sync, the stage to do, etc. A
|
||||
# list of currently understood and valid options is below. Multiple
|
||||
# options are seperated by space. All the words have to have the word
|
||||
# sync: in front or nothing will get used!
|
||||
#
|
||||
# Option Behaviour
|
||||
# stage1 Only do stage1 sync
|
||||
# stage2 Only do stage2 sync
|
||||
# all Do a complete sync
|
||||
# mhop Do a mhop sync, usually additionally to stage1
|
||||
# archive:foo Sync archive foo (if config for foo is available)
|
||||
# callback Call back when done (needs proper ssh setup for this to
|
||||
# work). It will always use the "command" callback:$HOSTNAME
|
||||
# where $HOSTNAME is the one defined below/in config and
|
||||
# will happen before slave mirrors are triggered.
|
||||
#
|
||||
# So to get us to sync all of the archive behind bpo and call back when
|
||||
# we are done, a trigger command of
|
||||
# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
|
||||
# trick.
|
||||
check_commandline() {
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
sync:stage1)
|
||||
SYNCSTAGE1="true"
|
||||
SYNCALL="false"
|
||||
;;
|
||||
sync:stage2)
|
||||
SYNCSTAGE2="true"
|
||||
SYNCALL="false"
|
||||
;;
|
||||
sync:callback)
|
||||
SYNCCALLBACK="true"
|
||||
;;
|
||||
sync:archive:*)
|
||||
ARCHIVE=${1##sync:archive:}
|
||||
# We do not like / or . in the remotely supplied archive name.
|
||||
ARCHIVE=${ARCHIVE//\/}
|
||||
ARCHIVE=${ARCHIVE//.}
|
||||
;;
|
||||
sync:all)
|
||||
SYNCALL="true"
|
||||
;;
|
||||
sync:mhop)
|
||||
SYNCMHOP="true"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown option ${1} ignored"
|
||||
;;
|
||||
esac
|
||||
shift # Check next set of parameters.
|
||||
done
|
||||
}
|
||||
|
||||
# All the stuff we want to do when we exit, no matter where
|
||||
cleanup() {
|
||||
trap - ERR TERM HUP INT QUIT EXIT
|
||||
# all done. Mail the log, exit.
|
||||
log "Mirrorsync done";
|
||||
|
||||
# Lets get a statistical value
|
||||
SPEED="unknown"
|
||||
if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then
|
||||
SPEED=$(
|
||||
SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log")
|
||||
set "nothing" ${SPEEDLINE}
|
||||
echo ${8:-""}
|
||||
)
|
||||
if [ -n "${SPEED}" ]; then
|
||||
SPEED=${SPEED%%.*}
|
||||
SPEED=$(( $SPEED / 1024 ))
|
||||
fi
|
||||
fi
|
||||
log "Rsync transfer speed: ${SPEED} KB/s"
|
||||
|
||||
if [ -n "${MAILTO}" ]; then
|
||||
# In case rsync had something on stderr
|
||||
if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
|
||||
mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
|
||||
fi
|
||||
if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
|
||||
# And the normal log
|
||||
MAILFILES="${LOG}"
|
||||
if [ "x${FULLLOGS}x" = "xtruex" ]; then
|
||||
# Someone wants full logs including rsync
|
||||
MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
|
||||
fi
|
||||
cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
|
||||
fi
|
||||
fi
|
||||
|
||||
savelog "${LOGDIR}/rsync-${NAME}.log"
|
||||
savelog "${LOGDIR}/rsync-${NAME}.error"
|
||||
savelog "$LOG" > /dev/null
|
||||
|
||||
rm -f "${LOCK}"
|
||||
}
|
||||
|
||||
# Check rsyncs return value
|
||||
check_rsync() {
|
||||
ret=$1
|
||||
msg=$2
|
||||
|
||||
# 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
|
||||
# and us re-running. If it's not, uplink is broken anyways.
|
||||
case "${ret}" in
|
||||
0) return 0;;
|
||||
24) return 0;;
|
||||
23) return 2;;
|
||||
30) return 2;;
|
||||
*)
|
||||
error "ERROR: ${msg}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
|
||||
# As what are we called?
|
||||
NAME="$(basename $0)"
|
||||
# The original command line arguments need to be saved!
|
||||
if [ $# -gt 0 ]; then
|
||||
ORIGINAL_COMMAND=$*
|
||||
else
|
||||
ORIGINAL_COMMAND=""
|
||||
fi
|
||||
|
||||
SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
|
||||
# Now, check if we got told about stuff via ssh
|
||||
if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
|
||||
# We deliberately add "nothing" and ignore it right again, to avoid
|
||||
# people from outside putting some set options in the first place,
|
||||
# making us parse them...
|
||||
set "nothing" "${SSH_ORIGINAL_COMMAND}"
|
||||
shift
|
||||
# Yes, unqouted $* here. Or the function will only see it as one
|
||||
# parameter, which doesnt help the case in it.
|
||||
check_commandline $*
|
||||
fi
|
||||
|
||||
# Now, we can locally override all the above variables by just putting
|
||||
# them into the .ssh/authorized_keys file forced command.
|
||||
if [ -n "${ORIGINAL_COMMAND}" ]; then
|
||||
set ${ORIGINAL_COMMAND}
|
||||
check_commandline $*
|
||||
fi
|
||||
|
||||
# If we have been told to do stuff for a different archive than default,
|
||||
# set the name accordingly.
|
||||
ARCHIVE=${ARCHIVE:-""}
|
||||
if [ -n "${ARCHIVE}" ]; then
|
||||
NAME="${NAME}-${ARCHIVE}"
|
||||
fi
|
||||
|
||||
# Now source the config for the archive we run on.
|
||||
# (Yes, people can also overwrite the options above in the config file
|
||||
# if they want to)
|
||||
if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then
|
||||
. "${BASEDIR}/etc/${NAME}.conf"
|
||||
else
|
||||
echo "Nono, you can't tell us about random archives. Bad boy!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
########################################################################
|
||||
# Config options go here. Feel free to overwrite them in the config #
|
||||
# file if you need to. #
|
||||
# On debian.org machines the defaults should be ok. #
|
||||
# #
|
||||
# The following extra variables can be defined in the config file: #
|
||||
# #
|
||||
# ARCH_EXCLUDE #
|
||||
# can be used to exclude a complete architecture from #
|
||||
# mirrorring. Use as space seperated list. #
|
||||
# Possible values are: #
|
||||
# alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, #
|
||||
# mipsel, mips, powerpc, s390, sparc, kfreebsd-i386, kfreebsd-amd64 #
|
||||
# and source. #
|
||||
# eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" #
|
||||
# #
|
||||
# An unset value will mirror all architectures #
|
||||
########################################################################
|
||||
|
||||
########################################################################
|
||||
# There should be nothing to edit here, use the config file #
|
||||
########################################################################
|
||||
MIRRORNAME=${MIRRORNAME:-$(hostname -f)}
|
||||
# Where to put logfiles in
|
||||
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
|
||||
# Our own logfile
|
||||
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
|
||||
|
||||
# Where should we put all the mirrored files?
|
||||
TO=${TO:-"/org/ftp.debian.org/ftp/"}
|
||||
|
||||
# used by log() and error()
|
||||
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
|
||||
|
||||
# Where to send mails about mirroring to?
|
||||
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
|
||||
# We are not on a debian.org host
|
||||
MAILTO=${MAILTO:-"root"}
|
||||
else
|
||||
# Yay, on a .debian.org host
|
||||
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
|
||||
fi
|
||||
# Want errors only or every log?
|
||||
ERRORSONLY=${ERRORSONLY:-"true"}
|
||||
# Want full logs, ie. including the rsync one?
|
||||
FULLLOGS=${FULLLOGS:-"false"}
|
||||
|
||||
# How many logfiles to keep
|
||||
LOGROTATE=${LOGROTATE:-14}
|
||||
|
||||
# Our lockfile
|
||||
LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
|
||||
# timeout for the lockfile, in case we have bash older than v4 (and no /proc)
|
||||
LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
|
||||
# Do we need another rsync run?
|
||||
UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
|
||||
# Trace file for mirror stats and checks (make sure we get full hostname)
|
||||
TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}
|
||||
|
||||
# rsync program
|
||||
RSYNC=${RSYNC:-rsync}
|
||||
# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
|
||||
# excluded files
|
||||
RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"}
|
||||
# limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited
|
||||
RSYNC_BW=${RSYNC_BW:-0}
|
||||
# Default rsync options for *every* rsync call
|
||||
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
|
||||
# Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files
|
||||
RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"}
|
||||
# Options for the second pass, where we do want everything, including deletion of old and now unused files
|
||||
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
|
||||
# Which rsync share to use on our upstream mirror?
|
||||
RSYNC_PATH=${RSYNC_PATH:-"ftp"}
|
||||
|
||||
# Now add the bwlimit option. As default is 0 we always add it, rsync interprets
|
||||
# 0 as unlimited, so this is safe.
|
||||
RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
|
||||
|
||||
# Connect from mirror.csclub
|
||||
RSYNC_OPTIONS="--address=129.97.134.71 ${RSYNC_OPTIONS}"
|
||||
|
||||
# We have no default host to sync from, but will error out if its unset
|
||||
RSYNC_HOST=${RSYNC_HOST:-""}
|
||||
# Error out if we have no host to sync from
|
||||
if [ -z "${RSYNC_HOST}" ]; then
|
||||
error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf"
|
||||
fi
|
||||
|
||||
# our username for the rsync share
|
||||
RSYNC_USER=${RSYNC_USER:-""}
|
||||
# the password
|
||||
RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
|
||||
|
||||
# a possible proxy
|
||||
RSYNC_PROXY=${RSYNC_PROXY:-""}
|
||||
|
||||
# Do we sync stage1?
|
||||
SYNCSTAGE1=${SYNCSTAGE1:-"false"}
|
||||
# Do we sync stage2?
|
||||
SYNCSTAGE2=${SYNCSTAGE2:-"false"}
|
||||
# Do we sync all?
|
||||
SYNCALL=${SYNCALL:-"true"}
|
||||
# Do we have a mhop sync?
|
||||
SYNCMHOP=${SYNCMHOP:-"false"}
|
||||
# Do we callback?
|
||||
SYNCCALLBACK=${SYNCCALLBACK:-"false"}
|
||||
# If we call back we need some more options defined in the config file.
|
||||
CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
|
||||
CALLBACKHOST=${CALLBACKHOST:-"none"}
|
||||
CALLBACKKEY=${CALLBACKKEY:-"none"}
|
||||
|
||||
# General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that!
|
||||
EXCLUDE=${EXCLUDE:-""}
|
||||
|
||||
# The temp directory used by rsync --delay-updates is not
|
||||
# world-readable remotely. Always exclude it to avoid errors.
|
||||
EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
|
||||
|
||||
SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""}
|
||||
ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
|
||||
# Exclude architectures defined in $ARCH_EXCLUDE
|
||||
for ARCH in ${ARCH_EXCLUDE}; do
|
||||
EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
|
||||
if [ "${ARCH}" = "source" ]; then
|
||||
if [ -z ${SOURCE_EXCLUDE} ]; then
|
||||
SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
# Hooks
|
||||
HOOK1=${HOOK1:-""}
|
||||
HOOK2=${HOOK2:-""}
|
||||
HOOK3=${HOOK3:-""}
|
||||
HOOK4=${HOOK4:-""}
|
||||
HOOK5=${HOOK5:-""}
|
||||
|
||||
# Are we a hub?
|
||||
HUB=${HUB:-"false"}
|
||||
|
||||
########################################################################
|
||||
# Really nothing to see below here. Only code follows. #
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
# Some sane defaults
|
||||
cd "${BASEDIR}"
|
||||
umask 022
|
||||
|
||||
# If we are here for the first time, create the
|
||||
# destination and the trace directory
|
||||
mkdir -p "${TO}/project/trace"
|
||||
|
||||
# Used to make sure we will have the archive fully and completly synced before
|
||||
# we stop, even if we get multiple pushes while this script is running.
|
||||
# Otherwise we can end up with a half-synced archive:
|
||||
# - get a push
|
||||
# - sync, while locked
|
||||
# - get another push. Of course no extra sync run then happens, we are locked.
|
||||
# - done. Archive not correctly synced, we don't have all the changes from the second push.
|
||||
touch "${UPDATEREQUIRED}"
|
||||
|
||||
# Check to see if another sync is in progress
|
||||
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
|
||||
if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then
|
||||
# We have a recent enough bash version, lets do it the easy way,
|
||||
# the lock will contain the right pid, thanks to $BASHPID
|
||||
if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
|
||||
# Process does either not exist or is not owned by us.
|
||||
echo "$$" > "${LOCK}"
|
||||
else
|
||||
echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
# Old bash, means we dont have the right pid in our lockfile
|
||||
# So take a different way - guess if it is still there by comparing its age.
|
||||
# Not optimal, but hey.
|
||||
stamptime=$(date --reference="${LOCK}" +%s)
|
||||
unixtime=$(date +%s)
|
||||
difference=$(( $unixtime - $stamptime ))
|
||||
if [ ${difference} -ge ${LOCKTIMEOUT} ]; then
|
||||
# Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
|
||||
echo "$$" > "${LOCK}"
|
||||
else
|
||||
echo "Unable to start rsync, lock file younger than one hour"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# When we exit normally we call cleanup on our own. Otherwise we want it called by
|
||||
# this trap. (We can not trap on EXIT, because that is called when the main script
|
||||
# exits. Which also happens when we background the mainroutine, ie. while we still
|
||||
# run!)
|
||||
trap cleanup ERR TERM HUP INT QUIT
|
||||
|
||||
# Start log by redirecting stdout and stderr there and closing stdin
|
||||
exec >"$LOG" 2>&1 <&-
|
||||
log "Mirrorsync start"
|
||||
|
||||
# Look who pushed us and note that in the log.
|
||||
PUSHFROM="${SSH_CONNECTION%%\ *}"
|
||||
if [ -n "${PUSHFROM}" ]; then
|
||||
log "We got pushed from ${PUSHFROM}"
|
||||
fi
|
||||
|
||||
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
|
||||
if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
|
||||
SYNCCALLBACK="false"
|
||||
error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
|
||||
fi
|
||||
fi
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=1
|
||||
HOOKSCR=${HOOK1}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
# Now, we might want to sync from anonymous too.
|
||||
# This is that deep in this script so hook1 could, if wanted, change things!
|
||||
if [ -z ${RSYNC_USER} ]; then
|
||||
RSYNCPTH="${RSYNC_HOST}"
|
||||
else
|
||||
RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
|
||||
fi
|
||||
|
||||
# Now do the actual mirroring, and run as long as we have an updaterequired file.
|
||||
export RSYNC_PASSWORD
|
||||
export RSYNC_PROXY
|
||||
|
||||
while [ -e "${UPDATEREQUIRED}" ]; do
|
||||
log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
|
||||
|
||||
# if we want stage1 *or* all
|
||||
if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
|
||||
while [ -e "${UPDATEREQUIRED}" ]; do
|
||||
rm -f "${UPDATEREQUIRED}"
|
||||
log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
|
||||
|
||||
set +e
|
||||
# Step one, sync everything except Packages/Releases
|
||||
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
|
||||
${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
|
||||
result=$?
|
||||
set -e
|
||||
|
||||
log "Back from rsync with returncode ${result}"
|
||||
done
|
||||
else
|
||||
# Fake a good resultcode
|
||||
result=0
|
||||
fi # Sync stage 1?
|
||||
rm -f "${UPDATEREQUIRED}"
|
||||
|
||||
set +e
|
||||
check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
|
||||
GO=$?
|
||||
set -e
|
||||
if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
|
||||
log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
|
||||
elif [ ${GO} -ne 0 ]; then
|
||||
exit 3
|
||||
fi
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=2
|
||||
HOOKSCR=${HOOK2}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
# if we want stage2 *or* all
|
||||
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
|
||||
log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
|
||||
|
||||
set +e
|
||||
# We are lucky, it worked. Now do step 2 and sync again, this time including
|
||||
# the packages/releases files
|
||||
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
|
||||
${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
|
||||
result=$?
|
||||
set -e
|
||||
|
||||
log "Back from rsync with returncode ${result}"
|
||||
else
|
||||
# Fake a good resultcode
|
||||
result=0
|
||||
fi # Sync stage 2?
|
||||
|
||||
set +e
|
||||
check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
|
||||
GO=$?
|
||||
set -e
|
||||
if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
|
||||
log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
|
||||
elif [ ${GO} -ne 0 ]; then
|
||||
exit 4
|
||||
fi
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=3
|
||||
HOOKSCR=${HOOK3}
|
||||
)
|
||||
hook $HOOK
|
||||
done
|
||||
|
||||
# We only update our tracefile when we had a stage2 or an all sync.
|
||||
# Otherwise we would update it after stage1 already, which is wrong.
|
||||
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
|
||||
if [ -d "$(dirname "${TO}/${TRACE}")" ]; then
|
||||
LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
|
||||
echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
|
||||
echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
|
||||
fi
|
||||
fi
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=4
|
||||
HOOKSCR=${HOOK4}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
|
||||
set +e
|
||||
callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
|
||||
set -e
|
||||
fi
|
||||
|
||||
# Remove the Archive-Update-in-Progress file before we push our downstreams.
|
||||
rm -f "${LOCK}"
|
||||
|
||||
if [ x${HUB} = "xtrue" ]; then
|
||||
# Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
|
||||
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
|
||||
RUNMIRRORARGS=""
|
||||
if [ -n "${ARCHIVE}" ]; then
|
||||
# We tell runmirrors about the archive we are running on.
|
||||
RUNMIRRORARGS="-a ${ARCHIVE}"
|
||||
fi
|
||||
# We also tell runmirrors that we are running it from within ftpsync, so it can change
|
||||
# the way it works with mhop based on that.
|
||||
RUNMIRRORARGS="${RUNMIRRORARGS} -f"
|
||||
|
||||
if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
|
||||
# This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
|
||||
# not get to this point.
|
||||
# So if that happens, tell runmirrors we are doing mhop
|
||||
RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
|
||||
elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
|
||||
RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
|
||||
elif [ "xtruex" = "x${SYNCALL}x" ]; then
|
||||
RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
|
||||
fi
|
||||
log "Trigger slave mirrors using ${RUNMIRRORARGS}"
|
||||
${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
|
||||
log "Trigger slave done"
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=5
|
||||
HOOKSCR=${HOOK5}
|
||||
)
|
||||
hook $HOOK
|
||||
fi
|
||||
fi
|
||||
|
||||
# All done, lets call cleanup
|
||||
cleanup
|
|
@ -0,0 +1,112 @@
|
|||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
set -u
|
||||
|
||||
# psuhpdo script for Debian
|
||||
#
|
||||
# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License as
|
||||
# published by the Free Software Foundation; version 2.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
||||
|
||||
# In case the admin somehow wants to have this script located someplace else,
|
||||
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
|
||||
BASEDIR=${BASEDIR:-"${HOME}"}
|
||||
|
||||
NAME="`basename $0`"
|
||||
|
||||
|
||||
# Read our config file
|
||||
. "${BASEDIR}/etc/${NAME}.conf"
|
||||
|
||||
# Source our common functions
|
||||
. "${BASEDIR}/etc/common"
|
||||
|
||||
# Set sane defaults if the configfile didn't do that for us.
|
||||
# The directory for our logfiles
|
||||
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
|
||||
# Our own logfile
|
||||
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
|
||||
# How many logfiles to keep
|
||||
LOGROTATE=${LOGROTATE:-14}
|
||||
# Our mirrorfile
|
||||
MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
|
||||
# used by log()
|
||||
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
|
||||
# extra ssh options we might want hostwide
|
||||
SSH_OPTS=${SSH_OPTS:-""}
|
||||
# Which ssh key to use?
|
||||
KEYFILE=${KEYFILE:-".ssh/pushpackages"}
|
||||
# which path to "mirror"
|
||||
MIRRORPATH=${MIRRORPATH:-"/org/packages.debian.org/mirror/"}
|
||||
# where to send mails to
|
||||
if [ "x$(hostname -s)x" != "x${MIRRORNAME%%.debian.org}x" ]; then
|
||||
# We are not on a debian.org host
|
||||
MAILTO=${MAILTO:-"root"}
|
||||
else
|
||||
# Yay, on a .debian.org host
|
||||
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
|
||||
fi
|
||||
|
||||
if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
|
||||
error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# Some sane defaults
|
||||
cd ${BASEDIR}
|
||||
umask 022
|
||||
|
||||
# Make sure we have our log and lock directories
|
||||
mkdir -p "${LOGDIR}"
|
||||
|
||||
trap 'log "Pdopush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT
|
||||
|
||||
log "Pushing pdo mirrors" >> ${LOG}
|
||||
|
||||
# From here on we do *NOT* want to exit on errors. We don't want to
|
||||
# stop pushing mirrors just because we can't reach one of them.
|
||||
set +e
|
||||
|
||||
|
||||
# Now read our mirrorfile and push the mirrors defined in there.
|
||||
# We use grep to easily sort out all lines having a # in front of them or are empty.
|
||||
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
|
||||
while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
|
||||
# Process the two options that can be left blank in the config
|
||||
if [ -z ${MPROTO} ]; then
|
||||
MPROTO=2
|
||||
fi
|
||||
if [ -z ${MKEYFILE} ]; then
|
||||
MKEYFILE="${BASEDIR}/${KEYFILE}"
|
||||
fi
|
||||
# Now, people can do stupid things and leave out the protocol, but
|
||||
# define a keyfile...
|
||||
if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then
|
||||
error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG}
|
||||
continue
|
||||
fi
|
||||
|
||||
# And finally, push the mirror
|
||||
log "Pushing ${MLNAME}" >> ${LOG}
|
||||
# This needs a limited ssh key on the other side, something like
|
||||
# no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa.....
|
||||
rsync -e "ssh -i ${MKEYFILE} -${MPROTO} ${SSH_OPTS}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log"
|
||||
log "Pushing ${MLNAME} done" >> ${LOG}
|
||||
savelog ${LOGDIR}${MLNAME}.log
|
||||
|
||||
set +e
|
||||
done
|
||||
|
||||
exit 0
|
|
@ -0,0 +1,286 @@
|
|||
#! /bin/bash
|
||||
|
||||
set -e
|
||||
set -u
|
||||
|
||||
# runmirrors script for Debian
|
||||
# Based losely on existing scripts, written by an unknown number of
|
||||
# different people over the years.
|
||||
#
|
||||
# Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License as
|
||||
# published by the Free Software Foundation; version 2.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
||||
|
||||
# In case the admin somehow wants to have this script located someplace else,
|
||||
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
|
||||
BASEDIR=${BASEDIR:-"${HOME}"}
|
||||
|
||||
NAME="$(basename $0)"
|
||||
|
||||
HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n
|
||||
Usage:\n\n
|
||||
|
||||
1.) a single parameter with NO leading -.\n
|
||||
\t This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n
|
||||
\t have us look for ${NAME}-security.{conf,mirror} files.\n\n
|
||||
|
||||
2.) using getopt style parameters:\n
|
||||
\t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n
|
||||
\t -k [TYPE] - Type of push. all, stage2, mhop. Default mhop.\n
|
||||
\t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n
|
||||
\t -h - Print this help and exit
|
||||
"
|
||||
# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle
|
||||
# it will not start with a -. If we find oldstyle we assume its only one, the config
|
||||
# name we run on.
|
||||
if [ $# -gt 0 ]; then
|
||||
if [ "x${1:0:1}x" != "x-x" ]; then
|
||||
# Yes, does not start with a -, so use it for the config name.
|
||||
CONF=${1:-""}
|
||||
if [ -n "${CONF}" ]; then
|
||||
NAME="${NAME}-${CONF}"
|
||||
fi
|
||||
else
|
||||
# Yeah well, new style, starting with - for getopts
|
||||
while getopts ':a:k:fh' OPTION ; do
|
||||
case $OPTION in
|
||||
a) CONF="${OPTARG}"
|
||||
if [ -n "${CONF}" ]; then
|
||||
NAME="${NAME}-${CONF}"
|
||||
fi
|
||||
;;
|
||||
k) PUSHKIND="${OPTARG}"
|
||||
;;
|
||||
f) FROMFTPSYNC="true"
|
||||
;;
|
||||
h) echo -e $HELP
|
||||
exit 0
|
||||
;;
|
||||
|
||||
*) echo "Invalid usage"
|
||||
echo -e $HELP
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
fi
|
||||
# Make sure the values are always defined, even if there was no commandline option
|
||||
# for them
|
||||
# Default config is empty
|
||||
CONF=${CONF:-""}
|
||||
|
||||
# Set the default to all, if we didnt get told about it. Currently
|
||||
# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1,
|
||||
# stage2 - staged push, second phase. Default is mhop.
|
||||
PUSHKIND=${PUSHKIND:-"mhop"}
|
||||
|
||||
# If we are pushed from within ftpsync. Default false.
|
||||
FROMFTPSYNC=${FROMFTPSYNC:-"false"}
|
||||
|
||||
########################################################################
|
||||
# Read our config file
|
||||
. "${BASEDIR}/etc/${NAME}.conf"
|
||||
|
||||
# Source our common functions
|
||||
. "${BASEDIR}/etc/common"
|
||||
|
||||
# Set sane defaults if the configfile didn't do that for us.
|
||||
# The directory for our logfiles
|
||||
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
|
||||
# Our own logfile
|
||||
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
|
||||
# Our lockfile directory
|
||||
LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"}
|
||||
# How many logfiles to keep
|
||||
LOGROTATE=${LOGROTATE:-14}
|
||||
# Our mirrorfile
|
||||
MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
|
||||
# used by log()
|
||||
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
|
||||
# extra ssh options we might want hostwide
|
||||
SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"}
|
||||
# Whats our archive name? We will also tell our leafs about it
|
||||
PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"}
|
||||
# How long to wait for mirrors to do stage1 if we have multi-stage syncing
|
||||
PUSHDELAY=${PUSHDELAY:-600}
|
||||
# Which ssh key to use?
|
||||
KEYFILE=${KEYFILE:-".ssh/pushmirror"}
|
||||
# where to send mails to
|
||||
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
|
||||
# We are not on a debian.org host
|
||||
MAILTO=${MAILTO:-"root"}
|
||||
else
|
||||
# Yay, on a .debian.org host
|
||||
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
|
||||
fi
|
||||
|
||||
if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
|
||||
error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> "${LOG}"
|
||||
exit 5
|
||||
fi
|
||||
|
||||
# Hooks
|
||||
HOOK1=${HOOK1:-""}
|
||||
HOOK2=${HOOK2:-""}
|
||||
HOOK3=${HOOK3:-""}
|
||||
|
||||
########################################################################
|
||||
|
||||
# Some sane defaults
|
||||
cd "${BASEDIR}"
|
||||
umask 022
|
||||
|
||||
# Make sure we have our log and lock directories
|
||||
mkdir -p "${LOGDIR}"
|
||||
mkdir -p "${LOCKDIR}"
|
||||
|
||||
trap 'log "Mirrorpush done" >> "${LOG}"; savelog "${LOG}" > /dev/null' EXIT
|
||||
|
||||
log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> "${LOG}"
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=1
|
||||
HOOKSCR=${HOOK1}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
# From here on we do *NOT* want to exit on errors. We don't want to
|
||||
# stop pushing mirrors just because we can't reach one of them.
|
||||
set +e
|
||||
|
||||
# Built up our list of 2-stage mirrors.
|
||||
PUSHLOCKS=""
|
||||
PUSHLOCKS=$(get2stage)
|
||||
|
||||
# In case we have it - remove. It is used to synchronize multi-stage mirroring
|
||||
rm -f "${LOCKDIR}/all_stage1"
|
||||
|
||||
# Now read our mirrorfile and push the mirrors defined in there.
|
||||
# We use grep to easily sort out all lines having a # in front of them or are empty.
|
||||
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
|
||||
while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
|
||||
if [ "x${MTYPE}x" = "xDELAYx" ]; then
|
||||
# We should wait a bit.
|
||||
if [ -z ${MLNAME} ]; then
|
||||
MLNAME=600
|
||||
fi
|
||||
log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
|
||||
sleep ${MLNAME}
|
||||
continue
|
||||
fi
|
||||
|
||||
# If we are told we have a mhop sync to do and are called from within ftpsync,
|
||||
# we will only look at staged/mhop entries and ignore the rest.
|
||||
if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then
|
||||
if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then
|
||||
continue
|
||||
fi
|
||||
fi
|
||||
|
||||
# Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken
|
||||
# as a set of options to give to ssh, we pass it without doing anything with it.
|
||||
# If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use,
|
||||
# and also means we look if there is one value more after a space. That value would then
|
||||
# be the ssh keyfile we use with -i. That gives us full flexibility for all
|
||||
# ssh options but doesn't destroy backwards compatibility.
|
||||
# If it is empty we assume proto 2 and the default keyfile.
|
||||
#
|
||||
# There is one bug in here. We will give out the master keyfile, even if there is a
|
||||
# "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the
|
||||
# target server. In the case both keys do some actions- the first one presented wins.
|
||||
# And this might not be what one wants.
|
||||
#
|
||||
# The only sane way to go around this, i think, is by dropping backward compability.
|
||||
# Which I don't really like.
|
||||
if [ -n "${MSSHOPT}" ]; then
|
||||
# So its not empty, lets check if it starts with a - and as such is a "new-style"
|
||||
# ssh options set.
|
||||
if [ "x${MSSHOPT:0:1}x" = "x-x" ]; then
|
||||
# Yes we start with a -
|
||||
SSHOPT="${MSSHOPT}"
|
||||
MPROTO="99"
|
||||
MKEYFILE="${BASEDIR}/${KEYFILE}"
|
||||
elif [ ${MSSHOPT:0:1} -eq 1 ] || [ ${MSSHOPT:0:1} -eq 2 ]; then
|
||||
# We do seem to have oldstyle options here.
|
||||
MPROTO=${MSSHOPT:0:1}
|
||||
MKEYFILE=${MSSHOPT:1}
|
||||
SSHOPT=""
|
||||
else
|
||||
error "I don't know what is configured for mirror ${MLNAME}"
|
||||
continue
|
||||
fi
|
||||
else
|
||||
MPROTO=2
|
||||
MKEYFILE="${BASEDIR}/${KEYFILE}"
|
||||
SSHOPT=""
|
||||
fi
|
||||
|
||||
# Built our array
|
||||
SIGNAL_OPTS=(
|
||||
MIRROR="${MLNAME}"
|
||||
HOSTNAME="${MHOSTNAME}"
|
||||
USERNAME="${MUSER}"
|
||||
SSHPROTO="${MPROTO}"
|
||||
SSHKEY="${MKEYFILE}"
|
||||
SSHOPTS="${SSHOPT/ /#}"
|
||||
PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
|
||||
PUSHTYPE="${MTYPE}"
|
||||
PUSHARCHIVE=${PUSHARCHIVE}
|
||||
PUSHKIND=${PUSHKIND}
|
||||
FROMFTPSYNC=${FROMFTPSYNC}
|
||||
)
|
||||
|
||||
# And finally, push the mirror
|
||||
log "Trigger ${MLNAME}" >> "${LOG}"
|
||||
signal "${SIGNAL_OPTS}" &
|
||||
log "Trigger for ${MLNAME} done" >> "${LOG}"
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=2
|
||||
HOOKSCR=${HOOK2}
|
||||
)
|
||||
hook $HOOK
|
||||
set +e
|
||||
done
|
||||
|
||||
# If we are run from within ftpsync *and* have an mhop push to send on, we have
|
||||
# to wait until the push is gone through and they all returned, or we will exit
|
||||
# much too early.
|
||||
# As the signal routine touches $LOCKDIR/all_stage1 when all are done, its
|
||||
# easy enough just to wait for that to appear. Of course we do the same game
|
||||
# with PUSHDELAY to not wait forever.
|
||||
if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
|
||||
tries=0
|
||||
# We do not wait forever
|
||||
while [ ${tries} -lt ${PUSHDELAY} ]; do
|
||||
if [ -f "${LOCKDIR}/all_stage1" ]; then
|
||||
break
|
||||
fi
|
||||
tries=$((tries + 5))
|
||||
sleep 5
|
||||
done
|
||||
|
||||
if [ ${tries} -ge ${PUSHDELAY} ]; then
|
||||
error "Failed to wait for our mirrors when sending mhop push down." >> "${LOG}"
|
||||
fi
|
||||
fi
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=3
|
||||
HOOKSCR=${HOOK3}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
exit 0
|
|
@ -0,0 +1,168 @@
|
|||
#!/usr/bin/perl -wT
|
||||
|
||||
# Copyright (c) 2006 Anthony Towns <ajt@debian.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation; either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
|
||||
use strict;
|
||||
use Fcntl ':flock';
|
||||
use File::Find;
|
||||
use POSIX qw(strftime);
|
||||
|
||||
# configuration:
|
||||
|
||||
my $local_dir = "/srv/ftp.debian.org/mirror";
|
||||
my $rsync_host = undef; #"merkel.debian.org";
|
||||
my $rsync_dir = undef; #"debian";
|
||||
|
||||
my $dest = "/srv/ftp.debian.org/rsync/typical";
|
||||
my $max_del = 1000;
|
||||
|
||||
$ENV{"PATH"} = "/bin:/usr/bin";
|
||||
|
||||
# program
|
||||
|
||||
my $hostname = `/bin/hostname -f`;
|
||||
die "bad hostname" unless $hostname =~ m/^([a-zA-Z0-9._-]+)/;
|
||||
$hostname = $1;
|
||||
|
||||
my $lockfile = "./Archive-Update-in-Progress-$hostname";
|
||||
|
||||
unless (open LKFILE, "> $dest/$lockfile" and flock(LKFILE, LOCK_EX)) {
|
||||
print "$hostname is unable to start sync, lock file exists\n";
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (defined $rsync_host && defined $rsync_dir) {
|
||||
system("rsync --links --hard-links --times --verbose --recursive"
|
||||
." --delay-updates --files-from :indices/files/typical.files"
|
||||
." rsync://$rsync_host/$rsync_dir/ $dest/");
|
||||
} else {
|
||||
open FILELIST, "< $local_dir/indices/files/typical.files"
|
||||
or die "typical.files index not found";
|
||||
while (<FILELIST>) {
|
||||
chomp;
|
||||
m/^(.*)$/; $_ = $1;
|
||||
my @l = lstat("$local_dir/$_");
|
||||
next unless (@l);
|
||||
|
||||
if (-l _) {
|
||||
my $lpath = readlink("$local_dir/$_");
|
||||
$lpath =~ m/^(.*)$/; $lpath = $1;
|
||||
if (-l "$dest/$_") {
|
||||
next if ($lpath eq readlink("$dest/$_"));
|
||||
}
|
||||
|
||||
unless (mk_dirname_as_dirs($dest, $_)) {
|
||||
print "E: couldn't create path for $_\n";
|
||||
next;
|
||||
}
|
||||
|
||||
if (-d "$dest/$_") {
|
||||
rename "$dest/$_", "$dest/$_.remove" or print "E: couldn't rename old dir $_ out of the way\n";
|
||||
} elsif (-e "$dest/$_") {
|
||||
unlink("$dest/$_") or print "E: couldn't unlink $_\n";
|
||||
}
|
||||
symlink($lpath, "$dest/$_") or print "E: couldn't create $_ as symlink to $lpath\n";
|
||||
next;
|
||||
}
|
||||
|
||||
next if (-d _);
|
||||
|
||||
unless (mk_dirname_as_dirs($dest, $_)) {
|
||||
print "E: couldn't create path for $_\n";
|
||||
next;
|
||||
}
|
||||
|
||||
my @d = lstat("$dest/$_");
|
||||
if (@d) {
|
||||
if (-d _) {
|
||||
rename("$dest/$_", "$dest/$_.remove") or print "E: couldn't rename old dir $_ out of the way\n";
|
||||
} else {
|
||||
next if (@l and @d and $l[0] == $d[0] and $l[1] == $d[1]);
|
||||
#next if (@l and @d and $l[7] == $d[7]);
|
||||
print "I: updating $_\n";
|
||||
unlink("$dest/$_");
|
||||
}
|
||||
}
|
||||
|
||||
link("$local_dir/$_", "$dest/$_") or print "E: couldn't link $_\n";
|
||||
}
|
||||
close(FILELIST);
|
||||
}
|
||||
|
||||
print "Files synced, now deleting any unnecessary files\n";
|
||||
|
||||
my %expected_files = ();
|
||||
open FILES, "< $dest/indices/files/typical.files"
|
||||
or die "typical.files index not found";
|
||||
while (<FILES>) {
|
||||
chomp;
|
||||
$expected_files{$_} = 1;
|
||||
}
|
||||
close(FILES);
|
||||
|
||||
chdir($dest);
|
||||
|
||||
my $del_count = 0;
|
||||
my $last = '';
|
||||
finddepth({wanted => \&wanted, no_chdir => 1}, ".");
|
||||
|
||||
open TRACE, "> $dest/project/trace/$hostname" or die "couldn't open trace";
|
||||
print TRACE strftime("%a %b %e %H:%M:%S UTC %Y", gmtime) . "\n";
|
||||
close TRACE;
|
||||
|
||||
close LKFILE;
|
||||
unlink("$dest/$lockfile");
|
||||
exit(0);
|
||||
|
||||
sub wanted {
|
||||
my ($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_);
|
||||
if (-d _) {
|
||||
if (substr($last, 0, length($_) + 1) ne "$_/") {
|
||||
print "Deleting empty directory: $_\n";
|
||||
$_ = m/^(.*)$/;
|
||||
my $f = $1;
|
||||
rmdir($f);
|
||||
} else {
|
||||
$last = $_;
|
||||
}
|
||||
} elsif ($_ =~ m|^\./project/trace/| or $_ eq $lockfile) {
|
||||
$last = $_;
|
||||
} elsif (defined $expected_files{$_}) {
|
||||
$last = $_;
|
||||
} elsif ($del_count < $max_del) {
|
||||
$del_count++;
|
||||
print "Deleting file: $_\n";
|
||||
$_ = m/^(.*)$/;
|
||||
my $f = $1;
|
||||
unlink($f);
|
||||
}
|
||||
}
|
||||
|
||||
sub mk_dirname_as_dirs {
|
||||
my ($base, $file) = @_;
|
||||
while ($file =~ m,^/*([^/]+)/+([^/].*)$,) {
|
||||
$file = $2;
|
||||
$base = "$base/$1";
|
||||
my @blah = lstat($base);
|
||||
if (!@blah) {
|
||||
mkdir($base, 0777);
|
||||
} elsif (-l _ or ! -d _) {
|
||||
print "SHOULD BE A DIRECTORY: $base\n";
|
||||
unlink($base);
|
||||
mkdir($base, 0777);
|
||||
}
|
||||
}
|
||||
1;
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
unset LC_CTYPE
|
||||
LANG=C
|
||||
HOST=`hostname -f`
|
||||
|
||||
cd ${HOME}/archvsync
|
||||
git pull
|
||||
|
||||
cd ${HOME}
|
||||
~/archvsync/bin/dircombine . . archvsync/ >/dev/null 2>&1
|
|
@ -0,0 +1,304 @@
|
|||
#! /bin/bash
|
||||
# No, we can not deal with sh alone.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
# ERR traps should be inherited from functions too. (And command
|
||||
# substitutions and subshells and whatnot, but for us the function is
|
||||
# the important part here)
|
||||
set -E
|
||||
|
||||
# websync script for Debian
|
||||
# Based losely on the old websync written by an
|
||||
# unknown number of different people over the years and ftpsync.
|
||||
#
|
||||
# Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org>
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or
|
||||
# modify it under the terms of the GNU General Public License as
|
||||
# published by the Free Software Foundation; version 2.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
# General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
|
||||
|
||||
# In case the admin somehow wants to have this script located someplace else,
|
||||
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
|
||||
# How the admin sets this isn't our place to deal with. One could use a wrapper
|
||||
# for that. Or pam_env. Or whatever fits in the local setup. :)
|
||||
BASEDIR=${BASEDIR:-"${HOME}"}
|
||||
|
||||
# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
|
||||
# by Joerg Jaspert and the Debian mirroradm group.
|
||||
# This is used to track which mirror is using which script version.
|
||||
VERSION="0815"
|
||||
|
||||
# Source our common functions
|
||||
. "${BASEDIR}/etc/common"
|
||||
|
||||
########################################################################
|
||||
########################################################################
|
||||
## functions ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
# All the stuff we want to do when we exit, no matter where
|
||||
cleanup() {
|
||||
trap - ERR TERM HUP INT QUIT EXIT
|
||||
# all done. Mail the log, exit.
|
||||
log "Mirrorsync done";
|
||||
if [ -n "${MAILTO}" ]; then
|
||||
# In case rsync had something on stderr
|
||||
if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
|
||||
mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
|
||||
fi
|
||||
if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
|
||||
# And the normal log
|
||||
MAILFILES="${LOG}"
|
||||
if [ "x${FULLLOGS}x" = "xtruex" ]; then
|
||||
# Someone wants full logs including rsync
|
||||
MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
|
||||
fi
|
||||
cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] web sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
|
||||
fi
|
||||
fi
|
||||
|
||||
savelog "${LOGDIR}/rsync-${NAME}.log"
|
||||
savelog "${LOGDIR}/rsync-${NAME}.error"
|
||||
savelog "$LOG" > /dev/null
|
||||
|
||||
rm -f "${LOCK}"
|
||||
}
|
||||
|
||||
|
||||
# Check rsyncs return value
|
||||
check_rsync() {
|
||||
|
||||
ret=$1
|
||||
msg=$2
|
||||
|
||||
# 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
|
||||
# and us re-running. If it's not, uplink is broken anyways.
|
||||
case "${ret}" in
|
||||
0) return 0;;
|
||||
24) return 0;;
|
||||
23) return 2;;
|
||||
30) return 2;;
|
||||
*)
|
||||
error "ERROR: ${msg}"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
# As what are we called?
|
||||
NAME="`basename $0`"
|
||||
|
||||
# Now source the config.
|
||||
. "${BASEDIR}/etc/${NAME}.conf"
|
||||
|
||||
########################################################################
|
||||
# Config options go here. Feel free to overwrite them in the config #
|
||||
# file if you need to. #
|
||||
# On debian.org machines the defaults should be ok. #
|
||||
########################################################################
|
||||
|
||||
########################################################################
|
||||
# There should be nothing to edit here, use the config file #
|
||||
########################################################################
|
||||
MIRRORNAME=${MIRRORNAME:-`hostname -f`}
|
||||
# Where to put logfiles in
|
||||
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
|
||||
# Our own logfile
|
||||
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
|
||||
|
||||
# Where should we put all the mirrored files?
|
||||
TO=${TO:-"/org/www.debian.org/www"}
|
||||
|
||||
# used by log() and error()
|
||||
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
|
||||
|
||||
# Where to send mails about mirroring to?
|
||||
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
|
||||
# We are not on a debian.org host
|
||||
MAILTO=${MAILTO:-"root"}
|
||||
else
|
||||
# Yay, on a .debian.org host
|
||||
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
|
||||
fi
|
||||
# Want errors only or every log?
|
||||
ERRORSONLY=${ERRORSONLY:-"true"}
|
||||
# Want full logs, ie. including the rsync one?
|
||||
FULLLOGS=${FULLLOGS:-"false"}
|
||||
|
||||
# How many logfiles to keep
|
||||
LOGROTATE=${LOGROTATE:-14}
|
||||
|
||||
# Our lockfile
|
||||
LOCK=${LOCK:-"${TO}/Website-Update-in-Progress-${MIRRORNAME}"}
|
||||
# Do we need another rsync run?
|
||||
UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}"
|
||||
# Trace file for mirror stats and checks (make sure we get full hostname)
|
||||
TRACE=${TRACE:-".project/trace/${MIRRORNAME}"}
|
||||
|
||||
# rsync program
|
||||
RSYNC=${RSYNC:-rsync}
|
||||
# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
|
||||
# excluded files
|
||||
RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Website-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Website-Update-Required-${MIRRORNAME}"}
|
||||
# Default rsync options for *every* rsync call
|
||||
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
|
||||
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
|
||||
# Which rsync share to use on our upstream mirror?
|
||||
RSYNC_PATH=${RSYNC_PATH:-"web.debian.org"}
|
||||
|
||||
# our username for the rsync share
|
||||
RSYNC_USER=${RSYNC_USER:-""}
|
||||
# the password
|
||||
RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
|
||||
|
||||
# a possible proxy
|
||||
RSYNC_PROXY=${RSYNC_PROXY:-""}
|
||||
|
||||
# General excludes.
|
||||
EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"}
|
||||
|
||||
# The temp directory used by rsync --delay-updates is not
|
||||
# world-readable remotely. Always exclude it to avoid errors.
|
||||
EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
|
||||
|
||||
# And site specific excludes, by default its the sponsor stuff that should be local to all (except templates)
|
||||
SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"}
|
||||
|
||||
# Hooks
|
||||
HOOK1=${HOOK1:-""}
|
||||
HOOK2=${HOOK2:-""}
|
||||
HOOK3=${HOOK3:-""}
|
||||
HOOK4=${HOOK4:-""}
|
||||
|
||||
# Are we a hub?
|
||||
HUB=${HUB:-"false"}
|
||||
|
||||
# Some sane defaults
|
||||
cd "${BASEDIR}"
|
||||
umask 022
|
||||
|
||||
# If we are here for the first time, create the
|
||||
# destination and the trace directory
|
||||
mkdir -p "${TO}/.project/trace"
|
||||
|
||||
# Used to make sure we will have the archive fully and completly synced before
|
||||
# we stop, even if we get multiple pushes while this script is running.
|
||||
# Otherwise we can end up with a half-synced archive:
|
||||
# - get a push
|
||||
# - sync, while locked
|
||||
# - get another push. Of course no extra sync run then happens, we are locked.
|
||||
# - done. Archive not correctly synced, we don't have all the changes from the second push.
|
||||
touch "${UPDATEREQUIRED}"
|
||||
|
||||
# Check to see if another sync is in progress
|
||||
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
|
||||
if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
|
||||
# Process does either not exist or is not owned by us.
|
||||
echo "$$" > "${LOCK}"
|
||||
else
|
||||
echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
trap cleanup EXIT ERR TERM HUP INT QUIT
|
||||
|
||||
# Start log by redirecting everything there.
|
||||
exec >"$LOG" 2>&1 </dev/null
|
||||
|
||||
# Look who pushed us and note that in the log.
|
||||
log "Mirrorsync start"
|
||||
PUSHFROM="${SSH_CONNECTION%%\ *}"
|
||||
if [ -n "${PUSHFROM}" ]; then
|
||||
log "We got pushed from ${PUSHFROM}"
|
||||
fi
|
||||
log "Acquired main lock"
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=1
|
||||
HOOKSCR=${HOOK1}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
# Now, we might want to sync from anonymous too.
|
||||
# This is that deep in this script so hook1 could, if wanted, change things!
|
||||
if [ -z ${RSYNC_USER} ]; then
|
||||
RSYNCPTH="${RSYNC_HOST}"
|
||||
else
|
||||
RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
|
||||
fi
|
||||
|
||||
# Now do the actual mirroring, and run as long as we have an updaterequired file.
|
||||
export RSYNC_PASSWORD
|
||||
export RSYNC_PROXY
|
||||
|
||||
while [ -e "${UPDATEREQUIRED}" ]; do
|
||||
log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
|
||||
|
||||
rm -f "${UPDATEREQUIRED}"
|
||||
log "Syncing: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
|
||||
|
||||
set +e
|
||||
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SITE_FILTER} \
|
||||
${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
|
||||
result=$?
|
||||
set -e
|
||||
|
||||
log "Back from rsync with returncode ${result}"
|
||||
|
||||
set +e
|
||||
check_rsync $result "Sync went wrong, got errorcode ${result}. Logfile: ${LOG}"
|
||||
GO=$?
|
||||
set -e
|
||||
|
||||
if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
|
||||
log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
|
||||
elif [ ${GO} -ne 0 ]; then
|
||||
exit 3
|
||||
fi
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=2
|
||||
HOOKSCR=${HOOK2}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
done
|
||||
|
||||
mkdir -p "${TO}/.project/trace"
|
||||
LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
|
||||
echo "Used websync version: ${VERSION}" >> "${TO}/${TRACE}"
|
||||
echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=3
|
||||
HOOKSCR=${HOOK3}
|
||||
)
|
||||
hook $HOOK
|
||||
|
||||
if [ x${HUB} = "xtrue" ]; then
|
||||
log "Trigger slave mirrors"
|
||||
${BASEDIR}/bin/runmirrors "websync"
|
||||
log "Trigger slave done"
|
||||
|
||||
HOOK=(
|
||||
HOOKNR=4
|
||||
HOOKSCR=${HOOK4}
|
||||
)
|
||||
hook $HOOK
|
||||
fi
|
||||
|
||||
# All done, rest is done by cleanup hook.
|
|
@ -0,0 +1,230 @@
|
|||
# -*- mode:sh -*-
|
||||
# Little common functions
|
||||
|
||||
# push a mirror attached to us.
|
||||
# Arguments (using an array named SIGNAL_OPTS):
|
||||
#
|
||||
# $MIRROR - Name for the mirror, also basename for the logfile
|
||||
# $HOSTNAME - Hostname to push to
|
||||
# $USERNAME - Username there
|
||||
# $SSHPROTO - Protocol version, either 1 or 2.
|
||||
# $SSHKEY - the ssh private key file to use for this push
|
||||
# $SSHOPTS - any other option ssh accepts, passed blindly, be careful
|
||||
# $PUSHLOCKOWN - own lockfile name to touch after stage1 in pushtype=staged
|
||||
# $PUSHTYPE - what kind of push should be done?
|
||||
# all - normal, just push once with ssh backgrounded and finish
|
||||
# staged - staged. first push stage1, then wait for $PUSHLOCKs to appear,
|
||||
# then push stage2
|
||||
# $PUSHARCHIVE - what archive to sync? (Multiple mirrors behind one ssh key!)
|
||||
# $PUSHCB - do we want a callback?
|
||||
# $PUSHKIND - whats going on? are we doing mhop push or already stage2?
|
||||
# $FROMFTPSYNC - set to true if we run from within ftpsync.
|
||||
#
|
||||
# This function assumes that the variable LOG is set to a directory where
|
||||
# logfiles can be written to.
|
||||
# Additionally $PUSHLOCKS has to be defined as a set of space delimited strings
|
||||
# (list of "lock"files) to wait for if you want pushtype=staged
|
||||
#
|
||||
# Pushes might be done in background (for type all).
|
||||
signal () {
|
||||
ARGS="SIGNAL_OPTS[*]"
|
||||
local ${!ARGS}
|
||||
|
||||
MIRROR=${MIRROR:-""}
|
||||
HOSTNAME=${HOSTNAME:-""}
|
||||
USERNAME=${USERNAME:-""}
|
||||
SSHPROTO=${SSHPROTO:-""}
|
||||
SSHKEY=${SSHKEY:-""}
|
||||
SSHOPTS=${SSHOPTS:-""}
|
||||
PUSHLOCKOWN=${PUSHLOCKOWN:-""}
|
||||
PUSHTYPE=${PUSHTYPE:-"all"}
|
||||
PUSHARCHIVE=${PUSHARCHIVE:-""}
|
||||
PUSHCB=${PUSHCB:-""}
|
||||
PUSHKIND=${PUSHKIND:-"all"}
|
||||
FROMFTPSYNC=${FROMFTPSYNC:-"false"}
|
||||
|
||||
# And now get # back to space...
|
||||
SSHOPTS=${SSHOPTS/\#/ }
|
||||
|
||||
# Defaults we always want, no matter what
|
||||
SSH_OPTIONS="-o user=${USERNAME} -o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
|
||||
|
||||
# If there are userdefined ssh options, add them.
|
||||
if [ -n "${SSH_OPTS}" ]; then
|
||||
SSH_OPTIONS="${SSH_OPTIONS} ${SSH_OPTS}"
|
||||
fi
|
||||
|
||||
# Does this machine need a special key?
|
||||
if [ -n "${SSHKEY}" ]; then
|
||||
SSH_OPTIONS="${SSH_OPTIONS} -i ${SSHKEY}"
|
||||
fi
|
||||
|
||||
# Does this machine have an extra own set of ssh options?
|
||||
if [ -n "${SSHOPTS}" ]; then
|
||||
SSH_OPTIONS="${SSH_OPTIONS} ${SSHOPTS}"
|
||||
fi
|
||||
|
||||
# Set the protocol version
|
||||
if [ ${SSHPROTO} -ne 1 ] && [ ${SSHPROTO} -ne 2 ] && [ ${SSHPROTO} -ne 99 ]; then
|
||||
# Idiots, we only want 1 or 2. Cant decide? Lets force 2.
|
||||
SSHPROTO=2
|
||||
fi
|
||||
if [ -n "${SSHPROTO}" ] && [ ${SSHPROTO} -ne 99 ]; then
|
||||
SSH_OPTIONS="${SSH_OPTIONS} -${SSHPROTO}"
|
||||
fi
|
||||
|
||||
date -u >> "${LOGDIR}/${MIRROR}.log"
|
||||
|
||||
PUSHARGS=""
|
||||
# PUSHARCHIVE empty or not, we always add the sync:archive: command to transfer.
|
||||
# Otherwise, if nothing else is added, ssh -f would not work ("no command to execute")
|
||||
# But ftpsync does treat "sync:archive:" as the main archive, so this works nicely.
|
||||
PUSHARGS="${PUSHARGS} sync:archive:${PUSHARCHIVE}"
|
||||
|
||||
# We have a callback wish, tell downstreams
|
||||
if [ -n "${PUSHCB}" ]; then
|
||||
PUSHARGS="${PUSHARGS} sync:callback"
|
||||
fi
|
||||
# If we are running an mhop push AND our downstream is one to receive it, tell it.
|
||||
if [ "xmhopx" = "x${PUSHKIND}x" ] && [ "xmhopx" = "x${PUSHTYPE}x" ]; then
|
||||
PUSHARGS="${PUSHARGS} sync:mhop"
|
||||
fi
|
||||
|
||||
if [ "xallx" = "x${PUSHTYPE}x" ]; then
|
||||
# Default normal "fire and forget" push. We background that, we do not care about the mirrors doings
|
||||
echo "Sending normal push" >> "${LOGDIR}/${MIRROR}.log"
|
||||
PUSHARGS1="sync:all"
|
||||
ssh -f $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log"
|
||||
elif [ "xstagedx" = "x${PUSHTYPE}x" ] || [ "xmhopx" = "x${PUSHTYPE}x" ]; then
|
||||
# Want a staged push. Fine, lets do that. Not backgrounded. We care about the mirrors doings.
|
||||
echo "Sending staged push" >> "${LOGDIR}/${MIRROR}.log"
|
||||
|
||||
# Only send stage1 if we havent already send it. When called with stage2, we already did.
|
||||
if [ "xstage2x" != "x${PUSHKIND}x" ]; then
|
||||
# Step1: Do a push to only sync stage1, do not background
|
||||
PUSHARGS1="sync:stage1"
|
||||
ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS1}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
|
||||
touch "${PUSHLOCKOWN}"
|
||||
|
||||
# Step2: Wait for all the other "lock"files to appear.
|
||||
tries=0
|
||||
# We do not wait forever
|
||||
while [ ${tries} -lt ${PUSHDELAY} ]; do
|
||||
total=0
|
||||
found=0
|
||||
for file in ${PUSHLOCKS}; do
|
||||
total=$((total + 1))
|
||||
if [ -f ${file} ]; then
|
||||
found=$((found + 1))
|
||||
fi
|
||||
done
|
||||
if [ ${total} -eq ${found} ] || [ -f "${LOCKDIR}/all_stage1" ]; then
|
||||
touch "${LOCKDIR}/all_stage1"
|
||||
break
|
||||
fi
|
||||
tries=$((tries + 5))
|
||||
sleep 5
|
||||
done
|
||||
# In case we did not have all PUSHLOCKS and still continued, note it
|
||||
# This is a little racy, especially if the other parts decide to do this
|
||||
# at the same time, but it wont hurt more than a mail too much, so I don't care much
|
||||
if [ ${tries} -ge ${PUSHDELAY} ]; then
|
||||
echo "Failed to wait for all other mirrors. Failed ones are:" >> "${LOGDIR}/${MIRROR}.log"
|
||||
for file in ${PUSHLOCKS}; do
|
||||
if [ ! -f ${file} ]; then
|
||||
echo "${file}" >> "${LOGDIR}/${MIRROR}.log"
|
||||
error "Missing Pushlockfile ${file} after waiting ${tries} second, continuing"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
rm -f "${PUSHLOCKOWN}"
|
||||
fi
|
||||
|
||||
# Step3: It either timed out or we have all the "lock"files, do the rest
|
||||
# If we are doing mhop AND are called from ftpsync - we now exit.
|
||||
# That way we notify our uplink that we and all our clients are done with their
|
||||
# stage1. It can then finish its own, and if all our upstreams downlinks are done,
|
||||
# it will send us stage2.
|
||||
# If we are not doing mhop or are not called from ftpsync, we start stage2
|
||||
if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
|
||||
return
|
||||
else
|
||||
PUSHARGS2="sync:stage2"
|
||||
echo "Now doing the second stage push" >> "${LOGDIR}/${MIRROR}.log"
|
||||
ssh $SSH_OPTIONS "${HOSTNAME}" "${PUSHARGS} ${PUSHARGS2}" >>"${LOGDIR}/${MIRROR}.log" 2>&1
|
||||
fi
|
||||
else
|
||||
# Can't decide? Then you get nothing.
|
||||
return
|
||||
fi
|
||||
}
|
||||
|
||||
# callback, used by ftpsync
|
||||
callback () {
|
||||
# Defaults we always want, no matter what
|
||||
SSH_OPTIONS="-o BatchMode=yes -o ServerAliveInterval=45 -o ConnectTimeout=45 -o PasswordAuthentication=no"
|
||||
ssh $SSH_OPTIONS -i "$3" -o"user $1" "$2" callback:${HOSTNAME}
|
||||
}
|
||||
|
||||
# log something (basically echo it together with a timestamp)
|
||||
#
|
||||
# Set $PROGRAM to a string to have it added to the output.
|
||||
log () {
|
||||
if [ -z "${PROGRAM}" ]; then
|
||||
echo "$(date +"%b %d %H:%M:%S") $(hostname -s) [$$] $@"
|
||||
else
|
||||
echo "$(date +"%b %d %H:%M:%S") $(hostname -s) ${PROGRAM}[$$]: $@"
|
||||
fi
|
||||
}
|
||||
|
||||
# log the message using log() but then also send a mail
|
||||
# to the address configured in MAILTO (if non-empty)
|
||||
error () {
|
||||
log "$@"
|
||||
if [ -z "${MAILTO}" ]; then
|
||||
echo "$@" | mail -e -s "[$PROGRAM@$(hostname -s)] ERROR [$$]" ${MAILTO}
|
||||
fi
|
||||
}
|
||||
|
||||
# run a hook
|
||||
# needs array variable HOOK setup with HOOKNR being a number an HOOKSCR
|
||||
# the script to run.
|
||||
hook () {
|
||||
ARGS='HOOK[@]'
|
||||
local "${!ARGS}"
|
||||
if [ -n "${HOOKSCR}" ]; then
|
||||
log "Running hook $HOOKNR: ${HOOKSCR}"
|
||||
set +e
|
||||
${HOOKSCR}
|
||||
result=$?
|
||||
set -e
|
||||
log "Back from hook $HOOKNR, got returncode ${result}"
|
||||
return $result
|
||||
else
|
||||
return 0
|
||||
fi
|
||||
}
|
||||
|
||||
# Return the list of 2-stage mirrors.
|
||||
get2stage() {
|
||||
egrep '^(staged|mhop)' "${MIRRORS}" | {
|
||||
while read MTYPE MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
|
||||
PUSHLOCKS="${LOCKDIR}/${MLNAME}.stage1 ${PUSHLOCKS}"
|
||||
done
|
||||
echo "$PUSHLOCKS"
|
||||
}
|
||||
}
|
||||
|
||||
# Rotate logfiles
|
||||
savelog() {
|
||||
torotate="$1"
|
||||
count=${2:-${LOGROTATE}}
|
||||
while [ ${count} -gt 0 ]; do
|
||||
prev=$(( count - 1 ))
|
||||
if [ -e "${torotate}.${prev}" ]; then
|
||||
mv "${torotate}.${prev}" "${torotate}.${count}"
|
||||
fi
|
||||
count=$prev
|
||||
done
|
||||
mv "${torotate}" "${torotate}.0"
|
||||
}
|
|
@ -0,0 +1,137 @@
|
|||
## Mirrorname. This is used for things like the trace file and should always
|
||||
## be the full hostname of the mirror.
|
||||
MIRRORNAME=mirror.csclub.uwaterloo.ca
|
||||
|
||||
## Destination of the mirrored files. Should be an empty directory.
|
||||
## CAREFUL, this directory will contain the mirror. Everything else
|
||||
## that might have happened to be in there WILL BE GONE after the mirror sync!
|
||||
TO="/mirror/root/debian"
|
||||
|
||||
## The upstream name of the rsync share.
|
||||
RSYNC_PATH="debian"
|
||||
|
||||
## The host we mirror from
|
||||
RSYNC_HOST="ftp.ca.debian.org"
|
||||
|
||||
## In case we need a user to access the rsync share at our upstream host
|
||||
#RSYNC_USER=
|
||||
|
||||
## If we need a user we also need a password
|
||||
#RSYNC_PASSWORD=
|
||||
|
||||
## In which directory should logfiles end up
|
||||
## Note that BASEDIR defaults to $HOME, but can be set before calling the
|
||||
## ftpsync script to any value you want (for example using pam_env)
|
||||
#LOGDIR="${BASEDIR}/log"
|
||||
|
||||
## Name of our own logfile.
|
||||
## Note that ${NAME} is set by the ftpsync script depending on the way it
|
||||
## is called. See README for a description of the multi-archive capability
|
||||
## and better always include ${NAME} in this path.
|
||||
#LOG="${LOGDIR}/${NAME}.log"
|
||||
|
||||
## The script can send logs (or error messages) to a mail address.
|
||||
## If this is unset it will default to the local root user unless it is run
|
||||
## on a .debian.org machine where it will default to the mirroradm people.
|
||||
MAILTO="mirror"
|
||||
|
||||
## If you do want a mail about every single sync, set this to false
|
||||
## Everything else will only send mails if a mirror sync fails
|
||||
#ERRORSONLY="true"
|
||||
|
||||
## If you want the logs to also include output of rsync, set this to true.
|
||||
## Careful, the logs can get pretty big, especially if it is the first mirror
|
||||
## run
|
||||
#FULLLOGS="false"
|
||||
|
||||
## If you do want to exclude files from the mirror run, put --exclude statements here.
|
||||
## See rsync(1) for the exact syntax, these are passed to rsync as written here.
|
||||
## DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH THIS, IT WILL NOT WORK!
|
||||
#EXCLUDE=""
|
||||
|
||||
## If you do want to exclude an architecture, this is for you.
|
||||
## Use as space seperated list.
|
||||
## Possible values are:
|
||||
## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64,
|
||||
## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh and sparc
|
||||
## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"
|
||||
## An unset value will mirror all architectures (default!)
|
||||
#ARCH_EXCLUDE=""
|
||||
|
||||
## Do we have leaf mirror to signal we are done and they should sync?
|
||||
## If so set it to true and make sure you configure runmirrors.mirrors
|
||||
## and runmirrors.conf for your need.
|
||||
#HUB=false
|
||||
|
||||
## We do create three logfiles for every run. To save space we rotate them, this
|
||||
## defines how many we keep
|
||||
#LOGROTATE=14
|
||||
|
||||
## Our own lockfile (only one sync should run at any time)
|
||||
#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}"
|
||||
|
||||
# Timeout for the lockfile, in case we have bash older than v4 (and no /proc)
|
||||
# LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
|
||||
|
||||
## The following file is used to make sure we will end up with a correctly
|
||||
## synced mirror even if we get multiple pushes in a short timeframe
|
||||
#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
|
||||
|
||||
## The trace file is used by a mirror check tool to see when we last
|
||||
## had a successful mirror sync. Make sure that it always ends up in
|
||||
## project/trace and always shows the full hostname.
|
||||
## This is *relative* to ${TO}
|
||||
#TRACE="project/trace/${MIRRORNAME}"
|
||||
|
||||
## We sync our mirror using rsync (everything else would be insane), so
|
||||
## we need a few options set.
|
||||
## The rsync program
|
||||
#RSYNC=rsync
|
||||
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
|
||||
## Default rsync options every rsync invocation sees.
|
||||
#RSYNC_OPTIONS="-rltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}"
|
||||
|
||||
## Options the first pass gets. We do not want the Packages/Source indices
|
||||
## here, and we also do not want to delete any files yet.
|
||||
#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*"
|
||||
|
||||
## Options the second pass gets. Now we want the Packages/Source indices too
|
||||
## and we also want to delete files. We also want to delete files that are
|
||||
## excluded.
|
||||
#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"
|
||||
|
||||
## You may establish the connection via a web proxy by setting the environment
|
||||
## variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note
|
||||
## that your web proxy's configuration must support proxy connections to port 873.
|
||||
# RSYNC_PROXY=
|
||||
|
||||
|
||||
## The following three options are used in case we want to "callback" the host
|
||||
## we got pushed from.
|
||||
#CALLBACKUSER="archvsync"
|
||||
#CALLBACKHOST="none"
|
||||
#CALLBACKKEY="none"
|
||||
|
||||
|
||||
## Hook scripts can be run at various places during the sync.
|
||||
## Leave them blank if you don't want any
|
||||
## Hook1: After lock is acquired, before first rsync
|
||||
## Hook2: After first rsync, if successful
|
||||
## Hook3: After second rsync, if successful
|
||||
## Hook4: Right before leaf mirror triggering
|
||||
## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true)
|
||||
##
|
||||
## Note that Hook3 and Hook4 are likely to be called directly after each other.
|
||||
## Difference is: Hook3 is called *every* time the second rsync was successful,
|
||||
## but even if the mirroring needs to re-run thanks to a second push.
|
||||
## Hook4 is only effective if we are done with mirroring.
|
||||
#HOOK1=
|
||||
#HOOK2=
|
||||
#HOOK3=
|
||||
#HOOK4=
|
||||
#HOOK5=
|
|
@ -0,0 +1,148 @@
|
|||
########################################################################
|
||||
########################################################################
|
||||
## This is a sample configuration file for the ftpsync mirror script. ##
|
||||
## Most of the values are commented out and just shown here for ##
|
||||
## completeness, together with their default value. ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
## Mirrorname. This is used for things like the trace file and should always
|
||||
## be the full hostname of the mirror.
|
||||
#MIRRORNAME=`hostname -f`
|
||||
|
||||
## Destination of the mirrored files. Should be an empty directory.
|
||||
## CAREFUL, this directory will contain the mirror. Everything else
|
||||
## that might have happened to be in there WILL BE GONE after the mirror sync!
|
||||
#TO="/org/ftp.debian.org/ftp/"
|
||||
|
||||
## The upstream name of the rsync share.
|
||||
#RSYNC_PATH="ftp"
|
||||
|
||||
## The host we mirror from
|
||||
#RSYNC_HOST=some.mirror.debian.org
|
||||
|
||||
## In case we need a user to access the rsync share at our upstream host
|
||||
#RSYNC_USER=
|
||||
|
||||
## If we need a user we also need a password
|
||||
#RSYNC_PASSWORD=
|
||||
|
||||
## In which directory should logfiles end up
|
||||
## Note that BASEDIR defaults to $HOME, but can be set before calling the
|
||||
## ftpsync script to any value you want (for example using pam_env)
|
||||
#LOGDIR="${BASEDIR}/log"
|
||||
|
||||
## Name of our own logfile.
|
||||
## Note that ${NAME} is set by the ftpsync script depending on the way it
|
||||
## is called. See README for a description of the multi-archive capability
|
||||
## and better always include ${NAME} in this path.
|
||||
#LOG="${LOGDIR}/${NAME}.log"
|
||||
|
||||
## The script can send logs (or error messages) to a mail address.
|
||||
## If this is unset it will default to the local root user unless it is run
|
||||
## on a .debian.org machine where it will default to the mirroradm people.
|
||||
#MAILTO="root"
|
||||
|
||||
## If you do want a mail about every single sync, set this to false
|
||||
## Everything else will only send mails if a mirror sync fails
|
||||
#ERRORSONLY="true"
|
||||
|
||||
## If you want the logs to also include output of rsync, set this to true.
|
||||
## Careful, the logs can get pretty big, especially if it is the first mirror
|
||||
## run
|
||||
#FULLLOGS="false"
|
||||
|
||||
## If you do want to exclude files from the mirror run, put --exclude statements here.
|
||||
## See rsync(1) for the exact syntax, these are passed to rsync as written here.
|
||||
## DO NOT TRY TO EXCLUDE ARCHITECTURES OR SUITES WITH THIS, IT WILL NOT WORK!
|
||||
#EXCLUDE=""
|
||||
|
||||
## If you do want to exclude an architecture, this is for you.
|
||||
## Use as space seperated list.
|
||||
## Possible values are:
|
||||
## alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, kfreebsd-amd64,
|
||||
## kfreebsd-i386, m68k, mipsel, mips, powerpc, s390, sh, sparc and source
|
||||
## eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc"
|
||||
## An unset value will mirror all architectures (default!)
|
||||
#ARCH_EXCLUDE=""
|
||||
|
||||
## Do we have leaf mirror to signal we are done and they should sync?
|
||||
## If so set it to true and make sure you configure runmirrors.mirrors
|
||||
## and runmirrors.conf for your need.
|
||||
#HUB=false
|
||||
|
||||
## We do create three logfiles for every run. To save space we rotate them, this
|
||||
## defines how many we keep
|
||||
#LOGROTATE=14
|
||||
|
||||
## Our own lockfile (only one sync should run at any time)
|
||||
#LOCK="${TO}/Archive-Update-in-Progress-${MIRRORNAME}"
|
||||
|
||||
# Timeout for the lockfile, in case we have bash older than v4 (and no /proc)
|
||||
# LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
|
||||
|
||||
## The following file is used to make sure we will end up with a correctly
|
||||
## synced mirror even if we get multiple pushes in a short timeframe
|
||||
#UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
|
||||
|
||||
## The trace file is used by a mirror check tool to see when we last
|
||||
## had a successful mirror sync. Make sure that it always ends up in
|
||||
## project/trace and always shows the full hostname.
|
||||
## This is *relative* to ${TO}
|
||||
#TRACE="project/trace/${MIRRORNAME}"
|
||||
|
||||
## We sync our mirror using rsync (everything else would be insane), so
|
||||
## we need a few options set.
|
||||
## The rsync program
|
||||
#RSYNC=rsync
|
||||
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
|
||||
## limit I/O bandwidth. Value is KBytes per second, unset or 0 means unlimited
|
||||
#RSYNC_BW=""
|
||||
|
||||
## Default rsync options every rsync invocation sees.
|
||||
#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}"
|
||||
|
||||
## Options the first pass gets. We do not want the Packages/Source indices
|
||||
## here, and we also do not want to delete any files yet.
|
||||
#RSYNC_OPTIONS1="--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"
|
||||
|
||||
## Options the second pass gets. Now we want the Packages/Source indices too
|
||||
## and we also want to delete files. We also want to delete files that are
|
||||
## excluded.
|
||||
#RSYNC_OPTIONS2="--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"
|
||||
|
||||
## You may establish the connection via a web proxy by setting the environment
|
||||
## variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note
|
||||
## that your web proxy's configuration must support proxy connections to port 873.
|
||||
# RSYNC_PROXY=
|
||||
|
||||
|
||||
## The following three options are used in case we want to "callback" the host
|
||||
## we got pushed from.
|
||||
#CALLBACKUSER="archvsync"
|
||||
#CALLBACKHOST="none"
|
||||
#CALLBACKKEY="none"
|
||||
|
||||
|
||||
## Hook scripts can be run at various places during the sync.
|
||||
## Leave them blank if you don't want any
|
||||
## Hook1: After lock is acquired, before first rsync
|
||||
## Hook2: After first rsync, if successful
|
||||
## Hook3: After second rsync, if successful
|
||||
## Hook4: Right before leaf mirror triggering
|
||||
## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true)
|
||||
##
|
||||
## Note that Hook3 and Hook4 are likely to be called directly after each other.
|
||||
## Difference is: Hook3 is called *every* time the second rsync was successful,
|
||||
## but even if the mirroring needs to re-run thanks to a second push.
|
||||
## Hook4 is only effective if we are done with mirroring.
|
||||
#HOOK1=
|
||||
#HOOK2=
|
||||
#HOOK3=
|
||||
#HOOK4=
|
||||
#HOOK5=
|
|
@ -0,0 +1,40 @@
|
|||
########################################################################
|
||||
########################################################################
|
||||
## This is a sample configuration file for the runmirror script. ##
|
||||
## Most of the values are commented out and just shown here for ##
|
||||
## completeness, together with their default value. ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
## Which ssh key to use?
|
||||
#KEYFILE=.ssh/pushmirror
|
||||
|
||||
## The directory for our logfiles
|
||||
#LOGDIR="${BASEDIR}/log"
|
||||
|
||||
## Our own logfile
|
||||
#LOG="${LOGDIR}/${NAME}.log"
|
||||
|
||||
## Our lockfile directory
|
||||
#LOCKDIR="${BASEDIR}/locks"
|
||||
|
||||
## We do create a logfile for every run. To save space we rotate it, this
|
||||
## defines how many we keep
|
||||
#LOGROTATE=14
|
||||
|
||||
## Our mirrorfile
|
||||
#MIRRORS="${BASEDIR}/etc/${NAME}.mirror"
|
||||
|
||||
## extra ssh options we might want. *hostwide*
|
||||
#SSH_OPTS=""
|
||||
|
||||
## The script can send logs (or error messages) to a mail address.
|
||||
## If this is unset it will default to the local root user unless it is run
|
||||
## on a .debian.org machine where it will default to the mirroradm people.
|
||||
#MAILTO="root"
|
||||
|
||||
## How long to wait for mirrors to do stage1 if we have multi-stage syncing
|
||||
#PUSHDELAY=240
|
||||
|
||||
## which path to push
|
||||
#MIRRORPATH="/org/packages.debian.org/mirror/"
|
|
@ -0,0 +1,21 @@
|
|||
# Definition of mirror hosts we push.
|
||||
# One mirror per line, with the following fields defined.
|
||||
#
|
||||
# ShortName HostName User SSHProtocol SSHKeyFile
|
||||
#
|
||||
# ShortName will be used as a shorthand in logfile outputs and for the logfile
|
||||
# where every ssh output gets redirected to.
|
||||
#
|
||||
# If no SSHKeyFile is given, the default from the config file applies.
|
||||
# If SSHProtocol is empty, it will default to 2, but if you want to
|
||||
# define a keyfile you HAVE TO set protocol too!
|
||||
#
|
||||
# Examples:
|
||||
#
|
||||
# piatti piatti.debian.org archvsync
|
||||
# One special value is allowed: DELAY
|
||||
# This word has to be on a line itself, followed by a space and a number.
|
||||
# nothing else, not even whitespace. It will trigger a pause of $number
|
||||
# seconds between the two mirrors. If no number is given it defaults to
|
||||
# 60 seconds.
|
||||
piatti piatti.debian.org archvsync
|
|
@ -0,0 +1,53 @@
|
|||
########################################################################
|
||||
########################################################################
|
||||
## This is a sample configuration file for the runmirror script. ##
|
||||
## Most of the values are commented out and just shown here for ##
|
||||
## completeness, together with their default value. ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
## Which ssh key to use?
|
||||
#KEYFILE=.ssh/pushmirror
|
||||
|
||||
## The directory for our logfiles
|
||||
#LOGDIR="${BASEDIR}/log"
|
||||
|
||||
## Our own logfile
|
||||
#LOG="${LOGDIR}/${NAME}.log"
|
||||
|
||||
## Our lockfile directory
|
||||
#LOCKDIR="${BASEDIR}/locks"
|
||||
|
||||
## We do create a logfile for every run. To save space we rotate it, this
|
||||
## defines how many we keep
|
||||
#LOGROTATE=14
|
||||
|
||||
## Our mirrorfile
|
||||
#MIRRORS="${BASEDIR}/etc/${NAME}.mirror"
|
||||
|
||||
## extra ssh options we might want. *hostwide*
|
||||
## By default, ignore ssh key change of leafs
|
||||
#SSH_OPTS="-o StrictHostKeyChecking=no"
|
||||
|
||||
## The script can send logs (or error messages) to a mail address.
|
||||
## If this is unset it will default to the local root user unless it is run
|
||||
## on a .debian.org machine where it will default to the mirroradm people.
|
||||
#MAILTO="root"
|
||||
|
||||
## Whats our archive name? We will also tell our leafs about it
|
||||
## This is usually empty, but if we are called as "runmirrors bpo"
|
||||
## it will default to bpo. This way one runmirrors script can serve
|
||||
## multiple archives, similar to what ftpsync does.
|
||||
#PUSHARCHIVE="${CONF}"
|
||||
|
||||
## How long to wait for mirrors to do stage1 if we have multi-stage syncing
|
||||
#PUSHDELAY=600
|
||||
|
||||
## Hook scripts can be run at various places.
|
||||
## Leave them blank/commented out if you don't want any
|
||||
## Hook1: After reading config, before doing the first real action
|
||||
## Hook2: Between two hosts to push
|
||||
## Hook3: When everything is done
|
||||
#HOOK1=""
|
||||
#HOOK2=""
|
||||
#HOOK3=""
|
|
@ -0,0 +1,72 @@
|
|||
# Definition of mirror hosts we push.
|
||||
# One mirror per line, with the following fields defined.
|
||||
#
|
||||
# Type ShortName HostName User SSHProtocol SSHKeyFile
|
||||
#
|
||||
# ALTERNATIVELY the line may look like
|
||||
#
|
||||
# Type ShortName HostName User -$SOMESSHOPTION
|
||||
#
|
||||
# The fields Type, ShortName, HostName and User are *mandantory*.
|
||||
#
|
||||
# Type is either all, staged or mhop, meaning:
|
||||
# all - do a "normal" push. Trigger them, go on.
|
||||
# staged - do a two-stage push, waiting for them after stage 2(and all others that
|
||||
# are staged) before doing stage2
|
||||
# mhop - send a multi-hop staged push. This will tell the mirror to initiate
|
||||
# a mhop/stage1 push to its staged/mhop mirrors and then exit.
|
||||
# When all mhop got back we then send the stage2 through to them.
|
||||
#
|
||||
# ShortName will be used as a shorthand in logfile outputs and for the logfile
|
||||
# where every ssh output gets redirected to.
|
||||
#
|
||||
# If no SSHKeyFile is given, the default from the config file applies.
|
||||
# If SSHProtocol is empty, it will default to 2, but if you want to
|
||||
# define a keyfile you HAVE TO set protocol too!
|
||||
#
|
||||
# With the ALTERNATIVE syntax you are able to use any special ssh option
|
||||
# you want just for one special mirror. The option after the username
|
||||
# then MUST start with a -, in which case the whole rest of the line is taken
|
||||
# as a set of options to give to ssh, it is passed through without doing
|
||||
# anything with it.
|
||||
#
|
||||
# There is one caveat here: Should you want to use the -i option to give
|
||||
# another ssh key to use, keep in mind that the master keyfile will
|
||||
# always be presented too! That is, ssh will show both keys to the other
|
||||
# side and the first one presented wins. Which might not be the key you
|
||||
# want. There is currently no way around this, as that would mean
|
||||
# dropping backward compatibility.
|
||||
#
|
||||
# Backwards compatibility:
|
||||
# An older runmirrors script will NOT run with a newer runmirrors.mirror file, but
|
||||
# a new runmirrors can run with an old runmirrors.mirror file. This should make updates
|
||||
# painless.
|
||||
#
|
||||
# Examples:
|
||||
# all eu.puccini puccini.debian.org archvsync 2
|
||||
#
|
||||
# -> will push puccini.debian.org, user archvsync, using ssh protocol 2
|
||||
# and the globally configured ssh key.
|
||||
#
|
||||
# all eu.puccini puccini.debian.org archvsync -p 2222
|
||||
#
|
||||
# -> will do the same as above, but use port 2222 to connect to.
|
||||
#
|
||||
# staged eu.puccini puccini.debian.org archvsync
|
||||
# staged eu.powell powell.debian.org archvsync
|
||||
#
|
||||
# -> will push both puccini and powell in stage1, waiting for both to
|
||||
# finish stage1 before stage2 gets pushed. The username will be archvsync.
|
||||
#
|
||||
# staged eu.puccini puccini.debian.org archvsync
|
||||
# mhop eu.powell powell.debian.org archvsync
|
||||
#
|
||||
# -> will do the same as above, but powell gets told about mhop and can then
|
||||
# push its own staged/mhop mirrors before returning. When both returned
|
||||
# then stage2 is sent to both.
|
||||
#
|
||||
# One special value is allowed: DELAY
|
||||
# This word has to be on a line itself, followed by a space and a number.
|
||||
# nothing else, not even whitespace. It will trigger a pause of $number
|
||||
# seconds between the two mirrors. If no number is given it defaults to
|
||||
# 600 seconds.
|
|
@ -0,0 +1,121 @@
|
|||
########################################################################
|
||||
########################################################################
|
||||
## This is a sample configuration file for the ftpsync mirror script. ##
|
||||
## Most of the values are commented out and just shown here for ##
|
||||
## completeness, together with their default value. ##
|
||||
########################################################################
|
||||
########################################################################
|
||||
|
||||
## Mirrorname. This is used for things like the trace file and should always
|
||||
## be the full hostname of the mirror.
|
||||
#MIRRORNAME=${MIRRORNAME:-`hostname -f`}
|
||||
|
||||
## Destination of the mirrored files. Should be an empty directory.
|
||||
## CAREFUL, this directory will contain the mirror. Everything else
|
||||
## that might have happened to be in there WILL BE GONE after the mirror sync!
|
||||
#TO=${TO:-"/org/www.debian.org/www"}
|
||||
|
||||
## The upstream name of the rsync share.
|
||||
#RSYNC_PATH="web.debian.org"
|
||||
|
||||
## The host we mirror from
|
||||
#RSYNC_HOST=www-master.debian.org
|
||||
|
||||
## In case we need a user to access the rsync share at our upstream host
|
||||
#RSYNC_USER=
|
||||
|
||||
## If we need a user we also need a password
|
||||
#RSYNC_PASSWORD=
|
||||
|
||||
## In which directory should logfiles end up
|
||||
## Note that BASEDIR defaults to $HOME, but can be set before calling the
|
||||
## ftpsync script to any value you want (for example using pam_env)
|
||||
#LOGDIR="${BASEDIR}/log"
|
||||
|
||||
## Name of our own logfile.
|
||||
## Note that ${NAME} is set by the websync script
|
||||
#LOG="${LOGDIR}/${NAME}.log"
|
||||
|
||||
## The script can send logs (or error messages) to a mail address.
|
||||
## If this is unset it will default to the local root user unless it is run
|
||||
## on a .debian.org machine where it will default to the mirroradm people.
|
||||
#MAILTO="root"
|
||||
|
||||
## If you do want a mail about every single sync, set this to false
|
||||
## Everything else will only send mails if a mirror sync fails
|
||||
#ERRORSONLY="true"
|
||||
|
||||
## If you want the logs to also include output of rsync, set this to true.
|
||||
## Careful, the logs can get pretty big, especially if it is the first mirror
|
||||
## run
|
||||
#FULLLOGS="false"
|
||||
|
||||
## If you do want to exclude files from the mirror run, put --exclude statements here.
|
||||
## See rsync(1) for the exact syntax, these are passed to rsync as written here.
|
||||
## Please do not use this except for rare cases and after you talked to us.
|
||||
## For the sponsor logos see SITE_FILTER
|
||||
#EXCLUDE=${EXCLUDE:-"--exclude ${HOSTNAME}"}
|
||||
|
||||
## And site specific excludes, by default its the sponsor stuff that should be local to all
|
||||
#SITE_FILTER=${SITE_FILTER:-"--include sponsor.deb.* --exclude sponsor_img.* --exclude sponsor.html --exclude sponsor.*.html --filter=protect_sponsor_img.* --filter=protect_sponsor.html --filter=protect_sponsor.*.html"}
|
||||
|
||||
## Do we have leaf mirror to signal we are done and they should sync?
|
||||
## If so set it to true and make sure you configure runmirrors-websync.mirrors
|
||||
## and runmirrors-websync.conf for your need.
|
||||
#HUB=false
|
||||
|
||||
## We do create three logfiles for every run. To save space we rotate them, this
|
||||
## defines how many we keep
|
||||
#LOGROTATE=14
|
||||
|
||||
## Our own lockfile (only one sync should run at any time)
|
||||
#LOCK="${TO}/Website-Update-in-Progress-${MIRRORNAME}"
|
||||
|
||||
## The following file is used to make sure we will end up with a correctly
|
||||
## synced mirror even if we get multiple pushes in a short timeframe
|
||||
#UPDATEREQUIRED="${TO}/Website-Update-Required-${MIRRORNAME}"
|
||||
|
||||
## The trace file is used by a mirror check tool to see when we last
|
||||
## had a successful mirror sync. Make sure that it always ends up in
|
||||
## .project/trace and always shows the full hostname.
|
||||
## This is *relative* to ${TO}
|
||||
#TRACE=".project/trace/${MIRRORNAME}"
|
||||
|
||||
## We sync our mirror using rsync (everything else would be insane), so
|
||||
## we need a few options set.
|
||||
## The rsync program
|
||||
#RSYNC=rsync
|
||||
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
## BE VERY CAREFUL WHEN YOU CHANGE THE RSYNC_OPTIONS! BETTER DON'T!
|
||||
|
||||
## Default rsync options every rsync invocation sees.
|
||||
#RSYNC_OPTIONS="-prltvHSB8192 --timeout 3600 --stats --exclude Archive-Update-in-Progress-${MIRRORNAME} --exclude ${TRACE} --exclude Archive-Update-Required-${MIRRORNAME}"
|
||||
|
||||
## Default rsync options
|
||||
#RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
|
||||
|
||||
## You may establish the connection via a web proxy by setting the environment
|
||||
## variable RSYNC_PROXY to a hostname:port pair pointing to your web proxy. Note
|
||||
## that your web proxy's configuration must support proxy connections to port 873.
|
||||
# RSYNC_PROXY=
|
||||
|
||||
## Hook scripts can be run at various places during the sync.
|
||||
## Leave them blank if you don't want any
|
||||
## Hook1: After lock is acquired, before first rsync
|
||||
## Hook2: After first rsync, if successful
|
||||
## Hook3: After second rsync, if successful
|
||||
## Hook4: Right before leaf mirror triggering
|
||||
## Hook5: After leaf mirror trigger, only if we have slave mirrors (HUB=true)
|
||||
##
|
||||
## Note that Hook3 and Hook4 are likely to be called directly after each other.
|
||||
## Difference is: Hook3 is called *every* time the second rsync was successful,
|
||||
## but even if the mirroring needs to re-run thanks to a second push.
|
||||
## Hook4 is only effective if we are done with mirroring.
|
||||
#HOOK1=
|
||||
#HOOK2=
|
||||
#HOOK3=
|
||||
#HOOK4=
|
||||
#HOOK5=
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1 @@
|
|||
~/bin/csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01
|
|
@ -0,0 +1,3 @@
|
|||
<!--#include virtual="/include/ubar.txt" -->
|
||||
<!-- The bandwidth bar program is available at:
|
||||
http://www.kernel.org/pub/software/web/bwbar/ -->
|
|
@ -0,0 +1,5 @@
|
|||
<div class="csclogo">
|
||||
<a href="http://mirror.csclub.uwaterloo.ca">
|
||||
<img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" />
|
||||
</a>
|
||||
</div>
|
|
@ -0,0 +1,7 @@
|
|||
img {
|
||||
border-width: 0;
|
||||
}
|
||||
|
||||
div.biglogo {
|
||||
height: 100px;
|
||||
}
|
Binary file not shown.
After Width: | Height: | Size: 1.1 KiB |
Binary file not shown.
After Width: | Height: | Size: 14 KiB |
|
@ -0,0 +1,11 @@
|
|||
*
|
||||
* Welcome to the University of Waterloo Computer Science Club Mirror
|
||||
*
|
||||
* http://csclub.uwaterloo.ca/
|
||||
*
|
||||
* Hardware funded by MEF (http://www.mef.uwaterloo.ca/)
|
||||
*
|
||||
* Admin Contact: systems-committee@csclub.uwaterloo.ca
|
||||
* Hostname: mirror.csclub.uwaterloo.ca
|
||||
* IP Address: 129.97.134.71
|
||||
*
|
|
@ -0,0 +1,2 @@
|
|||
User-agent: *
|
||||
Disallow: /
|
|
@ -0,0 +1,119 @@
|
|||
#!/usr/bin/python2.5
|
||||
import sys, os, re, gzip, bz2, hashlib
|
||||
|
||||
package_file_map = {
|
||||
'Packages' : file,
|
||||
'Packages.gz' : gzip.GzipFile,
|
||||
'Packages.bz2' : bz2.BZ2File,
|
||||
'Sources' : file,
|
||||
'Sources.gz' : gzip.GzipFile,
|
||||
'Sources.bz2' : bz2.BZ2File,
|
||||
}
|
||||
|
||||
def parse_packages_file(path):
|
||||
try:
|
||||
open_func = package_file_map[os.path.basename(path)]
|
||||
file = open_func(path)
|
||||
except IOError, e:
|
||||
print "WARNING: failed to open %s: %s" % (path, e)
|
||||
return {}
|
||||
cur_dict = {}
|
||||
key, value = None, ''
|
||||
ret_list = []
|
||||
while True:
|
||||
try:
|
||||
line = file.readline()
|
||||
except IOError, e:
|
||||
print "WARNING: failed to read %s: %s" % (path, e)
|
||||
print "WARNING: %s" % e
|
||||
return {}
|
||||
|
||||
# check if we are done with current value
|
||||
if (line == '' or line[0] == '\n' or line[0] != ' ') and key != None:
|
||||
cur_dict[key] = value
|
||||
|
||||
if line == '' or line == '\n': # done current block
|
||||
if cur_dict != {}:
|
||||
ret_list.append(cur_dict)
|
||||
cur_dict = {}
|
||||
key = None
|
||||
if line == '': break
|
||||
elif line[0] == ' ': # multi-line value
|
||||
value += '\n' + line[1:-1]
|
||||
else:
|
||||
if line[-1] == '\n': line = line[:-1]
|
||||
pos = line.find(':')
|
||||
key = line[:pos]
|
||||
if key == '': key = None
|
||||
value = line[pos+2:]
|
||||
return ret_list
|
||||
|
||||
def find_packages_files(path):
|
||||
files = []
|
||||
for file in os.listdir(path):
|
||||
file_path = "%s/%s" % (path, file)
|
||||
if os.path.islink(file_path):
|
||||
continue
|
||||
elif os.path.isdir(file_path):
|
||||
files += find_packages_files(file_path)
|
||||
elif file in package_file_map:
|
||||
files.append(file_path)
|
||||
return files
|
||||
|
||||
if len(sys.argv) != 2:
|
||||
print "Usage: debian-check-md5sum.py base-dir"
|
||||
sys.exit(1)
|
||||
base_dir = sys.argv[1]
|
||||
|
||||
all = {}
|
||||
files_regex = re.compile('(\S+)\s+(\S+)\s+(\S+)')
|
||||
for file in find_packages_files(base_dir):
|
||||
file_type = os.path.basename(file).split('.')[0]
|
||||
a = parse_packages_file(file)
|
||||
for package in parse_packages_file(file):
|
||||
if file_type == 'Packages':
|
||||
if 'Filename' in package:
|
||||
all[package['Filename']] = package
|
||||
elif file_type == 'Sources':
|
||||
files = package['Files'].split('\n')
|
||||
for file in files:
|
||||
if file == '': continue
|
||||
match = files_regex.match(file)
|
||||
file_path = '%s/%s' % (package['Directory'], match.group(3))
|
||||
all[file_path] = { 'MD5sum' : match.group(1) }
|
||||
print "NOTICE: need to check %d files" % len(all)
|
||||
|
||||
ret_val = 0
|
||||
block_size = 65536
|
||||
for (file, package) in all.iteritems():
|
||||
path = '%s/%s' % (base_dir, file)
|
||||
try:
|
||||
file = open(path, 'rb')
|
||||
except IOError:
|
||||
print "WARNING: missing %s" % path
|
||||
continue
|
||||
if 'SHA256' in package:
|
||||
md = hashlib.sha256()
|
||||
hash = package['SHA256']
|
||||
elif 'SHA1' in package:
|
||||
md = hashlib.sha1()
|
||||
hash = package['SHA1']
|
||||
elif 'MD5sum' in package:
|
||||
md = hashlib.md5()
|
||||
hash = package['MD5sum']
|
||||
else:
|
||||
print "WARNING: no hash found for %s" % path
|
||||
print package
|
||||
exit(1)
|
||||
while True:
|
||||
data = file.read(block_size)
|
||||
if data == '': break
|
||||
md.update(data)
|
||||
hash_calc = md.hexdigest()
|
||||
if hash == hash_calc:
|
||||
print "NOTICE: hash ok for %s [hash = %s]" % (path, hash)
|
||||
else:
|
||||
print "ERROR: hash mismatch for %s [hash = %s, hash_calc = %s]" % \
|
||||
(path, hash, hash_calc)
|
||||
ret_val = 1
|
||||
exit(ret_val)
|
|
@ -0,0 +1,22 @@
|
|||
# /etc/cron.d/csc-mirror: mirror cron jobs
|
||||
|
||||
# m h dom mon dow user command
|
||||
|
||||
# update orion routes
|
||||
30 5 * * * root /usr/local/sbin/update-orion-routes
|
||||
|
||||
# make torrents
|
||||
*/10 * * * * mirror /home/mirror/bin/make-torrents > /dev/null 2> /dev/null
|
||||
|
||||
# The rsync cron jobs are now run by a small script a2brenna wrote
|
||||
# that works a bit more intelligently than cron. For one thing, it
|
||||
# won't kick off a sync when one's already running. Please see
|
||||
# ~mirror/merlin.
|
||||
# -- mspang
|
||||
|
||||
# regenerate mirror index at 5:40 am on 14th & 28th of every month
|
||||
# feel free to run this manually if you've added or removed an
|
||||
# archive or some such thing
|
||||
#
|
||||
# Documented here: http://wiki.csclub.uwaterloo.ca/Mirror#Index
|
||||
40 5 */14 * * mirror cd /home/mirror/mirror-index && /home/mirror/mirror-index/make-index.py
|
|
@ -0,0 +1,48 @@
|
|||
# This file describes the network interfaces available on your system
|
||||
# and how to activate them. For more information, see interfaces(5).
|
||||
|
||||
# The loopback network interface
|
||||
auto lo
|
||||
iface lo inet loopback
|
||||
|
||||
# The routes added here will not be visible to the 'route' command; you
|
||||
# should use 'ip route show table foo' instead.
|
||||
|
||||
auto eth0
|
||||
iface eth0 inet static
|
||||
address 129.97.134.42
|
||||
netmask 255.255.255.0
|
||||
gateway 129.97.134.1
|
||||
|
||||
# campus routes are checked first and are maintained here
|
||||
up ip rule add from all lookup campus prio 1
|
||||
down ip rule del from all lookup campus prio 1
|
||||
up ip route add 129.97.0.0/16 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
down ip route del 129.97.0.0/16 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
up ip route add 10.0.0.0/8 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
down ip route del 10.0.0.0/8 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
up ip route add 172.16.0.0/20 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
down ip route del 172.16.0.0/20 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
up ip route add 192.168.0.0/16 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
down ip route del 192.168.0.0/16 via 129.97.134.1 dev eth0 table campus realm campus
|
||||
|
||||
# orion routes are checked second and are maintained by a cronjob
|
||||
up ip rule add from all lookup orion prio 2
|
||||
down ip rule del from all lookup orion prio 2
|
||||
|
||||
# Traffic shaping - 100M cogent, 200M orion, 700M campus.
|
||||
# Note that the border router is configured with a similar policy, but will
|
||||
# drop rather than queue excess packets. These rules keep them from dropping.
|
||||
up tc qdisc add dev eth0 parent root handle 1: htb default 2 r2q 10000
|
||||
up tc class add dev eth0 parent 1: classid 1:1 htb rate 1000Mbit
|
||||
up tc class add dev eth0 parent 1:1 classid 1:2 htb rate 100Mbit
|
||||
up tc class add dev eth0 parent 1:1 classid 1:3 htb rate 200Mbit
|
||||
up tc class add dev eth0 parent 1:1 classid 1:4 htb rate 700Mbit ceil 1000Mbit
|
||||
up tc filter add dev eth0 parent 1: protocol ip pref 2 route to orion flowid 1:3
|
||||
up tc filter add dev eth0 parent 1: protocol ip pref 1 route to campus flowid 1:4
|
||||
down tc qdisc del dev eth0 parent root
|
||||
|
||||
auto eth0:mirror
|
||||
iface eth0:mirror inet static
|
||||
address 129.97.134.71
|
||||
netmask 255.255.255.0
|
|
@ -0,0 +1,186 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# This file updates the orion routing table.
|
||||
# Put it at /usr/local/sbin/orionroutes.py
|
||||
|
||||
# Configuration
|
||||
ORION_TABLE = 1 # from /etc/iproute2/rt_tables
|
||||
ORION_REALMS = 1 # from /etc/iproute2/rt_realms
|
||||
ORION_VIAS = [ "66.97.23.33", "66.97.28.65", "129.97.1.46" ]
|
||||
ORION_GW = "129.97.134.1"
|
||||
ORION_SRC = "129.97.134.42"
|
||||
ORION_IFACE = "eth0"
|
||||
|
||||
# Don't touch anything beyond here
|
||||
|
||||
import sys, iplib, SubnetTree
|
||||
from ctypes import *
|
||||
|
||||
NETLINK_ROUTE = 0
|
||||
AF_UNSPEC = 0
|
||||
RT_SCOPE_UNIVERSE = 0
|
||||
RTPROT_STATIC = 4
|
||||
NLM_F_REPLACE = 0x100
|
||||
|
||||
def die(msg):
|
||||
sys.stderr.write("orionroutes.py: %s\n" % msg)
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
libnl = cdll.LoadLibrary("libnl.so.1")
|
||||
nl_geterror = CFUNCTYPE(c_char_p) (("nl_geterror", libnl), None)
|
||||
nl_handle_alloc = CFUNCTYPE(c_void_p) (("nl_handle_alloc", libnl), None)
|
||||
nl_connect = CFUNCTYPE(c_int, c_void_p, c_int) \
|
||||
(("nl_connect", libnl), ((1, "handle", None), (1, "type", NETLINK_ROUTE)))
|
||||
rtnl_route_alloc = CFUNCTYPE(c_void_p) (("rtnl_route_alloc", libnl), None)
|
||||
rtnl_link_alloc_cache = CFUNCTYPE(c_void_p, c_void_p) \
|
||||
(("rtnl_link_alloc_cache", libnl), ((1, "handle", None), ))
|
||||
rtnl_link_name2i = CFUNCTYPE(c_int, c_void_p, c_char_p) \
|
||||
(("rtnl_link_name2i", libnl), ((1, "cache", None), (1, "iface", -1)))
|
||||
rtnl_route_set_oif = CFUNCTYPE(c_void_p, c_void_p, c_int) \
|
||||
(("rtnl_route_set_oif", libnl), ((1, "route", None), (1, "iface", -1)))
|
||||
nl_cache_free = CFUNCTYPE(None, c_void_p) \
|
||||
(("nl_cache_free", libnl), ((1, "cache", None), ))
|
||||
nl_addr_parse = CFUNCTYPE(c_void_p, c_char_p, c_int) \
|
||||
(("nl_addr_parse", libnl), ((1, "dst", None), (1, "family", AF_UNSPEC)))
|
||||
rtnl_route_set_dst = CFUNCTYPE(c_int, c_void_p, c_void_p) \
|
||||
(("rtnl_route_set_dst", libnl), ((1, "route", None), (1, "dst", None)))
|
||||
rtnl_route_set_pref_src = CFUNCTYPE(c_int, c_void_p, c_void_p) \
|
||||
(("rtnl_route_set_pref_src", libnl), ((1, "route", None), (1, "src", None)))
|
||||
nl_addr_put = CFUNCTYPE(None, c_void_p) \
|
||||
(("nl_addr_put", libnl), ((1, "addr", None), ))
|
||||
rtnl_route_set_gateway = CFUNCTYPE(c_int, c_void_p, c_void_p) \
|
||||
(("rtnl_route_set_gateway", libnl), ((1, "route", None), (1, "gw", None)))
|
||||
rtnl_route_set_table = CFUNCTYPE(None, c_void_p, c_int) \
|
||||
(("rtnl_route_set_table", libnl), ((1, "route", None), (1, "table", -1)))
|
||||
rtnl_route_set_scope = CFUNCTYPE(None, c_void_p, c_int) \
|
||||
(("rtnl_route_set_scope", libnl), ((1, "route", None), (1, "scope", -1)))
|
||||
rtnl_route_set_protocol = CFUNCTYPE(None, c_void_p, c_int) \
|
||||
(("rtnl_route_set_protocol", libnl), ((1, "route", None), (1, "proto", -1)))
|
||||
rtnl_route_set_realms = CFUNCTYPE(None, c_void_p, c_int) \
|
||||
(("rtnl_route_set_realms", libnl), ((1, "route", None), (1, "realms", -1)))
|
||||
rtnl_route_add = CFUNCTYPE(c_int, c_void_p, c_void_p, c_int) \
|
||||
(("rtnl_route_add", libnl), ((1, "handle", None), (1, "route", None), (1, "flags", 0)))
|
||||
rtnl_route_put = CFUNCTYPE(None, c_void_p) \
|
||||
(("rtnl_route_put", libnl), ((1, "route", None), ))
|
||||
nl_handle_destroy = CFUNCTYPE(None, c_void_p) \
|
||||
(("nl_handle_destroy", libnl), ((1, "handle", None), ))
|
||||
rtnl_route_alloc_cache = CFUNCTYPE(c_void_p, c_void_p) \
|
||||
(("rtnl_route_alloc_cache", libnl), ((1, "handle", None), ))
|
||||
nl_cache_get_first = CFUNCTYPE(c_void_p, c_void_p) \
|
||||
(("nl_cache_get_first", libnl), ((1, "cache", None), ))
|
||||
rtnl_route_get_table = CFUNCTYPE(c_int, c_void_p) \
|
||||
(("rtnl_route_get_table", libnl), ((1, "route", None), ))
|
||||
rtnl_route_get_dst = CFUNCTYPE(c_void_p, c_void_p) \
|
||||
(("rtnl_route_get_dst", libnl), ((1, "route", None), ))
|
||||
nl_addr2str = CFUNCTYPE(c_char_p, c_void_p, c_char_p, c_int) \
|
||||
(("nl_addr2str", libnl), ((1, "addr", None), (1, "buffer", None), (1, "size", 0)))
|
||||
rtnl_route_del = CFUNCTYPE(c_int, c_void_p, c_void_p, c_int) \
|
||||
(("rtnl_route_del", libnl), ((1, "handle", None), (1, "route", None), (1, "flags", 0)))
|
||||
nl_cache_get_next = CFUNCTYPE(c_void_p, c_void_p) \
|
||||
(("nl_cache_get_next", libnl), ((1, "object", None), ))
|
||||
except Exception,e:
|
||||
die("Failed to load libnl: %s" % e)
|
||||
|
||||
def nl_die(func):
|
||||
die("%s: %s" % (func, nl_geterror()))
|
||||
|
||||
ips = [[] for i in range(33)]
|
||||
for line in sys.stdin:
|
||||
try:
|
||||
ip, mask, via = line.strip().split(',')[0:3]
|
||||
except KeyError, ValueError:
|
||||
die("Malformed line: %s" % line.strip())
|
||||
|
||||
if via not in ORION_VIAS:
|
||||
continue
|
||||
bits = int(iplib.IPv4NetMask(mask).get_bits())
|
||||
ips[bits].append(int(iplib.IPv4Address(ip)))
|
||||
|
||||
count = sum([len(ip_list) for ip_list in ips])
|
||||
if count < 10:
|
||||
die("Not enough routes (got %d)" % count)
|
||||
|
||||
cidrs = []
|
||||
for bits in range(32, 1, -1):
|
||||
ips[bits].sort()
|
||||
last_ip = 0
|
||||
for ip in ips[bits]:
|
||||
if ip != last_ip and (ip ^ last_ip) == (1 << (32 - bits)):
|
||||
ips[bits - 1].append(ip & (((1 << (bits - 1)) - 1) << (32 - (bits - 1))))
|
||||
last_ip = 0
|
||||
elif last_ip != 0:
|
||||
cidrs.append((iplib.IPv4Address(last_ip), bits))
|
||||
last_ip = ip
|
||||
if last_ip != 0:
|
||||
cidrs.append((iplib.IPv4Address(last_ip), bits))
|
||||
|
||||
nlh = nl_handle_alloc()
|
||||
if nlh == None: nl_die("nl_handle_alloc")
|
||||
if nl_connect(nlh, NETLINK_ROUTE) < 0: nl_die("nl_connect")
|
||||
|
||||
link_cache = rtnl_link_alloc_cache(nlh)
|
||||
if link_cache == None: nl_die("rtnl_link_alloc")
|
||||
iface = rtnl_link_name2i(link_cache, ORION_IFACE)
|
||||
if iface < 0: nl_die("rtnl_link_name2i")
|
||||
nl_cache_free(link_cache)
|
||||
|
||||
cidrs.sort(lambda (ip1, bits1), (ip2, bits2): cmp(ip1, ip2) if bits1 == bits2 else (bits1 - bits2))
|
||||
tree = SubnetTree.SubnetTree()
|
||||
for (ip, bits) in cidrs:
|
||||
if str(ip) not in tree:
|
||||
cidr = "%s/%s" % (ip, bits)
|
||||
tree[cidr] = None
|
||||
|
||||
route = rtnl_route_alloc()
|
||||
if route == None: nl_die("rtnl_route_alloc")
|
||||
|
||||
dstaddr = nl_addr_parse(cidr, AF_UNSPEC)
|
||||
if dstaddr == None: nl_die("nl_addr_parse(%s)" % cidr)
|
||||
if rtnl_route_set_dst(route, dstaddr) < 0: nl_die("rtnl_route_set_dst")
|
||||
nl_addr_put(dstaddr)
|
||||
|
||||
srcaddr = nl_addr_parse(ORION_SRC, AF_UNSPEC)
|
||||
if srcaddr == None: nl_die("nl_addr_parse(%s)" % ORION_SRC)
|
||||
if rtnl_route_set_pref_src(route, srcaddr) < 0: nl_die("nl_route_set_pref_src")
|
||||
nl_addr_put(srcaddr)
|
||||
|
||||
gwaddr = nl_addr_parse(ORION_GW, AF_UNSPEC)
|
||||
if gwaddr == None: nl_die("nl_addr_parse(%s)" % ORION_GW)
|
||||
if rtnl_route_set_gateway(route, gwaddr) < 0: nl_die("nl_route_set_gateway")
|
||||
nl_addr_put(gwaddr)
|
||||
|
||||
rtnl_route_set_oif(route, iface)
|
||||
rtnl_route_set_table(route, ORION_TABLE)
|
||||
rtnl_route_set_scope(route, RT_SCOPE_UNIVERSE)
|
||||
rtnl_route_set_protocol(route, RTPROT_STATIC)
|
||||
rtnl_route_set_realms(route, ORION_REALMS)
|
||||
|
||||
if rtnl_route_add(nlh, route, NLM_F_REPLACE) < 0: nl_die("rtnl_route_add(dst=%s)" % cidr)
|
||||
rtnl_route_put(route)
|
||||
|
||||
route_cache = rtnl_route_alloc_cache(nlh)
|
||||
if route_cache == None: nl_die("rtnl_route_alloc_cache")
|
||||
dstaddr_s = create_string_buffer(100)
|
||||
|
||||
route = nl_cache_get_first(route_cache)
|
||||
while route != None:
|
||||
table = rtnl_route_get_table(route)
|
||||
if table != ORION_TABLE:
|
||||
route = nl_cache_get_next(route)
|
||||
continue
|
||||
|
||||
dstaddr = rtnl_route_get_dst(route)
|
||||
if dstaddr == None:
|
||||
continue
|
||||
if nl_addr2str(dstaddr, dstaddr_s, sizeof(dstaddr_s)) == None: nl_die("nl_addr2str")
|
||||
dstaddr = str(repr(dstaddr_s.value)).strip('\'').split('/')[0]
|
||||
|
||||
if dstaddr not in tree:
|
||||
rtnl_route_del(nlh, route, 0)
|
||||
|
||||
route = nl_cache_get_next(route)
|
||||
|
||||
nl_cache_free(route_cache)
|
||||
|
||||
nl_handle_destroy(nlh)
|
|
@ -0,0 +1,11 @@
|
|||
# This file lives at /etc/iproute2/rt_realms
|
||||
|
||||
#
|
||||
# reserved values
|
||||
#
|
||||
0 cosmos
|
||||
#
|
||||
# local
|
||||
#
|
||||
1 orion
|
||||
2 campus
|
|
@ -0,0 +1,14 @@
|
|||
# This file lives at /etc/iproute2/rt_tables
|
||||
|
||||
#
|
||||
# reserved values
|
||||
#
|
||||
255 local
|
||||
254 main
|
||||
253 default
|
||||
0 unspec
|
||||
#
|
||||
# local
|
||||
#
|
||||
1 orion
|
||||
2 campus
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
# This file updates the orion routing table.
|
||||
# Put it at /usr/local/sbin/update-orion-routes.
|
||||
|
||||
wget --quiet -O - https://istns.uwaterloo.ca/borderroutes/borderroutes.txt | /usr/local/sbin/orionroutes.py
|
|
@ -0,0 +1,102 @@
|
|||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
#include <libgen.h>
|
||||
#include <netlink/route/class.h>
|
||||
#include <netlink/route/link.h>
|
||||
#include <netlink/cache-api.h>
|
||||
#include <netlink/object.h>
|
||||
#include "mirror-nl-glue.h"
|
||||
|
||||
static struct nl_cache *link_cache, *class_cache;
|
||||
static struct rtnl_link *eth;
|
||||
static int ifindex;
|
||||
|
||||
struct class_info cogent_class = { "cogent", "01:02", };
|
||||
struct class_info orion_class = { "orion", "01:03", };
|
||||
struct class_info campus_class = { "campus", "01:04", };
|
||||
|
||||
static struct nl_handle *nl_handle;
|
||||
|
||||
void die(const char *message) {
|
||||
fprintf(stderr, "fatal: %s\n", message);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void match_obj(struct nl_object *obj, void *arg) {
|
||||
struct nl_object *needle = *(struct nl_object **)arg;
|
||||
struct nl_object **ret = (struct nl_object **)arg + 1;
|
||||
|
||||
if (!*ret && nl_object_identical(obj, needle)) {
|
||||
nl_object_get(obj);
|
||||
*ret = obj;
|
||||
}
|
||||
}
|
||||
|
||||
static struct rtnl_class *get_class_by_id(char *id, int ifindex) {
|
||||
uint32_t handle;
|
||||
struct rtnl_class *needle;
|
||||
struct nl_object *magic[2];
|
||||
|
||||
if (rtnl_tc_str2handle(id, &handle))
|
||||
die("invalid id");
|
||||
|
||||
needle = rtnl_class_alloc();
|
||||
rtnl_class_set_ifindex(needle, ifindex);
|
||||
rtnl_class_set_handle(needle, handle);
|
||||
|
||||
magic[0] = (struct nl_object *)needle;
|
||||
magic[1] = (struct nl_object *)NULL;
|
||||
|
||||
nl_cache_foreach(class_cache, match_obj, magic);
|
||||
|
||||
rtnl_class_put(needle);
|
||||
return (struct rtnl_class *)magic[1];
|
||||
}
|
||||
|
||||
uint64_t get_class_byte_count(struct class_info *info) {
|
||||
struct rtnl_class *class = get_class_by_id(info->id, ifindex);
|
||||
uint64_t bytes;
|
||||
if (!class)
|
||||
die("class not found");
|
||||
bytes = rtnl_class_get_stat(class, RTNL_TC_BYTES);
|
||||
rtnl_class_put(class);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
void mirror_stats_refresh(void) {
|
||||
nl_cache_refill(nl_handle, class_cache);
|
||||
}
|
||||
|
||||
void mirror_stats_initialize(void) {
|
||||
nl_handle = nl_handle_alloc();
|
||||
if (!nl_handle)
|
||||
die("unable to allocate handle");
|
||||
|
||||
if (nl_connect(nl_handle, NETLINK_ROUTE) < 0)
|
||||
die("unable to connect to netlink");
|
||||
|
||||
link_cache = rtnl_link_alloc_cache(nl_handle);
|
||||
if (!link_cache)
|
||||
die("unable to allocate link cache");
|
||||
|
||||
eth = rtnl_link_get_by_name(link_cache, "eth0");
|
||||
if (!eth)
|
||||
die("unable to acquire eth0");
|
||||
ifindex = rtnl_link_get_ifindex(eth);
|
||||
|
||||
class_cache = rtnl_class_alloc_cache(nl_handle, ifindex);
|
||||
if (!class_cache)
|
||||
die("unable to allocate class cache");
|
||||
}
|
||||
|
||||
void mirror_stats_cleanup(void) {
|
||||
rtnl_link_put(eth);
|
||||
nl_cache_free(class_cache);
|
||||
nl_cache_free(link_cache);
|
||||
nl_close(nl_handle);
|
||||
nl_handle_destroy(nl_handle);
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
#include <libgen.h>
|
||||
#include <netlink/route/class.h>
|
||||
#include <netlink/route/link.h>
|
||||
#include <netlink/cache-api.h>
|
||||
#include <netlink/object.h>
|
||||
|
||||
struct class_info {
|
||||
char *name;
|
||||
char *id;
|
||||
};
|
||||
|
||||
extern struct class_info cogent_class;
|
||||
extern struct class_info orion_class;
|
||||
extern struct class_info campus_class;
|
||||
|
||||
void mirror_stats_refresh(void);
|
||||
void mirror_stats_initialize(void);
|
||||
void mirror_stats_cleanup(void);
|
||||
void die(const char *);
|
||||
uint64_t get_class_byte_count(struct class_info *);
|
|
@ -0,0 +1,40 @@
|
|||
#include "mirror-nl-glue.h"
|
||||
#include <rrd.h>
|
||||
|
||||
int main(void) {
|
||||
char *argv[3];
|
||||
unsigned long packet_count;
|
||||
|
||||
switch(fork()) {
|
||||
case -1:
|
||||
return -1;
|
||||
case 0:
|
||||
close(0);
|
||||
close(1);
|
||||
setsid();
|
||||
break;
|
||||
default:
|
||||
_exit(0);
|
||||
}
|
||||
|
||||
mirror_stats_initialize();
|
||||
argv[0] = malloc(1024);
|
||||
for (;;) {
|
||||
packet_count = get_class_byte_count(&cogent_class);
|
||||
snprintf(argv[0], 1024, "N:%lu", packet_count);
|
||||
rrd_update_r("/var/rrdtool/cogent.rrd", NULL, 1, argv);
|
||||
packet_count = get_class_byte_count(&orion_class);
|
||||
snprintf(argv[0], 1024, "N:%lu", packet_count);
|
||||
rrd_update_r("/var/rrdtool/orion.rrd", NULL, 1, argv);
|
||||
packet_count = get_class_byte_count(&campus_class);
|
||||
snprintf(argv[0], 1024, "N:%lu", packet_count);
|
||||
rrd_update_r("/var/rrdtool/campus.rrd", NULL, 1, argv);
|
||||
if (rrd_test_error()) {
|
||||
fprintf(stderr, "ERROR: %s\n", rrd_get_error());
|
||||
rrd_clear_error();
|
||||
}
|
||||
sleep(5);
|
||||
mirror_stats_refresh();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/sh
|
||||
/usr/bin/rrdtool graph /mirror/root/stats_monthly.png \
|
||||
-s -1m \
|
||||
--imgformat=PNG \
|
||||
--title='Mirror Traffic' \
|
||||
--base=1000 \
|
||||
--height=120 \
|
||||
--width=600 \
|
||||
--alt-autoscale-max \
|
||||
--lower-limit=0 \
|
||||
--vertical-label='bits per second' \
|
||||
--slope-mode \
|
||||
--font TITLE:10: \
|
||||
--font AXIS:8: \
|
||||
--font LEGEND:8: \
|
||||
--font UNIT:8: \
|
||||
DEF:a="/var/rrdtool/cogent.rrd":snmp_oid:AVERAGE \
|
||||
DEF:b="/var/rrdtool/orion.rrd":snmp_oid:AVERAGE \
|
||||
DEF:c="/var/rrdtool/campus.rrd":snmp_oid:AVERAGE \
|
||||
CDEF:cdefa=a,8,* \
|
||||
CDEF:cdefe=b,8,* \
|
||||
CDEF:cdefi=c,8,* \
|
||||
CDEF:cdefbc=TIME,1318318854,GT,a,a,UN,0,a,IF,IF,TIME,1318318854,GT,b,b,UN,0,b,IF,IF,TIME,1318318854,GT,c,c,UN,0,c,IF,IF,+,+,8,* \
|
||||
AREA:cdefa#157419FF:"Cogent" \
|
||||
GPRINT:cdefa:LAST:"Current\:%8.2lf%s" \
|
||||
GPRINT:cdefa:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefa:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
AREA:cdefe#00CF00FF:"Orion":STACK \
|
||||
GPRINT:cdefe:LAST:" Current\:%8.2lf%s" \
|
||||
GPRINT:cdefe:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefe:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
AREA:cdefi#EE5019FF:"Campus":STACK \
|
||||
GPRINT:cdefi:LAST:"Current\:%8.2lf%s" \
|
||||
GPRINT:cdefi:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefi:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
LINE1:cdefbc#000000FF:"Total" \
|
||||
GPRINT:cdefbc:LAST:" Current\:%8.2lf%s" \
|
||||
GPRINT:cdefbc:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefbc:MAX:"Maximum\:%8.2lf%s\n" >/dev/null 2>/dev/null
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/sh
|
||||
/usr/bin/rrdtool graph /mirror/root/stats_yearly.png \
|
||||
-s -1y \
|
||||
--imgformat=PNG \
|
||||
--title='Mirror Traffic' \
|
||||
--base=1000 \
|
||||
--height=120 \
|
||||
--width=600 \
|
||||
--alt-autoscale-max \
|
||||
--lower-limit=0 \
|
||||
--vertical-label='bits per second' \
|
||||
--slope-mode \
|
||||
--font TITLE:10: \
|
||||
--font AXIS:8: \
|
||||
--font LEGEND:8: \
|
||||
--font UNIT:8: \
|
||||
DEF:a="/var/rrdtool/cogent.rrd":snmp_oid:AVERAGE \
|
||||
DEF:b="/var/rrdtool/orion.rrd":snmp_oid:AVERAGE \
|
||||
DEF:c="/var/rrdtool/campus.rrd":snmp_oid:AVERAGE \
|
||||
CDEF:cdefa=a,8,* \
|
||||
CDEF:cdefe=b,8,* \
|
||||
CDEF:cdefi=c,8,* \
|
||||
CDEF:cdefbc=TIME,1318318854,GT,a,a,UN,0,a,IF,IF,TIME,1318318854,GT,b,b,UN,0,b,IF,IF,TIME,1318318854,GT,c,c,UN,0,c,IF,IF,+,+,8,* \
|
||||
AREA:cdefa#157419FF:"Cogent" \
|
||||
GPRINT:cdefa:LAST:"Current\:%8.2lf%s" \
|
||||
GPRINT:cdefa:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefa:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
AREA:cdefe#00CF00FF:"Orion":STACK \
|
||||
GPRINT:cdefe:LAST:" Current\:%8.2lf%s" \
|
||||
GPRINT:cdefe:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefe:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
AREA:cdefi#EE5019FF:"Campus":STACK \
|
||||
GPRINT:cdefi:LAST:"Current\:%8.2lf%s" \
|
||||
GPRINT:cdefi:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefi:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
LINE1:cdefbc#000000FF:"Total" \
|
||||
GPRINT:cdefbc:LAST:" Current\:%8.2lf%s" \
|
||||
GPRINT:cdefbc:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefbc:MAX:"Maximum\:%8.2lf%s\n" >/dev/null 2>/dev/null
|
|
@ -0,0 +1,38 @@
|
|||
#!/bin/sh
|
||||
/usr/bin/rrdtool graph /mirror/root/stats.png \
|
||||
--imgformat=PNG \
|
||||
--title='Mirror Traffic' \
|
||||
--base=1000 \
|
||||
--height=120 \
|
||||
--width=600 \
|
||||
--alt-autoscale-max \
|
||||
--lower-limit=0 \
|
||||
--vertical-label='bits per second' \
|
||||
--slope-mode \
|
||||
--font TITLE:10: \
|
||||
--font AXIS:8: \
|
||||
--font LEGEND:8: \
|
||||
--font UNIT:8: \
|
||||
DEF:a="/var/rrdtool/cogent.rrd":snmp_oid:AVERAGE \
|
||||
DEF:b="/var/rrdtool/orion.rrd":snmp_oid:AVERAGE \
|
||||
DEF:c="/var/rrdtool/campus.rrd":snmp_oid:AVERAGE \
|
||||
CDEF:cdefa=a,8,* \
|
||||
CDEF:cdefe=b,8,* \
|
||||
CDEF:cdefi=c,8,* \
|
||||
CDEF:cdefbc=TIME,1318318854,GT,a,a,UN,0,a,IF,IF,TIME,1318318854,GT,b,b,UN,0,b,IF,IF,TIME,1318318854,GT,c,c,UN,0,c,IF,IF,+,+,8,* \
|
||||
AREA:cdefa#157419FF:"Cogent" \
|
||||
GPRINT:cdefa:LAST:"Current\:%8.2lf%s" \
|
||||
GPRINT:cdefa:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefa:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
AREA:cdefe#00CF00FF:"Orion":STACK \
|
||||
GPRINT:cdefe:LAST:" Current\:%8.2lf%s" \
|
||||
GPRINT:cdefe:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefe:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
AREA:cdefi#EE5019FF:"Campus":STACK \
|
||||
GPRINT:cdefi:LAST:"Current\:%8.2lf%s" \
|
||||
GPRINT:cdefi:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefi:MAX:"Maximum\:%8.2lf%s\n" \
|
||||
LINE1:cdefbc#000000FF:"Total" \
|
||||
GPRINT:cdefbc:LAST:" Current\:%8.2lf%s" \
|
||||
GPRINT:cdefbc:AVERAGE:"Average\:%8.2lf%s" \
|
||||
GPRINT:cdefbc:MAX:"Maximum\:%8.2lf%s\n" >/dev/null 2>/dev/null
|
|
@ -0,0 +1,3 @@
|
|||
/csc-snmp-subagent
|
||||
/mirror-stats
|
||||
*.o
|
|
@ -0,0 +1,49 @@
|
|||
# this file goes at /etc/csc/mibs/CSC-MIB.txt
|
||||
# and make sure to copy snmp.conf
|
||||
|
||||
CSC-MIB DEFINITIONS ::= BEGIN
|
||||
|
||||
IMPORTS
|
||||
MODULE-IDENTITY, OBJECT-TYPE, Counter32, Gauge32, Counter64,
|
||||
Integer32, TimeTicks, mib-2, enterprises,
|
||||
NOTIFICATION-TYPE FROM SNMPv2-SMI
|
||||
TEXTUAL-CONVENTION, DisplayString,
|
||||
PhysAddress, TruthValue, RowStatus,
|
||||
TimeStamp, AutonomousType, TestAndIncr FROM SNMPv2-TC
|
||||
MODULE-COMPLIANCE, OBJECT-GROUP,
|
||||
NOTIFICATION-GROUP FROM SNMPv2-CONF
|
||||
snmpTraps FROM SNMPv2-MIB
|
||||
IANAifType FROM IANAifType-MIB;
|
||||
|
||||
csclub OBJECT IDENTIFIER ::= { enterprises 27934 }
|
||||
|
||||
cscMIB MODULE-IDENTITY
|
||||
LAST-UPDATED "200905080000Z"
|
||||
ORGANIZATION "University of Waterloo Computer Science Club"
|
||||
CONTACT-INFO "systems-committee@csclub.uwaterloo.ca"
|
||||
DESCRIPTION "Computer Science Club Local MIBs"
|
||||
REVISION "200905080000Z"
|
||||
DESCRIPTION "Initial revision"
|
||||
::= { csclub 2 }
|
||||
|
||||
mirror OBJECT IDENTIFIER ::= { cscMIB 2 }
|
||||
|
||||
cogentBytes OBJECT-TYPE
|
||||
SYNTAX Counter64
|
||||
MAX-ACCESS read-only
|
||||
STATUS current
|
||||
::= { mirror 1 }
|
||||
|
||||
orionBytes OBJECT-TYPE
|
||||
SYNTAX Counter64
|
||||
MAX-ACCESS read-only
|
||||
STATUS current
|
||||
::= { mirror 2 }
|
||||
|
||||
campusBytes OBJECT-TYPE
|
||||
SYNTAX Counter64
|
||||
MAX-ACCESS read-only
|
||||
STATUS current
|
||||
::= { mirror 3 }
|
||||
|
||||
END
|
|
@ -0,0 +1,11 @@
|
|||
LDFLAGS := -lnl $(shell net-snmp-config --base-agent-libs)
|
||||
CFLAGS := -g3 -O2 -Wall
|
||||
|
||||
all: mirror-stats csc-snmp-subagent
|
||||
|
||||
mirror-stats: mirror-stats.o mirror-nl-glue.o
|
||||
|
||||
csc-snmp-subagent: csc-snmp-subagent.o mirror-mib.o mirror-nl-glue.o
|
||||
|
||||
clean:
|
||||
rm -f *.o mirror-stats csc-snmp-subagent
|
|
@ -0,0 +1,221 @@
|
|||
/* generated from net-snmp-config */
|
||||
#include <net-snmp/net-snmp-config.h>
|
||||
#ifdef HAVE_SIGNAL
|
||||
#include <signal.h>
|
||||
#endif /* HAVE_SIGNAL */
|
||||
#include <net-snmp/net-snmp-includes.h>
|
||||
#include <net-snmp/agent/net-snmp-agent-includes.h>
|
||||
#include "mirror-mib.h"
|
||||
const char *app_name = "cscMIB";
|
||||
|
||||
extern int netsnmp_running;
|
||||
|
||||
#ifdef __GNUC__
|
||||
#define UNUSED __attribute__((unused))
|
||||
#else
|
||||
#define UNUSED
|
||||
#endif
|
||||
|
||||
RETSIGTYPE
|
||||
stop_server(UNUSED int a) {
|
||||
netsnmp_running = 0;
|
||||
}
|
||||
|
||||
static void
|
||||
usage(const char *prog)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"USAGE: %s [OPTIONS]\n"
|
||||
"\n"
|
||||
"OPTIONS:\n", prog);
|
||||
|
||||
fprintf(stderr,
|
||||
" -d\t\t\tdump all traffic\n"
|
||||
" -D TOKEN[,...]\tturn on debugging output for the specified "
|
||||
"TOKENs\n"
|
||||
"\t\t\t (ALL gives extremely verbose debugging output)\n"
|
||||
" -f\t\t\tDo not fork() from the calling shell.\n"
|
||||
" -h\t\t\tdisplay this help message\n"
|
||||
" -H\t\t\tdisplay a list of configuration file directives\n"
|
||||
" -L LOGOPTS\t\tToggle various defaults controlling logging:\n");
|
||||
snmp_log_options_usage("\t\t\t ", stderr);
|
||||
#ifndef DISABLE_MIB_LOADING
|
||||
fprintf(stderr,
|
||||
" -m MIB[:...]\t\tload given list of MIBs (ALL loads "
|
||||
"everything)\n"
|
||||
" -M DIR[:...]\t\tlook in given list of directories for MIBs\n");
|
||||
#endif /* DISABLE_MIB_LOADING */
|
||||
#ifndef DISABLE_MIB_LOADING
|
||||
fprintf(stderr,
|
||||
" -P MIBOPTS\t\tToggle various defaults controlling mib "
|
||||
"parsing:\n");
|
||||
snmp_mib_toggle_options_usage("\t\t\t ", stderr);
|
||||
#endif /* DISABLE_MIB_LOADING */
|
||||
fprintf(stderr,
|
||||
" -v\t\t\tdisplay package version number\n"
|
||||
" -x TRANSPORT\tconnect to master agent using TRANSPORT\n");
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void
|
||||
version(void)
|
||||
{
|
||||
fprintf(stderr, "NET-SNMP version: %s\n", netsnmp_get_version());
|
||||
exit(0);
|
||||
}
|
||||
|
||||
int
|
||||
main (int argc, char **argv)
|
||||
{
|
||||
int arg;
|
||||
char* cp = NULL;
|
||||
int dont_fork = 0, do_help = 0;
|
||||
|
||||
while ((arg = getopt(argc, argv, "dD:fhHL:"
|
||||
#ifndef DISABLE_MIB_LOADING
|
||||
"m:M:"
|
||||
#endif /* DISABLE_MIB_LOADING */
|
||||
"n:"
|
||||
#ifndef DISABLE_MIB_LOADING
|
||||
"P:"
|
||||
#endif /* DISABLE_MIB_LOADING */
|
||||
"vx:")) != EOF) {
|
||||
switch (arg) {
|
||||
case 'd':
|
||||
netsnmp_ds_set_boolean(NETSNMP_DS_LIBRARY_ID,
|
||||
NETSNMP_DS_LIB_DUMP_PACKET, 1);
|
||||
break;
|
||||
|
||||
case 'D':
|
||||
debug_register_tokens(optarg);
|
||||
snmp_set_do_debugging(1);
|
||||
break;
|
||||
|
||||
case 'f':
|
||||
dont_fork = 1;
|
||||
break;
|
||||
|
||||
case 'h':
|
||||
usage(argv[0]);
|
||||
break;
|
||||
|
||||
case 'H':
|
||||
do_help = 1;
|
||||
break;
|
||||
|
||||
case 'L':
|
||||
if (snmp_log_options(optarg, argc, argv) < 0) {
|
||||
exit(1);
|
||||
}
|
||||
break;
|
||||
|
||||
#ifndef DISABLE_MIB_LOADING
|
||||
case 'm':
|
||||
if (optarg != NULL) {
|
||||
setenv("MIBS", optarg, 1);
|
||||
} else {
|
||||
usage(argv[0]);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'M':
|
||||
if (optarg != NULL) {
|
||||
setenv("MIBDIRS", optarg, 1);
|
||||
} else {
|
||||
usage(argv[0]);
|
||||
}
|
||||
break;
|
||||
#endif /* DISABLE_MIB_LOADING */
|
||||
|
||||
case 'n':
|
||||
if (optarg != NULL) {
|
||||
app_name = optarg;
|
||||
netsnmp_ds_set_string(NETSNMP_DS_LIBRARY_ID,
|
||||
NETSNMP_DS_LIB_APPTYPE, app_name);
|
||||
} else {
|
||||
usage(argv[0]);
|
||||
}
|
||||
break;
|
||||
|
||||
#ifndef DISABLE_MIB_LOADING
|
||||
case 'P':
|
||||
cp = snmp_mib_toggle_options(optarg);
|
||||
if (cp != NULL) {
|
||||
fprintf(stderr, "Unknown parser option to -P: %c.\n", *cp);
|
||||
usage(argv[0]);
|
||||
}
|
||||
break;
|
||||
#endif /* DISABLE_MIB_LOADING */
|
||||
|
||||
case 'v':
|
||||
version();
|
||||
break;
|
||||
|
||||
case 'x':
|
||||
if (optarg != NULL) {
|
||||
netsnmp_ds_set_string(NETSNMP_DS_APPLICATION_ID,
|
||||
NETSNMP_DS_AGENT_X_SOCKET, optarg);
|
||||
} else {
|
||||
usage(argv[0]);
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
fprintf(stderr, "invalid option: -%c\n", arg);
|
||||
usage(argv[0]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (do_help) {
|
||||
netsnmp_ds_set_boolean(NETSNMP_DS_APPLICATION_ID,
|
||||
NETSNMP_DS_AGENT_NO_ROOT_ACCESS, 1);
|
||||
} else {
|
||||
/* we are a subagent */
|
||||
netsnmp_ds_set_boolean(NETSNMP_DS_APPLICATION_ID,
|
||||
NETSNMP_DS_AGENT_ROLE, 1);
|
||||
|
||||
if (!dont_fork) {
|
||||
if (netsnmp_daemonize(1, snmp_stderrlog_status()) != 0)
|
||||
exit(1);
|
||||
}
|
||||
|
||||
/* initialize tcpip, if necessary */
|
||||
SOCK_STARTUP;
|
||||
}
|
||||
|
||||
/* initialize the agent library */
|
||||
init_agent(app_name);
|
||||
|
||||
/* initialize your mib code here */
|
||||
init_mirror_mib();
|
||||
|
||||
/* cscMIB will be used to read cscMIB.conf files. */
|
||||
init_snmp("cscMIB");
|
||||
|
||||
if (do_help) {
|
||||
fprintf(stderr, "Configuration directives understood:\n");
|
||||
read_config_print_usage(" ");
|
||||
exit(0);
|
||||
}
|
||||
|
||||
/* In case we received a request to stop (kill -TERM or kill -INT) */
|
||||
netsnmp_running = 1;
|
||||
#ifdef SIGTERM
|
||||
signal(SIGTERM, stop_server);
|
||||
#endif
|
||||
#ifdef SIGINT
|
||||
signal(SIGINT, stop_server);
|
||||
#endif
|
||||
|
||||
/* main loop here... */
|
||||
while(netsnmp_running) {
|
||||
agent_check_and_process(1);
|
||||
}
|
||||
|
||||
/* at shutdown time */
|
||||
snmp_shutdown(app_name);
|
||||
SOCK_CLEANUP;
|
||||
exit(0);
|
||||
}
|
||||
|
|
@ -0,0 +1,109 @@
|
|||
/*
|
||||
* Note: this file originally auto-generated by mib2c using
|
||||
* : mib2c.scalar.conf 11805 2005-01-07 09:37:18Z dts12 $
|
||||
*/
|
||||
|
||||
#include <net-snmp/net-snmp-config.h>
|
||||
#include <net-snmp/net-snmp-includes.h>
|
||||
#include <net-snmp/agent/net-snmp-agent-includes.h>
|
||||
#include "mirror-mib.h"
|
||||
#include "mirror-nl-glue.h"
|
||||
|
||||
void
|
||||
init_mirror_mib(void)
|
||||
{
|
||||
static oid cogentBytes_oid[] =
|
||||
{ 1, 3, 6, 1, 4, 1, 27934, 2, 2, 1 };
|
||||
static oid orionBytes_oid[] =
|
||||
{ 1, 3, 6, 1, 4, 1, 27934, 2, 2, 2 };
|
||||
static oid campusBytes_oid[] =
|
||||
{ 1, 3, 6, 1, 4, 1, 27934, 2, 2, 3 };
|
||||
|
||||
DEBUGMSGTL(("mirror_mib", "Initializing\n"));
|
||||
|
||||
mirror_stats_initialize();
|
||||
|
||||
netsnmp_register_scalar(netsnmp_create_handler_registration
|
||||
("cogentBytes", handle_cogentBytes,
|
||||
cogentBytes_oid, OID_LENGTH(cogentBytes_oid),
|
||||
HANDLER_CAN_RONLY));
|
||||
netsnmp_register_scalar(netsnmp_create_handler_registration
|
||||
("orionBytes", handle_orionBytes,
|
||||
orionBytes_oid, OID_LENGTH(orionBytes_oid),
|
||||
HANDLER_CAN_RONLY));
|
||||
netsnmp_register_scalar(netsnmp_create_handler_registration
|
||||
("campusBytes", handle_campusBytes,
|
||||
campusBytes_oid, OID_LENGTH(campusBytes_oid),
|
||||
HANDLER_CAN_RONLY));
|
||||
}
|
||||
|
||||
void explode_counter64(uint64_t num, struct counter64 *counter) {
|
||||
counter->low = num & 0xFFFFFFFF;
|
||||
counter->high = (num >> 32) & 0xFFFFFFFF;
|
||||
}
|
||||
|
||||
int
|
||||
handle_cogentBytes(netsnmp_mib_handler *handler,
|
||||
netsnmp_handler_registration *reginfo,
|
||||
netsnmp_agent_request_info *reqinfo,
|
||||
netsnmp_request_info *requests)
|
||||
{
|
||||
struct counter64 counter;
|
||||
mirror_stats_refresh();
|
||||
explode_counter64(get_class_byte_count(&cogent_class), &counter);
|
||||
|
||||
switch (reqinfo->mode) {
|
||||
case MODE_GET:
|
||||
snmp_set_var_typed_value(requests->requestvb, ASN_COUNTER64,
|
||||
(u_char *)&counter, sizeof(counter));
|
||||
break;
|
||||
default:
|
||||
die("unknown mode");
|
||||
}
|
||||
|
||||
return SNMP_ERR_NOERROR;
|
||||
}
|
||||
|
||||
int
|
||||
handle_orionBytes(netsnmp_mib_handler *handler,
|
||||
netsnmp_handler_registration *reginfo,
|
||||
netsnmp_agent_request_info *reqinfo,
|
||||
netsnmp_request_info *requests)
|
||||
{
|
||||
struct counter64 counter;
|
||||
mirror_stats_refresh();
|
||||
explode_counter64(get_class_byte_count(&orion_class), &counter);
|
||||
|
||||
switch (reqinfo->mode) {
|
||||
case MODE_GET:
|
||||
snmp_set_var_typed_value(requests->requestvb, ASN_COUNTER64,
|
||||
(u_char *)&counter, sizeof(counter));
|
||||
break;
|
||||
default:
|
||||
die("unknown mode");
|
||||
}
|
||||
|
||||
return SNMP_ERR_NOERROR;
|
||||
}
|
||||
|
||||
int
|
||||
handle_campusBytes(netsnmp_mib_handler *handler,
|
||||
netsnmp_handler_registration *reginfo,
|
||||
netsnmp_agent_request_info *reqinfo,
|
||||
netsnmp_request_info *requests)
|
||||
{
|
||||
struct counter64 counter;
|
||||
mirror_stats_refresh();
|
||||
explode_counter64(get_class_byte_count(&campus_class), &counter);
|
||||
|
||||
switch (reqinfo->mode) {
|
||||
case MODE_GET:
|
||||
snmp_set_var_typed_value(requests->requestvb, ASN_COUNTER64,
|
||||
(u_char *)&counter, sizeof(counter));
|
||||
break;
|
||||
default:
|
||||
die("unknown mode");
|
||||
}
|
||||
|
||||
return SNMP_ERR_NOERROR;
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
#ifndef MIRRORMIB_H
|
||||
#define MIRRORMIB_H
|
||||
|
||||
void init_mirror_mib(void);
|
||||
Netsnmp_Node_Handler handle_cogentBytes;
|
||||
Netsnmp_Node_Handler handle_orionBytes;
|
||||
Netsnmp_Node_Handler handle_campusBytes;
|
||||
|
||||
#endif
|
|
@ -0,0 +1,102 @@
|
|||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
#include <libgen.h>
|
||||
#include <netlink/route/class.h>
|
||||
#include <netlink/route/link.h>
|
||||
#include <netlink/cache-api.h>
|
||||
#include <netlink/object.h>
|
||||
#include "mirror-nl-glue.h"
|
||||
|
||||
static struct nl_cache *link_cache, *class_cache;
|
||||
static struct rtnl_link *eth;
|
||||
static int ifindex;
|
||||
|
||||
struct class_info cogent_class = { "cogent", "01:02", };
|
||||
struct class_info orion_class = { "orion", "01:03", };
|
||||
struct class_info campus_class = { "campus", "01:04", };
|
||||
|
||||
static struct nl_handle *nl_handle;
|
||||
|
||||
void die(const char *message) {
|
||||
fprintf(stderr, "fatal: %s\n", message);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static void match_obj(struct nl_object *obj, void *arg) {
|
||||
struct nl_object *needle = *(struct nl_object **)arg;
|
||||
struct nl_object **ret = (struct nl_object **)arg + 1;
|
||||
|
||||
if (!*ret && nl_object_identical(obj, needle)) {
|
||||
nl_object_get(obj);
|
||||
*ret = obj;
|
||||
}
|
||||
}
|
||||
|
||||
static struct rtnl_class *get_class_by_id(char *id, int ifindex) {
|
||||
uint32_t handle;
|
||||
struct rtnl_class *needle;
|
||||
struct nl_object *magic[2];
|
||||
|
||||
if (rtnl_tc_str2handle(id, &handle))
|
||||
die("invalid id");
|
||||
|
||||
needle = rtnl_class_alloc();
|
||||
rtnl_class_set_ifindex(needle, ifindex);
|
||||
rtnl_class_set_handle(needle, handle);
|
||||
|
||||
magic[0] = (struct nl_object *)needle;
|
||||
magic[1] = (struct nl_object *)NULL;
|
||||
|
||||
nl_cache_foreach(class_cache, match_obj, magic);
|
||||
|
||||
rtnl_class_put(needle);
|
||||
return (struct rtnl_class *)magic[1];
|
||||
}
|
||||
|
||||
uint64_t get_class_byte_count(struct class_info *info) {
|
||||
struct rtnl_class *class = get_class_by_id(info->id, ifindex);
|
||||
uint64_t bytes;
|
||||
if (!class)
|
||||
die("class not found");
|
||||
bytes = rtnl_class_get_stat(class, RTNL_TC_BYTES);
|
||||
rtnl_class_put(class);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
void mirror_stats_refresh(void) {
|
||||
nl_cache_refill(nl_handle, class_cache);
|
||||
}
|
||||
|
||||
void mirror_stats_initialize(void) {
|
||||
nl_handle = nl_handle_alloc();
|
||||
if (!nl_handle)
|
||||
die("unable to allocate handle");
|
||||
|
||||
if (nl_connect(nl_handle, NETLINK_ROUTE) < 0)
|
||||
die("unable to connect to netlink");
|
||||
|
||||
link_cache = rtnl_link_alloc_cache(nl_handle);
|
||||
if (!link_cache)
|
||||
die("unable to allocate link cache");
|
||||
|
||||
eth = rtnl_link_get_by_name(link_cache, "eth0");
|
||||
if (!eth)
|
||||
die("unable to acquire eth0");
|
||||
ifindex = rtnl_link_get_ifindex(eth);
|
||||
|
||||
class_cache = rtnl_class_alloc_cache(nl_handle, ifindex);
|
||||
if (!class_cache)
|
||||
die("unable to allocate class cache");
|
||||
}
|
||||
|
||||
void mirror_stats_cleanup(void) {
|
||||
rtnl_link_put(eth);
|
||||
nl_cache_free(class_cache);
|
||||
nl_cache_free(link_cache);
|
||||
nl_close(nl_handle);
|
||||
nl_handle_destroy(nl_handle);
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
#include <libgen.h>
|
||||
#include <netlink/route/class.h>
|
||||
#include <netlink/route/link.h>
|
||||
#include <netlink/cache-api.h>
|
||||
#include <netlink/object.h>
|
||||
|
||||
struct class_info {
|
||||
char *name;
|
||||
char *id;
|
||||
};
|
||||
|
||||
extern struct class_info cogent_class;
|
||||
extern struct class_info orion_class;
|
||||
extern struct class_info campus_class;
|
||||
|
||||
void mirror_stats_refresh(void);
|
||||
void mirror_stats_initialize(void);
|
||||
void mirror_stats_cleanup(void);
|
||||
void die(const char *);
|
||||
uint64_t get_class_byte_count(struct class_info *);
|
|
@ -0,0 +1,12 @@
|
|||
#include "mirror-nl-glue.h"
|
||||
|
||||
int main(int argc, char *argv[]) {
|
||||
mirror_stats_initialize();
|
||||
for (;;) {
|
||||
printf("%s %"PRIu64"\n", cogent_class.id, get_class_byte_count(&cogent_class));
|
||||
printf("%s %"PRIu64"\n", orion_class.id, get_class_byte_count(&orion_class));
|
||||
printf("%s %"PRIu64"\n", campus_class.id, get_class_byte_count(&campus_class));
|
||||
sleep(1);
|
||||
mirror_stats_refresh();
|
||||
}
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
snmpwalk -v2c -cpublic mirror 1.3.6.1.4.1.27934.2.2
|
|
@ -0,0 +1,2 @@
|
|||
mibdirs /etc/csc/mibs:/usr/share/snmp/mibs
|
||||
mibs ALL
|
|
@ -0,0 +1,56 @@
|
|||
#!/bin/sh
|
||||
|
||||
. /lib/lsb/init-functions
|
||||
|
||||
PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin
|
||||
NAME=rtorrent
|
||||
PIDFILE=/var/run/$NAME.screen
|
||||
CHUSER=$NAME
|
||||
DAEMON=/usr/bin/rtorrent
|
||||
DAEMON_ARGS="-n -o try_import=/etc/rtorrent.rc"
|
||||
|
||||
do_start()
|
||||
{
|
||||
if [ -s $PIDFILE ] && kill -0 $(cat $PIDFILE) >/dev/null 2>&1; then
|
||||
exit 0
|
||||
fi
|
||||
log_daemon_msg "Starting" $NAME
|
||||
start-stop-daemon --start --quiet --background --pidfile $PIDFILE \
|
||||
--make-pidfile --exec /bin/su -- \
|
||||
$CHUSER -c "/usr/bin/screen -D -m -- $DAEMON $DAEMON_ARGS"
|
||||
log_end_msg 0
|
||||
}
|
||||
|
||||
do_stop()
|
||||
{
|
||||
log_daemon_msg "Stopping" $NAME
|
||||
start-stop-daemon --stop --quiet --pidfile $PIDFILE --oknodo
|
||||
log_end_msg 0
|
||||
}
|
||||
|
||||
do_status()
|
||||
{
|
||||
if [ -s $PIDFILE ] && kill -0 $(cat $PIDFILE) >/dev/null 2>&1; then
|
||||
exit 0
|
||||
else
|
||||
exit 4
|
||||
fi
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
do_start
|
||||
;;
|
||||
stop)
|
||||
do_stop
|
||||
;;
|
||||
restart)
|
||||
do_stop
|
||||
sleep 4
|
||||
do_start
|
||||
;;
|
||||
status)
|
||||
do_status
|
||||
esac
|
||||
|
||||
exit 0
|
|
@ -0,0 +1,100 @@
|
|||
# This is an example resource file for rTorrent. Copy to
|
||||
# ~/.rtorrent.rc and enable/modify the options as needed. Remember to
|
||||
# uncomment the options you wish to enable.
|
||||
|
||||
# Maximum and minimum number of peers to connect to per torrent.
|
||||
#min_peers = 40
|
||||
#max_peers = 100
|
||||
|
||||
# Same as above but for seeding completed torrents (-1 = same as downloading)
|
||||
#min_peers_seed = 10
|
||||
#max_peers_seed = 50
|
||||
|
||||
# Maximum number of simultanious uploads per torrent.
|
||||
#max_uploads = 15
|
||||
|
||||
# Global upload and download rate in KiB. "0" for unlimited.
|
||||
#download_rate = 0
|
||||
#upload_rate = 0
|
||||
|
||||
# Default directory to save the downloaded torrents.
|
||||
directory = /mirror/root/csclub
|
||||
|
||||
# Default session directory. Make sure you don't run multiple instance
|
||||
# of rtorrent using the same session directory. Perhaps using a
|
||||
# relative path?
|
||||
session = /var/lib/rtorrent/session
|
||||
|
||||
# Watch a directory for new torrents, and stop those that have been
|
||||
# deleted.
|
||||
schedule = watch_www_directory,1,5,load_start=/mirror/root/csclub/*.torrent
|
||||
schedule = untied_directory,5,5,remove_untied=
|
||||
|
||||
# Close torrents when diskspace is low.
|
||||
#schedule = low_diskspace,5,60,close_low_diskspace=100M
|
||||
|
||||
# Stop torrents when reaching upload ratio in percent,
|
||||
# when also reaching total upload in bytes, or when
|
||||
# reaching final upload ratio in percent.
|
||||
# example: stop at ratio 2.0 with at least 200 MB uploaded, or else ratio 20.0
|
||||
#schedule = ratio,60,60,stop_on_ratio=200,200M,2000
|
||||
|
||||
# The ip address reported to the tracker.
|
||||
#ip = 127.0.0.1
|
||||
#ip = rakshasa.no
|
||||
|
||||
# The ip address the listening socket and outgoing connections is
|
||||
# bound to.
|
||||
bind = mirror
|
||||
|
||||
# Port range to use for listening.
|
||||
port_range = 6900-6999
|
||||
|
||||
# Start opening ports at a random position within the port range.
|
||||
#port_random = no
|
||||
|
||||
# Check hash for finished torrents. Might be usefull until the bug is
|
||||
# fixed that causes lack of diskspace not to be properly reported.
|
||||
#check_hash = no
|
||||
|
||||
# Set whetever the client should try to connect to UDP trackers.
|
||||
#use_udp_trackers = yes
|
||||
|
||||
# Alternative calls to bind and ip that should handle dynamic ip's.
|
||||
#schedule = ip_tick,0,1800,ip=rakshasa
|
||||
#schedule = bind_tick,0,1800,bind=rakshasa
|
||||
|
||||
encryption = allow_incoming,prefer_plaintext
|
||||
|
||||
#
|
||||
# Do not modify the following parameters unless you know what you're doing.
|
||||
#
|
||||
|
||||
# Hash read-ahead controls how many MB to request the kernel to read
|
||||
# ahead. If the value is too low the disk may not be fully utilized,
|
||||
# while if too high the kernel might not be able to keep the read
|
||||
# pages in memory thus end up trashing.
|
||||
#hash_read_ahead = 10
|
||||
|
||||
# Interval between attempts to check the hash, in milliseconds.
|
||||
#hash_interval = 100
|
||||
|
||||
# Number of attempts to check the hash while using the mincore status,
|
||||
# before forcing. Overworked systems might need lower values to get a
|
||||
# decent hash checking rate.
|
||||
#hash_max_tries = 10
|
||||
|
||||
# Max number of files to keep open simultaniously.
|
||||
#max_open_files = 128
|
||||
|
||||
# Number of sockets to simultaneously keep open.
|
||||
#max_open_sockets = <no default>
|
||||
|
||||
|
||||
# Example of scheduling commands: Switch between two ip's every 5
|
||||
# seconds.
|
||||
#schedule = "ip_tick1,5,10,ip=torretta"
|
||||
#schedule = "ip_tick2,10,10,ip=lampedusa"
|
||||
|
||||
# Remove a scheduled event.
|
||||
#schedule_remove = "ip_tick1"
|
|
@ -0,0 +1,599 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="stylesheet" type="text/css" href="index.css" />
|
||||
<title>Computer Science Club Mirror</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="logo">
|
||||
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
|
||||
</div>
|
||||
We experienced a recent power outage which caused the RAID array hosting mirror data to need to be rebuilt. As a result, the mirrored data is temporarily read-only. We hope to resume synchronizing as soon as possible.
|
||||
|
||||
We apologize for any inconvenience this may cause you. <br />
|
||||
<div id="raid-progress"></div>
|
||||
<br />
|
||||
<div id="listing">
|
||||
<table>
|
||||
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/apache/">apache/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.apache.org/">apache.org</a>
|
||||
</td>
|
||||
|
||||
<td>32 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/archlinux/">archlinux/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.archlinux.org/">archlinux.org</a>
|
||||
</td>
|
||||
|
||||
<td>35 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/blastwave/">blastwave/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.blastwave.org/">blastwave.org</a>
|
||||
</td>
|
||||
|
||||
<td>14 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/centos/">centos/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.centos.org/">centos.org</a>
|
||||
</td>
|
||||
|
||||
<td>107 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/CPAN/">CPAN/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.cpan.org/">cpan.org</a>
|
||||
</td>
|
||||
|
||||
<td>7.6 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/CRAN/">CRAN/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://cran.r-project.org/">r-project.org</a>
|
||||
</td>
|
||||
|
||||
<td>54 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/csclub/">csclub/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://csclub.uwaterloo.ca/media/">csclub.uwaterloo.ca</a>
|
||||
</td>
|
||||
|
||||
<td>65 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/CTAN/">CTAN/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.ctan.org/">ctan.org</a>
|
||||
</td>
|
||||
|
||||
<td>19 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/cygwin/">cygwin/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.cygwin.com/">cygwin.com</a>
|
||||
</td>
|
||||
|
||||
<td>9.7 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/damnsmalllinux/">damnsmalllinux/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.damnsmalllinux.org/">damnsmalllinux.org</a>
|
||||
</td>
|
||||
|
||||
<td>18 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian/">debian/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.debian.org/">debian.org</a>
|
||||
</td>
|
||||
|
||||
<td>424 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-backports/">debian-backports/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://backports.debian.org/">backports.debian.org</a>
|
||||
</td>
|
||||
|
||||
<td>34 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-cd/">debian-cd/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.debian.org/CD/">debian.org</a>
|
||||
</td>
|
||||
|
||||
<td>77 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-multimedia/">debian-multimedia/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.debian-multimedia.org/">debian-multimedia.org</a>
|
||||
</td>
|
||||
|
||||
<td>5.0 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-ports/">debian-ports/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.debian-ports.org/">debian-ports.org</a>
|
||||
</td>
|
||||
|
||||
<td>63 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-security/">debian-security/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.debian.org/security/">debian.org</a>
|
||||
</td>
|
||||
|
||||
<td>45 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-unofficial/">debian-unofficial/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://unofficial.debian-maintainers.org/">debian-maintainers.org</a>
|
||||
</td>
|
||||
|
||||
<td>468 MB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/debian-volatile/">debian-volatile/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.debian.org/volatile/">debian.org</a>
|
||||
</td>
|
||||
|
||||
<td>2.5 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/eclipse/">eclipse/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.eclipse.org/">eclipse.org</a>
|
||||
</td>
|
||||
|
||||
<td>166 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/emdebian/">emdebian/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.emdebian.org/">emdebian.org</a>
|
||||
</td>
|
||||
|
||||
<td>2.8 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/fedora/">fedora/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.fedoraproject.org/">fedoraproject.org</a>
|
||||
</td>
|
||||
|
||||
<td>736 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/FreeBSD/">FreeBSD/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.freebsd.org/">freebsd.org</a>
|
||||
</td>
|
||||
|
||||
<td>1.9 TB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/gentoo-distfiles/">gentoo-distfiles/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.gentoo.org/">gentoo.org</a>
|
||||
</td>
|
||||
|
||||
<td>173 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/gentoo-portage/">gentoo-portage/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.gentoo.org/">gentoo.org</a>
|
||||
</td>
|
||||
|
||||
<td>612 MB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/gnome/">gnome/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.gnome.org/">gnome.org</a>
|
||||
</td>
|
||||
|
||||
<td>102 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/gnu/">gnu/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.gnu.org/">gnu.org</a>
|
||||
</td>
|
||||
|
||||
<td>24 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/gutenberg/">gutenberg/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.gutenberg.org">gutenberg.org</a>
|
||||
</td>
|
||||
|
||||
<td>511 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/kde/">kde/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.kde.org/">kde.org</a>
|
||||
</td>
|
||||
|
||||
<td>46 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/kernel.org/">kernel.org/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.kernel.org/">kernel.org</a>
|
||||
</td>
|
||||
|
||||
<td>149 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/linuxmint/">linuxmint/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.linuxmint.com/">linuxmint.com</a>
|
||||
</td>
|
||||
|
||||
<td>50 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/linuxmint-packages/">linuxmint-packages/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.linuxmint.com/">linuxmint.com</a>
|
||||
</td>
|
||||
|
||||
<td>5.3 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/mozdev/">mozdev/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.mozdev.org/">mozdev.org</a>
|
||||
</td>
|
||||
|
||||
<td>6.3 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/mozilla.org/">mozilla.org/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.mozilla.org/">mozilla.org</a>
|
||||
</td>
|
||||
|
||||
<td>165 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/mysql/">mysql/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.mysql.com/">mysql.com</a>
|
||||
</td>
|
||||
|
||||
<td>243 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/nongnu/">nongnu/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://savannah.nongnu.org/">nongnu.org</a>
|
||||
</td>
|
||||
|
||||
<td>18 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/OpenBSD/">OpenBSD/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.openbsd.org/">openbsd.org</a>
|
||||
</td>
|
||||
|
||||
<td>240 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/openoffice/">openoffice/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.openoffice.org/">openoffice.org</a>
|
||||
</td>
|
||||
|
||||
<td>125 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/opensuse/">opensuse/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.opensuse.org/">opensuse.org</a>
|
||||
</td>
|
||||
|
||||
<td>202 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/racket/">racket/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://racket-lang.org/">racket-lang.org</a>
|
||||
</td>
|
||||
|
||||
<td>11 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/slackware/">slackware/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.slackware.com/">slackware.com</a>
|
||||
</td>
|
||||
|
||||
<td>141 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/sunfreeware/">sunfreeware/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.sunfreeware.com/">sunfreeware.com</a>
|
||||
</td>
|
||||
|
||||
<td>80 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/ubuntu/">ubuntu/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.ubuntu.com/">ubuntu.com</a>
|
||||
</td>
|
||||
|
||||
<td>392 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/ubuntu-ports/">ubuntu-ports/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://ports.ubuntu.com/ubuntu-ports/">ports.ubuntu.com</a>
|
||||
</td>
|
||||
|
||||
<td>460 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/ubuntu-ports-releases/">ubuntu-ports-releases/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://cdimage.ubuntu.com/ports/releases/">ports.ubuntu.com</a>
|
||||
</td>
|
||||
|
||||
<td>40 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/ubuntu-releases/">ubuntu-releases/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://releases.ubuntu.com/">releases.ubuntu.com</a>
|
||||
</td>
|
||||
|
||||
<td>48 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/x.org/">x.org/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.x.org/">x.org</a>
|
||||
</td>
|
||||
|
||||
<td>5.7 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr>
|
||||
<td>
|
||||
<a href="/xubuntu-releases/">xubuntu-releases/</a>
|
||||
</td>
|
||||
|
||||
<td>
|
||||
<a href="http://www.xubuntu.org/">xubuntu.org</a>
|
||||
</td>
|
||||
|
||||
<td>24 GB</td>
|
||||
</tr>
|
||||
|
||||
<tr class="total">
|
||||
<td>Total</td>
|
||||
<td></td>
|
||||
<td>6.9 TB</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div id="footer">
|
||||
<p>This service is run by the <a href="http://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="http://www.student.math.uwaterloo.ca/~mefcom/home">Mathematics Endowment Fund</a><br />and support from the <a href="http://www.cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
*.pyc
|
||||
/logs
|
||||
/logs.*
|
||||
/rebuild_logs
|
||||
/stamps
|
||||
merlin.sock
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
#!/usr/bin/python2
|
||||
|
||||
import socket, sys
|
||||
try:
|
||||
command = sys.argv[1]
|
||||
except:
|
||||
print("bad command")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.connect('/home/mirror/merlin/merlin.sock')
|
||||
s.send(command)
|
||||
s.shutdown(socket.SHUT_WR)
|
||||
response = ''
|
||||
while True:
|
||||
data = s.recv(4096)
|
||||
if not data:
|
||||
break
|
||||
response = response + data
|
||||
s.close()
|
||||
if command=="status":
|
||||
print(response)
|
||||
else:
|
||||
print(response)
|
|
@ -0,0 +1,55 @@
|
|||
#! /bin/sh
|
||||
|
||||
### BEGIN INIT INFO
|
||||
# Provides: merlin
|
||||
# Required-Start: $remote_fs $syslog $network
|
||||
# Required-Stop: $remote_fs $syslog $network
|
||||
# Default-Start: 2 3 4 5
|
||||
# Default-Stop: 0 1 6
|
||||
# Short-Description: Merlin
|
||||
### END INIT INFO
|
||||
|
||||
set -e
|
||||
|
||||
test -f /home/mirror/merlin/merlin.py || exit 0
|
||||
|
||||
. /lib/lsb/init-functions
|
||||
|
||||
case "$1" in
|
||||
start)
|
||||
log_daemon_msg "Starting Merlin" "merlin"
|
||||
if start-stop-daemon --start --make-pidfile --background --quiet --oknodo --chuid mirror --pidfile /var/run/merlin.pid --chdir /home/mirror/merlin --exec /usr/bin/python -- merlin.py -d; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
stop)
|
||||
log_daemon_msg "Stopping Merlin" "merlin"
|
||||
if start-stop-daemon --stop --quiet --oknodo --user mirror --pidfile /var/run/merlin.pid; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
|
||||
restart|force-reload)
|
||||
log_daemon_msg "Restarting Merlin" "merlin"
|
||||
start-stop-daemon --stop --quiet --oknodo --user mirror --retry 30 --pidfile /var/run/merlin.pid
|
||||
if start-stop-daemon --start --make-pidfile --background --quiet --oknodo --chuid mirror --pidfile /var/run/merlin.pid --chdir /home/mirror/merlin --exec /usr/bin/python -- merlin.py -d; then
|
||||
log_end_msg 0
|
||||
else
|
||||
log_end_msg 1
|
||||
fi
|
||||
;;
|
||||
|
||||
status)
|
||||
status_of_proc -p /var/run/merlin.pid /usr/bin/python merlin && exit 0 || exit $?
|
||||
;;
|
||||
|
||||
*)
|
||||
log_action_msg "Usage: /etc/init.d/merlin {start|stop|force-reload|restart|status}"
|
||||
exit 1
|
||||
esac
|
||||
|
||||
exit 0
|
|
@ -0,0 +1,716 @@
|
|||
#!/usr/bin/python2
|
||||
|
||||
import time, sys, os, errno, logging, signal, copy, select, socket, grp
|
||||
|
||||
daily = 86400
|
||||
twice_daily = 86400 / 2
|
||||
hourly = 3600
|
||||
bi_hourly = 7200
|
||||
tri_hourly = 10800
|
||||
twice_hourly = 1800
|
||||
ten_minutely = 600
|
||||
five_minutely = 300
|
||||
maxtime = 86400
|
||||
mintime = 60
|
||||
jobs = {}
|
||||
MAX_JOBS = 6
|
||||
#earPath = '/home/mirror/merlin/merlin.sock'
|
||||
earPath = '/mirror/merlin/run/merlin.sock'
|
||||
cmd_buf_size = 4096
|
||||
|
||||
repos = {
|
||||
'debian': {
|
||||
'command': '~/bin/csc-sync-debian debian debian.mirror.rafal.ca debian',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'debian-cdimage': {
|
||||
# 'command': '~/bin/csc-sync-cdimage debian-cdimage cdimage.debian.org cdimage',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'ubuntu': {
|
||||
'command': '~/bin/csc-sync-debian ubuntu archive.ubuntu.com ubuntu drescher.canonical.com',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'ubuntu-ports': {
|
||||
'command': '~/bin/csc-sync-debian ubuntu-ports ports.ubuntu.com ubuntu-ports drescher.canonical.com',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'linuxmint-packages': {
|
||||
'command': '~/bin/csc-sync-debian linuxmint-packages rsync-packages.linuxmint.com packages',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'debian-multimedia': {
|
||||
'command': '~/bin/csc-sync-debian debian-multimedia www.deb-multimedia.org deb',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'debian-backports': {
|
||||
'command': '~/bin/csc-sync-debian debian-backports debian.mirror.rafal.ca debian-backports',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'debian-volatile': {
|
||||
# 'command': '~/bin/csc-sync-debian debian-volatile debian.mirror.rafal.ca debian-volatile',
|
||||
# 'interval': bi_hourly,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'debian-security': {
|
||||
'command': '~/bin/csc-sync-debian debian-security rsync.security.debian.org debian-security security-master.debian.org',
|
||||
'interval': twice_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'ubuntu-releases': {
|
||||
'command': '~/bin/csc-sync-standard ubuntu-releases rsync.releases.ubuntu.com releases',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'xubuntu-releases': {
|
||||
'command': '~/bin/csc-sync-standard xubuntu-releases cdimage.ubuntu.com cdimage/xubuntu/releases/',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'emdebian': {
|
||||
# 'command': '~/bin/csc-sync-badperms emdebian www.emdebian.org debian',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'puppylinux': {
|
||||
'command': '~/bin/csc-sync-standard puppylinux distro.ibiblio.org puppylinux',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'CPAN': {
|
||||
'command': '~/bin/csc-sync-standard CPAN cpan-rsync.perl.org CPAN',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'CRAN': {
|
||||
'command': '~/bin/csc-sync-ssh CRAN cran.r-project.org "" cran-rsync ~/.ssh/id_cran_rsa',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'CTAN': {
|
||||
'command': '~/bin/csc-sync-standard CTAN rsync.dante.ctan.org CTAN',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'openoffice': {
|
||||
# 'command': '~/bin/csc-sync-standard openoffice rsync.services.openoffice.org openoffice-extended',
|
||||
# 'command': '~/bin/csc-sync-standard openoffice ftp.snt.utwente.nl openoffice-extended',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'fedora-epel': {
|
||||
'command': '~/bin/csc-sync-standard fedora/epel mirrors.kernel.org fedora-epel && ~/bin/report_mirror >/dev/null',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'cygwin': {
|
||||
'command': '~/bin/csc-sync-standard cygwin cygwin.com cygwin-ftp',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'gnu': {
|
||||
#'command': '~/bin/csc-sync-standard gnu mirrors.ibiblio.org gnuftp/gnu/',
|
||||
'command': '~/bin/csc-sync-standard gnu ftp.gnu.org gnu',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'nongnu': {
|
||||
# 'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases --ignore-errors',
|
||||
'command': '~/bin/csc-sync-standard nongnu dl.sv.gnu.org releases',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'mysql': {
|
||||
#'command': '~/bin/csc-sync-standard mysql mysql.he.net mysql',
|
||||
'command': '~/bin/csc-sync-standard mysql rsync.mirrorservice.org ftp.mysql.com',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# No longer syncs, and no longer really relevant
|
||||
# 'mozdev': {
|
||||
# 'command': '~/bin/csc-sync-standard mozdev rsync.mozdev.org mozdev',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'gnome': {
|
||||
'command': '~/bin/csc-sync-standard gnome master.gnome.org gnomeftp gnome',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'damnsmalllinux': {
|
||||
'command': '~/bin/csc-sync-standard damnsmalllinux ftp.heanet.ie mirrors/damnsmalllinux.org/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'linuxmint': {
|
||||
'command': '~/bin/csc-sync-standard linuxmint pub.linuxmint.com pub',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'kernel.org-linux': {
|
||||
'command': '~/bin/csc-sync-standard kernel.org/linux rsync.kernel.org pub/linux/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'kernel.org-software': {
|
||||
'command': '~/bin/csc-sync-standard kernel.org/software rsync.kernel.org pub/software/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'apache': {
|
||||
'command': '~/bin/csc-sync-apache apache rsync.us.apache.org apache-dist',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'eclipse': {
|
||||
'command': '~/bin/csc-sync-standard eclipse download.eclipse.org eclipseMirror',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'kde': {
|
||||
'command': '~/bin/csc-sync-standard kde rsync.kde.org kdeftp',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'kde-applicationdata': {
|
||||
'command': '~/bin/csc-sync-standard kde-applicationdata rsync.kde.org applicationdata',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853)
|
||||
# so our IP is important.
|
||||
'archlinux': {
|
||||
#'command': '~/bin/csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux',
|
||||
'command': '~/bin/csc-sync-archlinux archlinux',
|
||||
'interval': five_minutely,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'debian-ports': {
|
||||
'command': '~/bin/csc-sync-standard debian-ports ftp.de.debian.org debian-ports',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'slackware': {
|
||||
'command': '~/bin/csc-sync-standard slackware slackware.cs.utah.edu slackware',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'debian-cd': {
|
||||
'command': '~/bin/csc-sync-debian-cd',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'x.org': {
|
||||
#'command': '~/bin/csc-sync-standard x.org xorg.freedesktop.org xorg-archive',
|
||||
#'command': '~/bin/csc-sync-standard x.org mirror.us.leaseweb.net xorg',
|
||||
'command': '~/bin/csc-sync-standard x.org rsync.mirrorservice.org ftp.x.org/pub',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'centos': {
|
||||
'command': '~/bin/csc-sync-standard centos us-msync.centos.org CentOS',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'opensuse': {
|
||||
'command': '~/bin/csc-sync-standard opensuse stage.opensuse.org opensuse-full/opensuse/ #"--exclude distribution/.timestamp_invisible"',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'FreeBSD': {
|
||||
# Has not updated since at least June 2018
|
||||
#'command': '~/bin/csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD',
|
||||
'command': '~/bin/csc-sync-standard FreeBSD ftp2.uk.freebsd.org ftp.freebsd.org/pub/FreeBSD/',
|
||||
#'command': '~/bin/csc-sync-standard FreeBSD ftp3.us.freebsd.org FreeBSD/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'fedora-enchilada': {
|
||||
# 'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null',
|
||||
'command': '~/bin/csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ && ~/bin/report_mirror >/dev/null',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'ubuntu-ports-releases': {
|
||||
'command': '~/bin/csc-sync-standard ubuntu-ports-releases cdimage.ubuntu.com cdimage/releases/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'gentoo-distfiles': {
|
||||
'command': '~/bin/csc-sync-gentoo',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'gentoo-portage': {
|
||||
'command': '~/bin/csc-sync-standard gentoo-portage rsync1.us.gentoo.org gentoo-portage',
|
||||
'interval': twice_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# This project is no longer available for mirroring
|
||||
# https://bugzilla.mozilla.org/show_bug.cgi?id=807543
|
||||
#'mozilla.org': {
|
||||
# 'command': '~/bin/csc-sync-standard mozilla.org releases-rsync.mozilla.org mozilla-releases',
|
||||
# 'interval': twice_hourly,
|
||||
# 'max-sync-time': maxtime,
|
||||
#},
|
||||
'gutenberg': {
|
||||
'command': '~/bin/csc-sync-standard gutenberg ftp@ftp.ibiblio.org gutenberg',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'racket-installers': {
|
||||
'command': '~/bin/csc-sync-wget racket/racket-installers https://mirror.racket-lang.org/installers/ 1',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'plt-bundles': {
|
||||
'command': '~/bin/csc-sync-standard racket/plt-bundles mirror.racket-lang.org plt-bundles',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'OpenBSD': {
|
||||
'command': '~/bin/csc-sync-standard OpenBSD ftp3.usa.openbsd.org ftp',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'xiph': {
|
||||
#'command': '~/bin/csc-sync-standard xiph downloads.xiph.org xiph/releases',
|
||||
'command': '~/bin/csc-sync-standard xiph ftp.osuosl.org xiph',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# We currently don't have the disk space
|
||||
'netbsd': {
|
||||
'command': '~/bin/csc-sync-standard NetBSD rsync.netbsd.org NetBSD',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'netbsd-pkgsrc': {
|
||||
'command': '~/bin/csc-sync-standard pkgsrc rsync.netbsd.org pkgsrc',
|
||||
#'command': '~/bin/csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'macports-release': {
|
||||
'command': '~/bin/csc-sync-standard MacPorts/release rsync.macports.org macports/release/',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'macports-distfiles': {
|
||||
'command': '~/bin/csc-sync-standard MacPorts/mpdistfiles rsync.macports.org macports/distfiles/',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'raspberrypi': {
|
||||
# 'command': '~/bin/csc-sync-standard raspberrypi mirrors.rit.edu rpi',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'sagemath': {
|
||||
#'command': '~/bin/csc-sync-standard sage mirror.clibre.uqam.ca sage',
|
||||
'command': '~/bin/csc-sync-standard sage rsync.sagemath.org sage',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'cs136': {
|
||||
# 'command': '~/bin/csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01',
|
||||
# 'interval': hourly,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'vlc': {
|
||||
'command': '~/bin/csc-sync-standard vlc rsync.videolan.org videolan-ftp',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'qtproject': {
|
||||
'command': '~/bin/csc-sync-standard qtproject master.qt.io qt-all',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'tdf': {
|
||||
'command': '~/bin/csc-sync-standard tdf rsync.documentfoundation.org tdf-pub',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'saltstack': {
|
||||
'command': '~/bin/csc-sync-s3 saltstack https://s3.repo.saltproject.io',
|
||||
'interval': daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
# 'kali': {
|
||||
# 'command': '~/bin/csc-sync-standard kali kali.mirror.globo.tech kali',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
# 'kali-images': {
|
||||
# 'command': '~/bin/csc-sync-standard kali-images kali.mirror.globo.tech kali-images',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
# },
|
||||
'alpine': {
|
||||
'command': '~/bin/csc-sync-standard alpine rsync.alpinelinux.org alpine',
|
||||
'interval': hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'raspbian': {
|
||||
'command': '~/bin/csc-sync-standard raspbian archive.raspbian.org archive',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'raspberrypi': {
|
||||
'command': '~/bin/csc-sync-standard-ipv6 raspberrypi apt-repo.raspberrypi.org archive',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'ipfire': {
|
||||
'command': '~/bin/csc-sync-standard ipfire rsync.ipfire.org full',
|
||||
'interval': hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'manjaro': {
|
||||
'command': '~/bin/csc-sync-standard manjaro mirrorservice.org repo.manjaro.org/repos/',
|
||||
'interval': hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'scientific': {
|
||||
'command': '~/bin/csc-sync-standard scientific rsync.scientificlinux.org scientific',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'mxlinux': {
|
||||
'command': '~/bin/csc-sync-standard mxlinux mirror.math.princeton.edu pub/mxlinux/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'mxlinux-iso': {
|
||||
'command': '~/bin/csc-sync-standard mxlinux-iso mirror.math.princeton.edu pub/mxlinux-iso/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
|
||||
'parabola': {
|
||||
'command': '~/bin/csc-sync-standard parabola repo.parabola.nu:875 repos/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
#'hyperbola-sources': {
|
||||
# 'command': '~/bin/csc-sync-chmod hyperbola/sources repo.hyperbola.info:52000 repo/',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
#},
|
||||
#'hyperbola-stable': {
|
||||
# 'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/stable repo.hyperbola.info:52012 repo/',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
#},
|
||||
#'hyperbola-testing': {
|
||||
# 'command': '~/bin/csc-sync-chmod hyperbola/gnu-plus-linux-libre/testing repo.hyperbola.info:52011 repo/',
|
||||
# 'interval': twice_daily,
|
||||
# 'max-sync-time': maxtime,
|
||||
#},
|
||||
'trisquel-packages': {
|
||||
'command': '~/bin/csc-sync-standard trisquel/packages rsync.trisquel.info trisquel.packages/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'trisquel-iso': {
|
||||
'command': '~/bin/csc-sync-standard trisquel/iso rsync.trisquel.info trisquel.iso/',
|
||||
'interval': twice_daily,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'almalinux': {
|
||||
'command': '~/bin/csc-sync-standard almalinux rsync.repo.almalinux.org almalinux/',
|
||||
'interval': bi_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
'ceph': {
|
||||
'command': '~/bin/csc-sync-ceph -q -s global -t ceph',
|
||||
'interval': tri_hourly,
|
||||
'max-sync-time': maxtime,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def mirror_status():
|
||||
out = []
|
||||
for x in repos:
|
||||
repository = repos[x]
|
||||
last_attempt = repository['last-attempt']
|
||||
next_attempt = repository['last-attempt'] + repository['interval']
|
||||
out.append( [x, last_attempt, next_attempt] )
|
||||
|
||||
out.sort(key= lambda x: x[2])
|
||||
|
||||
#turn floating point time values into human readable strings
|
||||
for x in out:
|
||||
for y in (1,2):
|
||||
x[y] = time.ctime(x[y])
|
||||
|
||||
out.insert(0,['Repository', 'Last Synced', 'Next Expected Sync'])
|
||||
|
||||
#calculate maximum width of each column
|
||||
widths = []
|
||||
i = 0
|
||||
while True:
|
||||
try:
|
||||
column_width = max([len(row[i]) for row in out])
|
||||
widths.append(column_width + 3)
|
||||
i = i + 1
|
||||
except:
|
||||
break
|
||||
|
||||
#compose table string and pad out columns
|
||||
status_string = '%s%s%s\n' % (out[0][0].ljust(widths[0]), out[0][1].ljust(widths[1]), out[0][2].ljust(widths[2]))
|
||||
for x in out[1:]:
|
||||
for y in (0,1,2):
|
||||
status_string = status_string + ('%s' % x[y].rjust(widths[y]))
|
||||
status_string = status_string + '\n'
|
||||
|
||||
return status_string
|
||||
|
||||
|
||||
def init_last_sync():
|
||||
|
||||
if not os.path.isdir('stamps'):
|
||||
os.mkdir('stamps')
|
||||
|
||||
now = time.time()
|
||||
|
||||
for repo in repos:
|
||||
try:
|
||||
last = os.stat('stamps/%s' % repo).st_mtime
|
||||
repos[repo]['last-sync'], repos[repo]['last-attempt'] = last, last
|
||||
logging.info('repo %s last synced %d seconds ago' % (repo, now - last))
|
||||
except OSError:
|
||||
repos[repo]['last-sync'], repos[repo]['last-attempt'] = 0, 0
|
||||
logging.warning('repo %s has never been synced' % repo)
|
||||
|
||||
|
||||
def update_last_attempt(repo, start_time):
|
||||
repos[repo]['last-attempt'] = start_time
|
||||
|
||||
def update_last_sync(repo, start_time, duration, exit_code):
|
||||
|
||||
repos[repo]['last-sync'] = start_time
|
||||
update_last_attempt(repo, start_time)
|
||||
|
||||
# touch the timestamp file
|
||||
with open('stamps/%s' % repo, 'w') as f:
|
||||
f.write('%d,%d,%d' % (start_time, duration, exit_code))
|
||||
# open('stamps/%s' % repo, 'w').close()
|
||||
|
||||
def handler(signum, frame):
|
||||
logging.info("Caught signal %d" % signum)
|
||||
|
||||
os.remove(earPath)
|
||||
|
||||
for job in jobs:
|
||||
os.kill(job, signal.SIGTERM)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
for job in jobs:
|
||||
os.kill(job, signal.SIGKILL)
|
||||
|
||||
try:
|
||||
while True:
|
||||
pid, status = os.wait()
|
||||
logging.info("Child process %d has terminated" % pid)
|
||||
|
||||
#ECHILD is thrown when there are no child processes left
|
||||
except OSError, e:
|
||||
if e.errno != errno.ECHILD:
|
||||
raise
|
||||
logging.info("All child processes terminated. Goodbye!")
|
||||
sys.exit()
|
||||
|
||||
def setup_logging():
|
||||
if not os.path.isdir('logs'):
|
||||
os.mkdir('logs')
|
||||
|
||||
try:
|
||||
os.unlink(earPath)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
if '-d' in sys.argv:
|
||||
logging.basicConfig(filename="/home/mirror/merlin/logs/merlin.log",
|
||||
level=logging.DEBUG, format="%(asctime)-15s %(message)s")
|
||||
else:
|
||||
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(message)s")
|
||||
|
||||
def sync(current, now):
|
||||
pid = os.fork()
|
||||
if not pid:
|
||||
try:
|
||||
logfd = os.open('logs/%s' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
|
||||
nulfd = os.open('/dev/null', os.O_RDONLY)
|
||||
os.dup2(nulfd, 0)
|
||||
os.dup2(logfd, 1)
|
||||
os.dup2(logfd, 2)
|
||||
os.close(logfd)
|
||||
os.close(nulfd)
|
||||
os.execv("/bin/sh", ['sh', '-c', repos[current]['command']])
|
||||
except OSError, e:
|
||||
print >>sys.stderr, 'failed to exec: %s' % e
|
||||
os.exit(1)
|
||||
|
||||
#There exists a race condition that manifests if merlin is asked to terminate ( by receiving signal ) after fork() but before the jobs table is updated.
|
||||
#Normally you would mask off signals to avoid attempting shutdown in that time period, but this is not supported in the version of Python for which merlin was written.
|
||||
#TODO:
|
||||
#The Linux Programming Interface 24.5 illustrates how to synchronize parent and child using signals. Might be applicable.
|
||||
|
||||
jobs[pid] = {'name': current, 'start_time': now, 'status': 'running'}
|
||||
|
||||
def zfssync(current):
|
||||
pid = os.fork()
|
||||
if not pid:
|
||||
try:
|
||||
logfd = os.open('logs/zfssync-%s.log' % current, os.O_WRONLY|os.O_APPEND|os.O_CREAT, 0644)
|
||||
nulfd = os.open('/dev/null', os.O_RDONLY)
|
||||
os.dup2(nulfd, 0)
|
||||
os.dup2(logfd, 1)
|
||||
os.dup2(logfd, 2)
|
||||
os.close(logfd)
|
||||
os.close(nulfd)
|
||||
os.execv("/home/mirror/bin/zfssync", ['zfssync', current])
|
||||
except OSError, e:
|
||||
print >>sys.stderr, 'failed to exec: %s' % e
|
||||
os.exit(1)
|
||||
|
||||
def await_command(ear):
|
||||
if select.select([ear],[],[],1) == ([ear],[],[]):
|
||||
#handle command heard on ear socket
|
||||
s, address = ear.accept()
|
||||
cmdstring = ''
|
||||
while True:
|
||||
data = s.recv(cmd_buf_size)
|
||||
if not data:
|
||||
break
|
||||
cmdstring = cmdstring + data
|
||||
cmdstring = cmdstring.split(':',1)
|
||||
command = cmdstring[0]
|
||||
try:
|
||||
if command == 'sync':
|
||||
try:
|
||||
arg = cmdstring[1]
|
||||
if arg in repos:
|
||||
if arg in (x['name'] for x in jobs.itervalues()):
|
||||
logging.info('Cannot force sync: %s. Already syncing.' % arg)
|
||||
s.send('Cannot force sync %s, already syncing.' % command)
|
||||
else:
|
||||
logging.info('Forcing sync: %s' % arg)
|
||||
s.send('Forcing sync: %s' % arg)
|
||||
sync(arg, time.time())
|
||||
else:
|
||||
logging.info('%s not tracked, cannot sync.' % arg)
|
||||
s.send('%s not tracked, cannot sync.' % arg)
|
||||
except:
|
||||
s.send('Could not parse sync command, forced sync fails.')
|
||||
raise
|
||||
|
||||
elif command == 'status':
|
||||
s.send(mirror_status())
|
||||
|
||||
else:
|
||||
logging.error('Received unrecognized command: %s' % command)
|
||||
s.send('Bad command: %s' % command)
|
||||
|
||||
s.close()
|
||||
|
||||
except socket.error, e:
|
||||
logging.error('Could not communicate with arthur over socket.')
|
||||
|
||||
def new_jobs(now):
|
||||
for current in repos:
|
||||
if len(jobs) >= MAX_JOBS:
|
||||
break
|
||||
if now <= repos[current]['last-attempt'] + mintime:
|
||||
continue
|
||||
if current in (x['name'] for x in jobs.itervalues()):
|
||||
continue
|
||||
when_due = repos[current]['last-sync'] + repos[current]['interval']
|
||||
if now >= when_due:
|
||||
logging.debug("syncing %s, due for %d seconds" % (current, now - when_due))
|
||||
sync(current, now)
|
||||
|
||||
def handle_completed(now):
|
||||
try:
|
||||
pid, status = os.waitpid(-1, os.WNOHANG) #waitpid() likes to throw an exception when there's nothing to wait for
|
||||
|
||||
#Don't know if we should be forever ignoring hung jobs if they eventually return...
|
||||
if pid != 0 and pid in jobs:
|
||||
job = jobs.pop(pid)
|
||||
|
||||
if os.WIFSIGNALED(status):
|
||||
logging.error("%s %s sync terminated with signal %d" % (job['status'], job['name'], os.WTERMSIG(status)))
|
||||
update_last_attempt(job['name'], job['start_time'])
|
||||
elif os.WIFEXITED(status):
|
||||
exit_status = os.WEXITSTATUS(status)
|
||||
logging.error("%s %s sync exited with status %d " % (job['status'], job['name'], exit_status))
|
||||
update_last_attempt(job['name'], job['start_time'])
|
||||
interval = repos[job['name']]['interval']
|
||||
sync_took = int(now - job['start_time'])
|
||||
next_sync = max(0, interval - sync_took)
|
||||
update_last_sync(job['name'], job['start_time'], sync_took, exit_status)
|
||||
logging.info('%s sync complete, took %d seconds, syncs again in %d seconds'
|
||||
% (job['name'], sync_took, next_sync))
|
||||
|
||||
zfssync(job['name'])
|
||||
|
||||
except OSError, e:
|
||||
if e.errno != errno.ECHILD:
|
||||
raise
|
||||
|
||||
def check_hung(now):
|
||||
for pid in jobs:
|
||||
#check if hung
|
||||
runtime = now - jobs[pid]['start_time']
|
||||
repo = jobs[pid]['name']
|
||||
if jobs[pid]['status'] == 'running' and runtime > repos[repo]['max-sync-time']:
|
||||
jobs[pid]['status'] = 'hung'
|
||||
logging.error("%s sync process hung, process pid %d" % (repo, pid))
|
||||
|
||||
def main():
|
||||
if not os.geteuid():
|
||||
print "Don't run merlin as root!"
|
||||
sys.exit(1);
|
||||
|
||||
setup_logging()
|
||||
|
||||
logging.info("Starting Merlin")
|
||||
|
||||
signal.signal(signal.SIGTERM, handler)
|
||||
signal.signal(signal.SIGINT, handler)
|
||||
|
||||
init_last_sync()
|
||||
|
||||
|
||||
old_umask = os.umask(0o002)
|
||||
ear = socket.socket(socket.AF_UNIX)
|
||||
ear.bind(earPath)
|
||||
ear.listen(1)
|
||||
os.umask(old_umask)
|
||||
os.chown(earPath, -1, grp.getgrnam("push").gr_gid)
|
||||
|
||||
while True:
|
||||
await_command(ear)
|
||||
|
||||
now = time.time()
|
||||
|
||||
new_jobs(now)
|
||||
|
||||
handle_completed(now)
|
||||
|
||||
check_hung(now)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
|
@ -0,0 +1,13 @@
|
|||
[Unit]
|
||||
Description=Manages synchronization of mirrored projects
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/home/mirror/merlin/merlin.py
|
||||
WorkingDirectory=/home/mirror/merlin
|
||||
User=mirror
|
||||
Group=mirror
|
||||
SyslogIdentifier=merlin
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
IFS=$'\n'
|
||||
for i in `cat merlin.py | sed -ne "s/^[ ]\+'command':[ ]*'\([^']\+\).*$/\1/pg"`; do
|
||||
while [ "`jobs|wc -l`" -ge 9 ]; do
|
||||
sleep 1
|
||||
done
|
||||
echo "$i"
|
||||
logfile="`echo "$i" | cut -d' ' -f2 | sed -e's/\//_/g'`"
|
||||
bash -c "$i" >rebuild_logs/$logfile 2>&1 &
|
||||
done
|
|
@ -0,0 +1,47 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
import merlin
|
||||
|
||||
merlin.repos = {
|
||||
'gnu': {
|
||||
'command': 'sleep 7',
|
||||
'interval': 30,
|
||||
'max-sync-time': 60,
|
||||
},
|
||||
'nongnu': {
|
||||
'command': 'sleep 10',
|
||||
'interval': 30,
|
||||
'max-sync-time': 60,
|
||||
},
|
||||
'mysql': {
|
||||
'command': 'sleep 30',
|
||||
'interval': 60,
|
||||
'max-sync-time': 60,
|
||||
},
|
||||
'mozdev': {
|
||||
'command': 'sleep 5',
|
||||
'interval': 5,
|
||||
'max-sync-time': 60,
|
||||
},
|
||||
'gnome': {
|
||||
'command': 'sleep 42',
|
||||
'interval': 10,
|
||||
'max-sync-time': 60,
|
||||
},
|
||||
'damnsmalllinux': {
|
||||
'command': 'sleep 3; exit 1',
|
||||
'interval': 15,
|
||||
'max-sync-time': 60,
|
||||
},
|
||||
'linuxmint': {
|
||||
'command': 'sleep 6',
|
||||
'interval': 20,
|
||||
'max-sync-time': 4,
|
||||
},
|
||||
}
|
||||
|
||||
merlin.mintime = 10
|
||||
merlin.earPath= 'merlin.sock'
|
||||
|
||||
if __name__ == '__main__':
|
||||
merlin.main()
|
|
@ -0,0 +1 @@
|
|||
("debian", "debian-backports", "debian-cd", "debian-multimedia", "debian-ports", "debian-security", "debian-unofficial", "debian-volatile", "ubuntu", "ubuntu-ports", "ubuntu-ports-releases", "ubuntu-releases", "xubuntu-releases")
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
#Dummy debian update script for testing
|
||||
echo "Updating debian-type mirror: $1"
|
||||
sleep 5
|
|
@ -0,0 +1 @@
|
|||
Wed Dec 30 03:17:50 UTC 2009
|
|
@ -0,0 +1 @@
|
|||
Sun Dec 27 03:17:50 UTC 2009
|
|
@ -0,0 +1 @@
|
|||
Sun Dec 27 03:17:50 UTC 2009
|
|
@ -0,0 +1 @@
|
|||
{'debian': 'foo'}
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
sleep 2
|
||||
echo "done syncing $1"
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
repos = {
|
||||
'gnu': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
'nongnu': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
'mysql': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
'mozdev': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
'gnome': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
'damnsmalllinux': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
'linuxmint': {
|
||||
'command': 'sleep 10',
|
||||
'interval': daily,
|
||||
},
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
Tue Dec 29 02:47:07 UTC 2009
|
|
@ -0,0 +1,553 @@
|
|||
projects: {}
|
||||
# CPAN:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: CPAN
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: CPAN
|
||||
# CRAN:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: CRAN
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: CRAN
|
||||
# CTAN:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: CTAN
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: CTAN
|
||||
## FreeBSD:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: FreeBSD
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: FreeBSD
|
||||
# macports-distfiles:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: MacPorts
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: MacPorts
|
||||
# macports-release:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: MacPorts
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: MacPorts
|
||||
## netbsd-pkgsrc:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: pkgsrc
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: pkgsrc
|
||||
## netbsd:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: NetBSD
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: NetBSD
|
||||
# OpenBSD:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: OpenBSD
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: OpenBSD
|
||||
# alpine:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: alpine
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: alpine
|
||||
## apache:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: apache
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: apache
|
||||
## archlinux:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: archlinux
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: archlinux
|
||||
# centos:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: centos
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: centos
|
||||
## csclub:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: csclub
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: csclub
|
||||
# cygwin:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: cygwin
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: cygwin
|
||||
## damnsmalllinux:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: damnsmalllinux
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: damnsmalllinux
|
||||
# debian:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian
|
||||
# debian-backports:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian-backports
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian-backports
|
||||
# debian-cd:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian-cd
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian-cd
|
||||
## debian-multimedia:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: debian-multimedia
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: debian-multimedia
|
||||
## debian-ports:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: debian-ports
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: debian-ports
|
||||
# debian-security:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian-security
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: debian-security
|
||||
## debian-volatile:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: debian-volatile
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: debian-volatile
|
||||
# eclipse:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: eclipse
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: eclipse
|
||||
# fedora-epel:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: fedora
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: fedora
|
||||
# fedora-enchilada:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: fedora
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: fedora
|
||||
## gentoo-distfiles:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: gentoo-distfiles
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: gentoo-distfiles
|
||||
## gentoo-portage:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: gentoo-portage
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: gentoo-portage
|
||||
## gnome:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: gnome
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: gnome
|
||||
# gnu:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: gnu
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: gnu
|
||||
# gutenberg:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: gutenberg
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: gutenberg
|
||||
## ipfire:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: ipfire
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: ipfire
|
||||
## kali:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: kali
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: kali
|
||||
## kali-images:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: kali-images
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: kali-images
|
||||
## kde:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: kde
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: kde
|
||||
## kernel.org-linux:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: kernel.org
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: kernel.org
|
||||
## kernel.org-software:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: kernel.org
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: kernel.org
|
||||
# linuxmint:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: linuxmint
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: linuxmint
|
||||
# linuxmint-packages:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: linuxmint-packages
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: linuxmint-packages
|
||||
# manjaro:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: manjaro
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: manjaro
|
||||
## mysql:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: mysql
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: mysql
|
||||
## nongnu:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: nongnu
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: nongnu
|
||||
## opensuse:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: opensuse
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: opensuse
|
||||
# pkgsrc:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: pkgsrc
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: pkgsrc
|
||||
## puppylinux:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: puppylinux
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: puppylinux
|
||||
## qtproject:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: qtproject
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: qtproject
|
||||
# plt-bundles:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: racket
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: racket
|
||||
# racket-installers:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: racket
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: racket
|
||||
# raspbian:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: raspbian
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: raspbian
|
||||
# raspberrypi:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: raspberrypi
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: raspberrypi
|
||||
## sagemath:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: sage
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: sage
|
||||
# saltstack:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror1
|
||||
# dataset: saltstack
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: saltstack
|
||||
## slackware:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: slackware
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: slackware
|
||||
## tdf:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror1
|
||||
## dataset: tdf
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: tdf
|
||||
# ubuntu:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: ubuntu
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: ubuntu
|
||||
## ubuntu-ports:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: ubuntu-ports
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: ubuntu-ports
|
||||
## ubuntu-ports-releases:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: ubuntu-ports-releases
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: ubuntu-ports-releases
|
||||
# ubuntu-releases:
|
||||
# hosts:
|
||||
# potassium-benzoate:
|
||||
# pool: cscmirror2
|
||||
# dataset: ubuntu-releases
|
||||
# phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
# pool: cscmirror1
|
||||
# dataset: ubuntu-releases
|
||||
## vlc:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: vlc
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: vlc
|
||||
## wics:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: wics
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: wics
|
||||
## x.org:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: x.org
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: x.org
|
||||
## xiph:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: xiph
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: xiph
|
||||
## xubuntu-releases:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: xubuntu-releases
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: xubuntu-releases
|
||||
## scientific:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: scientific
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: scientific
|
||||
## mxlinux:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: mxlinux
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: mxlinux
|
||||
## mxlinux-iso:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: mxlinux-iso
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: mxlinux-iso
|
||||
## parabola:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: parabola
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: parabola
|
||||
## hyperbola:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: hyperbola
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: hyperbola
|
||||
## trisquel:
|
||||
## hosts:
|
||||
## potassium-benzoate:
|
||||
## pool: cscmirror2
|
||||
## dataset: trisquel
|
||||
## phys-1002-201.cloud.cs.uwaterloo.ca:
|
||||
## pool: cscmirror1
|
||||
## dataset: trisquel
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue