Browse Source

fix gitignore

master
Max Erenberg 11 months ago committed by Mirror
parent
commit
6eea0c6584
  1. 1
      config/ADDRESS
  2. 1
      config/ADDRESS_V6
  3. 7
      debian/.bash_logout
  4. 11
      debian/.bash_profile
  5. 46
      debian/.bashrc
  6. 257
      debian/README
  7. 62
      debian/bin/dircombine
  8. 585
      debian/bin/ftpsync
  9. 112
      debian/bin/pushpdo
  10. 286
      debian/bin/runmirrors
  11. 168
      debian/bin/typicalsync
  12. 13
      debian/bin/udh
  13. 304
      debian/bin/websync
  14. 230
      debian/etc/common
  15. 137
      debian/etc/ftpsync.conf
  16. 148
      debian/etc/ftpsync.conf.sample
  17. 40
      debian/etc/pushpdo.conf.sample
  18. 21
      debian/etc/pushpdo.mirror.sample
  19. 53
      debian/etc/runmirrors.conf.sample
  20. 72
      debian/etc/runmirrors.mirror.sample
  21. 0
      debian/etc/secrets/.dummy
  22. 121
      debian/etc/websync.conf.sample
  23. 1371
      debian/mirrorcheck/bin/dmc-archive.pl
  24. 1371
      debian/mirrorcheck/bin/dmc.pl
  25. 0
      debian/mirrorcheck/www/.dummy
  26. 1
      foooooooo
  27. 3
      git_old/include/FOOTER.shtml
  28. 5
      git_old/include/HEADER.shtml
  29. 7
      git_old/include/default.css
  30. BIN
      git_old/include/favicon.ico
  31. BIN
      git_old/include/header.png
  32. 11
      git_old/include/motd.msg
  33. 2
      git_old/include/robots.txt
  34. 119
      git_old/misc/debian-check-md5sum
  35. 22
      git_old/routing/csc-mirror
  36. 48
      git_old/routing/interfaces
  37. 186
      git_old/routing/orionroutes.py
  38. 11
      git_old/routing/rt_realms
  39. 14
      git_old/routing/rt_tables
  40. 6
      git_old/routing/update-orion-routes
  41. 102
      git_old/rrdtool/mirror-nl-glue.c
  42. 25
      git_old/rrdtool/mirror-nl-glue.h
  43. 40
      git_old/rrdtool/mirror-rrd.c
  44. 39
      git_old/rrdtool/rrdgraph-monthly.sh
  45. 39
      git_old/rrdtool/rrdgraph-yearly.sh
  46. 38
      git_old/rrdtool/rrdgraph.sh
  47. 3
      git_old/snmp/.gitignore
  48. 49
      git_old/snmp/CSC-MIB.txt
  49. 11
      git_old/snmp/Makefile
  50. 221
      git_old/snmp/csc-snmp-subagent.c
  51. 109
      git_old/snmp/mirror-mib.c
  52. 9
      git_old/snmp/mirror-mib.h
  53. 102
      git_old/snmp/mirror-nl-glue.c
  54. 25
      git_old/snmp/mirror-nl-glue.h
  55. 12
      git_old/snmp/mirror-stats.c
  56. 1
      git_old/snmp/query_mib.sh
  57. 2
      git_old/snmp/snmp.conf
  58. 56
      git_old/torrents/rtorrent-init.d
  59. 100
      git_old/torrents/rtorrent.rc
  60. 599
      index.html__
  61. 1
      merlin
  62. 7
      merlin/.gitignore
  63. 25
      merlin/arthur.py
  64. 55
      merlin/init-script
  65. 716
      merlin/merlin.py
  66. 13
      merlin/merlin.service
  67. 10
      merlin/rebuild.sh
  68. 47
      merlin/test.py
  69. 1
      merlin/test/debianBased
  70. 5
      merlin/test/debian_update
  71. 1
      merlin/test/mirror/debian/trace/mirror.csclub.uwaterloo.ca
  72. 1
      merlin/test/mirror/hobo-linux/mirror.csclub.uwaterloo.ca
  73. 1
      merlin/test/mirror/ubuntu/trace/mirror.csclub.uwaterloo.ca
  74. 1
      merlin/test/mirrors
  75. 5
      merlin/test/sync
  76. 30
      merlin/test/test_repo_dict
  77. 1
      merlin/test/trace/leningradskaya.canonical.com
  78. 553
      merlin/zfssync.yml
  79. 76
      mirrormanager-client/report_mirror.conf
  80. 69
      old-crontab

1
config/ADDRESS

@ -0,0 +1 @@
129.97.134.71

1
config/ADDRESS_V6

@ -0,0 +1 @@
2620:101:f000:4901:c5c::f:1055

7
debian/.bash_logout vendored

@ -0,0 +1,7 @@
# ~/.bash_logout: executed by bash(1) when login shell exits.
# when leaving the console clear the screen to increase privacy
if [ "$SHLVL" = 1 ]; then
[ -x /usr/bin/clear_console ] && /usr/bin/clear_console -q
fi

11
debian/.bash_profile vendored

@ -0,0 +1,11 @@
# ~/.bash_profile: executed by bash(1) for login shells.
# include .bashrc if it exists
if [ -f ~/.bashrc ]; then
. ~/.bashrc
fi
# set PATH so it includes user's private bin if it exists
if [ -d ~/bin ] ; then
PATH=~/bin:"${PATH}"
fi

46
debian/.bashrc vendored

@ -0,0 +1,46 @@
# ~/.bashrc: executed by bash(1) for non-login shells.
# If not running interactively, don't do anything
[ -z "$PS1" ] && return
export HISTCONTROL=ignoreboth
# check the window size after each command and, if necessary,
# update the values of LINES and COLUMNS.
shopt -s checkwinsize
# make less more friendly for non-text input files, see lesspipe(1)
[ -x /usr/bin/lesspipe ] && eval "$(lesspipe)"
# A little nice prompt.
PS1='\[\033[01;33m\][`git branch 2>/dev/null|cut -f2 -d\* -s` ]\[\033[01;32m\]\u@\[\033[00;36m\]\h\[\033[01m\]:\[\033[01;34m\]\w\[\033[00m\]\$ '
# If this is an xterm set the title to user@host:dir
case "$TERM" in
xterm*|rxvt*)
PROMPT_COMMAND='echo -ne "\033]0;${USER}@${HOSTNAME}: ${PWD/$HOME/~}\007"'
;;
*)
;;
esac
# Alias definitions.
# enable color support of ls and also add handy aliases
eval "`dircolors -b`"
alias ls='ls --color=auto'
alias ll='ls -l'
alias la='ls -A'
alias l='ls -CF'
alias cp='cp -i'
alias mv='mv -i'
alias ..='cd ..'
# enable programmable completion features (you don't need to enable
# this, if it's already enabled in /etc/bash.bashrc and /etc/profile
# sources /etc/bash.bashrc).
if [ -f /etc/bash_completion ]; then
. /etc/bash_completion
fi

257
debian/README vendored

@ -0,0 +1,257 @@
Archvsync
=========
This is the central repository for the Debian mirror scripts. The scripts
in this repository are written for the purposes of maintaining a Debian
archive mirror (and shortly, a Debian bug mirror), but they should be
easily generalizable.
Currently the following scripts are available:
* ftpsync - Used to sync an archive using rsync
* runmirrors - Used to notify leaf nodes of available updates
* dircombine - Internal script to manage the mirror user's $HOME
on debian.org machines
* typicalsync - Generates a typical Debian mirror
* udh - We are lazy, just a shorthand to avoid typing the
commands, ignore... :)
Usage
=====
For impatient people, short usage instruction:
- Create a dedicated user for the whole mirror.
- Create a seperate directory for the mirror, writeable by the new user.
- Place the ftpsync script in the mirror user's $HOME/bin (or just $HOME)
- Place the ftpsync.conf.sample into $HOME/etc as ftpsync.conf and edit
it to suit your system. You should at the very least change the TO=
and RSYNC_HOST lines.
- Create $HOME/log (or wherever you point $LOGDIR to)
- Setup the .ssh/authorized_keys for the mirror user and place the public key of
your upstream mirror into it. Preface it with
no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="~/bin/ftpsync",from="IPADDRESS"
and replace $IPADDRESS with that of your upstream mirror.
- You are finished
In order to receive different pushes or syncs from different archives,
name the config file ftpsync-$ARCHIVE.conf and call the ftpsync script
with the commandline "sync:archive:$ARCHIVE". Replace $ARCHIVE with a
sensible value. If your upstream mirror pushes you using runmirrors
bundled together with this sync script, you do not need to add the
"sync:archive" parameter to the commandline, the scripts deal with it
automatically.
Debian mirror script minimum requirements
=========================================
As always, you may use whatever scripts you want for your Debian mirror,
but we *STRONGLY* recommend you to not invent your own. However, if you
want to be listed as a mirror it *MUST* support the following minimal
functionality:
- Must perform a 2-stage sync
The archive mirroring must be done in 2 stages. The first rsync run
must ignore the index files. The correct exclude options for the
first rsync run are:
--exclude Packages* --exclude Sources* --exclude Release* --exclude ls-lR*
The first stage must not delete any files.
The second stage should then transfer the above excluded files and
delete files that no longer belong on the mirror.
Rationale: If archive mirroring is done in a single stage, there will be
periods of time during which the index files will reference files not
yet mirrored.
- Must not ignore pushes whil(e|st) running.
If a push is received during a run of the mirror sync, it MUST NOT
be ignored. The whole synchronization process must be rerun.
Rationale: Most implementations of Debian mirror scripts will leave the
mirror in an inconsistent state in the event of a second push being
received while the first sync is still running. It is likely that in
the near future, the frequency of pushes will increase.
- Should understand multi-stage pushes.
The script should parse the arguments it gets via ssh, and if they
contain a hint to only sync stage1 or stage2, then ONLY those steps
SHOULD be performed.
Rationale: This enables us to coordinate the timing of the first
and second stage pushes and minimize the time during which the
archive is desynchronized. This is especially important for mirrors
that are involved in a round robin or GeoDNS setup.
The minimum arguments the script has to understand are:
sync:stage1 Only sync stage1
sync:stage2 Only sync stage2
sync:all Do everything. Default if none of stage1/2 are
present.
There are more possible arguments, for a complete list see the
ftpsync script in our git repository.
ftpsync
=======
This script is based on the old anonftpsync script. It has been rewritten
to add flexibilty and fix a number of outstanding issues.
Some of the advantages of the new version are:
- Nearly every aspect is configurable
- Correct support for multiple pushes
- Support for multi-stage archive synchronisations
- Support for hook scripts at various points
- Support for multiple archives, even if they are pushed using one ssh key
- Support for multi-hop, multi-stage archive synchronisations
Correct support for multiple pushes
-----------------------------------
When the script receives a second push while it is running and syncing
the archive it won't ignore it. Instead it will rerun the
synchronisation step to ensure the archive is correctly synchronised.
Scripts that fail to do that risk ending up with an inconsistent archive.
Can do multi-stage archive synchronisations
-------------------------------------------
The script can be told to only perform the first or second stage of the
archive synchronisation.
This enables us to send all the binary packages and sources to a
number of mirrors, and then tell all of them to sync the
Packages/Release files at once. This will keep the timeframe in which
the mirrors are out of sync very small and will greatly help things like
DNS RR entries or even the planned GeoDNS setup.
Multi-hop, multi-stage archive synchronisations
-----------------------------------------------
The script can be told to perform a multi-hop multi-stage archive
synchronisation.
This is basically the same as the multi-stage synchronisation
explained above, but enables the downstream mirror to push his own
staged/multi-hop downstreams before returning. This has the same
advantage than the multi-stage synchronisation but allows us to do
this over multiple level of mirrors. (Imagine one push going from
Europe to Australia, where then locally 3 others get updated before
stage2 is sent out. Instead of 4times transferring data from Europe to
Australia, just to have them all updated near instantly).
Can run hook scripts
--------------------
ftpsync currently allows 5 hook scripts to run at various points of the
mirror sync run.
Hook1: After lock is acquired, before first rsync
Hook2: After first rsync, if successful
Hook3: After second rsync, if successful
Hook4: Right before leaf mirror triggering
Hook5: After leaf mirror trigger (only if we have slave mirrors; HUB=true)
Note that Hook3 and Hook4 are likely to be called directly after each other.
The difference is that Hook3 is called *every* time the second rsync
succeeds even if the mirroring needs to re-run due to a second push.
Hook4 is only executed if mirroring is completed.
Support for multiple archives, even if they are pushed using one ssh key
------------------------------------------------------------------------
If you get multiple archives from your upstream mirror (say Debian,
Debian-Backports and Volatile), previously you had to use 3 different ssh
keys to be able to automagically synchronize them. This script can do it
all with just one key, if your upstream mirror tells you which archive.
See "Commandline/SSH options" below for further details.
For details of all available options, please see the extensive documentation
in the sample configuration file.
Commandline/SSH options
=======================
Script options may be set either on the local command line, or passed by
specifying an ssh "command". Local commandline options always have
precedence over the SSH_ORIGINAL_COMMAND ones.
Currently this script understands the options listed below. To make them
take effect they MUST be prepended by "sync:".
Option Behaviour
stage1 Only do stage1 sync
stage2 Only do stage2 sync
all Do a complete sync (default)
mhop Do a multi-hop sync
archive:foo Sync archive foo (if the file $HOME/etc/ftpsync-foo.conf
exists and is configured)
callback Call back when done (needs proper ssh setup for this to
work). It will always use the "command" callback:$HOSTNAME
where $HOSTNAME is the one defined in config and
will happen before slave mirrors are triggered.
So, to get the script to sync all of the archive behind bpo and call back when
it is complete, use an upstream trigger of
ssh $USER@$HOST sync:all sync:archive:bpo sync:callback
Mirror trace files
==================
Every mirror needs to have a 'trace' file under project/trace.
The file format is as follows:
The filename has to be the full hostname (eg. hostname -f), or in the
case of a mirror participating in RR DNS (where users will never use
the hostname) the name of the DNS RR entry, eg. security.debian.org
for the security rotation)
The content has (no leading spaces):
Sat Nov 8 13:20:22 UTC 2008
Used ftpsync version: 42
Running on host: steffani.debian.org
First line: Output of date -u
Second line: Freeform text containing the program name and version
Third line: Text "Running on host: " followed by hostname -f
The third line MUST NOT be the DNS RR name, even if the mirror is part
of it. It MUST BE the hosts own name. This is in contrast to the filename,
which SHOULD be the DNS RR name.
runmirrors
==========
This script is used to tell leaf mirrors that it is time to synchronize
their copy of the archive. This is done by parsing a mirror list and
using ssh to "push" the leaf nodes. You can read much more about the
principle behind the push at [1], essentially it tells the receiving
end to run a pre-defined script. As the whole setup is extremely limited
and the ssh key is not usable for anything else than the pre-defined
script this is the most secure method for such an action.
This script supports two types of pushes: The normal single stage push,
as well as the newer multi-stage push.
The normal push, as described above, will simply push the leaf node and
then go on with the other nodes.
The multi-staged push first pushes a mirror and tells it to only do a
stage1 sync run. Then it waits for the mirror (and all others being pushed
in the same run) to finish that run, before it tells all of the staged
mirrors to do the stage2 sync.
This way you can do a nearly-simultaneous update of multiple hosts.
This is useful in situations where periods of desynchronization should
be kept as small as possible. Examples of scenarios where this might be
useful include multiple hosts in a DNS Round Robin entry.
For details on the mirror list please see the documented
runmirrors.mirror.sample file.
[1] http://blog.ganneff.de/blog/2007/12/29/ssh-triggers.html

62
debian/bin/dircombine vendored

@ -0,0 +1,62 @@
#!/usr/bin/perl
# Uses symlinks to merge the files contained in a set of vcs
# checkouts to into a single directory. Keeps track of when files are
# removed from the merged directories and removes the symlinks.
#
# Only merges files that match the specified pattern.
#
# Note that the directories given to merge should be paths that will work
# for symlink targets from the destination directory (so either full paths,
# or they should be right inside the destination directory).
#
# Note that other files in the destination directory will be left as-is.
#
# Copyright 2006 by Joey Hess, licensed under the GPL.
if (! @ARGV) {
die "usage: dircombine include-pattern dest dir1 [dir2 ...]\n";
}
my $pattern=shift;
my $dest=shift;
foreach my $dir (@ARGV) {
my %known;
# Link in each thing from the dir.
opendir(DIR, $dir) || die "opendir: $!";
while ($_=readdir(DIR)) {
next if $_ eq '.' || $_ eq '..' || $_ eq 'known' || $_ eq '.svn' || $_ eq '.git' || $_ eq '.gitignore' || $_ eq '_darcs';
next unless /$pattern/;
$known{$_}=1;
if (! -l "$dest/$_" && -e "$dest/$_") {
print STDERR "$_ in $dir is also in $dest\n";
}
elsif (! -l "$dest/$_") {
system("ln", "-svf", "$dir/$_", $dest);
}
}
closedir(DIR);
# Remove anything that was previously linked in but is not in the
# dir anymore.
if (-e "$dir/known") {
open(KNOWN, "$dir/known") || die "open $dir/known: $!";
while (<KNOWN>) {
chomp;
if (! $known{$_}) {
system("rm", "-vf", "$dest/$_");
}
}
close KNOWN;
}
# Save state for next time.
open(KNOWN, ">$dir/known") || die "write $dir/known: $!";
foreach my $file (sort keys %known) {
print KNOWN "$file\n";
}
close KNOWN;
}

585
debian/bin/ftpsync vendored

@ -0,0 +1,585 @@
#! /bin/bash
# No, we can not deal with sh alone.
set -e
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the function is
# the important part here)
set -E
# ftpsync script for Debian
# Based losely on a number of existing scripts, written by an
# unknown number of different people over the years.
#
# Copyright (C) 2008,2009,2010,2011 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
# How the admin sets this isn't our place to deal with. One could use a wrapper
# for that. Or pam_env. Or whatever fits in the local setup. :)
BASEDIR=${BASEDIR:-"${HOME}"}
# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
# by Joerg Jaspert and the Debian mirroradm group.
# This is used to track which mirror is using which script version.
VERSION="80387"
# Source our common functions
. "${BASEDIR}/etc/common"
########################################################################
########################################################################
## functions ##
########################################################################
########################################################################
# We want to be able to get told what kind of sync we should do. This
# might be anything, from the archive to sync, the stage to do, etc. A
# list of currently understood and valid options is below. Multiple
# options are seperated by space. All the words have to have the word
# sync: in front or nothing will get used!
#
# Option Behaviour
# stage1 Only do stage1 sync
# stage2 Only do stage2 sync
# all Do a complete sync
# mhop Do a mhop sync, usually additionally to stage1
# archive:foo Sync archive foo (if config for foo is available)
# callback Call back when done (needs proper ssh setup for this to
# work). It will always use the "command" callback:$HOSTNAME
# where $HOSTNAME is the one defined below/in config and
# will happen before slave mirrors are triggered.
#
# So to get us to sync all of the archive behind bpo and call back when
# we are done, a trigger command of
# "ssh $USER@$HOST sync:all sync:archive:bpo sync:callback" will do the
# trick.
check_commandline() {
while [ $# -gt 0 ]; do
case "$1" in
sync:stage1)
SYNCSTAGE1="true"
SYNCALL="false"
;;
sync:stage2)
SYNCSTAGE2="true"
SYNCALL="false"
;;
sync:callback)
SYNCCALLBACK="true"
;;
sync:archive:*)
ARCHIVE=${1##sync:archive:}
# We do not like / or . in the remotely supplied archive name.
ARCHIVE=${ARCHIVE//\/}
ARCHIVE=${ARCHIVE//.}
;;
sync:all)
SYNCALL="true"
;;
sync:mhop)
SYNCMHOP="true"
;;
*)
echo "Unknown option ${1} ignored"
;;
esac
shift # Check next set of parameters.
done
}
# All the stuff we want to do when we exit, no matter where
cleanup() {
trap - ERR TERM HUP INT QUIT EXIT
# all done. Mail the log, exit.
log "Mirrorsync done";
# Lets get a statistical value
SPEED="unknown"
if [ -f "${LOGDIR}/rsync-${NAME}.log" ]; then
SPEED=$(
SPEEDLINE=$(egrep '[0-9.]+ bytes/sec' "${LOGDIR}/rsync-${NAME}.log")
set "nothing" ${SPEEDLINE}
echo ${8:-""}
)
if [ -n "${SPEED}" ]; then
SPEED=${SPEED%%.*}
SPEED=$(( $SPEED / 1024 ))
fi
fi
log "Rsync transfer speed: ${SPEED} KB/s"
if [ -n "${MAILTO}" ]; then
# In case rsync had something on stderr
if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"
fi
if [ "x${ERRORSONLY}x" = "xfalsex" ]; then
# And the normal log
MAILFILES="${LOG}"
if [ "x${FULLLOGS}x" = "xtruex" ]; then
# Someone wants full logs including rsync
MAILFILES="${MAILFILES} ${LOGDIR}/rsync-${NAME}.log"
fi
cat ${MAILFILES} | mail -e -s "[${PROGRAM}@$(hostname -s)] archive sync finished on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO}
fi
fi
savelog "${LOGDIR}/rsync-${NAME}.log"
savelog "${LOGDIR}/rsync-${NAME}.error"
savelog "$LOG" > /dev/null
rm -f "${LOCK}"
}
# Check rsyncs return value
check_rsync() {
ret=$1
msg=$2
# 24 - vanished source files. Ignored, that should be the target of $UPDATEREQUIRED
# and us re-running. If it's not, uplink is broken anyways.
case "${ret}" in
0) return 0;;
24) return 0;;
23) return 2;;
30) return 2;;
*)
error "ERROR: ${msg}"
return 1
;;
esac
}
########################################################################
########################################################################
# As what are we called?
NAME="$(basename $0)"
# The original command line arguments need to be saved!
if [ $# -gt 0 ]; then
ORIGINAL_COMMAND=$*
else
ORIGINAL_COMMAND=""
fi
SSH_ORIGINAL_COMMAND=${SSH_ORIGINAL_COMMAND:-""}
# Now, check if we got told about stuff via ssh
if [ -n "${SSH_ORIGINAL_COMMAND}" ]; then
# We deliberately add "nothing" and ignore it right again, to avoid
# people from outside putting some set options in the first place,
# making us parse them...
set "nothing" "${SSH_ORIGINAL_COMMAND}"
shift
# Yes, unqouted $* here. Or the function will only see it as one
# parameter, which doesnt help the case in it.
check_commandline $*
fi
# Now, we can locally override all the above variables by just putting
# them into the .ssh/authorized_keys file forced command.
if [ -n "${ORIGINAL_COMMAND}" ]; then
set ${ORIGINAL_COMMAND}
check_commandline $*
fi
# If we have been told to do stuff for a different archive than default,
# set the name accordingly.
ARCHIVE=${ARCHIVE:-""}
if [ -n "${ARCHIVE}" ]; then
NAME="${NAME}-${ARCHIVE}"
fi
# Now source the config for the archive we run on.
# (Yes, people can also overwrite the options above in the config file
# if they want to)
if [ -f "${BASEDIR}/etc/${NAME}.conf" ]; then
. "${BASEDIR}/etc/${NAME}.conf"
else
echo "Nono, you can't tell us about random archives. Bad boy!"
exit 1
fi
########################################################################
# Config options go here. Feel free to overwrite them in the config #
# file if you need to. #
# On debian.org machines the defaults should be ok. #
# #
# The following extra variables can be defined in the config file: #
# #
# ARCH_EXCLUDE #
# can be used to exclude a complete architecture from #
# mirrorring. Use as space seperated list. #
# Possible values are: #
# alpha, amd64, arm, armel, hppa, hurd-i386, i386, ia64, #
# mipsel, mips, powerpc, s390, sparc, kfreebsd-i386, kfreebsd-amd64 #
# and source. #
# eg. ARCH_EXCLUDE="alpha arm armel mipsel mips s390 sparc" #
# #
# An unset value will mirror all architectures #
########################################################################
########################################################################
# There should be nothing to edit here, use the config file #
########################################################################
MIRRORNAME=${MIRRORNAME:-$(hostname -f)}
# Where to put logfiles in
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
# Where should we put all the mirrored files?
TO=${TO:-"/org/ftp.debian.org/ftp/"}
# used by log() and error()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# Where to send mails about mirroring to?
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
# We are not on a debian.org host
MAILTO=${MAILTO:-"root"}
else
# Yay, on a .debian.org host
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
# Want errors only or every log?
ERRORSONLY=${ERRORSONLY:-"true"}
# Want full logs, ie. including the rsync one?
FULLLOGS=${FULLLOGS:-"false"}
# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}
# Our lockfile
LOCK=${LOCK:-"${TO}/Archive-Update-in-Progress-${MIRRORNAME}"}
# timeout for the lockfile, in case we have bash older than v4 (and no /proc)
LOCKTIMEOUT=${LOCKTIMEOUT:-3600}
# Do we need another rsync run?
UPDATEREQUIRED="${TO}/Archive-Update-Required-${MIRRORNAME}"
# Trace file for mirror stats and checks (make sure we get full hostname)
TRACE=${TRACE:-"project/trace/${MIRRORNAME}"}
# rsync program
RSYNC=${RSYNC:-rsync}
# Rsync filter rules. Used to protect various files we always want to keep, even if we otherwise delete
# excluded files
RSYNC_FILTER=${RSYNC_FILTER:-"--filter=protect_Archive-Update-in-Progress-${MIRRORNAME} --filter=protect_${TRACE} --filter=protect_Archive-Update-Required-${MIRRORNAME}"}
# limit I/O bandwidth. Value is KBytes per second, unset or 0 is unlimited
RSYNC_BW=${RSYNC_BW:-0}
# Default rsync options for *every* rsync call
RSYNC_OPTIONS=${RSYNC_OPTIONS:-"-prltvHSB8192 --timeout 3600 --stats ${RSYNC_FILTER}"}
# Options we only use in the first pass, where we do not want packages/sources to fly in yet and don't want to delete files
RSYNC_OPTIONS1=${RSYNC_OPTIONS1:-"--exclude Packages* --exclude Sources* --exclude Release* --exclude InRelease --exclude ls-lR*"}
# Options for the second pass, where we do want everything, including deletion of old and now unused files
RSYNC_OPTIONS2=${RSYNC_OPTIONS2:-"--max-delete=40000 --delay-updates --delete --delete-after --delete-excluded"}
# Which rsync share to use on our upstream mirror?
RSYNC_PATH=${RSYNC_PATH:-"ftp"}
# Now add the bwlimit option. As default is 0 we always add it, rsync interprets
# 0 as unlimited, so this is safe.
RSYNC_OPTIONS="--bwlimit=${RSYNC_BW} ${RSYNC_OPTIONS}"
# Connect from mirror.csclub
RSYNC_OPTIONS="--address=129.97.134.71 ${RSYNC_OPTIONS}"
# We have no default host to sync from, but will error out if its unset
RSYNC_HOST=${RSYNC_HOST:-""}
# Error out if we have no host to sync from
if [ -z "${RSYNC_HOST}" ]; then
error "Missing a host to mirror from, please set RSYNC_HOST variable in ${BASEDIR}/etc/${NAME}.conf"
fi
# our username for the rsync share
RSYNC_USER=${RSYNC_USER:-""}
# the password
RSYNC_PASSWORD=${RSYNC_PASSWORD:-""}
# a possible proxy
RSYNC_PROXY=${RSYNC_PROXY:-""}
# Do we sync stage1?
SYNCSTAGE1=${SYNCSTAGE1:-"false"}
# Do we sync stage2?
SYNCSTAGE2=${SYNCSTAGE2:-"false"}
# Do we sync all?
SYNCALL=${SYNCALL:-"true"}
# Do we have a mhop sync?
SYNCMHOP=${SYNCMHOP:-"false"}
# Do we callback?
SYNCCALLBACK=${SYNCCALLBACK:-"false"}
# If we call back we need some more options defined in the config file.
CALLBACKUSER=${CALLBACKUSER:-"archvsync"}
CALLBACKHOST=${CALLBACKHOST:-"none"}
CALLBACKKEY=${CALLBACKKEY:-"none"}
# General excludes. Don't list architecture specific stuff here, use ARCH_EXCLUDE for that!
EXCLUDE=${EXCLUDE:-""}
# The temp directory used by rsync --delay-updates is not
# world-readable remotely. Always exclude it to avoid errors.
EXCLUDE="${EXCLUDE} --exclude .~tmp~/"
SOURCE_EXCLUDE=${SOURCE_EXCLUDE:-""}
ARCH_EXCLUDE=${ARCH_EXCLUDE:-""}
# Exclude architectures defined in $ARCH_EXCLUDE
for ARCH in ${ARCH_EXCLUDE}; do
EXCLUDE="${EXCLUDE} --exclude binary-${ARCH}/ --exclude installer-${ARCH}/ --exclude Contents-${ARCH}.gz --exclude Contents-${ARCH}.bz2 --exclude Contents-${ARCH}.diff/ --exclude arch-${ARCH}.files --exclude arch-${ARCH}.list.gz --exclude *_${ARCH}.deb --exclude *_${ARCH}.udeb --exclude *_${ARCH}.changes"
if [ "${ARCH}" = "source" ]; then
if [ -z ${SOURCE_EXCLUDE} ]; then
SOURCE_EXCLUDE=" --exclude source/ --exclude *.tar.gz --exclude *.diff.gz --exclude *.tar.bz2 --exclude *.diff.bz2 --exclude *.dsc "
fi
fi
done
# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}
HOOK4=${HOOK4:-""}
HOOK5=${HOOK5:-""}
# Are we a hub?
HUB=${HUB:-"false"}
########################################################################
# Really nothing to see below here. Only code follows. #
########################################################################
########################################################################
# Some sane defaults
cd "${BASEDIR}"
umask 022
# If we are here for the first time, create the
# destination and the trace directory
mkdir -p "${TO}/project/trace"
# Used to make sure we will have the archive fully and completly synced before
# we stop, even if we get multiple pushes while this script is running.
# Otherwise we can end up with a half-synced archive:
# - get a push
# - sync, while locked
# - get another push. Of course no extra sync run then happens, we are locked.
# - done. Archive not correctly synced, we don't have all the changes from the second push.
touch "${UPDATEREQUIRED}"
# Check to see if another sync is in progress
if ! ( set -o noclobber; echo "$$" > "${LOCK}") 2> /dev/null; then
if [ ${BASH_VERSINFO[0]} -gt 3 ] || [ -L /proc/self ]; then
# We have a recent enough bash version, lets do it the easy way,
# the lock will contain the right pid, thanks to $BASHPID
if ! $(kill -0 $(cat ${LOCK}) 2>/dev/null); then
# Process does either not exist or is not owned by us.
echo "$$" > "${LOCK}"
else
echo "Unable to start rsync, lock file still exists, PID $(cat ${LOCK})"
exit 1
fi
else
# Old bash, means we dont have the right pid in our lockfile
# So take a different way - guess if it is still there by comparing its age.
# Not optimal, but hey.
stamptime=$(date --reference="${LOCK}" +%s)
unixtime=$(date +%s)
difference=$(( $unixtime - $stamptime ))
if [ ${difference} -ge ${LOCKTIMEOUT} ]; then
# Took longer than LOCKTIMEOUT minutes? Assume it broke and take the lock
echo "$$" > "${LOCK}"
else
echo "Unable to start rsync, lock file younger than one hour"
exit 1
fi
fi
fi
# When we exit normally we call cleanup on our own. Otherwise we want it called by
# this trap. (We can not trap on EXIT, because that is called when the main script
# exits. Which also happens when we background the mainroutine, ie. while we still
# run!)
trap cleanup ERR TERM HUP INT QUIT
# Start log by redirecting stdout and stderr there and closing stdin
exec >"$LOG" 2>&1 <&-
log "Mirrorsync start"
# Look who pushed us and note that in the log.
PUSHFROM="${SSH_CONNECTION%%\ *}"
if [ -n "${PUSHFROM}" ]; then
log "We got pushed from ${PUSHFROM}"
fi
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
if [ "xnonex" = "x${CALLBACKHOST}x" ] || [ "xnonex" = "x${CALLBACKKEY}x" ]; then
SYNCCALLBACK="false"
error "We are asked to call back, but we do not know where to and do not have a key, ignoring callback"
fi
fi
HOOK=(
HOOKNR=1
HOOKSCR=${HOOK1}
)
hook $HOOK
# Now, we might want to sync from anonymous too.
# This is that deep in this script so hook1 could, if wanted, change things!
if [ -z ${RSYNC_USER} ]; then
RSYNCPTH="${RSYNC_HOST}"
else
RSYNCPTH="${RSYNC_USER}@${RSYNC_HOST}"
fi
# Now do the actual mirroring, and run as long as we have an updaterequired file.
export RSYNC_PASSWORD
export RSYNC_PROXY
while [ -e "${UPDATEREQUIRED}" ]; do
log "Running mirrorsync, update is required, ${UPDATEREQUIRED} exists"
# if we want stage1 *or* all
if [ "xtruex" = "x${SYNCSTAGE1}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
while [ -e "${UPDATEREQUIRED}" ]; do
rm -f "${UPDATEREQUIRED}"
log "Running stage1: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
set +e
# Step one, sync everything except Packages/Releases
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS1} ${EXCLUDE} ${SOURCE_EXCLUDE} \
${RSYNCPTH}::${RSYNC_PATH} "${TO}" >"${LOGDIR}/rsync-${NAME}.log" 2>"${LOGDIR}/rsync-${NAME}.error"
result=$?
set -e
log "Back from rsync with returncode ${result}"
done
else
# Fake a good resultcode
result=0
fi # Sync stage 1?
rm -f "${UPDATEREQUIRED}"
set +e
check_rsync $result "Sync step 1 went wrong, got errorcode ${result}. Logfile: ${LOG}"
GO=$?
set -e
if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
elif [ ${GO} -ne 0 ]; then
exit 3
fi
HOOK=(
HOOKNR=2
HOOKSCR=${HOOK2}
)
hook $HOOK
# if we want stage2 *or* all
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
log "Running stage2: ${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} ${RSYNCPTH}::${RSYNC_PATH} ${TO}"
set +e
# We are lucky, it worked. Now do step 2 and sync again, this time including
# the packages/releases files
${RSYNC} ${RSYNC_OPTIONS} ${RSYNC_OPTIONS2} ${EXCLUDE} ${SOURCE_EXCLUDE} \
${RSYNCPTH}::${RSYNC_PATH} "${TO}" >>"${LOGDIR}/rsync-${NAME}.log" 2>>"${LOGDIR}/rsync-${NAME}.error"
result=$?
set -e
log "Back from rsync with returncode ${result}"
else
# Fake a good resultcode
result=0
fi # Sync stage 2?
set +e
check_rsync $result "Sync step 2 went wrong, got errorcode ${result}. Logfile: ${LOG}"
GO=$?
set -e
if [ ${GO} -eq 2 ] && [ -e "${UPDATEREQUIRED}" ]; then
log "We got error ${result} from rsync, but a second push went in hence ignoring this error for now"
elif [ ${GO} -ne 0 ]; then
exit 4
fi
HOOK=(
HOOKNR=3
HOOKSCR=${HOOK3}
)
hook $HOOK
done
# We only update our tracefile when we had a stage2 or an all sync.
# Otherwise we would update it after stage1 already, which is wrong.
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ]; then
if [ -d "$(dirname "${TO}/${TRACE}")" ]; then
LC_ALL=POSIX LANG=POSIX date -u > "${TO}/${TRACE}"
echo "Used ftpsync version: ${VERSION}" >> "${TO}/${TRACE}"
echo "Running on host: $(hostname -f)" >> "${TO}/${TRACE}"
fi
fi
HOOK=(
HOOKNR=4
HOOKSCR=${HOOK4}
)
hook $HOOK
if [ "xtruex" = "x${SYNCCALLBACK}x" ]; then
set +e
callback ${CALLBACKUSER} ${CALLBACKHOST} "${CALLBACKKEY}"
set -e
fi
# Remove the Archive-Update-in-Progress file before we push our downstreams.
rm -f "${LOCK}"
if [ x${HUB} = "xtrue" ]; then
# Trigger slave mirrors if we had a push for stage2 or all, or if its mhop
if [ "xtruex" = "x${SYNCSTAGE2}x" ] || [ "xtruex" = "x${SYNCALL}x" ] || [ "xtruex" = "x${SYNCMHOP}x" ]; then
RUNMIRRORARGS=""
if [ -n "${ARCHIVE}" ]; then
# We tell runmirrors about the archive we are running on.
RUNMIRRORARGS="-a ${ARCHIVE}"
fi
# We also tell runmirrors that we are running it from within ftpsync, so it can change
# the way it works with mhop based on that.
RUNMIRRORARGS="${RUNMIRRORARGS} -f"
if [ "xtruex" = "x${SYNCSTAGE1}x" ]; then
# This is true when we have a mhop sync. A normal multi-stage push sending stage1 will
# not get to this point.
# So if that happens, tell runmirrors we are doing mhop
RUNMIRRORARGS="${RUNMIRRORARGS} -k mhop"
elif [ "xtruex" = "x${SYNCSTAGE2}x" ]; then
RUNMIRRORARGS="${RUNMIRRORARGS} -k stage2"
elif [ "xtruex" = "x${SYNCALL}x" ]; then
RUNMIRRORARGS="${RUNMIRRORARGS} -k all"
fi
log "Trigger slave mirrors using ${RUNMIRRORARGS}"
${BASEDIR}/bin/runmirrors ${RUNMIRRORARGS}
log "Trigger slave done"
HOOK=(
HOOKNR=5
HOOKSCR=${HOOK5}
)
hook $HOOK
fi
fi
# All done, lets call cleanup
cleanup

112
debian/bin/pushpdo vendored

@ -0,0 +1,112 @@
#! /bin/bash
set -e
set -u
# psuhpdo script for Debian
#
# Copyright (C) 2008 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
BASEDIR=${BASEDIR:-"${HOME}"}
NAME="`basename $0`"
# Read our config file
. "${BASEDIR}/etc/${NAME}.conf"
# Source our common functions
. "${BASEDIR}/etc/common"
# Set sane defaults if the configfile didn't do that for us.
# The directory for our logfiles
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}
# Our mirrorfile
MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
# used by log()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# extra ssh options we might want hostwide
SSH_OPTS=${SSH_OPTS:-""}
# Which ssh key to use?
KEYFILE=${KEYFILE:-".ssh/pushpackages"}
# which path to "mirror"
MIRRORPATH=${MIRRORPATH:-"/org/packages.debian.org/mirror/"}
# where to send mails to
if [ "x$(hostname -s)x" != "x${MIRRORNAME%%.debian.org}x" ]; then
# We are not on a debian.org host
MAILTO=${MAILTO:-"root"}
else
# Yay, on a .debian.org host
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> ${LOG}
exit 5
fi
# Some sane defaults
cd ${BASEDIR}
umask 022
# Make sure we have our log and lock directories
mkdir -p "${LOGDIR}"
trap 'log "Pdopush done" >> ${LOG}; savelog "${LOG}" > /dev/null' EXIT
log "Pushing pdo mirrors" >> ${LOG}
# From here on we do *NOT* want to exit on errors. We don't want to
# stop pushing mirrors just because we can't reach one of them.
set +e
# Now read our mirrorfile and push the mirrors defined in there.
# We use grep to easily sort out all lines having a # in front of them or are empty.
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
while read MLNAME MHOSTNAME MUSER MPROTO MKEYFILE; do
# Process the two options that can be left blank in the config
if [ -z ${MPROTO} ]; then
MPROTO=2
fi
if [ -z ${MKEYFILE} ]; then
MKEYFILE="${BASEDIR}/${KEYFILE}"
fi
# Now, people can do stupid things and leave out the protocol, but
# define a keyfile...
if [ ${MPROTO} -ne 1 ] && [ ${MPROTO} -ne 2 ]; then
error "Need a correct ssh protocol version for ${MLNAME}, skipping" >> ${LOG}
continue
fi
# And finally, push the mirror
log "Pushing ${MLNAME}" >> ${LOG}
# This needs a limited ssh key on the other side, something like
# no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty,command="rsync --server -vlogDtpr . /srv/mirrors/packages.debian.org/",from="87.106.64.223,2001:8d8:80:11::35d,powell.debian.org" ssh-rsa.....
rsync -e "ssh -i ${MKEYFILE} -${MPROTO} ${SSH_OPTS}" -av --stats "${MIRRORPATH}" ${MUSER}@${MHOSTNAME}:/does/not/matter >"${LOGDIR}/${MLNAME}.log"
log "Pushing ${MLNAME} done" >> ${LOG}
savelog ${LOGDIR}${MLNAME}.log
set +e
done
exit 0

286
debian/bin/runmirrors vendored

@ -0,0 +1,286 @@
#! /bin/bash
set -e
set -u
# runmirrors script for Debian
# Based losely on existing scripts, written by an unknown number of
# different people over the years.
#
# Copyright (C) 2008, 2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
BASEDIR=${BASEDIR:-"${HOME}"}
NAME="$(basename $0)"
HELP="$0, (C) 2008, 2009 by Joerg Jaspert <joerg@debian.org>\n
Usage:\n\n
1.) a single parameter with NO leading -.\n
\t This will will then be used as the addition for our configfile. Ie. \`$0 security\` will\n
\t have us look for ${NAME}-security.{conf,mirror} files.\n\n
2.) using getopt style parameters:\n
\t -a [NAME] - Same as 1.) above, used for the config files. Default empty.\n
\t -k [TYPE] - Type of push. all, stage2, mhop. Default mhop.\n
\t -f - Run from within the mirrorscript ftpsync. Don't use from commandline!\n
\t -h - Print this help and exit
"
# If we got options, lets see if we use newstyle options, or oldstyle. If oldstyle
# it will not start with a -. If we find oldstyle we assume its only one, the config
# name we run on.
if [ $# -gt 0 ]; then
if [ "x${1:0:1}x" != "x-x" ]; then
# Yes, does not start with a -, so use it for the config name.
CONF=${1:-""}
if [ -n "${CONF}" ]; then
NAME="${NAME}-${CONF}"
fi
else
# Yeah well, new style, starting with - for getopts
while getopts ':a:k:fh' OPTION ; do
case $OPTION in
a) CONF="${OPTARG}"
if [ -n "${CONF}" ]; then
NAME="${NAME}-${CONF}"
fi
;;
k) PUSHKIND="${OPTARG}"
;;
f) FROMFTPSYNC="true"
;;
h) echo -e $HELP
exit 0
;;
*) echo "Invalid usage"
echo -e $HELP
exit 1
;;
esac
done
fi
fi
# Make sure the values are always defined, even if there was no commandline option
# for them
# Default config is empty
CONF=${CONF:-""}
# Set the default to all, if we didnt get told about it. Currently
# valid: all - normal push. mhop - multi-hop multi-stage push, this is stage1,
# stage2 - staged push, second phase. Default is mhop.
PUSHKIND=${PUSHKIND:-"mhop"}
# If we are pushed from within ftpsync. Default false.
FROMFTPSYNC=${FROMFTPSYNC:-"false"}
########################################################################
# Read our config file
. "${BASEDIR}/etc/${NAME}.conf"
# Source our common functions
. "${BASEDIR}/etc/common"
# Set sane defaults if the configfile didn't do that for us.
# The directory for our logfiles
LOGDIR=${LOGDIR:-"${BASEDIR}/log"}
# Our own logfile
LOG=${LOG:-"${LOGDIR}/${NAME}.log"}
# Our lockfile directory
LOCKDIR=${LOCKDIR:-"${BASEDIR}/locks"}
# How many logfiles to keep
LOGROTATE=${LOGROTATE:-14}
# Our mirrorfile
MIRRORS=${MIRRORS:-"${BASEDIR}/etc/${NAME}.mirror"}
# used by log()
PROGRAM=${PROGRAM:-"${NAME}-$(hostname -s)"}
# extra ssh options we might want hostwide
SSH_OPTS=${SSH_OPTS:-"-o StrictHostKeyChecking=no"}
# Whats our archive name? We will also tell our leafs about it
PUSHARCHIVE=${PUSHARCHIVE:-"${CONF}"}
# How long to wait for mirrors to do stage1 if we have multi-stage syncing
PUSHDELAY=${PUSHDELAY:-600}
# Which ssh key to use?
KEYFILE=${KEYFILE:-".ssh/pushmirror"}
# where to send mails to
if [ "x$(hostname -d)x" != "xdebian.orgx" ]; then
# We are not on a debian.org host
MAILTO=${MAILTO:-"root"}
else
# Yay, on a .debian.org host
MAILTO=${MAILTO:-"mirrorlogs@debian.org"}
fi
if ! [ -f "${BASEDIR}/${KEYFILE}" ]; then
error "SSH Key ${BASEDIR}/${KEYFILE} does not exist" >> "${LOG}"
exit 5
fi
# Hooks
HOOK1=${HOOK1:-""}
HOOK2=${HOOK2:-""}
HOOK3=${HOOK3:-""}
########################################################################
# Some sane defaults
cd "${BASEDIR}"
umask 022
# Make sure we have our log and lock directories
mkdir -p "${LOGDIR}"
mkdir -p "${LOCKDIR}"
trap 'log "Mirrorpush done" >> "${LOG}"; savelog "${LOG}" > /dev/null' EXIT
log "Pushing leaf mirrors. Inside ftpsync: ${FROMFTPSYNC}. Pushkind: ${PUSHKIND}" >> "${LOG}"
HOOK=(
HOOKNR=1
HOOKSCR=${HOOK1}
)
hook $HOOK
# From here on we do *NOT* want to exit on errors. We don't want to
# stop pushing mirrors just because we can't reach one of them.
set +e
# Built up our list of 2-stage mirrors.
PUSHLOCKS=""
PUSHLOCKS=$(get2stage)
# In case we have it - remove. It is used to synchronize multi-stage mirroring
rm -f "${LOCKDIR}/all_stage1"
# Now read our mirrorfile and push the mirrors defined in there.
# We use grep to easily sort out all lines having a # in front of them or are empty.
egrep -v '^[[:space:]]*(#|$)' "${MIRRORS}" |
while read MTYPE MLNAME MHOSTNAME MUSER MSSHOPT; do
if [ "x${MTYPE}x" = "xDELAYx" ]; then
# We should wait a bit.
if [ -z ${MLNAME} ]; then
MLNAME=600
fi
log "Delay of ${MLNAME} requested, sleeping" >> "${LOG}"
sleep ${MLNAME}
continue
fi
# If we are told we have a mhop sync to do and are called from within ftpsync,
# we will only look at staged/mhop entries and ignore the rest.
if [ "x${PUSHKIND}x" = "xmhopx" ] && [ "x${FROMFTPSYNC}x" = "xtruex" ]; then
if [ "x${MTYPE}x" != "xstagedx" ] && [ "x${MTYPE}x" != "xmhopx" ]; then
continue
fi
fi
# Now, MSSHOPT may start with a -. In that case the whole rest of the line is taken
# as a set of options to give to ssh, we pass it without doing anything with it.
# If it starts with a 1 or 2 then it will tell us about the ssh protocol version to use,
# and also means we look if there is one value more after a space. That value would then
# be the ssh keyfile we use with -i. That gives us full flexibility for all
# ssh options but doesn't destroy backwards compatibility.
# If it is empty we assume proto 2 and the default keyfile.
#
# There is one bug in here. We will give out the master keyfile, even if there is a
# "-i /bla/bla" in the options. ssh stuffs them together and presents two keys to the
# target server. In the case both keys do some actions- the first one presented wins.
# And this might not be what one wants.
#
# The only sane way to go around this, i think, is by dropping backward compability.
# Which I don't really like.
if [ -n "${MSSHOPT}" ]; then
# So its not empty, lets check if it starts with a - and as such is a "new-style"
# ssh options set.
if [ "x${MSSHOPT:0:1}x" = "x-x" ]; then
# Yes we start with a -
SSHOPT="${MSSHOPT}"
MPROTO="99"
MKEYFILE="${BASEDIR}/${KEYFILE}"
elif [ ${MSSHOPT:0:1} -eq 1 ] || [ ${MSSHOPT:0:1} -eq 2 ]; then
# We do seem to have oldstyle options here.
MPROTO=${MSSHOPT:0:1}
MKEYFILE=${MSSHOPT:1}
SSHOPT=""
else
error "I don't know what is configured for mirror ${MLNAME}"
continue
fi
else
MPROTO=2
MKEYFILE="${BASEDIR}/${KEYFILE}"
SSHOPT=""
fi
# Built our array
SIGNAL_OPTS=(
MIRROR="${MLNAME}"
HOSTNAME="${MHOSTNAME}"
USERNAME="${MUSER}"
SSHPROTO="${MPROTO}"
SSHKEY="${MKEYFILE}"
SSHOPTS="${SSHOPT/ /#}"
PUSHLOCKOWN="${LOCKDIR}/${MLNAME}.stage1"
PUSHTYPE="${MTYPE}"
PUSHARCHIVE=${PUSHARCHIVE}
PUSHKIND=${PUSHKIND}
FROMFTPSYNC=${FROMFTPSYNC}
)
# And finally, push the mirror
log "Trigger ${MLNAME}" >> "${LOG}"
signal "${SIGNAL_OPTS}" &
log "Trigger for ${MLNAME} done" >> "${LOG}"
HOOK=(
HOOKNR=2
HOOKSCR=${HOOK2}
)
hook $HOOK
set +e
done
# If we are run from within ftpsync *and* have an mhop push to send on, we have
# to wait until the push is gone through and they all returned, or we will exit
# much too early.
# As the signal routine touches $LOCKDIR/all_stage1 when all are done, its
# easy enough just to wait for that to appear. Of course we do the same game
# with PUSHDELAY to not wait forever.
if [ "xtruex" = "x${FROMFTPSYNC}x" ] && [ "xmhopx" = "x${PUSHKIND}x" ]; then
tries=0
# We do not wait forever
while [ ${tries} -lt ${PUSHDELAY} ]; do
if [ -f "${LOCKDIR}/all_stage1" ]; then
break
fi
tries=$((tries + 5))
sleep 5
done
if [ ${tries} -ge ${PUSHDELAY} ]; then
error "Failed to wait for our mirrors when sending mhop push down." >> "${LOG}"
fi
fi
HOOK=(
HOOKNR=3
HOOKSCR=${HOOK3}
)
hook $HOOK
exit 0

168
debian/bin/typicalsync vendored

@ -0,0 +1,168 @@
#!/usr/bin/perl -wT
# Copyright (c) 2006 Anthony Towns <ajt@debian.org>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
use strict;
use Fcntl ':flock';
use File::Find;
use POSIX qw(strftime);
# configuration:
my $local_dir = "/srv/ftp.debian.org/mirror";
my $rsync_host = undef; #"merkel.debian.org";
my $rsync_dir = undef; #"debian";
my $dest = "/srv/ftp.debian.org/rsync/typical";
my $max_del = 1000;
$ENV{"PATH"} = "/bin:/usr/bin";
# program
my $hostname = `/bin/hostname -f`;
die "bad hostname" unless $hostname =~ m/^([a-zA-Z0-9._-]+)/;
$hostname = $1;
my $lockfile = "./Archive-Update-in-Progress-$hostname";
unless (open LKFILE, "> $dest/$lockfile" and flock(LKFILE, LOCK_EX)) {
print "$hostname is unable to start sync, lock file exists\n";
exit(1);
}
if (defined $rsync_host && defined $rsync_dir) {
system("rsync --links --hard-links --times --verbose --recursive"
." --delay-updates --files-from :indices/files/typical.files"
." rsync://$rsync_host/$rsync_dir/ $dest/");
} else {
open FILELIST, "< $local_dir/indices/files/typical.files"
or die "typical.files index not found";
while (<FILELIST>) {
chomp;
m/^(.*)$/; $_ = $1;
my @l = lstat("$local_dir/$_");
next unless (@l);
if (-l _) {
my $lpath = readlink("$local_dir/$_");
$lpath =~ m/^(.*)$/; $lpath = $1;
if (-l "$dest/$_") {
next if ($lpath eq readlink("$dest/$_"));
}
unless (mk_dirname_as_dirs($dest, $_)) {
print "E: couldn't create path for $_\n";
next;
}
if (-d "$dest/$_") {
rename "$dest/$_", "$dest/$_.remove" or print "E: couldn't rename old dir $_ out of the way\n";
} elsif (-e "$dest/$_") {
unlink("$dest/$_") or print "E: couldn't unlink $_\n";
}
symlink($lpath, "$dest/$_") or print "E: couldn't create $_ as symlink to $lpath\n";
next;
}
next if (-d _);
unless (mk_dirname_as_dirs($dest, $_)) {
print "E: couldn't create path for $_\n";
next;
}
my @d = lstat("$dest/$_");
if (@d) {
if (-d _) {
rename("$dest/$_", "$dest/$_.remove") or print "E: couldn't rename old dir $_ out of the way\n";
} else {
next if (@l and @d and $l[0] == $d[0] and $l[1] == $d[1]);
#next if (@l and @d and $l[7] == $d[7]);
print "I: updating $_\n";
unlink("$dest/$_");
}
}
link("$local_dir/$_", "$dest/$_") or print "E: couldn't link $_\n";
}
close(FILELIST);
}
print "Files synced, now deleting any unnecessary files\n";
my %expected_files = ();
open FILES, "< $dest/indices/files/typical.files"
or die "typical.files index not found";
while (<FILES>) {
chomp;
$expected_files{$_} = 1;
}
close(FILES);
chdir($dest);
my $del_count = 0;
my $last = '';
finddepth({wanted => \&wanted, no_chdir => 1}, ".");
open TRACE, "> $dest/project/trace/$hostname" or die "couldn't open trace";
print TRACE strftime("%a %b %e %H:%M:%S UTC %Y", gmtime) . "\n";
close TRACE;
close LKFILE;
unlink("$dest/$lockfile");
exit(0);
sub wanted {
my ($dev,$ino,$mode,$nlink,$uid,$gid) = lstat($_);
if (-d _) {
if (substr($last, 0, length($_) + 1) ne "$_/") {
print "Deleting empty directory: $_\n";
$_ = m/^(.*)$/;
my $f = $1;
rmdir($f);
} else {
$last = $_;
}
} elsif ($_ =~ m|^\./project/trace/| or $_ eq $lockfile) {
$last = $_;
} elsif (defined $expected_files{$_}) {
$last = $_;
} elsif ($del_count < $max_del) {
$del_count++;
print "Deleting file: $_\n";
$_ = m/^(.*)$/;
my $f = $1;
unlink($f);
}
}
sub mk_dirname_as_dirs {
my ($base, $file) = @_;
while ($file =~ m,^/*([^/]+)/+([^/].*)$,) {
$file = $2;
$base = "$base/$1";
my @blah = lstat($base);
if (!@blah) {
mkdir($base, 0777);
} elsif (-l _ or ! -d _) {
print "SHOULD BE A DIRECTORY: $base\n";
unlink($base);
mkdir($base, 0777);
}
}
1;
}

13
debian/bin/udh vendored

@ -0,0 +1,13 @@
#!/bin/bash
set -e
unset LC_CTYPE
LANG=C
HOST=`hostname -f`
cd ${HOME}/archvsync
git pull
cd ${HOME}
~/archvsync/bin/dircombine . . archvsync/ >/dev/null 2>&1

304
debian/bin/websync vendored

@ -0,0 +1,304 @@
#! /bin/bash
# No, we can not deal with sh alone.
set -e
set -u
# ERR traps should be inherited from functions too. (And command
# substitutions and subshells and whatnot, but for us the function is
# the important part here)
set -E
# websync script for Debian
# Based losely on the old websync written by an
# unknown number of different people over the years and ftpsync.
#
# Copyright (C) 2008,2009 Joerg Jaspert <joerg@debian.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; version 2.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
# In case the admin somehow wants to have this script located someplace else,
# he can set BASEDIR, and we will take that. If it is unset we take ${HOME}
# How the admin sets this isn't our place to deal with. One could use a wrapper
# for that. Or pam_env. Or whatever fits in the local setup. :)
BASEDIR=${BASEDIR:-"${HOME}"}
# Script version. DO NOT CHANGE, *unless* you change the master copy maintained
# by Joerg Jaspert and the Debian mirroradm group.
# This is used to track which mirror is using which script version.
VERSION="0815"
# Source our common functions
. "${BASEDIR}/etc/common"
########################################################################
########################################################################
## functions ##
########################################################################
########################################################################
# All the stuff we want to do when we exit, no matter where
cleanup() {
trap - ERR TERM HUP INT QUIT EXIT
# all done. Mail the log, exit.
log "Mirrorsync done";
if [ -n "${MAILTO}" ]; then
# In case rsync had something on stderr
if [ -s "${LOGDIR}/rsync-${NAME}.error" ]; then
mail -e -s "[${PROGRAM}@$(hostname -s)] ($$) rsync ERROR on $(date +"%Y.%m.%d-%H:%M:%S")" ${MAILTO} < "${LOGDIR}/rsync-${NAME}.error"