diff --git a/bin/csc-sync-apache b/bin/csc-sync-apache new file mode 100755 index 0000000..038f500 --- /dev/null +++ b/bin/csc-sync-apache @@ -0,0 +1,23 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir rsync_host rsync_dir' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +RSYNC_HOST=$2 +RSYNC_DIR=$3 +ADDRESS=$(cat ~/config/ADDRESS) + +if test -n "$RSYNC_USER"; then + RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST +fi + +exec nice rsync -az --no-owner --no-group --delete --safe-links \ + --timeout=3600 -4 --address=$ADDRESS \ + --exclude .~tmp~/ \ + --quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ + $RSYNC_HOST::$RSYNC_DIR $TO diff --git a/bin/csc-sync-archlinux b/bin/csc-sync-archlinux new file mode 100755 index 0000000..baab075 --- /dev/null +++ b/bin/csc-sync-archlinux @@ -0,0 +1,93 @@ +#!/bin/bash + +# Source: https://git.server-speed.net/users/flo/bin/plain/syncrepo.sh +# (from https://bugs.archlinux.org/task/52853) + +if [ $# -lt 1 ]; then + echo "Usage: $0 local_dir" + exit 1 +fi + +umask 022 +LOCAL_DIR=$1 +ADDRESS=$(cat ~/config/ADDRESS) + +# This is a simple mirroring script. To save bandwidth it first checks a +# timestamp via HTTP and only runs rsync when the timestamp differs from the +# local copy. As of 2016, a single rsync run without changes transfers roughly +# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync +# is run every minute. Performing a simple check via HTTP first can thus save a +# lot of traffic. + +# Directory where the repo is stored locally. Example: /srv/repo +target="/mirror/root/$1" + +# Directory where files are downloaded to before being moved in place. +# This should be on the same filesystem as $target, but not a subdirectory of $target. +# Example: /srv/tmp +tmp="/home/mirror/tmp" + +# Lockfile path +lock="/home/mirror/tmp/mirrorsync-$1.lck" + +# If you want to limit the bandwidth used by rsync set this. +# Use 0 to disable the limit. +# The default unit is KiB (see man rsync /--bwlimit for more) +bwlimit=0 + +# The source URL of the mirror you want to sync from. +# If you are a tier 1 mirror use rsync.archlinux.org, for example like this: +# rsync://rsync.archlinux.org/ftp_tier1 +# Otherwise chose a tier 1 mirror from this list and use its rsync URL: +# https://www.archlinux.org/mirrors/ +source_url='rsync://rsync.archlinux.org/ftp_tier1' + +# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror. +# If you are a tier 1 mirror use: http://rsync.archlinux.org/lastupdate +# Otherwise use the HTTP(S) URL from your chosen mirror. +lastupdate_url='http://rsync.archlinux.org/lastupdate' + +#### END CONFIG + +[ ! -d "${target}" ] && mkdir -p "${target}" +[ ! -d "${tmp}" ] && mkdir -p "${tmp}" + +exec 9>"${lock}" +flock -n 9 || exit + +rsync_cmd() { + local -a cmd=(rsync -rtlH --safe-links --delete-after ${VERBOSE} "--timeout=600" "--contimeout=60" -p \ + --delay-updates --no-motd "--temp-dir=${tmp}" \ + "--log-file=/home/mirror/merlin/logs/$LOCAL_DIR.log" \ + --address=$ADDRESS) + + if stty &>/dev/null; then + cmd+=(-h -v --progress) + fi + + if ((bwlimit>0)); then + cmd+=("--bwlimit=$bwlimit") + fi + + "${cmd[@]}" "$@" +} + + +# if we are called without a tty (cronjob) only run when there are changes +if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl --interface $ADDRESS -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then + # keep lastsync file in sync for statistics generated by the Arch Linux website + rsync_cmd "$source_url/lastsync" "$target/lastsync" + exit 0 +fi + +# optional +# --exclude='*.links.tar.gz*' \ +# --exclude='/other' \ +# --exclude='/sources' \ +# --exclude='/iso' \ + +rsync_cmd \ + "${source_url}" \ + "${target}" + +#echo "Last sync was $(date -d @$(cat ${target}/lastsync))" diff --git a/bin/csc-sync-archlinux-old b/bin/csc-sync-archlinux-old new file mode 100755 index 0000000..69e8c0b --- /dev/null +++ b/bin/csc-sync-archlinux-old @@ -0,0 +1,66 @@ +#!/bin/bash + +# Source: https://git.server-speed.net/users/flo/bin/plain/syncrepo.sh +# as recommended in our request to become Tier 1: +# https://bugs.archlinux.org/task/52853 + +if [ $# -lt 1 ]; then + echo "Usage: $0 local_dir" + exit 1 +fi + +umask 022 +ADDRESS=$(cat ~/config/ADDRESS) + +# This is a simple mirroring script. To save bandwidth it first checks a +# timestamp via HTTP and only runs rsync when the timestamp differs from the +# local copy. As of 2016, a single rsync run without changes transfers roughly +# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync +# is run every minute. Performing a simple check via HTTP first can thus save a +# lot of traffic. + +home="/home/mirror" +target=/mirror/root/$1 +tmp="${home}/tmp" +lock="${tmp}/mirrorsync-$1.lck" +# NOTE: You'll probably want to change this or remove the --bwlimit setting in +# the rsync call below +bwlimit=4096 +# NOTE: most people reading this very likely need to change this since +# rsync.archlinux.org requires you to be a tier 1 mirror +source='rsync://rsync.archlinux.org/ftp_tier1' +lastupdate_url="http://rsync.archlinux.org/lastupdate" + +[ ! -d "${target}" ] && mkdir -p "${target}" +[ ! -d "${tmp}" ] && mkdir -p "${tmp}" + +exec 9>"${lock}" +flock -n 9 || exit + +# if we are called without a tty (cronjob) only run when there are changes +if ! tty -s && diff -b <(curl --interface $ADDRESS -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then + date +'%s' > "$target/lastsync" + exit 0 +fi + +if ! stty &>/dev/null; then + QUIET="-q" +fi + +# this can be added into the rsync: --bwlimit=$bwlimit \ +# --exclude='/iso' \ +# --exclude='*.links.tar.gz*' \ +# --exclude='/other' \ +# --exclude='/sources' \ + +rsync -rtlvH --safe-links --delete-after --progress -h ${QUIET} --timeout=600 --contimeout=60 -p \ + --delay-updates --no-motd \ + --temp-dir="${tmp}" \ + --stats \ + --log-file=$home/merlin/logs/$1.log \ + --address=$ADDRESS \ + ${source} \ + "${target}" + +date +'%s' > "$target/lastsync" +#echo "Last sync was $(date -d @$(cat ${target}/lastsync))" diff --git a/bin/csc-sync-badperms b/bin/csc-sync-badperms new file mode 100755 index 0000000..0dc376f --- /dev/null +++ b/bin/csc-sync-badperms @@ -0,0 +1,23 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir rsync_host rsync_dir' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +RSYNC_HOST=$2 +RSYNC_DIR=$3 +ADDRESS=$(cat ~/config/ADDRESS) + +if test -n "$RSYNC_USER"; then + RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST +fi + +exec nice rsync -aH --no-owner --no-group --chmod=o=rX --delete \ + --timeout=3600 -4 --address=$ADDRESS \ + --exclude .~tmp~/ \ + --quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ + $RSYNC_HOST::$RSYNC_DIR $TO diff --git a/bin/csc-sync-cdimage b/bin/csc-sync-cdimage new file mode 100755 index 0000000..0bfcb22 --- /dev/null +++ b/bin/csc-sync-cdimage @@ -0,0 +1,25 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir rsync_host rsync_dir' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +RSYNC_HOST=$2 +RSYNC_DIR=$3 +ADDRESS=$(cat ~/config/ADDRESS) + +if test -n "$RSYNC_USER"; then + RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST +fi + +mkdir -p $TO + +exec nice rsync -aH --no-owner --no-group --delete \ + --timeout=3600 -4 --address=$ADDRESS \ + --exclude ".*/" \ + --quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ + $RSYNC_HOST::$RSYNC_DIR $TO diff --git a/bin/csc-sync-ceph b/bin/csc-sync-ceph new file mode 100755 index 0000000..c65101e --- /dev/null +++ b/bin/csc-sync-ceph @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +set -e +# +# Script to mirror Ceph locally +# +# Please, choose a local source and do not sync in a shorter interval than +# 3 hours. +# +SILENT=0 + +# All available source mirrors +declare -A SOURCES +SOURCES[eu]="eu.ceph.com" +SOURCES[de]="de.ceph.com" +SOURCES[se]="se.ceph.com" +SOURCES[au]="au.ceph.com" +SOURCES[us]="download.ceph.com" +SOURCES[hk]="hk.ceph.com" +SOURCES[fr]="fr.ceph.com" +SOURCES[us-east]="us-east.ceph.com" +SOURCES[us-west]="us-west.ceph.com" +SOURCES[global]="download.ceph.com" + +function print_usage() { + echo "$0 [-q ] -s -t " +} + +while getopts ":qhs:t:" opt; do + case $opt in + q) + SILENT=1 + ;; + s) + SOURCE=$OPTARG + ;; + t) + TARGET=/mirror/root/$OPTARG + ;; + h) + HELP=1 + ;; + \?) + print_usage + exit 1 + ;; + esac +done + +if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then + print_usage + exit 1 +fi + +if [ ! -d "$TARGET" ]; then + echo "$TARGET is not a valid target directory" + exit 1 +fi + +for i in "${!SOURCES[@]}"; do + if [ "$i" == "$SOURCE" ]; then + SOURCE_HOST=${SOURCES[$i]} + fi +done + +if [ -z "$SOURCE_HOST" ]; then + echo -n "Please select one of the following sources:" + for i in "${!SOURCES[@]}"; do + echo -n " $i" + done + echo "" + exit 1 +fi + +RSYNC_OPTS="--stats --progress" +if [ $SILENT -eq 1 ]; then + RSYNC_OPTS="--quiet" +fi + +ADDRESS=$(cat ~/config/ADDRESS) +RSYNC_OPTS="$RSYNC_OPTS -4 --address=$ADDRESS" + +# We start a two-stage sync here for DEB and RPM +# Based on: https://www.debian.org/mirror/ftpmirror +# +# The idea is to prevent temporary situations where metadata points to files +# which do not exist +# + +# Exclude all metadata files +rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \ + --hard-links \ + --exclude Packages* \ + --exclude Sources* \ + --exclude Release* \ + --exclude InRelease \ + --exclude i18n/* \ + --exclude ls-lR* \ + --exclude repodata/* \ + ${TARGET} + +# Now also transfer the metadata and delete afterwards +rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \ + --hard-links --delete-after \ + ${TARGET} + diff --git a/bin/csc-sync-chmod b/bin/csc-sync-chmod new file mode 100755 index 0000000..58e7666 --- /dev/null +++ b/bin/csc-sync-chmod @@ -0,0 +1,27 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir rsync_host rsync_dir' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +RSYNC_HOST=$2 +RSYNC_DIR=$3 +ADDRESS=$(cat ~/config/ADDRESS) + +if test -n "$RSYNC_USER"; then + RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST +fi + +mkdir -p $TO + +exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \ + --timeout=3600 -4 --address=$ADDRESS \ + --exclude .~tmp~/ \ + --quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ + --chmod=u=rwX,go=rX \ + rsync://$RSYNC_HOST/$RSYNC_DIR $TO + #$RSYNC_HOST::$RSYNC_DIR $TO diff --git a/bin/csc-sync-debian b/bin/csc-sync-debian index 50f06cd..09e11ba 100755 --- a/bin/csc-sync-debian +++ b/bin/csc-sync-debian @@ -19,7 +19,7 @@ if [ $# = 5 ]; then TRACE_DIR=$5 fi LOGDIR=/var/log/mirror/$1_$2 -ADDRESS=129.97.134.71 +ADDRESS=$(cat ~/config/ADDRESS) mkdir -p $LOGDIR @@ -213,13 +213,12 @@ if [[ "$TRACE_HOST" != "" ]]; then rm -f $TRACE_NEW_FILE if [ "$TRACE_OLD_TIME" = "$TRACE_NEW_TIME" ]; then echo 'Trace file for' $RSYNC_HOST::$RSYNC_DIR \ - 'unchanged, not rsyncing.' >> $LOGFILE - exit 0 + 'unchanged!' >> $LOGFILE fi fi # First sync /pool -nice rsync -rlHtv \ +nice rsync -rlHtvp \ $TMP_EXCLUDE $EXCLUDE $SOURCE_EXCLUDE \ --timeout=3600 -4 --address=$ADDRESS \ $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1 @@ -227,7 +226,7 @@ result=$? if [ 0 = $result ]; then # Now sync the remaining stuff - nice rsync -rlHtv --delay-updates --delete-after \ + nice rsync -rlHtvp --delay-updates --delete-after \ --exclude "Archive-Update-in-Progress-${HOSTNAME}" \ --exclude "${TRACE_DIR}/${HOSTNAME}" \ --timeout=3600 -4 --address=$ADDRESS \ diff --git a/bin/csc-sync-debian-cd b/bin/csc-sync-debian-cd index 11abcbc..0c85034 100755 --- a/bin/csc-sync-debian-cd +++ b/bin/csc-sync-debian-cd @@ -7,7 +7,7 @@ TO=$TOP_DIR/root/debian-cd RSYNC_HOST=cdimage.debian.org RSYNC_DIR=debian-cd LOGDIR=/var/log/mirror/debian-cd_cdimage.debian.org -ADDRESS=129.97.134.71 +ADDRESS=$(cat ~/config/ADDRESS) mkdir -p $LOGDIR @@ -93,13 +93,8 @@ trap "rm -f $LOCK" exit set +e # Now sync the remaining stuff -nice rsync -rlHtv --delete \ - --include='*-businesscard.iso' \ - --include='*-netinst.iso' \ - --include='*-CD-1.iso' \ - --include='*-amd64-DVD-1.iso' \ - --include='*-i386-DVD-1.iso' \ - --exclude='*.iso' --timeout=3600 -4 --address=$ADDRESS $TMP_EXCLUDE \ +nice rsync -rlHtvp --delete \ + --timeout=3600 -4 --address=$ADDRESS $TMP_EXCLUDE \ --exclude "Archive-Update-in-Progress-${HOSTNAME}" \ $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1 if [[ "$?" != "0" ]]; then diff --git a/bin/csc-sync-gentoo b/bin/csc-sync-gentoo new file mode 100755 index 0000000..f5c9805 --- /dev/null +++ b/bin/csc-sync-gentoo @@ -0,0 +1,3 @@ +#!/bin/sh + +RSYNC_USER=gentoo exec ~/bin/csc-sync-standard gentoo-distfiles masterdistfiles.gentoo.org gentoo gentoo-distfiles diff --git a/bin/csc-sync-s3 b/bin/csc-sync-s3 new file mode 100755 index 0000000..ff3d7bb --- /dev/null +++ b/bin/csc-sync-s3 @@ -0,0 +1,16 @@ +#!/bin/bash -e + +if [ $# -lt 2 ]; then + echo 'Usage: sync local_dir endpoint' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +export RCLONE_CONFIG_S3_ENDPOINT=$2 RCLONE_CONFIG_S3_TYPE=s3 RCLONE_CONFIG_S3_PROVIDER=Other RCLONE_CONFIG_S3_ENV_AUTH=false +ADDRESS=$(cat ~/config/ADDRESS) + +mkdir -p $TO +cd $TO +exec nice rclone sync --fast-list --use-server-modtime --bind $ADDRESS s3:s3/ $TO diff --git a/bin/csc-sync-ssh b/bin/csc-sync-ssh new file mode 100755 index 0000000..1db4a5a --- /dev/null +++ b/bin/csc-sync-ssh @@ -0,0 +1,24 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir rsync_host rsync_dir ssh_user ssh_keyfile' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +RSYNC_HOST=$2 +RSYNC_DIR=$3 +SSH_USER=$4 +SSH_KEYFILE=$5 +RSYNC_HOST=$SSH_USER@$RSYNC_HOST + +exec nice rsync -aH --no-owner --no-group --delete \ + --timeout=3600 -4 \ + --exclude .~tmp~/ \ + --quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \ + -e "ssh -i $SSH_KEYFILE" \ + $RSYNC_HOST:$RSYNC_DIR/ $TO + +#134.71 needs to be used diff --git a/bin/csc-sync-standard b/bin/csc-sync-standard index e1ad951..9ba40d6 100755 --- a/bin/csc-sync-standard +++ b/bin/csc-sync-standard @@ -1,7 +1,7 @@ #!/bin/bash -e if [ $# -lt 3 ]; then - echo 'Usage: sync local_dir rsync_host rsync_dir' + echo 'Usage: sync local_dir rsync_host rsync_dir [password file]' exit 1 fi @@ -10,12 +10,22 @@ umask 002 TO=/mirror/root/$1 RSYNC_HOST=$2 RSYNC_DIR=$3 +RSYNC_PASSWORD_ARGS= +if [ $# -ge 4 ]; then + RSYNC_PASSWORD_ARGS="--password-file $HOME/passwords/$4" +fi +ADDRESS=$(cat ~/config/ADDRESS) + if test -n "$RSYNC_USER"; then RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST fi -exec nice rsync -aH --no-owner --no-group --delete \ - --timeout=3600 -4 --address=129.97.134.71 \ +mkdir -p $TO + +exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \ + --timeout=3600 -4 --address=$ADDRESS \ --exclude .~tmp~/ \ --quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ - $RSYNC_HOST::$RSYNC_DIR $TO + $RSYNC_PASSWORD_ARGS \ + rsync://$RSYNC_HOST/$RSYNC_DIR $TO + #$RSYNC_HOST::$RSYNC_DIR $TO diff --git a/bin/csc-sync-standard-ipv6 b/bin/csc-sync-standard-ipv6 new file mode 100755 index 0000000..f256eb4 --- /dev/null +++ b/bin/csc-sync-standard-ipv6 @@ -0,0 +1,25 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir rsync_host rsync_dir' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +RSYNC_HOST=$2 +RSYNC_DIR=$3 +ADDRESS=$(cat ~/config/ADDRESS_V6) + +if test -n "$RSYNC_USER"; then + RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST +fi + +mkdir -p $TO + +exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \ + --timeout=3600 -6 --address=$ADDRESS \ + --exclude .~tmp~/ \ + --quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ + $RSYNC_HOST::$RSYNC_DIR $TO diff --git a/bin/csc-sync-wget b/bin/csc-sync-wget new file mode 100755 index 0000000..c0430dc --- /dev/null +++ b/bin/csc-sync-wget @@ -0,0 +1,17 @@ +#!/bin/bash -e + +if [ $# -lt 3 ]; then + echo 'Usage: sync local_dir path cut' + exit 1 +fi + +umask 002 + +TO=/mirror/root/$1 +SOURCE_PATH=$2 +CUT=$3 +ADDRESS=$(cat ~/config/ADDRESS) + +mkdir -p $TO +cd $TO +exec nice wget -q --bind-address=$ADDRESS --mirror --no-parent --no-host-directories --cut-dirs=$CUT --content-disposition --execute robots=off --recursive --reject "*\?*" $SOURCE_PATH diff --git a/bin/report_mirror b/bin/report_mirror index 2f1054f..24a0514 100755 --- a/bin/report_mirror +++ b/bin/report_mirror @@ -49,7 +49,7 @@ def gen_dirtree(path): # }, # ... # } - # + # # 2009-03-09: MM's web app ignores the statfiles dict. So don't bother generating it. dirtree = {} @@ -192,7 +192,7 @@ def main(): parser = OptionParser(usage= sys.argv[0] + " [options]") parser.add_option("-c", "--config", dest="config", - default='/etc/mirrormanager-client/report_mirror.conf', + default='/home/mirror/mirrormanager-client/report_mirror.conf', help='Configuration filename (required)') parser.add_option("-s", "--stats", action="store_true", @@ -236,7 +236,7 @@ def main(): if options.debug: pp = pprint.PrettyPrinter(indent=4) pp.pprint(item.config) - + if options.output is not None: outfile = open(options.output, 'wb') outfile.write(p) @@ -265,7 +265,7 @@ def main(): print "Error checking in: %s. Please try again later." % (m[1]) except xmlrpclib.Fault: print "Error checking in. Connection closed before checkin complete. Please try again later." - sys.exit(1) + sys.exit(1) if __name__ == '__main__': diff --git a/bin/zfssync b/bin/zfssync new file mode 100755 index 0000000..38d2449 --- /dev/null +++ b/bin/zfssync @@ -0,0 +1,10 @@ +#!/bin/bash + +if [ $# -ne 1 ]; then + echo 'Usage: project' 1>&2 + exit 1 +fi + +exec 200>/tmp/$1.lock +flock 200 || exit 1 +sudo /usr/local/bin/zfssync.py $1