root
parent
a892861162
commit
e84c1e71bb
@ -0,0 +1,23 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir rsync_host rsync_dir' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
RSYNC_HOST=$2 |
||||
RSYNC_DIR=$3 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
if test -n "$RSYNC_USER"; then |
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST |
||||
fi |
||||
|
||||
exec nice rsync -az --no-owner --no-group --delete --safe-links \ |
||||
--timeout=3600 -4 --address=$ADDRESS \ |
||||
--exclude .~tmp~/ \ |
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ |
||||
$RSYNC_HOST::$RSYNC_DIR $TO |
||||
@ -0,0 +1,93 @@ |
||||
#!/bin/bash |
||||
|
||||
# Source: https://git.server-speed.net/users/flo/bin/plain/syncrepo.sh |
||||
# (from https://bugs.archlinux.org/task/52853) |
||||
|
||||
if [ $# -lt 1 ]; then |
||||
echo "Usage: $0 local_dir" |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 022 |
||||
LOCAL_DIR=$1 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
# This is a simple mirroring script. To save bandwidth it first checks a |
||||
# timestamp via HTTP and only runs rsync when the timestamp differs from the |
||||
# local copy. As of 2016, a single rsync run without changes transfers roughly |
||||
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync |
||||
# is run every minute. Performing a simple check via HTTP first can thus save a |
||||
# lot of traffic. |
||||
|
||||
# Directory where the repo is stored locally. Example: /srv/repo |
||||
target="/mirror/root/$1" |
||||
|
||||
# Directory where files are downloaded to before being moved in place. |
||||
# This should be on the same filesystem as $target, but not a subdirectory of $target. |
||||
# Example: /srv/tmp |
||||
tmp="/home/mirror/tmp" |
||||
|
||||
# Lockfile path |
||||
lock="/home/mirror/tmp/mirrorsync-$1.lck" |
||||
|
||||
# If you want to limit the bandwidth used by rsync set this. |
||||
# Use 0 to disable the limit. |
||||
# The default unit is KiB (see man rsync /--bwlimit for more) |
||||
bwlimit=0 |
||||
|
||||
# The source URL of the mirror you want to sync from. |
||||
# If you are a tier 1 mirror use rsync.archlinux.org, for example like this: |
||||
# rsync://rsync.archlinux.org/ftp_tier1 |
||||
# Otherwise chose a tier 1 mirror from this list and use its rsync URL: |
||||
# https://www.archlinux.org/mirrors/ |
||||
source_url='rsync://rsync.archlinux.org/ftp_tier1' |
||||
|
||||
# An HTTP(S) URL pointing to the 'lastupdate' file on your chosen mirror. |
||||
# If you are a tier 1 mirror use: http://rsync.archlinux.org/lastupdate |
||||
# Otherwise use the HTTP(S) URL from your chosen mirror. |
||||
lastupdate_url='http://rsync.archlinux.org/lastupdate' |
||||
|
||||
#### END CONFIG |
||||
|
||||
[ ! -d "${target}" ] && mkdir -p "${target}" |
||||
[ ! -d "${tmp}" ] && mkdir -p "${tmp}" |
||||
|
||||
exec 9>"${lock}" |
||||
flock -n 9 || exit |
||||
|
||||
rsync_cmd() { |
||||
local -a cmd=(rsync -rtlH --safe-links --delete-after ${VERBOSE} "--timeout=600" "--contimeout=60" -p \ |
||||
--delay-updates --no-motd "--temp-dir=${tmp}" \ |
||||
"--log-file=/home/mirror/merlin/logs/$LOCAL_DIR.log" \ |
||||
--address=$ADDRESS) |
||||
|
||||
if stty &>/dev/null; then |
||||
cmd+=(-h -v --progress) |
||||
fi |
||||
|
||||
if ((bwlimit>0)); then |
||||
cmd+=("--bwlimit=$bwlimit") |
||||
fi |
||||
|
||||
"${cmd[@]}" "$@" |
||||
} |
||||
|
||||
|
||||
# if we are called without a tty (cronjob) only run when there are changes |
||||
if ! tty -s && [[ -f "$target/lastupdate" ]] && diff -b <(curl --interface $ADDRESS -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then |
||||
# keep lastsync file in sync for statistics generated by the Arch Linux website |
||||
rsync_cmd "$source_url/lastsync" "$target/lastsync" |
||||
exit 0 |
||||
fi |
||||
|
||||
# optional |
||||
# --exclude='*.links.tar.gz*' \ |
||||
# --exclude='/other' \ |
||||
# --exclude='/sources' \ |
||||
# --exclude='/iso' \ |
||||
|
||||
rsync_cmd \ |
||||
"${source_url}" \ |
||||
"${target}" |
||||
|
||||
#echo "Last sync was $(date -d @$(cat ${target}/lastsync))" |
||||
@ -0,0 +1,66 @@ |
||||
#!/bin/bash |
||||
|
||||
# Source: https://git.server-speed.net/users/flo/bin/plain/syncrepo.sh |
||||
# as recommended in our request to become Tier 1: |
||||
# https://bugs.archlinux.org/task/52853 |
||||
|
||||
if [ $# -lt 1 ]; then |
||||
echo "Usage: $0 local_dir" |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 022 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
# This is a simple mirroring script. To save bandwidth it first checks a |
||||
# timestamp via HTTP and only runs rsync when the timestamp differs from the |
||||
# local copy. As of 2016, a single rsync run without changes transfers roughly |
||||
# 6MiB of data which adds up to roughly 250GiB of traffic per month when rsync |
||||
# is run every minute. Performing a simple check via HTTP first can thus save a |
||||
# lot of traffic. |
||||
|
||||
home="/home/mirror" |
||||
target=/mirror/root/$1 |
||||
tmp="${home}/tmp" |
||||
lock="${tmp}/mirrorsync-$1.lck" |
||||
# NOTE: You'll probably want to change this or remove the --bwlimit setting in |
||||
# the rsync call below |
||||
bwlimit=4096 |
||||
# NOTE: most people reading this very likely need to change this since |
||||
# rsync.archlinux.org requires you to be a tier 1 mirror |
||||
source='rsync://rsync.archlinux.org/ftp_tier1' |
||||
lastupdate_url="http://rsync.archlinux.org/lastupdate" |
||||
|
||||
[ ! -d "${target}" ] && mkdir -p "${target}" |
||||
[ ! -d "${tmp}" ] && mkdir -p "${tmp}" |
||||
|
||||
exec 9>"${lock}" |
||||
flock -n 9 || exit |
||||
|
||||
# if we are called without a tty (cronjob) only run when there are changes |
||||
if ! tty -s && diff -b <(curl --interface $ADDRESS -s "$lastupdate_url") "$target/lastupdate" >/dev/null; then |
||||
date +'%s' > "$target/lastsync" |
||||
exit 0 |
||||
fi |
||||
|
||||
if ! stty &>/dev/null; then |
||||
QUIET="-q" |
||||
fi |
||||
|
||||
# this can be added into the rsync: --bwlimit=$bwlimit \ |
||||
# --exclude='/iso' \ |
||||
# --exclude='*.links.tar.gz*' \ |
||||
# --exclude='/other' \ |
||||
# --exclude='/sources' \ |
||||
|
||||
rsync -rtlvH --safe-links --delete-after --progress -h ${QUIET} --timeout=600 --contimeout=60 -p \ |
||||
--delay-updates --no-motd \ |
||||
--temp-dir="${tmp}" \ |
||||
--stats \ |
||||
--log-file=$home/merlin/logs/$1.log \ |
||||
--address=$ADDRESS \ |
||||
${source} \ |
||||
"${target}" |
||||
|
||||
date +'%s' > "$target/lastsync" |
||||
#echo "Last sync was $(date -d @$(cat ${target}/lastsync))" |
||||
@ -0,0 +1,23 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir rsync_host rsync_dir' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
RSYNC_HOST=$2 |
||||
RSYNC_DIR=$3 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
if test -n "$RSYNC_USER"; then |
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST |
||||
fi |
||||
|
||||
exec nice rsync -aH --no-owner --no-group --chmod=o=rX --delete \ |
||||
--timeout=3600 -4 --address=$ADDRESS \ |
||||
--exclude .~tmp~/ \ |
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ |
||||
$RSYNC_HOST::$RSYNC_DIR $TO |
||||
@ -0,0 +1,25 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir rsync_host rsync_dir' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
RSYNC_HOST=$2 |
||||
RSYNC_DIR=$3 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
if test -n "$RSYNC_USER"; then |
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST |
||||
fi |
||||
|
||||
mkdir -p $TO |
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete \ |
||||
--timeout=3600 -4 --address=$ADDRESS \ |
||||
--exclude ".*/" \ |
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ |
||||
$RSYNC_HOST::$RSYNC_DIR $TO |
||||
@ -0,0 +1,105 @@ |
||||
#!/usr/bin/env bash |
||||
set -e |
||||
# |
||||
# Script to mirror Ceph locally |
||||
# |
||||
# Please, choose a local source and do not sync in a shorter interval than |
||||
# 3 hours. |
||||
# |
||||
SILENT=0 |
||||
|
||||
# All available source mirrors |
||||
declare -A SOURCES |
||||
SOURCES[eu]="eu.ceph.com" |
||||
SOURCES[de]="de.ceph.com" |
||||
SOURCES[se]="se.ceph.com" |
||||
SOURCES[au]="au.ceph.com" |
||||
SOURCES[us]="download.ceph.com" |
||||
SOURCES[hk]="hk.ceph.com" |
||||
SOURCES[fr]="fr.ceph.com" |
||||
SOURCES[us-east]="us-east.ceph.com" |
||||
SOURCES[us-west]="us-west.ceph.com" |
||||
SOURCES[global]="download.ceph.com" |
||||
|
||||
function print_usage() { |
||||
echo "$0 [-q ] -s <source mirror> -t <target directory>" |
||||
} |
||||
|
||||
while getopts ":qhs:t:" opt; do |
||||
case $opt in |
||||
q) |
||||
SILENT=1 |
||||
;; |
||||
s) |
||||
SOURCE=$OPTARG |
||||
;; |
||||
t) |
||||
TARGET=/mirror/root/$OPTARG |
||||
;; |
||||
h) |
||||
HELP=1 |
||||
;; |
||||
\?) |
||||
print_usage |
||||
exit 1 |
||||
;; |
||||
esac |
||||
done |
||||
|
||||
if [ ! -z "$HELP" ] || [ -z "$TARGET" ] || [ -z "$SOURCE" ]; then |
||||
print_usage |
||||
exit 1 |
||||
fi |
||||
|
||||
if [ ! -d "$TARGET" ]; then |
||||
echo "$TARGET is not a valid target directory" |
||||
exit 1 |
||||
fi |
||||
|
||||
for i in "${!SOURCES[@]}"; do |
||||
if [ "$i" == "$SOURCE" ]; then |
||||
SOURCE_HOST=${SOURCES[$i]} |
||||
fi |
||||
done |
||||
|
||||
if [ -z "$SOURCE_HOST" ]; then |
||||
echo -n "Please select one of the following sources:" |
||||
for i in "${!SOURCES[@]}"; do |
||||
echo -n " $i" |
||||
done |
||||
echo "" |
||||
exit 1 |
||||
fi |
||||
|
||||
RSYNC_OPTS="--stats --progress" |
||||
if [ $SILENT -eq 1 ]; then |
||||
RSYNC_OPTS="--quiet" |
||||
fi |
||||
|
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
RSYNC_OPTS="$RSYNC_OPTS -4 --address=$ADDRESS" |
||||
|
||||
# We start a two-stage sync here for DEB and RPM |
||||
# Based on: https://www.debian.org/mirror/ftpmirror |
||||
# |
||||
# The idea is to prevent temporary situations where metadata points to files |
||||
# which do not exist |
||||
# |
||||
|
||||
# Exclude all metadata files |
||||
rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \ |
||||
--hard-links \ |
||||
--exclude Packages* \ |
||||
--exclude Sources* \ |
||||
--exclude Release* \ |
||||
--exclude InRelease \ |
||||
--exclude i18n/* \ |
||||
--exclude ls-lR* \ |
||||
--exclude repodata/* \ |
||||
${TARGET} |
||||
|
||||
# Now also transfer the metadata and delete afterwards |
||||
rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \ |
||||
--hard-links --delete-after \ |
||||
${TARGET} |
||||
|
||||
@ -0,0 +1,27 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir rsync_host rsync_dir' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
RSYNC_HOST=$2 |
||||
RSYNC_DIR=$3 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
if test -n "$RSYNC_USER"; then |
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST |
||||
fi |
||||
|
||||
mkdir -p $TO |
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \ |
||||
--timeout=3600 -4 --address=$ADDRESS \ |
||||
--exclude .~tmp~/ \ |
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ |
||||
--chmod=u=rwX,go=rX \ |
||||
rsync://$RSYNC_HOST/$RSYNC_DIR $TO |
||||
#$RSYNC_HOST::$RSYNC_DIR $TO |
||||
@ -0,0 +1,3 @@ |
||||
#!/bin/sh |
||||
|
||||
RSYNC_USER=gentoo exec ~/bin/csc-sync-standard gentoo-distfiles masterdistfiles.gentoo.org gentoo gentoo-distfiles |
||||
@ -0,0 +1,16 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 2 ]; then |
||||
echo 'Usage: sync local_dir endpoint' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
export RCLONE_CONFIG_S3_ENDPOINT=$2 RCLONE_CONFIG_S3_TYPE=s3 RCLONE_CONFIG_S3_PROVIDER=Other RCLONE_CONFIG_S3_ENV_AUTH=false |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
mkdir -p $TO |
||||
cd $TO |
||||
exec nice rclone sync --fast-list --use-server-modtime --bind $ADDRESS s3:s3/ $TO |
||||
@ -0,0 +1,24 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir rsync_host rsync_dir ssh_user ssh_keyfile' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
RSYNC_HOST=$2 |
||||
RSYNC_DIR=$3 |
||||
SSH_USER=$4 |
||||
SSH_KEYFILE=$5 |
||||
RSYNC_HOST=$SSH_USER@$RSYNC_HOST |
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete \ |
||||
--timeout=3600 -4 \ |
||||
--exclude .~tmp~/ \ |
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer-ssh.log \ |
||||
-e "ssh -i $SSH_KEYFILE" \ |
||||
$RSYNC_HOST:$RSYNC_DIR/ $TO |
||||
|
||||
#134.71 needs to be used |
||||
@ -0,0 +1,25 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir rsync_host rsync_dir' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
RSYNC_HOST=$2 |
||||
RSYNC_DIR=$3 |
||||
ADDRESS=$(cat ~/config/ADDRESS_V6) |
||||
|
||||
if test -n "$RSYNC_USER"; then |
||||
RSYNC_HOST=$RSYNC_USER@$RSYNC_HOST |
||||
fi |
||||
|
||||
mkdir -p $TO |
||||
|
||||
exec nice rsync -aH --no-owner --no-group --delete-after --delay-updates --safe-links \ |
||||
--timeout=3600 -6 --address=$ADDRESS \ |
||||
--exclude .~tmp~/ \ |
||||
--quiet --stats --log-file=/home/mirror/merlin/logs/transfer.log \ |
||||
$RSYNC_HOST::$RSYNC_DIR $TO |
||||
@ -0,0 +1,17 @@ |
||||
#!/bin/bash -e |
||||
|
||||
if [ $# -lt 3 ]; then |
||||
echo 'Usage: sync local_dir path cut' |
||||
exit 1 |
||||
fi |
||||
|
||||
umask 002 |
||||
|
||||
TO=/mirror/root/$1 |
||||
SOURCE_PATH=$2 |
||||
CUT=$3 |
||||
ADDRESS=$(cat ~/config/ADDRESS) |
||||
|
||||
mkdir -p $TO |
||||
cd $TO |
||||
exec nice wget -q --bind-address=$ADDRESS --mirror --no-parent --no-host-directories --cut-dirs=$CUT --content-disposition --execute robots=off --recursive --reject "*\?*" $SOURCE_PATH |
||||
@ -0,0 +1,10 @@ |
||||
#!/bin/bash |
||||
|
||||
if [ $# -ne 1 ]; then |
||||
echo 'Usage: project' 1>&2 |
||||
exit 1 |
||||
fi |
||||
|
||||
exec 200>/tmp/$1.lock |
||||
flock 200 || exit 1 |
||||
sudo /usr/local/bin/zfssync.py $1 |
||||
Loading…
Reference in new issue