use quick-fedora-mirror for fedora repos
This commit is contained in:
parent
f9cb8a4176
commit
e900874833
|
@ -1,17 +1,21 @@
|
|||
!.gitignore
|
||||
*.swp
|
||||
.git_old/
|
||||
/dead.letter
|
||||
/*.bz2
|
||||
/.aptitude/
|
||||
/.bash_history
|
||||
/.bashrc
|
||||
/.cache/
|
||||
/.config/
|
||||
/.lesshst
|
||||
/.ssh/
|
||||
/.vim/
|
||||
/.vimrc
|
||||
/.viminfo
|
||||
/.wget-hsts
|
||||
/.zcompdump
|
||||
/go/
|
||||
/quick-fedora-mirror/timefile*
|
||||
/passwords/
|
||||
/tmp/
|
||||
/.bash_history
|
||||
/.lesshst
|
||||
/.vim/
|
||||
/.viminfo
|
||||
/.zcompdump
|
||||
/.aptitude/
|
||||
/.config/
|
||||
/.cache/
|
||||
/.ssh/
|
||||
/go/
|
||||
/.bashrc
|
||||
|
|
|
@ -18,6 +18,7 @@ const (
|
|||
TWICE_HOURLY = HOURLY / 2
|
||||
BI_HOURLY = HOURLY * 2
|
||||
TRI_HOURLY = HOURLY * 3
|
||||
QUAD_HOURLY = HOURLY * 4
|
||||
TEN_MINUTELY = 600
|
||||
FIVE_MINUTELY = 300
|
||||
MINUTELY = 60
|
||||
|
@ -43,6 +44,7 @@ var frequencies = map[string]int{
|
|||
"twice-hourly": TWICE_HOURLY,
|
||||
"bi-hourly": BI_HOURLY,
|
||||
"tri-hourly": TRI_HOURLY,
|
||||
"quad-hourly": QUAD_HOURLY,
|
||||
"ten-minutely": TEN_MINUTELY,
|
||||
"five-minutely": FIVE_MINUTELY,
|
||||
"minutely": MINUTELY,
|
||||
|
@ -262,7 +264,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
|
|||
panic("Missing local download location for " + repo.Name)
|
||||
} else if repo.SyncType == "csc-sync-ftpsync" && repo.FtpsyncArchive == "" {
|
||||
panic("Missing ftpsync archive for " + repo.Name)
|
||||
} else if repo.SyncType != "csc-sync-ftpsync" && repo.RsyncHost == "" {
|
||||
} else if repo.SyncType != "csc-sync-ftpsync" && repo.SyncType != "csc-sync-fedora" && repo.RsyncHost == "" {
|
||||
panic("Missing rsync host for " + repo.Name)
|
||||
}
|
||||
|
||||
|
|
|
@ -28,8 +28,9 @@ var levels = map[int]string{
|
|||
ERROR: "[ERROR]",
|
||||
}
|
||||
|
||||
var outLogger = log.New(os.Stdout, "", log.LstdFlags)
|
||||
var errLogger = log.New(os.Stderr, "", log.LstdFlags)
|
||||
// We don't need to log the date/time because journald takes care of that for us
|
||||
var outLogger = log.New(os.Stdout, "", 0)
|
||||
var errLogger = log.New(os.Stderr, "", 0)
|
||||
|
||||
// log to stdout
|
||||
func OutLog(v ...interface{}) {
|
||||
|
@ -77,7 +78,7 @@ func (logger *Logger) Debug(v ...interface{}) {
|
|||
|
||||
// write information to the logfile and to stdout
|
||||
func (logger *Logger) Info(v ...interface{}) {
|
||||
OutLog(append([]interface{}{"[" + logger.name + "]"}, v...))
|
||||
OutLog(append([]interface{}{"[" + logger.name + "]"}, v...)...)
|
||||
logger.log(INFO, v...)
|
||||
}
|
||||
|
||||
|
@ -88,6 +89,6 @@ func (logger *Logger) Warning(v ...interface{}) {
|
|||
|
||||
// write errors to the logfile and to stderr
|
||||
func (logger *Logger) Error(v ...interface{}) {
|
||||
ErrLog(append([]interface{}{"[" + logger.name + "]"}, v...))
|
||||
ErrLog(append([]interface{}{"[" + logger.name + "]"}, v...)...)
|
||||
logger.log(ERROR, v...)
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ sock_path = /mirror/merlin/run/merlin.sock
|
|||
|
||||
[debian]
|
||||
sync_type = csc-sync-ftpsync
|
||||
frequency = bi-hourly
|
||||
frequency = quad-hourly
|
||||
ftpsync_archive = debian
|
||||
|
||||
[ubuntu]
|
||||
|
@ -111,14 +111,6 @@ local_dir = CTAN
|
|||
rsync_host = rsync.dante.ctan.org
|
||||
rsync_dir = CTAN
|
||||
|
||||
[fedora-epel]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
local_dir = fedora/epel
|
||||
rsync_host = mirrors.kernel.org
|
||||
rsync_dir = fedora-epel
|
||||
|
||||
[cygwin]
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
|
@ -284,14 +276,11 @@ local_dir = FreeBSD
|
|||
rsync_host = ftp2.uk.freebsd.org
|
||||
rsync_dir = ftp.freebsd.org/pub/FreeBSD/
|
||||
|
||||
[fedora-enchilada]
|
||||
verbose = true
|
||||
; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null
|
||||
sync_type = csc-sync-standard
|
||||
; This handles both fedora/linux and fedora/epel
|
||||
; See ~/quick-fedora-mirror/quick-fedora-mirror.conf
|
||||
[fedora]
|
||||
sync_type = csc-sync-fedora
|
||||
frequency = bi-hourly
|
||||
local_dir = fedora/linux
|
||||
rsync_host = mirrors.kernel.org
|
||||
rsync_dir = fedora-enchilada/linux/
|
||||
|
||||
[ubuntu-ports-releases]
|
||||
sync_type = csc-sync-standard
|
||||
|
|
|
@ -3,11 +3,12 @@ Description=Manages synchronization of mirrored projects
|
|||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/home/mirror/merlin/merlin --config=/home/mirror/merlin/merlin-config.ini
|
||||
ExecStart=/home/mirror/merlin/merlin --config=./merlin-config.ini
|
||||
WorkingDirectory=/home/mirror/merlin
|
||||
User=mirror
|
||||
Group=mirror
|
||||
SyslogIdentifier=merlin
|
||||
ExecReload=kill -HUP $MAINPID
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
|
@ -55,6 +55,8 @@ func getSyncCommand(repo *config.Repo) (cmds [][]string) {
|
|||
return append(cmds, cscSyncDebianStep1(repo), cscSyncDebianStep2(repo))
|
||||
case "csc-sync-debian-cd":
|
||||
return append(cmds, cscSyncDebianCD(repo))
|
||||
case "csc-sync-fedora":
|
||||
return append(cmds, cscSyncFedora(repo))
|
||||
case "csc-sync-ftpsync":
|
||||
return append(cmds, cscSyncFtpsync(repo))
|
||||
case "csc-sync-s3":
|
||||
|
@ -291,6 +293,16 @@ func cscSyncDebianCD(repo *config.Repo) []string {
|
|||
return args
|
||||
}
|
||||
|
||||
func cscSyncFedora(repo *config.Repo) []string {
|
||||
// Make sure that repo.RepoLogFile (default: ~/merlin/log/fedora.log)
|
||||
// is the same as the LOGFILE setting in ~/quick-fedora-mirror/quick-fedora-mirror.conf
|
||||
return []string{
|
||||
"sh", "-c",
|
||||
"truncate --size=0 " + repo.RepoLogFile + "; " +
|
||||
"cd ~/quick-fedora-mirror && ./quick-fedora-mirror > " + repo.RsyncLogFile,
|
||||
}
|
||||
}
|
||||
|
||||
func cscSyncFtpsync(repo *config.Repo) []string {
|
||||
// ftpsync configs are in /home/mirror/ftpsync/
|
||||
return []string{"ftpsync", "sync:archive:" + repo.FtpsyncArchive}
|
||||
|
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,196 @@
|
|||
# Configuration file for quick-fedora-mirror
|
||||
# This file is sourced by the shell and must be in valid sh syntax.
|
||||
|
||||
#### Required settings
|
||||
# Required: The the directory holding your copy of all of the modules you
|
||||
# mirror. Does not include any module name
|
||||
DESTD=/mirror/root/fedora
|
||||
|
||||
# Required: The file in which to store the last mirror time.
|
||||
# Note: this really should not be in the repository itself.
|
||||
TIMEFILE=/home/mirror/quick-fedora-mirror/timefile
|
||||
|
||||
# Other settings
|
||||
# The remote host to rsync from, not including a module name
|
||||
REMOTE=rsync://mirrors.rit.edu
|
||||
|
||||
# The master module, which holds the other modules
|
||||
# Note that the mirror you pull from MUST have this master module. You cannot
|
||||
# use quick-fedora-mirror to pull from a host which does not have a master
|
||||
# module containing the other modules.
|
||||
# MASTERMODULE=fedora-buffet
|
||||
|
||||
# Tier 1 Fedora mirrors should uncomment the following to get the proper
|
||||
# pre-bitflip content.
|
||||
# MASTERMODULE=fedora-buffet0
|
||||
# PREBITFLIP=1
|
||||
|
||||
# Define if the entire repository (all modules under fedora-buffet) should be
|
||||
# mirrored. If set, MODULES (below) is ignored
|
||||
# MIRRORBUFFET=
|
||||
|
||||
# An array containing the modules to be mirrored
|
||||
# MODULES=(fedora-enchilada fedora-epel)
|
||||
MODULES=(fedora fedora-epel)
|
||||
|
||||
# The name of the file holding the file list on the mirror host Note: the
|
||||
# string '$mdir' will be replaced with the name of the current module directory
|
||||
# in context, and so the '$' must be escaped or the string quoted.
|
||||
# FILELIST='fullfiletimelist-$mdir'
|
||||
|
||||
# An array of extra file lists to be transferred. They won't be processed, but
|
||||
# will implicitly be included in every transfer because file lists can't be
|
||||
# included in the file lists. '$mdir' is substituted as above.
|
||||
# Note that if you change this, you will want to do a run with -a to pick up
|
||||
# those extra files even in unchanged modules.
|
||||
# EXTRAFILES=(fullfilelist imagelist-\$mdir)
|
||||
|
||||
# Mapping of MODULES to directory names, as an associative array
|
||||
# MODULEMAPPING=(fedora-alt alt fedora-archive archive
|
||||
# fedora-enchilada fedora fedora-epel epel)
|
||||
MODULEMAPPING=(fedora fedora fedora-epel epel)
|
||||
|
||||
# curl binary (only if MirrorManager checkins are enabled; see below).
|
||||
# CURL=/usr/bin/curl
|
||||
|
||||
# rsync binary
|
||||
# RSYNC=/usr/bin/rsync
|
||||
|
||||
# Rsync timeout value, in seconds
|
||||
# Will be passed to rsync via --timeout.
|
||||
# RSYNCTIMEOUT=600
|
||||
|
||||
# Array of default options to pass to rsync
|
||||
# Will be modified automatically according to VERBOSITY level; no need to set
|
||||
# -q, -v or --info here.
|
||||
#
|
||||
# You can add excludes here, but the script will always detect those files as
|
||||
# missing and will add them back to the file list. This may generate
|
||||
# complaints from rsync, but should not cause any problems, though it is almost
|
||||
# certainly better to use the provided FILTEREXP to do exclusion instead.
|
||||
#
|
||||
# Note that some of these options may be required for proper operation of the
|
||||
# script. You really should be careful if you change things here, as the
|
||||
# default value is carefully crafted and rsync can react in unexpected ways to
|
||||
# seeminly compatible sets of arguments.
|
||||
# RSYNCOPTS=(-aSH -f 'R .~tmp~' --stats --preallocate --delay-updates --out-format='@ %i %n%L')
|
||||
|
||||
# By default quick-fedora-mirror will try to detect and recover from an aborted
|
||||
# rsync run by moving any already downloaded files into place before
|
||||
# processing, Define NORSYNCRECOVERY (to anything) to prevent this.
|
||||
# NORSYNCRECOVERY=
|
||||
|
||||
# Define KEEPDIRTIMES (to anything) to make a third rsync call which restores
|
||||
# the timestamps of any directories which were modified after file removal.
|
||||
# This won't ensure that timestamps are always up to date, but its good enough
|
||||
# if you don't modify your repository locally. Maintaining directory
|
||||
# timestamps isn't important for mirroring in any case.
|
||||
# KEEPDIRTIMES=
|
||||
|
||||
# DEFINE CHECKIN_SITE and CHECKIN_PASSWORD to perform a mirrormanager checkin
|
||||
# at the completion of the run if rsync succeeded without error. These values
|
||||
# should match the master site name and site password you have configured in
|
||||
# mirrormanager.
|
||||
# CHECKIN_HOST will default to the output of the hostname command; you must set
|
||||
# it manually if that does not match the hostname you have configured in
|
||||
# mirrormanager.
|
||||
# The "curl", "bzip2" and "base64" commands must be installed in order to
|
||||
# perform mirrormanager checkins.
|
||||
# CHECKIN_SITE=
|
||||
# CHECKIN_PASSWORD=
|
||||
# CHECKIN_HOST=
|
||||
|
||||
# If you have configured multiple mirrormanager hostnames to virtually
|
||||
# spread the modules/categories you mirror between them, then you can configure
|
||||
# per-module checkin hosts with variables liks the following.
|
||||
# If a specific CHECKIN_HOST_* setting is not configured, then CHECKIN_HOST
|
||||
# above will be used.
|
||||
# Note that the module name is used, not the mirrormanager category, and that
|
||||
# '-' in the module name is replaced by '_' to make a legal variable name.
|
||||
# Most sites will hnot need this; it is only necessary when you mirror too many
|
||||
# modules for mirrormanager to handle in a single checkin or crawl.
|
||||
# CHECKIN_HOST_fedora_archive=
|
||||
# CHECKIN_HOST_fedora_alt=
|
||||
|
||||
# Verbosity levels - info sent to stdout; usually this gets mailed to root when
|
||||
# being run by cron
|
||||
# 0 - quiet
|
||||
# 1 - quiet if no changes, otherwise basic transfer info
|
||||
# 2 - no -q to rsync
|
||||
# 3 - very slightly more verbosity
|
||||
# 4 - One -v to rsync
|
||||
# 5 - Another -v to rsync
|
||||
# 6 - Output of all settings
|
||||
# 7 - Add --progress to rsync
|
||||
# 8 - Shell script trace
|
||||
# VERBOSE=0
|
||||
VERBOSE=4
|
||||
|
||||
# Define (to anything) to enable logging to the systemd journal (via
|
||||
# systemd-cat). the identifier "quick-fedora-mirror" is used, so logs can be
|
||||
# retrieved with: journalctl -t quick-fedora-mirror
|
||||
# LOGJOURNAL=
|
||||
|
||||
# Define to a full path to enable logging to that file.
|
||||
# The provided file must already exist and be writable.
|
||||
# Is only considered if LOGJOURNAL above is not defined.
|
||||
# LOGFILE=
|
||||
LOGFILE=/home/mirror/merlin/log/fedora.log
|
||||
|
||||
# Logging fields - Each character selects a piece of information to log.
|
||||
# @ - Absolutely everything.
|
||||
# a - aborted run recovery
|
||||
# A - each recovered file from an aborted run
|
||||
# c - rsync calls
|
||||
# C - rsync call completions
|
||||
# d - File/directory deletion start/end
|
||||
# D - all file/directory deletes
|
||||
# e - minor errors
|
||||
# E - serious errors
|
||||
# F - all transferred files (not impl)
|
||||
# g - file list generation start/end
|
||||
# k - lock contention
|
||||
# l - per-module local file list generation (recursive find) start/end
|
||||
# L - file list generation counts
|
||||
# m - mirrormanager checkin
|
||||
# M - mirrormanager checkin detail
|
||||
# n - lack of updates in a run
|
||||
# N - lack of updates in a module
|
||||
# o - remote file list download start/end
|
||||
# p - module processing start/end
|
||||
# p - per-module module processing start/end
|
||||
# r - run start
|
||||
# R - run end
|
||||
# s - Basic transfer statistics
|
||||
# S - Detailed transfer statistics
|
||||
# t - directory time updates (not impl)
|
||||
# LOGITEMS=aeElrRs
|
||||
|
||||
# When q-f-m fails to run becuse it is already running, it checks the time
|
||||
# since the last successful run. If that is larger than this value, it logs a
|
||||
# serious error. Thid helps to detect a hung run or issues with slow
|
||||
# transfers.
|
||||
# WARNDELAY=$((60 * 60 * 24)) # One day
|
||||
|
||||
# When q-f-m encounters an error calling rsync, it may (depending on the error)
|
||||
# sleep and retry. It will always sleep with exponential backup. Set
|
||||
# MAXRETRIES to limit the number of times it retries.
|
||||
# MAXRETRIES=10
|
||||
|
||||
# mktemp will be called after this file is sourced to make a temporary
|
||||
# directory. This directory can contain a large amount of data, and that data
|
||||
# is specified by the server. If your /tmp is small and you are concerned
|
||||
# about the server potentially sending extra-huge files and filling things up,
|
||||
# you can set TMPDIR here.
|
||||
# TMPDIR=
|
||||
|
||||
# A regular expression used to filter the file lists. It must be quoted (or
|
||||
# very carefully escaped). Entries matching this expression will not be synced
|
||||
# and are expected not to be present locally. They will also be ignored by
|
||||
# quick-fedora-hardlink. Cannot contain commas. Run against the file list that
|
||||
# includes sizes (by quick-fedora-mirror) and the fullfiletimelist (by
|
||||
# quick-fedora-hardlink), so don't use expressions that would match those
|
||||
# metadata (which are digit strings and single characters). Example is a heavy
|
||||
# filter which gives you an x86_64-only mirror with source packages, debuginfo
|
||||
# packages, Alpha and Beta releases, and most image files excluded.
|
||||
# FILTEREXP='(/i386|/armhfp|/source|/SRPMS|/debug/|\.iso|\.img|\.qcow2|\.raw\.xz|\.box|/releases/test)'
|
Loading…
Reference in New Issue