use quick-fedora-mirror for fedora repos

.gitignore

@@ -1,17 +1,21 @@
 !.gitignore
+*.swp
 .git_old/
 /dead.letter
 /*.bz2
+/.aptitude/
+/.bash_history
+/.bashrc
+/.cache/
+/.config/
+/.lesshst
+/.ssh/
+/.vim/
+/.vimrc
+/.viminfo
+/.wget-hsts
+/.zcompdump
+/go/
+/quick-fedora-mirror/timefile*
 /passwords/
 /tmp/
-/.bash_history
-/.lesshst
-/.vim/
-/.viminfo
-/.zcompdump
-/.aptitude/
-/.config/
-/.cache/
-/.ssh/
-/go/
-/.bashrc

merlin/config/config.go

@@ -18,6 +18,7 @@ const (
 	TWICE_HOURLY  = HOURLY / 2
 	BI_HOURLY     = HOURLY * 2
 	TRI_HOURLY    = HOURLY * 3
+	QUAD_HOURLY   = HOURLY * 4
 	TEN_MINUTELY  = 600
 	FIVE_MINUTELY = 300
 	MINUTELY      = 60
@@ -43,6 +44,7 @@ var frequencies = map[string]int{
 	"twice-hourly":  TWICE_HOURLY,
 	"bi-hourly":     BI_HOURLY,
 	"tri-hourly":    TRI_HOURLY,
+	"quad-hourly":   QUAD_HOURLY,
 	"ten-minutely":  TEN_MINUTELY,
 	"five-minutely": FIVE_MINUTELY,
 	"minutely":      MINUTELY,
@@ -262,7 +264,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
 			panic("Missing local download location for " + repo.Name)
 		} else if repo.SyncType == "csc-sync-ftpsync" && repo.FtpsyncArchive == "" {
 			panic("Missing ftpsync archive for " + repo.Name)
-		} else if repo.SyncType != "csc-sync-ftpsync" && repo.RsyncHost == "" {
+		} else if repo.SyncType != "csc-sync-ftpsync" && repo.SyncType != "csc-sync-fedora" && repo.RsyncHost == "" {
 			panic("Missing rsync host for " + repo.Name)
 		}
 

merlin/logger/logger.go

@@ -28,8 +28,9 @@ var levels = map[int]string{
 	ERROR:   "[ERROR]",
 }
 
-var outLogger = log.New(os.Stdout, "", log.LstdFlags)
-var errLogger = log.New(os.Stderr, "", log.LstdFlags)
+// We don't need to log the date/time because journald takes care of that for us
+var outLogger = log.New(os.Stdout, "", 0)
+var errLogger = log.New(os.Stderr, "", 0)
 
 // log to stdout
 func OutLog(v ...interface{}) {
@@ -77,7 +78,7 @@ func (logger *Logger) Debug(v ...interface{}) {
 
 // write information to the logfile and to stdout
 func (logger *Logger) Info(v ...interface{}) {
-	OutLog(append([]interface{}{"[" + logger.name + "]"}, v...))
+	OutLog(append([]interface{}{"[" + logger.name + "]"}, v...)...)
 	logger.log(INFO, v...)
 }
 
@@ -88,6 +89,6 @@ func (logger *Logger) Warning(v ...interface{}) {
 
 // write errors to the logfile and to stderr
 func (logger *Logger) Error(v ...interface{}) {
-	ErrLog(append([]interface{}{"[" + logger.name + "]"}, v...))
+	ErrLog(append([]interface{}{"[" + logger.name + "]"}, v...)...)
 	logger.log(ERROR, v...)
 }

merlin/merlin-config.ini

@@ -14,7 +14,7 @@ sock_path = /mirror/merlin/run/merlin.sock
 
 [debian]
 sync_type = csc-sync-ftpsync
-frequency = bi-hourly
+frequency = quad-hourly
 ftpsync_archive = debian
 
 [ubuntu]
@@ -111,14 +111,6 @@ local_dir = CTAN
 rsync_host = rsync.dante.ctan.org
 rsync_dir = CTAN
 
-[fedora-epel]
-verbose = true
-sync_type = csc-sync-standard
-frequency = bi-hourly
-local_dir = fedora/epel
-rsync_host = mirrors.kernel.org
-rsync_dir = fedora-epel
-
 [cygwin]
 sync_type = csc-sync-standard
 frequency = twice-daily
@@ -284,14 +276,11 @@ local_dir = FreeBSD
 rsync_host = ftp2.uk.freebsd.org
 rsync_dir = ftp.freebsd.org/pub/FreeBSD/
 
-[fedora-enchilada]
-verbose = true
-; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null
-sync_type = csc-sync-standard
+; This handles both fedora/linux and fedora/epel
+; See ~/quick-fedora-mirror/quick-fedora-mirror.conf
+[fedora]
+sync_type = csc-sync-fedora
 frequency = bi-hourly
-local_dir = fedora/linux
-rsync_host = mirrors.kernel.org
-rsync_dir = fedora-enchilada/linux/
 
 [ubuntu-ports-releases]
 sync_type = csc-sync-standard

merlin/merlin-go.service

@@ -3,11 +3,12 @@ Description=Manages synchronization of mirrored projects
 After=network.target
 
 [Service]
-ExecStart=/home/mirror/merlin/merlin --config=/home/mirror/merlin/merlin-config.ini
+ExecStart=/home/mirror/merlin/merlin --config=./merlin-config.ini
 WorkingDirectory=/home/mirror/merlin
 User=mirror
 Group=mirror
 SyslogIdentifier=merlin
+ExecReload=kill -HUP $MAINPID
 
 [Install]
 WantedBy=multi-user.target

merlin/sync/sync.go

@@ -55,6 +55,8 @@ func getSyncCommand(repo *config.Repo) (cmds [][]string) {
 		return append(cmds, cscSyncDebianStep1(repo), cscSyncDebianStep2(repo))
 	case "csc-sync-debian-cd":
 		return append(cmds, cscSyncDebianCD(repo))
+	case "csc-sync-fedora":
+		return append(cmds, cscSyncFedora(repo))
 	case "csc-sync-ftpsync":
 		return append(cmds, cscSyncFtpsync(repo))
 	case "csc-sync-s3":
@@ -291,6 +293,16 @@ func cscSyncDebianCD(repo *config.Repo) []string {
 	return args
 }
 
+func cscSyncFedora(repo *config.Repo) []string {
+	// Make sure that repo.RepoLogFile (default: ~/merlin/log/fedora.log)
+	// is the same as the LOGFILE setting in ~/quick-fedora-mirror/quick-fedora-mirror.conf
+	return []string{
+		"sh", "-c",
+		"truncate --size=0 " + repo.RepoLogFile + "; " +
+			"cd ~/quick-fedora-mirror && ./quick-fedora-mirror > " + repo.RsyncLogFile,
+	}
+}
+
 func cscSyncFtpsync(repo *config.Repo) []string {
 	// ftpsync configs are in /home/mirror/ftpsync/
 	return []string{"ftpsync", "sync:archive:" + repo.FtpsyncArchive}

quick-fedora-mirror/quick-fedora-mirror.conf

@@ -0,0 +1,196 @@
+# Configuration file for quick-fedora-mirror
+# This file is sourced by the shell and must be in valid sh syntax.
+
+#### Required settings
+# Required: The the directory holding your copy of all of the modules you
+# mirror.  Does not include any module name
+DESTD=/mirror/root/fedora
+
+# Required: The file in which to store the last mirror time.
+# Note: this really should not be in the repository itself.
+TIMEFILE=/home/mirror/quick-fedora-mirror/timefile
+
+# Other settings
+# The remote host to rsync from, not including a module name
+REMOTE=rsync://mirrors.rit.edu
+
+# The master module, which holds the other modules
+# Note that the mirror you pull from MUST have this master module.  You cannot
+# use quick-fedora-mirror to pull from a host which does not have a master
+# module containing the other modules.
+# MASTERMODULE=fedora-buffet
+
+# Tier 1 Fedora mirrors should uncomment the following to get the proper
+# pre-bitflip content.
+# MASTERMODULE=fedora-buffet0
+# PREBITFLIP=1
+
+# Define if the entire repository (all modules under fedora-buffet) should be
+# mirrored.  If set, MODULES (below) is ignored
+# MIRRORBUFFET=
+
+# An array containing the modules to be mirrored
+# MODULES=(fedora-enchilada fedora-epel)
+MODULES=(fedora fedora-epel)
+
+# The name of the file holding the file list on the mirror host Note: the
+# string '$mdir' will be replaced with the name of the current module directory
+# in context, and so the '$' must be escaped or the string quoted.
+# FILELIST='fullfiletimelist-$mdir'
+
+# An array of extra file lists to be transferred.  They won't be processed, but
+# will implicitly be included in every transfer because file lists can't be
+# included in the file lists.  '$mdir' is substituted as above.
+# Note that if you change this, you will want to do a run with -a to pick up
+# those extra files even in unchanged modules.
+# EXTRAFILES=(fullfilelist imagelist-\$mdir)
+
+# Mapping of MODULES to directory names, as an associative array
+# MODULEMAPPING=(fedora-alt alt  fedora-archive archive
+#                fedora-enchilada fedora  fedora-epel epel)
+MODULEMAPPING=(fedora fedora  fedora-epel epel)
+
+# curl binary (only if MirrorManager checkins are enabled; see below).
+# CURL=/usr/bin/curl
+
+# rsync binary
+# RSYNC=/usr/bin/rsync
+
+# Rsync timeout value, in seconds
+# Will be passed to rsync via --timeout.
+# RSYNCTIMEOUT=600
+
+# Array of default options to pass to rsync
+# Will be modified automatically according to VERBOSITY level; no need to set
+# -q, -v or --info here.
+#
+# You can add excludes here, but the script will always detect those files as
+# missing and will add them back to the file list.  This may generate
+# complaints from rsync, but should not cause any problems, though it is almost
+# certainly better to use the provided FILTEREXP to do exclusion instead.
+#
+# Note that some of these options may be required for proper operation of the
+# script.  You really should be careful if you change things here, as the
+# default value is carefully crafted and rsync can react in unexpected ways to
+# seeminly compatible sets of arguments.
+# RSYNCOPTS=(-aSH -f 'R .~tmp~' --stats --preallocate --delay-updates --out-format='@ %i  %n%L')
+
+# By default quick-fedora-mirror will try to detect and recover from an aborted
+# rsync run by moving any already downloaded files into place before
+# processing,  Define NORSYNCRECOVERY (to anything) to prevent this.
+# NORSYNCRECOVERY=
+
+# Define KEEPDIRTIMES (to anything) to make a third rsync call which restores
+# the timestamps of any directories which were modified after file removal.
+# This won't ensure that timestamps are always up to date, but its good enough
+# if you don't modify your repository locally.  Maintaining directory
+# timestamps isn't important for mirroring in any case.
+# KEEPDIRTIMES=
+
+# DEFINE CHECKIN_SITE and CHECKIN_PASSWORD to perform a mirrormanager checkin
+# at the completion of the run if rsync succeeded without error.  These values
+# should match the master site name and site password you have configured in
+# mirrormanager.
+# CHECKIN_HOST will default to the output of the hostname command; you must set
+# it manually if that does not match the hostname you have configured in
+# mirrormanager.
+# The "curl", "bzip2" and "base64" commands must be installed in order to
+# perform mirrormanager checkins.
+# CHECKIN_SITE=
+# CHECKIN_PASSWORD=
+# CHECKIN_HOST=
+
+# If you have configured multiple mirrormanager hostnames to virtually
+# spread the modules/categories you mirror between them, then you can configure
+# per-module checkin hosts with variables liks the following.
+# If a specific CHECKIN_HOST_* setting is not configured, then CHECKIN_HOST
+# above will be used.
+# Note that the module name is used, not the mirrormanager category, and that
+# '-' in the module name is replaced by '_' to make a legal variable name.
+# Most sites will hnot need this; it is only necessary when you mirror too many
+# modules for mirrormanager to handle in a single checkin or crawl.
+# CHECKIN_HOST_fedora_archive=
+# CHECKIN_HOST_fedora_alt=
+
+# Verbosity levels - info sent to stdout; usually this gets mailed to root when
+# being run by cron
+# 0 - quiet
+# 1 - quiet if no changes, otherwise basic transfer info
+# 2 - no -q to rsync
+# 3 - very slightly more verbosity
+# 4 - One -v to rsync
+# 5 - Another -v to rsync
+# 6 - Output of all settings
+# 7 - Add --progress to rsync
+# 8 - Shell script trace
+# VERBOSE=0
+VERBOSE=4
+
+# Define (to anything) to enable logging to the systemd journal (via
+# systemd-cat).  the identifier "quick-fedora-mirror" is used, so logs can be
+# retrieved with: journalctl -t quick-fedora-mirror
+# LOGJOURNAL=
+
+# Define to a full path to enable logging to that file.
+# The provided file must already exist and be writable.
+# Is only considered if LOGJOURNAL above is not defined.
+# LOGFILE=
+LOGFILE=/home/mirror/merlin/log/fedora.log
+
+# Logging fields - Each character selects a piece of information to log.
+# @ - Absolutely everything.
+# a - aborted run recovery
+# A - each recovered file from an aborted run
+# c - rsync calls
+# C - rsync call completions
+# d - File/directory deletion start/end
+# D - all file/directory deletes
+# e - minor errors
+# E - serious errors
+# F - all transferred files (not impl)
+# g - file list generation start/end
+# k - lock contention
+# l - per-module local file list generation (recursive find) start/end
+# L - file list generation counts
+# m - mirrormanager checkin
+# M - mirrormanager checkin detail
+# n - lack of updates in a run
+# N - lack of updates in a module
+# o - remote file list download start/end
+# p - module processing start/end
+# p - per-module module processing start/end
+# r - run start
+# R - run end
+# s - Basic transfer statistics
+# S - Detailed transfer statistics
+# t - directory time updates (not impl)
+# LOGITEMS=aeElrRs
+
+# When q-f-m fails to run becuse it is already running, it checks the time
+# since the last successful run.  If that is larger than this value, it logs a
+# serious error.  Thid helps to detect a hung run or issues with slow
+# transfers.
+# WARNDELAY=$((60 * 60 * 24)) # One day
+
+# When q-f-m encounters an error calling rsync, it may (depending on the error)
+# sleep and retry.  It will always sleep with exponential backup.  Set
+# MAXRETRIES to limit the number of times it retries.
+# MAXRETRIES=10
+
+# mktemp will be called after this file is sourced to make a temporary
+# directory.  This directory can contain a large amount of data, and that data
+# is specified by the server.  If your /tmp is small and you are concerned
+# about the server potentially sending extra-huge files and filling things up,
+# you can set TMPDIR here.
+# TMPDIR=
+
+# A regular expression used to filter the file lists. It must be quoted (or
+# very carefully escaped). Entries matching this expression will not be synced
+# and are expected not to be present locally. They will also be ignored by
+# quick-fedora-hardlink. Cannot contain commas. Run against the file list that
+# includes sizes (by quick-fedora-mirror) and the fullfiletimelist (by
+# quick-fedora-hardlink), so don't use expressions that would match those
+# metadata (which are digit strings and single characters). Example is a heavy
+# filter which gives you an x86_64-only mirror with source packages, debuginfo
+# packages, Alpha and Beta releases, and most image files excluded.
+# FILTEREXP='(/i386|/armhfp|/source|/SRPMS|/debug/|\.iso|\.img|\.qcow2|\.raw\.xz|\.box|/releases/test)'