247 lines
6.7 KiB
Go
247 lines
6.7 KiB
Go
|
package sync
|
||
|
|
||
|
import (
|
||
|
"fmt"
|
||
|
"math/rand"
|
||
|
"strconv"
|
||
|
|
||
|
"git.csclub.uwaterloo.ca/public/merlin/config"
|
||
|
)
|
||
|
|
||
|
func buildRsyncHost(repo *config.Repo) string {
|
||
|
if repo.RsyncUser != "" {
|
||
|
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||
|
}
|
||
|
return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
|
||
|
}
|
||
|
|
||
|
func buildRsyncDaemonHost(repo *config.Repo) string {
|
||
|
if repo.RsyncUser != "" {
|
||
|
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||
|
}
|
||
|
return repo.RsyncHost + "::" + repo.RsyncDir
|
||
|
}
|
||
|
|
||
|
func buildRsyncSSHHost(repo *config.Repo) string {
|
||
|
if repo.RsyncUser != "" {
|
||
|
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||
|
}
|
||
|
return repo.RsyncHost + ":" + repo.RsyncDir
|
||
|
}
|
||
|
|
||
|
func buildDownloadDir(repo *config.Repo) string {
|
||
|
return config.Conf.DownloadDir + "/" + repo.LocalDir
|
||
|
}
|
||
|
|
||
|
func cscSyncApache(repo *config.Repo) []string {
|
||
|
args := []string{
|
||
|
"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
|
||
|
"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
|
||
|
"--exclude", ".~tmp~/",
|
||
|
"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
}
|
||
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncArchLinux(repo *config.Repo) []string {
|
||
|
|
||
|
tempDir := "" // is this option even needed?
|
||
|
|
||
|
// add option for verbose flag?
|
||
|
args := []string{
|
||
|
"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
|
||
|
"--contimeout=60", "-p", "--delay-updates", "--no-motd",
|
||
|
"--temp-dir=" + tempDir, "--log-file=" + repo.RsyncLogFile, "--address=" + config.Conf.IPv4Address,
|
||
|
}
|
||
|
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
|
||
|
}
|
||
|
|
||
|
func cscSyncBadPerms(repo *config.Repo) []string {
|
||
|
args := []string{
|
||
|
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
|
||
|
"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
|
||
|
"--exclude", ".~tmp~/",
|
||
|
"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
}
|
||
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
// TODO ceph
|
||
|
|
||
|
func cscSyncCDImage(repo *config.Repo) []string {
|
||
|
args := []string{
|
||
|
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
|
||
|
"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
|
||
|
"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
}
|
||
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncChmod(repo *config.Repo) []string {
|
||
|
args := []string{
|
||
|
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
|
||
|
"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
|
||
|
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
"--chmod=u=rwX,go=rX",
|
||
|
}
|
||
|
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncDebian(repo *config.Repo) []string {
|
||
|
|
||
|
// sync /pool
|
||
|
args := []string{"nice", "rsync", "-rlHtvp",
|
||
|
"--exclude", ".~tmp~/", "--timeout=3600", "-4",
|
||
|
"--address=" + config.Conf.IPv4Address,
|
||
|
}
|
||
|
// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncDebianCD(repo *config.Repo) []string {
|
||
|
|
||
|
// this is basically the same as CSCSyncDebian, except it has an extra --exclude
|
||
|
args := []string{"nice", "rsync", "-rlHtvp", "--delete",
|
||
|
"--exclude", ".~tmp~/", "--timeout=3600", "-4",
|
||
|
"--address=" + config.Conf.IPv4Address,
|
||
|
// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
|
||
|
}
|
||
|
// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncGentoo(repo *config.Repo) []string {
|
||
|
repo.RsyncUser = "gentoo"
|
||
|
repo.PasswordFile = "gentoo-distfiles"
|
||
|
return cscSyncStandard(repo)
|
||
|
}
|
||
|
|
||
|
// TODO s3
|
||
|
|
||
|
func cscSyncSSH(repo *config.Repo) []string {
|
||
|
|
||
|
args := []string{
|
||
|
"rsync", "-aH", "--no-owner", "--no-group", "--delete",
|
||
|
"--timeout=3600", "-4",
|
||
|
"--exclude", ".~tmp~/",
|
||
|
"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
}
|
||
|
// not sure if we should be assuming ssh identity file is the password file
|
||
|
args = append(args, "-e", fmt.Sprintf("'ssh -i %s'", repo.PasswordFile))
|
||
|
args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncStandard(repo *config.Repo) []string {
|
||
|
args := []string{
|
||
|
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||
|
"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
|
||
|
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
}
|
||
|
if repo.PasswordFile != "" {
|
||
|
filename := config.Conf.PasswordDir + "/" + repo.PasswordFile
|
||
|
args = append(args, "--password-file", filename)
|
||
|
}
|
||
|
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
func cscSyncStandardIPV6(repo *config.Repo) []string {
|
||
|
args := []string{
|
||
|
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||
|
"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + config.Conf.IPv4Address,
|
||
|
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
|
||
|
}
|
||
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
// for testing, to be removed later
|
||
|
func cscSyncDummy(repo *config.Repo) []string {
|
||
|
|
||
|
sleepDur := strconv.FormatInt(rand.Int63n(10)+5, 10)
|
||
|
args := []string{"sleep", sleepDur}
|
||
|
|
||
|
return args
|
||
|
}
|
||
|
|
||
|
// executes a particular sync job depending on repo.SyncType.
|
||
|
func getSyncCommand(repo *config.Repo) []string {
|
||
|
/*
|
||
|
# scripts used by merlin.py
|
||
|
csc-sync-debian
|
||
|
csc-sync-standard
|
||
|
csc-sync-ssh
|
||
|
csc-sync-apache
|
||
|
csc-sync-archlinux
|
||
|
csc-sync-debian-cd
|
||
|
csc-sync-gentoo
|
||
|
csc-sync-wget
|
||
|
csc-sync-s3
|
||
|
csc-sync-standard-ipv6
|
||
|
csc-sync-ceph
|
||
|
|
||
|
zfssync
|
||
|
report_mirror (what is this?)
|
||
|
|
||
|
# other things in bin/
|
||
|
csc-sync-archlinux-old
|
||
|
csc-sync-cdimage
|
||
|
csc-sync-chmod
|
||
|
csc-sync-badperms
|
||
|
make-torrents
|
||
|
ubuntu-releases-sync
|
||
|
*/
|
||
|
switch repo.SyncType {
|
||
|
|
||
|
case "csc-sync-apache":
|
||
|
return cscSyncApache(repo)
|
||
|
case "csc-sync-archlinux":
|
||
|
return cscSyncArchLinux(repo)
|
||
|
case "csc-sync-badperms":
|
||
|
return cscSyncBadPerms(repo)
|
||
|
case "csc-sync-cdimage":
|
||
|
return cscSyncCDImage(repo)
|
||
|
// case "csc-sync-ceph":
|
||
|
// return cscSyncCeph(repo)
|
||
|
case "csc-sync-chmod":
|
||
|
return cscSyncChmod(repo)
|
||
|
case "csc-sync-debian":
|
||
|
return cscSyncDebian(repo)
|
||
|
case "csc-sync-debian-cd":
|
||
|
return cscSyncDebianCD(repo)
|
||
|
case "csc-sync-gentoo":
|
||
|
return cscSyncGentoo(repo)
|
||
|
// case "csc-sync-s3":
|
||
|
// return cscSyncS3(repo)
|
||
|
case "csc-sync-ssh":
|
||
|
return cscSyncSSH(repo)
|
||
|
case "csc-sync-standard":
|
||
|
return cscSyncStandard(repo)
|
||
|
case "csc-sync-standard-ipv6":
|
||
|
return cscSyncStandardIPV6(repo)
|
||
|
// case "csc-sync-wget":
|
||
|
// return cscSyncWget(repo)
|
||
|
case "csc-sync-dummy":
|
||
|
return cscSyncDummy(repo)
|
||
|
default:
|
||
|
repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
|
||
|
}
|
||
|
|
||
|
return []string{}
|
||
|
}
|