356 lines
9.0 KiB
Go
356 lines
9.0 KiB
Go
package sync
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"git.csclub.uwaterloo.ca/public/merlin/config"
|
|
)
|
|
|
|
func buildRsyncHost(repo *config.Repo) string {
|
|
rsyncHost := repo.RsyncHost
|
|
if repo.RsyncUser != "" {
|
|
rsyncHost = repo.RsyncUser + "@" + rsyncHost
|
|
}
|
|
return "rsync://" + rsyncHost + "/" + repo.RsyncDir
|
|
}
|
|
|
|
func buildRsyncDaemonHost(repo *config.Repo) string {
|
|
rsyncHost := repo.RsyncHost
|
|
if repo.RsyncUser != "" {
|
|
rsyncHost = repo.RsyncUser + "@" + rsyncHost
|
|
}
|
|
return rsyncHost + "::" + repo.RsyncDir
|
|
}
|
|
|
|
func buildRsyncSSHHost(repo *config.Repo) string {
|
|
rsyncHost := repo.RsyncHost
|
|
if repo.RsyncUser != "" {
|
|
rsyncHost = repo.RsyncUser + "@" + rsyncHost
|
|
}
|
|
return rsyncHost + ":" + repo.RsyncDir
|
|
}
|
|
|
|
func buildDownloadDir(repo *config.Repo) string {
|
|
return filepath.Join(config.Conf.DownloadDir, repo.LocalDir)
|
|
}
|
|
|
|
const (
|
|
noOwnerNoGroup = 1 << iota
|
|
timeout3600
|
|
excludeTmp
|
|
logFile
|
|
quiet
|
|
ipv4
|
|
ipv6
|
|
delete
|
|
delayUpdatesDeleteAfter
|
|
// adds base arguments for setting timeout, a logfile, and including quiet
|
|
baseFlags = timeout3600 | logFile | quiet
|
|
// adds standard arguments timeout, logging, quiet, no owner/group, and excluding .~tmp~/
|
|
stdFlags = baseFlags | noOwnerNoGroup | excludeTmp
|
|
)
|
|
|
|
func addConditionalFlags(repo *config.Repo, flags int) []string {
|
|
args := []string{}
|
|
if flags&noOwnerNoGroup != 0 {
|
|
args = append(args, "--no-owner", "--no-group")
|
|
}
|
|
if flags&timeout3600 != 0 {
|
|
args = append(args, "--timeout=3600")
|
|
}
|
|
if flags&excludeTmp != 0 {
|
|
args = append(args, "--exclude", ".~tmp~/")
|
|
}
|
|
if flags&logFile != 0 {
|
|
args = append(args, "--log-file="+repo.RsyncLogFile)
|
|
}
|
|
if flags&quiet != 0 && !repo.Verbose {
|
|
args = append(args, "--quiet")
|
|
}
|
|
|
|
if flags&ipv4 != 0 {
|
|
args = append(args, "-4", "--address="+config.Conf.IPv4Address)
|
|
} else if flags&ipv6 != 0 {
|
|
args = append(args, "-6", "--address="+config.Conf.IPv6Address)
|
|
}
|
|
|
|
if flags&delete != 0 {
|
|
args = append(args, "--delete")
|
|
}
|
|
if flags&delayUpdatesDeleteAfter != 0 {
|
|
args = append(args, "--delay-updates", "--delete-after")
|
|
}
|
|
|
|
// add bwlimit (bandwidth limit) only when MaxRsyncIO is set to positive value
|
|
if repo.MaxRsyncIO > 0 {
|
|
args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO))
|
|
}
|
|
if repo.Verbose {
|
|
args = append(args, "-vv")
|
|
}
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncApache(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-az",
|
|
"--delete",
|
|
"--safe-links",
|
|
"--stats",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, stdFlags|ipv4)...)
|
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncArchLinux(repo *config.Repo) []string {
|
|
tempDir := "/home/mirror/tmp"
|
|
|
|
args := []string{
|
|
"rsync", "-rtlHp",
|
|
"--safe-links",
|
|
"--timeout=600",
|
|
"--contimeout=60",
|
|
"--no-motd",
|
|
"--temp-dir=" + tempDir,
|
|
"--address=" + config.Conf.IPv4Address,
|
|
}
|
|
args = append(args, addConditionalFlags(repo, logFile|delayUpdatesDeleteAfter)...)
|
|
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncBadPerms(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-aH",
|
|
"--chmod=o=rX",
|
|
"--stats",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delete)...)
|
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncCDImage(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-aH",
|
|
"--exclude", "\".*/\"",
|
|
"--stats",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, baseFlags|noOwnerNoGroup|ipv4|delete)...)
|
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncCeph(repo *config.Repo) []string {
|
|
args := []string{
|
|
"rsync", "--stats", "--progress",
|
|
repo.RsyncHost + "::ceph",
|
|
"--recursive", "--times", "--links",
|
|
"--hard-links",
|
|
"--exclude", "Packages*",
|
|
"--exclude", "Sources*",
|
|
"--exclude", "Release*",
|
|
"--exclude", "InRelease",
|
|
"--exclude", "i18n/*",
|
|
"--exclude", "ls-lR*",
|
|
"--exclude", "repodata/*",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, quiet|ipv4)...)
|
|
args = append(args, buildDownloadDir(repo))
|
|
|
|
args = append(args, []string{
|
|
"&&", "rsync", "--stats", "--progress",
|
|
repo.RsyncHost + "::ceph",
|
|
"--recursive", "--times", "--links",
|
|
"--hard-links", "--delete-after",
|
|
}...)
|
|
args = append(args, addConditionalFlags(repo, quiet|ipv4)...)
|
|
args = append(args, buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncChmod(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-aH",
|
|
"--safe-links",
|
|
"--stats",
|
|
"--chmod=u=rwX,go=rX",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...)
|
|
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncDebian(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-rlHtvp",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4)...)
|
|
args = append(args, buildRsyncDaemonHost(repo)+"/pool/", buildDownloadDir(repo)+"/pool/")
|
|
|
|
args = append(args, []string{
|
|
"&&", "nice", "rsync", "-rlHtvp",
|
|
"--exclude", filepath.Join("project/trace", config.Conf.Hostname),
|
|
}...)
|
|
args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delayUpdatesDeleteAfter)...)
|
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncDebianCD(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-rlHtvp",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delete)...)
|
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncGentoo(repo *config.Repo) []string {
|
|
repo.RsyncUser = "gentoo"
|
|
repo.PasswordFile = "gentoo-distfiles"
|
|
return cscSyncStandard(repo)
|
|
}
|
|
|
|
// TODO: check for special stuff that rcloning S3 needs
|
|
func cscSyncS3(repo *config.Repo) []string {
|
|
args := []string{
|
|
// RsyncHost is just a regular host (https://s3.repo.saltproject.io) in this case
|
|
"RCLONE_CONFIG_S3_ENDPOINT=" + repo.RsyncHost,
|
|
"RCLONE_CONFIG_S3_TYPE=s3",
|
|
"RCLONE_CONFIG_S3_PROVIDER=Other",
|
|
"RCLONE_CONFIG_S3_ENV_AUTH=false",
|
|
"nice", "rclone", "sync",
|
|
"--fast-list",
|
|
"--use-server-modtime",
|
|
"--bind", config.Conf.IPv4Address,
|
|
"s3:s3/",
|
|
}
|
|
args = append(args, buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncSSH(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-aH",
|
|
"--stats", "-4",
|
|
}
|
|
args = append(args, addConditionalFlags(repo, stdFlags|delete)...)
|
|
// PasswordFile should point to the SSH_KEYFILE
|
|
args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile))
|
|
args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncStandard(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-aH",
|
|
"--safe-links",
|
|
"--stats",
|
|
}
|
|
|
|
if repo.PasswordFile != "" {
|
|
args = append(args, "--password-file", repo.PasswordFile)
|
|
}
|
|
|
|
args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...)
|
|
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncStandardIPV6(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "rsync", "-aH",
|
|
"--safe-links",
|
|
"--stats",
|
|
}
|
|
|
|
if repo.PasswordFile != "" {
|
|
args = append(args, "--password-file", repo.PasswordFile)
|
|
}
|
|
|
|
args = append(args, addConditionalFlags(repo, stdFlags|ipv6|delayUpdatesDeleteAfter)...)
|
|
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
|
|
|
return args
|
|
}
|
|
|
|
func cscSyncWget(repo *config.Repo) []string {
|
|
args := []string{
|
|
"nice", "wget", "-q",
|
|
"--bind-address=" + config.Conf.IPv4Address,
|
|
"--mirror",
|
|
"--no-parent",
|
|
"--no-host-directories",
|
|
"--cut-dirs=1",
|
|
"--content-disposition",
|
|
"--execute",
|
|
"robots=off",
|
|
"--recursive",
|
|
"--reject", "\"*\\?*\"",
|
|
"--directory-prefix=" + buildDownloadDir(repo),
|
|
// RsyncHost is just a regular host (https://mirror.racket-lang.org/installers/) in this case
|
|
repo.RsyncHost,
|
|
}
|
|
|
|
return args
|
|
}
|
|
|
|
// executes a particular sync job depending on repo.SyncType.
|
|
func getSyncCommand(repo *config.Repo) (args []string) {
|
|
// check that the download directory exists
|
|
if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) {
|
|
repo.Logger.Error(err.Error())
|
|
return
|
|
}
|
|
|
|
switch repo.SyncType {
|
|
case "csc-sync-apache":
|
|
return cscSyncApache(repo)
|
|
case "csc-sync-archlinux":
|
|
return cscSyncArchLinux(repo)
|
|
case "csc-sync-badperms":
|
|
return cscSyncBadPerms(repo)
|
|
case "csc-sync-cdimage":
|
|
return cscSyncCDImage(repo)
|
|
case "csc-sync-ceph":
|
|
return cscSyncCeph(repo)
|
|
case "csc-sync-chmod":
|
|
return cscSyncChmod(repo)
|
|
case "csc-sync-debian":
|
|
return cscSyncDebian(repo)
|
|
case "csc-sync-debian-cd":
|
|
return cscSyncDebianCD(repo)
|
|
case "csc-sync-gentoo":
|
|
return cscSyncGentoo(repo)
|
|
case "csc-sync-s3":
|
|
return cscSyncS3(repo)
|
|
case "csc-sync-ssh":
|
|
return cscSyncSSH(repo)
|
|
case "csc-sync-standard":
|
|
return cscSyncStandard(repo)
|
|
case "csc-sync-standard-ipv6":
|
|
return cscSyncStandardIPV6(repo)
|
|
case "csc-sync-wget":
|
|
return cscSyncWget(repo)
|
|
default:
|
|
repo.Logger.Error("Unrecognized sync type: " + repo.SyncType)
|
|
return
|
|
}
|
|
}
|