mirror/merlin/sync/command.go

package sync

import (
	"fmt"
	"os"
	"path/filepath"

	"git.csclub.uwaterloo.ca/public/merlin/config"
)

func buildRsyncHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
}

func buildRsyncDaemonHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + "::" + repo.RsyncDir
}

func buildRsyncSSHHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + ":" + repo.RsyncDir
}

func buildDownloadDir(repo *config.Repo) string {
	return filepath.Join(config.Conf.DownloadDir, repo.LocalDir)
}

func cscSyncApache(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncArchLinux(repo *config.Repo) []string {

	tempDir := "" // is this option even needed?

	// add option for verbose flag?
	args := []string{
		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
		"--temp-dir=" + tempDir, "--log-file=" + repo.RsyncLogFile, "--address=" + config.Conf.IPv4Address,
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args

}

func cscSyncBadPerms(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncCDImage(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncCeph(repo *config.Repo) []string {

	args := []string{
		"rsync", "--stats", "--progress", "--quiet", "-4", "--address=" + config.Conf.IPv4Address,
		repo.RsyncHost + "::ceph",
		"--recursive", "--times", "--links", "--hard-links",
		"--exclude", "Packages*",
		"--exclude", "Sources*",
		"--exclude", "Release*",
		"--exclude", "InRelease",
		"--exclude", "i18n/*",
		"--exclude", "ls-lR*",
		"--exclude", "repodata/*",
	}
	args = append(args, buildDownloadDir(repo))

	/* might need to implement two staged sync */
	/*
		rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
		--hard-links --delete-after \
		${TARGET}
	*/
	return args

}

func cscSyncChmod(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
		"--chmod=u=rwX,go=rX",
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncDebian(repo *config.Repo) []string {

	// sync /pool
	args := []string{"nice", "rsync", "-rlHtvp",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + config.Conf.IPv4Address,
	}
	// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
	return args
}

func cscSyncDebianCD(repo *config.Repo) []string {

	// this is basically the same as CSCSyncDebian, except it has an extra --exclude
	args := []string{"nice", "rsync", "-rlHtvp", "--delete",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + config.Conf.IPv4Address,
		// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
	}
	// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
	return args
}

func cscSyncGentoo(repo *config.Repo) []string {
	repo.RsyncUser = "gentoo"
	repo.PasswordFile = "gentoo-distfiles"
	return cscSyncStandard(repo)
}

func cscSyncS3(repo *config.Repo) []string {

	// set these env vars before using
	// export RCLONE_CONFIG_S3_ENDPOINT=https://s3.repo.saltproject.io RCLONE_CONFIG_S3_TYPE=s3 RCLONE_CONFIG_S3_PROVIDER=Other RCLONE_CONFIG_S3_ENV_AUTH=false

	args := []string{
		"nice", "rclone", "sync", "--fast-list", "--use-server-modtime",
		"--bind", config.Conf.IPv4Address,
		"s3:s3/",
	}
	args = append(args, buildDownloadDir(repo))

	return args
}

func cscSyncSSH(repo *config.Repo) []string {

	args := []string{
		"rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4",
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	// not sure if we should be assuming ssh identity file is the password file
	args = append(args, "-e", fmt.Sprintf("'ssh -i %s'", repo.PasswordFile))
	args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncStandard(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	if repo.PasswordFile != "" {
		args = append(args, "--password-file", repo.PasswordFile)
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncStandardIPV6(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncDummy(repo *config.Repo) []string {
	args := []string{"sleep", "1"}

	return args
}

// executes a particular sync job depending on repo.SyncType.
func getSyncCommand(repo *config.Repo) (args []string) {
	if repo.SyncType == "csc-sync-dummy" {
		return cscSyncDummy(repo)
	}

	// check that the download directory exists
	if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) {
		repo.Logger.Error(err.Error())
		return
	}

	switch repo.SyncType {
	case "csc-sync-apache":
		args = cscSyncApache(repo)
	case "csc-sync-archlinux":
		args = cscSyncArchLinux(repo)
	case "csc-sync-badperms":
		args = cscSyncBadPerms(repo)
	case "csc-sync-cdimage":
		args = cscSyncCDImage(repo)
	case "csc-sync-ceph":
		args = cscSyncCeph(repo)
	case "csc-sync-chmod":
		args = cscSyncChmod(repo)
	case "csc-sync-debian":
		args = cscSyncDebian(repo)
	case "csc-sync-debian-cd":
		args = cscSyncDebianCD(repo)
	case "csc-sync-gentoo":
		args = cscSyncGentoo(repo)
	case "csc-sync-s3":
		args = cscSyncS3(repo)
	case "csc-sync-ssh":
		args = cscSyncSSH(repo)
	case "csc-sync-standard":
		args = cscSyncStandard(repo)
	case "csc-sync-standard-ipv6":
		args = cscSyncStandardIPV6(repo)
		// case "csc-sync-wget":
		// 	return cscSyncWget(repo)
	default:
		repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
		return
	}

	if repo.MaxRsyncIO >= 0 {
		args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO))
	}
	// TODO: remove buildDownloadDir and check what cscSyncDebian and co are about
	// (the ones that do not append buildDownloadDir at the end)
	args = append(args, buildDownloadDir(repo))

	return
}