mirror/merlin/sync/command.go

package sync

import (
	"fmt"
	"os"

	"git.csclub.uwaterloo.ca/public/merlin/config"
)

func buildRsyncHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
}

func buildRsyncDaemonHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + "::" + repo.RsyncDir
}

func buildRsyncSSHHost(repo *config.Repo) string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + ":" + repo.RsyncDir
}

// TODO: remove this (repo.LocalDir should be the full path)
func buildDownloadDir(repo *config.Repo) string {
	return config.Conf.DownloadDir + "/" + repo.LocalDir
}

func cscSyncApache(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncArchLinux(repo *config.Repo) []string {

	tempDir := "" // is this option even needed?

	// add option for verbose flag?
	args := []string{
		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
		"--temp-dir=" + tempDir, "--log-file=" + repo.RsyncLogFile, "--address=" + config.Conf.IPv4Address,
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args

}

func cscSyncBadPerms(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncCDImage(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncCeph(repo *config.Repo) []string {

	args := []string{
		"rsync", "--stats", "--progress", "--quiet", "-4", "--address=" + config.Conf.IPv4Address,
		repo.RsyncHost + "::ceph",
		"--recursive", "--times", "--links", "--hard-links",
		"--exclude", "Packages*",
		"--exclude", "Sources*",
		"--exclude", "Release*",
		"--exclude", "InRelease",
		"--exclude", "i18n/*",
		"--exclude", "ls-lR*",
		"--exclude", "repodata/*",
	}
	args = append(args, buildDownloadDir(repo))

	/* might need to implement two staged sync */
	/*
		rsync ${RSYNC_OPTS} ${SOURCE_HOST}::ceph --recursive --times --links \
		--hard-links --delete-after \
		${TARGET}
	*/
	return args

}

func cscSyncChmod(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
		"--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
		"--chmod=u=rwX,go=rX",
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncDebian(repo *config.Repo) []string {

	// sync /pool
	args := []string{"nice", "rsync", "-rlHtvp",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + config.Conf.IPv4Address,
	}
	// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
	return args
}

func cscSyncDebianCD(repo *config.Repo) []string {

	// this is basically the same as CSCSyncDebian, except it has an extra --exclude
	args := []string{"nice", "rsync", "-rlHtvp", "--delete",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + config.Conf.IPv4Address,
		// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
	}
	// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
	return args
}

func cscSyncGentoo(repo *config.Repo) []string {
	repo.RsyncUser = "gentoo"
	repo.PasswordFile = "gentoo-distfiles"
	return cscSyncStandard(repo)
}

func cscSyncS3(repo *config.Repo) []string {

	// set these env vars before using
	// export RCLONE_CONFIG_S3_ENDPOINT=https://s3.repo.saltproject.io RCLONE_CONFIG_S3_TYPE=s3 RCLONE_CONFIG_S3_PROVIDER=Other RCLONE_CONFIG_S3_ENV_AUTH=false

	args := []string{
		"nice", "rclone", "sync", "--fast-list", "--use-server-modtime",
		"--bind", config.Conf.IPv4Address,
		"s3:s3/",
	}
	args = append(args, buildDownloadDir(repo))

	return args
}

func cscSyncSSH(repo *config.Repo) []string {

	args := []string{
		"rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4",
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	// not sure if we should be assuming ssh identity file is the password file
	args = append(args, "-e", fmt.Sprintf("'ssh -i %s'", repo.PasswordFile))
	args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncStandard(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	if repo.PasswordFile != "" {
		filename := config.Conf.PasswordDir + "/" + repo.PasswordFile
		args = append(args, "--password-file", filename)
	}
	args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncStandardIPV6(repo *config.Repo) []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + config.Conf.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))

	return args
}

func cscSyncDummy(repo *config.Repo) []string {
	args := []string{"sleep", "1"}

	return args
}

// executes a particular sync job depending on repo.SyncType.
func getSyncCommand(repo *config.Repo) (args []string) {
	/*
		# scripts used by merlin.py
		csc-sync-debian
		csc-sync-standard
		csc-sync-ssh
		csc-sync-apache
		csc-sync-archlinux
		csc-sync-debian-cd
		csc-sync-gentoo
		csc-sync-wget
		csc-sync-s3
		csc-sync-standard-ipv6
		csc-sync-ceph

		zfssync
		report_mirror (what is this?)

		# other things in bin/
		csc-sync-archlinux-old
		csc-sync-cdimage
		csc-sync-chmod
		csc-sync-badperms
		make-torrents
		ubuntu-releases-sync
	*/
	if repo.SyncType == "csc-sync-dummy" {
		return cscSyncDummy(repo)
	}

	localDir := buildDownloadDir(repo)
	if err := os.MkdirAll(localDir, 0775); err != nil {
		repo.Logger.Error(err.Error())
		return
	}

	switch repo.SyncType {
	case "csc-sync-apache":
		return cscSyncApache(repo)
	case "csc-sync-archlinux":
		return cscSyncArchLinux(repo)
	case "csc-sync-badperms":
		return cscSyncBadPerms(repo)
	case "csc-sync-cdimage":
		return cscSyncCDImage(repo)
	case "csc-sync-ceph":
		return cscSyncCeph(repo)
	case "csc-sync-chmod":
		return cscSyncChmod(repo)
	case "csc-sync-debian":
		return cscSyncDebian(repo)
	case "csc-sync-debian-cd":
		return cscSyncDebianCD(repo)
	case "csc-sync-gentoo":
		return cscSyncGentoo(repo)
	case "csc-sync-s3":
		return cscSyncS3(repo)
	case "csc-sync-ssh":
		return cscSyncSSH(repo)
	case "csc-sync-standard":
		return cscSyncStandard(repo)
	case "csc-sync-standard-ipv6":
		return cscSyncStandardIPV6(repo)
		// case "csc-sync-wget":
		// 	return cscSyncWget(repo)
	default:
		repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
	}
	return
}