mirror/merlin/common/sync.go

package common

import (
	"fmt"
	"math/rand"
	"os"
	"strconv"
)

func (repo *Repo) buildRsyncHost() string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
}

func (repo *Repo) buildRsyncDaemonHost() string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + "::" + repo.RsyncDir
}

func (repo *Repo) buildRsyncSSHHost() string {
	if repo.RsyncUser != "" {
		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
	}
	return repo.RsyncHost + ":" + repo.RsyncDir
}

func (repo *Repo) buildDownloadDir() string {
	return repo.cfg.DownloadDir + "/" + repo.LocalDir
}

func (repo *Repo) CSCSyncApache() []string {
	args := []string{
		"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())

	return args
}

func (repo *Repo) CSCSyncArchLinux() []string {

	tempDir := "" // is this option even needed?

	// add option for verbose flag?
	args := []string{
		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
		"--temp-dir=" + tempDir, "--log-file=" + repo.RsyncLogFile, "--address=" + repo.cfg.IPv4Address,
	}
	args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())

	return args

}

func (repo *Repo) CSCSyncBadPerms() []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())

	return args
}

// TODO ceph

func (repo *Repo) CSCSyncCDImage() []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
		"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())

	return args
}

func (repo *Repo) CSCSyncChmod() []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
		"--chmod=u=rwX,go=rX",
	}
	args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())

	return args
}

func (repo *Repo) CSCSyncDebian() []string {

	// sync /pool
	args := []string{"nice", "rsync", "-rlHtvp",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + repo.cfg.IPv4Address,
	}
	// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
	return args
}

func (repo *Repo) CSCSyncDebianCD() []string {

	// this is basically the same as CSCSyncDebian, except it has an extra --exclude
	args := []string{"nice", "rsync", "-rlHtvp", "--delete",
		"--exclude", ".~tmp~/", "--timeout=3600", "-4",
		"--address=" + repo.cfg.IPv4Address,
		// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
	}
	// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
	return args
}

func (repo *Repo) CSCSyncGentoo() []string {
	repo.RsyncUser = "gentoo"
	repo.PasswordFile = "gentoo-distfiles"
	return repo.CSCSyncStandard()
}

// TODO s3

func (repo *Repo) CSCSyncSSH() []string {

	args := []string{
		"rsync", "-aH", "--no-owner", "--no-group", "--delete",
		"--timeout=3600", "-4",
		"--exclude", ".~tmp~/",
		"--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	// not sure if we should be assuming ssh identity file is the password file
	args = append(args, "-e", fmt.Sprintf("'ssh -i %s'", repo.PasswordFile))
	args = append(args, repo.buildRsyncSSHHost(), repo.buildDownloadDir())

	return args
}

func (repo *Repo) CSCSyncStandard() []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	if repo.PasswordFile != "" {
		filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
		args = append(args, "--password-file", filename)
	}
	args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())

	return args
}

func (repo *Repo) CSCSyncStandardIPV6() []string {
	args := []string{
		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
		"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + repo.cfg.IPv4Address,
		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile,
	}
	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())

	return args
}

// for testing, to be removed later
func (repo *Repo) CSCSyncDummy() []string {

	sleepDur := strconv.FormatInt(rand.Int63n(10)+5, 10)
	args := []string{"sleep", sleepDur}

	return args
}

// executes a particular sync job depending on repo.SyncType.
func (repo *Repo) getSyncCommand() []string {
	/*
		# scripts used by merlin.py
		csc-sync-debian
		csc-sync-standard
		csc-sync-ssh
		csc-sync-apache
		csc-sync-archlinux
		csc-sync-debian-cd
		csc-sync-gentoo
		csc-sync-wget
		csc-sync-s3
		csc-sync-standard-ipv6
		csc-sync-ceph

		zfssync
		report_mirror (what is this?)

		# other things in bin/
		csc-sync-archlinux-old
		csc-sync-cdimage
		csc-sync-chmod
		csc-sync-badperms
		make-torrents
		ubuntu-releases-sync
	*/
	switch repo.SyncType {

	case "csc-sync-apache":
		return repo.CSCSyncApache()
	case "csc-sync-archlinux":
		return repo.CSCSyncArchLinux()
	case "csc-sync-badperms":
		return repo.CSCSyncBadPerms()
	case "csc-sync-cdimage":
		return repo.CSCSyncCDImage()
	// case "csc-sync-ceph":
	// 	return repo.CSCSyncCeph()
	case "csc-sync-chmod":
		return repo.CSCSyncChmod()
	case "csc-sync-debian":
		return repo.CSCSyncDebian()
	case "csc-sync-debian-cd":
		return repo.CSCSyncDebianCD()
	case "csc-sync-gentoo":
		return repo.CSCSyncGentoo()
	// case "csc-sync-s3":
	// 	return repo.CSCSyncS3()
	case "csc-sync-ssh":
		return repo.CSCSyncSSH()
	case "csc-sync-standard":
		return repo.CSCSyncStandard()
	case "csc-sync-standard-ipv6":
		return repo.CSCSyncStandardIPV6()
		// case "csc-sync-wget":
		// 	return repo.CSCSyncWget()
	case "csc-sync-dummy":
		return repo.CSCSyncDummy()
	default:
		repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
	}

	return []string{}
}

func (repo *Repo) StartSyncJob() {
	status := FAILURE
	defer func() {
		repo.DoneChan <- Result{
			Name: repo.Name,
			Exit: status,
		}
	}()

	localDir := repo.buildDownloadDir()
	err := os.MkdirAll(localDir, 0775)
	if err != nil {
		err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
		// I'm not sure if logger can handle error so just use the string?
		repo.Logger.Error(err.Error())
		return
	}

	args := repo.getSyncCommand()
	if repo.PasswordFile != "" {
		filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
		args = append(args, "--password-file", filename)
	}
	args = append(args, repo.buildRsyncHost(), localDir)

	ch := SpawnProcess(repo, args)
	if ch == nil {
		// SpawnProcess will have already logged error
		return
	}
	cmd := <-ch
	switch cmd.ProcessState.ExitCode() {
	case 0:
		status = SUCCESS
	case -1:
		status = TERMINATED
		// default is already FAILURE
	}
}