package sync import ( "fmt" "math/rand" "strconv" "git.csclub.uwaterloo.ca/public/merlin/config" ) func buildRsyncHost(repo *config.Repo) string { if repo.RsyncUser != "" { repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost } return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir } func buildRsyncDaemonHost(repo *config.Repo) string { if repo.RsyncUser != "" { repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost } return repo.RsyncHost + "::" + repo.RsyncDir } func buildRsyncSSHHost(repo *config.Repo) string { if repo.RsyncUser != "" { repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost } return repo.RsyncHost + ":" + repo.RsyncDir } func buildDownloadDir(repo *config.Repo) string { return config.Conf.DownloadDir + "/" + repo.LocalDir } func cscSyncApache(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, } args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncArchLinux(repo *config.Repo) []string { tempDir := "" // is this option even needed? // add option for verbose flag? args := []string{ "rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600", "--contimeout=60", "-p", "--delay-updates", "--no-motd", "--temp-dir=" + tempDir, "--log-file=" + repo.RsyncLogFile, "--address=" + config.Conf.IPv4Address, } args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncBadPerms(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, } args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } // TODO ceph func cscSyncCDImage(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, "--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, } args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncChmod(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, "--chmod=u=rwX,go=rX", } args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncDebian(repo *config.Repo) []string { // sync /pool args := []string{"nice", "rsync", "-rlHtvp", "--exclude", ".~tmp~/", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, } // $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1 return args } func cscSyncDebianCD(repo *config.Repo) []string { // this is basically the same as CSCSyncDebian, except it has an extra --exclude args := []string{"nice", "rsync", "-rlHtvp", "--delete", "--exclude", ".~tmp~/", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, // "--exclude", "Archive-Update-in-Progress-${HOSTNAME}" } // $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1 return args } func cscSyncGentoo(repo *config.Repo) []string { repo.RsyncUser = "gentoo" repo.PasswordFile = "gentoo-distfiles" return cscSyncStandard(repo) } // TODO s3 func cscSyncSSH(repo *config.Repo) []string { args := []string{ "rsync", "-aH", "--no-owner", "--no-group", "--delete", "--timeout=3600", "-4", "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, } // not sure if we should be assuming ssh identity file is the password file args = append(args, "-e", fmt.Sprintf("'ssh -i %s'", repo.PasswordFile)) args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo)) return args } func cscSyncStandard(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + config.Conf.IPv4Address, "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, } if repo.PasswordFile != "" { filename := config.Conf.PasswordDir + "/" + repo.PasswordFile args = append(args, "--password-file", filename) } args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncStandardIPV6(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + config.Conf.IPv4Address, "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.RsyncLogFile, } args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } // for testing, to be removed later func cscSyncDummy(repo *config.Repo) []string { sleepDur := strconv.FormatInt(rand.Int63n(10)+5, 10) args := []string{"sleep", sleepDur} return args } // executes a particular sync job depending on repo.SyncType. func getSyncCommand(repo *config.Repo) []string { /* # scripts used by merlin.py csc-sync-debian csc-sync-standard csc-sync-ssh csc-sync-apache csc-sync-archlinux csc-sync-debian-cd csc-sync-gentoo csc-sync-wget csc-sync-s3 csc-sync-standard-ipv6 csc-sync-ceph zfssync report_mirror (what is this?) # other things in bin/ csc-sync-archlinux-old csc-sync-cdimage csc-sync-chmod csc-sync-badperms make-torrents ubuntu-releases-sync */ switch repo.SyncType { case "csc-sync-apache": return cscSyncApache(repo) case "csc-sync-archlinux": return cscSyncArchLinux(repo) case "csc-sync-badperms": return cscSyncBadPerms(repo) case "csc-sync-cdimage": return cscSyncCDImage(repo) // case "csc-sync-ceph": // return cscSyncCeph(repo) case "csc-sync-chmod": return cscSyncChmod(repo) case "csc-sync-debian": return cscSyncDebian(repo) case "csc-sync-debian-cd": return cscSyncDebianCD(repo) case "csc-sync-gentoo": return cscSyncGentoo(repo) // case "csc-sync-s3": // return cscSyncS3(repo) case "csc-sync-ssh": return cscSyncSSH(repo) case "csc-sync-standard": return cscSyncStandard(repo) case "csc-sync-standard-ipv6": return cscSyncStandardIPV6(repo) // case "csc-sync-wget": // return cscSyncWget(repo) case "csc-sync-dummy": return cscSyncDummy(repo) default: repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'") } return []string{} }