package sync import ( "fmt" "os" "path/filepath" "git.csclub.uwaterloo.ca/public/merlin/config" ) func runRepoSync(repo *config.Repo) (status int) { status = config.FAILURE cmds := getSyncCommand(repo) if len(cmds) == 0 { repo.Logger.Error() return } // run every step of the sync command as long as the previous step was successful for i, args := range cmds { repo.Logger.Debug(fmt.Sprintf("Running step %d of sync", i)) status = spawnProcessAndWait(repo, args) if status != config.SUCCESS { // spawnProcessAndWait will have already logged error return } } return } // executes a particular sync job depending on repo.SyncType. func getSyncCommand(repo *config.Repo) (cmds [][]string) { // check that the download directory exists if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) { repo.Logger.Error(err.Error()) return } switch repo.SyncType { case "csc-sync-apache": return append(cmds, cscSyncApache(repo)) case "csc-sync-archlinux": return append(cmds, cscSyncArchLinux(repo)) case "csc-sync-badperms": return append(cmds, cscSyncBadPerms(repo)) case "csc-sync-cdimage": return append(cmds, cscSyncCDImage(repo)) case "csc-sync-ceph": return append(cmds, cscSyncCephStep1(repo), cscSyncCephStep2(repo)) case "csc-sync-chmod": return append(cmds, cscSyncChmod(repo)) case "csc-sync-debian": return append(cmds, cscSyncDebianStep1(repo), cscSyncDebianStep2(repo)) case "csc-sync-debian-cd": return append(cmds, cscSyncDebianCD(repo)) case "csc-sync-gentoo": return append(cmds, cscSyncGentoo(repo)) case "csc-sync-s3": return append(cmds, cscSyncS3(repo)) case "csc-sync-ssh": return append(cmds, cscSyncSSH(repo)) case "csc-sync-standard": return append(cmds, cscSyncStandard(repo)) case "csc-sync-standard-ipv6": return append(cmds, cscSyncStandardIPV6(repo)) case "csc-sync-wget": return append(cmds, cscSyncWget(repo)) default: repo.Logger.Error("Unrecognized sync type: " + repo.SyncType) return } } const ( noOwnerNoGroup = 1 << iota timeout3600 excludeTmp logFile quiet ipv4 ipv6 delete delayUpdatesDeleteAfter // adds base arguments for setting timeout, a logfile, and including quiet baseFlags = timeout3600 | logFile | quiet // adds standard arguments timeout, logging, quiet, no owner/group, and excluding .~tmp~/ stdFlags = baseFlags | noOwnerNoGroup | excludeTmp ) func addConditionalFlags(repo *config.Repo, flags int) []string { args := []string{} if flags&noOwnerNoGroup != 0 { args = append(args, "--no-owner", "--no-group") } if flags&timeout3600 != 0 { args = append(args, "--timeout=3600") } if flags&excludeTmp != 0 { args = append(args, "--exclude", ".~tmp~/") } if flags&logFile != 0 { args = append(args, "--log-file="+repo.RsyncLogFile) } if flags&quiet != 0 && !repo.Verbose { args = append(args, "--quiet") } if flags&ipv4 != 0 { args = append(args, "-4", "--address="+config.Conf.IPv4Address) } else if flags&ipv6 != 0 { args = append(args, "-6", "--address="+config.Conf.IPv6Address) } if flags&delete != 0 { args = append(args, "--delete") } if flags&delayUpdatesDeleteAfter != 0 { args = append(args, "--delay-updates", "--delete-after") } // add bwlimit (bandwidth limit) only when MaxRsyncIO is set to positive value if repo.MaxRsyncIO > 0 { args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO)) } if repo.Verbose { args = append(args, "-vv") } return args } func buildRsyncHost(repo *config.Repo) string { rsyncHost := repo.RsyncHost if repo.RsyncUser != "" { rsyncHost = repo.RsyncUser + "@" + rsyncHost } return "rsync://" + rsyncHost + "/" + repo.RsyncDir } func buildRsyncDaemonHost(repo *config.Repo) string { rsyncHost := repo.RsyncHost if repo.RsyncUser != "" { rsyncHost = repo.RsyncUser + "@" + rsyncHost } return rsyncHost + "::" + repo.RsyncDir } func buildRsyncSSHHost(repo *config.Repo) string { rsyncHost := repo.RsyncHost if repo.RsyncUser != "" { rsyncHost = repo.RsyncUser + "@" + rsyncHost } return rsyncHost + ":" + repo.RsyncDir } func buildDownloadDir(repo *config.Repo) string { return filepath.Join(config.Conf.DownloadDir, repo.LocalDir) } func cscSyncApache(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-az", "--delete", "--safe-links", "--stats", } args = append(args, addConditionalFlags(repo, stdFlags|ipv4)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncArchLinux(repo *config.Repo) []string { tempDir := "/home/mirror/tmp" args := []string{ "rsync", "-rtlHp", "--safe-links", "--timeout=600", "--contimeout=60", "--no-motd", "--temp-dir=" + tempDir, "--address=" + config.Conf.IPv4Address, } args = append(args, addConditionalFlags(repo, logFile|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncBadPerms(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--chmod=o=rX", "--stats", } args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delete)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncCDImage(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--exclude", "\".*/\"", "--stats", } args = append(args, addConditionalFlags(repo, baseFlags|noOwnerNoGroup|ipv4|delete)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncCephStep1(repo *config.Repo) []string { args := []string{ "rsync", "--stats", "--progress", repo.RsyncHost + "::ceph", "--recursive", "--times", "--links", "--hard-links", "--exclude", "Packages*", "--exclude", "Sources*", "--exclude", "Release*", "--exclude", "InRelease", "--exclude", "i18n/*", "--exclude", "ls-lR*", "--exclude", "repodata/*", } args = append(args, addConditionalFlags(repo, quiet|ipv4)...) args = append(args, buildDownloadDir(repo)) return args } func cscSyncCephStep2(repo *config.Repo) []string { args := []string{ "&&", "rsync", "--stats", "--progress", repo.RsyncHost + "::ceph", "--recursive", "--times", "--links", "--hard-links", "--delete-after", } args = append(args, addConditionalFlags(repo, quiet|ipv4)...) args = append(args, buildDownloadDir(repo)) return args } func cscSyncChmod(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--safe-links", "--stats", "--chmod=u=rwX,go=rX", } args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncDebianStep1(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-rlHtvp", } args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4)...) args = append(args, buildRsyncDaemonHost(repo)+"/pool/", buildDownloadDir(repo)+"/pool/") return args } func cscSyncDebianStep2(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-rlHtvp", "--exclude", filepath.Join("project/trace", config.Conf.Hostname), } args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncDebianCD(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-rlHtvp", } args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delete)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncGentoo(repo *config.Repo) []string { repo.RsyncUser = "gentoo" repo.PasswordFile = "gentoo-distfiles" return cscSyncStandard(repo) } // TODO: check for special stuff that rcloning S3 needs func cscSyncS3(repo *config.Repo) []string { args := []string{ // RsyncHost is just a regular host (https://s3.repo.saltproject.io) in this case "RCLONE_CONFIG_S3_ENDPOINT=" + repo.RsyncHost, "RCLONE_CONFIG_S3_TYPE=s3", "RCLONE_CONFIG_S3_PROVIDER=Other", "RCLONE_CONFIG_S3_ENV_AUTH=false", "nice", "rclone", "sync", "--fast-list", "--use-server-modtime", "--bind", config.Conf.IPv4Address, "s3:s3/", } args = append(args, buildDownloadDir(repo)) return args } func cscSyncSSH(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--stats", "-4", } args = append(args, addConditionalFlags(repo, stdFlags|delete)...) // PasswordFile should point to the SSH_KEYFILE args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile)) args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo)) return args } func cscSyncStandard(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--safe-links", "--stats", } if repo.PasswordFile != "" { args = append(args, "--password-file", repo.PasswordFile) } args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncStandardIPV6(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--safe-links", "--stats", } if repo.PasswordFile != "" { args = append(args, "--password-file", repo.PasswordFile) } args = append(args, addConditionalFlags(repo, stdFlags|ipv6|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args } func cscSyncWget(repo *config.Repo) []string { args := []string{ "nice", "wget", "-q", "--bind-address=" + config.Conf.IPv4Address, "--mirror", "--no-parent", "--no-host-directories", "--cut-dirs=1", "--content-disposition", "--execute", "robots=off", "--recursive", "--reject", "\"*\\?*\"", "--directory-prefix=" + buildDownloadDir(repo), // RsyncHost is just a regular host (https://mirror.racket-lang.org/installers/) in this case repo.RsyncHost, } return args }