From 7a3c8c1c93cdd7ee9f5f2413c522cb381cfdce8b Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Sun, 24 Jul 2022 17:18:12 +0000 Subject: [PATCH] fix up csc-sync-debian --- merlin/Handoff.md | 58 +++++++++------ merlin/README.md | 3 +- merlin/config/config.go | 10 +-- merlin/merlin-config-all.ini | 4 +- merlin/merlin-config-test.ini | 6 +- merlin/merlin-config.ini | 68 ++++++++++++++++- merlin/sync/interface.go | 1 + merlin/sync/process.go | 4 +- merlin/sync/sync.go | 51 ++++++------- merlin/sync/trace.go | 135 +++++++++++++++++++++------------- 10 files changed, 220 insertions(+), 120 deletions(-) diff --git a/merlin/Handoff.md b/merlin/Handoff.md index 691a992..0a135c5 100644 --- a/merlin/Handoff.md +++ b/merlin/Handoff.md @@ -1,18 +1,18 @@ ## Status of moving from merlin to merlin-go -- [ ] debian -- [ ] ubuntu -- [ ] ubuntu-ports -- [ ] linuxmint-packages -- [ ] debian-multimedia -- [ ] debian-backports -- [ ] debian-security +- [ ] debian < +- [ ] ubuntu < +- [ ] ubuntu-ports < +- [ ] linuxmint-packages < +- [ ] debian-multimedia < +- [ ] debian-backports < +- [ ] debian-security < - [x] ubuntu-releases - [x] xubuntu-releases - [ ] puppylinux (broken) - [ ] CPAN (unstable) -- [ ] CRAN + - [x] CTAN -- [ ] fedora-epel < +- [ ] fedora-epel (some attrs not transfered) - [x] cygwin - [x] gnu - [x] nongnu @@ -26,20 +26,20 @@ - [ ] eclipse (unstable) - [x] kde - [x] kde-applicationdata -- [ ] archlinux + - [x] artixlinx - [x] slackware -- [ ] debian-cd + - [x] x.org - [x] centos -- [ ] opensuse < +- [x] opensuse - [x] FreeBSD -- [ ] fedora-enchilada < +- [ ] fedora-enchilada (broken: some attrs not transfered) - [x] ubuntu-ports-releases -- [ ] gentoo-distfiles + - [x] gentoo-portage -- [ ] gutenberg (broken) -- [ ] racket-installers +- [ ] gutenberg (unstable) + - [ ] plt-bundles (broken) - [x] xiph - [ ] netbsd (broken) @@ -48,11 +48,10 @@ - [x] sagemath - [x] vlc - [x] tdf -- [ ] saltstack -- [x] alpine -- [ ] raspbian (high disk usage + long runtime + failing) -- [ ] raspberrypi -- [ ] ipfire (broken) + +- [ ] alpine (unstable) +- [ ] raspbian (broekn: high disk usage + long runtime + failing) +- [ ] ipfire (unstable) - [x] manjaro - [x] mxlinux - [x] mxlinux-iso @@ -60,4 +59,19 @@ - [x] trisquel-packages - [x] trisquel-iso - [x] almalinux -- [ ] ceph + + +### Other Repos +- debian-cdimage +- debian-volatile +- emdebian +- openoffice +- mozdev +- mozilla.org +- raspberrypi +- cs136 +- kali +- kali-images +- hyperbola-sources +- hyperbola-stable +- hyperbola-testing \ No newline at end of file diff --git a/merlin/README.md b/merlin/README.md index f711390..b1b3461 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -13,11 +13,12 @@ Then configure `merlin-config.ini` and run using `./merlin` ### Nice Features To Add - detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes) -- get the config or the rsync commond that a repo will sync with using a cli tool +- get the config or the rsync command that a repo will sync with using a cli tool - improve conversion from exit status enum to string - sort `arthur status` by last time synced - last sync runtime, time until next sync - when sync fails, then make a copy of the rsync logs +- implementation of `trace` is ad hoc and can be improved ### Completed - [x] add bwlimit option for each rsync process diff --git a/merlin/config/config.go b/merlin/config/config.go index 4b7931a..d597f53 100644 --- a/merlin/config/config.go +++ b/merlin/config/config.go @@ -22,13 +22,13 @@ const ( FIVE_MINUTELY = 300 MINUTELY = 60 + DEAFULT_HOSTNAME = "mirror.csclub.uwaterloo.ca" DEFAULT_MAX_JOBS = 6 DEFAULT_MAX_TIME = DAILY / 4 DEFAULT_MAX_RSYNC_IO = 0 DEFAULT_SYNC_TYPE = "csc-sync-standard" DEFAULT_FREQUENCY_STRING = "bi-hourly" DEFAULT_DOWNLOAD_DIR = "/mirror/root" - DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace" DEFAULT_STATE_DIR = "/home/mirror/merlin/state" DEFAULT_LOG_DIR = "/home/mirror/merlin/log" DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync" @@ -62,6 +62,7 @@ type SyncResult struct { } type Config struct { + Hostname string `ini:"hostname"` // the IPv4 addresses to use for rsync IPv4Address string `ini:"ipv4_address"` // the IPv6 addresses to use for rsync @@ -79,8 +80,6 @@ type Config struct { DefaultFrequencyStr string `ini:"default_frequency"` // the base directory where rsync should download files to DownloadDir string `ini:"download_dir"` - // directory where the trace of each repo is saved - TraceDir string `ini:"trace_dir"` // directory where the state of each repo is saved StateDir string `ini:"state_dir"` // directory where merlin will store the merlin logs for each repo @@ -113,7 +112,7 @@ type Repo struct { // where to download the files for this repo (relative to Conf.DownloadDir) LocalDir string `ini:"local_dir"` // the address to the trace file (how this url will be used depends on SyncType) - TraceUrl string `ini:"trace_url"` + TraceHost string `ini:"trace_host"` // the address to the remote host to rsync from RsyncHost string `ini:"rsync_host"` // the remote directory on the rsync host (optional) @@ -170,13 +169,13 @@ var ( func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struct{}) { // create a new config with the default values then load config from file newConf := Config{ + Hostname: DEAFULT_HOSTNAME, MaxJobs: DEFAULT_MAX_JOBS, DefaultMaxTime: DEFAULT_MAX_TIME, DefaultMaxRsyncIO: DEFAULT_MAX_RSYNC_IO, DefaultSyncType: DEFAULT_SYNC_TYPE, DefaultFrequencyStr: DEFAULT_FREQUENCY_STRING, DownloadDir: DEFAULT_DOWNLOAD_DIR, - TraceDir: DEFAULT_TRACE_DIR, StateDir: DEFAULT_STATE_DIR, RepoLogDir: DEFAULT_LOG_DIR, RsyncLogDir: DEFAULT_RSYNC_LOG_DIR, @@ -201,7 +200,6 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc // create directories for _, dir := range []string{ - newConf.TraceDir, newConf.StateDir, newConf.RepoLogDir, newConf.RsyncLogDir, diff --git a/merlin/merlin-config-all.ini b/merlin/merlin-config-all.ini index 7e43b60..ec1e733 100644 --- a/merlin/merlin-config-all.ini +++ b/merlin/merlin-config-all.ini @@ -40,7 +40,7 @@ frequency = bi-hourly local_dir = ubuntu rsync_host = archive.ubuntu.com rsync_dir = ubuntu -; trace_host = drescher.canonical.com +trace_host = drescher.canonical.com [ubuntu-ports] sync_type = csc-sync-debian @@ -48,7 +48,7 @@ frequency = bi-hourly local_dir = ubuntu-ports rsync_host = ports.ubuntu.com rsync_dir = ubuntu-ports -; trace_host = drescher.canonical.com +trace_host = drescher.canonical.com [linuxmint-packages] sync_type = csc-sync-debian diff --git a/merlin/merlin-config-test.ini b/merlin/merlin-config-test.ini index cde8f8e..8811079 100644 --- a/merlin/merlin-config-test.ini +++ b/merlin/merlin-config-test.ini @@ -15,15 +15,15 @@ sock_path = /mirror/merlin/run/merlin-go.sock [ubuntu] dry_run = true -sync_type = csc-sync-standard +sync_type = csc-sync-debian frequency = five-minutely local_dir = ubuntu rsync_host = rsync.releases.ubuntu.com rsync_dir = releases -rsync_exclude = distribution/.timestamp_invisiable, .ignore_me +; rsync_exclude = distribution/.timestamp_invisiable, .ignore_me ; add to the repo struct ; RsyncExclude []string `ini:"rsync_exclude"` ; then iterate over RsyncExclude: -; for _, exclude := range repo.RsyncExclude {} \ No newline at end of file +; for _, exclude := range repo.RsyncExclude {} diff --git a/merlin/merlin-config.ini b/merlin/merlin-config.ini index 49cd190..bb47b6f 100644 --- a/merlin/merlin-config.ini +++ b/merlin/merlin-config.ini @@ -13,6 +13,72 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync sock_path = /mirror/merlin/run/merlin-go.sock +[debian] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = debian +rsync_host = debian.mirror.rafal.ca +rsync_dir = debian + +[ubuntu] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = ubuntu +rsync_host = archive.ubuntu.com +rsync_dir = ubuntu +trace_host = drescher.canonical.com + +[ubuntu-ports] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = ubuntu-ports +rsync_host = ports.ubuntu.com +rsync_dir = ubuntu-ports +trace_host = drescher.canonical.com + +[linuxmint-packages] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = linuxmint-packages +rsync_host = rsync-packages.linuxmint.com +rsync_dir = packages + +[debian-multimedia] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = debian-multimedia +rsync_host = www.deb-multimedia.org +rsync_dir = deb + +[debian-backports] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = debian-backports +rsync_host = debian.mirror.rafal.ca +rsync_dir = debian-backports + +[debian-security] +dry_run = true +verbose = true +sync_type = csc-sync-debian +frequency = twice-hourly +local_dir = debian-security +rsync_host = rsync.security.debian.org +rsync_dir = debian-security +trace_host = security-master.debian.org + [ubuntu-releases] sync_type = csc-sync-standard frequency = bi-hourly @@ -183,7 +249,6 @@ rsync_host = us-msync.centos.org rsync_dir = CentOS [opensuse] -verbose = true ; unsure what this setting is supposed to do ; #"--exclude distribution/.timestamp_invisible" sync_type = csc-sync-standard @@ -295,6 +360,7 @@ rsync_host = rsync.documentfoundation.org rsync_dir = tdf-pub [alpine] +verbose = true sync_type = csc-sync-standard frequency = hourly local_dir = alpine diff --git a/merlin/sync/interface.go b/merlin/sync/interface.go index 3a88357..7be313e 100644 --- a/merlin/sync/interface.go +++ b/merlin/sync/interface.go @@ -50,5 +50,6 @@ func SyncCompleted(repo *config.Repo, exit int) { if exit == config.SUCCESS { go zfsSync(repo) + go postSyncTraceUpdate(repo) } } diff --git a/merlin/sync/process.go b/merlin/sync/process.go index 11d280a..c4b3514 100644 --- a/merlin/sync/process.go +++ b/merlin/sync/process.go @@ -87,12 +87,12 @@ func startRepoSync(repo *config.Repo) { } }() - if repo.TraceUrl != "" { + if repo.TraceHost != "" { repo.Logger.Debug("Checking for changes") continueSync := true status, continueSync = checkIfSyncNeeded(repo) - if !continueSync { + if !continueSync || status == config.TERMINATED { return } diff --git a/merlin/sync/sync.go b/merlin/sync/sync.go index ed78e3c..fae9013 100644 --- a/merlin/sync/sync.go +++ b/merlin/sync/sync.go @@ -39,12 +39,17 @@ func buildDownloadDir(repo *config.Repo) string { const ( noOwnerNoGroup = 1 << iota timeout3600 + excludeTmp logFile quiet ipv4 ipv6 delete delayUpdatesDeleteAfter + // adds base arguments for setting timeout, a logfile, and including quiet + baseFlags = timeout3600 | logFile | quiet + // adds standard arguments timeout, logging, quiet, no owner/group, and excluding .~tmp~/ + stdFlags = baseFlags | noOwnerNoGroup | excludeTmp ) func addConditionalFlags(repo *config.Repo, flags int) []string { @@ -55,10 +60,13 @@ func addConditionalFlags(repo *config.Repo, flags int) []string { if flags&timeout3600 != 0 { args = append(args, "--timeout=3600") } + if flags&excludeTmp != 0 { + args = append(args, "--exclude", ".~tmp~/") + } if flags&logFile != 0 { args = append(args, "--log-file="+repo.RsyncLogFile) } - if flags&quiet != 0 { + if flags&quiet != 0 && !repo.Verbose { args = append(args, "--quiet") } @@ -92,9 +100,8 @@ func cscSyncApache(repo *config.Repo) []string { "--delete", "--safe-links", "--stats", - "--exclude", ".~tmp~/", } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4)...) + args = append(args, addConditionalFlags(repo, stdFlags|ipv4)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args @@ -122,10 +129,9 @@ func cscSyncBadPerms(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--chmod=o=rX", - "--exclude", ".~tmp~/", "--stats", } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delete)...) + args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delete)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args @@ -137,7 +143,7 @@ func cscSyncCDImage(repo *config.Repo) []string { "--exclude", "\".*/\"", "--stats", } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delete)...) + args = append(args, addConditionalFlags(repo, baseFlags|noOwnerNoGroup|ipv4|delete)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args @@ -176,49 +182,37 @@ func cscSyncChmod(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--safe-links", - "--exclude", ".~tmp~/", "--stats", "--chmod=u=rwX,go=rX", } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delayUpdatesDeleteAfter)...) + args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args } func cscSyncDebian(repo *config.Repo) []string { - // some repos specify a trace file but does not exit script even if trace files are the same - // https://git.csclub.uwaterloo.ca/public/mirror/src/branch/go/bin/csc-sync-debian#L205-L218 - - // ubuntu repos are trying to get trace/drescher.canonical.com but this file does not exist - // http://archive.ubuntu.com/ubuntu/project/trace/ - // if are adding a trace check ensure it can handle when the file does not exist on the remote - args := []string{ "nice", "rsync", "-rlHtvp", - "--exclude", ".~tmp~/", } - args = append(args, addConditionalFlags(repo, timeout3600|logFile|quiet|ipv4)...) + args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4)...) args = append(args, buildRsyncDaemonHost(repo)+"/pool/", buildDownloadDir(repo)+"/pool/") args = append(args, []string{ "&&", "nice", "rsync", "-rlHtvp", - "--exclude", ".~tmp~/", + "--exclude", filepath.Join("project/trace", config.Conf.Hostname), }...) - args = append(args, addConditionalFlags(repo, timeout3600|logFile|quiet|ipv4|delayUpdatesDeleteAfter)...) + args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) - // TODO: run 'LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"' after the sync is done - return args } func cscSyncDebianCD(repo *config.Repo) []string { args := []string{ - "nice", "rsync", "-rlHtvp4", - "--exclude", ".~tmp~/", + "nice", "rsync", "-rlHtvp", } - args = append(args, addConditionalFlags(repo, timeout3600|logFile|quiet|ipv4|delete)...) + args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delete)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args @@ -252,10 +246,9 @@ func cscSyncS3(repo *config.Repo) []string { func cscSyncSSH(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", - "--exclude", ".~tmp~/", "--stats", "-4", } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|delete)...) + args = append(args, addConditionalFlags(repo, stdFlags|delete)...) // PasswordFile should point to the SSH_KEYFILE args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile)) args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo)) @@ -267,7 +260,6 @@ func cscSyncStandard(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--safe-links", - "--exclude", ".~tmp~/", "--stats", } @@ -275,7 +267,7 @@ func cscSyncStandard(repo *config.Repo) []string { args = append(args, "--password-file", repo.PasswordFile) } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delayUpdatesDeleteAfter)...) + args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncHost(repo), buildDownloadDir(repo)) return args @@ -285,7 +277,6 @@ func cscSyncStandardIPV6(repo *config.Repo) []string { args := []string{ "nice", "rsync", "-aH", "--safe-links", - "--exclude", ".~tmp~/", "--stats", } @@ -293,7 +284,7 @@ func cscSyncStandardIPV6(repo *config.Repo) []string { args = append(args, "--password-file", repo.PasswordFile) } - args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv6|delayUpdatesDeleteAfter)...) + args = append(args, addConditionalFlags(repo, stdFlags|ipv6|delayUpdatesDeleteAfter)...) args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo)) return args diff --git a/merlin/sync/trace.go b/merlin/sync/trace.go index 8caeba7..64ef7bf 100644 --- a/merlin/sync/trace.go +++ b/merlin/sync/trace.go @@ -2,24 +2,26 @@ package sync import ( "os" - "path" + "path/filepath" "strings" + "time" "git.csclub.uwaterloo.ca/public/merlin/config" ) -func cscTraceArchLinux(repo *config.Repo, destPath string) []string { +func cscTraceArchLinux(repo *config.Repo, newTime string) []string { args := []string{ "curl", - "-s", repo.TraceUrl, - "-o", destPath, + "--interface", config.Conf.IPv4Address, + "-s", repo.TraceHost, + "-o", newTime, } return args } // return false iff both files exist, are non-empty, and have the same contents -func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool { +func cscTraceArchLinuxDiff(repo *config.Repo, newTime string) bool { readFile := func(file string) string { f, err := os.ReadFile(file) if err != nil { @@ -29,8 +31,8 @@ func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool { return strings.TrimSpace(string(f)) } - file1 := readFile(destPath) - file2 := readFile(path.Join(buildDownloadDir(repo), "lastupdate")) + file1 := readFile(newTime) + file2 := readFile(filepath.Join(buildDownloadDir(repo), "lastupdate")) if file1 == "" || file2 == "" { return true @@ -38,6 +40,7 @@ func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool { return file1 != file2 } +// update the lastsync file func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) { rsyncDir := repo.RsyncDir localDir := repo.LocalDir @@ -53,21 +56,20 @@ func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) { return args } -func cscTraceDebian(repo *config.Repo, destPath string) []string { +func cscTraceDebian(repo *config.Repo, newTime string) []string { args := []string{ "nice", "rsync", "-tv", "-4", "--address=" + config.Conf.IPv4Address, "--quiet", - // "--log-file=" + repo.RepoLogFile, - repo.RsyncHost + "::" + path.Join(repo.RsyncDir, "project/trace", repo.TraceUrl), - destPath, + repo.RsyncHost + "::" + filepath.Join(repo.RsyncDir, "project/trace", repo.TraceHost), + newTime, } return args } // return false iff both files exist and were modified at the same time -func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { +func cscTraceDebianDiff(repo *config.Repo, newTime string) bool { statFile := func(file string) int64 { f, err := os.Stat(file) if err != nil { @@ -78,8 +80,8 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { return f.ModTime().Unix() } - file1 := statFile(destPath) - file2 := statFile(path.Join(buildDownloadDir(repo), "project/trace", "lastupdate")) + file1 := statFile(newTime) + file2 := statFile(filepath.Join(buildDownloadDir(repo), "project/trace", repo.TraceHost)) if file1 == 0 || file2 == 0 { return true @@ -88,15 +90,44 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { return file1 != file2 } +// update our trace file's modification date by writing the current time +func cscTraceDebianUpdate(repo *config.Repo) { + target := filepath.Join(buildDownloadDir(repo), "project/trace", config.Conf.Hostname) + + os.MkdirAll(filepath.Join(buildDownloadDir(repo), "project/trace"), 0755) + + f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + repo.Logger.Error("Unable to open trace file: " + target) + return + } + if _, err = f.WriteString(time.Now().UTC().Format(time.RFC1123)); err != nil { + repo.Logger.Error("Unable to write to trace file: " + target) + return + } +} + // some repos want us to check a last modification time file before performing a full sync // returns a status to update the original sync's status along with if the sync should continue func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { // default return values are the default status and to continue the sync status = config.FAILURE continueSync = true + if repo.DryRun { + return + } + runCommand := func(args []string) int { + ch := spawnProcess(repo, args) + if ch == nil { + // spawnSyncProcess will have already logged error + return 1 + } + cmd := <-ch + return cmd.ProcessState.ExitCode() + } // create a temp file - temp, err := os.CreateTemp("/tmp", ".tmp.merlin."+repo.Name+"-") + temp, err := os.CreateTemp("/tmp", "merlin-"+repo.Name+"-trace-*") if err != nil { repo.Logger.Error(err.Error()) return @@ -104,7 +135,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { temp.Close() defer os.Remove(temp.Name()) - // get the trace command + // get the trace command to copy a last updated file var args []string switch repo.SyncType { case "csc-sync-archlinux": @@ -116,16 +147,10 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { return } - ch := spawnProcess(repo, args) - if ch == nil { - // spawnSyncProcess will have already logged error - return - } - - cmd := <-ch - if cmd.ProcessState.ExitCode() != 0 { + exit := runCommand(args) + if exit != 0 { // if the process was terminated then don't continue to sync - if cmd.ProcessState.ExitCode() == -1 { + if exit == -1 { return config.TERMINATED, false } return @@ -138,37 +163,41 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { case "csc-sync-debian": filesDiffer = cscTraceDebianDiff(repo, temp.Name()) } + // debian still syncs even if the trace file is the same + if !filesDiffer && repo.SyncType == "csc-sync-debian" { + repo.Logger.Error("trace file for " + repo.RsyncHost + " unchanged") + return + } + if filesDiffer { + // continue sync if files differ + return + } - // if the files are the same then don't continue to sync - if !filesDiffer { - repo.Logger.Debug("No changes found; no sync made") - - // archlinux wants to sync a "lastsync" file even if lastupdate prevents sync - var args []string - switch repo.SyncType { - case "csc-sync-archlinux": - args = cscTraceArchLinuxUpdate(repo) - default: - return config.SUCCESS, false - } - - ch := spawnProcess(repo, args) - if ch == nil { - // spawnSyncProcess will have already logged error - return - } - - cmd := <-ch - if cmd.ProcessState.ExitCode() != 0 { - // if the process was terminated then don't continue to sync - if cmd.ProcessState.ExitCode() == -1 { - return config.TERMINATED, false - } - return - } + repo.Logger.Debug("No changes found; full sync will not be made") + // archlinux wants to sync a "lastsync" file if lastupdate prevents sync + switch repo.SyncType { + case "csc-sync-archlinux": + args = cscTraceArchLinuxUpdate(repo) + default: return config.SUCCESS, false } - return + exit = runCommand(args) + if exit != 0 { + // if the process was terminated then don't continue to sync + if exit == -1 { + return config.TERMINATED, false + } + return + } + + return config.SUCCESS, false +} + +// csc-sync-debian wants to update a trace file after the repo is done syncing +func postSyncTraceUpdate(repo *config.Repo) { + if repo.SyncType == "csc-sync-debian" { + cscTraceDebianUpdate(repo) + } }