|
|
|
|
@ -2,24 +2,26 @@ package sync |
|
|
|
|
|
|
|
|
|
import ( |
|
|
|
|
"os" |
|
|
|
|
"path" |
|
|
|
|
"path/filepath" |
|
|
|
|
"strings" |
|
|
|
|
"time" |
|
|
|
|
|
|
|
|
|
"git.csclub.uwaterloo.ca/public/merlin/config" |
|
|
|
|
) |
|
|
|
|
|
|
|
|
|
func cscTraceArchLinux(repo *config.Repo, destPath string) []string { |
|
|
|
|
func cscTraceArchLinux(repo *config.Repo, newTime string) []string { |
|
|
|
|
args := []string{ |
|
|
|
|
"curl", |
|
|
|
|
"-s", repo.TraceUrl, |
|
|
|
|
"-o", destPath, |
|
|
|
|
"--interface", config.Conf.IPv4Address, |
|
|
|
|
"-s", repo.TraceHost, |
|
|
|
|
"-o", newTime, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return args |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// return false iff both files exist, are non-empty, and have the same contents
|
|
|
|
|
func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool { |
|
|
|
|
func cscTraceArchLinuxDiff(repo *config.Repo, newTime string) bool { |
|
|
|
|
readFile := func(file string) string { |
|
|
|
|
f, err := os.ReadFile(file) |
|
|
|
|
if err != nil { |
|
|
|
|
@ -29,8 +31,8 @@ func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool { |
|
|
|
|
return strings.TrimSpace(string(f)) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
file1 := readFile(destPath) |
|
|
|
|
file2 := readFile(path.Join(buildDownloadDir(repo), "lastupdate")) |
|
|
|
|
file1 := readFile(newTime) |
|
|
|
|
file2 := readFile(filepath.Join(buildDownloadDir(repo), "lastupdate")) |
|
|
|
|
|
|
|
|
|
if file1 == "" || file2 == "" { |
|
|
|
|
return true |
|
|
|
|
@ -38,6 +40,7 @@ func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool { |
|
|
|
|
return file1 != file2 |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// update the lastsync file
|
|
|
|
|
func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) { |
|
|
|
|
rsyncDir := repo.RsyncDir |
|
|
|
|
localDir := repo.LocalDir |
|
|
|
|
@ -53,21 +56,20 @@ func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) { |
|
|
|
|
return args |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
func cscTraceDebian(repo *config.Repo, destPath string) []string { |
|
|
|
|
func cscTraceDebian(repo *config.Repo, newTime string) []string { |
|
|
|
|
args := []string{ |
|
|
|
|
"nice", "rsync", "-tv", "-4", |
|
|
|
|
"--address=" + config.Conf.IPv4Address, |
|
|
|
|
"--quiet", |
|
|
|
|
// "--log-file=" + repo.RepoLogFile,
|
|
|
|
|
repo.RsyncHost + "::" + path.Join(repo.RsyncDir, "project/trace", repo.TraceUrl), |
|
|
|
|
destPath, |
|
|
|
|
repo.RsyncHost + "::" + filepath.Join(repo.RsyncDir, "project/trace", repo.TraceHost), |
|
|
|
|
newTime, |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return args |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// return false iff both files exist and were modified at the same time
|
|
|
|
|
func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { |
|
|
|
|
func cscTraceDebianDiff(repo *config.Repo, newTime string) bool { |
|
|
|
|
statFile := func(file string) int64 { |
|
|
|
|
f, err := os.Stat(file) |
|
|
|
|
if err != nil { |
|
|
|
|
@ -78,8 +80,8 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { |
|
|
|
|
return f.ModTime().Unix() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
file1 := statFile(destPath) |
|
|
|
|
file2 := statFile(path.Join(buildDownloadDir(repo), "project/trace", "lastupdate")) |
|
|
|
|
file1 := statFile(newTime) |
|
|
|
|
file2 := statFile(filepath.Join(buildDownloadDir(repo), "project/trace", repo.TraceHost)) |
|
|
|
|
|
|
|
|
|
if file1 == 0 || file2 == 0 { |
|
|
|
|
return true |
|
|
|
|
@ -88,15 +90,44 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { |
|
|
|
|
return file1 != file2 |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// update our trace file's modification date by writing the current time
|
|
|
|
|
func cscTraceDebianUpdate(repo *config.Repo) { |
|
|
|
|
target := filepath.Join(buildDownloadDir(repo), "project/trace", config.Conf.Hostname) |
|
|
|
|
|
|
|
|
|
os.MkdirAll(filepath.Join(buildDownloadDir(repo), "project/trace"), 0755) |
|
|
|
|
|
|
|
|
|
f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) |
|
|
|
|
if err != nil { |
|
|
|
|
repo.Logger.Error("Unable to open trace file: " + target) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
if _, err = f.WriteString(time.Now().UTC().Format(time.RFC1123)); err != nil { |
|
|
|
|
repo.Logger.Error("Unable to write to trace file: " + target) |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// some repos want us to check a last modification time file before performing a full sync
|
|
|
|
|
// returns a status to update the original sync's status along with if the sync should continue
|
|
|
|
|
func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { |
|
|
|
|
// default return values are the default status and to continue the sync
|
|
|
|
|
status = config.FAILURE |
|
|
|
|
continueSync = true |
|
|
|
|
if repo.DryRun { |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
runCommand := func(args []string) int { |
|
|
|
|
ch := spawnProcess(repo, args) |
|
|
|
|
if ch == nil { |
|
|
|
|
// spawnSyncProcess will have already logged error
|
|
|
|
|
return 1 |
|
|
|
|
} |
|
|
|
|
cmd := <-ch |
|
|
|
|
return cmd.ProcessState.ExitCode() |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// create a temp file
|
|
|
|
|
temp, err := os.CreateTemp("/tmp", ".tmp.merlin."+repo.Name+"-") |
|
|
|
|
temp, err := os.CreateTemp("/tmp", "merlin-"+repo.Name+"-trace-*") |
|
|
|
|
if err != nil { |
|
|
|
|
repo.Logger.Error(err.Error()) |
|
|
|
|
return |
|
|
|
|
@ -104,7 +135,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { |
|
|
|
|
temp.Close() |
|
|
|
|
defer os.Remove(temp.Name()) |
|
|
|
|
|
|
|
|
|
// get the trace command
|
|
|
|
|
// get the trace command to copy a last updated file
|
|
|
|
|
var args []string |
|
|
|
|
switch repo.SyncType { |
|
|
|
|
case "csc-sync-archlinux": |
|
|
|
|
@ -116,16 +147,10 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
ch := spawnProcess(repo, args) |
|
|
|
|
if ch == nil { |
|
|
|
|
// spawnSyncProcess will have already logged error
|
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
cmd := <-ch |
|
|
|
|
if cmd.ProcessState.ExitCode() != 0 { |
|
|
|
|
exit := runCommand(args) |
|
|
|
|
if exit != 0 { |
|
|
|
|
// if the process was terminated then don't continue to sync
|
|
|
|
|
if cmd.ProcessState.ExitCode() == -1 { |
|
|
|
|
if exit == -1 { |
|
|
|
|
return config.TERMINATED, false |
|
|
|
|
} |
|
|
|
|
return |
|
|
|
|
@ -138,37 +163,41 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { |
|
|
|
|
case "csc-sync-debian": |
|
|
|
|
filesDiffer = cscTraceDebianDiff(repo, temp.Name()) |
|
|
|
|
} |
|
|
|
|
// debian still syncs even if the trace file is the same
|
|
|
|
|
if !filesDiffer && repo.SyncType == "csc-sync-debian" { |
|
|
|
|
repo.Logger.Error("trace file for " + repo.RsyncHost + " unchanged") |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
if filesDiffer { |
|
|
|
|
// continue sync if files differ
|
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// if the files are the same then don't continue to sync
|
|
|
|
|
if !filesDiffer { |
|
|
|
|
repo.Logger.Debug("No changes found; no sync made") |
|
|
|
|
repo.Logger.Debug("No changes found; full sync will not be made") |
|
|
|
|
|
|
|
|
|
// archlinux wants to sync a "lastsync" file even if lastupdate prevents sync
|
|
|
|
|
var args []string |
|
|
|
|
switch repo.SyncType { |
|
|
|
|
case "csc-sync-archlinux": |
|
|
|
|
args = cscTraceArchLinuxUpdate(repo) |
|
|
|
|
default: |
|
|
|
|
return config.SUCCESS, false |
|
|
|
|
} |
|
|
|
|
// archlinux wants to sync a "lastsync" file if lastupdate prevents sync
|
|
|
|
|
switch repo.SyncType { |
|
|
|
|
case "csc-sync-archlinux": |
|
|
|
|
args = cscTraceArchLinuxUpdate(repo) |
|
|
|
|
default: |
|
|
|
|
return config.SUCCESS, false |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
ch := spawnProcess(repo, args) |
|
|
|
|
if ch == nil { |
|
|
|
|
// spawnSyncProcess will have already logged error
|
|
|
|
|
return |
|
|
|
|
exit = runCommand(args) |
|
|
|
|
if exit != 0 { |
|
|
|
|
// if the process was terminated then don't continue to sync
|
|
|
|
|
if exit == -1 { |
|
|
|
|
return config.TERMINATED, false |
|
|
|
|
} |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
cmd := <-ch |
|
|
|
|
if cmd.ProcessState.ExitCode() != 0 { |
|
|
|
|
// if the process was terminated then don't continue to sync
|
|
|
|
|
if cmd.ProcessState.ExitCode() == -1 { |
|
|
|
|
return config.TERMINATED, false |
|
|
|
|
} |
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
return config.SUCCESS, false |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return config.SUCCESS, false |
|
|
|
|
// csc-sync-debian wants to update a trace file after the repo is done syncing
|
|
|
|
|
func postSyncTraceUpdate(repo *config.Repo) { |
|
|
|
|
if repo.SyncType == "csc-sync-debian" { |
|
|
|
|
cscTraceDebianUpdate(repo) |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
return |
|
|
|
|
} |
|
|
|
|
|
Andrew Wang
8 months ago