Andrew Wang
8 months ago
parent
b3d4abd885
commit
8675e13d42
@ -0,0 +1,94 @@ |
||||
package sync |
||||
|
||||
import ( |
||||
"io" |
||||
"os" |
||||
"os/exec" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"git.csclub.uwaterloo.ca/public/merlin/config" |
||||
) |
||||
|
||||
func postRepoSync(repo *config.Repo, exit int) { |
||||
if repo.DryRun { |
||||
repo.Logger.Debug("post sync not run because in dry run mode") |
||||
return |
||||
} |
||||
|
||||
switch exit { |
||||
case config.SUCCESS: |
||||
go zfsSync(repo) |
||||
go postSyncTraceUpdate(repo) |
||||
return |
||||
|
||||
case config.FAILURE: |
||||
go backupRsyncFailLog(repo) |
||||
return |
||||
} |
||||
} |
||||
|
||||
func zfsSync(repo *config.Repo) { |
||||
// we are not using zfs snapshots at the moment
|
||||
repo.Logger.Debug("Would run a zfssync if not disabled") |
||||
return |
||||
|
||||
out, err := exec.Command("/bin/sh", "/home/mirror/bin/zfssync", repo.Name).CombinedOutput() |
||||
if err != nil { |
||||
repo.Logger.Error(err) |
||||
} else { |
||||
f, err := os.OpenFile(repo.ZfssyncLogFile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) |
||||
if err != nil { |
||||
repo.Logger.Error(err.Error()) |
||||
} else { |
||||
f.Write(out) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// updates a trace file after the repo has finished syncing
|
||||
func postSyncTraceUpdate(repo *config.Repo) { |
||||
switch repo.SyncType { |
||||
case "csc-sync-debian": |
||||
cscPostDebian(repo) |
||||
} |
||||
} |
||||
|
||||
// update our trace file's modification date by writing the current time
|
||||
func cscPostDebian(repo *config.Repo) { |
||||
targetDir := filepath.Join(buildDownloadDir(repo), "project/trace") |
||||
target := filepath.Join(targetDir, config.Conf.Hostname) |
||||
|
||||
os.MkdirAll(targetDir, 0755) |
||||
f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) |
||||
|
||||
if err != nil { |
||||
repo.Logger.Error("Unable to open trace file: " + target) |
||||
return |
||||
} |
||||
if _, err = f.WriteString(time.Now().UTC().Format(time.RFC1123)); err != nil { |
||||
repo.Logger.Error("Unable to write to trace file: " + target) |
||||
return |
||||
} |
||||
} |
||||
|
||||
func backupRsyncFailLog(repo *config.Repo) { |
||||
src := repo.RsyncLogFile |
||||
dest := repo.RsyncLogFile + ".fail" |
||||
|
||||
fin, err := os.Open(src) |
||||
if err != nil { |
||||
repo.Logger.Error(err.Error()) |
||||
} |
||||
defer fin.Close() |
||||
|
||||
fout, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) |
||||
if err != nil { |
||||
repo.Logger.Error(err.Error()) |
||||
} |
||||
defer fout.Close() |
||||
|
||||
if _, err = io.Copy(fout, fin); err != nil { |
||||
repo.Logger.Error(err.Error()) |
||||
} |
||||
} |
||||
@ -0,0 +1,165 @@ |
||||
package sync |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
|
||||
"git.csclub.uwaterloo.ca/public/merlin/config" |
||||
) |
||||
|
||||
// jobs to complete before repo sync command should be run
|
||||
// when done is false: status can be ignored and sync command should continue
|
||||
// when done is true: sync should exit with status
|
||||
func preRepoSync(repo *config.Repo) (status int, done bool) { |
||||
status = config.SUCCESS |
||||
done = false |
||||
|
||||
// clear the rsync log file
|
||||
if repo.RsyncLogFile != "" { |
||||
err := os.Truncate(repo.RsyncLogFile, 0) |
||||
if err != nil { |
||||
status = config.FAILURE |
||||
repo.Logger.Error("Error while trying to clear logfile: " + repo.RepoLogFile) |
||||
} |
||||
} |
||||
|
||||
if repo.TraceHost != "" { |
||||
if status, done = checkIfRepoSyncNeeded(repo); done { |
||||
|
||||
// run alternative jobs if a full sync will not be done
|
||||
if exit := noRepoSyncNeeded(repo); exit != config.SUCCESS { |
||||
status = exit |
||||
} |
||||
return |
||||
} |
||||
} |
||||
|
||||
return |
||||
} |
||||
|
||||
// check if the repo should be synced by comparing their trace file with our own
|
||||
// returns the exit status of the command used to retrieve the repo's trace file
|
||||
// returns true for done if and only if no errors occur and it is confirmed that
|
||||
// the trace files are the same or that a termination signal was given
|
||||
// (csc-sync-debian will always sync so done in its case will always be false)
|
||||
func checkIfRepoSyncNeeded(repo *config.Repo) (status int, done bool) { |
||||
status = config.FAILURE |
||||
done = false |
||||
|
||||
if repo.DryRun { |
||||
repo.Logger.Debug("dry running so assuming that trace file was changed") |
||||
return config.SUCCESS, false |
||||
} |
||||
|
||||
repo.Logger.Debug("Retrieving the repo's trace file") |
||||
|
||||
temp, err := os.CreateTemp("/tmp", "merlin-"+repo.Name+"-trace-*") |
||||
if err != nil { |
||||
repo.Logger.Error(err.Error()) |
||||
return |
||||
} |
||||
temp.Close() |
||||
defer os.Remove(temp.Name()) |
||||
|
||||
// get the trace command to copy the last update file
|
||||
var cmd []string |
||||
switch repo.SyncType { |
||||
case "csc-sync-archlinux": |
||||
cmd = cscTraceArchLinux(repo, temp.Name()) |
||||
case "csc-sync-debian": |
||||
cmd = cscTraceDebian(repo, temp.Name()) |
||||
default: |
||||
repo.Logger.Error("Trace files are not implemented for sync type '" + repo.SyncType + "'") |
||||
return |
||||
} |
||||
|
||||
status = spawnProcessAndWait(repo, cmd) |
||||
if status != config.SUCCESS { |
||||
if status == config.TERMINATED { |
||||
done = true |
||||
} |
||||
return |
||||
} |
||||
|
||||
repo.Logger.Debug("Comparing the repo's trace file with our own") |
||||
|
||||
// diff returns true if files are the same (if files are the same then sync is done)
|
||||
switch repo.SyncType { |
||||
case "csc-sync-archlinux": |
||||
done = cscTraceArchLinuxDiff(repo, temp.Name()) |
||||
case "csc-sync-debian": |
||||
done = cscTraceDebianDiff(repo, temp.Name()) |
||||
} |
||||
|
||||
if done { |
||||
repo.Logger.Info("trace file for " + repo.RsyncHost + " unchanged") |
||||
|
||||
// debian syncs even when trace file is unchanged
|
||||
if repo.SyncType == "csc-sync-debian" { |
||||
done = false |
||||
} |
||||
return |
||||
} |
||||
|
||||
repo.Logger.Debug("trace file changes found; will attempt to sync") |
||||
return |
||||
} |
||||
|
||||
// jobs to do when a full sync will not be run
|
||||
func noRepoSyncNeeded(repo *config.Repo) (status int) { |
||||
status = config.SUCCESS |
||||
|
||||
var args []string |
||||
switch repo.SyncType { |
||||
case "csc-sync-archlinux": |
||||
args = cscTraceArchLinuxUpdate(repo) |
||||
} |
||||
|
||||
status = spawnProcessAndWait(repo, args) |
||||
return |
||||
} |
||||
|
||||
func cscTraceArchLinux(repo *config.Repo, newTime string) []string { |
||||
args := []string{ |
||||
"curl", |
||||
"--interface", config.Conf.IPv4Address, |
||||
"-s", repo.TraceHost, |
||||
"-o", newTime, |
||||
} |
||||
|
||||
return args |
||||
} |
||||
|
||||
func cscTraceDebian(repo *config.Repo, newTime string) []string { |
||||
args := []string{ |
||||
"nice", "rsync", "-tv", "--quiet", |
||||
"-4", "--address=" + config.Conf.IPv4Address, |
||||
repo.RsyncHost + "::" + filepath.Join(repo.RsyncDir, "project/trace", repo.TraceHost), |
||||
newTime, |
||||
} |
||||
|
||||
return args |
||||
} |
||||
|
||||
func cscTraceArchLinuxDiff(repo *config.Repo, newTime string) bool { |
||||
return diffFileContent(repo, newTime, filepath.Join(buildDownloadDir(repo), "lastupdate")) |
||||
} |
||||
|
||||
func cscTraceDebianDiff(repo *config.Repo, newTime string) bool { |
||||
return diffFileTime(repo, newTime, filepath.Join(buildDownloadDir(repo), "project/trace", repo.TraceHost)) |
||||
} |
||||
|
||||
func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) { |
||||
rsyncDir := repo.RsyncDir |
||||
localDir := repo.LocalDir |
||||
|
||||
repo.RsyncDir = repo.RsyncDir + "/lastsync" |
||||
repo.LocalDir = repo.LocalDir + "/lastsync" |
||||
|
||||
args = cscSyncArchLinux(repo) |
||||
|
||||
repo.RsyncDir = rsyncDir |
||||
repo.LocalDir = localDir |
||||
|
||||
return args |
||||
} |
||||
@ -1,88 +0,0 @@ |
||||
package sync |
||||
|
||||
import ( |
||||
"path/filepath" |
||||
"reflect" |
||||
"testing" |
||||
|
||||
"git.csclub.uwaterloo.ca/public/merlin/config" |
||||
"git.csclub.uwaterloo.ca/public/merlin/logger" |
||||
) |
||||
|
||||
func dummyRepoConf(name string, syncType string, frequencyStr string, localDir string, rsyncHost string, rsyncDir string) *config.Repo { |
||||
doneChan := make(chan config.SyncResult) |
||||
stopChan := make(chan struct{}) |
||||
|
||||
repoLogFile := filepath.Join("test_files", name, name+".log") |
||||
logger := logger.NewLogger(name, repoLogFile, false) |
||||
|
||||
return &config.Repo{ |
||||
Name: name, |
||||
SyncType: syncType, |
||||
FrequencyStr: frequencyStr, |
||||
Frequency: 0, |
||||
MaxTime: config.DEFAULT_MAX_TIME, |
||||
LocalDir: localDir, |
||||
RsyncHost: rsyncHost, |
||||
RsyncDir: rsyncDir, |
||||
RsyncUser: "", |
||||
PasswordFile: "", |
||||
StateFile: "", |
||||
RepoLogFile: repoLogFile, |
||||
Logger: logger, |
||||
RsyncLogFile: "/tmp/log/" + name + "-rsync.log", |
||||
ZfssyncLogFile: "", |
||||
DoneChan: doneChan, |
||||
StopChan: stopChan, |
||||
State: &config.RepoState{ |
||||
IsRunning: false, |
||||
LastAttemptStartTime: 0, |
||||
LastAttemptRunTime: 0, |
||||
LastAttemptExit: config.NOT_RUN_YET, |
||||
}, |
||||
} |
||||
} |
||||
|
||||
func TestGetSyncCommand(t *testing.T) { |
||||
|
||||
config.Conf.DownloadDir = "test_files" |
||||
config.Conf.IPv4Address = "0.0.0.0" |
||||
config.Conf.DownloadDir = "/tmp/mirror" |
||||
|
||||
testData := []struct { |
||||
repoConf *config.Repo |
||||
expected []string |
||||
}{ |
||||
{ |
||||
repoConf: dummyRepoConf("ubuntu-releases", "csc-sync-standard", "bi-hourly", "ubuntu-releases", "rsync.releases.ubuntu.com", "releases"), |
||||
expected: []string{ |
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", |
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=0.0.0.0", |
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=/tmp/log/ubuntu-releases-rsync.log", |
||||
"rsync://rsync.releases.ubuntu.com/releases", "/tmp/mirror/ubuntu-releases", |
||||
}, |
||||
}, |
||||
{ |
||||
repoConf: dummyRepoConf("raspberrypi", "csc-sync-standard-ipv6", "bi-hourly", "raspberrypi", "apt-repo.raspberrypi.org", "archive"), |
||||
expected: []string{ |
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", |
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=0.0.0.0", |
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=/tmp/log/raspberrypi-rsync.log", |
||||
"apt-repo.raspberrypi.org::archive", "/tmp/mirror/raspberrypi", |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
for _, test := range testData { |
||||
|
||||
syncCommand := getSyncCommand(test.repoConf) |
||||
|
||||
// check for correct command output
|
||||
if !reflect.DeepEqual(syncCommand, test.expected) { |
||||
t.Errorf("Invalid command string for %s repo\nRECIEVED:\n%+v\nEXPECTED:\n%+v\n", test.repoConf.Name, syncCommand, test.expected) |
||||
} |
||||
|
||||
// check if download dir was created
|
||||
|
||||
} |
||||
} |
||||
@ -1,203 +0,0 @@ |
||||
package sync |
||||
|
||||
import ( |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
"time" |
||||
|
||||
"git.csclub.uwaterloo.ca/public/merlin/config" |
||||
) |
||||
|
||||
func cscTraceArchLinux(repo *config.Repo, newTime string) []string { |
||||
args := []string{ |
||||
"curl", |
||||
"--interface", config.Conf.IPv4Address, |
||||
"-s", repo.TraceHost, |
||||
"-o", newTime, |
||||
} |
||||
|
||||
return args |
||||
} |
||||
|
||||
// return false iff both files exist, are non-empty, and have the same contents
|
||||
func cscTraceArchLinuxDiff(repo *config.Repo, newTime string) bool { |
||||
readFile := func(file string) string { |
||||
f, err := os.ReadFile(file) |
||||
if err != nil { |
||||
repo.Logger.Debug(err.Error()) |
||||
return "" |
||||
} |
||||
return strings.TrimSpace(string(f)) |
||||
} |
||||
|
||||
file1 := readFile(newTime) |
||||
file2 := readFile(filepath.Join(buildDownloadDir(repo), "lastupdate")) |
||||
|
||||
if file1 == "" || file2 == "" { |
||||
return true |
||||
} |
||||
return file1 != file2 |
||||
} |
||||
|
||||
// update the lastsync file
|
||||
func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) { |
||||
rsyncDir := repo.RsyncDir |
||||
localDir := repo.LocalDir |
||||
|
||||
repo.RsyncDir = repo.RsyncDir + "/lastsync" |
||||
repo.LocalDir = repo.LocalDir + "/lastsync" |
||||
|
||||
args = cscSyncArchLinux(repo) |
||||
|
||||
repo.RsyncDir = rsyncDir |
||||
repo.LocalDir = localDir |
||||
|
||||
return args |
||||
} |
||||
|
||||
func cscTraceDebian(repo *config.Repo, newTime string) []string { |
||||
args := []string{ |
||||
"nice", "rsync", "-tv", "-4", |
||||
"--address=" + config.Conf.IPv4Address, |
||||
"--quiet", |
||||
repo.RsyncHost + "::" + filepath.Join(repo.RsyncDir, "project/trace", repo.TraceHost), |
||||
newTime, |
||||
} |
||||
|
||||
return args |
||||
} |
||||
|
||||
// return false iff both files exist and were modified at the same time
|
||||
func cscTraceDebianDiff(repo *config.Repo, newTime string) bool { |
||||
statFile := func(file string) int64 { |
||||
f, err := os.Stat(file) |
||||
if err != nil { |
||||
repo.Logger.Debug("Error while trying to stat file: " + file) |
||||
return 0 |
||||
} |
||||
|
||||
return f.ModTime().Unix() |
||||
} |
||||
|
||||
file1 := statFile(newTime) |
||||
file2 := statFile(filepath.Join(buildDownloadDir(repo), "project/trace", repo.TraceHost)) |
||||
|
||||
if file1 == 0 || file2 == 0 { |
||||
return true |
||||
} |
||||
|
||||
return file1 != file2 |
||||
} |
||||
|
||||
// update our trace file's modification date by writing the current time
|
||||
func cscTraceDebianUpdate(repo *config.Repo) { |
||||
target := filepath.Join(buildDownloadDir(repo), "project/trace", config.Conf.Hostname) |
||||
|
||||
os.MkdirAll(filepath.Join(buildDownloadDir(repo), "project/trace"), 0755) |
||||
|
||||
f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) |
||||
if err != nil { |
||||
repo.Logger.Error("Unable to open trace file: " + target) |
||||
return |
||||
} |
||||
if _, err = f.WriteString(time.Now().UTC().Format(time.RFC1123)); err != nil { |
||||
repo.Logger.Error("Unable to write to trace file: " + target) |
||||
return |
||||
} |
||||
} |
||||
|
||||
// some repos want us to check a last modification time file before performing a full sync
|
||||
// returns a status to update the original sync's status along with if the sync should continue
|
||||
func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { |
||||
// default return values are the default status and to continue the sync
|
||||
status = config.FAILURE |
||||
continueSync = true |
||||
if repo.DryRun { |
||||
return |
||||
} |
||||
runCommand := func(args []string) int { |
||||
ch := spawnProcess(repo, args) |
||||
if ch == nil { |
||||
// spawnSyncProcess will have already logged error
|
||||
return 1 |
||||
} |
||||
cmd := <-ch |
||||
return cmd.ProcessState.ExitCode() |
||||
} |
||||
|
||||
// create a temp file
|
||||
temp, err := os.CreateTemp("/tmp", "merlin-"+repo.Name+"-trace-*") |
||||
if err != nil { |
||||
repo.Logger.Error(err.Error()) |
||||
return |
||||
} |
||||
temp.Close() |
||||
defer os.Remove(temp.Name()) |
||||
|
||||
// get the trace command to copy a last updated file
|
||||
var args []string |
||||
switch repo.SyncType { |
||||
case "csc-sync-archlinux": |
||||
args = cscTraceArchLinux(repo, temp.Name()) |
||||
case "csc-sync-debian": |
||||
args = cscTraceDebian(repo, temp.Name()) |
||||
default: |
||||
repo.Logger.Error("Trace files are not implemented for sync type '" + repo.SyncType + "'") |
||||
return |
||||
} |
||||
|
||||
exit := runCommand(args) |
||||
if exit != 0 { |
||||
// if the process was terminated then don't continue to sync
|
||||
if exit == -1 { |
||||
return config.TERMINATED, false |
||||
} |
||||
return |
||||
} |
||||
|
||||
filesDiffer := true |
||||
switch repo.SyncType { |
||||
case "csc-sync-archlinux": |
||||
filesDiffer = cscTraceArchLinuxDiff(repo, temp.Name()) |
||||
case "csc-sync-debian": |
||||
filesDiffer = cscTraceDebianDiff(repo, temp.Name()) |
||||
} |
||||
// debian still syncs even if the trace file is the same
|
||||
if !filesDiffer && repo.SyncType == "csc-sync-debian" { |
||||
repo.Logger.Error("trace file for " + repo.RsyncHost + " unchanged") |
||||
return |
||||
} |
||||
if filesDiffer { |
||||
// continue sync if files differ
|
||||
return |
||||
} |
||||
|
||||
repo.Logger.Debug("No changes found; full sync will not be made") |
||||
|
||||
// archlinux wants to sync a "lastsync" file if lastupdate prevents sync
|
||||
switch repo.SyncType { |
||||
case "csc-sync-archlinux": |
||||
args = cscTraceArchLinuxUpdate(repo) |
||||
default: |
||||
return config.SUCCESS, false |
||||
} |
||||
|
||||
exit = runCommand(args) |
||||
if exit != 0 { |
||||
// if the process was terminated then don't continue to sync
|
||||
if exit == -1 { |
||||
return config.TERMINATED, false |
||||
} |
||||
return |
||||
} |
||||
|
||||
return config.SUCCESS, false |
||||
} |
||||
|
||||
// csc-sync-debian wants to update a trace file after the repo is done syncing
|
||||
func postSyncTraceUpdate(repo *config.Repo) { |
||||
if repo.SyncType == "csc-sync-debian" { |
||||
cscTraceDebianUpdate(repo) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue