From 392396d1a0afc5cdf6509766300463a384b92bbf Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Thu, 23 Jun 2022 02:43:42 +0000 Subject: [PATCH] Bump version, add verbosity and dry run options --- merlin/README.md | 8 +++++++- merlin/arthur/arthur_test.go | 2 +- merlin/config/config.go | 12 +++++++----- merlin/go.mod | 6 +++--- merlin/go.sum | 10 ++++------ merlin/logger/logger.go | 18 +++++++++++------- merlin/sync/process.go | 5 +++++ merlin/sync/sync.go | 11 +++-------- merlin/sync/sync_test.go | 2 +- merlin/sync/trace.go | 8 ++++---- 10 files changed, 46 insertions(+), 36 deletions(-) diff --git a/merlin/README.md b/merlin/README.md index 9d205d0..9ec4ef4 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -5,9 +5,15 @@ This folder contains the code for merlin (which does the actual syncing) and art Check out the the [mirror env](https://git.csclub.uwaterloo.ca/public/mirror-env) for a testing environment +### Usage +``` +go build merlin.go +``` +Then configure `merlin-config.ini` and run using `./merlin` + ### Nice Features To Add - detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes) -- set user and group in config +- get the config or the rsync commond that a repo will sync with using a cli tool ### Completed - [x] add bwlimit option for each rsync process diff --git a/merlin/arthur/arthur_test.go b/merlin/arthur/arthur_test.go index e7c2673..27b8cd1 100644 --- a/merlin/arthur/arthur_test.go +++ b/merlin/arthur/arthur_test.go @@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) { SyncType: "csc-sync-dummy", Frequency: 7 * 86400, MaxTime: 30, - Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs"), + Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false), StateFile: "/tmp/merlin_force_sync_test_state", DoneChan: doneChan, State: &config.RepoState{ diff --git a/merlin/config/config.go b/merlin/config/config.go index a06d494..cedcc7e 100644 --- a/merlin/config/config.go +++ b/merlin/config/config.go @@ -91,10 +91,6 @@ type Config struct { ZfssyncLogDir string `ini:"zfssync_logs_dir"` // path to the unix socket for arthur to use for communication SockPath string `ini:"sock_path"` - // // TODO: add dry run (default: false) - // DryRun bool `ini:"dry_run"` - // // TODO: add setting for verbosity (default: false) - // Verbose bool `ini:"verbose"` } type Repo struct { @@ -106,6 +102,8 @@ type Repo struct { FrequencyStr string `ini:"frequency"` // the desired interval (in seconds) between successive runs Frequency int `ini:"-"` + // instead of spawning a process sleep for 50 seconds instead (default: false) + DryRun bool `ini:"dry_run"` // the maximum time (in seconds) that each child process of this repo // can for before being killed MaxTime int `ini:"max_time"` @@ -134,6 +132,8 @@ type Repo struct { RsyncLogFile string `ini:"-"` // full file path for file logging this repo's zfssync ZfssyncLogFile string `ini:"-"` + // add the "-vv" flag to rsync commands and enable the Debug log (default: false) + Verbose bool `ini:"verbose"` // the repo will write its name and status in a Result struct to DoneChan // when it has finished a job (shared by all repos) DoneChan chan<- SyncResult `ini:"-"` @@ -224,6 +224,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc Name: repoName, SyncType: newConf.DefaultSyncType, FrequencyStr: newConf.DefaultFrequencyStr, + DryRun: false, MaxTime: newConf.DefaultMaxTime, MaxRsyncIO: newConf.DefaultMaxRsyncIO, LocalDir: repoName, @@ -231,6 +232,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log", RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log", ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log", + Verbose: false, DoneChan: doneChan, StopChan: stopChan, } @@ -267,7 +269,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc ) // create the logger and load the state - repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile) + repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile, repo.Verbose) repo.State = &RepoState{ IsRunning: false, LastAttemptStartTime: 0, diff --git a/merlin/go.mod b/merlin/go.mod index 256c295..b44648b 100644 --- a/merlin/go.mod +++ b/merlin/go.mod @@ -1,10 +1,10 @@ module git.csclub.uwaterloo.ca/public/merlin require ( - github.com/davecgh/go-spew v1.1.0 + github.com/davecgh/go-spew v1.1.1 github.com/stretchr/testify v1.7.0 // indirect - golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 + golang.org/x/sys v0.0.0-20220209214540-3681064d5158 gopkg.in/ini.v1 v1.63.2 ) -go 1.13 +go 1.16 diff --git a/merlin/go.sum b/merlin/go.sum index 3552215..cc970e4 100644 --- a/merlin/go.sum +++ b/merlin/go.sum @@ -1,15 +1,13 @@ -github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97 h1:Nyfs+rh56aORy2tGMI9GCYEqTfePwL1v47qOzebfv/o= -github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97/go.mod h1:WO1d2m1QDzkoPcgn9lgHVMi7qQR5j3jxYjIIvMTHpC0= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw= -golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/merlin/logger/logger.go b/merlin/logger/logger.go index a6ab73b..6ff8f66 100644 --- a/merlin/logger/logger.go +++ b/merlin/logger/logger.go @@ -7,8 +7,9 @@ import ( type Logger struct { *log.Logger - name string - file string + name string + file string + verbose bool } // DEBUG/WARNING will only be written to file @@ -41,11 +42,12 @@ func ErrLog(v ...interface{}) { } // initialize a logger -func NewLogger(name, file string) *Logger { +func NewLogger(name, file string, verbose bool) *Logger { logger := Logger{ - Logger: log.New(os.Stderr, "", log.LstdFlags), - name: name, - file: file, + Logger: log.New(os.Stderr, "", log.LstdFlags), + name: name, + file: file, + verbose: verbose, } return &logger } @@ -68,7 +70,9 @@ func (logger *Logger) log(level int, v ...interface{}) { // write debug information to the logfile func (logger *Logger) Debug(v ...interface{}) { - logger.log(DEBUG, v...) + if logger.verbose { + logger.log(DEBUG, v...) + } } // write information to the logfile and to stdout diff --git a/merlin/sync/process.go b/merlin/sync/process.go index b598005..7838d15 100644 --- a/merlin/sync/process.go +++ b/merlin/sync/process.go @@ -17,6 +17,10 @@ import ( // or nil if it was unable to start a process. func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) { repo.Logger.Debug(fmt.Sprintf("Running the command: %v", args)) + if repo.DryRun { + repo.Logger.Debug("Dry running for 50 seconds") + args = []string{"sleep", "50"} + } cmd := exec.Command(args[0], args[1:]...) repo.Logger.Debug("Starting process") @@ -127,6 +131,7 @@ func startRepoSync(repo *config.Repo) { func zfsSync(repo *config.Repo) { // we are not using zfs snapshots at the moment + repo.Logger.Debug("Would run a zfssync if not disabled") return out, err := exec.Command("/home/mirror/bin/zfssync", repo.Name).CombinedOutput() diff --git a/merlin/sync/sync.go b/merlin/sync/sync.go index 40b62c3..ed78e3c 100644 --- a/merlin/sync/sync.go +++ b/merlin/sync/sync.go @@ -79,6 +79,9 @@ func addConditionalFlags(repo *config.Repo, flags int) []string { if repo.MaxRsyncIO > 0 { args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO)) } + if repo.Verbose { + args = append(args, "-vv") + } return args } @@ -317,16 +320,8 @@ func cscSyncWget(repo *config.Repo) []string { return args } -func cscSyncDummy(repo *config.Repo) []string { - return []string{"sleep", "10"} -} - // executes a particular sync job depending on repo.SyncType. func getSyncCommand(repo *config.Repo) (args []string) { - if repo.SyncType == "csc-sync-dummy" { - return cscSyncDummy(repo) - } - // check that the download directory exists if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) { repo.Logger.Error(err.Error()) diff --git a/merlin/sync/sync_test.go b/merlin/sync/sync_test.go index 86ae190..ca46741 100644 --- a/merlin/sync/sync_test.go +++ b/merlin/sync/sync_test.go @@ -14,7 +14,7 @@ func dummyRepoConf(name string, syncType string, frequencyStr string, localDir s stopChan := make(chan struct{}) repoLogFile := filepath.Join("test_files", name, name+".log") - logger := logger.NewLogger(name, repoLogFile) + logger := logger.NewLogger(name, repoLogFile, false) return &config.Repo{ Name: name, diff --git a/merlin/sync/trace.go b/merlin/sync/trace.go index e866ebd..8caeba7 100644 --- a/merlin/sync/trace.go +++ b/merlin/sync/trace.go @@ -88,8 +88,10 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool { return file1 != file2 } +// some repos want us to check a last modification time file before performing a full sync +// returns a status to update the original sync's status along with if the sync should continue func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { - // default is to keep the default status and continue the sync + // default return values are the default status and to continue the sync status = config.FAILURE continueSync = true @@ -110,9 +112,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) { case "csc-sync-debian": args = cscTraceDebian(repo, temp.Name()) default: - if repo.SyncType != "csc-sync-dummy" { - repo.Logger.Error("Trace files are not implemented for sync type ", "'"+repo.SyncType+"'") - } + repo.Logger.Error("Trace files are not implemented for sync type '" + repo.SyncType + "'") return }