Bump version, add verbosity and dry run options

merlin/README.md

@@ -5,9 +5,15 @@ This folder contains the code for merlin (which does the actual syncing) and art
 
 Check out the the [mirror env](https://git.csclub.uwaterloo.ca/public/mirror-env) for a testing environment
 
+### Usage
+```
+go build merlin.go
+```
+Then configure `merlin-config.ini` and run using `./merlin`
+
 ### Nice Features To Add
 - detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes)
-- set user and group in config
+- get the config or the rsync commond that a repo will sync with using a cli tool
 
 ### Completed
 - [x] add bwlimit option for each rsync process

merlin/arthur/arthur_test.go

@@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
 		SyncType:  "csc-sync-dummy",
 		Frequency: 7 * 86400,
 		MaxTime:   30,
-		Logger:    logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs"),
+		Logger:    logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
 		StateFile: "/tmp/merlin_force_sync_test_state",
 		DoneChan:  doneChan,
 		State: &config.RepoState{

merlin/config/config.go

@@ -91,10 +91,6 @@ type Config struct {
 	ZfssyncLogDir string `ini:"zfssync_logs_dir"`
 	// path to the unix socket for arthur to use for communication
 	SockPath string `ini:"sock_path"`
-	// // TODO: add dry run (default: false)
-	// DryRun bool `ini:"dry_run"`
-	// // TODO: add setting for verbosity (default: false)
-	// Verbose bool `ini:"verbose"`
 }
 
 type Repo struct {
@@ -106,6 +102,8 @@ type Repo struct {
 	FrequencyStr string `ini:"frequency"`
 	// the desired interval (in seconds) between successive runs
 	Frequency int `ini:"-"`
+	// instead of spawning a process sleep for 50 seconds instead (default: false)
+	DryRun bool `ini:"dry_run"`
 	// the maximum time (in seconds) that each child process of this repo
 	// can for before being killed
 	MaxTime int `ini:"max_time"`
@@ -134,6 +132,8 @@ type Repo struct {
 	RsyncLogFile string `ini:"-"`
 	// full file path for file logging this repo's zfssync
 	ZfssyncLogFile string `ini:"-"`
+	// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
+	Verbose bool `ini:"verbose"`
 	// the repo will write its name and status in a Result struct to DoneChan
 	// when it has finished a job (shared by all repos)
 	DoneChan chan<- SyncResult `ini:"-"`
@@ -224,6 +224,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
 			Name:           repoName,
 			SyncType:       newConf.DefaultSyncType,
 			FrequencyStr:   newConf.DefaultFrequencyStr,
+			DryRun:         false,
 			MaxTime:        newConf.DefaultMaxTime,
 			MaxRsyncIO:     newConf.DefaultMaxRsyncIO,
 			LocalDir:       repoName,
@@ -231,6 +232,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
 			RepoLogFile:    filepath.Join(newConf.RepoLogDir, repoName) + ".log",
 			RsyncLogFile:   filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
 			ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
+			Verbose:        false,
 			DoneChan:       doneChan,
 			StopChan:       stopChan,
 		}
@@ -267,7 +269,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
 		)
 
 		// create the logger and load the state
-		repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile)
+		repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile, repo.Verbose)
 		repo.State = &RepoState{
 			IsRunning:            false,
 			LastAttemptStartTime: 0,

merlin/go.mod

@@ -1,10 +1,10 @@
 module git.csclub.uwaterloo.ca/public/merlin
 
 require (
-	github.com/davecgh/go-spew v1.1.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/stretchr/testify v1.7.0 // indirect
-	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158
 	gopkg.in/ini.v1 v1.63.2
 )
 
-go 1.13
+go 1.16

merlin/go.sum

@@ -1,15 +1,13 @@
-github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97 h1:Nyfs+rh56aORy2tGMI9GCYEqTfePwL1v47qOzebfv/o=
-github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97/go.mod h1:WO1d2m1QDzkoPcgn9lgHVMi7qQR5j3jxYjIIvMTHpC0=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
-golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
+golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
 gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

merlin/logger/logger.go

@@ -7,8 +7,9 @@ import (
 
 type Logger struct {
 	*log.Logger
-	name string
-	file string
+	name    string
+	file    string
+	verbose bool
 }
 
 // DEBUG/WARNING will only be written to file
@@ -41,11 +42,12 @@ func ErrLog(v ...interface{}) {
 }
 
 // initialize a logger
-func NewLogger(name, file string) *Logger {
+func NewLogger(name, file string, verbose bool) *Logger {
 	logger := Logger{
-		Logger: log.New(os.Stderr, "", log.LstdFlags),
-		name:   name,
-		file:   file,
+		Logger:  log.New(os.Stderr, "", log.LstdFlags),
+		name:    name,
+		file:    file,
+		verbose: verbose,
 	}
 	return &logger
 }
@@ -68,7 +70,9 @@ func (logger *Logger) log(level int, v ...interface{}) {
 
 // write debug information to the logfile
 func (logger *Logger) Debug(v ...interface{}) {
-	logger.log(DEBUG, v...)
+	if logger.verbose {
+		logger.log(DEBUG, v...)
+	}
 }
 
 // write information to the logfile and to stdout

merlin/sync/process.go

@@ -17,6 +17,10 @@ import (
 // or nil if it was unable to start a process.
 func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) {
 	repo.Logger.Debug(fmt.Sprintf("Running the command: %v", args))
+	if repo.DryRun {
+		repo.Logger.Debug("Dry running for 50 seconds")
+		args = []string{"sleep", "50"}
+	}
 	cmd := exec.Command(args[0], args[1:]...)
 	repo.Logger.Debug("Starting process")
 
@@ -127,6 +131,7 @@ func startRepoSync(repo *config.Repo) {
 
 func zfsSync(repo *config.Repo) {
 	// we are not using zfs snapshots at the moment
+	repo.Logger.Debug("Would run a zfssync if not disabled")
 	return
 
 	out, err := exec.Command("/home/mirror/bin/zfssync", repo.Name).CombinedOutput()

merlin/sync/sync.go

@@ -79,6 +79,9 @@ func addConditionalFlags(repo *config.Repo, flags int) []string {
 	if repo.MaxRsyncIO > 0 {
 		args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO))
 	}
+	if repo.Verbose {
+		args = append(args, "-vv")
+	}
 
 	return args
 }
@@ -317,16 +320,8 @@ func cscSyncWget(repo *config.Repo) []string {
 	return args
 }
 
-func cscSyncDummy(repo *config.Repo) []string {
-	return []string{"sleep", "10"}
-}
-
 // executes a particular sync job depending on repo.SyncType.
 func getSyncCommand(repo *config.Repo) (args []string) {
-	if repo.SyncType == "csc-sync-dummy" {
-		return cscSyncDummy(repo)
-	}
-
 	// check that the download directory exists
 	if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) {
 		repo.Logger.Error(err.Error())

merlin/sync/sync_test.go

@@ -14,7 +14,7 @@ func dummyRepoConf(name string, syncType string, frequencyStr string, localDir s
 	stopChan := make(chan struct{})
 
 	repoLogFile := filepath.Join("test_files", name, name+".log")
-	logger := logger.NewLogger(name, repoLogFile)
+	logger := logger.NewLogger(name, repoLogFile, false)
 
 	return &config.Repo{
 		Name:           name,

merlin/sync/trace.go

@@ -88,8 +88,10 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool {
 	return file1 != file2
 }
 
+// some repos want us to check a last modification time file before performing a full sync
+// returns a status to update the original sync's status along with if the sync should continue
 func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
-	// default is to keep the default status and continue the sync
+	// default return values are the default status and to continue the sync
 	status = config.FAILURE
 	continueSync = true
 
@@ -110,9 +112,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
 	case "csc-sync-debian":
 		args = cscTraceDebian(repo, temp.Name())
 	default:
-		if repo.SyncType != "csc-sync-dummy" {
-			repo.Logger.Error("Trace files are not implemented for sync type ", "'"+repo.SyncType+"'")
-		}
+		repo.Logger.Error("Trace files are not implemented for sync type '" + repo.SyncType + "'")
 		return
 	}