Bump version, add verbosity and dry run options

pull/3/head
Andrew Wang 5 months ago
parent c386fb2578
commit 392396d1a0
  1. 8
      merlin/README.md
  2. 2
      merlin/arthur/arthur_test.go
  3. 12
      merlin/config/config.go
  4. 6
      merlin/go.mod
  5. 10
      merlin/go.sum
  6. 18
      merlin/logger/logger.go
  7. 5
      merlin/sync/process.go
  8. 11
      merlin/sync/sync.go
  9. 2
      merlin/sync/sync_test.go
  10. 8
      merlin/sync/trace.go

@ -5,9 +5,15 @@ This folder contains the code for merlin (which does the actual syncing) and art
Check out the the [mirror env](https://git.csclub.uwaterloo.ca/public/mirror-env) for a testing environment
### Usage
```
go build merlin.go
```
Then configure `merlin-config.ini` and run using `./merlin`
### Nice Features To Add
- detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes)
- set user and group in config
- get the config or the rsync commond that a repo will sync with using a cli tool
### Completed
- [x] add bwlimit option for each rsync process

@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
SyncType: "csc-sync-dummy",
Frequency: 7 * 86400,
MaxTime: 30,
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs"),
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
StateFile: "/tmp/merlin_force_sync_test_state",
DoneChan: doneChan,
State: &config.RepoState{

@ -91,10 +91,6 @@ type Config struct {
ZfssyncLogDir string `ini:"zfssync_logs_dir"`
// path to the unix socket for arthur to use for communication
SockPath string `ini:"sock_path"`
// // TODO: add dry run (default: false)
// DryRun bool `ini:"dry_run"`
// // TODO: add setting for verbosity (default: false)
// Verbose bool `ini:"verbose"`
}
type Repo struct {
@ -106,6 +102,8 @@ type Repo struct {
FrequencyStr string `ini:"frequency"`
// the desired interval (in seconds) between successive runs
Frequency int `ini:"-"`
// instead of spawning a process sleep for 50 seconds instead (default: false)
DryRun bool `ini:"dry_run"`
// the maximum time (in seconds) that each child process of this repo
// can for before being killed
MaxTime int `ini:"max_time"`
@ -134,6 +132,8 @@ type Repo struct {
RsyncLogFile string `ini:"-"`
// full file path for file logging this repo's zfssync
ZfssyncLogFile string `ini:"-"`
// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
Verbose bool `ini:"verbose"`
// the repo will write its name and status in a Result struct to DoneChan
// when it has finished a job (shared by all repos)
DoneChan chan<- SyncResult `ini:"-"`
@ -224,6 +224,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
Name: repoName,
SyncType: newConf.DefaultSyncType,
FrequencyStr: newConf.DefaultFrequencyStr,
DryRun: false,
MaxTime: newConf.DefaultMaxTime,
MaxRsyncIO: newConf.DefaultMaxRsyncIO,
LocalDir: repoName,
@ -231,6 +232,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log",
RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
Verbose: false,
DoneChan: doneChan,
StopChan: stopChan,
}
@ -267,7 +269,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
)
// create the logger and load the state
repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile)
repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile, repo.Verbose)
repo.State = &RepoState{
IsRunning: false,
LastAttemptStartTime: 0,

@ -1,10 +1,10 @@
module git.csclub.uwaterloo.ca/public/merlin
require (
github.com/davecgh/go-spew v1.1.0
github.com/davecgh/go-spew v1.1.1
github.com/stretchr/testify v1.7.0 // indirect
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
golang.org/x/sys v0.0.0-20220209214540-3681064d5158
gopkg.in/ini.v1 v1.63.2
)
go 1.13
go 1.16

@ -1,15 +1,13 @@
github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97 h1:Nyfs+rh56aORy2tGMI9GCYEqTfePwL1v47qOzebfv/o=
github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97/go.mod h1:WO1d2m1QDzkoPcgn9lgHVMi7qQR5j3jxYjIIvMTHpC0=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=

@ -7,8 +7,9 @@ import (
type Logger struct {
*log.Logger
name string
file string
name string
file string
verbose bool
}
// DEBUG/WARNING will only be written to file
@ -41,11 +42,12 @@ func ErrLog(v ...interface{}) {
}
// initialize a logger
func NewLogger(name, file string) *Logger {
func NewLogger(name, file string, verbose bool) *Logger {
logger := Logger{
Logger: log.New(os.Stderr, "", log.LstdFlags),
name: name,
file: file,
Logger: log.New(os.Stderr, "", log.LstdFlags),
name: name,
file: file,
verbose: verbose,
}
return &logger
}
@ -68,7 +70,9 @@ func (logger *Logger) log(level int, v ...interface{}) {
// write debug information to the logfile
func (logger *Logger) Debug(v ...interface{}) {
logger.log(DEBUG, v...)
if logger.verbose {
logger.log(DEBUG, v...)
}
}
// write information to the logfile and to stdout

@ -17,6 +17,10 @@ import (
// or nil if it was unable to start a process.
func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) {
repo.Logger.Debug(fmt.Sprintf("Running the command: %v", args))
if repo.DryRun {
repo.Logger.Debug("Dry running for 50 seconds")
args = []string{"sleep", "50"}
}
cmd := exec.Command(args[0], args[1:]...)
repo.Logger.Debug("Starting process")
@ -127,6 +131,7 @@ func startRepoSync(repo *config.Repo) {
func zfsSync(repo *config.Repo) {
// we are not using zfs snapshots at the moment
repo.Logger.Debug("Would run a zfssync if not disabled")
return
out, err := exec.Command("/home/mirror/bin/zfssync", repo.Name).CombinedOutput()

@ -79,6 +79,9 @@ func addConditionalFlags(repo *config.Repo, flags int) []string {
if repo.MaxRsyncIO > 0 {
args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO))
}
if repo.Verbose {
args = append(args, "-vv")
}
return args
}
@ -317,16 +320,8 @@ func cscSyncWget(repo *config.Repo) []string {
return args
}
func cscSyncDummy(repo *config.Repo) []string {
return []string{"sleep", "10"}
}
// executes a particular sync job depending on repo.SyncType.
func getSyncCommand(repo *config.Repo) (args []string) {
if repo.SyncType == "csc-sync-dummy" {
return cscSyncDummy(repo)
}
// check that the download directory exists
if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) {
repo.Logger.Error(err.Error())

@ -14,7 +14,7 @@ func dummyRepoConf(name string, syncType string, frequencyStr string, localDir s
stopChan := make(chan struct{})
repoLogFile := filepath.Join("test_files", name, name+".log")
logger := logger.NewLogger(name, repoLogFile)
logger := logger.NewLogger(name, repoLogFile, false)
return &config.Repo{
Name: name,

@ -88,8 +88,10 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool {
return file1 != file2
}
// some repos want us to check a last modification time file before performing a full sync
// returns a status to update the original sync's status along with if the sync should continue
func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
// default is to keep the default status and continue the sync
// default return values are the default status and to continue the sync
status = config.FAILURE
continueSync = true
@ -110,9 +112,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
case "csc-sync-debian":
args = cscTraceDebian(repo, temp.Name())
default:
if repo.SyncType != "csc-sync-dummy" {
repo.Logger.Error("Trace files are not implemented for sync type ", "'"+repo.SyncType+"'")
}
repo.Logger.Error("Trace files are not implemented for sync type '" + repo.SyncType + "'")
return
}

Loading…
Cancel
Save