Bump version, add verbosity and dry run options
This commit is contained in:
parent
c386fb2578
commit
392396d1a0
|
@ -5,9 +5,15 @@ This folder contains the code for merlin (which does the actual syncing) and art
|
|||
|
||||
Check out the the [mirror env](https://git.csclub.uwaterloo.ca/public/mirror-env) for a testing environment
|
||||
|
||||
### Usage
|
||||
```
|
||||
go build merlin.go
|
||||
```
|
||||
Then configure `merlin-config.ini` and run using `./merlin`
|
||||
|
||||
### Nice Features To Add
|
||||
- detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes)
|
||||
- set user and group in config
|
||||
- get the config or the rsync commond that a repo will sync with using a cli tool
|
||||
|
||||
### Completed
|
||||
- [x] add bwlimit option for each rsync process
|
||||
|
|
|
@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
|
|||
SyncType: "csc-sync-dummy",
|
||||
Frequency: 7 * 86400,
|
||||
MaxTime: 30,
|
||||
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs"),
|
||||
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
|
||||
StateFile: "/tmp/merlin_force_sync_test_state",
|
||||
DoneChan: doneChan,
|
||||
State: &config.RepoState{
|
||||
|
|
|
@ -91,10 +91,6 @@ type Config struct {
|
|||
ZfssyncLogDir string `ini:"zfssync_logs_dir"`
|
||||
// path to the unix socket for arthur to use for communication
|
||||
SockPath string `ini:"sock_path"`
|
||||
// // TODO: add dry run (default: false)
|
||||
// DryRun bool `ini:"dry_run"`
|
||||
// // TODO: add setting for verbosity (default: false)
|
||||
// Verbose bool `ini:"verbose"`
|
||||
}
|
||||
|
||||
type Repo struct {
|
||||
|
@ -106,6 +102,8 @@ type Repo struct {
|
|||
FrequencyStr string `ini:"frequency"`
|
||||
// the desired interval (in seconds) between successive runs
|
||||
Frequency int `ini:"-"`
|
||||
// instead of spawning a process sleep for 50 seconds instead (default: false)
|
||||
DryRun bool `ini:"dry_run"`
|
||||
// the maximum time (in seconds) that each child process of this repo
|
||||
// can for before being killed
|
||||
MaxTime int `ini:"max_time"`
|
||||
|
@ -134,6 +132,8 @@ type Repo struct {
|
|||
RsyncLogFile string `ini:"-"`
|
||||
// full file path for file logging this repo's zfssync
|
||||
ZfssyncLogFile string `ini:"-"`
|
||||
// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
|
||||
Verbose bool `ini:"verbose"`
|
||||
// the repo will write its name and status in a Result struct to DoneChan
|
||||
// when it has finished a job (shared by all repos)
|
||||
DoneChan chan<- SyncResult `ini:"-"`
|
||||
|
@ -224,6 +224,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
|
|||
Name: repoName,
|
||||
SyncType: newConf.DefaultSyncType,
|
||||
FrequencyStr: newConf.DefaultFrequencyStr,
|
||||
DryRun: false,
|
||||
MaxTime: newConf.DefaultMaxTime,
|
||||
MaxRsyncIO: newConf.DefaultMaxRsyncIO,
|
||||
LocalDir: repoName,
|
||||
|
@ -231,6 +232,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
|
|||
RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log",
|
||||
RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
|
||||
ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
|
||||
Verbose: false,
|
||||
DoneChan: doneChan,
|
||||
StopChan: stopChan,
|
||||
}
|
||||
|
@ -267,7 +269,7 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
|
|||
)
|
||||
|
||||
// create the logger and load the state
|
||||
repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile)
|
||||
repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile, repo.Verbose)
|
||||
repo.State = &RepoState{
|
||||
IsRunning: false,
|
||||
LastAttemptStartTime: 0,
|
||||
|
|
|
@ -1,10 +1,10 @@
|
|||
module git.csclub.uwaterloo.ca/public/merlin
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/stretchr/testify v1.7.0 // indirect
|
||||
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158
|
||||
gopkg.in/ini.v1 v1.63.2
|
||||
)
|
||||
|
||||
go 1.13
|
||||
go 1.16
|
||||
|
|
|
@ -1,15 +1,13 @@
|
|||
github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97 h1:Nyfs+rh56aORy2tGMI9GCYEqTfePwL1v47qOzebfv/o=
|
||||
github.com/andelf/go-curl v0.0.0-20200630032108-fd49ff24ed97/go.mod h1:WO1d2m1QDzkoPcgn9lgHVMi7qQR5j3jxYjIIvMTHpC0=
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
|
||||
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c=
|
||||
golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c=
|
||||
gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
|
|
|
@ -7,8 +7,9 @@ import (
|
|||
|
||||
type Logger struct {
|
||||
*log.Logger
|
||||
name string
|
||||
file string
|
||||
name string
|
||||
file string
|
||||
verbose bool
|
||||
}
|
||||
|
||||
// DEBUG/WARNING will only be written to file
|
||||
|
@ -41,11 +42,12 @@ func ErrLog(v ...interface{}) {
|
|||
}
|
||||
|
||||
// initialize a logger
|
||||
func NewLogger(name, file string) *Logger {
|
||||
func NewLogger(name, file string, verbose bool) *Logger {
|
||||
logger := Logger{
|
||||
Logger: log.New(os.Stderr, "", log.LstdFlags),
|
||||
name: name,
|
||||
file: file,
|
||||
Logger: log.New(os.Stderr, "", log.LstdFlags),
|
||||
name: name,
|
||||
file: file,
|
||||
verbose: verbose,
|
||||
}
|
||||
return &logger
|
||||
}
|
||||
|
@ -68,7 +70,9 @@ func (logger *Logger) log(level int, v ...interface{}) {
|
|||
|
||||
// write debug information to the logfile
|
||||
func (logger *Logger) Debug(v ...interface{}) {
|
||||
logger.log(DEBUG, v...)
|
||||
if logger.verbose {
|
||||
logger.log(DEBUG, v...)
|
||||
}
|
||||
}
|
||||
|
||||
// write information to the logfile and to stdout
|
||||
|
|
|
@ -17,6 +17,10 @@ import (
|
|||
// or nil if it was unable to start a process.
|
||||
func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) {
|
||||
repo.Logger.Debug(fmt.Sprintf("Running the command: %v", args))
|
||||
if repo.DryRun {
|
||||
repo.Logger.Debug("Dry running for 50 seconds")
|
||||
args = []string{"sleep", "50"}
|
||||
}
|
||||
cmd := exec.Command(args[0], args[1:]...)
|
||||
repo.Logger.Debug("Starting process")
|
||||
|
||||
|
@ -127,6 +131,7 @@ func startRepoSync(repo *config.Repo) {
|
|||
|
||||
func zfsSync(repo *config.Repo) {
|
||||
// we are not using zfs snapshots at the moment
|
||||
repo.Logger.Debug("Would run a zfssync if not disabled")
|
||||
return
|
||||
|
||||
out, err := exec.Command("/home/mirror/bin/zfssync", repo.Name).CombinedOutput()
|
||||
|
|
|
@ -79,6 +79,9 @@ func addConditionalFlags(repo *config.Repo, flags int) []string {
|
|||
if repo.MaxRsyncIO > 0 {
|
||||
args = append(args, fmt.Sprintf("--bwlimit=%d", repo.MaxRsyncIO))
|
||||
}
|
||||
if repo.Verbose {
|
||||
args = append(args, "-vv")
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
@ -317,16 +320,8 @@ func cscSyncWget(repo *config.Repo) []string {
|
|||
return args
|
||||
}
|
||||
|
||||
func cscSyncDummy(repo *config.Repo) []string {
|
||||
return []string{"sleep", "10"}
|
||||
}
|
||||
|
||||
// executes a particular sync job depending on repo.SyncType.
|
||||
func getSyncCommand(repo *config.Repo) (args []string) {
|
||||
if repo.SyncType == "csc-sync-dummy" {
|
||||
return cscSyncDummy(repo)
|
||||
}
|
||||
|
||||
// check that the download directory exists
|
||||
if _, err := os.Stat(buildDownloadDir(repo)); os.IsNotExist(err) {
|
||||
repo.Logger.Error(err.Error())
|
||||
|
|
|
@ -14,7 +14,7 @@ func dummyRepoConf(name string, syncType string, frequencyStr string, localDir s
|
|||
stopChan := make(chan struct{})
|
||||
|
||||
repoLogFile := filepath.Join("test_files", name, name+".log")
|
||||
logger := logger.NewLogger(name, repoLogFile)
|
||||
logger := logger.NewLogger(name, repoLogFile, false)
|
||||
|
||||
return &config.Repo{
|
||||
Name: name,
|
||||
|
|
|
@ -88,8 +88,10 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool {
|
|||
return file1 != file2
|
||||
}
|
||||
|
||||
// some repos want us to check a last modification time file before performing a full sync
|
||||
// returns a status to update the original sync's status along with if the sync should continue
|
||||
func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
|
||||
// default is to keep the default status and continue the sync
|
||||
// default return values are the default status and to continue the sync
|
||||
status = config.FAILURE
|
||||
continueSync = true
|
||||
|
||||
|
@ -110,9 +112,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
|
|||
case "csc-sync-debian":
|
||||
args = cscTraceDebian(repo, temp.Name())
|
||||
default:
|
||||
if repo.SyncType != "csc-sync-dummy" {
|
||||
repo.Logger.Error("Trace files are not implemented for sync type ", "'"+repo.SyncType+"'")
|
||||
}
|
||||
repo.Logger.Error("Trace files are not implemented for sync type '" + repo.SyncType + "'")
|
||||
return
|
||||
}
|
||||
|
||||
|
|
Loading…
Reference in New Issue