From fe1071ac88aafee41fec130d3c808c3ee7594913 Mon Sep 17 00:00:00 2001 From: Daniel Liu Date: Sat, 6 Nov 2021 23:07:59 -0400 Subject: [PATCH] more sync scripts --- merlin/README.md | 18 +++- merlin/common/sync.go | 212 +++++++++++++++++++++++++++--------------- 2 files changed, 153 insertions(+), 77 deletions(-) diff --git a/merlin/README.md b/merlin/README.md index 3ababa3..b354bae 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -13,6 +13,22 @@ arthur (which sends commands to merlin). - [ ] detect if an rsync process is stuck (\*\*) - [ ] place each rsync process in a separate cgroup (\*\*\*) +### TODO FOR SYNC SCRIPTS +- [ ] apache +- [ ] archlinux +- [ ] badperms +- [ ] cdimage +- [ ] ceph +- [ ] chmod +- [ ] debian +- [ ] debian-cd +- [ ] gentoo +- [ ] s3 +- [ ] ssh +- [x] standard +- [ ] standard-ipv6 +- [ ] wget + \* This is optional because the current version of merlin doesn't support it. If it turns out to be complicated, then don't worry about it. If you decide to implement it, use SIGHUP as the reload signal. @@ -29,4 +45,4 @@ stdout/stderr of the rsync process. - [x] add all repos to merlin-config.ini (\*) - [x] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this -\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc) \ No newline at end of file +\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc) diff --git a/merlin/common/sync.go b/merlin/common/sync.go index ea15051..4a991cd 100644 --- a/merlin/common/sync.go +++ b/merlin/common/sync.go @@ -5,11 +5,6 @@ import ( "os" ) -/* -if sync functions are similar enough, possibly create a common sync function -that takes in the rsync command to run instead -*/ - func (repo *Repo) buildRsyncHost() string { if repo.RsyncUser != "" { repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost @@ -17,55 +12,81 @@ func (repo *Repo) buildRsyncHost() string { return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir } -// CSCSyncStandard performs a standard rsync job. -func (repo *Repo) CSCSyncStandard() { - status := FAILURE - defer func() { - repo.DoneChan <- Result{ - Name: repo.Name, - Exit: status, - } - }() - - localDir := repo.cfg.DownloadDir + "/" + repo.LocalDir - err := os.MkdirAll(localDir, 0775) - if err != nil { - err = fmt.Errorf("Could not create directory %s: %w", localDir, err) - repo.Logger.Error(err) - return +func (repo *Repo) buildRsyncDaemonHost() string { + if repo.RsyncUser != "" { + repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost } - address := repo.cfg.IPv4Address - logFile := repo.LogFile + return repo.RsyncHost + "::" + repo.RsyncDir +} + +func (repo *Repo) buildDownloadDir() string { + return repo.cfg.DownloadDir + "/" + repo.LocalDir +} + +func (repo *Repo) CSCSyncApache() []string { args := []string{ - "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", - "--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + address, - "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + logFile, + "nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links", + "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address, + "--exclude", ".~tmp~/", + "--quiet", "--stats", "--log-file=" + repo.LogFile, } - if repo.PasswordFile != "" { - filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile - args = append(args, "--password-file", filename) - } - args = append(args, repo.buildRsyncHost(), localDir) + args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir()) - ch := SpawnProcess(repo, args) - if ch == nil { - // Log that something failed? - return - } - cmd := <-ch - switch cmd.ProcessState.ExitCode() { - case 0: - status = SUCCESS - case -1: - status = TERMINATED - // default is already FAILURE - } + return args } -func (repo *Repo) CSCSyncStandardIPV6() { +func (repo *Repo) CSCSyncArchLinux() []string { + + tempDir := "" // is this option even needed? + + // add option for verbose flag? + args := []string{ + "rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600", + "--contimeout=60", "-p", "--delay-updates", "--no-motd", + "--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address, + } + args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir()) + + return args + } -func (repo *Repo) CSCSyncDebian() { +func (repo *Repo) CSCSyncBadPerms() []string { + args := []string{ + "nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete", + "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address, + "--exclude", ".~tmp~/", + "--quiet", "--stats", "--log-file=" + repo.LogFile, + } + args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir()) + + return args + } + +func (repo *Repo) CSCSyncCDImage() []string { + args := []string{ + "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete", + "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address, + "--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.LogFile, + } + args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir()) + + return args + } + +func (repo *Repo) CSCSyncChmod() []string { + args := []string{ + "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links", + "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address, + "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile, + "--chmod=u=rwX,go=rX", + } + args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir()) + + return args + } + +func (repo *Repo) CSCSyncDebian() []string { // sync /pool args := []string{"nice", "rsync", "-rlHtvp", @@ -73,9 +94,10 @@ func (repo *Repo) CSCSyncDebian() { "--address=" + repo.cfg.IPv4Address, } // $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1 + return args } -func (repo *Repo) CSCSyncDebianCD() { +func (repo *Repo) CSCSyncDebianCD() []string { // this is basically the same as CSCSyncDebian, except it has an extra --exclude args := []string{"nice", "rsync", "-rlHtvp", "--delete", @@ -84,44 +106,45 @@ func (repo *Repo) CSCSyncDebianCD() { // "--exclude", "Archive-Update-in-Progress-${HOSTNAME}" } // $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1 + return args } -func (repo *Repo) CSCSyncArchLinux() { - status := FAILURE +func (repo *Repo) CSCSyncGentoo() []string { + repo.RsyncUser = "gentoo" + repo.PasswordFile = "gentoo-distfiles" + return repo.CSCSyncStandard() +} - tempDir := "" // is this option even needed? - localDir := "" // temp - - // add option for verbose flag? +func (repo *Repo) CSCSyncStandard() []string { args := []string{ - "rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600", - "--contimeout=60", "-p", "--delay-updates", "--no-motd", - "--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address, + "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", + "--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address, + "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile, } - args = append(args, repo.buildRsyncHost(), localDir) - - ch := SpawnProcess(repo, args) - if ch == nil { - return + if repo.PasswordFile != "" { + filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile + args = append(args, "--password-file", filename) } + args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir()) - cmd := <-ch - switch cmd.ProcessState.ExitCode() { - case 0: - status = SUCCESS - case -1: - status = TERMINATED - } - - _ = status + return args } -func (repo *Repo) CSCSyncGentoo() { +func (repo *Repo) CSCSyncStandardIPV6() []string { + args := []string{ + "nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", + "--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + repo.cfg.IPv4Address, + "--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile, + } + args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir()) + return args } -// StartSyncJob executes a particular sync job depending on repo.SyncType. -func (repo *Repo) StartSyncJob() { + + +// executes a particular sync job depending on repo.SyncType. +func (repo *Repo) getSyncCommand() []string { switch repo.SyncType { /* # scripts used by merlin.py @@ -149,13 +172,50 @@ func (repo *Repo) StartSyncJob() { ubuntu-releases-sync */ case "csc-sync-debian": - repo.CSCSyncDebian() case "csc-sync-arch": - repo.CSCSyncArchLinux() case "csc-sync-standard": - repo.CSCSyncStandard() default: repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'") + } + + return []string{} +} + +func (repo *Repo) StartSyncJob() { + status := FAILURE + defer func() { + repo.DoneChan <- Result{ + Name: repo.Name, + Exit: status, + } + }() + + localDir := repo.buildDownloadDir() + err := os.MkdirAll(localDir, 0775) + if err != nil { + err = fmt.Errorf("Could not create directory %s: %w", localDir, err) + repo.Logger.Error(err) return } + + args := repo.getSyncCommand() + if repo.PasswordFile != "" { + filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile + args = append(args, "--password-file", filename) + } + args = append(args, repo.buildRsyncHost(), localDir) + + ch := SpawnProcess(repo, args) + if ch == nil { + // Log that something failed? + return + } + cmd := <-ch + switch cmd.ProcessState.ExitCode() { + case 0: + status = SUCCESS + case -1: + status = TERMINATED + // default is already FAILURE + } }