fix merge confict
This commit is contained in:
Andrew Wang
commit
395d7d616f
|
|
@ -1,16 +1,24 @@
|
|||
This folder contains the code for merlin (which does the actual syncing) and
|
||||
arthur (which sends commands to merlin).
|
||||
This folder contains the code for merlin (which does the actual syncing) and arthur (which sends commands to merlin).
|
||||
|
||||
### In Progress
|
||||
- [ ] implement all sync types (csc-sync-debian, csc-sync-apache, etc.)
|
||||
- [ ] use separate log file for each child process (currently sharing stdout/stderr with parent)
|
||||
- [ ] implement zfssync in merlin (just invoke the existing Python script)
|
||||
|
||||
\* This is optional because the current version of merlin doesn't support it. If it turns
|
||||
out to be complicated, then don't worry about it. If you decide to implement it, use SIGHUP
|
||||
as the reload signal.
|
||||
### TODO FOR SYNC SCRIPTS
|
||||
- [ ] apache
|
||||
- [ ] archlinux
|
||||
- [ ] badperms
|
||||
- [ ] cdimage
|
||||
- [ ] ceph
|
||||
- [ ] chmod
|
||||
- [ ] debian
|
||||
- [ ] debian-cd
|
||||
- [ ] gentoo
|
||||
- [ ] s3
|
||||
- [ ] ssh
|
||||
- [x] standard
|
||||
- [ ] standard-ipv6
|
||||
- [ ] wget
|
||||
|
||||
### TODO
|
||||
- [ ] implement zfssync in merlin (just invoke the existing Python script)
|
||||
- [ ] use separate log file for each child process (currently sharing stdout/stderr with parent)
|
||||
- [ ] detect if an rsync process is stuck (\*\*)
|
||||
- [ ] place each rsync process in a separate cgroup (\*\*\*)
|
||||
|
||||
|
|
@ -31,4 +39,4 @@ stdout/stderr of the rsync process.
|
|||
|
||||
\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc)
|
||||
|
||||
\*\* may be a good idea to stop all repo syncs? (or just the ones not in the new config after the reload)
|
||||
\*\* may be a good idea to stop all repo syncs? (or just the ones not in the new config after the reload?)
|
||||
|
|
|
|||
|
|
@ -2,16 +2,9 @@ package common
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
/*
|
||||
if sync functions are similar enough, possibly create a common sync function
|
||||
that takes in the rsync command to run instead
|
||||
*/
|
||||
|
||||
func (repo *Repo) buildRsyncHost() string {
|
||||
if repo.RsyncUser != "" {
|
||||
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||||
|
|
@ -19,54 +12,30 @@ func (repo *Repo) buildRsyncHost() string {
|
|||
return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
|
||||
}
|
||||
|
||||
// CSCSyncStandard performs a standard rsync job.
|
||||
func (repo *Repo) CSCSyncStandard() {
|
||||
status := FAILURE
|
||||
defer func() {
|
||||
repo.DoneChan <- Result{
|
||||
Name: repo.Name,
|
||||
Exit: status,
|
||||
}
|
||||
}()
|
||||
|
||||
localDir := repo.cfg.DownloadDir + "/" + repo.LocalDir
|
||||
err := os.MkdirAll(localDir, 0775)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
|
||||
repo.Logger.Error(err)
|
||||
return
|
||||
}
|
||||
address := repo.cfg.IPv4Address
|
||||
logFile := repo.LogFile
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + logFile,
|
||||
}
|
||||
if repo.PasswordFile != "" {
|
||||
filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
|
||||
args = append(args, "--password-file", filename)
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), localDir)
|
||||
|
||||
ch := SpawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// Log that something failed?
|
||||
return
|
||||
}
|
||||
cmd := <-ch
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
// default is already FAILURE
|
||||
func (repo *Repo) buildRsyncDaemonHost() string {
|
||||
if repo.RsyncUser != "" {
|
||||
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||||
}
|
||||
return repo.RsyncHost + "::" + repo.RsyncDir
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncArchLinux() {
|
||||
func (repo *Repo) buildDownloadDir() string {
|
||||
return repo.cfg.DownloadDir + "/" + repo.LocalDir
|
||||
}
|
||||
|
||||
status := FAILURE
|
||||
func (repo *Repo) CSCSyncApache() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/",
|
||||
"--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncArchLinux() []string {
|
||||
|
||||
tempDir := "" // is this option even needed?
|
||||
|
||||
|
|
@ -76,53 +45,104 @@ func (repo *Repo) CSCSyncArchLinux() {
|
|||
"--contimeout=60", "-p", "--delay-updates", "--no-motd",
|
||||
"--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address,
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
|
||||
|
||||
ch := SpawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
return
|
||||
}
|
||||
return args
|
||||
|
||||
cmd := <-ch
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
}
|
||||
|
||||
_ = status
|
||||
}
|
||||
|
||||
// does nothing for 5 to 15 seconds then returns
|
||||
func (repo *Repo) CSCSyncDummy() {
|
||||
status := FAILURE
|
||||
defer func() {
|
||||
repo.DoneChan <- Result{
|
||||
Name: repo.Name,
|
||||
Exit: status,
|
||||
}
|
||||
}()
|
||||
sleepDur := strconv.FormatInt(rand.Int63n(10)+5, 10)
|
||||
args := []string{"sleep", sleepDur}
|
||||
ch := SpawnProcess(repo, args)
|
||||
|
||||
if ch == nil {
|
||||
return
|
||||
func (repo *Repo) CSCSyncBadPerms() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/",
|
||||
"--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
cmd := <-ch
|
||||
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
// StartSyncJob executes a particular sync job depending on repo.SyncType.
|
||||
func (repo *Repo) StartSyncJob() {
|
||||
func (repo *Repo) CSCSyncCDImage() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncChmod() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
"--chmod=u=rwX,go=rX",
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncDebian() []string {
|
||||
|
||||
// sync /pool
|
||||
args := []string{"nice", "rsync", "-rlHtvp",
|
||||
"--exclude", ".~tmp~/", "--timeout=3600", "-4",
|
||||
"--address=" + repo.cfg.IPv4Address,
|
||||
}
|
||||
// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncDebianCD() []string {
|
||||
|
||||
// this is basically the same as CSCSyncDebian, except it has an extra --exclude
|
||||
args := []string{"nice", "rsync", "-rlHtvp", "--delete",
|
||||
"--exclude", ".~tmp~/", "--timeout=3600", "-4",
|
||||
"--address=" + repo.cfg.IPv4Address,
|
||||
// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
|
||||
}
|
||||
// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncGentoo() []string {
|
||||
repo.RsyncUser = "gentoo"
|
||||
repo.PasswordFile = "gentoo-distfiles"
|
||||
return repo.CSCSyncStandard()
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncStandard() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
if repo.PasswordFile != "" {
|
||||
filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
|
||||
args = append(args, "--password-file", filename)
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncStandardIPV6() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
// executes a particular sync job depending on repo.SyncType.
|
||||
func (repo *Repo) getSyncCommand() []string {
|
||||
switch repo.SyncType {
|
||||
/*
|
||||
# scripts used by merlin.py
|
||||
|
|
@ -149,12 +169,79 @@ func (repo *Repo) StartSyncJob() {
|
|||
make-torrents
|
||||
ubuntu-releases-sync
|
||||
*/
|
||||
case "csc-sync-debian":
|
||||
case "csc-sync-arch":
|
||||
repo.CSCSyncArchLinux()
|
||||
case "csc-sync-standard":
|
||||
repo.CSCSyncStandard()
|
||||
default:
|
||||
repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
|
||||
}
|
||||
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func (repo *Repo) StartSyncJob() {
|
||||
status := FAILURE
|
||||
defer func() {
|
||||
repo.DoneChan <- Result{
|
||||
Name: repo.Name,
|
||||
Exit: status,
|
||||
}
|
||||
}()
|
||||
|
||||
localDir := repo.buildDownloadDir()
|
||||
err := os.MkdirAll(localDir, 0775)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
|
||||
repo.Logger.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
args := repo.getSyncCommand()
|
||||
if repo.PasswordFile != "" {
|
||||
filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
|
||||
args = append(args, "--password-file", filename)
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), localDir)
|
||||
|
||||
ch := SpawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// Log that something failed?
|
||||
return
|
||||
}
|
||||
cmd := <-ch
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
// default is already FAILURE
|
||||
}
|
||||
}
|
||||
|
||||
// // does nothing for 5 to 15 seconds then returns
|
||||
// func (repo *Repo) CSCSyncDummy() {
|
||||
// status := FAILURE
|
||||
// defer func() {
|
||||
// repo.DoneChan <- Result{
|
||||
// Name: repo.Name,
|
||||
// Exit: status,
|
||||
// }
|
||||
// }()
|
||||
// sleepDur := strconv.FormatInt(rand.Int63n(10)+5, 10)
|
||||
// args := []string{"sleep", sleepDur}
|
||||
// ch := SpawnProcess(repo, args)
|
||||
|
||||
// if ch == nil {
|
||||
// return
|
||||
// }
|
||||
|
||||
// cmd := <-ch
|
||||
|
||||
// switch cmd.ProcessState.ExitCode() {
|
||||
|
||||
// case 0:
|
||||
// status = SUCCESS
|
||||
// case -1:
|
||||
// status = TERMINATED
|
||||
// }
|
||||
// }
|
||||
|
|
|
|||
Loading…
Reference in New Issue