more sync scripts
This commit is contained in:
Daniel Liu
parent
c3e1bf7e3e
commit
fe1071ac88
|
|
@ -13,6 +13,22 @@ arthur (which sends commands to merlin).
|
|||
- [ ] detect if an rsync process is stuck (\*\*)
|
||||
- [ ] place each rsync process in a separate cgroup (\*\*\*)
|
||||
|
||||
### TODO FOR SYNC SCRIPTS
|
||||
- [ ] apache
|
||||
- [ ] archlinux
|
||||
- [ ] badperms
|
||||
- [ ] cdimage
|
||||
- [ ] ceph
|
||||
- [ ] chmod
|
||||
- [ ] debian
|
||||
- [ ] debian-cd
|
||||
- [ ] gentoo
|
||||
- [ ] s3
|
||||
- [ ] ssh
|
||||
- [x] standard
|
||||
- [ ] standard-ipv6
|
||||
- [ ] wget
|
||||
|
||||
\* This is optional because the current version of merlin doesn't support it. If it turns
|
||||
out to be complicated, then don't worry about it. If you decide to implement it, use SIGHUP
|
||||
as the reload signal.
|
||||
|
|
@ -29,4 +45,4 @@ stdout/stderr of the rsync process.
|
|||
- [x] add all repos to merlin-config.ini (\*)
|
||||
- [x] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this
|
||||
|
||||
\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc)
|
||||
\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc)
|
||||
|
|
|
|||
|
|
@ -5,11 +5,6 @@ import (
|
|||
"os"
|
||||
)
|
||||
|
||||
/*
|
||||
if sync functions are similar enough, possibly create a common sync function
|
||||
that takes in the rsync command to run instead
|
||||
*/
|
||||
|
||||
func (repo *Repo) buildRsyncHost() string {
|
||||
if repo.RsyncUser != "" {
|
||||
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||||
|
|
@ -17,55 +12,81 @@ func (repo *Repo) buildRsyncHost() string {
|
|||
return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
|
||||
}
|
||||
|
||||
// CSCSyncStandard performs a standard rsync job.
|
||||
func (repo *Repo) CSCSyncStandard() {
|
||||
status := FAILURE
|
||||
defer func() {
|
||||
repo.DoneChan <- Result{
|
||||
Name: repo.Name,
|
||||
Exit: status,
|
||||
}
|
||||
}()
|
||||
|
||||
localDir := repo.cfg.DownloadDir + "/" + repo.LocalDir
|
||||
err := os.MkdirAll(localDir, 0775)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
|
||||
repo.Logger.Error(err)
|
||||
return
|
||||
func (repo *Repo) buildRsyncDaemonHost() string {
|
||||
if repo.RsyncUser != "" {
|
||||
repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
|
||||
}
|
||||
address := repo.cfg.IPv4Address
|
||||
logFile := repo.LogFile
|
||||
return repo.RsyncHost + "::" + repo.RsyncDir
|
||||
}
|
||||
|
||||
func (repo *Repo) buildDownloadDir() string {
|
||||
return repo.cfg.DownloadDir + "/" + repo.LocalDir
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncApache() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + logFile,
|
||||
"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/",
|
||||
"--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
if repo.PasswordFile != "" {
|
||||
filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
|
||||
args = append(args, "--password-file", filename)
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), localDir)
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
ch := SpawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// Log that something failed?
|
||||
return
|
||||
}
|
||||
cmd := <-ch
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
// default is already FAILURE
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncStandardIPV6() {
|
||||
func (repo *Repo) CSCSyncArchLinux() []string {
|
||||
|
||||
tempDir := "" // is this option even needed?
|
||||
|
||||
// add option for verbose flag?
|
||||
args := []string{
|
||||
"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
|
||||
"--contimeout=60", "-p", "--delay-updates", "--no-motd",
|
||||
"--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address,
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncDebian() {
|
||||
func (repo *Repo) CSCSyncBadPerms() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/",
|
||||
"--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncCDImage() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncChmod() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
|
||||
"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
"--chmod=u=rwX,go=rX",
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncDebian() []string {
|
||||
|
||||
// sync /pool
|
||||
args := []string{"nice", "rsync", "-rlHtvp",
|
||||
|
|
@ -73,9 +94,10 @@ func (repo *Repo) CSCSyncDebian() {
|
|||
"--address=" + repo.cfg.IPv4Address,
|
||||
}
|
||||
// $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncDebianCD() {
|
||||
func (repo *Repo) CSCSyncDebianCD() []string {
|
||||
|
||||
// this is basically the same as CSCSyncDebian, except it has an extra --exclude
|
||||
args := []string{"nice", "rsync", "-rlHtvp", "--delete",
|
||||
|
|
@ -84,44 +106,45 @@ func (repo *Repo) CSCSyncDebianCD() {
|
|||
// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
|
||||
}
|
||||
// $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncArchLinux() {
|
||||
status := FAILURE
|
||||
func (repo *Repo) CSCSyncGentoo() []string {
|
||||
repo.RsyncUser = "gentoo"
|
||||
repo.PasswordFile = "gentoo-distfiles"
|
||||
return repo.CSCSyncStandard()
|
||||
}
|
||||
|
||||
tempDir := "" // is this option even needed?
|
||||
localDir := "" // temp
|
||||
|
||||
// add option for verbose flag?
|
||||
func (repo *Repo) CSCSyncStandard() []string {
|
||||
args := []string{
|
||||
"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
|
||||
"--contimeout=60", "-p", "--delay-updates", "--no-motd",
|
||||
"--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address,
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), localDir)
|
||||
|
||||
ch := SpawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
return
|
||||
if repo.PasswordFile != "" {
|
||||
filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
|
||||
args = append(args, "--password-file", filename)
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
|
||||
|
||||
cmd := <-ch
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
}
|
||||
|
||||
_ = status
|
||||
return args
|
||||
}
|
||||
|
||||
func (repo *Repo) CSCSyncGentoo() {
|
||||
func (repo *Repo) CSCSyncStandardIPV6() []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
|
||||
"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + repo.cfg.IPv4Address,
|
||||
"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
|
||||
}
|
||||
args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
// StartSyncJob executes a particular sync job depending on repo.SyncType.
|
||||
func (repo *Repo) StartSyncJob() {
|
||||
|
||||
|
||||
// executes a particular sync job depending on repo.SyncType.
|
||||
func (repo *Repo) getSyncCommand() []string {
|
||||
switch repo.SyncType {
|
||||
/*
|
||||
# scripts used by merlin.py
|
||||
|
|
@ -149,13 +172,50 @@ func (repo *Repo) StartSyncJob() {
|
|||
ubuntu-releases-sync
|
||||
*/
|
||||
case "csc-sync-debian":
|
||||
repo.CSCSyncDebian()
|
||||
case "csc-sync-arch":
|
||||
repo.CSCSyncArchLinux()
|
||||
case "csc-sync-standard":
|
||||
repo.CSCSyncStandard()
|
||||
default:
|
||||
repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
|
||||
}
|
||||
|
||||
return []string{}
|
||||
}
|
||||
|
||||
func (repo *Repo) StartSyncJob() {
|
||||
status := FAILURE
|
||||
defer func() {
|
||||
repo.DoneChan <- Result{
|
||||
Name: repo.Name,
|
||||
Exit: status,
|
||||
}
|
||||
}()
|
||||
|
||||
localDir := repo.buildDownloadDir()
|
||||
err := os.MkdirAll(localDir, 0775)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
|
||||
repo.Logger.Error(err)
|
||||
return
|
||||
}
|
||||
|
||||
args := repo.getSyncCommand()
|
||||
if repo.PasswordFile != "" {
|
||||
filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
|
||||
args = append(args, "--password-file", filename)
|
||||
}
|
||||
args = append(args, repo.buildRsyncHost(), localDir)
|
||||
|
||||
ch := SpawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// Log that something failed?
|
||||
return
|
||||
}
|
||||
cmd := <-ch
|
||||
switch cmd.ProcessState.ExitCode() {
|
||||
case 0:
|
||||
status = SUCCESS
|
||||
case -1:
|
||||
status = TERMINATED
|
||||
// default is already FAILURE
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue