more sync scripts

merlin/README.md

@@ -13,6 +13,22 @@ arthur (which sends commands to merlin).
 - [ ] detect if an rsync process is stuck (\*\*)
 - [ ] place each rsync process in a separate cgroup (\*\*\*)
 
+### TODO FOR SYNC SCRIPTS
+- [ ] apache
+- [ ] archlinux
+- [ ] badperms
+- [ ] cdimage
+- [ ] ceph
+- [ ] chmod
+- [ ] debian
+- [ ] debian-cd
+- [ ] gentoo
+- [ ] s3
+- [ ] ssh
+- [x] standard
+- [ ] standard-ipv6
+- [ ] wget
+
 \* This is optional because the current version of merlin doesn't support it. If it turns
 out to be complicated, then don't worry about it. If you decide to implement it, use SIGHUP
 as the reload signal.
@@ -29,4 +45,4 @@ stdout/stderr of the rsync process.
 - [x] add all repos to merlin-config.ini (\*)
 - [x] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this
 
-\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc)
+\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc)

merlin/common/sync.go

@@ -5,11 +5,6 @@ import (
 	"os"
 )
 
-/*
-if sync functions are similar enough, possibly create a common sync function
-that takes in the rsync command to run instead
-*/
-
 func (repo *Repo) buildRsyncHost() string {
 	if repo.RsyncUser != "" {
 		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
@@ -17,55 +12,81 @@ func (repo *Repo) buildRsyncHost() string {
 	return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
 }
 
-// CSCSyncStandard performs a standard rsync job.
+func (repo *Repo) buildRsyncDaemonHost() string {
-func (repo *Repo) CSCSyncStandard() {
+	if repo.RsyncUser != "" {
-	status := FAILURE
+		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
-	defer func() {
-		repo.DoneChan <- Result{
-			Name: repo.Name,
-			Exit: status,
-		}
-	}()
-
-	localDir := repo.cfg.DownloadDir + "/" + repo.LocalDir
-	err := os.MkdirAll(localDir, 0775)
-	if err != nil {
-		err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
-		repo.Logger.Error(err)
-		return
 	}
-	address := repo.cfg.IPv4Address
+	return repo.RsyncHost + "::" + repo.RsyncDir
-	logFile := repo.LogFile
+}
+
+func (repo *Repo) buildDownloadDir() string {
+	return repo.cfg.DownloadDir + "/" + repo.LocalDir
+}
+
+func (repo *Repo) CSCSyncApache() []string {
 	args := []string{
-		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
+		"nice", "rsync", "-az", "--no-owner", "--no-group", "--delete", "--safe-links",
-		"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + address,
+		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
-		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + logFile,
+		"--exclude", ".~tmp~/",
+		"--quiet", "--stats", "--log-file=" + repo.LogFile,
 	}
-	if repo.PasswordFile != "" {
+	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
-		filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
-		args = append(args, "--password-file", filename)
-	}
-	args = append(args, repo.buildRsyncHost(), localDir)
 
-	ch := SpawnProcess(repo, args)
+	return args
-	if ch == nil {
-		// Log that something failed?
-		return
-	}
-	cmd := <-ch
-	switch cmd.ProcessState.ExitCode() {
-	case 0:
-		status = SUCCESS
-	case -1:
-		status = TERMINATED
-		// default is already FAILURE
-	}
 }
 
-func (repo *Repo) CSCSyncStandardIPV6() {
+func (repo *Repo) CSCSyncArchLinux() []string {
+
+	tempDir := "" // is this option even needed?
+
+	// add option for verbose flag?
+	args := []string{
+		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
+		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
+		"--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address,
+	}
+	args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
+
+	return args
+
 }
 
-func (repo *Repo) CSCSyncDebian() {
+func (repo *Repo) CSCSyncBadPerms() []string {
+	args := []string{
+		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--chmod=o=rX", "--delete",
+		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
+		"--exclude", ".~tmp~/",
+		"--quiet", "--stats", "--log-file=" + repo.LogFile,
+	}
+	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
+
+	return args
+ }
+
+func (repo *Repo) CSCSyncCDImage() []string {
+	args := []string{
+		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete",
+		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
+		"--exclude", ".*/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
+	}
+	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
+
+	return args
+ }
+
+func (repo *Repo) CSCSyncChmod() []string {
+	args := []string{
+		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after", "--delay-updates", "--safe-links",
+		"--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
+		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
+		"--chmod=u=rwX,go=rX",
+	}
+	args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
+
+	return args
+ }
+
+func (repo *Repo) CSCSyncDebian() []string {
 
 	// sync /pool
 	args := []string{"nice", "rsync", "-rlHtvp",
@@ -73,9 +94,10 @@ func (repo *Repo) CSCSyncDebian() {
 		"--address=" + repo.cfg.IPv4Address,
 	}
 		 // $RSYNC_HOST::$RSYNC_DIR/pool/ $TO/pool/ >> $LOGFILE 2>&1
+	return args
 }
 
-func (repo *Repo) CSCSyncDebianCD() {
+func (repo *Repo) CSCSyncDebianCD() []string {
 
 	// this is basically the same as CSCSyncDebian, except it has an extra --exclude
 	args := []string{"nice", "rsync", "-rlHtvp", "--delete",
@@ -84,44 +106,45 @@ func (repo *Repo) CSCSyncDebianCD() {
 		// "--exclude", "Archive-Update-in-Progress-${HOSTNAME}"
 	}
      // $RSYNC_HOST::$RSYNC_DIR $TO >> $LOGFILE 2>&1
+	return args
 }
 
-func (repo *Repo) CSCSyncArchLinux() {
+func (repo *Repo) CSCSyncGentoo() []string {
-	status := FAILURE
+	repo.RsyncUser = "gentoo"
+	repo.PasswordFile = "gentoo-distfiles"
+	return repo.CSCSyncStandard()
+}
 
-	tempDir := "" // is this option even needed?
+func (repo *Repo) CSCSyncStandard() []string {
-	localDir := "" // temp
-
-	// add option for verbose flag?
 	args := []string{
-		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
+		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
-		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
+		"--delay-updates", "--safe-links", "--timeout=3600", "-4", "--address=" + repo.cfg.IPv4Address,
-		"--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address,
+		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
 	}
-	args = append(args, repo.buildRsyncHost(), localDir)
+	if repo.PasswordFile != "" {
-
+		filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
-	ch := SpawnProcess(repo, args)
+		args = append(args, "--password-file", filename)
-	if ch == nil {
-		return
 	}
+	args = append(args, repo.buildRsyncHost(), repo.buildDownloadDir())
 
-	cmd := <-ch
+	return args
-	switch cmd.ProcessState.ExitCode() {
-	case 0:
-		status = SUCCESS
-	case -1:
-		status = TERMINATED
-	}
-
-	_ = status
 }
 
-func (repo *Repo) CSCSyncGentoo() {
+func (repo *Repo) CSCSyncStandardIPV6() []string {
+	args := []string{
+		"nice", "rsync", "-aH", "--no-owner", "--no-group", "--delete-after",
+		"--delay-updates", "--safe-links", "--timeout=3600", "-6", "--address=" + repo.cfg.IPv4Address,
+		"--exclude", ".~tmp~/", "--quiet", "--stats", "--log-file=" + repo.LogFile,
+	}
+	args = append(args, repo.buildRsyncDaemonHost(), repo.buildDownloadDir())
 
+	return args
 }
 
-// StartSyncJob executes a particular sync job depending on repo.SyncType.
+
-func (repo *Repo) StartSyncJob() {
+
+// executes a particular sync job depending on repo.SyncType.
+func (repo *Repo) getSyncCommand() []string {
 	switch repo.SyncType {
 	/*
 		# scripts used by merlin.py
@@ -149,13 +172,50 @@ func (repo *Repo) StartSyncJob() {
 		ubuntu-releases-sync
 	*/
 	case "csc-sync-debian":
-		repo.CSCSyncDebian()
 	case "csc-sync-arch":
-		repo.CSCSyncArchLinux()
 	case "csc-sync-standard":
-		repo.CSCSyncStandard()
 	default:
 		repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
+	}
+
+	return []string{}
+}
+
+func (repo *Repo) StartSyncJob() {
+	status := FAILURE
+	defer func() {
+		repo.DoneChan <- Result{
+			Name: repo.Name,
+			Exit: status,
+		}
+	}()
+
+	localDir := repo.buildDownloadDir()
+	err := os.MkdirAll(localDir, 0775)
+	if err != nil {
+		err = fmt.Errorf("Could not create directory %s: %w", localDir, err)
+		repo.Logger.Error(err)
 		return
 	}
+
+	args := repo.getSyncCommand()
+	if repo.PasswordFile != "" {
+		filename := repo.cfg.PasswordDir + "/" + repo.PasswordFile
+		args = append(args, "--password-file", filename)
+	}
+	args = append(args, repo.buildRsyncHost(), localDir)
+
+	ch := SpawnProcess(repo, args)
+	if ch == nil {
+		// Log that something failed?
+		return
+	}
+	cmd := <-ch
+	switch cmd.ProcessState.ExitCode() {
+	case 0:
+		status = SUCCESS
+	case -1:
+		status = TERMINATED
+		// default is already FAILURE
+	}
 }