start merlin_old and make changes to merlin go

merlin/README.md

@@ -1,19 +1,16 @@
 This folder contains the code for merlin (which does the actual syncing) and
 arthur (which sends commands to merlin).
 
-## Checklist
+### In Progress
-- [x] write process manager
 - [ ] implement all sync types (csc-sync-debian, csc-sync-apache, etc.)
-- [ ] save state (last attempted time, last attempted status) for each repo,
+- [ ] use separate log file for each child process (currently sharing stdout/stderr with parent) (does this mean different from repo?)
-      and restore state on startup (e.g. use JSON/INI file for each repo)
+    - different log dir for each repo?
-- [ ] use separate log file for each child process (currently sharing stdout/stderr with parent)
+- [ ] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this
-- [ ] calculate difference between the scheduled time of a job and the time at which it actually
+
-      ran; log this
+### TODO
-- [ ] add all repos to merlin-config.ini
 - [ ] listen on Unix socket in merlin
 - [ ] implement arthur.go (commands: sync and status)
 - [ ] implement zfssync in merlin (just invoke the existing Python script)
-- [ ] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this
 - [ ] allow dynamic reloading in merlin (\*)
 - [ ] detect if an rsync process is stuck (\*\*)
 - [ ] place each rsync process in a separate cgroup (\*\*\*)
@@ -26,3 +23,11 @@ as the reload signal.
 stdout/stderr of the rsync process.
 
 \*\*\* I need to do more research into this - don't worry about it for now.
+
+### Completed
+- [x] write process manager
+- [x] save state (last attempted time, last attempted status) for each repo, and restore state on startup (e.g. use JSON/INI file for each repo)
+- [x] calculate difference between the scheduled time of a job and the time at which it actually ran; log this
+- [x] add all repos to merlin-config.ini (\*)
+
+\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc)

merlin/common/common.go

@@ -11,9 +11,9 @@ const (
 	DAILY         = 86400
 	TWICE_DAILY   = DAILY / 2
 	HOURLY        = 3600
+	TWICE_HOURLY  = HOURLY / 2
 	BI_HOURLY     = HOURLY * 2
 	TRI_HOURLY    = HOURLY * 3
-	TWICE_HOURLY  = HOURLY / 2
 	TEN_MINUTELY  = 600
 	FIVE_MINUTELY = 300
 
@@ -21,20 +21,20 @@ const (
 
 	DEFAULT_MAX_JOBS     = 6
 	DEFAULT_MAX_TIME     = DAILY / 4
-	DEFAULT_SOCK_PATH    = "/run/merlin/merlin.sock"
-	DEFAULT_STATE_PATH   = "/home/mirror/merlin/states"
-	DEFAULT_LOG_FILE     = "/home/mirror/merlin/logs/transfer.log"
-	DEFAULT_PASSWORD_DIR = "/home/mirror/passwords"
 	DEFAULT_DOWNLOAD_DIR = "/mirror/root"
+	DEFAULT_PASSWORD_DIR = "/home/mirror/passwords"
+	DEFAULT_LOG_DIR      = "/home/mirror/merlin/logs"
+	DEFAULT_STATE_DIR    = "/home/mirror/merlin/states"
+	DEFAULT_SOCK_PATH    = "/run/merlin/merlin.sock"
 )
 
 var frequencies = map[string]int{
 	"daily":         DAILY,
 	"twice-daily":   TWICE_DAILY,
 	"hourly":        HOURLY,
+	"twice-hourly":  TWICE_HOURLY,
 	"bi-hourly":     BI_HOURLY,
 	"tri-hourly":    TRI_HOURLY,
-	"twice-hourly":  TWICE_HOURLY,
 	"ten-minutely":  TEN_MINUTELY,
 	"five-minutely": FIVE_MINUTELY,
 }
@@ -43,15 +43,24 @@ var frequencies = map[string]int{
 const (
 	NOT_RUN_YET = iota
 	SUCCESS
-	FAILURE    // change to TIMEOUT?
+	FAILURE
 	TERMINATED // was killed by a signal
 )
 
+type Result struct {
+	Name string
+	Exit int
+}
+
 type Repo struct {
+	// the name of this repo
+	Name string `ini:"-"`
 	// this should be one of "csc-sync-standard", etc.
 	SyncType string `ini:"sync_type"`
 	// a human-readable frequency, e.g. "bi-hourly"
 	FrequencyStr string `ini:"frequency"`
+	// the desired interval (in seconds) between successive runs
+	Frequency int `ini:"-"`
 	// the maximum time (in seconds) that each child process of this repo
 	// can for before being killed
 	MaxTime int `ini:"max_time"`
@@ -68,85 +77,112 @@ type Repo struct {
 	PasswordFile string `ini:"password_file"`
 	// the log file for rsync (optional, defaults to Config.LogFile)
 	LogFile string `ini:"log_file"`
-	// the desired interval (in seconds) between successive runs
+	// a reference to the logger
-	Frequency int `ini:"-"`
-	// the name of this repo
-	Name   string  `ini:"-"`
 	Logger *Logger `ini:"-"`
+	// the repo will write its name and status in a Result struct to DoneChan
+	// when it has finished a job
+	DoneChan chan<- Result `ini:"-"`
 	// the repo should stop syncing if StopChan is closed
 	StopChan <-chan bool `ini:"-"`
-	// the repo will write its name to DoneChan when it has finished a job
+	// a struct that stores the repo's status
-	// (successfully or unsuccessfully)
+	State RepoState `ini:"-"`
-	DoneChan chan<- string `ini:"-"`
 	// a reference to the global config
 	cfg *Config `ini:"-"`
-	// a struct that stores the repo's status
-	state State `ini:"-"`
 }
 
 type Config struct {
 	// the maximum number of jobs allowed to execute concurrently
 	MaxJobs int `ini:"max_jobs"`
-	// the default MaxTime for each repo
-	MaxTime int `ini:"max_time"`
-	// the Unix socket path which arthur will use to communicate with us
-	SockPath string `ini:"sock_path"`
-	// the path to where the state of each repo's sync is saved
-	StatePath string `ini:"states_path"`
-	// the default LogFile for each repo
-	LogFile string `ini:"log_file"`
 	// the IP addresses to use for rsync
 	IPv4Address string `ini:"ipv4_address"`
 	IPv6Address string `ini:"ipv6_address"`
-	// the directory where rsync passwords are stored
+	// the default sync type
-	PasswordDir string `ini:"password_dir"`
+	SyncType string `ini:"default_sync_type"`
+	// the default frequency string for the repos
+	FrequencyStr string `ini:"default_frequency"`
+	// the default MaxTime for each repo
+	MaxTime int `ini:"default_max_time"`
 	// the directory where rsync should download files
 	DownloadDir string `ini:"download_dir"`
+	// the directory where rsync passwords are stored
+	PasswordDir string `ini:"password_dir"`
+	// the path where merlin will store the logs for each repo synced
+	LogDir string `ini:"log_dir"`
+	// the path to where the state of each repo's sync is saved
+	StateDir string `ini:"states_dir"`
+	// the Unix socket path which arthur will use to communicate with us
+	SockPath string `ini:"sock_path"`
 	// the DoneChan for each repo (shared instance)
-	DoneChan <-chan string `ini:"-"`
+	DoneChan <-chan Result `ini:"-"`
 	// a list of all of the repos
 	Repos []*Repo `ini:"-"`
 }
 
-type State struct {
+// This should only be modified by the main thread
+type RepoState struct {
 	// these are stored in the states folder
+	// whether this repo is running a job or not
+	IsRunning bool `ini:"is_running"`
+	// whether the last attempt was successful or not
+	LastAttemptExit int `ini:"last_attempt_exit"`
 	// the Unix epoch timestamp at which this repo last attempted a job
 	LastAttemptTime int64 `ini:"last_attempt_time"`
 	// the number of seconds this repo ran for during its last attempted job
 	LastAttemptRunTime int64 `ini:"last_attempt_runtime"`
-	// whether the last attempt was successful or not
-	LastAttemptStatus int `ini:"last_attempt_status"`
-	// whether this repo is running a job or not
-	IsRunning bool `ini:"is_running"`
-}
-
-// IsRunning returns true if the repo is currently running a sync job.
-func (repo *Repo) IsRunning() bool {
-	return repo.state.IsRunning
 }
 
 // RunIfScheduled starts a sync job for this repo if more than repo.Frequency
 // seconds have elapsed since its last job.
 // It returns true iff a job is started.
 func (repo *Repo) RunIfScheduled() bool {
-	// sanity check; don't run if a job is already running
+	// don't run if a job is already running
-	if repo.state.IsRunning {
+	if repo.State.IsRunning {
 		return false
 	}
-	if time.Now().Unix()-repo.state.LastAttemptTime > int64(repo.Frequency) {
+
-		// this should be set in the caller's thread so that the check
+	// this should be set in the caller's thread so that the if will work
-		// above will always work
+	curTime := time.Now().Unix()
-		repo.state.IsRunning = true
+	if curTime-repo.State.LastAttemptTime > int64(repo.Frequency) {
+		repo.State.IsRunning = true
+		repo.State.LastAttemptTime = curTime
+		repo.SaveState()
 		go repo.StartSyncJob()
 		return true
 	}
 	return false
 }
 
+// save the save the current state of a repo to a file
+func (repo *Repo) SaveState() {
+	state_cfg := ini.Empty()
+	if err := ini.ReflectFrom(state_cfg, &repo.State); err != nil {
+		repo.Logger.Error(err)
+	}
+	file, err := os.OpenFile(repo.cfg.StateDir+"/"+repo.Name, os.O_RDWR|os.O_CREATE, 0644)
+	if err != nil {
+		repo.Logger.Error(err)
+	}
+	if _, err := state_cfg.WriteTo(file); err != nil {
+		repo.Logger.Error(err)
+	}
+}
+
+// update the repo state with the last attempt time and exit now that the job is done
+// TODO: rename and reorginize
+// TODO: method before and after sync job
+func (repo *Repo) JobDone(exit int) {
+	repoState := repo.State
+	repoState.IsRunning = false
+	repoState.LastAttemptExit = exit
+	repoState.LastAttemptTime = time.Now().Unix() - repoState.LastAttemptTime
+	repo.SaveState()
+}
+
 // GetConfig reads the config from a JSON file, initializes default values,
 // and initializes the non-configurable fields of each repo.
 // It returns a Config.
 func GetConfig() Config {
+	// add global configuration in cfg
 	data, err := ini.Load(CONFIG_PATH)
 	if err != nil {
 		panic(err)
@@ -154,69 +190,74 @@ func GetConfig() Config {
 	cfg := Config{
 		MaxJobs:     DEFAULT_MAX_JOBS,
 		MaxTime:     DEFAULT_MAX_TIME,
-		SockPath:    DEFAULT_SOCK_PATH,
-		StatePath:   DEFAULT_STATE_PATH,
-		LogFile:     DEFAULT_LOG_FILE,
 		PasswordDir: DEFAULT_PASSWORD_DIR,
 		DownloadDir: DEFAULT_DOWNLOAD_DIR,
+		LogDir:      DEFAULT_LOG_DIR,
+		StateDir:    DEFAULT_STATE_DIR,
+		SockPath:    DEFAULT_SOCK_PATH,
 		Repos:       make([]*Repo, 0),
 	}
 	if err := data.MapTo(&cfg); err != nil {
 		panic(err)
 	}
 
-	if err := os.MkdirAll(cfg.StatePath, 0755); err != nil {
+	// validate global configuration
-		panic("Could not create states path at " + cfg.StatePath)
+	if err := os.MkdirAll(cfg.LogDir, 0755); err != nil {
+		panic("Could not create log path at " + cfg.LogDir)
+	} else if err := os.MkdirAll(cfg.StateDir, 0755); err != nil {
+		panic("Could not create states path at " + cfg.StateDir)
 	} else if cfg.IPv4Address == "" {
 		panic("Missing IPv4 address from config")
 	} else if cfg.IPv6Address == "" {
 		panic("Missing IPv6 address from config")
 	}
 
-	doneChan := make(chan string)
+	// add each repo configuration to cfg
+	doneChan := make(chan Result)
 	cfg.DoneChan = doneChan
 	for _, section := range data.Sections() {
 		if section.Name() == "DEFAULT" {
 			continue
 		}
-		repo := Repo{Name: section.Name()}
+
+		// create the repo configuration
+		repo := Repo{
+			Name:         section.Name(),
+			SyncType:     cfg.SyncType,
+			FrequencyStr: cfg.FrequencyStr,
+			MaxTime:      cfg.MaxTime,
+		}
 		if err := section.MapTo(&repo); err != nil {
 			panic(err)
 		}
-
 		repo.Frequency = frequencies[repo.FrequencyStr]
-		if repo.MaxTime == 0 {
+		if repo.SyncType == "" {
-			repo.MaxTime = cfg.MaxTime
+			panic("Missing sync type from " + repo.Name)
+		} else if repo.Frequency == 0 {
+			panic("Missing frequency from " + repo.Name)
 		}
 		repo.Logger = NewLogger(repo.Name)
 		repo.DoneChan = doneChan
 		repo.StopChan = make(chan bool, 1)
 		repo.cfg = &cfg
 
-		repo.state = State{
+		// create the default repo state configuration from a file
+		repoStateFile := cfg.StateDir + "/" + repo.Name
+		repo.State = RepoState{
+			IsRunning:          false,
+			LastAttemptExit:    NOT_RUN_YET,
 			LastAttemptTime:    0,
 			LastAttemptRunTime: 0,
-			LastAttemptStatus:  NOT_RUN_YET,
-			IsRunning:          false,
 		}
-
+		// when repoStatusPath does not exist, create it and write the default state
-		repo_state_file := cfg.StatePath + "/" + repo.Name
+		// overwise overwrite repo.state
-
+		if _, err := os.Stat(repoStateFile); err != nil {
-		if _, err := os.Stat(repo_state_file); err != nil {
+			repo.SaveState()
-			// when repo_status_path does not exist then save the default config
+		} else if err := ini.MapTo(&repo.State, repoStateFile); err != nil {
-			repo_state := ini.Empty()
-			ini.ReflectFrom(repo_state, &repo.state)
-			file, err := os.OpenFile(repo_state_file, os.O_RDWR|os.O_CREATE, 0644)
-			if err != nil {
-				panic(err)
-			}
-			if _, err := repo_state.WriteTo(file); err != nil {
-				panic(err)
-			}
-		} else if err := ini.MapTo(&repo.state, repo_state_file); err != nil {
-			// when it does exist attempt to the grab its configs
 			panic(err)
 		}
+		// must be initially not running, otherwise will never run
+		repo.State.IsRunning = false
 
 		cfg.Repos = append(cfg.Repos, &repo)
 	}
@@ -225,17 +266,3 @@ func GetConfig() Config {
 	}
 	return cfg
 }
-
-func (repo *Repo) SaveState() {
-	cfg := ini.Empty()
-	if err := ini.ReflectFrom(cfg, &repo.state); err != nil {
-		repo.Logger.Error(err)
-	}
-	file, err := os.OpenFile(repo.cfg.StatePath+"/"+repo.Name, os.O_RDWR|os.O_CREATE, 0644)
-	if err != nil {
-		repo.Logger.Error(err)
-	}
-	if _, err := cfg.WriteTo(file); err != nil {
-		repo.Logger.Error(err)
-	}
-}

merlin/common/process_manager.go

@@ -14,6 +14,7 @@ import (
 // It returns a channel through which a Cmd will be sent once it has finished,
 // or nil if it was unable to start a process.
 func SpawnProcess(repo *Repo, args []string) (ch <-chan *exec.Cmd) {
+	// TODO change stdout and stderr to something else
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
@@ -25,6 +26,7 @@ func SpawnProcess(repo *Repo, args []string) (ch <-chan *exec.Cmd) {
 		repo.Logger.Error(err)
 		return
 	}
+
 	cmdChan := make(chan *exec.Cmd)
 	ch = cmdChan
 	procDoneChan := make(chan bool, 1)
@@ -42,6 +44,7 @@ func SpawnProcess(repo *Repo, args []string) (ch <-chan *exec.Cmd) {
 			repo.Logger.Debug("Process has stopped.")
 		}
 	}
+
 	go func() {
 		cmd.Wait()
 		procDoneChan <- true

merlin/common/sync.go

@@ -3,7 +3,6 @@ package common
 import (
 	"fmt"
 	"os"
-	"time"
 )
 
 /*
@@ -15,22 +14,19 @@ func (repo *Repo) buildRsyncHost() string {
 	if repo.RsyncUser != "" {
 		repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost
 	}
-	return "rsync://"+repo.RsyncHost+"/"+repo.RsyncDir
+	return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir
 }
 
 // CSCSyncStandard performs a standard rsync job.
 func (repo *Repo) CSCSyncStandard() {
-	startTime := time.Now().Unix()
 	status := FAILURE
 	// https://medium.com/@manandharsabbir/go-lang-defer-statement-arguments-evaluated-at-defer-execution-b2c4a1687c6c
 	// will defer actaully wait till end to function to set the vars?
 	defer func() {
-		repo.state.LastAttemptTime = startTime
+		repo.DoneChan <- Result{
-		repo.state.LastAttemptRunTime = time.Now().Unix() - startTime
+			Name: repo.Name,
-		repo.state.LastAttemptStatus = status
+			Exit: status,
-		repo.state.IsRunning = false
+		}
-		repo.SaveState()
-		repo.DoneChan <- repo.Name
 	}()
 
 	localDir := repo.cfg.DownloadDir + "/" + repo.LocalDir
@@ -77,11 +73,13 @@ func (repo *Repo) CSCSyncArchLinux() {
 	args := []string{
 		"rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600",
 		"--contimeout=60", "-p", "--delay-updates", "--no-motd",
-		"--temp-dir="+tempDir, "--log-file="+repo.LogFile, "--address="+repo.cfg.IPv4Address,
+		"--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address,
 	}
 
 	ch := SpawnProcess(repo, args)
-	if ch == nil { return }
+	if ch == nil {
+		return
+	}
 
 	cmd := <-ch
 	switch cmd.ProcessState.ExitCode() {
@@ -128,5 +126,6 @@ func (repo *Repo) StartSyncJob() {
 		repo.CSCSyncStandard()
 	default:
 		repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'")
+		return
 	}
 }

merlin/merlin-config-all.ini

@@ -0,0 +1,572 @@
+max_jobs = 6
+
+sock_path = /run/merlin/merlin.sock
+states_path = /home/test-mirror/merlin/states
+
+# log_path? (just spes a dir and make files)
+log_file = /tmp/test-mirror/test.log
+
+ipv4_address = 129.97.134.129
+ipv6_address = 2620:101:f000:4901:c5c::129
+download_dir = /tmp/test-mirror
+
+; 5 hours
+; default_max_time = 18000
+# add default max_time
+# add default frequency
+# add default sync_type
+
+
+
+[debian]
+sync_type = csc-sync-debian
+frequency = bi-hourly
+local_dir = debian
+rsync_host = debian.mirror.rafal.ca
+rsync_dir = debian
+
+; [debian-cdimage]
+; sync_type = csc-sync-cdimage
+; frequency = twice-daily
+; local_dir = debian-cdimage
+; rsync_host = cdimage.debian.org
+; rsync_dir = cdimage
+
+[ubuntu]
+sync_type = csc-sync-debian
+frequency = bi-hourly
+local_dir = ubuntu
+rsync_host = archive.ubuntu.com
+rsync_dir = ubuntu
+; trace_host = drescher.canonical.com
+
+[ubuntu-ports]
+sync_type = csc-sync-debian
+frequency = bi-hourly
+local_dir = ubuntu-ports
+rsync_host = ports.ubuntu.com
+rsync_dir = ubuntu-ports
+; trace_host = drescher.canonical.com
+
+[linuxmint-packages]
+sync_type = csc-sync-debian
+frequency = bi-hourly
+local_dir = linuxmint-packages
+rsync_host = rsync-packages.linuxmint.com
+rsync_dir = packages
+
+[debian-multimedia]
+sync_type = csc-sync-debian
+frequency = bi-hourly
+local_dir = debian-multimedia
+rsync_host = www.deb-multimedia.org
+rsync_dir = deb
+
+[debian-backports]
+sync_type = csc-sync-debian
+frequency = bi-hourly
+local_dir = debian-backports
+rsync_host = debian.mirror.rafal.ca
+rsync_dir = debian-backports
+
+; [debian-volatile]
+; sync_type = csc-sync-debian
+; frequency = bi-hourly
+; local_dir = debian-volatile
+; rsync_host = debian.mirror.rafal.ca
+; rsync_dir = debian-volatile
+
+[debian-security]
+sync_type = csc-sync-debian
+frequency = twice-hourly
+local_dir = debian-security
+rsync_host = rsync.security.debian.org
+rsync_dir = debian-security
+; trace_host = security-master.debian.org
+
+[ubuntu-releases]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = ubuntu-releases
+rsync_host = rsync.releases.ubuntu.com
+rsync_dir = releases
+
+[xubuntu-releases]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = xubuntu-releases
+rsync_host = cdimage.ubuntu.com
+rsync_dir = cdimage/xubuntu/releases/
+
+; [emdebian]
+; sync_type = csc-sync-badperms
+; frequency = twice-daily
+; local_dir = emdebian
+; rsync_host = www.emdebian.org
+; rsync_dir = debian
+
+[puppylinux]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = puppylinux
+rsync_host = distro.ibiblio.org
+rsync_dir = puppylinux
+
+[CPAN]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = CPAN
+rsync_host = cpan-rsync.perl.org
+rsync_dir = CPAN
+
+[CRAN]
+sync_type = csc-sync-ssh
+frequency = twice-daily
+local_dir = CRAN
+rsync_host = cran.r-project.org 
+rsync_dir = "" 
+; rsync_user = cran-rsync ~/.ssh/id_cran_rsa
+; password_file = ~/.ssh/id_cran_rsa
+
+[CTAN]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = CTAN
+rsync_host = rsync.dante.ctan.org
+rsync_dir = CTAN
+
+; [openoffice]
+; sync_type = csc-sync-standard
+; frequency = twice-daily
+; local_dir = openoffice
+; rsync_host = rsync.services.openoffice.org
+; rsync_host = ftp.snt.utwente.nl
+; rsync_dir = openoffice-extended
+
+[fedora-epel]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = fedora/epel
+rsync_host = mirrors.kernel.org
+rsync_dir = fedora-epel
+; ~/bin/report_mirror > /dev/null
+
+[cygwin]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = cygwin
+rsync_host = cygwin.com
+rsync_dir = cygwin-ftp
+
+[gnu]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = gnu
+; rsync_host = mirrors.ibiblio.org
+; rsync_dir = gnuftp/gnu/
+rsync_host = ftp.gnu.org
+rsync_dir = gnu
+
+[nongnu]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = nongnu
+rsync_host = dl.sv.gnu.org
+rsync_dir = releases
+
+[mysql]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = mysql
+; rsync_host = mysql.he.net
+; rsync_dir = mysql
+rsync_host = rsync.mirrorservice.org
+rsync_dir = ftp.mysql.com
+
+; No longer syncs, and no longer really relevant
+; [mozdev]
+; sync_type = csc-sync-standard
+; frequency = twice-daily
+; local_dir = mozdev
+; rsync_host = rsync.mozdev.org
+; rsync_dir = mozdev
+
+[gnome]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = gnome
+rsync_host = master.gnome.org
+rsync_dir = gnomeftp
+; password_file = gnome
+
+[damnsmalllinux]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = damnsmalllinux
+rsync_host = ftp.heanet.ie
+rsync_dir = mirrors/damnsmalllinux.org/
+
+[linuxmint]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = linuxmint
+rsync_host = pub.linuxmint.com
+rsync_dir = pub
+
+[kernel.org-linux]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = kernel.org/linux
+rsync_host = rsync.kernel.org
+rsync_dir = pub/linux/
+
+[kernel.org-software]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = kernel.org/software
+rsync_host = rsync.kernel.org
+rsync_dir = pub/software/
+
+[apache]
+sync_type = csc-sync-apache
+frequency = twice-daily
+local_dir = apache
+rsync_host = rsync.us.apache.org
+rsync_dir = apache-dist
+
+[eclipse]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = eclipse
+rsync_host = download.eclipse.org
+rsync_dir = eclipseMirror
+
+[kde]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = kde
+rsync_host = rsync.kde.org
+rsync_dir = kdeftp
+
+[kde-applicationdata]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = kde-applicationdata
+rsync_host = rsync.kde.org
+rsync_dir = applicationdata
+
+; We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853)
+; so our IP is important.
+[archlinux]
+; csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux
+sync_type = csc-sync-archlinux
+frequency = five-minutely
+local_dir = archlinux
+
+[debian-ports]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = debian-ports
+rsync_host = ftp.de.debian.org
+rsync_dir = debian-ports
+
+[slackware]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = slackware
+rsync_host = slackware.cs.utah.edu
+rsync_dir = slackware
+
+[debian-cd]
+sync_type = csc-sync-debian-cd
+frequency = twice-daily
+
+[x.org]
+; csc-sync-standard x.org xorg.freedesktop.org xorg-archive
+; csc-sync-standard x.org mirror.us.leaseweb.net xorg
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = x.org
+rsync_host = rsync.mirrorservice.org
+rsync_dir = ftp.x.org/pub
+
+[centos]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = centos
+rsync_host = us-msync.centos.org
+rsync_dir = CentOS
+
+[opensuse]
+; "--exclude distribution/.timestamp_invisible"
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = opensuse
+rsync_host = stage.opensuse.org
+rsync_dir = opensuse-full/opensuse/
+
+[FreeBSD]
+; Has not updated since at least June 2018
+; csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD
+sync_type = csc-sync-standard
+; csc-sync-standard FreeBSD ftp3.us.freebsd.org FreeBSD/
+frequency = twice-daily
+local_dir = FreeBSD
+rsync_host = ftp2.uk.freebsd.org
+rsync_dir = ftp.freebsd.org/pub/FreeBSD/
+
+[fedora-enchilada]
+; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = fedora/linux
+rsync_host = mirrors.kernel.org
+rsync_dir = fedora-enchilada/linux/
+; ~/bin/report_mirror >/dev/null
+
+[ubuntu-ports-releases]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = ubuntu-ports-releases
+rsync_host = cdimage.ubuntu.com
+rsync_dir = cdimage/releases/
+
+[gentoo-distfiles]
+sync_type = csc-sync-gentoo
+frequency = bi-hourly
+
+[gentoo-portage]
+sync_type = csc-sync-standard
+frequency = twice_hourly
+local_dir = gentoo-portage
+rsync_host = rsync1.us.gentoo.org
+rsync_dir = gentoo-portage
+
+; This project is no longer available for mirroring
+; https://bugzilla.mozilla.org/show_bug.cgi?id=807543
+; [mozilla.org]
+;    csc-sync-standard mozilla.org releases-rsync.mozilla.org mozilla-releases
+;    frequency = twice_hourly
+
+[gutenberg]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = gutenberg
+rsync_host = ftp@ftp.ibiblio.org
+rsync_dir = gutenberg
+
+; I donno what csc-sync-wget is doing
+[racket-installers]
+sync_type = csc-sync-wget
+frequency = twice-daily
+local_dir = racket/racket-installers
+; path = https://mirror.racket-lang.org/installers/
+; cut = 1
+
+[plt-bundles]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = racket/plt-bundles
+rsync_host = mirror.racket-lang.org
+rsync_dir = plt-bundles
+
+[OpenBSD]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = OpenBSD
+rsync_host = ftp3.usa.openbsd.org
+rsync_dir = ftp
+
+[xiph]
+; csc-sync-standard xiph downloads.xiph.org xiph/releases
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = xiph
+rsync_host = ftp.osuosl.org
+rsync_dir = xiph
+
+; We currently don't have the disk space
+[netbsd]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = NetBSD
+rsync_host = rsync.netbsd.org
+rsync_dir = NetBSD
+
+[netbsd-pkgsrc]
+sync_type = csc-sync-standard
+; csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/
+frequency = twice-daily
+local_dir = pkgsrc
+rsync_host = rsync.netbsd.org
+rsync_dir = pkgsrc
+
+[macports-release]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = MacPorts/release
+rsync_host = rsync.macports.org
+rsync_dir = macports/release/
+
+[macports-distfiles]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = MacPorts/mpdistfiles
+rsync_host = rsync.macports.org
+rsync_dir = macports/distfiles/
+
+; [raspberrypi]
+; sync_type = csc-sync-standard
+; frequency = twice-daily
+; local_dir = raspberrypi
+; rsync_host = mirrors.rit.edu
+; rsync_dir = rpi
+
+[sagemath]
+; csc-sync-standard sage mirror.clibre.uqam.ca sage
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = sage
+rsync_host = rsync.sagemath.org
+rsync_dir = sage
+
+; [cs136]
+; csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01
+; frequency = hourly
+
+[vlc]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = vlc
+rsync_host = rsync.videolan.org
+rsync_dir = videolan-ftp
+
+[qtproject]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = qtproject
+rsync_host = master.qt.io
+rsync_dir = qt-all
+
+[tdf]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = tdf
+rsync_host = rsync.documentfoundation.org
+rsync_dir = tdf-pub
+
+[saltstack]
+sync_type = csc-sync-s3
+frequency = daily
+local_dir = saltstack
+; endpoint = https://s3.repo.saltproject.io
+
+; [kali]
+; csc-sync-standard kali kali.mirror.globo.tech kali
+; frequency = twice-daily
+
+; [kali-images]
+; csc-sync-standard kali-images kali.mirror.globo.tech kali-images
+; frequency = twice-daily
+
+[alpine]
+sync_type = csc-sync-standard
+frequency = hourly
+local_dir = alpine
+rsync_host = rsync.alpinelinux.org
+rsync_dir = alpine
+
+[raspbian]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = raspbian
+rsync_host = archive.raspbian.org
+rsync_dir = archive
+
+[raspberrypi]
+sync_type = csc-sync-standard-ipv6
+frequency = bi-hourly
+local_dir = raspberrypi
+rsync_host = apt-repo.raspberrypi.org
+rsync_dir = archive
+
+[ipfire]
+sync_type = csc-sync-standard
+frequency = hourly
+local_dir = ipfire
+rsync_host = rsync.ipfire.org
+rsync_dir = full
+
+[manjaro]
+sync_type = csc-sync-standard
+frequency = hourly
+local_dir = manjaro
+rsync_host = mirrorservice.org
+rsync_dir = repo.manjaro.org/repos/
+
+[scientific]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = scientific
+rsync_host = rsync.scientificlinux.org
+rsync_dir = scientific
+
+[mxlinux]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = mxlinux
+rsync_host = mirror.math.princeton.edu
+rsync_dir = pub/mxlinux/
+
+[mxlinux-iso]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = mxlinux-iso
+rsync_host = mirror.math.princeton.edu
+rsync_dir = pub/mxlinux-iso/
+
+
+[parabola]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = parabola
+rsync_host = repo.parabola.nu:875
+rsync_dir = repos/
+
+; [hyperbola-sources]
+; csc-sync-chmod hyperbola/sources repo.hyperbola.info:52000 repo/
+; frequency = twice-daily
+
+; [hyperbola-stable]
+; csc-sync-chmod hyperbola/gnu-plus-linux-libre/stable repo.hyperbola.info:52012 repo/
+; frequency = twice-daily
+
+; [hyperbola-testing]
+; csc-sync-chmod hyperbola/gnu-plus-linux-libre/testing repo.hyperbola.info:52011 repo/
+; frequency = twice-daily
+
+[trisquel-packages]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = trisquel/packages
+rsync_host = rsync.trisquel.info
+rsync_dir = trisquel.packages/
+
+[trisquel-iso]
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = trisquel/iso
+rsync_host = rsync.trisquel.info
+rsync_dir = trisquel.iso/
+
+[almalinux]
+sync_type = csc-sync-standard
+frequency = bi-hourly
+local_dir = almalinux
+rsync_host = rsync.repo.almalinux.org
+rsync_dir = almalinux/
+
+; donno what is happening here
+[ceph]
+; csc-sync-ceph -q -s global -t ceph
+frequency = tri-hourly

merlin/merlin.go

@@ -2,9 +2,10 @@ package main
 
 import (
 	"fmt"
+	"time"
+
 	"git.csclub.uwaterloo.ca/public/merlin/common"
 	"golang.org/x/sys/unix"
-	"time"
 )
 
 func main() {
@@ -15,7 +16,12 @@ func main() {
 
 	unix.Umask(002)
 
+	// TODO: add debug log in main that job has started
 	repos := cfg.Repos
+	repoMap := make(map[string]*common.Repo)
+	for _, repo := range repos {
+		repoMap[repo.Name] = repo
+	}
 	doneChan := cfg.DoneChan
 	repoIdx := 0
 	numJobsRunning := 0
@@ -26,7 +32,8 @@ func main() {
 		startIdx := repoIdx
 		for numJobsRunning < cfg.MaxJobs {
 			repo := repos[repoIdx]
-			if !repo.IsRunning() && repo.RunIfScheduled() {
+			// attempt to run repo and increment when a job is started
+			if repo.RunIfScheduled() {
 				numJobsRunning++
 			}
 			repoIdx = (repoIdx + 1) % len(repos)
@@ -36,10 +43,13 @@ func main() {
 			}
 		}
 	}
+
 	runAsManyAsPossible()
 	for {
 		select {
-		case <-doneChan:
+		case result := <-doneChan:
+			// move this into a method in common.go
+			repoMap[result.Name].JobDone(result.Exit)
 			numJobsRunning--
 		case <-time.After(1 * time.Minute):
 		}

merlin/ubuntu-releases

@@ -1,4 +0,0 @@
-; move to states folder
-last_run = 0
-runtime = 0
-status = 0