diff --git a/merlin/README.md b/merlin/README.md index f5b7d73..2637a70 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -1,19 +1,16 @@ This folder contains the code for merlin (which does the actual syncing) and arthur (which sends commands to merlin). -## Checklist -- [x] write process manager +### In Progress - [ ] implement all sync types (csc-sync-debian, csc-sync-apache, etc.) -- [ ] save state (last attempted time, last attempted status) for each repo, - and restore state on startup (e.g. use JSON/INI file for each repo) -- [ ] use separate log file for each child process (currently sharing stdout/stderr with parent) -- [ ] calculate difference between the scheduled time of a job and the time at which it actually - ran; log this -- [ ] add all repos to merlin-config.ini +- [ ] use separate log file for each child process (currently sharing stdout/stderr with parent) (does this mean different from repo?) + - different log dir for each repo? +- [ ] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this + +### TODO - [ ] listen on Unix socket in merlin - [ ] implement arthur.go (commands: sync and status) - [ ] implement zfssync in merlin (just invoke the existing Python script) -- [ ] handle termination signals in merlin (SIGINT, SIGTERM); close stopChan for this - [ ] allow dynamic reloading in merlin (\*) - [ ] detect if an rsync process is stuck (\*\*) - [ ] place each rsync process in a separate cgroup (\*\*\*) @@ -26,3 +23,11 @@ as the reload signal. stdout/stderr of the rsync process. \*\*\* I need to do more research into this - don't worry about it for now. + +### Completed +- [x] write process manager +- [x] save state (last attempted time, last attempted status) for each repo, and restore state on startup (e.g. use JSON/INI file for each repo) +- [x] calculate difference between the scheduled time of a job and the time at which it actually ran; log this +- [x] add all repos to merlin-config.ini (\*) + +\* there are some parts that I don't understand (trace_host, csc-sync-ceph, csc-sync-saltstack, etc) \ No newline at end of file diff --git a/merlin/common/common.go b/merlin/common/common.go index 4a8adfc..0394126 100644 --- a/merlin/common/common.go +++ b/merlin/common/common.go @@ -11,9 +11,9 @@ const ( DAILY = 86400 TWICE_DAILY = DAILY / 2 HOURLY = 3600 + TWICE_HOURLY = HOURLY / 2 BI_HOURLY = HOURLY * 2 TRI_HOURLY = HOURLY * 3 - TWICE_HOURLY = HOURLY / 2 TEN_MINUTELY = 600 FIVE_MINUTELY = 300 @@ -21,20 +21,20 @@ const ( DEFAULT_MAX_JOBS = 6 DEFAULT_MAX_TIME = DAILY / 4 - DEFAULT_SOCK_PATH = "/run/merlin/merlin.sock" - DEFAULT_STATE_PATH = "/home/mirror/merlin/states" - DEFAULT_LOG_FILE = "/home/mirror/merlin/logs/transfer.log" - DEFAULT_PASSWORD_DIR = "/home/mirror/passwords" DEFAULT_DOWNLOAD_DIR = "/mirror/root" + DEFAULT_PASSWORD_DIR = "/home/mirror/passwords" + DEFAULT_LOG_DIR = "/home/mirror/merlin/logs" + DEFAULT_STATE_DIR = "/home/mirror/merlin/states" + DEFAULT_SOCK_PATH = "/run/merlin/merlin.sock" ) var frequencies = map[string]int{ "daily": DAILY, "twice-daily": TWICE_DAILY, "hourly": HOURLY, + "twice-hourly": TWICE_HOURLY, "bi-hourly": BI_HOURLY, "tri-hourly": TRI_HOURLY, - "twice-hourly": TWICE_HOURLY, "ten-minutely": TEN_MINUTELY, "five-minutely": FIVE_MINUTELY, } @@ -43,15 +43,24 @@ var frequencies = map[string]int{ const ( NOT_RUN_YET = iota SUCCESS - FAILURE // change to TIMEOUT? + FAILURE TERMINATED // was killed by a signal ) +type Result struct { + Name string + Exit int +} + type Repo struct { + // the name of this repo + Name string `ini:"-"` // this should be one of "csc-sync-standard", etc. SyncType string `ini:"sync_type"` // a human-readable frequency, e.g. "bi-hourly" FrequencyStr string `ini:"frequency"` + // the desired interval (in seconds) between successive runs + Frequency int `ini:"-"` // the maximum time (in seconds) that each child process of this repo // can for before being killed MaxTime int `ini:"max_time"` @@ -68,85 +77,112 @@ type Repo struct { PasswordFile string `ini:"password_file"` // the log file for rsync (optional, defaults to Config.LogFile) LogFile string `ini:"log_file"` - // the desired interval (in seconds) between successive runs - Frequency int `ini:"-"` - // the name of this repo - Name string `ini:"-"` + // a reference to the logger Logger *Logger `ini:"-"` + // the repo will write its name and status in a Result struct to DoneChan + // when it has finished a job + DoneChan chan<- Result `ini:"-"` // the repo should stop syncing if StopChan is closed StopChan <-chan bool `ini:"-"` - // the repo will write its name to DoneChan when it has finished a job - // (successfully or unsuccessfully) - DoneChan chan<- string `ini:"-"` + // a struct that stores the repo's status + State RepoState `ini:"-"` // a reference to the global config cfg *Config `ini:"-"` - // a struct that stores the repo's status - state State `ini:"-"` } type Config struct { // the maximum number of jobs allowed to execute concurrently MaxJobs int `ini:"max_jobs"` - // the default MaxTime for each repo - MaxTime int `ini:"max_time"` - // the Unix socket path which arthur will use to communicate with us - SockPath string `ini:"sock_path"` - // the path to where the state of each repo's sync is saved - StatePath string `ini:"states_path"` - // the default LogFile for each repo - LogFile string `ini:"log_file"` // the IP addresses to use for rsync IPv4Address string `ini:"ipv4_address"` IPv6Address string `ini:"ipv6_address"` - // the directory where rsync passwords are stored - PasswordDir string `ini:"password_dir"` + // the default sync type + SyncType string `ini:"default_sync_type"` + // the default frequency string for the repos + FrequencyStr string `ini:"default_frequency"` + // the default MaxTime for each repo + MaxTime int `ini:"default_max_time"` // the directory where rsync should download files DownloadDir string `ini:"download_dir"` + // the directory where rsync passwords are stored + PasswordDir string `ini:"password_dir"` + // the path where merlin will store the logs for each repo synced + LogDir string `ini:"log_dir"` + // the path to where the state of each repo's sync is saved + StateDir string `ini:"states_dir"` + // the Unix socket path which arthur will use to communicate with us + SockPath string `ini:"sock_path"` // the DoneChan for each repo (shared instance) - DoneChan <-chan string `ini:"-"` + DoneChan <-chan Result `ini:"-"` // a list of all of the repos Repos []*Repo `ini:"-"` } -type State struct { +// This should only be modified by the main thread +type RepoState struct { // these are stored in the states folder + // whether this repo is running a job or not + IsRunning bool `ini:"is_running"` + // whether the last attempt was successful or not + LastAttemptExit int `ini:"last_attempt_exit"` // the Unix epoch timestamp at which this repo last attempted a job LastAttemptTime int64 `ini:"last_attempt_time"` // the number of seconds this repo ran for during its last attempted job LastAttemptRunTime int64 `ini:"last_attempt_runtime"` - // whether the last attempt was successful or not - LastAttemptStatus int `ini:"last_attempt_status"` - // whether this repo is running a job or not - IsRunning bool `ini:"is_running"` -} - -// IsRunning returns true if the repo is currently running a sync job. -func (repo *Repo) IsRunning() bool { - return repo.state.IsRunning } // RunIfScheduled starts a sync job for this repo if more than repo.Frequency // seconds have elapsed since its last job. // It returns true iff a job is started. func (repo *Repo) RunIfScheduled() bool { - // sanity check; don't run if a job is already running - if repo.state.IsRunning { + // don't run if a job is already running + if repo.State.IsRunning { return false } - if time.Now().Unix()-repo.state.LastAttemptTime > int64(repo.Frequency) { - // this should be set in the caller's thread so that the check - // above will always work - repo.state.IsRunning = true + + // this should be set in the caller's thread so that the if will work + curTime := time.Now().Unix() + if curTime-repo.State.LastAttemptTime > int64(repo.Frequency) { + repo.State.IsRunning = true + repo.State.LastAttemptTime = curTime + repo.SaveState() go repo.StartSyncJob() return true } return false } +// save the save the current state of a repo to a file +func (repo *Repo) SaveState() { + state_cfg := ini.Empty() + if err := ini.ReflectFrom(state_cfg, &repo.State); err != nil { + repo.Logger.Error(err) + } + file, err := os.OpenFile(repo.cfg.StateDir+"/"+repo.Name, os.O_RDWR|os.O_CREATE, 0644) + if err != nil { + repo.Logger.Error(err) + } + if _, err := state_cfg.WriteTo(file); err != nil { + repo.Logger.Error(err) + } +} + +// update the repo state with the last attempt time and exit now that the job is done +// TODO: rename and reorginize +// TODO: method before and after sync job +func (repo *Repo) JobDone(exit int) { + repoState := repo.State + repoState.IsRunning = false + repoState.LastAttemptExit = exit + repoState.LastAttemptTime = time.Now().Unix() - repoState.LastAttemptTime + repo.SaveState() +} + // GetConfig reads the config from a JSON file, initializes default values, // and initializes the non-configurable fields of each repo. // It returns a Config. func GetConfig() Config { + // add global configuration in cfg data, err := ini.Load(CONFIG_PATH) if err != nil { panic(err) @@ -154,69 +190,74 @@ func GetConfig() Config { cfg := Config{ MaxJobs: DEFAULT_MAX_JOBS, MaxTime: DEFAULT_MAX_TIME, - SockPath: DEFAULT_SOCK_PATH, - StatePath: DEFAULT_STATE_PATH, - LogFile: DEFAULT_LOG_FILE, PasswordDir: DEFAULT_PASSWORD_DIR, DownloadDir: DEFAULT_DOWNLOAD_DIR, + LogDir: DEFAULT_LOG_DIR, + StateDir: DEFAULT_STATE_DIR, + SockPath: DEFAULT_SOCK_PATH, Repos: make([]*Repo, 0), } if err := data.MapTo(&cfg); err != nil { panic(err) } - if err := os.MkdirAll(cfg.StatePath, 0755); err != nil { - panic("Could not create states path at " + cfg.StatePath) + // validate global configuration + if err := os.MkdirAll(cfg.LogDir, 0755); err != nil { + panic("Could not create log path at " + cfg.LogDir) + } else if err := os.MkdirAll(cfg.StateDir, 0755); err != nil { + panic("Could not create states path at " + cfg.StateDir) } else if cfg.IPv4Address == "" { panic("Missing IPv4 address from config") } else if cfg.IPv6Address == "" { panic("Missing IPv6 address from config") } - doneChan := make(chan string) + // add each repo configuration to cfg + doneChan := make(chan Result) cfg.DoneChan = doneChan for _, section := range data.Sections() { if section.Name() == "DEFAULT" { continue } - repo := Repo{Name: section.Name()} + + // create the repo configuration + repo := Repo{ + Name: section.Name(), + SyncType: cfg.SyncType, + FrequencyStr: cfg.FrequencyStr, + MaxTime: cfg.MaxTime, + } if err := section.MapTo(&repo); err != nil { panic(err) } - repo.Frequency = frequencies[repo.FrequencyStr] - if repo.MaxTime == 0 { - repo.MaxTime = cfg.MaxTime + if repo.SyncType == "" { + panic("Missing sync type from " + repo.Name) + } else if repo.Frequency == 0 { + panic("Missing frequency from " + repo.Name) } repo.Logger = NewLogger(repo.Name) repo.DoneChan = doneChan repo.StopChan = make(chan bool, 1) repo.cfg = &cfg - repo.state = State{ + // create the default repo state configuration from a file + repoStateFile := cfg.StateDir + "/" + repo.Name + repo.State = RepoState{ + IsRunning: false, + LastAttemptExit: NOT_RUN_YET, LastAttemptTime: 0, LastAttemptRunTime: 0, - LastAttemptStatus: NOT_RUN_YET, - IsRunning: false, } - - repo_state_file := cfg.StatePath + "/" + repo.Name - - if _, err := os.Stat(repo_state_file); err != nil { - // when repo_status_path does not exist then save the default config - repo_state := ini.Empty() - ini.ReflectFrom(repo_state, &repo.state) - file, err := os.OpenFile(repo_state_file, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - panic(err) - } - if _, err := repo_state.WriteTo(file); err != nil { - panic(err) - } - } else if err := ini.MapTo(&repo.state, repo_state_file); err != nil { - // when it does exist attempt to the grab its configs + // when repoStatusPath does not exist, create it and write the default state + // overwise overwrite repo.state + if _, err := os.Stat(repoStateFile); err != nil { + repo.SaveState() + } else if err := ini.MapTo(&repo.State, repoStateFile); err != nil { panic(err) } + // must be initially not running, otherwise will never run + repo.State.IsRunning = false cfg.Repos = append(cfg.Repos, &repo) } @@ -225,17 +266,3 @@ func GetConfig() Config { } return cfg } - -func (repo *Repo) SaveState() { - cfg := ini.Empty() - if err := ini.ReflectFrom(cfg, &repo.state); err != nil { - repo.Logger.Error(err) - } - file, err := os.OpenFile(repo.cfg.StatePath+"/"+repo.Name, os.O_RDWR|os.O_CREATE, 0644) - if err != nil { - repo.Logger.Error(err) - } - if _, err := cfg.WriteTo(file); err != nil { - repo.Logger.Error(err) - } -} diff --git a/merlin/common/process_manager.go b/merlin/common/process_manager.go index 7e6fb36..0236c68 100644 --- a/merlin/common/process_manager.go +++ b/merlin/common/process_manager.go @@ -14,6 +14,7 @@ import ( // It returns a channel through which a Cmd will be sent once it has finished, // or nil if it was unable to start a process. func SpawnProcess(repo *Repo, args []string) (ch <-chan *exec.Cmd) { + // TODO change stdout and stderr to something else cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -25,6 +26,7 @@ func SpawnProcess(repo *Repo, args []string) (ch <-chan *exec.Cmd) { repo.Logger.Error(err) return } + cmdChan := make(chan *exec.Cmd) ch = cmdChan procDoneChan := make(chan bool, 1) @@ -42,6 +44,7 @@ func SpawnProcess(repo *Repo, args []string) (ch <-chan *exec.Cmd) { repo.Logger.Debug("Process has stopped.") } } + go func() { cmd.Wait() procDoneChan <- true diff --git a/merlin/common/sync.go b/merlin/common/sync.go index 68108a0..0356320 100644 --- a/merlin/common/sync.go +++ b/merlin/common/sync.go @@ -3,7 +3,6 @@ package common import ( "fmt" "os" - "time" ) /* @@ -15,22 +14,19 @@ func (repo *Repo) buildRsyncHost() string { if repo.RsyncUser != "" { repo.RsyncHost = repo.RsyncUser + "@" + repo.RsyncHost } - return "rsync://"+repo.RsyncHost+"/"+repo.RsyncDir + return "rsync://" + repo.RsyncHost + "/" + repo.RsyncDir } // CSCSyncStandard performs a standard rsync job. func (repo *Repo) CSCSyncStandard() { - startTime := time.Now().Unix() status := FAILURE // https://medium.com/@manandharsabbir/go-lang-defer-statement-arguments-evaluated-at-defer-execution-b2c4a1687c6c // will defer actaully wait till end to function to set the vars? defer func() { - repo.state.LastAttemptTime = startTime - repo.state.LastAttemptRunTime = time.Now().Unix() - startTime - repo.state.LastAttemptStatus = status - repo.state.IsRunning = false - repo.SaveState() - repo.DoneChan <- repo.Name + repo.DoneChan <- Result{ + Name: repo.Name, + Exit: status, + } }() localDir := repo.cfg.DownloadDir + "/" + repo.LocalDir @@ -77,11 +73,13 @@ func (repo *Repo) CSCSyncArchLinux() { args := []string{ "rsync", "-rtlH", "--safe-links", "--delete-after", "--timeout=600", "--contimeout=60", "-p", "--delay-updates", "--no-motd", - "--temp-dir="+tempDir, "--log-file="+repo.LogFile, "--address="+repo.cfg.IPv4Address, + "--temp-dir=" + tempDir, "--log-file=" + repo.LogFile, "--address=" + repo.cfg.IPv4Address, } ch := SpawnProcess(repo, args) - if ch == nil { return } + if ch == nil { + return + } cmd := <-ch switch cmd.ProcessState.ExitCode() { @@ -128,5 +126,6 @@ func (repo *Repo) StartSyncJob() { repo.CSCSyncStandard() default: repo.Logger.Error("Unrecognized sync type", "'"+repo.SyncType+"'") + return } } diff --git a/merlin/merlin-config-all.ini b/merlin/merlin-config-all.ini new file mode 100644 index 0000000..06ae1c8 --- /dev/null +++ b/merlin/merlin-config-all.ini @@ -0,0 +1,572 @@ +max_jobs = 6 + +sock_path = /run/merlin/merlin.sock +states_path = /home/test-mirror/merlin/states + +# log_path? (just spes a dir and make files) +log_file = /tmp/test-mirror/test.log + +ipv4_address = 129.97.134.129 +ipv6_address = 2620:101:f000:4901:c5c::129 +download_dir = /tmp/test-mirror + +; 5 hours +; default_max_time = 18000 +# add default max_time +# add default frequency +# add default sync_type + + + +[debian] +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = debian +rsync_host = debian.mirror.rafal.ca +rsync_dir = debian + +; [debian-cdimage] +; sync_type = csc-sync-cdimage +; frequency = twice-daily +; local_dir = debian-cdimage +; rsync_host = cdimage.debian.org +; rsync_dir = cdimage + +[ubuntu] +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = ubuntu +rsync_host = archive.ubuntu.com +rsync_dir = ubuntu +; trace_host = drescher.canonical.com + +[ubuntu-ports] +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = ubuntu-ports +rsync_host = ports.ubuntu.com +rsync_dir = ubuntu-ports +; trace_host = drescher.canonical.com + +[linuxmint-packages] +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = linuxmint-packages +rsync_host = rsync-packages.linuxmint.com +rsync_dir = packages + +[debian-multimedia] +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = debian-multimedia +rsync_host = www.deb-multimedia.org +rsync_dir = deb + +[debian-backports] +sync_type = csc-sync-debian +frequency = bi-hourly +local_dir = debian-backports +rsync_host = debian.mirror.rafal.ca +rsync_dir = debian-backports + +; [debian-volatile] +; sync_type = csc-sync-debian +; frequency = bi-hourly +; local_dir = debian-volatile +; rsync_host = debian.mirror.rafal.ca +; rsync_dir = debian-volatile + +[debian-security] +sync_type = csc-sync-debian +frequency = twice-hourly +local_dir = debian-security +rsync_host = rsync.security.debian.org +rsync_dir = debian-security +; trace_host = security-master.debian.org + +[ubuntu-releases] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = ubuntu-releases +rsync_host = rsync.releases.ubuntu.com +rsync_dir = releases + +[xubuntu-releases] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = xubuntu-releases +rsync_host = cdimage.ubuntu.com +rsync_dir = cdimage/xubuntu/releases/ + +; [emdebian] +; sync_type = csc-sync-badperms +; frequency = twice-daily +; local_dir = emdebian +; rsync_host = www.emdebian.org +; rsync_dir = debian + +[puppylinux] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = puppylinux +rsync_host = distro.ibiblio.org +rsync_dir = puppylinux + +[CPAN] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = CPAN +rsync_host = cpan-rsync.perl.org +rsync_dir = CPAN + +[CRAN] +sync_type = csc-sync-ssh +frequency = twice-daily +local_dir = CRAN +rsync_host = cran.r-project.org +rsync_dir = "" +; rsync_user = cran-rsync ~/.ssh/id_cran_rsa +; password_file = ~/.ssh/id_cran_rsa + +[CTAN] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = CTAN +rsync_host = rsync.dante.ctan.org +rsync_dir = CTAN + +; [openoffice] +; sync_type = csc-sync-standard +; frequency = twice-daily +; local_dir = openoffice +; rsync_host = rsync.services.openoffice.org +; rsync_host = ftp.snt.utwente.nl +; rsync_dir = openoffice-extended + +[fedora-epel] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = fedora/epel +rsync_host = mirrors.kernel.org +rsync_dir = fedora-epel +; ~/bin/report_mirror > /dev/null + +[cygwin] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = cygwin +rsync_host = cygwin.com +rsync_dir = cygwin-ftp + +[gnu] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = gnu +; rsync_host = mirrors.ibiblio.org +; rsync_dir = gnuftp/gnu/ +rsync_host = ftp.gnu.org +rsync_dir = gnu + +[nongnu] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = nongnu +rsync_host = dl.sv.gnu.org +rsync_dir = releases + +[mysql] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = mysql +; rsync_host = mysql.he.net +; rsync_dir = mysql +rsync_host = rsync.mirrorservice.org +rsync_dir = ftp.mysql.com + +; No longer syncs, and no longer really relevant +; [mozdev] +; sync_type = csc-sync-standard +; frequency = twice-daily +; local_dir = mozdev +; rsync_host = rsync.mozdev.org +; rsync_dir = mozdev + +[gnome] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = gnome +rsync_host = master.gnome.org +rsync_dir = gnomeftp +; password_file = gnome + +[damnsmalllinux] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = damnsmalllinux +rsync_host = ftp.heanet.ie +rsync_dir = mirrors/damnsmalllinux.org/ + +[linuxmint] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = linuxmint +rsync_host = pub.linuxmint.com +rsync_dir = pub + +[kernel.org-linux] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = kernel.org/linux +rsync_host = rsync.kernel.org +rsync_dir = pub/linux/ + +[kernel.org-software] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = kernel.org/software +rsync_host = rsync.kernel.org +rsync_dir = pub/software/ + +[apache] +sync_type = csc-sync-apache +frequency = twice-daily +local_dir = apache +rsync_host = rsync.us.apache.org +rsync_dir = apache-dist + +[eclipse] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = eclipse +rsync_host = download.eclipse.org +rsync_dir = eclipseMirror + +[kde] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = kde +rsync_host = rsync.kde.org +rsync_dir = kdeftp + +[kde-applicationdata] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = kde-applicationdata +rsync_host = rsync.kde.org +rsync_dir = applicationdata + +; We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853) +; so our IP is important. +[archlinux] +; csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux +sync_type = csc-sync-archlinux +frequency = five-minutely +local_dir = archlinux + +[debian-ports] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = debian-ports +rsync_host = ftp.de.debian.org +rsync_dir = debian-ports + +[slackware] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = slackware +rsync_host = slackware.cs.utah.edu +rsync_dir = slackware + +[debian-cd] +sync_type = csc-sync-debian-cd +frequency = twice-daily + +[x.org] +; csc-sync-standard x.org xorg.freedesktop.org xorg-archive +; csc-sync-standard x.org mirror.us.leaseweb.net xorg +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = x.org +rsync_host = rsync.mirrorservice.org +rsync_dir = ftp.x.org/pub + +[centos] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = centos +rsync_host = us-msync.centos.org +rsync_dir = CentOS + +[opensuse] +; "--exclude distribution/.timestamp_invisible" +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = opensuse +rsync_host = stage.opensuse.org +rsync_dir = opensuse-full/opensuse/ + +[FreeBSD] +; Has not updated since at least June 2018 +; csc-sync-standard FreeBSD ftp10.us.freebsd.org FreeBSD +sync_type = csc-sync-standard +; csc-sync-standard FreeBSD ftp3.us.freebsd.org FreeBSD/ +frequency = twice-daily +local_dir = FreeBSD +rsync_host = ftp2.uk.freebsd.org +rsync_dir = ftp.freebsd.org/pub/FreeBSD/ + +[fedora-enchilada] +; csc-sync-standard fedora/linux mirrors.kernel.org fedora-enchilada/linux/ --ignore-errors && ~/bin/report_mirror >/dev/null +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = fedora/linux +rsync_host = mirrors.kernel.org +rsync_dir = fedora-enchilada/linux/ +; ~/bin/report_mirror >/dev/null + +[ubuntu-ports-releases] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = ubuntu-ports-releases +rsync_host = cdimage.ubuntu.com +rsync_dir = cdimage/releases/ + +[gentoo-distfiles] +sync_type = csc-sync-gentoo +frequency = bi-hourly + +[gentoo-portage] +sync_type = csc-sync-standard +frequency = twice_hourly +local_dir = gentoo-portage +rsync_host = rsync1.us.gentoo.org +rsync_dir = gentoo-portage + +; This project is no longer available for mirroring +; https://bugzilla.mozilla.org/show_bug.cgi?id=807543 +; [mozilla.org] +; csc-sync-standard mozilla.org releases-rsync.mozilla.org mozilla-releases +; frequency = twice_hourly + +[gutenberg] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = gutenberg +rsync_host = ftp@ftp.ibiblio.org +rsync_dir = gutenberg + +; I donno what csc-sync-wget is doing +[racket-installers] +sync_type = csc-sync-wget +frequency = twice-daily +local_dir = racket/racket-installers +; path = https://mirror.racket-lang.org/installers/ +; cut = 1 + +[plt-bundles] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = racket/plt-bundles +rsync_host = mirror.racket-lang.org +rsync_dir = plt-bundles + +[OpenBSD] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = OpenBSD +rsync_host = ftp3.usa.openbsd.org +rsync_dir = ftp + +[xiph] +; csc-sync-standard xiph downloads.xiph.org xiph/releases +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = xiph +rsync_host = ftp.osuosl.org +rsync_dir = xiph + +; We currently don't have the disk space +[netbsd] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = NetBSD +rsync_host = rsync.netbsd.org +rsync_dir = NetBSD + +[netbsd-pkgsrc] +sync_type = csc-sync-standard +; csc-sync-standard pkgsrc rsync3.jp.netbsd.org pub/pkgsrc/ +frequency = twice-daily +local_dir = pkgsrc +rsync_host = rsync.netbsd.org +rsync_dir = pkgsrc + +[macports-release] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = MacPorts/release +rsync_host = rsync.macports.org +rsync_dir = macports/release/ + +[macports-distfiles] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = MacPorts/mpdistfiles +rsync_host = rsync.macports.org +rsync_dir = macports/distfiles/ + +; [raspberrypi] +; sync_type = csc-sync-standard +; frequency = twice-daily +; local_dir = raspberrypi +; rsync_host = mirrors.rit.edu +; rsync_dir = rpi + +[sagemath] +; csc-sync-standard sage mirror.clibre.uqam.ca sage +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = sage +rsync_host = rsync.sagemath.org +rsync_dir = sage + +; [cs136] +; csc-sync-ssh uw-coursewear/cs136 linux024.student.cs.uwaterloo.ca /u/cs136/mirror.uwaterloo.ca csc01 ~/.ssh/id_rsa_csc01 +; frequency = hourly + +[vlc] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = vlc +rsync_host = rsync.videolan.org +rsync_dir = videolan-ftp + +[qtproject] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = qtproject +rsync_host = master.qt.io +rsync_dir = qt-all + +[tdf] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = tdf +rsync_host = rsync.documentfoundation.org +rsync_dir = tdf-pub + +[saltstack] +sync_type = csc-sync-s3 +frequency = daily +local_dir = saltstack +; endpoint = https://s3.repo.saltproject.io + +; [kali] +; csc-sync-standard kali kali.mirror.globo.tech kali +; frequency = twice-daily + +; [kali-images] +; csc-sync-standard kali-images kali.mirror.globo.tech kali-images +; frequency = twice-daily + +[alpine] +sync_type = csc-sync-standard +frequency = hourly +local_dir = alpine +rsync_host = rsync.alpinelinux.org +rsync_dir = alpine + +[raspbian] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = raspbian +rsync_host = archive.raspbian.org +rsync_dir = archive + +[raspberrypi] +sync_type = csc-sync-standard-ipv6 +frequency = bi-hourly +local_dir = raspberrypi +rsync_host = apt-repo.raspberrypi.org +rsync_dir = archive + +[ipfire] +sync_type = csc-sync-standard +frequency = hourly +local_dir = ipfire +rsync_host = rsync.ipfire.org +rsync_dir = full + +[manjaro] +sync_type = csc-sync-standard +frequency = hourly +local_dir = manjaro +rsync_host = mirrorservice.org +rsync_dir = repo.manjaro.org/repos/ + +[scientific] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = scientific +rsync_host = rsync.scientificlinux.org +rsync_dir = scientific + +[mxlinux] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = mxlinux +rsync_host = mirror.math.princeton.edu +rsync_dir = pub/mxlinux/ + +[mxlinux-iso] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = mxlinux-iso +rsync_host = mirror.math.princeton.edu +rsync_dir = pub/mxlinux-iso/ + + +[parabola] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = parabola +rsync_host = repo.parabola.nu:875 +rsync_dir = repos/ + +; [hyperbola-sources] +; csc-sync-chmod hyperbola/sources repo.hyperbola.info:52000 repo/ +; frequency = twice-daily + +; [hyperbola-stable] +; csc-sync-chmod hyperbola/gnu-plus-linux-libre/stable repo.hyperbola.info:52012 repo/ +; frequency = twice-daily + +; [hyperbola-testing] +; csc-sync-chmod hyperbola/gnu-plus-linux-libre/testing repo.hyperbola.info:52011 repo/ +; frequency = twice-daily + +[trisquel-packages] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = trisquel/packages +rsync_host = rsync.trisquel.info +rsync_dir = trisquel.packages/ + +[trisquel-iso] +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = trisquel/iso +rsync_host = rsync.trisquel.info +rsync_dir = trisquel.iso/ + +[almalinux] +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = almalinux +rsync_host = rsync.repo.almalinux.org +rsync_dir = almalinux/ + +; donno what is happening here +[ceph] +; csc-sync-ceph -q -s global -t ceph +frequency = tri-hourly diff --git a/merlin/merlin.go b/merlin/merlin.go index 2d534e3..202a90a 100644 --- a/merlin/merlin.go +++ b/merlin/merlin.go @@ -2,9 +2,10 @@ package main import ( "fmt" + "time" + "git.csclub.uwaterloo.ca/public/merlin/common" "golang.org/x/sys/unix" - "time" ) func main() { @@ -15,7 +16,12 @@ func main() { unix.Umask(002) + // TODO: add debug log in main that job has started repos := cfg.Repos + repoMap := make(map[string]*common.Repo) + for _, repo := range repos { + repoMap[repo.Name] = repo + } doneChan := cfg.DoneChan repoIdx := 0 numJobsRunning := 0 @@ -26,7 +32,8 @@ func main() { startIdx := repoIdx for numJobsRunning < cfg.MaxJobs { repo := repos[repoIdx] - if !repo.IsRunning() && repo.RunIfScheduled() { + // attempt to run repo and increment when a job is started + if repo.RunIfScheduled() { numJobsRunning++ } repoIdx = (repoIdx + 1) % len(repos) @@ -36,10 +43,13 @@ func main() { } } } + runAsManyAsPossible() for { select { - case <-doneChan: + case result := <-doneChan: + // move this into a method in common.go + repoMap[result.Name].JobDone(result.Exit) numJobsRunning-- case <-time.After(1 * time.Minute): } diff --git a/merlin/ubuntu-releases b/merlin/ubuntu-releases deleted file mode 100644 index a8e57ef..0000000 --- a/merlin/ubuntu-releases +++ /dev/null @@ -1,4 +0,0 @@ -; move to states folder -last_run = 0 -runtime = 0 -status = 0 \ No newline at end of file diff --git a/merlin/arthur.py b/merlin_old/arthur.py similarity index 100% rename from merlin/arthur.py rename to merlin_old/arthur.py diff --git a/merlin/init-script b/merlin_old/init-script similarity index 100% rename from merlin/init-script rename to merlin_old/init-script diff --git a/merlin/merlin.py b/merlin_old/merlin.py similarity index 100% rename from merlin/merlin.py rename to merlin_old/merlin.py diff --git a/merlin/merlin.service b/merlin_old/merlin.service similarity index 100% rename from merlin/merlin.service rename to merlin_old/merlin.service diff --git a/merlin/rebuild.sh b/merlin_old/rebuild.sh similarity index 100% rename from merlin/rebuild.sh rename to merlin_old/rebuild.sh diff --git a/merlin/test.py b/merlin_old/test.py similarity index 100% rename from merlin/test.py rename to merlin_old/test.py diff --git a/merlin/zfssync.yml b/merlin_old/zfssync.yml similarity index 100% rename from merlin/zfssync.yml rename to merlin_old/zfssync.yml