Move more repos to merlin-go

merlin/Handoff.md

@@ -0,0 +1,63 @@
+## Status of moving from merlin to merlin-go
+- [ ] debian
+- [ ] ubuntu
+- [ ] ubuntu-ports
+- [ ] linuxmint-packages
+- [ ] debian-multimedia
+- [ ] debian-backports
+- [ ] debian-security
+- [ ] ubuntu-releases
+- [ ] xubuntu-releases
+- [ ] puppylinux <
+- [x] CPAN
+- [ ] CRAN
+- [ ] CTAN <
+- [ ] fedora-epel
+- [ ] cygwin <
+- [ ] gnu <
+- [ ] nognu <
+- [ ] mysql <
+- [ ] gnome <
+- [ ] damnsmalllinux <
+- [ ] linuxmint <
+- [ ] kernel.org-linux <
+- [ ] kernel.org-software <
+- [ ] apache <
+- [ ] eclipse <
+- [ ] kde
+- [ ] kde-appplicationdata
+- [ ] archlinux
+- [ ] artixlinux
+- [ ] slackware
+- [ ] debian-cd
+- [ ] x.org
+- [ ] centos
+- [ ] opensuse
+- [ ] FreeBSD
+- [ ] fedora-enchilada
+- [ ] ubuntu-ports-releases
+- [ ] gentoo-distfiles
+- [ ] gentoo-portage
+- [ ] gutenberg
+- [ ] racket-installers
+- [ ] plt-bundles
+- [ ] xiph
+- [ ] netbsd
+- [ ] macports-release
+- [ ] macports-distfiles
+- [ ] sagemath
+- [ ] vlc
+- [ ] tdf
+- [ ] saltstack
+- [ ] alpine
+- [ ] raspbian
+- [ ] raspberrypi
+- [ ] ipfire
+- [ ] manjaro
+- [ ] mxlinux
+- [ ] mxlinux-iso
+- [ ] parabola
+- [ ] trisquel-packages
+- [ ] trisquel-iso
+- [ ] almalinux
+- [ ] ceph

merlin/arthur/arthur.go

@@ -85,7 +85,7 @@ func ForceSync(conn net.Conn, repoName string) (newSync bool) {
 
 	if repo, isInMap := config.RepoMap[repoName]; isInMap {
 		logger.OutLog("Attempting to force sync of " + repoName)
-		if sync.SyncIfPossible(repo) {
+		if sync.SyncIfPossible(repo, true) {
 			conn.Write([]byte("Forced sync for " + repoName))
 			newSync = true
 		} else {

merlin/arthur/arthur_test.go

@@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
 		SyncType:  "csc-sync-dummy",
 		Frequency: 7 * 86400,
 		MaxTime:   30,
-		Logger:    logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
+		Logger:    logger.NewLogger("nux", "/tmp/merlin_force_sync_test_log", false),
 		StateFile: "/tmp/merlin_force_sync_test_state",
 		DoneChan:  doneChan,
 		State: &config.RepoState{

merlin/config/config.go

@@ -30,9 +30,9 @@ const (
 	DEFAULT_DOWNLOAD_DIR     = "/mirror/root"
 	DEFAULT_TRACE_DIR        = "/home/mirror/merlin/trace"
 	DEFAULT_STATE_DIR        = "/home/mirror/merlin/state"
-	DEFAULT_LOG_DIR          = "/home/mirror/merlin/logs"
-	DEFAULT_RSYNC_LOG_DIR    = "/home/mirror/merlin/logs-rsync"
-	DEFAULT_ZFSSYNC_LOG_DIR  = "/home/mirror/merlin/logs-zfssync"
+	DEFAULT_LOG_DIR          = "/home/mirror/merlin/log"
+	DEFAULT_RSYNC_LOG_DIR    = "/home/mirror/merlin/log-rsync"
+	DEFAULT_ZFSSYNC_LOG_DIR  = "/home/mirror/merlin/log-zfssync"
 	DEFAULT_SOCK_PATH        = "/mirror/merlin/run/merlin.sock"
 )
 
@@ -84,11 +84,11 @@ type Config struct {
 	// directory where the state of each repo is saved
 	StateDir string `ini:"state_dir"`
 	// directory where merlin will store the merlin logs for each repo
-	RepoLogDir string `ini:"repo_logs_dir"`
+	RepoLogDir string `ini:"repo_log_dir"`
 	// directory to store the rsync logs for each repo
-	RsyncLogDir string `ini:"rsync_logs_dir"`
+	RsyncLogDir string `ini:"rsync_log_dir"`
 	// directory to store the zfssync logs for each repo
-	ZfssyncLogDir string `ini:"zfssync_logs_dir"`
+	ZfssyncLogDir string `ini:"zfssync_log_dir"`
 	// path to the unix socket for arthur to use for communication
 	SockPath string `ini:"sock_path"`
 }

merlin/config/config_test.go

@@ -46,7 +46,7 @@ func TestLoadConfig(t *testing.T) {
 		DefaultMaxRsyncIO:   0,
 		DownloadDir:         "test_files/download",
 		StateDir:            "test_files",
-		RepoLogDir:          "test_files/logs",
+		RepoLogDir:          "test_files/log",
 		RsyncLogDir:         "test_files/rsync",
 		ZfssyncLogDir:       "test_files/zfssync",
 		SockPath:            "test_files/test.sock",
@@ -62,7 +62,7 @@ func TestLoadConfig(t *testing.T) {
 		RsyncHost:      "rsync.releases.eelinux.ca",
 		RsyncDir:       "releases",
 		StateFile:      "test_files/eelinux",
-		RepoLogFile:    "test_files/logs/eelinux.log",
+		RepoLogFile:    "test_files/log/eelinux.log",
 		Logger:         Repos[0].Logger,
 		RsyncLogFile:   "test_files/rsync/eelinux-rsync.log",
 		ZfssyncLogFile: "test_files/zfssync/eelinux-zfssync.log",
@@ -86,7 +86,7 @@ func TestLoadConfig(t *testing.T) {
 		RsyncHost:      "rsync.releases.yoland.io",
 		RsyncDir:       "releases",
 		StateFile:      "test_files/yoland",
-		RepoLogFile:    "test_files/logs/yoland.log",
+		RepoLogFile:    "test_files/log/yoland.log",
 		Logger:         Repos[1].Logger,
 		RsyncLogFile:   "test_files/rsync/yoland-rsync.log",
 		ZfssyncLogFile: "test_files/zfssync/yoland-zfssync.log",
@@ -117,7 +117,7 @@ func TestLoadConfig(t *testing.T) {
 	}
 
 	os.Remove("test_files/yoland")
-	os.RemoveAll("test_files/logs")
+	os.RemoveAll("test_files/log")
 	os.RemoveAll("test_files/rsync")
 	os.RemoveAll("test_files/zfssync")
 }

merlin/config/config_test.ini

@@ -8,9 +8,9 @@ default_frequency = daily
 default_max_time = 1000
 download_dir = test_files/download
 state_dir = test_files
-repo_logs_dir = test_files/logs
-rsync_logs_dir = test_files/rsync
-zfssync_logs_dir = test_files/zfssync
+repo_log_dir = test_files/log
+rsync_log_dir = test_files/rsync
+zfssync_log_dir = test_files/zfssync
 sock_path = test_files/test.sock
 
 [eelinux]

merlin/merlin-config.ini

@@ -1,19 +1,140 @@
 ipv4_address = 129.97.134.71
 ipv6_address = 2620:101:f000:4901:c5c::f:1055
 
+; max_jobs = 6
+
 download_dir = /mirror/root
 trace_dir = /home/mirror-go/merlin/trace
 state_dir = /home/mirror-go/merlin/state
 
-repo_logs_dir = /home/mirror-go/merlin/logs
-rsync_logs_dir = /home/mirror-go/merlin/logs-rsync
-zfssync_logs_dir = /home/mirror-go/merlin/logs-zfssync
+repo_log_dir = /home/mirror-go/merlin/log
+rsync_log_dir = /home/mirror-go/merlin/log-rsync
+zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
 
 sock_path = /mirror/merlin/run/merlin-go.sock
 
+[puppylinux]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = puppylinux
+rsync_host = distro.ibiblio.org
+rsync_dir = puppylinux
+
 [CPAN]
+verbose = true
 sync_type = csc-sync-standard
 frequency = twice-daily
 local_dir = CPAN
 rsync_host = cpan-rsync.perl.org
 rsync_dir = CPAN
+
+[CTAN]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = CTAN
+rsync_host = rsync.dante.ctan.org
+rsync_dir = CTAN
+
+[cygwin]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = cygwin
+rsync_host = cygwin.com
+rsync_dir = cygwin-ftp
+
+[gnu]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = gnu
+rsync_host = ftp.gnu.org
+rsync_dir = gnu
+
+[nognu]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = nongnu
+rsync_host = dl.sv.gnu.org
+rsync_dir = releases
+
+[mysql]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = mysql
+rsync_host = rsync.mirrorservice.org
+rsync_dir = ftp.mysql.com
+
+[gnome]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = gnome
+rsync_host = master.gnome.org
+rsync_dir = gnomeftp
+password_file = /home/mirror/passwords/gnome
+
+[damnsmalllinux]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = damnsmalllinux
+rsync_host = ftp.heanet.ie
+rsync_dir = mirrors/damnsmalllinux.org/
+
+[linuxmint]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = linuxmint
+rsync_host = pub.linuxmint.com
+rsync_dir = pub
+
+[kernel.org-linux]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = kernel.org/linux
+rsync_host = rsync.kernel.org
+rsync_dir = pub/linux/
+
+[kernel.org-software]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = kernel.org/software
+rsync_host = rsync.kernel.org
+rsync_dir = pub/software/
+
+[apache]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = apache
+rsync_host = rsync.us.apache.org
+rsync_dir = apache-dist
+
+[eclipse]
+verbose = true
+dry_run = true
+sync_type = csc-sync-standard
+frequency = twice-daily
+local_dir = eclipse
+rsync_host = download.eclipse.org
+rsync_dir = eclipseMirror

merlin/merlin-go.service

@@ -3,8 +3,8 @@ Description=Manages synchronization of mirrored projects
 After=network.target
 
 [Service]
-ExecStart=/home/mirror/merlin-go/merlin /home/mirror/merlin-go/merlin-config.ini
-WorkingDirectory=/home/mirror/merlin
+ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
+WorkingDirectory=/home/mirror-go/merlin
 User=mirror
 Group=mirror
 SyslogIdentifier=merlin-go

merlin/merlin.go

@@ -76,7 +76,7 @@ func main() {
 		startIdx := repoIdx
 
 		for numJobsRunning < config.Conf.MaxJobs {
-			if sync.SyncIfPossible(config.Repos[repoIdx]) {
+			if sync.SyncIfPossible(config.Repos[repoIdx], false) {
 				numJobsRunning++
 			}
 			repoIdx = (repoIdx + 1) % len(config.Repos)

merlin/sync/interface.go

@@ -9,13 +9,13 @@ import (
 
 // Start sync job for the repo if more than repo.Frequency seconds have elapsed since its last job
 // and is not currently running. Returns true iff a job is started.
-func SyncIfPossible(repo *config.Repo) bool {
+func SyncIfPossible(repo *config.Repo, force bool) bool {
 	if repo.State.IsRunning {
 		return false
 	}
 
 	curTime := time.Now().Unix()
-	if curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
+	if force || curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
 		repo.State.IsRunning = true
 		repo.State.LastAttemptStartTime = curTime
 		repo.SaveState()