diff --git a/merlin/Handoff.md b/merlin/Handoff.md new file mode 100644 index 0000000..41cf7ee --- /dev/null +++ b/merlin/Handoff.md @@ -0,0 +1,63 @@ +## Status of moving from merlin to merlin-go +- [ ] debian +- [ ] ubuntu +- [ ] ubuntu-ports +- [ ] linuxmint-packages +- [ ] debian-multimedia +- [ ] debian-backports +- [ ] debian-security +- [ ] ubuntu-releases +- [ ] xubuntu-releases +- [ ] puppylinux < +- [x] CPAN +- [ ] CRAN +- [ ] CTAN < +- [ ] fedora-epel +- [ ] cygwin < +- [ ] gnu < +- [ ] nognu < +- [ ] mysql < +- [ ] gnome < +- [ ] damnsmalllinux < +- [ ] linuxmint < +- [ ] kernel.org-linux < +- [ ] kernel.org-software < +- [ ] apache < +- [ ] eclipse < +- [ ] kde +- [ ] kde-appplicationdata +- [ ] archlinux +- [ ] artixlinux +- [ ] slackware +- [ ] debian-cd +- [ ] x.org +- [ ] centos +- [ ] opensuse +- [ ] FreeBSD +- [ ] fedora-enchilada +- [ ] ubuntu-ports-releases +- [ ] gentoo-distfiles +- [ ] gentoo-portage +- [ ] gutenberg +- [ ] racket-installers +- [ ] plt-bundles +- [ ] xiph +- [ ] netbsd +- [ ] macports-release +- [ ] macports-distfiles +- [ ] sagemath +- [ ] vlc +- [ ] tdf +- [ ] saltstack +- [ ] alpine +- [ ] raspbian +- [ ] raspberrypi +- [ ] ipfire +- [ ] manjaro +- [ ] mxlinux +- [ ] mxlinux-iso +- [ ] parabola +- [ ] trisquel-packages +- [ ] trisquel-iso +- [ ] almalinux +- [ ] ceph \ No newline at end of file diff --git a/merlin/arthur/arthur.go b/merlin/arthur/arthur.go index baf69f4..df59ad6 100644 --- a/merlin/arthur/arthur.go +++ b/merlin/arthur/arthur.go @@ -85,7 +85,7 @@ func ForceSync(conn net.Conn, repoName string) (newSync bool) { if repo, isInMap := config.RepoMap[repoName]; isInMap { logger.OutLog("Attempting to force sync of " + repoName) - if sync.SyncIfPossible(repo) { + if sync.SyncIfPossible(repo, true) { conn.Write([]byte("Forced sync for " + repoName)) newSync = true } else { diff --git a/merlin/arthur/arthur_test.go b/merlin/arthur/arthur_test.go index 27b8cd1..b97eed2 100644 --- a/merlin/arthur/arthur_test.go +++ b/merlin/arthur/arthur_test.go @@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) { SyncType: "csc-sync-dummy", Frequency: 7 * 86400, MaxTime: 30, - Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false), + Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_log", false), StateFile: "/tmp/merlin_force_sync_test_state", DoneChan: doneChan, State: &config.RepoState{ diff --git a/merlin/cmd/arthur/arthur b/merlin/cmd/arthur/arthur new file mode 100755 index 0000000..a3594b1 Binary files /dev/null and b/merlin/cmd/arthur/arthur differ diff --git a/merlin/config/config.go b/merlin/config/config.go index cedcc7e..4b7931a 100644 --- a/merlin/config/config.go +++ b/merlin/config/config.go @@ -30,9 +30,9 @@ const ( DEFAULT_DOWNLOAD_DIR = "/mirror/root" DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace" DEFAULT_STATE_DIR = "/home/mirror/merlin/state" - DEFAULT_LOG_DIR = "/home/mirror/merlin/logs" - DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/logs-rsync" - DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/logs-zfssync" + DEFAULT_LOG_DIR = "/home/mirror/merlin/log" + DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync" + DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/log-zfssync" DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock" ) @@ -84,11 +84,11 @@ type Config struct { // directory where the state of each repo is saved StateDir string `ini:"state_dir"` // directory where merlin will store the merlin logs for each repo - RepoLogDir string `ini:"repo_logs_dir"` + RepoLogDir string `ini:"repo_log_dir"` // directory to store the rsync logs for each repo - RsyncLogDir string `ini:"rsync_logs_dir"` + RsyncLogDir string `ini:"rsync_log_dir"` // directory to store the zfssync logs for each repo - ZfssyncLogDir string `ini:"zfssync_logs_dir"` + ZfssyncLogDir string `ini:"zfssync_log_dir"` // path to the unix socket for arthur to use for communication SockPath string `ini:"sock_path"` } diff --git a/merlin/config/config_test.go b/merlin/config/config_test.go index e54de2e..7493a02 100644 --- a/merlin/config/config_test.go +++ b/merlin/config/config_test.go @@ -46,7 +46,7 @@ func TestLoadConfig(t *testing.T) { DefaultMaxRsyncIO: 0, DownloadDir: "test_files/download", StateDir: "test_files", - RepoLogDir: "test_files/logs", + RepoLogDir: "test_files/log", RsyncLogDir: "test_files/rsync", ZfssyncLogDir: "test_files/zfssync", SockPath: "test_files/test.sock", @@ -62,7 +62,7 @@ func TestLoadConfig(t *testing.T) { RsyncHost: "rsync.releases.eelinux.ca", RsyncDir: "releases", StateFile: "test_files/eelinux", - RepoLogFile: "test_files/logs/eelinux.log", + RepoLogFile: "test_files/log/eelinux.log", Logger: Repos[0].Logger, RsyncLogFile: "test_files/rsync/eelinux-rsync.log", ZfssyncLogFile: "test_files/zfssync/eelinux-zfssync.log", @@ -86,7 +86,7 @@ func TestLoadConfig(t *testing.T) { RsyncHost: "rsync.releases.yoland.io", RsyncDir: "releases", StateFile: "test_files/yoland", - RepoLogFile: "test_files/logs/yoland.log", + RepoLogFile: "test_files/log/yoland.log", Logger: Repos[1].Logger, RsyncLogFile: "test_files/rsync/yoland-rsync.log", ZfssyncLogFile: "test_files/zfssync/yoland-zfssync.log", @@ -117,7 +117,7 @@ func TestLoadConfig(t *testing.T) { } os.Remove("test_files/yoland") - os.RemoveAll("test_files/logs") + os.RemoveAll("test_files/log") os.RemoveAll("test_files/rsync") os.RemoveAll("test_files/zfssync") } diff --git a/merlin/config/config_test.ini b/merlin/config/config_test.ini index 721f7dc..ee10fad 100644 --- a/merlin/config/config_test.ini +++ b/merlin/config/config_test.ini @@ -8,9 +8,9 @@ default_frequency = daily default_max_time = 1000 download_dir = test_files/download state_dir = test_files -repo_logs_dir = test_files/logs -rsync_logs_dir = test_files/rsync -zfssync_logs_dir = test_files/zfssync +repo_log_dir = test_files/log +rsync_log_dir = test_files/rsync +zfssync_log_dir = test_files/zfssync sock_path = test_files/test.sock [eelinux] diff --git a/merlin/merlin-config.ini b/merlin/merlin-config.ini index 08d145e..d1b1d62 100644 --- a/merlin/merlin-config.ini +++ b/merlin/merlin-config.ini @@ -1,19 +1,140 @@ ipv4_address = 129.97.134.71 ipv6_address = 2620:101:f000:4901:c5c::f:1055 +; max_jobs = 6 + download_dir = /mirror/root trace_dir = /home/mirror-go/merlin/trace state_dir = /home/mirror-go/merlin/state -repo_logs_dir = /home/mirror-go/merlin/logs -rsync_logs_dir = /home/mirror-go/merlin/logs-rsync -zfssync_logs_dir = /home/mirror-go/merlin/logs-zfssync +repo_log_dir = /home/mirror-go/merlin/log +rsync_log_dir = /home/mirror-go/merlin/log-rsync +zfssync_log_dir = /home/mirror-go/merlin/log-zfssync sock_path = /mirror/merlin/run/merlin-go.sock +[puppylinux] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = puppylinux +rsync_host = distro.ibiblio.org +rsync_dir = puppylinux + [CPAN] +verbose = true sync_type = csc-sync-standard frequency = twice-daily local_dir = CPAN rsync_host = cpan-rsync.perl.org rsync_dir = CPAN + +[CTAN] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = CTAN +rsync_host = rsync.dante.ctan.org +rsync_dir = CTAN + +[cygwin] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = cygwin +rsync_host = cygwin.com +rsync_dir = cygwin-ftp + +[gnu] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = gnu +rsync_host = ftp.gnu.org +rsync_dir = gnu + +[nognu] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = nongnu +rsync_host = dl.sv.gnu.org +rsync_dir = releases + +[mysql] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = mysql +rsync_host = rsync.mirrorservice.org +rsync_dir = ftp.mysql.com + +[gnome] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = gnome +rsync_host = master.gnome.org +rsync_dir = gnomeftp +password_file = /home/mirror/passwords/gnome + +[damnsmalllinux] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = damnsmalllinux +rsync_host = ftp.heanet.ie +rsync_dir = mirrors/damnsmalllinux.org/ + +[linuxmint] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = linuxmint +rsync_host = pub.linuxmint.com +rsync_dir = pub + +[kernel.org-linux] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = kernel.org/linux +rsync_host = rsync.kernel.org +rsync_dir = pub/linux/ + +[kernel.org-software] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = kernel.org/software +rsync_host = rsync.kernel.org +rsync_dir = pub/software/ + +[apache] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = apache +rsync_host = rsync.us.apache.org +rsync_dir = apache-dist + +[eclipse] +verbose = true +dry_run = true +sync_type = csc-sync-standard +frequency = twice-daily +local_dir = eclipse +rsync_host = download.eclipse.org +rsync_dir = eclipseMirror \ No newline at end of file diff --git a/merlin/merlin-go.service b/merlin/merlin-go.service index 71a649a..03971a0 100644 --- a/merlin/merlin-go.service +++ b/merlin/merlin-go.service @@ -3,8 +3,8 @@ Description=Manages synchronization of mirrored projects After=network.target [Service] -ExecStart=/home/mirror/merlin-go/merlin /home/mirror/merlin-go/merlin-config.ini -WorkingDirectory=/home/mirror/merlin +ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini +WorkingDirectory=/home/mirror-go/merlin User=mirror Group=mirror SyslogIdentifier=merlin-go diff --git a/merlin/merlin.go b/merlin/merlin.go index dc6d0c1..17f49c6 100644 --- a/merlin/merlin.go +++ b/merlin/merlin.go @@ -76,7 +76,7 @@ func main() { startIdx := repoIdx for numJobsRunning < config.Conf.MaxJobs { - if sync.SyncIfPossible(config.Repos[repoIdx]) { + if sync.SyncIfPossible(config.Repos[repoIdx], false) { numJobsRunning++ } repoIdx = (repoIdx + 1) % len(config.Repos) diff --git a/merlin/sync/interface.go b/merlin/sync/interface.go index 02a5f94..3a88357 100644 --- a/merlin/sync/interface.go +++ b/merlin/sync/interface.go @@ -9,13 +9,13 @@ import ( // Start sync job for the repo if more than repo.Frequency seconds have elapsed since its last job // and is not currently running. Returns true iff a job is started. -func SyncIfPossible(repo *config.Repo) bool { +func SyncIfPossible(repo *config.Repo, force bool) bool { if repo.State.IsRunning { return false } curTime := time.Now().Unix() - if curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) { + if force || curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) { repo.State.IsRunning = true repo.State.LastAttemptStartTime = curTime repo.SaveState()