Move more repos to merlin-go
This commit is contained in:
parent
392396d1a0
commit
2492d533f0
|
@ -0,0 +1,63 @@
|
||||||
|
## Status of moving from merlin to merlin-go
|
||||||
|
- [ ] debian
|
||||||
|
- [ ] ubuntu
|
||||||
|
- [ ] ubuntu-ports
|
||||||
|
- [ ] linuxmint-packages
|
||||||
|
- [ ] debian-multimedia
|
||||||
|
- [ ] debian-backports
|
||||||
|
- [ ] debian-security
|
||||||
|
- [ ] ubuntu-releases
|
||||||
|
- [ ] xubuntu-releases
|
||||||
|
- [ ] puppylinux <
|
||||||
|
- [x] CPAN
|
||||||
|
- [ ] CRAN
|
||||||
|
- [ ] CTAN <
|
||||||
|
- [ ] fedora-epel
|
||||||
|
- [ ] cygwin <
|
||||||
|
- [ ] gnu <
|
||||||
|
- [ ] nognu <
|
||||||
|
- [ ] mysql <
|
||||||
|
- [ ] gnome <
|
||||||
|
- [ ] damnsmalllinux <
|
||||||
|
- [ ] linuxmint <
|
||||||
|
- [ ] kernel.org-linux <
|
||||||
|
- [ ] kernel.org-software <
|
||||||
|
- [ ] apache <
|
||||||
|
- [ ] eclipse <
|
||||||
|
- [ ] kde
|
||||||
|
- [ ] kde-appplicationdata
|
||||||
|
- [ ] archlinux
|
||||||
|
- [ ] artixlinux
|
||||||
|
- [ ] slackware
|
||||||
|
- [ ] debian-cd
|
||||||
|
- [ ] x.org
|
||||||
|
- [ ] centos
|
||||||
|
- [ ] opensuse
|
||||||
|
- [ ] FreeBSD
|
||||||
|
- [ ] fedora-enchilada
|
||||||
|
- [ ] ubuntu-ports-releases
|
||||||
|
- [ ] gentoo-distfiles
|
||||||
|
- [ ] gentoo-portage
|
||||||
|
- [ ] gutenberg
|
||||||
|
- [ ] racket-installers
|
||||||
|
- [ ] plt-bundles
|
||||||
|
- [ ] xiph
|
||||||
|
- [ ] netbsd
|
||||||
|
- [ ] macports-release
|
||||||
|
- [ ] macports-distfiles
|
||||||
|
- [ ] sagemath
|
||||||
|
- [ ] vlc
|
||||||
|
- [ ] tdf
|
||||||
|
- [ ] saltstack
|
||||||
|
- [ ] alpine
|
||||||
|
- [ ] raspbian
|
||||||
|
- [ ] raspberrypi
|
||||||
|
- [ ] ipfire
|
||||||
|
- [ ] manjaro
|
||||||
|
- [ ] mxlinux
|
||||||
|
- [ ] mxlinux-iso
|
||||||
|
- [ ] parabola
|
||||||
|
- [ ] trisquel-packages
|
||||||
|
- [ ] trisquel-iso
|
||||||
|
- [ ] almalinux
|
||||||
|
- [ ] ceph
|
|
@ -85,7 +85,7 @@ func ForceSync(conn net.Conn, repoName string) (newSync bool) {
|
||||||
|
|
||||||
if repo, isInMap := config.RepoMap[repoName]; isInMap {
|
if repo, isInMap := config.RepoMap[repoName]; isInMap {
|
||||||
logger.OutLog("Attempting to force sync of " + repoName)
|
logger.OutLog("Attempting to force sync of " + repoName)
|
||||||
if sync.SyncIfPossible(repo) {
|
if sync.SyncIfPossible(repo, true) {
|
||||||
conn.Write([]byte("Forced sync for " + repoName))
|
conn.Write([]byte("Forced sync for " + repoName))
|
||||||
newSync = true
|
newSync = true
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
|
||||||
SyncType: "csc-sync-dummy",
|
SyncType: "csc-sync-dummy",
|
||||||
Frequency: 7 * 86400,
|
Frequency: 7 * 86400,
|
||||||
MaxTime: 30,
|
MaxTime: 30,
|
||||||
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
|
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_log", false),
|
||||||
StateFile: "/tmp/merlin_force_sync_test_state",
|
StateFile: "/tmp/merlin_force_sync_test_state",
|
||||||
DoneChan: doneChan,
|
DoneChan: doneChan,
|
||||||
State: &config.RepoState{
|
State: &config.RepoState{
|
||||||
|
|
Binary file not shown.
|
@ -30,9 +30,9 @@ const (
|
||||||
DEFAULT_DOWNLOAD_DIR = "/mirror/root"
|
DEFAULT_DOWNLOAD_DIR = "/mirror/root"
|
||||||
DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace"
|
DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace"
|
||||||
DEFAULT_STATE_DIR = "/home/mirror/merlin/state"
|
DEFAULT_STATE_DIR = "/home/mirror/merlin/state"
|
||||||
DEFAULT_LOG_DIR = "/home/mirror/merlin/logs"
|
DEFAULT_LOG_DIR = "/home/mirror/merlin/log"
|
||||||
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/logs-rsync"
|
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync"
|
||||||
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/logs-zfssync"
|
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/log-zfssync"
|
||||||
DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock"
|
DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -84,11 +84,11 @@ type Config struct {
|
||||||
// directory where the state of each repo is saved
|
// directory where the state of each repo is saved
|
||||||
StateDir string `ini:"state_dir"`
|
StateDir string `ini:"state_dir"`
|
||||||
// directory where merlin will store the merlin logs for each repo
|
// directory where merlin will store the merlin logs for each repo
|
||||||
RepoLogDir string `ini:"repo_logs_dir"`
|
RepoLogDir string `ini:"repo_log_dir"`
|
||||||
// directory to store the rsync logs for each repo
|
// directory to store the rsync logs for each repo
|
||||||
RsyncLogDir string `ini:"rsync_logs_dir"`
|
RsyncLogDir string `ini:"rsync_log_dir"`
|
||||||
// directory to store the zfssync logs for each repo
|
// directory to store the zfssync logs for each repo
|
||||||
ZfssyncLogDir string `ini:"zfssync_logs_dir"`
|
ZfssyncLogDir string `ini:"zfssync_log_dir"`
|
||||||
// path to the unix socket for arthur to use for communication
|
// path to the unix socket for arthur to use for communication
|
||||||
SockPath string `ini:"sock_path"`
|
SockPath string `ini:"sock_path"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,7 +46,7 @@ func TestLoadConfig(t *testing.T) {
|
||||||
DefaultMaxRsyncIO: 0,
|
DefaultMaxRsyncIO: 0,
|
||||||
DownloadDir: "test_files/download",
|
DownloadDir: "test_files/download",
|
||||||
StateDir: "test_files",
|
StateDir: "test_files",
|
||||||
RepoLogDir: "test_files/logs",
|
RepoLogDir: "test_files/log",
|
||||||
RsyncLogDir: "test_files/rsync",
|
RsyncLogDir: "test_files/rsync",
|
||||||
ZfssyncLogDir: "test_files/zfssync",
|
ZfssyncLogDir: "test_files/zfssync",
|
||||||
SockPath: "test_files/test.sock",
|
SockPath: "test_files/test.sock",
|
||||||
|
@ -62,7 +62,7 @@ func TestLoadConfig(t *testing.T) {
|
||||||
RsyncHost: "rsync.releases.eelinux.ca",
|
RsyncHost: "rsync.releases.eelinux.ca",
|
||||||
RsyncDir: "releases",
|
RsyncDir: "releases",
|
||||||
StateFile: "test_files/eelinux",
|
StateFile: "test_files/eelinux",
|
||||||
RepoLogFile: "test_files/logs/eelinux.log",
|
RepoLogFile: "test_files/log/eelinux.log",
|
||||||
Logger: Repos[0].Logger,
|
Logger: Repos[0].Logger,
|
||||||
RsyncLogFile: "test_files/rsync/eelinux-rsync.log",
|
RsyncLogFile: "test_files/rsync/eelinux-rsync.log",
|
||||||
ZfssyncLogFile: "test_files/zfssync/eelinux-zfssync.log",
|
ZfssyncLogFile: "test_files/zfssync/eelinux-zfssync.log",
|
||||||
|
@ -86,7 +86,7 @@ func TestLoadConfig(t *testing.T) {
|
||||||
RsyncHost: "rsync.releases.yoland.io",
|
RsyncHost: "rsync.releases.yoland.io",
|
||||||
RsyncDir: "releases",
|
RsyncDir: "releases",
|
||||||
StateFile: "test_files/yoland",
|
StateFile: "test_files/yoland",
|
||||||
RepoLogFile: "test_files/logs/yoland.log",
|
RepoLogFile: "test_files/log/yoland.log",
|
||||||
Logger: Repos[1].Logger,
|
Logger: Repos[1].Logger,
|
||||||
RsyncLogFile: "test_files/rsync/yoland-rsync.log",
|
RsyncLogFile: "test_files/rsync/yoland-rsync.log",
|
||||||
ZfssyncLogFile: "test_files/zfssync/yoland-zfssync.log",
|
ZfssyncLogFile: "test_files/zfssync/yoland-zfssync.log",
|
||||||
|
@ -117,7 +117,7 @@ func TestLoadConfig(t *testing.T) {
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Remove("test_files/yoland")
|
os.Remove("test_files/yoland")
|
||||||
os.RemoveAll("test_files/logs")
|
os.RemoveAll("test_files/log")
|
||||||
os.RemoveAll("test_files/rsync")
|
os.RemoveAll("test_files/rsync")
|
||||||
os.RemoveAll("test_files/zfssync")
|
os.RemoveAll("test_files/zfssync")
|
||||||
}
|
}
|
||||||
|
|
|
@ -8,9 +8,9 @@ default_frequency = daily
|
||||||
default_max_time = 1000
|
default_max_time = 1000
|
||||||
download_dir = test_files/download
|
download_dir = test_files/download
|
||||||
state_dir = test_files
|
state_dir = test_files
|
||||||
repo_logs_dir = test_files/logs
|
repo_log_dir = test_files/log
|
||||||
rsync_logs_dir = test_files/rsync
|
rsync_log_dir = test_files/rsync
|
||||||
zfssync_logs_dir = test_files/zfssync
|
zfssync_log_dir = test_files/zfssync
|
||||||
sock_path = test_files/test.sock
|
sock_path = test_files/test.sock
|
||||||
|
|
||||||
[eelinux]
|
[eelinux]
|
||||||
|
|
|
@ -1,19 +1,140 @@
|
||||||
ipv4_address = 129.97.134.71
|
ipv4_address = 129.97.134.71
|
||||||
ipv6_address = 2620:101:f000:4901:c5c::f:1055
|
ipv6_address = 2620:101:f000:4901:c5c::f:1055
|
||||||
|
|
||||||
|
; max_jobs = 6
|
||||||
|
|
||||||
download_dir = /mirror/root
|
download_dir = /mirror/root
|
||||||
trace_dir = /home/mirror-go/merlin/trace
|
trace_dir = /home/mirror-go/merlin/trace
|
||||||
state_dir = /home/mirror-go/merlin/state
|
state_dir = /home/mirror-go/merlin/state
|
||||||
|
|
||||||
repo_logs_dir = /home/mirror-go/merlin/logs
|
repo_log_dir = /home/mirror-go/merlin/log
|
||||||
rsync_logs_dir = /home/mirror-go/merlin/logs-rsync
|
rsync_log_dir = /home/mirror-go/merlin/log-rsync
|
||||||
zfssync_logs_dir = /home/mirror-go/merlin/logs-zfssync
|
zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
|
||||||
|
|
||||||
sock_path = /mirror/merlin/run/merlin-go.sock
|
sock_path = /mirror/merlin/run/merlin-go.sock
|
||||||
|
|
||||||
|
[puppylinux]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = puppylinux
|
||||||
|
rsync_host = distro.ibiblio.org
|
||||||
|
rsync_dir = puppylinux
|
||||||
|
|
||||||
[CPAN]
|
[CPAN]
|
||||||
|
verbose = true
|
||||||
sync_type = csc-sync-standard
|
sync_type = csc-sync-standard
|
||||||
frequency = twice-daily
|
frequency = twice-daily
|
||||||
local_dir = CPAN
|
local_dir = CPAN
|
||||||
rsync_host = cpan-rsync.perl.org
|
rsync_host = cpan-rsync.perl.org
|
||||||
rsync_dir = CPAN
|
rsync_dir = CPAN
|
||||||
|
|
||||||
|
[CTAN]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = CTAN
|
||||||
|
rsync_host = rsync.dante.ctan.org
|
||||||
|
rsync_dir = CTAN
|
||||||
|
|
||||||
|
[cygwin]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = cygwin
|
||||||
|
rsync_host = cygwin.com
|
||||||
|
rsync_dir = cygwin-ftp
|
||||||
|
|
||||||
|
[gnu]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = gnu
|
||||||
|
rsync_host = ftp.gnu.org
|
||||||
|
rsync_dir = gnu
|
||||||
|
|
||||||
|
[nognu]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = nongnu
|
||||||
|
rsync_host = dl.sv.gnu.org
|
||||||
|
rsync_dir = releases
|
||||||
|
|
||||||
|
[mysql]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = mysql
|
||||||
|
rsync_host = rsync.mirrorservice.org
|
||||||
|
rsync_dir = ftp.mysql.com
|
||||||
|
|
||||||
|
[gnome]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = gnome
|
||||||
|
rsync_host = master.gnome.org
|
||||||
|
rsync_dir = gnomeftp
|
||||||
|
password_file = /home/mirror/passwords/gnome
|
||||||
|
|
||||||
|
[damnsmalllinux]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = damnsmalllinux
|
||||||
|
rsync_host = ftp.heanet.ie
|
||||||
|
rsync_dir = mirrors/damnsmalllinux.org/
|
||||||
|
|
||||||
|
[linuxmint]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = linuxmint
|
||||||
|
rsync_host = pub.linuxmint.com
|
||||||
|
rsync_dir = pub
|
||||||
|
|
||||||
|
[kernel.org-linux]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = kernel.org/linux
|
||||||
|
rsync_host = rsync.kernel.org
|
||||||
|
rsync_dir = pub/linux/
|
||||||
|
|
||||||
|
[kernel.org-software]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = kernel.org/software
|
||||||
|
rsync_host = rsync.kernel.org
|
||||||
|
rsync_dir = pub/software/
|
||||||
|
|
||||||
|
[apache]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = apache
|
||||||
|
rsync_host = rsync.us.apache.org
|
||||||
|
rsync_dir = apache-dist
|
||||||
|
|
||||||
|
[eclipse]
|
||||||
|
verbose = true
|
||||||
|
dry_run = true
|
||||||
|
sync_type = csc-sync-standard
|
||||||
|
frequency = twice-daily
|
||||||
|
local_dir = eclipse
|
||||||
|
rsync_host = download.eclipse.org
|
||||||
|
rsync_dir = eclipseMirror
|
|
@ -3,8 +3,8 @@ Description=Manages synchronization of mirrored projects
|
||||||
After=network.target
|
After=network.target
|
||||||
|
|
||||||
[Service]
|
[Service]
|
||||||
ExecStart=/home/mirror/merlin-go/merlin /home/mirror/merlin-go/merlin-config.ini
|
ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
|
||||||
WorkingDirectory=/home/mirror/merlin
|
WorkingDirectory=/home/mirror-go/merlin
|
||||||
User=mirror
|
User=mirror
|
||||||
Group=mirror
|
Group=mirror
|
||||||
SyslogIdentifier=merlin-go
|
SyslogIdentifier=merlin-go
|
||||||
|
|
|
@ -76,7 +76,7 @@ func main() {
|
||||||
startIdx := repoIdx
|
startIdx := repoIdx
|
||||||
|
|
||||||
for numJobsRunning < config.Conf.MaxJobs {
|
for numJobsRunning < config.Conf.MaxJobs {
|
||||||
if sync.SyncIfPossible(config.Repos[repoIdx]) {
|
if sync.SyncIfPossible(config.Repos[repoIdx], false) {
|
||||||
numJobsRunning++
|
numJobsRunning++
|
||||||
}
|
}
|
||||||
repoIdx = (repoIdx + 1) % len(config.Repos)
|
repoIdx = (repoIdx + 1) % len(config.Repos)
|
||||||
|
|
|
@ -9,13 +9,13 @@ import (
|
||||||
|
|
||||||
// Start sync job for the repo if more than repo.Frequency seconds have elapsed since its last job
|
// Start sync job for the repo if more than repo.Frequency seconds have elapsed since its last job
|
||||||
// and is not currently running. Returns true iff a job is started.
|
// and is not currently running. Returns true iff a job is started.
|
||||||
func SyncIfPossible(repo *config.Repo) bool {
|
func SyncIfPossible(repo *config.Repo, force bool) bool {
|
||||||
if repo.State.IsRunning {
|
if repo.State.IsRunning {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
curTime := time.Now().Unix()
|
curTime := time.Now().Unix()
|
||||||
if curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
|
if force || curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
|
||||||
repo.State.IsRunning = true
|
repo.State.IsRunning = true
|
||||||
repo.State.LastAttemptStartTime = curTime
|
repo.State.LastAttemptStartTime = curTime
|
||||||
repo.SaveState()
|
repo.SaveState()
|
||||||
|
|
Loading…
Reference in New Issue