Move more repos to merlin-go
This commit is contained in:
parent
392396d1a0
commit
2492d533f0
|
@ -0,0 +1,63 @@
|
|||
## Status of moving from merlin to merlin-go
|
||||
- [ ] debian
|
||||
- [ ] ubuntu
|
||||
- [ ] ubuntu-ports
|
||||
- [ ] linuxmint-packages
|
||||
- [ ] debian-multimedia
|
||||
- [ ] debian-backports
|
||||
- [ ] debian-security
|
||||
- [ ] ubuntu-releases
|
||||
- [ ] xubuntu-releases
|
||||
- [ ] puppylinux <
|
||||
- [x] CPAN
|
||||
- [ ] CRAN
|
||||
- [ ] CTAN <
|
||||
- [ ] fedora-epel
|
||||
- [ ] cygwin <
|
||||
- [ ] gnu <
|
||||
- [ ] nognu <
|
||||
- [ ] mysql <
|
||||
- [ ] gnome <
|
||||
- [ ] damnsmalllinux <
|
||||
- [ ] linuxmint <
|
||||
- [ ] kernel.org-linux <
|
||||
- [ ] kernel.org-software <
|
||||
- [ ] apache <
|
||||
- [ ] eclipse <
|
||||
- [ ] kde
|
||||
- [ ] kde-appplicationdata
|
||||
- [ ] archlinux
|
||||
- [ ] artixlinux
|
||||
- [ ] slackware
|
||||
- [ ] debian-cd
|
||||
- [ ] x.org
|
||||
- [ ] centos
|
||||
- [ ] opensuse
|
||||
- [ ] FreeBSD
|
||||
- [ ] fedora-enchilada
|
||||
- [ ] ubuntu-ports-releases
|
||||
- [ ] gentoo-distfiles
|
||||
- [ ] gentoo-portage
|
||||
- [ ] gutenberg
|
||||
- [ ] racket-installers
|
||||
- [ ] plt-bundles
|
||||
- [ ] xiph
|
||||
- [ ] netbsd
|
||||
- [ ] macports-release
|
||||
- [ ] macports-distfiles
|
||||
- [ ] sagemath
|
||||
- [ ] vlc
|
||||
- [ ] tdf
|
||||
- [ ] saltstack
|
||||
- [ ] alpine
|
||||
- [ ] raspbian
|
||||
- [ ] raspberrypi
|
||||
- [ ] ipfire
|
||||
- [ ] manjaro
|
||||
- [ ] mxlinux
|
||||
- [ ] mxlinux-iso
|
||||
- [ ] parabola
|
||||
- [ ] trisquel-packages
|
||||
- [ ] trisquel-iso
|
||||
- [ ] almalinux
|
||||
- [ ] ceph
|
|
@ -85,7 +85,7 @@ func ForceSync(conn net.Conn, repoName string) (newSync bool) {
|
|||
|
||||
if repo, isInMap := config.RepoMap[repoName]; isInMap {
|
||||
logger.OutLog("Attempting to force sync of " + repoName)
|
||||
if sync.SyncIfPossible(repo) {
|
||||
if sync.SyncIfPossible(repo, true) {
|
||||
conn.Write([]byte("Forced sync for " + repoName))
|
||||
newSync = true
|
||||
} else {
|
||||
|
|
|
@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
|
|||
SyncType: "csc-sync-dummy",
|
||||
Frequency: 7 * 86400,
|
||||
MaxTime: 30,
|
||||
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
|
||||
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_log", false),
|
||||
StateFile: "/tmp/merlin_force_sync_test_state",
|
||||
DoneChan: doneChan,
|
||||
State: &config.RepoState{
|
||||
|
|
Binary file not shown.
|
@ -30,9 +30,9 @@ const (
|
|||
DEFAULT_DOWNLOAD_DIR = "/mirror/root"
|
||||
DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace"
|
||||
DEFAULT_STATE_DIR = "/home/mirror/merlin/state"
|
||||
DEFAULT_LOG_DIR = "/home/mirror/merlin/logs"
|
||||
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/logs-rsync"
|
||||
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/logs-zfssync"
|
||||
DEFAULT_LOG_DIR = "/home/mirror/merlin/log"
|
||||
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync"
|
||||
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/log-zfssync"
|
||||
DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock"
|
||||
)
|
||||
|
||||
|
@ -84,11 +84,11 @@ type Config struct {
|
|||
// directory where the state of each repo is saved
|
||||
StateDir string `ini:"state_dir"`
|
||||
// directory where merlin will store the merlin logs for each repo
|
||||
RepoLogDir string `ini:"repo_logs_dir"`
|
||||
RepoLogDir string `ini:"repo_log_dir"`
|
||||
// directory to store the rsync logs for each repo
|
||||
RsyncLogDir string `ini:"rsync_logs_dir"`
|
||||
RsyncLogDir string `ini:"rsync_log_dir"`
|
||||
// directory to store the zfssync logs for each repo
|
||||
ZfssyncLogDir string `ini:"zfssync_logs_dir"`
|
||||
ZfssyncLogDir string `ini:"zfssync_log_dir"`
|
||||
// path to the unix socket for arthur to use for communication
|
||||
SockPath string `ini:"sock_path"`
|
||||
}
|
||||
|
|
|
@ -46,7 +46,7 @@ func TestLoadConfig(t *testing.T) {
|
|||
DefaultMaxRsyncIO: 0,
|
||||
DownloadDir: "test_files/download",
|
||||
StateDir: "test_files",
|
||||
RepoLogDir: "test_files/logs",
|
||||
RepoLogDir: "test_files/log",
|
||||
RsyncLogDir: "test_files/rsync",
|
||||
ZfssyncLogDir: "test_files/zfssync",
|
||||
SockPath: "test_files/test.sock",
|
||||
|
@ -62,7 +62,7 @@ func TestLoadConfig(t *testing.T) {
|
|||
RsyncHost: "rsync.releases.eelinux.ca",
|
||||
RsyncDir: "releases",
|
||||
StateFile: "test_files/eelinux",
|
||||
RepoLogFile: "test_files/logs/eelinux.log",
|
||||
RepoLogFile: "test_files/log/eelinux.log",
|
||||
Logger: Repos[0].Logger,
|
||||
RsyncLogFile: "test_files/rsync/eelinux-rsync.log",
|
||||
ZfssyncLogFile: "test_files/zfssync/eelinux-zfssync.log",
|
||||
|
@ -86,7 +86,7 @@ func TestLoadConfig(t *testing.T) {
|
|||
RsyncHost: "rsync.releases.yoland.io",
|
||||
RsyncDir: "releases",
|
||||
StateFile: "test_files/yoland",
|
||||
RepoLogFile: "test_files/logs/yoland.log",
|
||||
RepoLogFile: "test_files/log/yoland.log",
|
||||
Logger: Repos[1].Logger,
|
||||
RsyncLogFile: "test_files/rsync/yoland-rsync.log",
|
||||
ZfssyncLogFile: "test_files/zfssync/yoland-zfssync.log",
|
||||
|
@ -117,7 +117,7 @@ func TestLoadConfig(t *testing.T) {
|
|||
}
|
||||
|
||||
os.Remove("test_files/yoland")
|
||||
os.RemoveAll("test_files/logs")
|
||||
os.RemoveAll("test_files/log")
|
||||
os.RemoveAll("test_files/rsync")
|
||||
os.RemoveAll("test_files/zfssync")
|
||||
}
|
||||
|
|
|
@ -8,9 +8,9 @@ default_frequency = daily
|
|||
default_max_time = 1000
|
||||
download_dir = test_files/download
|
||||
state_dir = test_files
|
||||
repo_logs_dir = test_files/logs
|
||||
rsync_logs_dir = test_files/rsync
|
||||
zfssync_logs_dir = test_files/zfssync
|
||||
repo_log_dir = test_files/log
|
||||
rsync_log_dir = test_files/rsync
|
||||
zfssync_log_dir = test_files/zfssync
|
||||
sock_path = test_files/test.sock
|
||||
|
||||
[eelinux]
|
||||
|
|
|
@ -1,19 +1,140 @@
|
|||
ipv4_address = 129.97.134.71
|
||||
ipv6_address = 2620:101:f000:4901:c5c::f:1055
|
||||
|
||||
; max_jobs = 6
|
||||
|
||||
download_dir = /mirror/root
|
||||
trace_dir = /home/mirror-go/merlin/trace
|
||||
state_dir = /home/mirror-go/merlin/state
|
||||
|
||||
repo_logs_dir = /home/mirror-go/merlin/logs
|
||||
rsync_logs_dir = /home/mirror-go/merlin/logs-rsync
|
||||
zfssync_logs_dir = /home/mirror-go/merlin/logs-zfssync
|
||||
repo_log_dir = /home/mirror-go/merlin/log
|
||||
rsync_log_dir = /home/mirror-go/merlin/log-rsync
|
||||
zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
|
||||
|
||||
sock_path = /mirror/merlin/run/merlin-go.sock
|
||||
|
||||
[puppylinux]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = puppylinux
|
||||
rsync_host = distro.ibiblio.org
|
||||
rsync_dir = puppylinux
|
||||
|
||||
[CPAN]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = CPAN
|
||||
rsync_host = cpan-rsync.perl.org
|
||||
rsync_dir = CPAN
|
||||
|
||||
[CTAN]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = CTAN
|
||||
rsync_host = rsync.dante.ctan.org
|
||||
rsync_dir = CTAN
|
||||
|
||||
[cygwin]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = cygwin
|
||||
rsync_host = cygwin.com
|
||||
rsync_dir = cygwin-ftp
|
||||
|
||||
[gnu]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = gnu
|
||||
rsync_host = ftp.gnu.org
|
||||
rsync_dir = gnu
|
||||
|
||||
[nognu]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = nongnu
|
||||
rsync_host = dl.sv.gnu.org
|
||||
rsync_dir = releases
|
||||
|
||||
[mysql]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = mysql
|
||||
rsync_host = rsync.mirrorservice.org
|
||||
rsync_dir = ftp.mysql.com
|
||||
|
||||
[gnome]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = gnome
|
||||
rsync_host = master.gnome.org
|
||||
rsync_dir = gnomeftp
|
||||
password_file = /home/mirror/passwords/gnome
|
||||
|
||||
[damnsmalllinux]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = damnsmalllinux
|
||||
rsync_host = ftp.heanet.ie
|
||||
rsync_dir = mirrors/damnsmalllinux.org/
|
||||
|
||||
[linuxmint]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = linuxmint
|
||||
rsync_host = pub.linuxmint.com
|
||||
rsync_dir = pub
|
||||
|
||||
[kernel.org-linux]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = kernel.org/linux
|
||||
rsync_host = rsync.kernel.org
|
||||
rsync_dir = pub/linux/
|
||||
|
||||
[kernel.org-software]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = kernel.org/software
|
||||
rsync_host = rsync.kernel.org
|
||||
rsync_dir = pub/software/
|
||||
|
||||
[apache]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = apache
|
||||
rsync_host = rsync.us.apache.org
|
||||
rsync_dir = apache-dist
|
||||
|
||||
[eclipse]
|
||||
verbose = true
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = twice-daily
|
||||
local_dir = eclipse
|
||||
rsync_host = download.eclipse.org
|
||||
rsync_dir = eclipseMirror
|
|
@ -3,8 +3,8 @@ Description=Manages synchronization of mirrored projects
|
|||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/home/mirror/merlin-go/merlin /home/mirror/merlin-go/merlin-config.ini
|
||||
WorkingDirectory=/home/mirror/merlin
|
||||
ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
|
||||
WorkingDirectory=/home/mirror-go/merlin
|
||||
User=mirror
|
||||
Group=mirror
|
||||
SyslogIdentifier=merlin-go
|
||||
|
|
|
@ -76,7 +76,7 @@ func main() {
|
|||
startIdx := repoIdx
|
||||
|
||||
for numJobsRunning < config.Conf.MaxJobs {
|
||||
if sync.SyncIfPossible(config.Repos[repoIdx]) {
|
||||
if sync.SyncIfPossible(config.Repos[repoIdx], false) {
|
||||
numJobsRunning++
|
||||
}
|
||||
repoIdx = (repoIdx + 1) % len(config.Repos)
|
||||
|
|
|
@ -9,13 +9,13 @@ import (
|
|||
|
||||
// Start sync job for the repo if more than repo.Frequency seconds have elapsed since its last job
|
||||
// and is not currently running. Returns true iff a job is started.
|
||||
func SyncIfPossible(repo *config.Repo) bool {
|
||||
func SyncIfPossible(repo *config.Repo, force bool) bool {
|
||||
if repo.State.IsRunning {
|
||||
return false
|
||||
}
|
||||
|
||||
curTime := time.Now().Unix()
|
||||
if curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
|
||||
if force || curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
|
||||
repo.State.IsRunning = true
|
||||
repo.State.LastAttemptStartTime = curTime
|
||||
repo.SaveState()
|
||||
|
|
Loading…
Reference in New Issue