public
/
mirror
10
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Move more repos to merlin-go

This commit is contained in:
Andrew Wang 2022-06-28 03:36:51 +00:00
parent 392396d1a0
commit 2492d533f0
11 changed files with 207 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
merlin/Handoff.md Normal file
View File

@ -0,0 +1,63 @@
## Status of moving from merlin to merlin-go
- [ ] debian
- [ ] ubuntu
- [ ] ubuntu-ports
- [ ] linuxmint-packages
- [ ] debian-multimedia
- [ ] debian-backports
- [ ] debian-security
- [ ] ubuntu-releases
- [ ] xubuntu-releases
- [ ] puppylinux <
- [x] CPAN
- [ ] CRAN
- [ ] CTAN <
- [ ] fedora-epel
- [ ] cygwin <
- [ ] gnu <
- [ ] nognu <
- [ ] mysql <
- [ ] gnome <
- [ ] damnsmalllinux <
- [ ] linuxmint <
- [ ] kernel.org-linux <
- [ ] kernel.org-software <
- [ ] apache <
- [ ] eclipse <
- [ ] kde
- [ ] kde-appplicationdata
- [ ] archlinux
- [ ] artixlinux
- [ ] slackware
- [ ] debian-cd
- [ ] x.org
- [ ] centos
- [ ] opensuse
- [ ] FreeBSD
- [ ] fedora-enchilada
- [ ] ubuntu-ports-releases
- [ ] gentoo-distfiles
- [ ] gentoo-portage
- [ ] gutenberg
- [ ] racket-installers
- [ ] plt-bundles
- [ ] xiph
- [ ] netbsd
- [ ] macports-release
- [ ] macports-distfiles
- [ ] sagemath
- [ ] vlc
- [ ] tdf
- [ ] saltstack
- [ ] alpine
- [ ] raspbian
- [ ] raspberrypi
- [ ] ipfire
- [ ] manjaro
- [ ] mxlinux
- [ ] mxlinux-iso
- [ ] parabola
- [ ] trisquel-packages
- [ ] trisquel-iso
- [ ] almalinux
- [ ] ceph

2
merlin/arthur/arthur.go
View File

@ -85,7 +85,7 @@ func ForceSync(conn net.Conn, repoName string) (newSync bool) {
if repo, isInMap := config.RepoMap[repoName]; isInMap {
logger.OutLog("Attempting to force sync of " + repoName)
if sync.SyncIfPossible(repo) {
if sync.SyncIfPossible(repo, true) {
conn.Write([]byte("Forced sync for " + repoName))
newSync = true
} else {

2
merlin/arthur/arthur_test.go
View File

@ -114,7 +114,7 @@ func TestForceSync(t *testing.T) {
SyncType: "csc-sync-dummy",
Frequency: 7 * 86400,
MaxTime: 30,
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_logs", false),
Logger: logger.NewLogger("nux", "/tmp/merlin_force_sync_test_log", false),
StateFile: "/tmp/merlin_force_sync_test_state",
DoneChan: doneChan,
State: &config.RepoState{

BIN
merlin/cmd/arthur/arthur Executable file
View File

Binary file not shown.

12
merlin/config/config.go
View File

@ -30,9 +30,9 @@ const (
DEFAULT_DOWNLOAD_DIR = "/mirror/root"
DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace"
DEFAULT_STATE_DIR = "/home/mirror/merlin/state"
DEFAULT_LOG_DIR = "/home/mirror/merlin/logs"
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/logs-rsync"
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/logs-zfssync"
DEFAULT_LOG_DIR = "/home/mirror/merlin/log"
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync"
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/log-zfssync"
DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock"
)
@ -84,11 +84,11 @@ type Config struct {
// directory where the state of each repo is saved
StateDir string `ini:"state_dir"`
// directory where merlin will store the merlin logs for each repo
RepoLogDir string `ini:"repo_logs_dir"`
RepoLogDir string `ini:"repo_log_dir"`
// directory to store the rsync logs for each repo
RsyncLogDir string `ini:"rsync_logs_dir"`
RsyncLogDir string `ini:"rsync_log_dir"`
// directory to store the zfssync logs for each repo
ZfssyncLogDir string `ini:"zfssync_logs_dir"`
ZfssyncLogDir string `ini:"zfssync_log_dir"`
// path to the unix socket for arthur to use for communication
SockPath string `ini:"sock_path"`
}

8
merlin/config/config_test.go
View File

@ -46,7 +46,7 @@ func TestLoadConfig(t *testing.T) {
DefaultMaxRsyncIO: 0,
DownloadDir: "test_files/download",
StateDir: "test_files",
RepoLogDir: "test_files/logs",
RepoLogDir: "test_files/log",
RsyncLogDir: "test_files/rsync",
ZfssyncLogDir: "test_files/zfssync",
SockPath: "test_files/test.sock",
@ -62,7 +62,7 @@ func TestLoadConfig(t *testing.T) {
RsyncHost: "rsync.releases.eelinux.ca",
RsyncDir: "releases",
StateFile: "test_files/eelinux",
RepoLogFile: "test_files/logs/eelinux.log",
RepoLogFile: "test_files/log/eelinux.log",
Logger: Repos[0].Logger,
RsyncLogFile: "test_files/rsync/eelinux-rsync.log",
ZfssyncLogFile: "test_files/zfssync/eelinux-zfssync.log",
@ -86,7 +86,7 @@ func TestLoadConfig(t *testing.T) {
RsyncHost: "rsync.releases.yoland.io",
RsyncDir: "releases",
StateFile: "test_files/yoland",
RepoLogFile: "test_files/logs/yoland.log",
RepoLogFile: "test_files/log/yoland.log",
Logger: Repos[1].Logger,
RsyncLogFile: "test_files/rsync/yoland-rsync.log",
ZfssyncLogFile: "test_files/zfssync/yoland-zfssync.log",
@ -117,7 +117,7 @@ func TestLoadConfig(t *testing.T) {
}
os.Remove("test_files/yoland")
os.RemoveAll("test_files/logs")
os.RemoveAll("test_files/log")
os.RemoveAll("test_files/rsync")
os.RemoveAll("test_files/zfssync")
}

6
merlin/config/config_test.ini
View File

@ -8,9 +8,9 @@ default_frequency = daily
default_max_time = 1000
download_dir = test_files/download
state_dir = test_files
repo_logs_dir = test_files/logs
rsync_logs_dir = test_files/rsync
zfssync_logs_dir = test_files/zfssync
repo_log_dir = test_files/log
rsync_log_dir = test_files/rsync
zfssync_log_dir = test_files/zfssync
sock_path = test_files/test.sock
[eelinux]

127
merlin/merlin-config.ini
View File

@ -1,19 +1,140 @@
ipv4_address = 129.97.134.71
ipv6_address = 2620:101:f000:4901:c5c::f:1055
; max_jobs = 6
download_dir = /mirror/root
trace_dir = /home/mirror-go/merlin/trace
state_dir = /home/mirror-go/merlin/state
repo_logs_dir = /home/mirror-go/merlin/logs
rsync_logs_dir = /home/mirror-go/merlin/logs-rsync
zfssync_logs_dir = /home/mirror-go/merlin/logs-zfssync
repo_log_dir = /home/mirror-go/merlin/log
rsync_log_dir = /home/mirror-go/merlin/log-rsync
zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
sock_path = /mirror/merlin/run/merlin-go.sock
[puppylinux]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = puppylinux
rsync_host = distro.ibiblio.org
rsync_dir = puppylinux
[CPAN]
verbose = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = CPAN
rsync_host = cpan-rsync.perl.org
rsync_dir = CPAN
[CTAN]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = CTAN
rsync_host = rsync.dante.ctan.org
rsync_dir = CTAN
[cygwin]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = cygwin
rsync_host = cygwin.com
rsync_dir = cygwin-ftp
[gnu]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = gnu
rsync_host = ftp.gnu.org
rsync_dir = gnu
[nognu]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = nongnu
rsync_host = dl.sv.gnu.org
rsync_dir = releases
[mysql]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = mysql
rsync_host = rsync.mirrorservice.org
rsync_dir = ftp.mysql.com
[gnome]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = gnome
rsync_host = master.gnome.org
rsync_dir = gnomeftp
password_file = /home/mirror/passwords/gnome
[damnsmalllinux]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = damnsmalllinux
rsync_host = ftp.heanet.ie
rsync_dir = mirrors/damnsmalllinux.org/
[linuxmint]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = linuxmint
rsync_host = pub.linuxmint.com
rsync_dir = pub
[kernel.org-linux]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = kernel.org/linux
rsync_host = rsync.kernel.org
rsync_dir = pub/linux/
[kernel.org-software]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = kernel.org/software
rsync_host = rsync.kernel.org
rsync_dir = pub/software/
[apache]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = apache
rsync_host = rsync.us.apache.org
rsync_dir = apache-dist
[eclipse]
verbose = true
dry_run = true
sync_type = csc-sync-standard
frequency = twice-daily
local_dir = eclipse
rsync_host = download.eclipse.org
rsync_dir = eclipseMirror

4
merlin/merlin-go.service
View File

@ -3,8 +3,8 @@ Description=Manages synchronization of mirrored projects
After=network.target
[Service]
ExecStart=/home/mirror/merlin-go/merlin /home/mirror/merlin-go/merlin-config.ini
WorkingDirectory=/home/mirror/merlin
ExecStart=/home/mirror-go/merlin/merlin /home/mirror-go/merlin/merlin-config.ini
WorkingDirectory=/home/mirror-go/merlin
User=mirror
Group=mirror
SyslogIdentifier=merlin-go

2
merlin/merlin.go
View File

@ -76,7 +76,7 @@ func main() {
startIdx := repoIdx
for numJobsRunning < config.Conf.MaxJobs {
if sync.SyncIfPossible(config.Repos[repoIdx]) {
if sync.SyncIfPossible(config.Repos[repoIdx], false) {
numJobsRunning++
}
repoIdx = (repoIdx + 1) % len(config.Repos)

4
merlin/sync/interface.go
View File

@ -9,13 +9,13 @@ import (
// Start sync job for the repo if more than repo.Frequency seconds have elapsed since its last job
// and is not currently running. Returns true iff a job is started.
func SyncIfPossible(repo *config.Repo) bool {
func SyncIfPossible(repo *config.Repo, force bool) bool {
if repo.State.IsRunning {
return false
}
curTime := time.Now().Unix()
if curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
if force || curTime-repo.State.LastAttemptStartTime > int64(repo.Frequency) {
repo.State.IsRunning = true
repo.State.LastAttemptStartTime = curTime
repo.SaveState()