diff --git a/merlin/Handoff.md b/merlin/Handoff.md index 9f6d36d..dd767d1 100644 --- a/merlin/Handoff.md +++ b/merlin/Handoff.md @@ -1,7 +1,7 @@ ## Status of moving from merlin to merlin-go - [x] debian -- [ ] ubuntu < -- [ ] ubuntu-ports < +- [ ] ubuntu (successful but too slow) +- [ ] ubuntu-ports (successful but too slow) - [x] linuxmint-packages - [x] debian-multimedia - [ ] debian-backports (part 1 broken) @@ -10,7 +10,7 @@ - [x] xubuntu-releases - [ ] puppylinux (broken) - [ ] CPAN (unstable) - +- [ ] CRAN < - [x] CTAN - [ ] fedora-epel (some attrs not transfered) - [x] cygwin @@ -26,20 +26,20 @@ - [ ] eclipse (unstable) - [x] kde - [x] kde-applicationdata - +- [ ] archlinux < - [x] artixlinx - [x] slackware - +- [ ] debian-cd < - [x] x.org - [x] centos - [x] opensuse - [x] FreeBSD - [ ] fedora-enchilada (broken: some attrs not transfered) - [x] ubuntu-ports-releases - +- [ ] gentoo-distfiles < - [x] gentoo-portage - [ ] gutenberg (unstable) - +- [ ] racket-installers < - [ ] plt-bundles (broken) - [x] xiph - [ ] netbsd (broken) @@ -48,9 +48,10 @@ - [x] sagemath - [x] vlc - [x] tdf - +- [ ] saltstack < - [ ] alpine (unstable) -- [ ] raspbian (broekn: high disk usage + long runtime + failing) +- [ ] raspbian (broken: too slow and failing) +- [ ] raspberrypi < - [ ] ipfire (unstable) - [x] manjaro - [x] mxlinux @@ -59,7 +60,7 @@ - [x] trisquel-packages - [x] trisquel-iso - [x] almalinux - +- [ ] ceph < ### Other Repos - debian-cdimage diff --git a/merlin/README.md b/merlin/README.md index 9b3f3b9..403d46a 100644 --- a/merlin/README.md +++ b/merlin/README.md @@ -16,6 +16,8 @@ Then configure `merlin-config.ini` and run using `./merlin` - detect if config file is changed and automatically reload (fsnotify) - sort `arthur status` by last time synced - respond to `arthur` without blocking main thread +- state files appear to be getting extra newlines +- log rotation for log files (also maybe state reporting to prometheus) - split off arthur with a more featureful arthur for debugging - last sync runtime, time until next sync - get expected rsync command or repo config diff --git a/merlin/config/config.go b/merlin/config/config.go index cb0899d..cf1f85a 100644 --- a/merlin/config/config.go +++ b/merlin/config/config.go @@ -61,9 +61,9 @@ func StatusToString(status int) string { case TERMINATED: return "terminated" case SUCCESS: - return "success" + return "completed" case FAILURE: - return "failure" + return "failed" case NOT_RUN_YET: return "not run yet" } diff --git a/merlin/merlin-config.ini b/merlin/merlin-config.ini index ae2e236..e7fc695 100644 --- a/merlin/merlin-config.ini +++ b/merlin/merlin-config.ini @@ -13,6 +13,8 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync sock_path = /mirror/merlin/run/merlin-go.sock +; note: the .ssh keys are only located on /home/mirror/.ssh not at /home/mirror-go/.ssh + [debian] sync_type = csc-sync-debian frequency = bi-hourly @@ -21,6 +23,7 @@ rsync_host = debian.mirror.rafal.ca rsync_dir = debian [ubuntu] +verbose = true sync_type = csc-sync-debian frequency = bi-hourly local_dir = ubuntu @@ -29,6 +32,7 @@ rsync_dir = ubuntu trace_host = drescher.canonical.com [ubuntu-ports] +verbose = true sync_type = csc-sync-debian frequency = bi-hourly local_dir = ubuntu-ports @@ -51,6 +55,7 @@ rsync_host = www.deb-multimedia.org rsync_dir = deb [debian-backports] +verbose = true sync_type = csc-sync-debian frequency = bi-hourly local_dir = debian-backports @@ -95,6 +100,16 @@ local_dir = CPAN rsync_host = cpan-rsync.perl.org rsync_dir = CPAN +[CRAN] +verbose = true +sync_type = csc-sync-ssh +frequency = twice-daily +local_dir = CRAN +rsync_user = cran-rsync +rsync_host = cran.r-project.org +rsync_dir = / +password_file = ~/.ssh/id_cran_rsa + [CTAN] sync_type = csc-sync-standard frequency = twice-daily @@ -204,6 +219,18 @@ local_dir = kde-applicationdata rsync_host = rsync.kde.org rsync_dir = applicationdata +; We are a Tier 1 arch mirror (https://bugs.archlinux.org/task/52853) so our IP is important. +; our rsync_host is rsync://rsync.archlinux.org/ftp_tier1 +[archlinux] +; csc-sync-standard archlinux archlinux.mirror.rafal.ca archlinux +verbose = true +sync_type = csc-sync-archlinux +frequency = five-minutely +local_dir = archlinux +rsync_host = rsync.archlinux.org +rsync_dir = ftp_tier1 +trace_host = http://rsync.archlinux.org/lastupdate + [artixlinux] sync_type = csc-sync-standard frequency = bi-hourly @@ -218,6 +245,14 @@ local_dir = slackware rsync_host = slackware.cs.utah.edu rsync_dir = slackware +[debian-cd] +verbose = true +sync_type = csc-sync-debian-cd +frequency = twice-daily +local_dir = debian-cd +rsync_host = cdimage.debian.org +rsync_dir = debian-cd + [x.org] ; csc-sync-standard x.org xorg.freedesktop.org xorg-archive ; csc-sync-standard x.org mirror.us.leaseweb.net xorg @@ -269,6 +304,16 @@ local_dir = ubuntu-ports-releases rsync_host = cdimage.ubuntu.com rsync_dir = cdimage/releases/ +[gentoo-distfiles] +verbose = true +sync_type = csc-sync-standard +frequency = bi-hourly +local_dir = gentoo-distfiles +rsync_user = gentoo +rsync_host = masterdistfiles.gentoo.org +rsync_dir = gentoo +password_file = /home/mirror-go/passwords/gentoo-distfiles + [gentoo-portage] sync_type = csc-sync-standard frequency = twice-hourly @@ -284,6 +329,14 @@ local_dir = gutenberg rsync_host = ftp@ftp.ibiblio.org rsync_dir = gutenberg +[racket-installers] +verbose = true +; --cut-dirs=1 is hardcoded for csc-sync-wget +sync_type = csc-sync-wget +frequency = twice-daily +local_dir = racket/racket-installers +rsync_host = https://mirror.racket-lang.org/installers/ + [plt-bundles] verbose = true sync_type = csc-sync-standard @@ -345,6 +398,13 @@ local_dir = tdf rsync_host = rsync.documentfoundation.org rsync_dir = tdf-pub +[saltstack] +verbose = true +sync_type = csc-sync-s3 +frequency = daily +local_dir = saltstack +rsync_host = https://s3.repo.saltproject.io + [alpine] verbose = true sync_type = csc-sync-standard @@ -361,6 +421,14 @@ local_dir = raspbian rsync_host = raspbian.freemirror.org rsync_dir = raspbian +[raspberrypi] +verbose = true +sync_type = csc-sync-standard-ipv6 +frequency = bi-hourly +local_dir = raspberrypi +rsync_host = apt-repo.raspberrypi.org +rsync_dir = archive + [ipfire] verbose = true sync_type = csc-sync-standard @@ -417,3 +485,10 @@ frequency = bi-hourly local_dir = almalinux rsync_host = rsync.repo.almalinux.org rsync_dir = almalinux/ + +[ceph] +verbose = true +sync_type = csc-sync-ceph +frequency = tri-hourly +local_dir = ceph +rsync_host = download.ceph.com \ No newline at end of file diff --git a/merlin/sync/sync.go b/merlin/sync/sync.go index 7c97198..4ead8f4 100644 --- a/merlin/sync/sync.go +++ b/merlin/sync/sync.go @@ -55,8 +55,6 @@ func getSyncCommand(repo *config.Repo) (cmds [][]string) { return append(cmds, cscSyncDebianStep1(repo), cscSyncDebianStep2(repo)) case "csc-sync-debian-cd": return append(cmds, cscSyncDebianCD(repo)) - case "csc-sync-gentoo": - return append(cmds, cscSyncGentoo(repo)) case "csc-sync-s3": return append(cmds, cscSyncS3(repo)) case "csc-sync-ssh": @@ -216,7 +214,7 @@ func cscSyncCDImage(repo *config.Repo) []string { func cscSyncCephStep1(repo *config.Repo) []string { args := []string{ - "rsync", "--stats", "--progress", + "nice", "rsync", "--stats", "--progress", repo.RsyncHost + "::ceph", "--recursive", "--times", "--links", "--hard-links", @@ -236,7 +234,7 @@ func cscSyncCephStep1(repo *config.Repo) []string { func cscSyncCephStep2(repo *config.Repo) []string { args := []string{ - "&&", "rsync", "--stats", "--progress", + "nice", "rsync", "--stats", "--progress", repo.RsyncHost + "::ceph", "--recursive", "--times", "--links", "--hard-links", "--delete-after", @@ -291,24 +289,17 @@ func cscSyncDebianCD(repo *config.Repo) []string { return args } -func cscSyncGentoo(repo *config.Repo) []string { - repo.RsyncUser = "gentoo" - repo.PasswordFile = "gentoo-distfiles" - return cscSyncStandard(repo) -} - -// TODO: check for special stuff that rcloning S3 needs func cscSyncS3(repo *config.Repo) []string { + // could not find a argument for "RCLONE_CONFIG_S3_TYPE=s3" + // so an adhoc solution was put in spawnProcess (if possible remove) args := []string{ - // RsyncHost is just a regular host (https://s3.repo.saltproject.io) in this case - "RCLONE_CONFIG_S3_ENDPOINT=" + repo.RsyncHost, - "RCLONE_CONFIG_S3_TYPE=s3", - "RCLONE_CONFIG_S3_PROVIDER=Other", - "RCLONE_CONFIG_S3_ENV_AUTH=false", "nice", "rclone", "sync", "--fast-list", "--use-server-modtime", - "--bind", config.Conf.IPv4Address, + "--bind=" + config.Conf.IPv4Address, + "--s3-endpoint=" + repo.RsyncHost, // RCLONE_CONFIG_S3_ENDPOINT=$2 + "--s3-provider=Other", // RCLONE_CONFIG_S3_PROVIDER=Other + "--s3-env-auth=false", // RCLONE_CONFIG_S3_ENV_AUTH=false "s3:s3/", } args = append(args, buildDownloadDir(repo)) @@ -322,8 +313,7 @@ func cscSyncSSH(repo *config.Repo) []string { "--stats", "-4", } args = append(args, addConditionalFlags(repo, stdFlags|delete)...) - // PasswordFile should point to the SSH_KEYFILE - args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile)) + args = append(args, "-e", fmt.Sprintf("\"ssh -b %s -i %s\"", config.Conf.IPv4Address, repo.PasswordFile)) args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo)) return args @@ -372,12 +362,10 @@ func cscSyncWget(repo *config.Repo) []string { "--no-host-directories", "--cut-dirs=1", "--content-disposition", - "--execute", - "robots=off", + "--execute", "robots=off", "--recursive", "--reject", "\"*\\?*\"", "--directory-prefix=" + buildDownloadDir(repo), - // RsyncHost is just a regular host (https://mirror.racket-lang.org/installers/) in this case repo.RsyncHost, } diff --git a/merlin/sync/utils.go b/merlin/sync/utils.go index 11334c2..6f90f35 100644 --- a/merlin/sync/utils.go +++ b/merlin/sync/utils.go @@ -28,6 +28,14 @@ func spawnProcess(repo *config.Repo, args []string) (ch <-chan *exec.Cmd) { } cmd := exec.Command(args[0], args[1:]...) + + // This appears to be the only way to set env variables for exec.Command + // The other env varaibles for rclone s3 were converted to arguments + // but I was unable to find a way to convert this one. + if repo.SyncType == "csc-sync-s3" { + cmd.Env = append(cmd.Env, "RCLONE_CONFIG_S3_TYPE=s3") + } + repo.Logger.Debug("Starting process") if err := cmd.Start(); err != nil {