fix up csc-sync-debian
This commit is contained in:
parent
dcac3ad47b
commit
7a3c8c1c93
|
@ -1,18 +1,18 @@
|
|||
## Status of moving from merlin to merlin-go
|
||||
- [ ] debian
|
||||
- [ ] ubuntu
|
||||
- [ ] ubuntu-ports
|
||||
- [ ] linuxmint-packages
|
||||
- [ ] debian-multimedia
|
||||
- [ ] debian-backports
|
||||
- [ ] debian-security
|
||||
- [ ] debian <
|
||||
- [ ] ubuntu <
|
||||
- [ ] ubuntu-ports <
|
||||
- [ ] linuxmint-packages <
|
||||
- [ ] debian-multimedia <
|
||||
- [ ] debian-backports <
|
||||
- [ ] debian-security <
|
||||
- [x] ubuntu-releases
|
||||
- [x] xubuntu-releases
|
||||
- [ ] puppylinux (broken)
|
||||
- [ ] CPAN (unstable)
|
||||
- [ ] CRAN
|
||||
<!-- - [ ] CRAN -->
|
||||
- [x] CTAN
|
||||
- [ ] fedora-epel <
|
||||
- [ ] fedora-epel (some attrs not transfered)
|
||||
- [x] cygwin
|
||||
- [x] gnu
|
||||
- [x] nongnu
|
||||
|
@ -26,20 +26,20 @@
|
|||
- [ ] eclipse (unstable)
|
||||
- [x] kde
|
||||
- [x] kde-applicationdata
|
||||
- [ ] archlinux
|
||||
<!-- - [ ] archlinux -->
|
||||
- [x] artixlinx
|
||||
- [x] slackware
|
||||
- [ ] debian-cd
|
||||
<!-- - [ ] debian-cd -->
|
||||
- [x] x.org
|
||||
- [x] centos
|
||||
- [ ] opensuse <
|
||||
- [x] opensuse
|
||||
- [x] FreeBSD
|
||||
- [ ] fedora-enchilada <
|
||||
- [ ] fedora-enchilada (broken: some attrs not transfered)
|
||||
- [x] ubuntu-ports-releases
|
||||
- [ ] gentoo-distfiles
|
||||
<!-- - [ ] gentoo-distfiles -->
|
||||
- [x] gentoo-portage
|
||||
- [ ] gutenberg (broken)
|
||||
- [ ] racket-installers
|
||||
- [ ] gutenberg (unstable)
|
||||
<!-- - [ ] racket-installers -->
|
||||
- [ ] plt-bundles (broken)
|
||||
- [x] xiph
|
||||
- [ ] netbsd (broken)
|
||||
|
@ -48,11 +48,10 @@
|
|||
- [x] sagemath
|
||||
- [x] vlc
|
||||
- [x] tdf
|
||||
- [ ] saltstack
|
||||
- [x] alpine
|
||||
- [ ] raspbian (high disk usage + long runtime + failing)
|
||||
- [ ] raspberrypi
|
||||
- [ ] ipfire (broken)
|
||||
<!-- - [ ] saltstack -->
|
||||
- [ ] alpine (unstable)
|
||||
- [ ] raspbian (broekn: high disk usage + long runtime + failing)
|
||||
- [ ] ipfire (unstable)
|
||||
- [x] manjaro
|
||||
- [x] mxlinux
|
||||
- [x] mxlinux-iso
|
||||
|
@ -60,4 +59,19 @@
|
|||
- [x] trisquel-packages
|
||||
- [x] trisquel-iso
|
||||
- [x] almalinux
|
||||
- [ ] ceph
|
||||
<!-- - [ ] ceph -->
|
||||
|
||||
### Other Repos
|
||||
- debian-cdimage
|
||||
- debian-volatile
|
||||
- emdebian
|
||||
- openoffice
|
||||
- mozdev
|
||||
- mozilla.org
|
||||
- raspberrypi
|
||||
- cs136
|
||||
- kali
|
||||
- kali-images
|
||||
- hyperbola-sources
|
||||
- hyperbola-stable
|
||||
- hyperbola-testing
|
|
@ -13,11 +13,12 @@ Then configure `merlin-config.ini` and run using `./merlin`
|
|||
|
||||
### Nice Features To Add
|
||||
- detect if an rsync process is stuck (watch the stdout/stderr of the rsync processes)
|
||||
- get the config or the rsync commond that a repo will sync with using a cli tool
|
||||
- get the config or the rsync command that a repo will sync with using a cli tool
|
||||
- improve conversion from exit status enum to string
|
||||
- sort `arthur status` by last time synced
|
||||
- last sync runtime, time until next sync
|
||||
- when sync fails, then make a copy of the rsync logs
|
||||
- implementation of `trace` is ad hoc and can be improved
|
||||
|
||||
### Completed
|
||||
- [x] add bwlimit option for each rsync process
|
||||
|
|
|
@ -22,13 +22,13 @@ const (
|
|||
FIVE_MINUTELY = 300
|
||||
MINUTELY = 60
|
||||
|
||||
DEAFULT_HOSTNAME = "mirror.csclub.uwaterloo.ca"
|
||||
DEFAULT_MAX_JOBS = 6
|
||||
DEFAULT_MAX_TIME = DAILY / 4
|
||||
DEFAULT_MAX_RSYNC_IO = 0
|
||||
DEFAULT_SYNC_TYPE = "csc-sync-standard"
|
||||
DEFAULT_FREQUENCY_STRING = "bi-hourly"
|
||||
DEFAULT_DOWNLOAD_DIR = "/mirror/root"
|
||||
DEFAULT_TRACE_DIR = "/home/mirror/merlin/trace"
|
||||
DEFAULT_STATE_DIR = "/home/mirror/merlin/state"
|
||||
DEFAULT_LOG_DIR = "/home/mirror/merlin/log"
|
||||
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync"
|
||||
|
@ -62,6 +62,7 @@ type SyncResult struct {
|
|||
}
|
||||
|
||||
type Config struct {
|
||||
Hostname string `ini:"hostname"`
|
||||
// the IPv4 addresses to use for rsync
|
||||
IPv4Address string `ini:"ipv4_address"`
|
||||
// the IPv6 addresses to use for rsync
|
||||
|
@ -79,8 +80,6 @@ type Config struct {
|
|||
DefaultFrequencyStr string `ini:"default_frequency"`
|
||||
// the base directory where rsync should download files to
|
||||
DownloadDir string `ini:"download_dir"`
|
||||
// directory where the trace of each repo is saved
|
||||
TraceDir string `ini:"trace_dir"`
|
||||
// directory where the state of each repo is saved
|
||||
StateDir string `ini:"state_dir"`
|
||||
// directory where merlin will store the merlin logs for each repo
|
||||
|
@ -113,7 +112,7 @@ type Repo struct {
|
|||
// where to download the files for this repo (relative to Conf.DownloadDir)
|
||||
LocalDir string `ini:"local_dir"`
|
||||
// the address to the trace file (how this url will be used depends on SyncType)
|
||||
TraceUrl string `ini:"trace_url"`
|
||||
TraceHost string `ini:"trace_host"`
|
||||
// the address to the remote host to rsync from
|
||||
RsyncHost string `ini:"rsync_host"`
|
||||
// the remote directory on the rsync host (optional)
|
||||
|
@ -170,13 +169,13 @@ var (
|
|||
func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struct{}) {
|
||||
// create a new config with the default values then load config from file
|
||||
newConf := Config{
|
||||
Hostname: DEAFULT_HOSTNAME,
|
||||
MaxJobs: DEFAULT_MAX_JOBS,
|
||||
DefaultMaxTime: DEFAULT_MAX_TIME,
|
||||
DefaultMaxRsyncIO: DEFAULT_MAX_RSYNC_IO,
|
||||
DefaultSyncType: DEFAULT_SYNC_TYPE,
|
||||
DefaultFrequencyStr: DEFAULT_FREQUENCY_STRING,
|
||||
DownloadDir: DEFAULT_DOWNLOAD_DIR,
|
||||
TraceDir: DEFAULT_TRACE_DIR,
|
||||
StateDir: DEFAULT_STATE_DIR,
|
||||
RepoLogDir: DEFAULT_LOG_DIR,
|
||||
RsyncLogDir: DEFAULT_RSYNC_LOG_DIR,
|
||||
|
@ -201,7 +200,6 @@ func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struc
|
|||
|
||||
// create directories
|
||||
for _, dir := range []string{
|
||||
newConf.TraceDir,
|
||||
newConf.StateDir,
|
||||
newConf.RepoLogDir,
|
||||
newConf.RsyncLogDir,
|
||||
|
|
|
@ -40,7 +40,7 @@ frequency = bi-hourly
|
|||
local_dir = ubuntu
|
||||
rsync_host = archive.ubuntu.com
|
||||
rsync_dir = ubuntu
|
||||
; trace_host = drescher.canonical.com
|
||||
trace_host = drescher.canonical.com
|
||||
|
||||
[ubuntu-ports]
|
||||
sync_type = csc-sync-debian
|
||||
|
@ -48,7 +48,7 @@ frequency = bi-hourly
|
|||
local_dir = ubuntu-ports
|
||||
rsync_host = ports.ubuntu.com
|
||||
rsync_dir = ubuntu-ports
|
||||
; trace_host = drescher.canonical.com
|
||||
trace_host = drescher.canonical.com
|
||||
|
||||
[linuxmint-packages]
|
||||
sync_type = csc-sync-debian
|
||||
|
|
|
@ -15,15 +15,15 @@ sock_path = /mirror/merlin/run/merlin-go.sock
|
|||
|
||||
[ubuntu]
|
||||
dry_run = true
|
||||
sync_type = csc-sync-standard
|
||||
sync_type = csc-sync-debian
|
||||
frequency = five-minutely
|
||||
local_dir = ubuntu
|
||||
rsync_host = rsync.releases.ubuntu.com
|
||||
rsync_dir = releases
|
||||
rsync_exclude = distribution/.timestamp_invisiable, .ignore_me
|
||||
; rsync_exclude = distribution/.timestamp_invisiable, .ignore_me
|
||||
|
||||
; add to the repo struct
|
||||
; RsyncExclude []string `ini:"rsync_exclude"`
|
||||
|
||||
; then iterate over RsyncExclude:
|
||||
; for _, exclude := range repo.RsyncExclude {}
|
||||
; for _, exclude := range repo.RsyncExclude {}
|
||||
|
|
|
@ -13,6 +13,72 @@ zfssync_log_dir = /home/mirror-go/merlin/log-zfssync
|
|||
|
||||
sock_path = /mirror/merlin/run/merlin-go.sock
|
||||
|
||||
[debian]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = bi-hourly
|
||||
local_dir = debian
|
||||
rsync_host = debian.mirror.rafal.ca
|
||||
rsync_dir = debian
|
||||
|
||||
[ubuntu]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = bi-hourly
|
||||
local_dir = ubuntu
|
||||
rsync_host = archive.ubuntu.com
|
||||
rsync_dir = ubuntu
|
||||
trace_host = drescher.canonical.com
|
||||
|
||||
[ubuntu-ports]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = bi-hourly
|
||||
local_dir = ubuntu-ports
|
||||
rsync_host = ports.ubuntu.com
|
||||
rsync_dir = ubuntu-ports
|
||||
trace_host = drescher.canonical.com
|
||||
|
||||
[linuxmint-packages]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = bi-hourly
|
||||
local_dir = linuxmint-packages
|
||||
rsync_host = rsync-packages.linuxmint.com
|
||||
rsync_dir = packages
|
||||
|
||||
[debian-multimedia]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = bi-hourly
|
||||
local_dir = debian-multimedia
|
||||
rsync_host = www.deb-multimedia.org
|
||||
rsync_dir = deb
|
||||
|
||||
[debian-backports]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = bi-hourly
|
||||
local_dir = debian-backports
|
||||
rsync_host = debian.mirror.rafal.ca
|
||||
rsync_dir = debian-backports
|
||||
|
||||
[debian-security]
|
||||
dry_run = true
|
||||
verbose = true
|
||||
sync_type = csc-sync-debian
|
||||
frequency = twice-hourly
|
||||
local_dir = debian-security
|
||||
rsync_host = rsync.security.debian.org
|
||||
rsync_dir = debian-security
|
||||
trace_host = security-master.debian.org
|
||||
|
||||
[ubuntu-releases]
|
||||
sync_type = csc-sync-standard
|
||||
frequency = bi-hourly
|
||||
|
@ -183,7 +249,6 @@ rsync_host = us-msync.centos.org
|
|||
rsync_dir = CentOS
|
||||
|
||||
[opensuse]
|
||||
verbose = true
|
||||
; unsure what this setting is supposed to do
|
||||
; #"--exclude distribution/.timestamp_invisible"
|
||||
sync_type = csc-sync-standard
|
||||
|
@ -295,6 +360,7 @@ rsync_host = rsync.documentfoundation.org
|
|||
rsync_dir = tdf-pub
|
||||
|
||||
[alpine]
|
||||
verbose = true
|
||||
sync_type = csc-sync-standard
|
||||
frequency = hourly
|
||||
local_dir = alpine
|
||||
|
|
|
@ -50,5 +50,6 @@ func SyncCompleted(repo *config.Repo, exit int) {
|
|||
|
||||
if exit == config.SUCCESS {
|
||||
go zfsSync(repo)
|
||||
go postSyncTraceUpdate(repo)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -87,12 +87,12 @@ func startRepoSync(repo *config.Repo) {
|
|||
}
|
||||
}()
|
||||
|
||||
if repo.TraceUrl != "" {
|
||||
if repo.TraceHost != "" {
|
||||
repo.Logger.Debug("Checking for changes")
|
||||
|
||||
continueSync := true
|
||||
status, continueSync = checkIfSyncNeeded(repo)
|
||||
if !continueSync {
|
||||
if !continueSync || status == config.TERMINATED {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -39,12 +39,17 @@ func buildDownloadDir(repo *config.Repo) string {
|
|||
const (
|
||||
noOwnerNoGroup = 1 << iota
|
||||
timeout3600
|
||||
excludeTmp
|
||||
logFile
|
||||
quiet
|
||||
ipv4
|
||||
ipv6
|
||||
delete
|
||||
delayUpdatesDeleteAfter
|
||||
// adds base arguments for setting timeout, a logfile, and including quiet
|
||||
baseFlags = timeout3600 | logFile | quiet
|
||||
// adds standard arguments timeout, logging, quiet, no owner/group, and excluding .~tmp~/
|
||||
stdFlags = baseFlags | noOwnerNoGroup | excludeTmp
|
||||
)
|
||||
|
||||
func addConditionalFlags(repo *config.Repo, flags int) []string {
|
||||
|
@ -55,10 +60,13 @@ func addConditionalFlags(repo *config.Repo, flags int) []string {
|
|||
if flags&timeout3600 != 0 {
|
||||
args = append(args, "--timeout=3600")
|
||||
}
|
||||
if flags&excludeTmp != 0 {
|
||||
args = append(args, "--exclude", ".~tmp~/")
|
||||
}
|
||||
if flags&logFile != 0 {
|
||||
args = append(args, "--log-file="+repo.RsyncLogFile)
|
||||
}
|
||||
if flags&quiet != 0 {
|
||||
if flags&quiet != 0 && !repo.Verbose {
|
||||
args = append(args, "--quiet")
|
||||
}
|
||||
|
||||
|
@ -92,9 +100,8 @@ func cscSyncApache(repo *config.Repo) []string {
|
|||
"--delete",
|
||||
"--safe-links",
|
||||
"--stats",
|
||||
"--exclude", ".~tmp~/",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4)...)
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|ipv4)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
@ -122,10 +129,9 @@ func cscSyncBadPerms(repo *config.Repo) []string {
|
|||
args := []string{
|
||||
"nice", "rsync", "-aH",
|
||||
"--chmod=o=rX",
|
||||
"--exclude", ".~tmp~/",
|
||||
"--stats",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delete)...)
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delete)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
@ -137,7 +143,7 @@ func cscSyncCDImage(repo *config.Repo) []string {
|
|||
"--exclude", "\".*/\"",
|
||||
"--stats",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delete)...)
|
||||
args = append(args, addConditionalFlags(repo, baseFlags|noOwnerNoGroup|ipv4|delete)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
@ -176,49 +182,37 @@ func cscSyncChmod(repo *config.Repo) []string {
|
|||
args := []string{
|
||||
"nice", "rsync", "-aH",
|
||||
"--safe-links",
|
||||
"--exclude", ".~tmp~/",
|
||||
"--stats",
|
||||
"--chmod=u=rwX,go=rX",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func cscSyncDebian(repo *config.Repo) []string {
|
||||
// some repos specify a trace file but does not exit script even if trace files are the same
|
||||
// https://git.csclub.uwaterloo.ca/public/mirror/src/branch/go/bin/csc-sync-debian#L205-L218
|
||||
|
||||
// ubuntu repos are trying to get trace/drescher.canonical.com but this file does not exist
|
||||
// http://archive.ubuntu.com/ubuntu/project/trace/
|
||||
// if are adding a trace check ensure it can handle when the file does not exist on the remote
|
||||
|
||||
args := []string{
|
||||
"nice", "rsync", "-rlHtvp",
|
||||
"--exclude", ".~tmp~/",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, timeout3600|logFile|quiet|ipv4)...)
|
||||
args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo)+"/pool/", buildDownloadDir(repo)+"/pool/")
|
||||
|
||||
args = append(args, []string{
|
||||
"&&", "nice", "rsync", "-rlHtvp",
|
||||
"--exclude", ".~tmp~/",
|
||||
"--exclude", filepath.Join("project/trace", config.Conf.Hostname),
|
||||
}...)
|
||||
args = append(args, addConditionalFlags(repo, timeout3600|logFile|quiet|ipv4|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||||
|
||||
// TODO: run 'LANG=C date -u > "${TO}/${TRACE_DIR}/${HOSTNAME}"' after the sync is done
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
func cscSyncDebianCD(repo *config.Repo) []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-rlHtvp4",
|
||||
"--exclude", ".~tmp~/",
|
||||
"nice", "rsync", "-rlHtvp",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, timeout3600|logFile|quiet|ipv4|delete)...)
|
||||
args = append(args, addConditionalFlags(repo, baseFlags|excludeTmp|ipv4|delete)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
@ -252,10 +246,9 @@ func cscSyncS3(repo *config.Repo) []string {
|
|||
func cscSyncSSH(repo *config.Repo) []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-aH",
|
||||
"--exclude", ".~tmp~/",
|
||||
"--stats", "-4",
|
||||
}
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|delete)...)
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|delete)...)
|
||||
// PasswordFile should point to the SSH_KEYFILE
|
||||
args = append(args, "-e", fmt.Sprintf("'ssh -b %s -i %s'", config.Conf.IPv4Address, repo.PasswordFile))
|
||||
args = append(args, buildRsyncSSHHost(repo), buildDownloadDir(repo))
|
||||
|
@ -267,7 +260,6 @@ func cscSyncStandard(repo *config.Repo) []string {
|
|||
args := []string{
|
||||
"nice", "rsync", "-aH",
|
||||
"--safe-links",
|
||||
"--exclude", ".~tmp~/",
|
||||
"--stats",
|
||||
}
|
||||
|
||||
|
@ -275,7 +267,7 @@ func cscSyncStandard(repo *config.Repo) []string {
|
|||
args = append(args, "--password-file", repo.PasswordFile)
|
||||
}
|
||||
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv4|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|ipv4|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, buildRsyncHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
@ -285,7 +277,6 @@ func cscSyncStandardIPV6(repo *config.Repo) []string {
|
|||
args := []string{
|
||||
"nice", "rsync", "-aH",
|
||||
"--safe-links",
|
||||
"--exclude", ".~tmp~/",
|
||||
"--stats",
|
||||
}
|
||||
|
||||
|
@ -293,7 +284,7 @@ func cscSyncStandardIPV6(repo *config.Repo) []string {
|
|||
args = append(args, "--password-file", repo.PasswordFile)
|
||||
}
|
||||
|
||||
args = append(args, addConditionalFlags(repo, noOwnerNoGroup|timeout3600|logFile|quiet|ipv6|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, addConditionalFlags(repo, stdFlags|ipv6|delayUpdatesDeleteAfter)...)
|
||||
args = append(args, buildRsyncDaemonHost(repo), buildDownloadDir(repo))
|
||||
|
||||
return args
|
||||
|
|
|
@ -2,24 +2,26 @@ package sync
|
|||
|
||||
import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"git.csclub.uwaterloo.ca/public/merlin/config"
|
||||
)
|
||||
|
||||
func cscTraceArchLinux(repo *config.Repo, destPath string) []string {
|
||||
func cscTraceArchLinux(repo *config.Repo, newTime string) []string {
|
||||
args := []string{
|
||||
"curl",
|
||||
"-s", repo.TraceUrl,
|
||||
"-o", destPath,
|
||||
"--interface", config.Conf.IPv4Address,
|
||||
"-s", repo.TraceHost,
|
||||
"-o", newTime,
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
// return false iff both files exist, are non-empty, and have the same contents
|
||||
func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool {
|
||||
func cscTraceArchLinuxDiff(repo *config.Repo, newTime string) bool {
|
||||
readFile := func(file string) string {
|
||||
f, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
|
@ -29,8 +31,8 @@ func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool {
|
|||
return strings.TrimSpace(string(f))
|
||||
}
|
||||
|
||||
file1 := readFile(destPath)
|
||||
file2 := readFile(path.Join(buildDownloadDir(repo), "lastupdate"))
|
||||
file1 := readFile(newTime)
|
||||
file2 := readFile(filepath.Join(buildDownloadDir(repo), "lastupdate"))
|
||||
|
||||
if file1 == "" || file2 == "" {
|
||||
return true
|
||||
|
@ -38,6 +40,7 @@ func cscTraceArchLinuxDiff(repo *config.Repo, destPath string) bool {
|
|||
return file1 != file2
|
||||
}
|
||||
|
||||
// update the lastsync file
|
||||
func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) {
|
||||
rsyncDir := repo.RsyncDir
|
||||
localDir := repo.LocalDir
|
||||
|
@ -53,21 +56,20 @@ func cscTraceArchLinuxUpdate(repo *config.Repo) (args []string) {
|
|||
return args
|
||||
}
|
||||
|
||||
func cscTraceDebian(repo *config.Repo, destPath string) []string {
|
||||
func cscTraceDebian(repo *config.Repo, newTime string) []string {
|
||||
args := []string{
|
||||
"nice", "rsync", "-tv", "-4",
|
||||
"--address=" + config.Conf.IPv4Address,
|
||||
"--quiet",
|
||||
// "--log-file=" + repo.RepoLogFile,
|
||||
repo.RsyncHost + "::" + path.Join(repo.RsyncDir, "project/trace", repo.TraceUrl),
|
||||
destPath,
|
||||
repo.RsyncHost + "::" + filepath.Join(repo.RsyncDir, "project/trace", repo.TraceHost),
|
||||
newTime,
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
// return false iff both files exist and were modified at the same time
|
||||
func cscTraceDebianDiff(repo *config.Repo, destPath string) bool {
|
||||
func cscTraceDebianDiff(repo *config.Repo, newTime string) bool {
|
||||
statFile := func(file string) int64 {
|
||||
f, err := os.Stat(file)
|
||||
if err != nil {
|
||||
|
@ -78,8 +80,8 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool {
|
|||
return f.ModTime().Unix()
|
||||
}
|
||||
|
||||
file1 := statFile(destPath)
|
||||
file2 := statFile(path.Join(buildDownloadDir(repo), "project/trace", "lastupdate"))
|
||||
file1 := statFile(newTime)
|
||||
file2 := statFile(filepath.Join(buildDownloadDir(repo), "project/trace", repo.TraceHost))
|
||||
|
||||
if file1 == 0 || file2 == 0 {
|
||||
return true
|
||||
|
@ -88,15 +90,44 @@ func cscTraceDebianDiff(repo *config.Repo, destPath string) bool {
|
|||
return file1 != file2
|
||||
}
|
||||
|
||||
// update our trace file's modification date by writing the current time
|
||||
func cscTraceDebianUpdate(repo *config.Repo) {
|
||||
target := filepath.Join(buildDownloadDir(repo), "project/trace", config.Conf.Hostname)
|
||||
|
||||
os.MkdirAll(filepath.Join(buildDownloadDir(repo), "project/trace"), 0755)
|
||||
|
||||
f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
repo.Logger.Error("Unable to open trace file: " + target)
|
||||
return
|
||||
}
|
||||
if _, err = f.WriteString(time.Now().UTC().Format(time.RFC1123)); err != nil {
|
||||
repo.Logger.Error("Unable to write to trace file: " + target)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// some repos want us to check a last modification time file before performing a full sync
|
||||
// returns a status to update the original sync's status along with if the sync should continue
|
||||
func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
|
||||
// default return values are the default status and to continue the sync
|
||||
status = config.FAILURE
|
||||
continueSync = true
|
||||
if repo.DryRun {
|
||||
return
|
||||
}
|
||||
runCommand := func(args []string) int {
|
||||
ch := spawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// spawnSyncProcess will have already logged error
|
||||
return 1
|
||||
}
|
||||
cmd := <-ch
|
||||
return cmd.ProcessState.ExitCode()
|
||||
}
|
||||
|
||||
// create a temp file
|
||||
temp, err := os.CreateTemp("/tmp", ".tmp.merlin."+repo.Name+"-")
|
||||
temp, err := os.CreateTemp("/tmp", "merlin-"+repo.Name+"-trace-*")
|
||||
if err != nil {
|
||||
repo.Logger.Error(err.Error())
|
||||
return
|
||||
|
@ -104,7 +135,7 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
|
|||
temp.Close()
|
||||
defer os.Remove(temp.Name())
|
||||
|
||||
// get the trace command
|
||||
// get the trace command to copy a last updated file
|
||||
var args []string
|
||||
switch repo.SyncType {
|
||||
case "csc-sync-archlinux":
|
||||
|
@ -116,16 +147,10 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
|
|||
return
|
||||
}
|
||||
|
||||
ch := spawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// spawnSyncProcess will have already logged error
|
||||
return
|
||||
}
|
||||
|
||||
cmd := <-ch
|
||||
if cmd.ProcessState.ExitCode() != 0 {
|
||||
exit := runCommand(args)
|
||||
if exit != 0 {
|
||||
// if the process was terminated then don't continue to sync
|
||||
if cmd.ProcessState.ExitCode() == -1 {
|
||||
if exit == -1 {
|
||||
return config.TERMINATED, false
|
||||
}
|
||||
return
|
||||
|
@ -138,37 +163,41 @@ func checkIfSyncNeeded(repo *config.Repo) (status int, continueSync bool) {
|
|||
case "csc-sync-debian":
|
||||
filesDiffer = cscTraceDebianDiff(repo, temp.Name())
|
||||
}
|
||||
// debian still syncs even if the trace file is the same
|
||||
if !filesDiffer && repo.SyncType == "csc-sync-debian" {
|
||||
repo.Logger.Error("trace file for " + repo.RsyncHost + " unchanged")
|
||||
return
|
||||
}
|
||||
if filesDiffer {
|
||||
// continue sync if files differ
|
||||
return
|
||||
}
|
||||
|
||||
// if the files are the same then don't continue to sync
|
||||
if !filesDiffer {
|
||||
repo.Logger.Debug("No changes found; no sync made")
|
||||
|
||||
// archlinux wants to sync a "lastsync" file even if lastupdate prevents sync
|
||||
var args []string
|
||||
switch repo.SyncType {
|
||||
case "csc-sync-archlinux":
|
||||
args = cscTraceArchLinuxUpdate(repo)
|
||||
default:
|
||||
return config.SUCCESS, false
|
||||
}
|
||||
|
||||
ch := spawnProcess(repo, args)
|
||||
if ch == nil {
|
||||
// spawnSyncProcess will have already logged error
|
||||
return
|
||||
}
|
||||
|
||||
cmd := <-ch
|
||||
if cmd.ProcessState.ExitCode() != 0 {
|
||||
// if the process was terminated then don't continue to sync
|
||||
if cmd.ProcessState.ExitCode() == -1 {
|
||||
return config.TERMINATED, false
|
||||
}
|
||||
return
|
||||
}
|
||||
repo.Logger.Debug("No changes found; full sync will not be made")
|
||||
|
||||
// archlinux wants to sync a "lastsync" file if lastupdate prevents sync
|
||||
switch repo.SyncType {
|
||||
case "csc-sync-archlinux":
|
||||
args = cscTraceArchLinuxUpdate(repo)
|
||||
default:
|
||||
return config.SUCCESS, false
|
||||
}
|
||||
|
||||
return
|
||||
exit = runCommand(args)
|
||||
if exit != 0 {
|
||||
// if the process was terminated then don't continue to sync
|
||||
if exit == -1 {
|
||||
return config.TERMINATED, false
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
return config.SUCCESS, false
|
||||
}
|
||||
|
||||
// csc-sync-debian wants to update a trace file after the repo is done syncing
|
||||
func postSyncTraceUpdate(repo *config.Repo) {
|
||||
if repo.SyncType == "csc-sync-debian" {
|
||||
cscTraceDebianUpdate(repo)
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue