311 lines
10 KiB
Go
311 lines
10 KiB
Go
package config
|
|
|
|
import (
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
|
|
"gopkg.in/ini.v1"
|
|
|
|
"git.csclub.uwaterloo.ca/public/merlin/logger"
|
|
)
|
|
|
|
const (
|
|
DAILY = 86400
|
|
TWICE_DAILY = DAILY / 2
|
|
HOURLY = 3600
|
|
TWICE_HOURLY = HOURLY / 2
|
|
BI_HOURLY = HOURLY * 2
|
|
TRI_HOURLY = HOURLY * 3
|
|
TEN_MINUTELY = 600
|
|
FIVE_MINUTELY = 300
|
|
MINUTELY = 60
|
|
|
|
DEAFULT_HOSTNAME = "mirror.csclub.uwaterloo.ca"
|
|
DEFAULT_MAX_JOBS = 6
|
|
DEFAULT_MAX_TIME = DAILY / 4
|
|
DEFAULT_MAX_RSYNC_IO = 0
|
|
DEFAULT_SYNC_TYPE = "csc-sync-standard"
|
|
DEFAULT_FREQUENCY_STRING = "bi-hourly"
|
|
DEFAULT_DOWNLOAD_DIR = "/mirror/root"
|
|
DEFAULT_STATE_DIR = "/home/mirror/merlin/state"
|
|
DEFAULT_LOG_DIR = "/home/mirror/merlin/log"
|
|
DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync"
|
|
DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/log-zfssync"
|
|
DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock"
|
|
)
|
|
|
|
var frequencies = map[string]int{
|
|
"daily": DAILY,
|
|
"twice-daily": TWICE_DAILY,
|
|
"hourly": HOURLY,
|
|
"twice-hourly": TWICE_HOURLY,
|
|
"bi-hourly": BI_HOURLY,
|
|
"tri-hourly": TRI_HOURLY,
|
|
"ten-minutely": TEN_MINUTELY,
|
|
"five-minutely": FIVE_MINUTELY,
|
|
"minutely": MINUTELY,
|
|
}
|
|
|
|
// Last job attempt statuses
|
|
const (
|
|
NOT_RUN_YET = iota
|
|
SUCCESS
|
|
FAILURE
|
|
TERMINATED // was killed by a signal
|
|
)
|
|
|
|
type SyncResult struct {
|
|
Name string
|
|
Exit int
|
|
}
|
|
|
|
type Config struct {
|
|
Hostname string `ini:"hostname"`
|
|
// the IPv4 addresses to use for rsync
|
|
IPv4Address string `ini:"ipv4_address"`
|
|
// the IPv6 addresses to use for rsync
|
|
IPv6Address string `ini:"ipv6_address"`
|
|
// the maximum number of jobs allowed to execute concurrently
|
|
MaxJobs int `ini:"max_jobs"`
|
|
// the default maximum time before killing rsync proccess
|
|
DefaultMaxTime int `ini:"default_max_time"`
|
|
// the default value for the maximum bandwidth a repo can use while syncing
|
|
// (set to 0 for unlimited)
|
|
DefaultMaxRsyncIO int `ini:"default_max_rsync_io"`
|
|
// the default sync type
|
|
DefaultSyncType string `ini:"default_sync_type"`
|
|
// the default sync frequency string
|
|
DefaultFrequencyStr string `ini:"default_frequency"`
|
|
// the base directory where rsync should download files to
|
|
DownloadDir string `ini:"download_dir"`
|
|
// directory where the state of each repo is saved
|
|
StateDir string `ini:"state_dir"`
|
|
// directory where merlin will store the merlin logs for each repo
|
|
RepoLogDir string `ini:"repo_log_dir"`
|
|
// directory to store the rsync logs for each repo
|
|
RsyncLogDir string `ini:"rsync_log_dir"`
|
|
// directory to store the zfssync logs for each repo
|
|
ZfssyncLogDir string `ini:"zfssync_log_dir"`
|
|
// path to the unix socket for arthur to use for communication
|
|
SockPath string `ini:"sock_path"`
|
|
}
|
|
|
|
type Repo struct {
|
|
// the name of this repo
|
|
Name string `ini:"-"`
|
|
// this should be one of "csc-sync-standard", etc.
|
|
SyncType string `ini:"sync_type"`
|
|
// a human-readable frequency, e.g. "bi-hourly"
|
|
FrequencyStr string `ini:"frequency"`
|
|
// the desired interval (in seconds) between successive runs
|
|
Frequency int `ini:"-"`
|
|
// instead of spawning a process sleep for 50 seconds instead (default: false)
|
|
DryRun bool `ini:"dry_run"`
|
|
// the maximum time (in seconds) that each child process of this repo
|
|
// can for before being killed
|
|
MaxTime int `ini:"max_time"`
|
|
// limit the amount of bandwidth a repo can use while syncing
|
|
// (set to 0 to disable the limit) (unit is KiB)
|
|
MaxRsyncIO int `ini:"max_rsync_io"`
|
|
// where to download the files for this repo (relative to Conf.DownloadDir)
|
|
LocalDir string `ini:"local_dir"`
|
|
// the address to the trace file (how this url will be used depends on SyncType)
|
|
TraceHost string `ini:"trace_host"`
|
|
// the address to the remote host to rsync from
|
|
RsyncHost string `ini:"rsync_host"`
|
|
// the remote directory on the rsync host (optional)
|
|
RsyncDir string `ini:"rsync_dir"`
|
|
// the rsync user (optional)
|
|
RsyncUser string `ini:"rsync_user"`
|
|
// full path to file storing the password for rsync (optional)
|
|
PasswordFile string `ini:"password_file"`
|
|
// full path to file storing the repo sync state
|
|
StateFile string `ini:"-"`
|
|
// full path for file storing general logging of this repo
|
|
RepoLogFile string `ini:"-"`
|
|
// a pointer to the general logger
|
|
Logger *logger.Logger `ini:"-"`
|
|
// full file path for file logging this repo's rsync
|
|
RsyncLogFile string `ini:"-"`
|
|
// full file path for file logging this repo's zfssync
|
|
ZfssyncLogFile string `ini:"-"`
|
|
// add the "-vv" flag to rsync commands and enable the Debug log (default: false)
|
|
Verbose bool `ini:"verbose"`
|
|
// the repo will write its name and status in a Result struct to DoneChan
|
|
// when it has finished a job (shared by all repos)
|
|
DoneChan chan<- SyncResult `ini:"-"`
|
|
// repos should stop syncing if StopChan is closed (shared by all repos)
|
|
StopChan chan struct{} `ini:"-"`
|
|
// a struct that stores the repo's status
|
|
State *RepoState `ini:"-"`
|
|
}
|
|
|
|
// This should only be modified by the main thread
|
|
type RepoState struct {
|
|
// these are stored in the states folder
|
|
// whether this repo is running a job or not
|
|
IsRunning bool `ini:"is_running"`
|
|
// the Unix epoch timestamp at which this repo last attempted a job
|
|
LastAttemptStartTime int64 `ini:"last_attempt_time"`
|
|
// the number of seconds this repo ran for during its last attempted job
|
|
LastAttemptRunTime int64 `ini:"last_attempt_runtime"`
|
|
// whether the last attempt was successful or not
|
|
LastAttemptExit int `ini:"last_attempt_exit"`
|
|
}
|
|
|
|
var (
|
|
// the global config
|
|
Conf Config
|
|
// the global array of repos (iterate over repos)
|
|
Repos []*Repo
|
|
// the global map of repos (find repo by name)
|
|
RepoMap map[string]*Repo
|
|
)
|
|
|
|
// LoadConfig initializes the default config values then overrides them with
|
|
// the values it reads from the INI config file
|
|
func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struct{}) {
|
|
// create a new config with the default values then load config from file
|
|
newConf := Config{
|
|
Hostname: DEAFULT_HOSTNAME,
|
|
MaxJobs: DEFAULT_MAX_JOBS,
|
|
DefaultMaxTime: DEFAULT_MAX_TIME,
|
|
DefaultMaxRsyncIO: DEFAULT_MAX_RSYNC_IO,
|
|
DefaultSyncType: DEFAULT_SYNC_TYPE,
|
|
DefaultFrequencyStr: DEFAULT_FREQUENCY_STRING,
|
|
DownloadDir: DEFAULT_DOWNLOAD_DIR,
|
|
StateDir: DEFAULT_STATE_DIR,
|
|
RepoLogDir: DEFAULT_LOG_DIR,
|
|
RsyncLogDir: DEFAULT_RSYNC_LOG_DIR,
|
|
ZfssyncLogDir: DEFAULT_ZFSSYNC_LOG_DIR,
|
|
SockPath: DEFAULT_SOCK_PATH,
|
|
}
|
|
iniInfo, err := ini.Load(configPath)
|
|
panicIfErr(err)
|
|
err = iniInfo.MapTo(&newConf)
|
|
panicIfErr(err)
|
|
|
|
// check newConf for possible configuration errors
|
|
if newConf.IPv4Address == "" {
|
|
panic("Missing IPv4 address from config")
|
|
} else if newConf.IPv6Address == "" {
|
|
panic("Missing IPv6 address from config")
|
|
} else if _, check := frequencies[newConf.DefaultFrequencyStr]; !check {
|
|
panic(fmt.Errorf("%s is not a valid frequency", newConf.DefaultFrequencyStr))
|
|
} else if _, err := os.Stat(newConf.DownloadDir); errors.Is(err, os.ErrNotExist) {
|
|
panic(fmt.Errorf("the directory %s does not exist", newConf.DownloadDir))
|
|
}
|
|
|
|
// create directories
|
|
for _, dir := range []string{
|
|
newConf.StateDir,
|
|
newConf.RepoLogDir,
|
|
newConf.RsyncLogDir,
|
|
newConf.ZfssyncLogDir,
|
|
} {
|
|
err := os.MkdirAll(dir, 0755)
|
|
panicIfErr(err)
|
|
}
|
|
|
|
var newRepos []*Repo
|
|
var check bool
|
|
for _, section := range iniInfo.Sections() {
|
|
repoName := section.Name()
|
|
if repoName == "DEFAULT" {
|
|
continue
|
|
}
|
|
|
|
// set the default values for the repo then load from file
|
|
repo := Repo{
|
|
Name: repoName,
|
|
SyncType: newConf.DefaultSyncType,
|
|
FrequencyStr: newConf.DefaultFrequencyStr,
|
|
DryRun: false,
|
|
MaxTime: newConf.DefaultMaxTime,
|
|
MaxRsyncIO: newConf.DefaultMaxRsyncIO,
|
|
LocalDir: repoName,
|
|
StateFile: filepath.Join(newConf.StateDir, repoName),
|
|
RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log",
|
|
RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log",
|
|
ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log",
|
|
Verbose: false,
|
|
DoneChan: doneChan,
|
|
StopChan: stopChan,
|
|
}
|
|
err := section.MapTo(&repo)
|
|
panicIfErr(err)
|
|
|
|
// checks for validity of repo configuration
|
|
if repo.Frequency, check = frequencies[repo.FrequencyStr]; !check {
|
|
panic("Missing or invalid frequency for " + repo.Name)
|
|
} else if repo.SyncType == "" {
|
|
panic("Missing sync type from " + repo.Name)
|
|
} else if repo.LocalDir == "" {
|
|
panic("Missing local download location for " + repo.Name)
|
|
} else if repo.RsyncHost == "" {
|
|
panic("Missing rsync host for " + repo.Name)
|
|
}
|
|
|
|
// check that full download path and password file (if defined) exists
|
|
localDirFull := filepath.Join(newConf.DownloadDir, repo.LocalDir)
|
|
if _, err := os.Stat(localDirFull); errors.Is(err, os.ErrNotExist) {
|
|
panic("the path " + localDirFull + " does not exist")
|
|
} else if repo.PasswordFile != "" {
|
|
if _, err := os.Stat(repo.PasswordFile); errors.Is(err, os.ErrNotExist) {
|
|
panic("the file " + repo.PasswordFile + " does not exist")
|
|
}
|
|
}
|
|
|
|
// make sure the log and state files exists for the repo
|
|
touch(
|
|
repo.StateFile,
|
|
repo.RepoLogFile,
|
|
repo.RsyncLogFile,
|
|
repo.ZfssyncLogFile,
|
|
)
|
|
|
|
// create the logger and load the state
|
|
repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile, repo.Verbose)
|
|
repo.State = &RepoState{
|
|
IsRunning: false,
|
|
LastAttemptStartTime: 0,
|
|
LastAttemptRunTime: 0,
|
|
LastAttemptExit: NOT_RUN_YET,
|
|
}
|
|
err = ini.MapTo(&repo.State, repo.StateFile)
|
|
panicIfErr(err)
|
|
repo.SaveState()
|
|
|
|
newRepos = append(newRepos, &repo)
|
|
}
|
|
|
|
if len(newRepos) == 0 {
|
|
panic("no repos found in config")
|
|
}
|
|
|
|
// set the global variables after config and repos are fully loaded
|
|
Conf = newConf
|
|
Repos = newRepos
|
|
RepoMap = make(map[string]*Repo)
|
|
for _, repo := range Repos {
|
|
RepoMap[repo.Name] = repo
|
|
}
|
|
}
|
|
|
|
// Save the current state of the repo to the state file
|
|
func (repo *Repo) SaveState() {
|
|
state_cfg := ini.Empty()
|
|
if err := ini.ReflectFrom(state_cfg, &repo.State); err != nil {
|
|
repo.Logger.Error(err.Error())
|
|
}
|
|
file, err := os.OpenFile(repo.StateFile, os.O_RDWR|os.O_CREATE, 0644)
|
|
if err != nil {
|
|
repo.Logger.Error(err.Error())
|
|
}
|
|
if _, err := state_cfg.WriteTo(file); err != nil {
|
|
repo.Logger.Error(err.Error())
|
|
}
|
|
}
|