package config import ( "errors" "fmt" "os" "path/filepath" "gopkg.in/ini.v1" "git.csclub.uwaterloo.ca/public/merlin/logger" ) const ( DAILY = 86400 TWICE_DAILY = DAILY / 2 HOURLY = 3600 TWICE_HOURLY = HOURLY / 2 BI_HOURLY = HOURLY * 2 TRI_HOURLY = HOURLY * 3 TEN_MINUTELY = 600 FIVE_MINUTELY = 300 MINUTELY = 60 DEAFULT_HOSTNAME = "mirror.csclub.uwaterloo.ca" DEFAULT_MAX_JOBS = 6 DEFAULT_MAX_TIME = DAILY / 4 DEFAULT_MAX_RSYNC_IO = 0 DEFAULT_SYNC_TYPE = "csc-sync-standard" DEFAULT_FREQUENCY_STRING = "bi-hourly" DEFAULT_DOWNLOAD_DIR = "/mirror/root" DEFAULT_STATE_DIR = "/home/mirror/merlin/state" DEFAULT_LOG_DIR = "/home/mirror/merlin/log" DEFAULT_RSYNC_LOG_DIR = "/home/mirror/merlin/log-rsync" DEFAULT_ZFSSYNC_LOG_DIR = "/home/mirror/merlin/log-zfssync" DEFAULT_SOCK_PATH = "/mirror/merlin/run/merlin.sock" ) var frequencies = map[string]int{ "daily": DAILY, "twice-daily": TWICE_DAILY, "hourly": HOURLY, "twice-hourly": TWICE_HOURLY, "bi-hourly": BI_HOURLY, "tri-hourly": TRI_HOURLY, "ten-minutely": TEN_MINUTELY, "five-minutely": FIVE_MINUTELY, "minutely": MINUTELY, } // Last job attempt statuses const ( NOT_RUN_YET = iota SUCCESS FAILURE TERMINATED // was killed by a signal ) type SyncResult struct { Name string Exit int } type Config struct { Hostname string `ini:"hostname"` // the IPv4 addresses to use for rsync IPv4Address string `ini:"ipv4_address"` // the IPv6 addresses to use for rsync IPv6Address string `ini:"ipv6_address"` // the maximum number of jobs allowed to execute concurrently MaxJobs int `ini:"max_jobs"` // the default maximum time before killing rsync proccess DefaultMaxTime int `ini:"default_max_time"` // the default value for the maximum bandwidth a repo can use while syncing // (set to 0 for unlimited) DefaultMaxRsyncIO int `ini:"default_max_rsync_io"` // the default sync type DefaultSyncType string `ini:"default_sync_type"` // the default sync frequency string DefaultFrequencyStr string `ini:"default_frequency"` // the base directory where rsync should download files to DownloadDir string `ini:"download_dir"` // directory where the state of each repo is saved StateDir string `ini:"state_dir"` // directory where merlin will store the merlin logs for each repo RepoLogDir string `ini:"repo_log_dir"` // directory to store the rsync logs for each repo RsyncLogDir string `ini:"rsync_log_dir"` // directory to store the zfssync logs for each repo ZfssyncLogDir string `ini:"zfssync_log_dir"` // path to the unix socket for arthur to use for communication SockPath string `ini:"sock_path"` } type Repo struct { // the name of this repo Name string `ini:"-"` // this should be one of "csc-sync-standard", etc. SyncType string `ini:"sync_type"` // a human-readable frequency, e.g. "bi-hourly" FrequencyStr string `ini:"frequency"` // the desired interval (in seconds) between successive runs Frequency int `ini:"-"` // instead of spawning a process sleep for 50 seconds instead (default: false) DryRun bool `ini:"dry_run"` // the maximum time (in seconds) that each child process of this repo // can for before being killed MaxTime int `ini:"max_time"` // limit the amount of bandwidth a repo can use while syncing // (set to 0 to disable the limit) (unit is KiB) MaxRsyncIO int `ini:"max_rsync_io"` // where to download the files for this repo (relative to Conf.DownloadDir) LocalDir string `ini:"local_dir"` // the address to the trace file (how this url will be used depends on SyncType) TraceHost string `ini:"trace_host"` // the address to the remote host to rsync from RsyncHost string `ini:"rsync_host"` // the remote directory on the rsync host (optional) RsyncDir string `ini:"rsync_dir"` // the rsync user (optional) RsyncUser string `ini:"rsync_user"` // full path to file storing the password for rsync (optional) PasswordFile string `ini:"password_file"` // full path to file storing the repo sync state StateFile string `ini:"-"` // full path for file storing general logging of this repo RepoLogFile string `ini:"-"` // a pointer to the general logger Logger *logger.Logger `ini:"-"` // full file path for file logging this repo's rsync RsyncLogFile string `ini:"-"` // full file path for file logging this repo's zfssync ZfssyncLogFile string `ini:"-"` // add the "-vv" flag to rsync commands and enable the Debug log (default: false) Verbose bool `ini:"verbose"` // the repo will write its name and status in a Result struct to DoneChan // when it has finished a job (shared by all repos) DoneChan chan<- SyncResult `ini:"-"` // repos should stop syncing if StopChan is closed (shared by all repos) StopChan chan struct{} `ini:"-"` // a struct that stores the repo's status State *RepoState `ini:"-"` } // This should only be modified by the main thread type RepoState struct { // these are stored in the states folder // whether this repo is running a job or not IsRunning bool `ini:"is_running"` // the Unix epoch timestamp at which this repo last attempted a job LastAttemptStartTime int64 `ini:"last_attempt_time"` // the number of seconds this repo ran for during its last attempted job LastAttemptRunTime int64 `ini:"last_attempt_runtime"` // whether the last attempt was successful or not LastAttemptExit int `ini:"last_attempt_exit"` } var ( // the global config Conf Config // the global array of repos (iterate over repos) Repos []*Repo // the global map of repos (find repo by name) RepoMap map[string]*Repo ) // LoadConfig initializes the default config values then overrides them with // the values it reads from the INI config file func LoadConfig(configPath string, doneChan chan SyncResult, stopChan chan struct{}) { // create a new config with the default values then load config from file newConf := Config{ Hostname: DEAFULT_HOSTNAME, MaxJobs: DEFAULT_MAX_JOBS, DefaultMaxTime: DEFAULT_MAX_TIME, DefaultMaxRsyncIO: DEFAULT_MAX_RSYNC_IO, DefaultSyncType: DEFAULT_SYNC_TYPE, DefaultFrequencyStr: DEFAULT_FREQUENCY_STRING, DownloadDir: DEFAULT_DOWNLOAD_DIR, StateDir: DEFAULT_STATE_DIR, RepoLogDir: DEFAULT_LOG_DIR, RsyncLogDir: DEFAULT_RSYNC_LOG_DIR, ZfssyncLogDir: DEFAULT_ZFSSYNC_LOG_DIR, SockPath: DEFAULT_SOCK_PATH, } iniInfo, err := ini.Load(configPath) panicIfErr(err) err = iniInfo.MapTo(&newConf) panicIfErr(err) // check newConf for possible configuration errors if newConf.IPv4Address == "" { panic("Missing IPv4 address from config") } else if newConf.IPv6Address == "" { panic("Missing IPv6 address from config") } else if _, check := frequencies[newConf.DefaultFrequencyStr]; !check { panic(fmt.Errorf("%s is not a valid frequency", newConf.DefaultFrequencyStr)) } else if _, err := os.Stat(newConf.DownloadDir); errors.Is(err, os.ErrNotExist) { panic(fmt.Errorf("the directory %s does not exist", newConf.DownloadDir)) } // create directories for _, dir := range []string{ newConf.StateDir, newConf.RepoLogDir, newConf.RsyncLogDir, newConf.ZfssyncLogDir, } { err := os.MkdirAll(dir, 0755) panicIfErr(err) } var newRepos []*Repo var check bool for _, section := range iniInfo.Sections() { repoName := section.Name() if repoName == "DEFAULT" { continue } // set the default values for the repo then load from file repo := Repo{ Name: repoName, SyncType: newConf.DefaultSyncType, FrequencyStr: newConf.DefaultFrequencyStr, DryRun: false, MaxTime: newConf.DefaultMaxTime, MaxRsyncIO: newConf.DefaultMaxRsyncIO, LocalDir: repoName, StateFile: filepath.Join(newConf.StateDir, repoName), RepoLogFile: filepath.Join(newConf.RepoLogDir, repoName) + ".log", RsyncLogFile: filepath.Join(newConf.RsyncLogDir, repoName) + "-rsync.log", ZfssyncLogFile: filepath.Join(newConf.ZfssyncLogDir, repoName) + "-zfssync.log", Verbose: false, DoneChan: doneChan, StopChan: stopChan, } err := section.MapTo(&repo) panicIfErr(err) // checks for validity of repo configuration if repo.Frequency, check = frequencies[repo.FrequencyStr]; !check { panic("Missing or invalid frequency for " + repo.Name) } else if repo.SyncType == "" { panic("Missing sync type from " + repo.Name) } else if repo.LocalDir == "" { panic("Missing local download location for " + repo.Name) } else if repo.RsyncHost == "" { panic("Missing rsync host for " + repo.Name) } // check that full download path and password file (if defined) exists localDirFull := filepath.Join(newConf.DownloadDir, repo.LocalDir) if _, err := os.Stat(localDirFull); errors.Is(err, os.ErrNotExist) { panic("the path " + localDirFull + " does not exist") } else if repo.PasswordFile != "" { if _, err := os.Stat(repo.PasswordFile); errors.Is(err, os.ErrNotExist) { panic("the file " + repo.PasswordFile + " does not exist") } } // make sure the log and state files exists for the repo touch( repo.StateFile, repo.RepoLogFile, repo.RsyncLogFile, repo.ZfssyncLogFile, ) // create the logger and load the state repo.Logger = logger.NewLogger(repo.Name, repo.RepoLogFile, repo.Verbose) repo.State = &RepoState{ IsRunning: false, LastAttemptStartTime: 0, LastAttemptRunTime: 0, LastAttemptExit: NOT_RUN_YET, } err = ini.MapTo(&repo.State, repo.StateFile) panicIfErr(err) repo.SaveState() newRepos = append(newRepos, &repo) } if len(newRepos) == 0 { panic("no repos found in config") } // set the global variables after config and repos are fully loaded Conf = newConf Repos = newRepos RepoMap = make(map[string]*Repo) for _, repo := range Repos { RepoMap[repo.Name] = repo } } // Save the current state of the repo to the state file func (repo *Repo) SaveState() { state_cfg := ini.Empty() if err := ini.ReflectFrom(state_cfg, &repo.State); err != nil { repo.Logger.Error(err.Error()) } file, err := os.OpenFile(repo.StateFile, os.O_RDWR|os.O_CREATE, 0644) if err != nil { repo.Logger.Error(err.Error()) } if _, err := state_cfg.WriteTo(file); err != nil { repo.Logger.Error(err.Error()) } }