mirror/merlin/common/common.go

package common

import (
	"os"
	"time"

	ini "gopkg.in/ini.v1"
)

const (
	DAILY         = 86400
	TWICE_DAILY   = DAILY / 2
	HOURLY        = 3600
	TWICE_HOURLY  = HOURLY / 2
	BI_HOURLY     = HOURLY * 2
	TRI_HOURLY    = HOURLY * 3
	TEN_MINUTELY  = 600
	FIVE_MINUTELY = 300

	CONFIG_PATH = "merlin-config.ini"

	DEFAULT_MAX_JOBS     = 6
	DEFAULT_MAX_TIME     = DAILY / 4
	DEFAULT_DOWNLOAD_DIR = "/mirror/root"
	DEFAULT_PASSWORD_DIR = "/home/mirror/passwords"
	DEFAULT_LOG_DIR      = "/home/mirror/merlin/logs"
	DEFAULT_STATE_DIR    = "/home/mirror/merlin/states"
	DEFAULT_SOCK_PATH    = "/run/merlin/merlin.sock"
)

var frequencies = map[string]int{
	"daily":         DAILY,
	"twice-daily":   TWICE_DAILY,
	"hourly":        HOURLY,
	"twice-hourly":  TWICE_HOURLY,
	"bi-hourly":     BI_HOURLY,
	"tri-hourly":    TRI_HOURLY,
	"ten-minutely":  TEN_MINUTELY,
	"five-minutely": FIVE_MINUTELY,
}

// Last job attempt statuses
const (
	NOT_RUN_YET = iota
	SUCCESS
	FAILURE
	TERMINATED // was killed by a signal
)

type Result struct {
	Name string
	Exit int
}

type Repo struct {
	// the name of this repo
	Name string `ini:"-"`
	// this should be one of "csc-sync-standard", etc.
	SyncType string `ini:"sync_type"`
	// a human-readable frequency, e.g. "bi-hourly"
	FrequencyStr string `ini:"frequency"`
	// the desired interval (in seconds) between successive runs
	Frequency int `ini:"-"`
	// the maximum time (in seconds) that each child process of this repo
	// can for before being killed
	MaxTime int `ini:"max_time"`
	// where to download the files for this repo (relative to the download
	// dir in the config)
	LocalDir string `ini:"local_dir"`
	// the remote host to rsync from
	RsyncHost string `ini:"rsync_host"`
	// the remote directory on the rsync host
	RsyncDir string `ini:"rsync_dir"`
	// the rsync user (optional)
	RsyncUser string `ini:"rsync_user"`
	// the file storing the password for rsync (optional)
	PasswordFile string `ini:"password_file"`
	// the log file for rsync (optional, defaults to Config.LogFile)
	LogFile string `ini:"log_file"`
	// a reference to the logger
	Logger *Logger `ini:"-"`
	// the repo will write its name and status in a Result struct to DoneChan
	// when it has finished a job
	DoneChan chan<- Result `ini:"-"`
	// the repo should stop syncing if StopChan is closed
	StopChan chan bool `ini:"-"`
	// a struct that stores the repo's status
	State RepoState `ini:"-"`
	// a reference to the global config
	cfg *Config `ini:"-"`
}

type Config struct {
	// the maximum number of jobs allowed to execute concurrently
	MaxJobs int `ini:"max_jobs"`
	// the IP addresses to use for rsync
	IPv4Address string `ini:"ipv4_address"`
	IPv6Address string `ini:"ipv6_address"`
	// the default sync type
	SyncType string `ini:"default_sync_type"`
	// the default frequency string for the repos
	FrequencyStr string `ini:"default_frequency"`
	// the default MaxTime for each repo
	MaxTime int `ini:"default_max_time"`
	// the directory where rsync should download files
	DownloadDir string `ini:"download_dir"`
	// the directory where rsync passwords are stored
	PasswordDir string `ini:"password_dir"`
	// the path where merlin will store the logs for each repo synced
	LogDir string `ini:"log_dir"`
	// the path to where the state of each repo's sync is saved
	StateDir string `ini:"states_dir"`
	// the Unix socket path which arthur will use to communicate with us
	SockPath string `ini:"sock_path"`
	// the DoneChan for each repo (shared instance)
	DoneChan <-chan Result `ini:"-"`
	// a list of all of the repos
	Repos []*Repo `ini:"-"`
}

// This should only be modified by the main thread
type RepoState struct {
	// these are stored in the states folder
	// whether this repo is running a job or not
	IsRunning bool `ini:"is_running"`
	// whether the last attempt was successful or not
	LastAttemptExit int `ini:"last_attempt_exit"`
	// the Unix epoch timestamp at which this repo last attempted a job
	LastAttemptTime int64 `ini:"last_attempt_time"`
	// the number of seconds this repo ran for during its last attempted job
	LastAttemptRunTime int64 `ini:"last_attempt_runtime"`
}

// save the save the current state of a repo to a file
func (repo *Repo) SaveState() {
	state_cfg := ini.Empty()
	if err := ini.ReflectFrom(state_cfg, &repo.State); err != nil {
		repo.Logger.Error(err)
	}
	file, err := os.OpenFile(repo.cfg.StateDir+"/"+repo.Name, os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		repo.Logger.Error(err)
	}
	if _, err := state_cfg.WriteTo(file); err != nil {
		repo.Logger.Error(err)
	}
}

// RunIfScheduled starts a sync job for this repo if more than repo.Frequency
// seconds have elapsed since its last job.
// It returns true iff a job is started.
func (repo *Repo) RunIfPossible() bool {
	// don't run if a job is already running
	if repo.State.IsRunning {
		return false
	}

	// this should be set in the caller's thread so that the "if" will work
	curTime := time.Now().Unix()
	if curTime-repo.State.LastAttemptTime > int64(repo.Frequency) {
		repo.State.IsRunning = true
		repo.State.LastAttemptTime = curTime
		repo.SaveState()
		go repo.StartSyncJob()
		return true
	}
	return false
}

// update the repo state with the last attempt time and exit now that the job is done
func (repo *Repo) SyncExit(exit int) {
	repoState := repo.State
	repoState.IsRunning = false
	repoState.LastAttemptExit = exit
	repoState.LastAttemptTime = time.Now().Unix() - repoState.LastAttemptTime
	// repo.Logger.Debug(fmt.Sprintf("Process exited after %d seconds", repoState.LastAttemptTime))
	repo.SaveState()
}

// GetConfig reads the config from a JSON file, initializes default values,
// and initializes the non-configurable fields of each repo.
// It returns a Config.
func GetConfig() Config {
	// add global configuration in cfg
	data, err := ini.Load(CONFIG_PATH)
	if err != nil {
		panic(err)
	}
	cfg := Config{
		MaxJobs:     DEFAULT_MAX_JOBS,
		MaxTime:     DEFAULT_MAX_TIME,
		PasswordDir: DEFAULT_PASSWORD_DIR,
		DownloadDir: DEFAULT_DOWNLOAD_DIR,
		LogDir:      DEFAULT_LOG_DIR,
		StateDir:    DEFAULT_STATE_DIR,
		SockPath:    DEFAULT_SOCK_PATH,
		Repos:       make([]*Repo, 0),
	}
	if err := data.MapTo(&cfg); err != nil {
		panic(err)
	}

	// validate global configuration
	if err := os.MkdirAll(cfg.LogDir, 0755); err != nil {
		panic("Could not create log path at " + cfg.LogDir)
	} else if err := os.MkdirAll(cfg.StateDir, 0755); err != nil {
		panic("Could not create states path at " + cfg.StateDir)
	} else if cfg.IPv4Address == "" {
		panic("Missing IPv4 address from config")
	} else if cfg.IPv6Address == "" {
		panic("Missing IPv6 address from config")
	}

	// buffered to prevent possible race condition
	doneChan := make(chan Result, cfg.MaxJobs)
	cfg.DoneChan = doneChan

	// add each repo configuration to cfg
	for _, section := range data.Sections() {
		if section.Name() == "DEFAULT" {
			continue
		}

		// create the repo configuration
		repo := Repo{
			Name:         section.Name(),
			SyncType:     cfg.SyncType,
			FrequencyStr: cfg.FrequencyStr,
			MaxTime:      cfg.MaxTime,
		}
		if err := section.MapTo(&repo); err != nil {
			panic(err)
		}
		repo.Frequency = frequencies[repo.FrequencyStr]
		if repo.SyncType == "" {
			panic("Missing sync type from " + repo.Name)
		} else if repo.Frequency == 0 {
			panic("Missing frequency from " + repo.Name)
		}
		repo.Logger = NewLogger(repo.Name)
		repo.DoneChan = doneChan
		repo.StopChan = make(chan bool)
		repo.cfg = &cfg

		// create the default repo state
		repo.State = RepoState{
			IsRunning:          false,
			LastAttemptExit:    NOT_RUN_YET,
			LastAttemptTime:    0,
			LastAttemptRunTime: 0,
		}

		// create the state file if it does not exist otherwise sync the state
		repoStateFile := cfg.StateDir + "/" + repo.Name
		if _, err := os.Stat(repoStateFile); err != nil {
			repo.SaveState()
		} else if err := ini.MapTo(&repo.State, repoStateFile); err != nil {
			panic(err)
		}

		// repo state must be initially not running, otherwise will never run
		repo.State.IsRunning = false

		// append a reference to the new repo in the slice of repos
		cfg.Repos = append(cfg.Repos, &repo)
	}
	if len(cfg.Repos) == 0 {
		panic("No repos found in config")
	}
	return cfg
}