update latest checkers

Signed-off-by: Nathan13888 <29968201+Nathan13888@users.noreply.github.com>

checkers/almalinux.go

@@ -0,0 +1,3 @@
+package checkers
+
+// https://mirrors.almalinux.org/

checkers/debian.go

@@ -21,6 +21,7 @@ var DebianProject Project = Project{
 			}
 
 			// Modelled after: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/debian.py
+			// NOTE: cloned in debiancd, debianmultimedia, debianports, debiansecurity
 
 			csc_url := config.MirrorBaseURL + data.CSC + data.File
 			upstream_url := data.Upstream + data.File
@@ -64,12 +65,7 @@ var DebianProject Project = Project{
 
 			delta := (upstream_utc_time - CSC_utc_time)
 
-			if delta < data.OOSInterval && delta > -data.OOSInterval {
-				return true, nil
-			}
-
-			// we've got problems :/
-			return false, nil
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
 		}),
 	},
 }

checkers/debiancd.go

@@ -0,0 +1,62 @@
+package checkers
+
+import (
+	"strings"
+	"time"
+
+	"github.com/Nathan13888/mirror-checker2/v2/config"
+)
+
+var DebianCDProject Project = Project{
+	Name:          "debiancd",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("debiancd", true, func(*Project) (bool, error) {
+			// based on debian.go checker
+			// config sanity check
+			data := EnabledProjects["debiancd"].Properties
+			err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
+			if err != nil {
+				return false, GetError(err, "DebianCD", "config sanity check")
+			}
+
+			csc_url := config.MirrorBaseURL + data.CSC + data.File
+			upstream_url := data.Upstream + data.File
+
+			csc_body, err := httpGET(csc_url)
+			if err != nil {
+				return false, GetError(err, "DebianCD", "getting CSC file")
+			}
+			upstream_body, err := httpGET(upstream_url)
+			if err != nil {
+				return false, GetError(err, "DebianCD", "getting upstream file")
+			}
+
+			CSC := string(csc_body)
+			upstream := string(upstream_body)
+
+			if CSC == upstream {
+				return true, nil
+			}
+
+			date_format := "Mon 2 Jan 15:04:05 MST 2006"
+
+			CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianCD", "parsing CSC date")
+			}
+			CSC_utc_time := CSC_date.Unix()
+
+			upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianCD", "parsing upstream date")
+			}
+			upstream_utc_time := upstream_date.Unix()
+
+			delta := (upstream_utc_time - CSC_utc_time)
+
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
+		}),
+	},
+}

checkers/debianmultimedia.go

@@ -0,0 +1,62 @@
+package checkers
+
+import (
+	"strings"
+	"time"
+
+	"github.com/Nathan13888/mirror-checker2/v2/config"
+)
+
+var DebianMultimediaProject Project = Project{
+	Name:          "debianmultimedia",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("debianmultimedia", true, func(*Project) (bool, error) {
+			// based on debian.go checker
+			// config sanity check
+			data := EnabledProjects["debianmultimedia"].Properties
+			err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
+			if err != nil {
+				return false, GetError(err, "DebianMultimedia", "config sanity check")
+			}
+
+			csc_url := config.MirrorBaseURL + data.CSC + data.File
+			upstream_url := data.Upstream + data.File
+
+			csc_body, err := httpGET(csc_url)
+			if err != nil {
+				return false, GetError(err, "DebianMultimedia", "getting CSC file")
+			}
+			upstream_body, err := httpGET(upstream_url)
+			if err != nil {
+				return false, GetError(err, "DebianMultimedia", "getting upstream file")
+			}
+
+			CSC := string(csc_body)
+			upstream := string(upstream_body)
+
+			if CSC == upstream {
+				return true, nil
+			}
+
+			date_format := "Mon Jan 2 15:04:05 MST 2006"
+
+			CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianMultimedia", "parsing CSC date")
+			}
+			CSC_utc_time := CSC_date.Unix()
+
+			upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianMultimedia", "parsing upstream date")
+			}
+			upstream_utc_time := upstream_date.Unix()
+
+			delta := (upstream_utc_time - CSC_utc_time)
+
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
+		}),
+	},
+}

checkers/debianports.go

@@ -0,0 +1,62 @@
+package checkers
+
+import (
+	"strings"
+	"time"
+
+	"github.com/Nathan13888/mirror-checker2/v2/config"
+)
+
+var DebianPortsProject Project = Project{
+	Name:          "debianports",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("debianports", true, func(*Project) (bool, error) {
+			// based on debian.go checker
+			// config sanity check
+			data := EnabledProjects["debianports"].Properties
+			err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
+			if err != nil {
+				return false, GetError(err, "DebianPorts", "config sanity check")
+			}
+
+			csc_url := config.MirrorBaseURL + data.CSC + data.File
+			upstream_url := data.Upstream + data.File
+
+			csc_body, err := httpGET(csc_url)
+			if err != nil {
+				return false, GetError(err, "DebianPorts", "getting CSC file")
+			}
+			upstream_body, err := httpGET(upstream_url)
+			if err != nil {
+				return false, GetError(err, "DebianPorts", "getting upstream file")
+			}
+
+			CSC := string(csc_body)
+			upstream := string(upstream_body)
+
+			if CSC == upstream {
+				return true, nil
+			}
+
+			date_format := "Mon Jan 2 15:04:05 MST 2006"
+
+			CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianPorts", "parsing CSC date")
+			}
+			CSC_utc_time := CSC_date.Unix()
+
+			upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianPorts", "parsing upstream date")
+			}
+			upstream_utc_time := upstream_date.Unix()
+
+			delta := (upstream_utc_time - CSC_utc_time)
+
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
+		}),
+	},
+}

checkers/debiansecurity.go

@@ -0,0 +1,62 @@
+package checkers
+
+import (
+	"strings"
+	"time"
+
+	"github.com/Nathan13888/mirror-checker2/v2/config"
+)
+
+var DebianSecurityProject Project = Project{
+	Name:          "debiansecurity",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("debiansecurity", true, func(*Project) (bool, error) {
+			// based on debian.go checker
+			// config sanity check
+			data := EnabledProjects["debiansecurity"].Properties
+			err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
+			if err != nil {
+				return false, GetError(err, "DebianSecurity", "config sanity check")
+			}
+
+			csc_url := config.MirrorBaseURL + data.CSC + data.File
+			upstream_url := data.Upstream + data.File
+
+			csc_body, err := httpGET(csc_url)
+			if err != nil {
+				return false, GetError(err, "DebianSecurity", "getting CSC file")
+			}
+			upstream_body, err := httpGET(upstream_url)
+			if err != nil {
+				return false, GetError(err, "DebianSecurity", "getting upstream file")
+			}
+
+			CSC := string(csc_body)
+			upstream := string(upstream_body)
+
+			if CSC == upstream {
+				return true, nil
+			}
+
+			date_format := "Mon Jan 2 15:04:05 MST 2006"
+
+			CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianSecurity", "parsing CSC date")
+			}
+			CSC_utc_time := CSC_date.Unix()
+
+			upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
+			if err != nil {
+				return false, GetError(err, "DebianSecurity", "parsing upstream date")
+			}
+			upstream_utc_time := upstream_date.Unix()
+
+			delta := (upstream_utc_time - CSC_utc_time)
+
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
+		}),
+	},
+}

checkers/default.go

@@ -38,16 +38,16 @@ var SupportedProjects map[string]*Project = map[string]*Project{
 	// "cran":&CranProject,
 	// "ctan":&CtanProject,
 	// "cygwin":&CygwinProject,
-	"debian": &DebianProject,
-	// "debiancd":&DebianCdProject,
-	// "debianmultimedia":&DebianMultimediaProject,
-	// "debianports":&DebianPortsProject,
-	// "debiansecurity":&DebianSecurityProject,
+	"debian":           &DebianProject,
+	"debiancd":         &DebianCDProject,
+	"debianmultimedia": &DebianMultimediaProject,
+	"debianports":      &DebianPortsProject,
+	"debiansecurity":   &DebianSecurityProject,
 	// "eclipse":&EclipseProject,
-	// "fedora":&FedoraProject,
+	"fedora": &FedoraProject,
 	// "freebsd":&FreebsdProject,
-	// "gentoodistfiles":&GentooDistFilesProject,
-	// "gentooportage":&GentooPortageProject,
+	"gentoodistfiles": &GentooDistProject,
+	"gentooportage":   &GentooPortageProject,
 	// "gnome":&GnomeProject,
 	// "gnu":&GnuProject,
 	// "gutenberg":&GutenbergProject,
@@ -58,7 +58,7 @@ var SupportedProjects map[string]*Project = map[string]*Project{
 	// "linuxmint":&LinuxMintProject,
 	// "linuxmint_packages":&LinuxMintPackagesProject,
 	// "macports":&MacPortsProject,
-	// "manjaro":&MajaroProject,
+	"manjaro": &ManjaroProject,
 	// "mxlinux":&MxLinuxProject,
 	// "mxlinux_iso":&MxLinuxIsoProject,
 	// "mysql":&MysqlProject,
@@ -105,12 +105,12 @@ func LoadDefaultProjects() {
 	// LoadProject("ctan")
 	// LoadProject("cygwin")
 	LoadProject("debian")
-	// LoadProject("debiancd")
-	// LoadProject("debianmultimedia")
-	// LoadProject("debianports")
-	// LoadProject("debiansecurity")
+	LoadProject("debiancd")
+	LoadProject("debianmultimedia")
+	LoadProject("debianports")
+	LoadProject("debiansecurity")
 	// LoadProject("eclipse")
-	// LoadProject("fedora")
+	LoadProject("fedora")
 	// LoadProject("freebsd")
 	// LoadProject("gentoodistfiles")
 	// LoadProject("gentooportage")

checkers/fedora.go

@@ -0,0 +1,63 @@
+package checkers
+
+import (
+	"time"
+
+	"github.com/Nathan13888/mirror-checker2/v2/config"
+)
+
+var FedoraProject Project = Project{
+	Name:          "fedora",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("fedora", true, func(*Project) (bool, error) {
+			// config sanity check
+			data := EnabledProjects["fedora"].Properties
+			err := AssertStrings(config.MirrorBaseURL, data.CSC, data.File, data.Upstream)
+			if err != nil {
+				return false, GetError(err, "Fedora", "config sanity check")
+			}
+
+			// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/fedora.py
+
+			csc_url := config.MirrorBaseURL + data.CSC + data.File
+			upstream_url := data.Upstream + data.File
+
+			csc_body, err := httpGET(csc_url)
+			if err != nil {
+				return false, GetError(err, "Fedora", "getting CSC file")
+			}
+			upstream_body, err := httpGET(upstream_url)
+			if err != nil {
+				return false, GetError(err, "Fedora", "getting upstream file")
+			}
+
+			CSC := string(csc_body)
+			upstream := string(upstream_body)
+
+			if CSC == upstream {
+				return true, nil
+			}
+
+			// # Date example: Fedora-Rawhide-20220725.n.1
+			data_format := "20060102"
+
+			CSC_date, err := time.Parse(data_format, CSC[15:23])
+			if err != nil {
+				return false, GetError(err, "Fedora", "parsing CSC date")
+			}
+			CSC_utc_time := CSC_date.Unix()
+
+			upstream_date, err := time.Parse(data_format, upstream[15:23])
+			if err != nil {
+				return false, GetError(err, "Fedora", "parsing upstream date")
+			}
+			upstream_utc_time := upstream_date.Unix()
+
+			delta := (upstream_utc_time - CSC_utc_time)
+
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
+		}),
+	},
+}

checkers/gentoodist.go

@@ -0,0 +1,61 @@
+package checkers
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/Nathan13888/mirror-checker2/v2/config"
+)
+
+var GentooDistProject Project = Project{
+	Name:          "gentoodistfiles",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("gentoodistfiles", true, func(*Project) (bool, error) {
+			// config sanity check
+			data := EnabledProjects["gentoodistfiles"].Properties
+
+			// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/gentoodistfiles.py
+
+			// 	csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
+			// upstream_url = data[project]["upstream"] + data[project]["file"]
+			csc_url := config.MirrorBaseURL + data.CSC + data.File
+			upstream_url := data.Upstream + data.File
+
+			// req = requests.get(csc_url)
+			// req.raise_for_status()
+			// CSC = req.text
+			csc_body, err := httpGET(csc_url)
+			if err != nil {
+				return false, GetError(err, "GentooDistFiles", "getting CSC file")
+			}
+
+			upstream_body, err := httpGET(upstream_url)
+			if err != nil {
+				return false, GetError(err, "GentooDistFiles", "getting upstream file")
+			}
+
+			CSC := string(csc_body)
+			upstream := string(upstream_body)
+
+			if CSC == upstream {
+				return true, nil
+			}
+
+			// parse time as int
+			CSC_utc_time, err := strconv.ParseInt(strings.TrimSpace(CSC[0:11]), 10, 64)
+			if err != nil {
+				return false, GetError(err, "GentooDistFiles", "parsing CSC date")
+			}
+			upstream_utc_time, err := strconv.ParseInt(strings.TrimSpace(upstream[0:11]), 10, 64)
+			if err != nil {
+				return false, GetError(err, "GentooDistFiles", "parsing upstream date")
+			}
+
+			delta := (upstream_utc_time - CSC_utc_time)
+
+			return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
+		}),
+	},
+}

checkers/gentooportage.go

@@ -0,0 +1,53 @@
+package checkers
+
+import (
+	"regexp"
+	"strings"
+	"time"
+)
+
+var GentooPortageProject Project = Project{
+	Name:          "gentooportage",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("gentooportage", true, func(*Project) (bool, error) {
+			// config sanity check
+			data := EnabledProjects["gentooportage"].Properties
+			// TODO: assert
+			err := AssertStrings()
+			if err != nil {
+				return false, GetError(err, "GentooPortage", "config sanity check")
+			}
+
+			// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/gentooportage.py
+
+			upstream_body, err := httpGET(data.Upstream)
+			if err != nil {
+				return false, GetError(err, "GentooPortage", "getting upstream file")
+			}
+			page := string(upstream_body)
+
+			indexOfFile := strings.Index(page, "rsync4.ca.gentoo.org")
+			if indexOfFile == -1 {
+				return false, GetError(nil, "GentooPortage", "no index of file")
+			}
+
+			re := regexp.MustCompile(`(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)`)
+			m := re.FindStringSubmatch(page[indexOfFile:])
+			if len(m) == 0 || len(m[0]) == 0 {
+				return false, GetError(nil, "GentooPortage", "no matches for regex in file")
+			}
+
+			// fmt.Println(m[0])
+			// eg. duration: "20 minutes"
+			duration, err := getTimeDelta(m[0])
+			if err != nil {
+				return false, GetError(err, "GentooPortage", "parsing duration")
+			}
+
+			// Return whether the duration is less than or equal to the out_of_sync_interval
+			return (duration <= time.Duration(data.OOSInterval)*time.Second), nil
+		}),
+	},
+}

checkers/manjaro.go

@@ -0,0 +1,60 @@
+package checkers
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var ManjaroProject Project = Project{
+	Name:          "manjaro",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("manjaro", true, func(*Project) (bool, error) {
+			// config sanity check
+			data := EnabledProjects["manjaro"].Properties
+			// TODO: assert
+			err := AssertStrings()
+			if err != nil {
+				return false, GetError(err, "Manjaro", "config sanity check")
+			}
+
+			// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/manjaro.py
+
+			upstream_body, err := httpGET(data.Upstream)
+			if err != nil {
+				return false, GetError(err, "Manjaro", "getting upstream file")
+			}
+			page := string(upstream_body)
+
+			indexOfFile := strings.Index(page, "mirror.csclub.uwaterloo.ca/manjaro")
+			if indexOfFile == -1 {
+				return false, GetError(nil, "Manjaro", "no index of file")
+			}
+
+			re := regexp.MustCompile(`(?P<hours>\d+):(?P<minutes>\d+)`)
+			m := re.FindStringSubmatch(page[indexOfFile:])
+			if len(m) == 0 || len(m[0]) == 0 {
+				return false, GetError(nil, "Manjaro", "no matches for regex in file")
+			}
+
+			// fmt.Println(m[0])
+			// eg. 02:33
+			split := strings.Split(m[0], ":")
+			hours, err := strconv.Atoi(split[0])
+			if err != nil {
+				return false, GetError(err, "Manjaro", "parsing hours")
+			}
+			minutes, err := strconv.Atoi(split[1])
+			if err != nil {
+				return false, GetError(err, "Manjaro", "parsing minutes")
+			}
+
+			duration := time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute
+
+			return (duration <= time.Duration(data.OOSInterval)*time.Second), nil
+		}),
+	},
+}

checkers/ubuntu.go

@@ -0,0 +1,39 @@
+package checkers
+
+import (
+	"strings"
+
+	"github.com/rs/zerolog/log"
+)
+
+var UbuntuProject Project = Project{
+	Name:          "ubuntu",
+	Properties:    DefaultProjectProperties,
+	NumOfCheckers: 1,
+	Checkers: []*ProjectChecker{
+		GetDefaultChecker("ubuntu", true, func(*Project) (bool, error) {
+			data := EnabledProjects["ubuntu"].Properties
+			err := AssertStrings(data.Upstream)
+			if err != nil {
+				return false, GetError(err, "Ubuntu", "config sanity check")
+			}
+
+			// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/ubuntu.py
+			res, err := httpGET(data.Upstream)
+			if err != nil {
+				return false, GetError(err, "Ubuntu", "getting upstream data")
+			}
+			page := string(res)
+
+			// count occurences of "Up to date"
+			count := strings.Count(page, "Up to date")
+
+			NUM_UBUNTU_RELEASES := 24 // TODO: should be updated automatically (from another source)
+			THRESHOLD := 5            // it would be pretty bad if we don't update this checker for 5 (major) ubuntu releases
+
+			log.Debug().Str("project", "Ubuntu").Int("count", count).Msg("counted occurences of 'Up to date'")
+
+			return count >= NUM_UBUNTU_RELEASES && count < NUM_UBUNTU_RELEASES+THRESHOLD, nil
+		}),
+	},
+}

checkers/utils.go

@@ -5,11 +5,35 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"
 
 	"github.com/rs/zerolog/log"
 )
 
+// duration parser
+func getTimeDelta(input string) (time.Duration, error) {
+	input = strings.TrimSpace(input)
+	parts := strings.Split(input, " ")
+
+	number, err := strconv.Atoi(parts[0])
+	if err != nil {
+		return 0, err
+	}
+
+	switch parts[1] {
+	case "minute", "minutes":
+		return time.Duration(number) * time.Minute, nil
+	case "hour", "hours":
+		return time.Duration(number) * time.Hour, nil
+	case "day", "days":
+		return time.Duration(number) * 24 * time.Hour, nil
+	default:
+		return 0, fmt.Errorf("unknown time unit")
+	}
+}
+
 // http helpers
 func httpGET(url string) ([]byte, error) {
 	res, err := http.Get(url)

config/config.go

@@ -5,6 +5,5 @@ import (
 )
 
 var MirrorBaseURL = "http://mirror.csclub.uwaterloo.ca/" // TODO: must start with https://
-var NUM_UBUNTU_RELEASES = 18                             // TODO: should be updated automatically
 
 var loggingTimeFormat = time.RFC3339