Compare commits

...

No commits in common. "ng" and "master" have entirely different histories.
ng ... master

96 changed files with 2510 additions and 2273 deletions

152
.gitignore vendored
View File

@ -1,22 +1,142 @@
TODO.md
bin/
.vscode/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# From https://raw.githubusercontent.com/github/gitignore/main/Go.gitignore
*.exe
*.exe~
*.dll
# C extensions
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Dependency directories (remove the comment below to include it)
# vendor/
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Go workspace file
go.work
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
# Idea
.idea/
*~

View File

@ -1,26 +0,0 @@
.DEFAULT_GOAL := build-n-run
run:
./bin/mc2
build:
go build -o bin/mc2 -ldflags "\
-X 'mc2/config.BuildVersion=$$(git rev-parse --abbrev-ref HEAD)' \
-X 'mc2/config.BuildUser=$$(id -u -n)' \
-X 'mc2/config.BuildTime=$$(date)' \
-X 'mc2/config.BuildGOOS=$$(go env GOOS)' \
-X 'mc2/config.BuildARCH=$$(go env GOARCH)' \
-s -w"
build-n-run:
make build
make run
docker-build:
docker build -t mc2 .
docker-run:
docker run --rm -p 4200:4200 mc2
test:
go test -v ./...

104
README.md
View File

@ -1,38 +1,94 @@
# Mirror Checker 2
# Mirror Checker
To be written in the future....
This mirror status checker determines whether CSC mirror is up-to-date with upstream.
## How To Run
## How to test
A configuration file may be provided through standard input. Without a configuration file, execute `python main.py`. By default, all the available distributions will be checked. With a configuration file, execute `python main.py < name_of_config_file.in`, for example, `python main.py < example.in`. In this case, only the distributions listed in the configuration file will be checked.
## Dev Notes
How the program works: We first have a general mirror check class called project.py which checks whether the timestamp in the directory of the mirror is in-sync with the upstream. Then, for each CSC mirror, a class is built which inherits from the general project.py class but often overrides the original check function with a check function specific to the mirror. A few big themes are: some check a mirror status tracker provided by the project mirrored; some check all the Release files for each version in a distro etc. website information which all the mirror checker classes need is stored in the data.json file.
## How to build
Future notes: Because many of the mirror checkers are built very specific to each mirror. A slight change in the way the project manages their mirror-related websites, public repos etc. can drastically influence whether the mirror checker works correctly or not. These problems are also unfortunately very hard to detect, so it's important that CSC actively maintain the mirror checker so that it works as intended in the long term.
Download dependencies:
```
go get -u
```
Extra notes: A test client for individual mirror checker classes is provided as test.py. To use it, simply change all occurrences of the imported project class
## Examples
## Resources
```
go run . check debian ubuntu < data/example.in # from the project root directory, read the config and run all checks for the specified projects
```
- [CSC Mirror](http://mirror.csclub.uwaterloo.ca/)
- [Debian Mirror Status Checker](https://mirror-master.debian.org/status/mirror-status.html)
- [Debian Mirror Status Checker Code](https://salsa.debian.org/mirror-team/mirror/status)
## Commands
After building...
## Notes (for devs)
...
> **From the original mirror checker**: Future notes: Because many of the mirror checkers are built very specific to each mirror. A slight change in the way the project manages their mirror-related websites, public repos etc. can drastically influence whether the mirror checker works correctly or not. These problems are also unfortunately very hard to detect, so it's important that CSC actively maintain the mirror checker so that it works as intended in the long term.
>
> Extra notes: A test client for individual mirror checker classes is provided as test.py. To use it, simply change all occurrences of the imported project class
if we can just view their repo online, we only have to remember the link for their repo and then check the latest timestamp in their repo the same way we check ours
even if the date relies on a specific file in their repo, we can still find the right link for it
to find repos of the mirrored projects to check, just search "projectName mirrors"
## Checker Information
- almalinux
- alpine
- apache
- archlinux
- centos
- ceph
- CPAN
- CRAN: https://cran.r-project.org/mirmon_report.html has a mirror tracker
- csclub: for now, this is the upstream itself, so it needs not to be checked
- CTAN: https://www.ctan.org/mirrors/mirmon has a mirror tracker
- Cygwin
- damnsmalllinux: http://distro.ibiblio.org/damnsmall/ not checking this, since it's abandoned
- debian
- debian-backports: this is a legacy thing, no longer have to check
- debian-cd
- debian-multimedia
- debian-ports
- debian-security
- debian-volatile: this is a legacy thing, no longer have to check
- eclipse
- emacsconf: for now, this is the upstream itself, so it needs not to be checked
- fedora
- freeBSD
- gentoo-distfiles
- gentoo-portage
- gnome
- GNU
- gutenberg
- ipfire
- kde
- kde-applicationdata
- kernel
- linuxmint: https://mirrors.edge.kernel.org/linuxmint/ candidate for brute force looping
- linuxmint-packages: https://mirrors.edge.kernel.org/linuxmint-packages/ Checking the timestamp of either the Release file or the Packages file should suffice.
- macPorts: only distfiles has public repo, no timestamp, too large to loop through, comparing ports.tar.gz in distfiles
- manjaro
- mxlinux
- mxlinux-iso: this one seems out of sync on the official tracker for 134 days, which is weird
- mysql: http://mirrors.sunsite.dk/mysql/
- NetBSD: http://ftp.netbsd.org/pub/NetBSD/ checking timestamps of change files in different versions, and SHA512, MD5 files in the isos of different versions
- nongnu: http://download.savannah.nongnu.org/releases/ https://savannah.gnu.org/maintenance/Mirmon/ http://download.savannah.gnu.org/mirmon/savannah/
- openbsd
- opensuse: http://download.opensuse.org/ check Update.repo files in folders inside the update folder, not checking tumbleweed-non-oss/ and tumbleweed/ temporarily
- parabola: https://repo.parabola.nu/ https://www.parabola.nu/mirrors/status/
- pkgsrc
- puppylinux: https://distro.ibiblio.org/puppylinux/ check the ISO files or htm files in the folders starting with puppy
- qtproject: https://download.qt.io/
- racket: https://mirror.racket-lang.org/installers/ make sure that we have the latest version number under racket-installers
- raspberry pi: https://archive.raspberrypi.org/ Checking the timestamp of either the Release file or the Packages file should suffice.
- raspbian: http://archive.raspbian.org/ snapshotindex.txt is most likely a timestamp, tho i'm not sure. also i think our mirror is completely outdated, it's not listed on official mirror list
- sagemath: same source tarballs as them (the sage-*.tar.gz files under 'Source Code')
- salt stack: checking the "Latest release" text under the 'About' header
- scientific: https://scientificlinux.org/downloads/sl-mirrors/ not checking this one since it's abandoned
- slackware: https://mirrors.slackware.com/slackware/ check whether we have each release and whether the timestamp for CHECKSUMS.md5 in each release is the same, for slackware-iso, just make sure that our list of directories is the same
- tdf: https://download.documentfoundation.org/
- trisquel: http://archive.trisquel.info/trisquel/ checking Release file for all versions in packages/dist and md5sum.txt in iso/ with two other mirrors
- ubuntu: https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive
- ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/ checking the Release files in dists
- ubuntu-ports-releases: https://cdimage.ubuntu.com/releases/ has public repo, no timestamp, no status tracker, brute force looped it
- ubuntu-releases: https://releases.ubuntu.com/
- vlc: http://download.videolan.org/pub/videolan/
- x.org: https://www.x.org/releases/ check all of the files under each directory under /x.org/individual/, and make sure that we have all of the files which the upstream has, ignoring the xcb folder
- Xiph: https://ftp.osuosl.org/pub/xiph/releases/ loop through each directory in xiph/releases/ and trying to compare the timestamp of the checksum files
- xubuntu-releases: https://cdimage.ubuntu.com/xubuntu/releases/ candidate for brute force looping since it has few folders

View File

@ -1,3 +0,0 @@
package checkers
// https://mirrors.almalinux.org/

View File

@ -1,246 +0,0 @@
package checkers
import (
"bufio"
"errors"
"fmt"
"os"
"strconv"
"sync"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
)
// TODO: uuid for each job
// TODO: job history
type JobDescription struct {
Group *JobGroup
Checker *ProjectChecker
Callback CheckerResultCallback
}
type JobGroup struct {
Projects []string
Name string
Jobs []*JobDescription
Results *(chan CheckerResult)
FinalStatus *(chan CheckerStatus)
Wg *sync.WaitGroup
WorkerID int
}
////////////////////////////////////////////////////////////////////////////////
// TODO: run check without callback
// TODO: create "custom" job groups??
// Given a project, all checks are initiated.
func (p *Project) RunChecks(callback CheckerResultCallback) (*JobGroup, error) {
checks := p.Checkers
n := len(checks)
if n == 0 {
return nil, errors.New("No checkers found for project.")
}
if n != p.NumOfCheckers {
return nil, errors.New("Number of checkers does not match project config.")
}
// TODO: assert job group properties?
log.Debug().Msgf("Running %d checks for project %s", n, p.Name)
var jg JobGroup
statusResult := make(chan CheckerStatus)
jobs := make([]*JobDescription, n)
for i, c := range p.Checkers {
jobs[i] = &JobDescription{
Group: &jg,
Checker: c,
Callback: callback,
}
}
results := make(chan CheckerResult, n)
jg = JobGroup{
Projects: []string{p.Name},
Name: fmt.Sprintf("check_%s_project", p.Name),
Jobs: jobs,
Results: &results,
FinalStatus: &statusResult,
Wg: &sync.WaitGroup{},
}
for i, jd := range jobs {
log.Info().Str("project", p.Name).
Str("check_name", jd.Checker.Name).
Msgf("Running check %d", i)
jd.QueueJob(callback)
}
go func() {
// Wait for all checks to complete
log.Debug().
Str("project", p.Name).
Str("job_group", jg.Name).
Msgf("Waiting for %d checks to complete.", n)
for i := 0; i < n; i++ {
res := <-results
log.Debug().Str("project", p.Name).
Str("job_group", jg.Name).
Str("worker_id", strconv.Itoa(jg.WorkerID)).
Str("checker_name", res.CheckerName).
Msg("Received checker result.")
if res.Status == CHECKER_ERROR {
// TODO: log checker UUID
log.Error().Err(res.Error).Msg("Checker returned error.")
statusResult <- CHECKER_ERROR
// TODO: stop exisiting jobs
return
}
if res.Status == CHECKER_FAIL {
// TODO: log checker UUID
log.Debug().Msg("Checker failed.")
statusResult <- CHECKER_FAIL
return
}
}
close(results)
statusResult <- CHECKER_SUCCESS
}()
return &jg, nil
}
func (jd *JobDescription) addLogProps(evt *zerolog.Event) *zerolog.Event {
return evt.
Str("project", jd.Group.Projects[0]).
Str("job_group", jd.Group.Name).
Str("worker_id", strconv.Itoa(jd.Group.WorkerID)).
Str("checker_name", jd.Checker.Name)
}
func (jd *JobDescription) LogInfo() *zerolog.Event {
return jd.addLogProps(log.Info())
}
func (jd *JobDescription) LogError() *zerolog.Event {
return jd.addLogProps(log.Error())
}
func ReadEnabledFromStdin() error {
scanner := bufio.NewScanner(os.Stdin)
scanner.Split(bufio.ScanWords)
for scanner.Scan() {
token := scanner.Text()
_, err := LoadProject(token)
if err != nil {
return err
}
}
return scanner.Err()
}
////////////////////////////////////////////////////////////////////////////////
const WORKER_COUNT = 5
func StartWorkers() {
log.Debug().Msgf("Starting %d workers.", WORKER_COUNT)
for i := 0; i < WORKER_COUNT; i++ {
go worker(i)
}
}
// TODO: available worker?
// var workerAvailable sync.WaitGroup
var jobs = make(chan JobDescription, WORKER_COUNT)
func (job JobDescription) QueueJob(callback CheckerResultCallback) {
if job.Group.Wg != nil {
job.Group.Wg.Add(1)
}
// TODO: workerAvailable.Wait() // wait for a worker to be available
jobs <- job
log.Debug().Str("checker_name", job.Checker.Name).Msg("Queued checker.")
}
func StopWorkers() {
log.Debug().Msg("Stopping workers.")
close(jobs)
// TODO: stop in process jobs???
}
// TODO: id to uuid?
func worker(id int) {
for j := range jobs {
var res CheckerResult = DefaultCheckerResult
res.CheckerName = j.Checker.Name
res.ProjectName = j.Group.Name
res.Time = time.Now()
log.Debug().Str("project", j.Checker.Name).
Str("worker_id", strconv.FormatInt(int64(id), 10)).
Msgf("Running check.")
success, err := j.Checker.CheckProject()
res.EndTime = time.Now()
if err != nil {
res.Status = CHECKER_ERROR
res.Error = err
} else if success {
res.Status = CHECKER_SUCCESS
} else {
res.Status = CHECKER_FAIL
}
*j.Group.Results <- res
if j.Group.Wg != nil {
j.Group.Wg.Done() // called before callback to prevent unnecessary waiting
}
if j.Callback != nil {
j.Callback(res)
} else {
log.Debug().Str("project", j.Checker.Name).
Str("worker_id", strconv.FormatInt(int64(id), 10)).
Msgf("No callback registered.")
}
}
}
////////////////////////////////////////////////////////////////////////////////
// Looks up a project by name, and returns only if the project is enabled.
func GetProject(name string) (*Project, error) {
res, exists := EnabledProjects[name]
if !exists {
return res, errors.New("requested project not found")
}
return res, nil
}
// Loads a project by name, and returns if the project is found.
func LoadProject(name string) (*Project, error) {
res, exists := SupportedProjects[name]
if !exists {
return res, errors.New("requested project not found")
}
if res.Properties.CSC == "unknown" {
log.Debug().Str("csc", res.Properties.CSC).
Msgf("Requested project %s has default properties.", name)
return res, errors.New("requested project has invalid properties (check the project config)")
}
log.Debug().Msgf("Loading Project %s", name)
EnabledProjects[name] = res
return res, nil
}

View File

@ -1,49 +0,0 @@
package checkers
import (
"encoding/json"
"os"
"github.com/Nathan13888/mirror-checker2/v2/config"
"github.com/rs/zerolog/log"
)
type MirrorData map[string]ProjectProperties
func LoadFromFile(path string) error {
raw_data, err := os.ReadFile(path)
if err != nil {
return err
}
var data MirrorData
err = json.Unmarshal(raw_data, &data)
if err != nil {
return err
}
// TODO: load in alphabetical order
// Access the parsed data
for proj, prop := range data {
log.Info().Str("project", proj).Msg("Enabled Project.")
sp, exists := SupportedProjects[config.NormalizeName(proj)]
if !exists {
log.Warn().Str("project", proj).Msg("Project not supported.")
continue
}
sp.Properties = prop
log.Debug().
Str("distribution", proj).
// Time("out_of_sync_since", prop.OOSSince).
Int64("out_of_sync_interval", prop.OOSInterval).
Str("csc", prop.CSC).
Str("upstream", prop.Upstream).
Str("file", prop.File).
Msg("Loaded project config.")
}
return nil
}

View File

@ -1,71 +0,0 @@
package checkers
import (
"strings"
"time"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var DebianProject Project = Project{
Name: "debian",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("debian", true, func(*Project) (bool, error) {
// config sanity check
data := EnabledProjects["debian"].Properties
err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
if err != nil {
return false, GetError(err, "Debian", "config sanity check")
}
// Modelled after: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/debian.py
// NOTE: cloned in debiancd, debianmultimedia, debianports, debiansecurity
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
// make HTTP GET request to csc_url
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "Debian", "getting CSC file")
}
// make HTTP GET request to upstream_url
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "Debian", "getting upstream file")
}
// parse bodies as string
CSC := string(csc_body)
upstream := string(upstream_body)
// same date stamp
if CSC == upstream {
return true, nil
}
// check if delta is within threshold
// equiv to: "%a %b %d %H:%M:%S UTC %Y"
date_format := "Mon Jan 2 15:04:05 MST 2006"
CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
if err != nil {
return false, GetError(err, "Debian", "parsing CSC date")
}
CSC_utc_time := CSC_date.Unix()
upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
if err != nil {
return false, GetError(err, "Debian", "parsing upstream date")
}
upstream_utc_time := upstream_date.Unix()
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,62 +0,0 @@
package checkers
import (
"strings"
"time"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var DebianCDProject Project = Project{
Name: "debiancd",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("debiancd", true, func(*Project) (bool, error) {
// based on debian.go checker
// config sanity check
data := EnabledProjects["debiancd"].Properties
err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
if err != nil {
return false, GetError(err, "DebianCD", "config sanity check")
}
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "DebianCD", "getting CSC file")
}
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "DebianCD", "getting upstream file")
}
CSC := string(csc_body)
upstream := string(upstream_body)
if CSC == upstream {
return true, nil
}
date_format := "Mon 2 Jan 15:04:05 MST 2006"
CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
if err != nil {
return false, GetError(err, "DebianCD", "parsing CSC date")
}
CSC_utc_time := CSC_date.Unix()
upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
if err != nil {
return false, GetError(err, "DebianCD", "parsing upstream date")
}
upstream_utc_time := upstream_date.Unix()
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,62 +0,0 @@
package checkers
import (
"strings"
"time"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var DebianMultimediaProject Project = Project{
Name: "debianmultimedia",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("debianmultimedia", true, func(*Project) (bool, error) {
// based on debian.go checker
// config sanity check
data := EnabledProjects["debianmultimedia"].Properties
err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
if err != nil {
return false, GetError(err, "DebianMultimedia", "config sanity check")
}
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "DebianMultimedia", "getting CSC file")
}
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "DebianMultimedia", "getting upstream file")
}
CSC := string(csc_body)
upstream := string(upstream_body)
if CSC == upstream {
return true, nil
}
date_format := "Mon Jan 2 15:04:05 MST 2006"
CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
if err != nil {
return false, GetError(err, "DebianMultimedia", "parsing CSC date")
}
CSC_utc_time := CSC_date.Unix()
upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
if err != nil {
return false, GetError(err, "DebianMultimedia", "parsing upstream date")
}
upstream_utc_time := upstream_date.Unix()
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,62 +0,0 @@
package checkers
import (
"strings"
"time"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var DebianPortsProject Project = Project{
Name: "debianports",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("debianports", true, func(*Project) (bool, error) {
// based on debian.go checker
// config sanity check
data := EnabledProjects["debianports"].Properties
err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
if err != nil {
return false, GetError(err, "DebianPorts", "config sanity check")
}
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "DebianPorts", "getting CSC file")
}
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "DebianPorts", "getting upstream file")
}
CSC := string(csc_body)
upstream := string(upstream_body)
if CSC == upstream {
return true, nil
}
date_format := "Mon Jan 2 15:04:05 MST 2006"
CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
if err != nil {
return false, GetError(err, "DebianPorts", "parsing CSC date")
}
CSC_utc_time := CSC_date.Unix()
upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
if err != nil {
return false, GetError(err, "DebianPorts", "parsing upstream date")
}
upstream_utc_time := upstream_date.Unix()
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,62 +0,0 @@
package checkers
import (
"strings"
"time"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var DebianSecurityProject Project = Project{
Name: "debiansecurity",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("debiansecurity", true, func(*Project) (bool, error) {
// based on debian.go checker
// config sanity check
data := EnabledProjects["debiansecurity"].Properties
err := AssertStrings(config.MirrorBaseURL, data.Upstream, data.File)
if err != nil {
return false, GetError(err, "DebianSecurity", "config sanity check")
}
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "DebianSecurity", "getting CSC file")
}
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "DebianSecurity", "getting upstream file")
}
CSC := string(csc_body)
upstream := string(upstream_body)
if CSC == upstream {
return true, nil
}
date_format := "Mon Jan 2 15:04:05 MST 2006"
CSC_date, err := time.Parse(date_format, strings.Split(CSC, "\n")[0])
if err != nil {
return false, GetError(err, "DebianSecurity", "parsing CSC date")
}
CSC_utc_time := CSC_date.Unix()
upstream_date, err := time.Parse(date_format, strings.Split(upstream, "\n")[0])
if err != nil {
return false, GetError(err, "DebianSecurity", "parsing upstream date")
}
upstream_utc_time := upstream_date.Unix()
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,199 +0,0 @@
package checkers
import (
"fmt"
"time"
"github.com/rs/zerolog/log"
)
////////////////////////////////////////////////////////////////////////////////
var DefaultProjectProperties = ProjectProperties{
OOSSince: time.Now(),
OOSInterval: 3600,
CSC: "unknown",
Upstream: "unknown",
File: "unknown",
}
var DefaultCheckerResult CheckerResult = CheckerResult{
ProjectName: "unknown",
CheckerName: "unknown",
Status: CHECKER_PROGRESS,
}
////////////////////////////////////////////////////////////////////////////////
// All "projects" which have been implemented.
var SupportedProjects map[string]*Project = map[string]*Project{
// "almalinux":&AlmaLinuxProject,
// "alpine":&AlpineProject,
// "apache":&ApacheProject,
// "arch":&ArchProject,
// "artix":&ArtixProject,
// "centos":&CentosProject,
// "ceph":&CephProject,
// "cpan":&CpanProject,
// "cran":&CranProject,
// "ctan":&CtanProject,
// "cygwin":&CygwinProject,
"debian": &DebianProject,
"debiancd": &DebianCDProject,
"debianmultimedia": &DebianMultimediaProject,
"debianports": &DebianPortsProject,
"debiansecurity": &DebianSecurityProject,
// "eclipse":&EclipseProject,
"fedora": &FedoraProject,
// "freebsd":&FreebsdProject,
"gentoodistfiles": &GentooDistProject,
"gentooportage": &GentooPortageProject,
// "gnome":&GnomeProject,
// "gnu":&GnuProject,
// "gutenberg":&GutenbergProject,
// "ipfire":&IpfireProject,
// "kde":&KdeProject,
// "kdeapplicationdata":&KdeApplicationDataProject,
// "kernel":&KernelProject,
// "linuxmint":&LinuxMintProject,
// "linuxmint_packages":&LinuxMintPackagesProject,
// "macports":&MacPortsProject,
"manjaro": &ManjaroProject,
// "mxlinux":&MxLinuxProject,
// "mxlinux_iso":&MxLinuxIsoProject,
// "mysql":&MysqlProject,
// "netbsd":&NetBsdProject,
// "nongnu":&NongnuProject,
// "openbsd":&OpenbsdProject,
// "opensuse":&OpensuseProject,
// "parabola":&ParabolaProject,
// "pkgsrc":&PkgsrcProject,
// "puppy_linux":&PuppyLinuxProject,
// "qtproject":&QtProject,
// "racket":&RacketProject,
// "raspberrypi":&RaspberrypiProject,
// "raspbian":&RaspbianProject,
// "sage":&SageProject,
// "saltstack":&SaltStackProject,
// "slackware":&SlackwareProject,
// "tdf":&TdfProject,
// "trisquel":&TrisquelProject,
"ubuntu": &UbuntuProject,
// "ubuntu_ports":&UbuntuPortsProject,
// "ubuntu_ports_releases":&UbuntuPortsReleasesProject,
// "ubuntu_releases":&UbuntuReleasesProject,
// "vlc":&VlcProject,
// "x_org":&XorgProject,
// "xiph":&XiphProject,
// "xubuntu_releases":&XubuntuReleasesProject,
}
func LoadDefaultProjects() {
// Load all projects that's implemented
log.Info().Msg("Loading Default Projects.")
// LoadProject("almalinux")
// LoadProject("alpine")
// LoadProject("apache")
// LoadProject("arch")
// LoadProject("artix")
// LoadProject("centos")
// LoadProject("ceph")
// LoadProject("cpan")
// LoadProject("cran")
// LoadProject("ctan")
// LoadProject("cygwin")
LoadProject("debian")
LoadProject("debiancd")
LoadProject("debianmultimedia")
LoadProject("debianports")
LoadProject("debiansecurity")
// LoadProject("eclipse")
LoadProject("fedora")
// LoadProject("freebsd")
// LoadProject("gentoodistfiles")
// LoadProject("gentooportage")
// LoadProject("gnome")
// LoadProject("gnu")
// LoadProject("gutenberg")
// LoadProject("ipfire")
// LoadProject("kde")
// LoadProject("kdeapplicationdata")
// LoadProject("kernel")
// LoadProject("linuxmint")
// LoadProject("linuxmint_packages")
// LoadProject("macports")
// LoadProject("manjaro")
// LoadProject("mxlinux")
// LoadProject("mxlinux_iso")
// LoadProject("mysql")
// LoadProject("netbsd")
// LoadProject("nongnu")
// LoadProject("openbsd")
// LoadProject("opensuse")
// LoadProject("parabola")
// LoadProject("pkgsrc")
// LoadProject("puppy_linux")
// LoadProject("qtproject")
// LoadProject("racket")
// LoadProject("raspberrypi")
// LoadProject("raspbian")
// LoadProject("sage")
// LoadProject("saltstack")
// LoadProject("slackware")
// LoadProject("tdf")
// LoadProject("trisquel")
LoadProject("ubuntu")
// LoadProject("ubuntu_ports")
// LoadProject("ubuntu_ports_releases")
// LoadProject("ubuntu_releases")
// LoadProject("vlc")
// LoadProject("x_org")
// LoadProject("xiph")
// LoadProject("xubuntu_releases")
}
func AssertStrings(s ...string) error {
for i, str := range s {
if str == "" {
return fmt.Errorf("failed assert: string %d of %d is empty", i+1, len(s))
}
}
return nil
}
// TODO: add checker logger
func GetDefaultChecker(name string, def bool, f func(*Project) (bool, error)) *ProjectChecker {
checkerName := name
if def {
checkerName += "_default"
}
// configure checker function
var checkerFunc func() (bool, error)
if f != nil {
// has custom checker function
checkerFunc = func() (bool, error) {
proj, err := GetProject(name)
if err != nil {
return false, err
}
return f(proj)
}
} else {
// no custom checker function
checkerFunc = func() (bool, error) {
return false, fmt.Errorf("project '%s' not supported", name)
}
}
checker := ProjectChecker{
Name: checkerName,
CheckProject: checkerFunc,
Default: def,
}
return &checker
}

View File

@ -1,63 +0,0 @@
package checkers
import (
"time"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var FedoraProject Project = Project{
Name: "fedora",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("fedora", true, func(*Project) (bool, error) {
// config sanity check
data := EnabledProjects["fedora"].Properties
err := AssertStrings(config.MirrorBaseURL, data.CSC, data.File, data.Upstream)
if err != nil {
return false, GetError(err, "Fedora", "config sanity check")
}
// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/fedora.py
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "Fedora", "getting CSC file")
}
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "Fedora", "getting upstream file")
}
CSC := string(csc_body)
upstream := string(upstream_body)
if CSC == upstream {
return true, nil
}
// # Date example: Fedora-Rawhide-20220725.n.1
data_format := "20060102"
CSC_date, err := time.Parse(data_format, CSC[15:23])
if err != nil {
return false, GetError(err, "Fedora", "parsing CSC date")
}
CSC_utc_time := CSC_date.Unix()
upstream_date, err := time.Parse(data_format, upstream[15:23])
if err != nil {
return false, GetError(err, "Fedora", "parsing upstream date")
}
upstream_utc_time := upstream_date.Unix()
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,61 +0,0 @@
package checkers
import (
"strconv"
"strings"
"github.com/Nathan13888/mirror-checker2/v2/config"
)
var GentooDistProject Project = Project{
Name: "gentoodistfiles",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("gentoodistfiles", true, func(*Project) (bool, error) {
// config sanity check
data := EnabledProjects["gentoodistfiles"].Properties
// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/gentoodistfiles.py
// csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
// upstream_url = data[project]["upstream"] + data[project]["file"]
csc_url := config.MirrorBaseURL + data.CSC + data.File
upstream_url := data.Upstream + data.File
// req = requests.get(csc_url)
// req.raise_for_status()
// CSC = req.text
csc_body, err := httpGET(csc_url)
if err != nil {
return false, GetError(err, "GentooDistFiles", "getting CSC file")
}
upstream_body, err := httpGET(upstream_url)
if err != nil {
return false, GetError(err, "GentooDistFiles", "getting upstream file")
}
CSC := string(csc_body)
upstream := string(upstream_body)
if CSC == upstream {
return true, nil
}
// parse time as int
CSC_utc_time, err := strconv.ParseInt(strings.TrimSpace(CSC[0:11]), 10, 64)
if err != nil {
return false, GetError(err, "GentooDistFiles", "parsing CSC date")
}
upstream_utc_time, err := strconv.ParseInt(strings.TrimSpace(upstream[0:11]), 10, 64)
if err != nil {
return false, GetError(err, "GentooDistFiles", "parsing upstream date")
}
delta := (upstream_utc_time - CSC_utc_time)
return (delta < data.OOSInterval && delta > -data.OOSInterval), nil
}),
},
}

View File

@ -1,53 +0,0 @@
package checkers
import (
"regexp"
"strings"
"time"
)
var GentooPortageProject Project = Project{
Name: "gentooportage",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("gentooportage", true, func(*Project) (bool, error) {
// config sanity check
data := EnabledProjects["gentooportage"].Properties
// TODO: assert
err := AssertStrings()
if err != nil {
return false, GetError(err, "GentooPortage", "config sanity check")
}
// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/gentooportage.py
upstream_body, err := httpGET(data.Upstream)
if err != nil {
return false, GetError(err, "GentooPortage", "getting upstream file")
}
page := string(upstream_body)
indexOfFile := strings.Index(page, "rsync4.ca.gentoo.org")
if indexOfFile == -1 {
return false, GetError(nil, "GentooPortage", "no index of file")
}
re := regexp.MustCompile(`(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)`)
m := re.FindStringSubmatch(page[indexOfFile:])
if len(m) == 0 || len(m[0]) == 0 {
return false, GetError(nil, "GentooPortage", "no matches for regex in file")
}
// fmt.Println(m[0])
// eg. duration: "20 minutes"
duration, err := getTimeDelta(m[0])
if err != nil {
return false, GetError(err, "GentooPortage", "parsing duration")
}
// Return whether the duration is less than or equal to the out_of_sync_interval
return (duration <= time.Duration(data.OOSInterval)*time.Second), nil
}),
},
}

View File

@ -1,50 +0,0 @@
package checkers
import {
"fmt"
"io/ioutil"
"net/http"
"regexp"
"strings"
"time"
}
var LinuxMintProject Project = Project{
Name: "linuxmint",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker {
GetDefaultChecker("linuxmint", true, func(*Project) (bool, error) {
data := EnabledProjects["linuxmint"].Properties
err := AssertStrings(data.Upstream, data.OOSInterval)
if err != nil {
return false, GetError(err, "Linux Mint", "config sanity check")
}
upstream_body, err := httpGET(data.Upstream)
if err != nil {
return false, GetError(err, "Linux Mint", "getting upstream file")
}
page := string(upstream_body)
indexOfFile := strings.Index(page, "mirror.csclub.uwaterloo.ca/linuxmint")
if indexOfFile == -1 {
return false, GetError(nil, "Linux Mint", "no index of file")
}
re := regexp.MustCompile(`(?P<hours>\d+):(?P<minutes>\d+)`)
m := re.FindStringSubmatch(page[indexOfFile:])
if len(m) == 0 || len(m[0]) == 0 {
return false, GetError(nil, "Linux Mint", "no matches for regex in file")
}
split := strings.Split(m[0], ":")
hours, err := strconv.Atoi(split[0])
if err != nil {
return false, GetError(err, "Linux Mint", "parsing hours")
}
minutes, err := strconv.Atoi(split[1])
if err != nil {
return false, GetError(err, "Linux Mint", "parsing minutes")
}
duration := time.Duration(hours) * time.Hour + time.Duration(minutes) * time.Minute
return (duration <= time.Duration(data.OOSInterval) * time.Second), nil
}),
},
}

View File

@ -1,60 +0,0 @@
package checkers
import (
"regexp"
"strconv"
"strings"
"time"
)
var ManjaroProject Project = Project{
Name: "manjaro",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("manjaro", true, func(*Project) (bool, error) {
// config sanity check
data := EnabledProjects["manjaro"].Properties
// TODO: assert
err := AssertStrings()
if err != nil {
return false, GetError(err, "Manjaro", "config sanity check")
}
// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/manjaro.py
upstream_body, err := httpGET(data.Upstream)
if err != nil {
return false, GetError(err, "Manjaro", "getting upstream file")
}
page := string(upstream_body)
indexOfFile := strings.Index(page, "mirror.csclub.uwaterloo.ca/manjaro")
if indexOfFile == -1 {
return false, GetError(nil, "Manjaro", "no index of file")
}
re := regexp.MustCompile(`(?P<hours>\d+):(?P<minutes>\d+)`)
m := re.FindStringSubmatch(page[indexOfFile:])
if len(m) == 0 || len(m[0]) == 0 {
return false, GetError(nil, "Manjaro", "no matches for regex in file")
}
// fmt.Println(m[0])
// eg. 02:33
split := strings.Split(m[0], ":")
hours, err := strconv.Atoi(split[0])
if err != nil {
return false, GetError(err, "Manjaro", "parsing hours")
}
minutes, err := strconv.Atoi(split[1])
if err != nil {
return false, GetError(err, "Manjaro", "parsing minutes")
}
duration := time.Duration(hours)*time.Hour + time.Duration(minutes)*time.Minute
return (duration <= time.Duration(data.OOSInterval)*time.Second), nil
}),
},
}

View File

@ -1,55 +0,0 @@
package checkers
import "time"
// "Projects" which have valid configs.
var EnabledProjects = make(map[string]*Project)
// TODO: refactor all errors into variables
////////////////////////////////////////////////////////////////////////////////
// Project, Project Properties, Project Checker
type Project struct {
Name string
Properties ProjectProperties
NumOfCheckers int
Checkers []*ProjectChecker
}
type ProjectProperties struct {
OOSSince time.Time //`json:"out_of_sync_since"`
OOSInterval int64 `json:"out_of_sync_interval"` // in seconds
CSC string `json:"csc"`
Mirrors []string `json:"mirrors"`
Upstream string `json:"upstream"`
File string `json:"file"`
}
type ProjectChecker struct {
Name string `json:"name"`
CheckProject CPFunc
Default bool `json:"default"`
// TODO: other severity levels?
}
type CPFunc func() (bool, error)
// //////////////////////////////////////////////////////////////////////////////
// Checker Status and Checker Result
type CheckerStatus string
var CHECKER_SUCCESS CheckerStatus = "success"
var CHECKER_PROGRESS CheckerStatus = "progress"
var CHECKER_ERROR CheckerStatus = "error"
var CHECKER_FAIL CheckerStatus = "fail"
type CheckerResult struct {
ProjectName string `json:"project_name"`
CheckerName string `json:"checker_name"`
Time time.Time `json:"start_time"`
EndTime time.Time `json:"end_time"`
Status CheckerStatus `json:"status"`
Error error `json:"error"`
}
type CheckerResultCallback func(CheckerResult)

View File

@ -1,39 +0,0 @@
package checkers
import (
"strings"
"github.com/rs/zerolog/log"
)
var UbuntuProject Project = Project{
Name: "ubuntu",
Properties: DefaultProjectProperties,
NumOfCheckers: 1,
Checkers: []*ProjectChecker{
GetDefaultChecker("ubuntu", true, func(*Project) (bool, error) {
data := EnabledProjects["ubuntu"].Properties
err := AssertStrings(data.Upstream)
if err != nil {
return false, GetError(err, "Ubuntu", "config sanity check")
}
// SOURCE: https://git.csclub.uwaterloo.ca/public/mirror-checker/src/branch/master/projects/ubuntu.py
res, err := httpGET(data.Upstream)
if err != nil {
return false, GetError(err, "Ubuntu", "getting upstream data")
}
page := string(res)
// count occurences of "Up to date"
count := strings.Count(page, "Up to date")
NUM_UBUNTU_RELEASES := 24 // TODO: should be updated automatically (from another source)
THRESHOLD := 5 // it would be pretty bad if we don't update this checker for 5 (major) ubuntu releases
log.Debug().Str("project", "Ubuntu").Int("count", count).Msg("counted occurences of 'Up to date'")
return count >= NUM_UBUNTU_RELEASES && count < NUM_UBUNTU_RELEASES+THRESHOLD, nil
}),
},
}