Compare commits

..

19 Commits

Author SHA1 Message Date
Raymond Li 90bde5a754 Merge pull request 'Fixed slackware checker' (#12) from jtoft-rt-4117-slackware into master 4 months ago
Justin Toft 55f3e7bcbe
Fixed slackware checker 4 months ago
Justin Toft 860d5c8e44 Fixed 6 broken mirror checkers (#11) 4 months ago
Raymond Li 65b06ac0a8 Merge pull request 'Fixed broken ubuntu and ubuntu_ports mirror checks' (#10) from jtoft-rt-4117 into master 6 months ago
Justin Toft 5afebc8030 Fixed broken ubuntu and ubuntu_ports mirror checks 6 months ago
Raymond Li 54d8a47944 Merge pull request 'Update time stamp parsing in cran, ctan, mxlinux' (#9) from time-parsing into master 8 months ago
Rio Liu 789f0cd662 update time stamp parsing in cran, ctan, mxlinux 8 months ago
Raymond Li e1ef917af0 Merge pull request 'fix CTAN && trust CPAN's report' (#8) from y266shen/mirror-checker:feature-retry into master 8 months ago
Yiao Shen 0571f7353a
projects/ctan: make Python happy 8 months ago
Yiao Shen 612ec9d04b
projects/cpan: trust CPAN's mirror status 9 months ago
Raymond Li 73bc2b5ade Merge pull request 'various mirror fixes' (#7) from y266shen/mirror-checker:feature-retry into master 9 months ago
Yiao Shen 9f345c7b0c
main: display how long a mirror has been out-of-sync 9 months ago
Yiao Shen 660b566715
projects/freebsd: correctly parse FreeBSD's timestamp file 9 months ago
Yiao Shen f408922a96
projects/apache: correctly parse Apache's timestamp file 9 months ago
Raymond Li 4a3cedfb07 Merge pull request 'implement retry' (#6) from y266shen/mirror-checker:feature-retry into master 9 months ago
Yiao Shen 76f7863a85
main: retry 3 times before reporting errors 9 months ago
Raymond Li 250f358f4e Merge pull request 'parallize mirror checking and remove special case matching' (#5) from multithread into master 10 months ago
Rio Liu 45af282f83 parallize mirror checking and remove special case matching 10 months ago
Raymond Li 3a793c5f01 Merge pull request 'fix GentooDistfiles, opensuse, tdf checking' (#4) from d9hui/mirror-checker:master into master 10 months ago
  1. 847
      data.json
  2. 101
      main.py
  3. 32
      projects/apache.py
  4. 2
      projects/artix.py
  5. 18
      projects/cpan.py
  6. 6
      projects/cran.py
  7. 7
      projects/ctan.py
  8. 32
      projects/debian.py
  9. 35
      projects/debiancd.py
  10. 35
      projects/fedora.py
  11. 32
      projects/freebsd.py
  12. 30
      projects/gentoodistfiles.py
  13. 4
      projects/gentooportage.py
  14. 3
      projects/manjaro.py
  15. 6
      projects/mxlinux.py
  16. 3
      projects/mxlinux_iso.py
  17. 27
      projects/nongnu.py
  18. 4
      projects/raspberrypi.py
  19. 2
      projects/slackware.py
  20. 3
      projects/ubuntu.py
  21. 8
      projects/ubuntu_releases.py
  22. 1
      shared.py

@ -1,424 +1,425 @@
{
"AlmaLinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME"
},
"Alpine": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated"
},
"Apache": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "apache/",
"upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt"
},
"Arch": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate"
},
"Artix": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "artixlinux/",
"upstream": "https://mirror1.artixlinux.org/repos/"
},
"CentOS": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME"
},
"Ceph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ceph/",
"upstream": "https://download.ceph.com/",
"file": "timestamp"
},
"CPAN": {
"out_of_sync_interval": 172800
},
"cran": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://cran.r-project.org/mirmon_report.html",
"file": ""
},
"ctan": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://www.ctan.org/mirrors/mirmon",
"file": ""
},
"Cygwin": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "cygwin/",
"upstream": "https://cygwin.com/pub/cygwin/",
"file": "x86/sha512.sum"
},
"Debian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master"
},
"DebianCD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org"
},
"DebianMultimedia": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org"
},
"DebianPorts": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org",
"exclude": true
},
"DebianSecurity": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-security/project/trace/master"
},
"Eclipse": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "eclipse/",
"upstream": "http://download.eclipse.org/",
"file": "TIME"
},
"Fedora": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID"
},
"FreeBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP"
},
"GentooDistfiles": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gentoo-distfiles/",
"upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local"
},
"GentooPortage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/",
"upstream": "https://mirrorstats.gentoo.org/rsync/",
"upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest"
},
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt"
},
"Gutenberg": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs"
},
"IPFire": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800
},
"KDE": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR"
},
"KDEApplicationData": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/",
"file": "last-updated"
},
"Kernel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc"
},
"linuxmint": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint/",
"file": ""
},
"linuxmint_packages": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint-packages/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint-packages/",
"file": "dists/"
},
"macports": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "MacPorts/mpdistfiles/",
"upstream": "https://distfiles.macports.org/",
"file": "ports.tar.gz"
},
"manjaro": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.manjaro.org/",
"file": ""
},
"mxlinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://rsync-mxlinux.org/mirmon/packages.html",
"file": ""
},
"mxlinux_iso": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mxlinux-iso/",
"upstream": "http://rsync-mxlinux.org/mirmon/index.html",
"mirrors": [
"http://mirror.its.dal.ca/mxlinux-cd/",
"http://mirror.umd.edu/mxlinux-iso/"
],
"file": ""
},
"mySQL": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mysql/",
"upstream": "http://mirrors.sunsite.dk/mysql/",
"file": "last-updated.txt"
},
"netbsd": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "NetBSD/",
"upstream": "http://ftp.netbsd.org/pub/NetBSD/",
"file": ""
},
"nongnu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "nongnu/",
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
"file": "00_TIME.txt"
},
"OpenBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp",
"exclude": true
},
"opensuse": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "opensuse/update/",
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
"file": ""
},
"parabola": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "parabola/",
"upstream": "https://repo.parabola.nu/",
"file": "lastsync"
},
"pkgsrc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "pkgsrc/",
"upstream": "http://ftp.netbsd.org/pub/pkgsrc/",
"file": "MIRROR-TIMESTAMP",
"exclude": true
},
"puppy_linux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "puppylinux/",
"upstream": "https://distro.ibiblio.org/puppylinux/",
"file": ""
},
"qtproject": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "qtproject/",
"upstream": "https://download.qt.io/",
"file": "timestamp.txt",
"exclude": true
},
"racket": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "racket/racket-installers/",
"upstream": "https://mirror.racket-lang.org/installers/",
"file": ""
},
"raspberrypi": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "raspberrypi/debian/",
"upstream": "https://archive.raspberrypi.org/debian/",
"file": "dists/"
},
"raspbian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "raspbian/",
"upstream": "http://archive.raspbian.org/",
"file": "snapshotindex.txt"
},
"sage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "",
"file": "sage/src/index.html"
},
"saltstack": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "saltstack/",
"upstream": "https://repo.saltproject.io/",
"file": ""
},
"slackware": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "slackware/",
"upstream": "https://mirrors.slackware.com/slackware/",
"file": ""
},
"tdf": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "tdf/",
"upstream": "https://download.documentfoundation.org/",
"file": "TIMESTAMP"
},
"trisquel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "trisquel/",
"upstream": "http://rsync.trisquel.info/trisquel/dists/",
"mirrors": [
"https://mirror.fsf.org/trisquel-images/",
"http://mirrors.ocf.berkeley.edu/trisquel-images/"
],
"file": ""
},
"ubuntu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": ""
},
"ubuntu_ports": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports/",
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
"file": "dists/"
},
"ubuntu_ports_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports-releases/",
"upstream": "https://cdimage.ubuntu.com/releases/",
"file": ""
},
"ubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release",
"file": ""
},
"vlc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "vlc/",
"upstream": "http://download.videolan.org/pub/videolan/",
"file": "trace"
},
"x_org": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "x.org/individual/",
"upstream": "https://www.x.org/releases/individual/",
"file": ""
},
"xiph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xiph/releases/",
"upstream": "https://ftp.osuosl.org/pub/xiph/releases/",
"file": ""
},
"xubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xubuntu-releases/",
"upstream": "https://cdimage.ubuntu.com/xubuntu/releases/",
"file": ""
}
}
"AlmaLinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME"
},
"Alpine": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated"
},
"Apache": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "apache/",
"upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt"
},
"Arch": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate"
},
"Artix": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "artixlinux/",
"upstream": "https://mirror1.artixlinux.org/repos/"
},
"CentOS": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME"
},
"Ceph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ceph/",
"upstream": "https://download.ceph.com/",
"file": "timestamp"
},
"CPAN": {
"out_of_sync_interval": 172800,
"out_of_sync_since": null
},
"cran": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://cran.r-project.org/mirmon_report.html",
"file": ""
},
"ctan": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://www.ctan.org/mirrors/mirmon",
"file": ""
},
"Cygwin": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "cygwin/",
"upstream": "https://cygwin.com/pub/cygwin/",
"file": "x86/sha512.sum"
},
"Debian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master"
},
"DebianCD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org"
},
"DebianMultimedia": {
"out_of_sync_since": 1659116719,
"out_of_sync_interval": 86400,
"csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org"
},
"DebianPorts": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org",
"exclude": true
},
"DebianSecurity": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-security/project/trace/master"
},
"Eclipse": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "eclipse/",
"upstream": "http://download.eclipse.org/",
"file": "TIME"
},
"Fedora": {
"out_of_sync_since": null,
"out_of_sync_interval": 259200,
"csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID"
},
"FreeBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP"
},
"GentooDistfiles": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gentoo-distfiles/",
"upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local"
},
"GentooPortage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/",
"upstream": "https://mirrorstats.gentoo.org/rsync/",
"upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest"
},
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt"
},
"Gutenberg": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs"
},
"IPFire": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800
},
"KDE": {
"out_of_sync_since": 1659116720,
"out_of_sync_interval": 86400,
"csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR"
},
"KDEApplicationData": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/",
"file": "last-updated"
},
"Kernel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc"
},
"linuxmint": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint/",
"file": ""
},
"linuxmint_packages": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint-packages/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint-packages/",
"file": "dists/"
},
"macports": {
"out_of_sync_since": 1642827723,
"out_of_sync_interval": 86400,
"csc": "MacPorts/mpdistfiles/",
"upstream": "https://distfiles.macports.org/",
"file": "ports.tar.gz"
},
"manjaro": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.manjaro.org/",
"file": ""
},
"mxlinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://rsync-mxlinux.org/mirmon/packages.html",
"file": ""
},
"mxlinux_iso": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mxlinux-iso/",
"upstream": "http://rsync-mxlinux.org/mirmon/index.html",
"mirrors": [
"http://mirror.its.dal.ca/mxlinux-cd/",
"http://mirror.umd.edu/mxlinux-iso/"
],
"file": ""
},
"mySQL": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mysql/",
"upstream": "http://mirrors.sunsite.dk/mysql/",
"file": "last-updated.txt"
},
"netbsd": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "NetBSD/",
"upstream": "http://ftp.netbsd.org/pub/NetBSD/",
"file": ""
},
"nongnu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "nongnu/",
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
"file": "00_TIME.txt"
},
"OpenBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp",
"exclude": true
},
"opensuse": {
"out_of_sync_since": 1648699331,
"out_of_sync_interval": 86400,
"csc": "opensuse/update/",
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
"file": ""
},
"parabola": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "parabola/",
"upstream": "https://repo.parabola.nu/",
"file": "lastsync"
},
"pkgsrc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "pkgsrc/",
"upstream": "http://ftp.netbsd.org/pub/pkgsrc/",
"file": "MIRROR-TIMESTAMP",
"exclude": true
},
"puppy_linux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "puppylinux/",
"upstream": "https://distro.ibiblio.org/puppylinux/",
"file": ""
},
"qtproject": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "qtproject/",
"upstream": "https://download.qt.io/",
"file": "timestamp.txt",
"exclude": true
},
"racket": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "racket/racket-installers/",
"upstream": "https://mirror.racket-lang.org/installers/",
"file": ""
},
"raspberrypi": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "raspberrypi/debian/",
"upstream": "https://archive.raspberrypi.org/debian/",
"file": "dists/"
},
"raspbian": {
"out_of_sync_since": 1659116721,
"out_of_sync_interval": 86400,
"csc": "raspbian/",
"upstream": "http://archive.raspbian.org/",
"file": "snapshotindex.txt"
},
"sage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "",
"file": "sage/src/index.html"
},
"saltstack": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "saltstack/",
"upstream": "https://repo.saltproject.io/",
"file": ""
},
"slackware": {
"out_of_sync_since": 1642827723,
"out_of_sync_interval": 86400,
"csc": "slackware/",
"upstream": "https://mirrors.slackware.com/slackware/",
"file": ""
},
"tdf": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "tdf/",
"upstream": "https://download.documentfoundation.org/",
"file": "TIMESTAMP"
},
"trisquel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "trisquel/",
"upstream": "http://rsync.trisquel.info/trisquel/dists/",
"mirrors": [
"https://mirror.fsf.org/trisquel-images/",
"http://mirrors.ocf.berkeley.edu/trisquel-images/"
],
"file": ""
},
"ubuntu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": ""
},
"ubuntu_ports": {
"out_of_sync_since": 1651550528,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports/",
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
"file": "dists/"
},
"ubuntu_ports_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports-releases/",
"upstream": "https://cdimage.ubuntu.com/releases/",
"file": ""
},
"ubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": ""
},
"vlc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "vlc/",
"upstream": "http://download.videolan.org/pub/videolan/",
"file": "trace"
},
"x_org": {
"out_of_sync_since": 1657512131,
"out_of_sync_interval": 86400,
"csc": "x.org/individual/",
"upstream": "https://www.x.org/releases/individual/",
"file": ""
},
"xiph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xiph/releases/",
"upstream": "https://ftp.osuosl.org/pub/xiph/releases/",
"file": ""
},
"xubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xubuntu-releases/",
"upstream": "https://cdimage.ubuntu.com/xubuntu/releases/",
"file": ""
}
}

@ -7,59 +7,88 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
import time
import sys
import requests
from multiprocessing import Pool, Manager
from typing import Optional
from time import sleep, localtime, strftime
from projects import *
import json
NUM_THREAD = 16
MAX_RETRY = 3
RETRY_TIMEOUT = 30 # In seconds
def safe_print(*args, **kwargs):
# When run with 'chronic' and 'timeout', stdout gets suppressed
# due to buffering. Make sure to always flush the output.
print(*args, **kwargs, flush=True)
# Return None if no error occurs and a string for error message otherwise
def check_project(args) -> Optional[str]:
current_time = int(time.time())
project, data = args
try:
project_class = getattr(sys.modules[__name__], project)
if __name__ == "__main__":
# Skip projects we no longer mirror
if data[project].get('exclude', False):
return None
checker_result = project_class.check(data, project, current_time)
if checker_result:
data[project]["out_of_sync_since"] = None
return None
elif (data[project]["out_of_sync_since"] is not None
and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
now_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
duration = current_time - data[project]["out_of_sync_since"]
return f"{project} out-of-sync at {now_str} for {duration}s"
else:
data[project]["out_of_sync_since"] = current_time
return None
except requests.exceptions.RequestException as err:
return f"{project}\n{err}"
def check_project_with_retry(args) -> bool:
project, _ = args
errs = []
for _ in range(MAX_RETRY):
res = check_project(args)
if res == None:
safe_print(f"Success: {project} up-to-date")
return True
else:
errs.append(res)
# Do nothing, try again later
sleep(RETRY_TIMEOUT)
# Max try reached, print errors
safe_print(f"Error: {project}")
for reason in errs:
safe_print(f" {reason}")
exit_code = 0
return False
def main():
data_file = 'data.json'
if len(sys.argv) > 1:
data_file = sys.argv[1]
manager = Manager()
data = json.load(open(data_file))
sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
with Pool(NUM_THREAD) as pool:
all_pass = all(pool.imap(check_project_with_retry, ((k, sync_data) for k in data.keys())))
current_time = int(time.time())
for project in data:
try:
project_class = getattr(sys.modules[__name__], project)
# Skip projects we no longer mirror
if data[project].get('exclude', False):
continue
checker_result = project_class.check(data, project, current_time)
if project in ["CPAN", "ubuntu_releases", "manjaro", "mxlinux", "cran", "ctan", "gentooportage", "Artix"]:
if checker_result:
safe_print(f"Success: {project} up-to-date")
else:
safe_print(f"Failure: {project} out-of-sync")
# Exit with non-zero status if any of the projects are not up-to-date
exit_code = 1
continue
if checker_result:
data[project]["out_of_sync_since"] = None
elif data[project]["out_of_sync_since"] is None:
data[project]["out_of_sync_since"] = current_time
elif current_time - data[project]["out_of_sync_since"] \
> data[project]["out_of_sync_interval"]:
safe_print(f"Failure: {project} out-of-sync")
# Exit with non-zero status if any of the projects are not up-to-date
exit_code = 1
continue
safe_print(f"Success: {project} up-to-date")
except requests.exceptions.RequestException as err:
safe_print(f"Error: {project}\n{err}")
with open(data_file, "w", encoding="utf-8") as file:
json.dump(data, file, indent='\t')
json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')
sys.exit(exit_code)
sys.exit(0 if all_pass else 1)
if __name__ == "__main__":
main()

@ -3,7 +3,35 @@ Contains Apache class
"""
from project import Project
from shared import CSC_MIRROR
import requests
class Apache(Project):
"""Apache class"""
"""Apache class"""
# Apache's time file has two segments, so we need a special function
# Example: 1648323001 rsync-he-fi
def check(data, project, current_time):
"""Check if project packages are up-to-date"""
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
try:
return get_timestamp_from_apache(upstream) - get_timestamp_from_apache(CSC) < data[project]["out_of_sync_interval"]
except ValueError:
print("failed to parse apache")
return False
def get_timestamp_from_apache(s: str) -> int:
real_time = s.split(" ")[0]
return int(real_time)

@ -39,6 +39,6 @@ class Artix(Project):
if outdated_since is not None:
data[project]['out_of_sync_since'] = int(outdated_since.timestamp())
return (datetime.now() - outdated_since).total_seconds() < data[project]['out_of_sync_interval']
return current_time - data[project]['out_of_sync_since'] < data[project]['out_of_sync_interval']
return True

@ -9,12 +9,14 @@ from shared import CSC_MIRROR
class CPAN(Project):
"""CPAN class"""
"""CPAN class"""
@staticmethod
def check(data, project, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[project]["out_of_sync_interval"]
return False
@staticmethod
def check(data, project, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/" and mirror["last_status"] == "ok":
# This is an improvised method: report we're good if CPAN think we are good
# Change this to a more precise method if you find a better way to do it
return True
return False

@ -8,7 +8,6 @@ from shared import CSC_MIRROR
import requests
import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
import re
import pandas as pd
@ -19,8 +18,9 @@ class cran(Project):
page = requests.get(data[project]["upstream"]).text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
duration = pd.to_timedelta(m.group(0))
data[project]["out_of_sync_since"] = current_time - duration.total_seconds()
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')

@ -7,8 +7,7 @@ from project import Project
from shared import CSC_MIRROR
import requests
import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
from datetime import datetime, timedelta
import re
import pandas as pd
@ -19,8 +18,10 @@ class ctan(Project):
page = requests.get(data[project]["upstream"]).text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
duration = pd.to_timedelta(m.group(0))
data[project]["out_of_sync_since"] = datetime.now() - duration
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')

@ -3,7 +3,37 @@ Contains Debian class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class Debian(Project):
"""Debian class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
CSC_date = datetime.strptime(CSC.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
CSC_utc_time = time.mktime(CSC_date.timetuple())
upstream_date = datetime.strptime(upstream.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
upstream_utc_time = time.mktime(upstream_date.timetuple())
try:
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -3,7 +3,40 @@ Contains DebianCD class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class DebianCD(Project):
"""DebianCD class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
# Date Format Example: Sun 27 Mar 00:20:12 UTC 2022
date_format = "%a %d %b %H:%M:%S UTC %Y\n"
CSC_date = datetime.strptime(CSC, date_format)
CSC_utc_time = time.mktime(CSC_date.timetuple())
upstream_date = datetime.strptime(upstream, date_format)
upstream_utc_time = time.mktime(upstream_date.timetuple())
try:
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -3,7 +3,40 @@ Contains Fedora class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class Fedora(Project):
"""Fedora class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
# Date example: Fedora-Rawhide-20220725.n.1
date_format = "%Y%m%d"
CSC_date = datetime.strptime(CSC[15:23], "%Y%m%d")
CSC_utc_time = time.mktime(CSC_date.timetuple())
upstream_date = datetime.strptime(upstream[15:23], "%Y%m%d")
upstream_utc_time = time.mktime(upstream_date.timetuple())
try:
return upstream_utc_time - CSC_utc_time < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -3,7 +3,35 @@ Contains FreeBSD class
"""
from project import Project
from shared import CSC_MIRROR
import requests
class FreeBSD(Project):
"""FreeBSD class"""
"""FreeBSD class"""
# FreeBSD's time file has two segments, so we need a special function
# Example TIMESTAMP file: 1648308600 2022-03-26T15:30:00 UTC
def check(data, project, current_time):
"""Check if project packages are up-to-date"""
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
try:
return get_timestamp_for_freebsd(upstream) - get_timestamp_for_freebsd(CSC) < data[project]["out_of_sync_interval"]
except ValueError:
print("failed to parse apache")
return False
def get_timestamp_for_freebsd(s: str) -> int:
real_time = s.split(" ")[0]
return int(real_time)

@ -3,7 +3,35 @@ Contains GentooDistfiles class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class GentooDistfiles(Project):
"""GentooDistfiles class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
CSC_utc_time = CSC[0:11]
upstream_utc_time = upstream[0:11]
try:
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -9,7 +9,6 @@ from project import Project
import requests
import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
import re
import pandas as pd