Compare commits

...

19 Commits

Author SHA1 Message Date
Raymond Li 90bde5a754 Merge pull request 'Fixed slackware checker' (#12) from jtoft-rt-4117-slackware into master 2 months ago
Justin Toft 55f3e7bcbe
Fixed slackware checker 2 months ago
Justin Toft 860d5c8e44 Fixed 6 broken mirror checkers (#11) 2 months ago
Raymond Li 65b06ac0a8 Merge pull request 'Fixed broken ubuntu and ubuntu_ports mirror checks' (#10) from jtoft-rt-4117 into master 4 months ago
Justin Toft 5afebc8030 Fixed broken ubuntu and ubuntu_ports mirror checks 4 months ago
Raymond Li 54d8a47944 Merge pull request 'Update time stamp parsing in cran, ctan, mxlinux' (#9) from time-parsing into master 6 months ago
Rio Liu 789f0cd662 update time stamp parsing in cran, ctan, mxlinux 6 months ago
Raymond Li e1ef917af0 Merge pull request 'fix CTAN && trust CPAN's report' (#8) from y266shen/mirror-checker:feature-retry into master 6 months ago
Yiao Shen 0571f7353a
projects/ctan: make Python happy 6 months ago
Yiao Shen 612ec9d04b
projects/cpan: trust CPAN's mirror status 6 months ago
Raymond Li 73bc2b5ade Merge pull request 'various mirror fixes' (#7) from y266shen/mirror-checker:feature-retry into master 6 months ago
Yiao Shen 9f345c7b0c
main: display how long a mirror has been out-of-sync 6 months ago
Yiao Shen 660b566715
projects/freebsd: correctly parse FreeBSD's timestamp file 6 months ago
Yiao Shen f408922a96
projects/apache: correctly parse Apache's timestamp file 6 months ago
Raymond Li 4a3cedfb07 Merge pull request 'implement retry' (#6) from y266shen/mirror-checker:feature-retry into master 6 months ago
Yiao Shen 76f7863a85
main: retry 3 times before reporting errors 6 months ago
Raymond Li 250f358f4e Merge pull request 'parallize mirror checking and remove special case matching' (#5) from multithread into master 8 months ago
Rio Liu 45af282f83 parallize mirror checking and remove special case matching 8 months ago
Raymond Li 3a793c5f01 Merge pull request 'fix GentooDistfiles, opensuse, tdf checking' (#4) from d9hui/mirror-checker:master into master 8 months ago
  1. 847
      data.json
  2. 101
      main.py
  3. 32
      projects/apache.py
  4. 2
      projects/artix.py
  5. 18
      projects/cpan.py
  6. 6
      projects/cran.py
  7. 7
      projects/ctan.py
  8. 32
      projects/debian.py
  9. 35
      projects/debiancd.py
  10. 35
      projects/fedora.py
  11. 32
      projects/freebsd.py
  12. 30
      projects/gentoodistfiles.py
  13. 4
      projects/gentooportage.py
  14. 3
      projects/manjaro.py
  15. 6
      projects/mxlinux.py
  16. 3
      projects/mxlinux_iso.py
  17. 27
      projects/nongnu.py
  18. 4
      projects/raspberrypi.py
  19. 2
      projects/slackware.py
  20. 3
      projects/ubuntu.py
  21. 8
      projects/ubuntu_releases.py
  22. 1
      shared.py

@ -1,424 +1,425 @@
{
"AlmaLinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME"
},
"Alpine": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated"
},
"Apache": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "apache/",
"upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt"
},
"Arch": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate"
},
"Artix": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "artixlinux/",
"upstream": "https://mirror1.artixlinux.org/repos/"
},
"CentOS": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME"
},
"Ceph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ceph/",
"upstream": "https://download.ceph.com/",
"file": "timestamp"
},
"CPAN": {
"out_of_sync_interval": 172800
},
"cran": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://cran.r-project.org/mirmon_report.html",
"file": ""
},
"ctan": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://www.ctan.org/mirrors/mirmon",
"file": ""
},
"Cygwin": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "cygwin/",
"upstream": "https://cygwin.com/pub/cygwin/",
"file": "x86/sha512.sum"
},
"Debian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master"
},
"DebianCD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org"
},
"DebianMultimedia": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org"
},
"DebianPorts": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org",
"exclude": true
},
"DebianSecurity": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-security/project/trace/master"
},
"Eclipse": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "eclipse/",
"upstream": "http://download.eclipse.org/",
"file": "TIME"
},
"Fedora": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID"
},
"FreeBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP"
},
"GentooDistfiles": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gentoo-distfiles/",
"upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local"
},
"GentooPortage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/",
"upstream": "https://mirrorstats.gentoo.org/rsync/",
"upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest"
},
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt"
},
"Gutenberg": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs"
},
"IPFire": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800
},
"KDE": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR"
},
"KDEApplicationData": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/",
"file": "last-updated"
},
"Kernel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc"
},
"linuxmint": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint/",
"file": ""
},
"linuxmint_packages": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint-packages/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint-packages/",
"file": "dists/"
},
"macports": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "MacPorts/mpdistfiles/",
"upstream": "https://distfiles.macports.org/",
"file": "ports.tar.gz"
},
"manjaro": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.manjaro.org/",
"file": ""
},
"mxlinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://rsync-mxlinux.org/mirmon/packages.html",
"file": ""
},
"mxlinux_iso": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mxlinux-iso/",
"upstream": "http://rsync-mxlinux.org/mirmon/index.html",
"mirrors": [
"http://mirror.its.dal.ca/mxlinux-cd/",
"http://mirror.umd.edu/mxlinux-iso/"
],
"file": ""
},
"mySQL": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mysql/",
"upstream": "http://mirrors.sunsite.dk/mysql/",
"file": "last-updated.txt"
},
"netbsd": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "NetBSD/",
"upstream": "http://ftp.netbsd.org/pub/NetBSD/",
"file": ""
},
"nongnu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "nongnu/",
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
"file": "00_TIME.txt"
},
"OpenBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp",
"exclude": true
},
"opensuse": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "opensuse/update/",
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
"file": ""
},
"parabola": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "parabola/",
"upstream": "https://repo.parabola.nu/",
"file": "lastsync"
},
"pkgsrc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "pkgsrc/",
"upstream": "http://ftp.netbsd.org/pub/pkgsrc/",
"file": "MIRROR-TIMESTAMP",
"exclude": true
},
"puppy_linux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "puppylinux/",
"upstream": "https://distro.ibiblio.org/puppylinux/",
"file": ""
},
"qtproject": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "qtproject/",
"upstream": "https://download.qt.io/",
"file": "timestamp.txt",
"exclude": true
},
"racket": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "racket/racket-installers/",
"upstream": "https://mirror.racket-lang.org/installers/",
"file": ""
},
"raspberrypi": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "raspberrypi/debian/",
"upstream": "https://archive.raspberrypi.org/debian/",
"file": "dists/"
},
"raspbian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "raspbian/",
"upstream": "http://archive.raspbian.org/",
"file": "snapshotindex.txt"
},
"sage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "",
"file": "sage/src/index.html"
},
"saltstack": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "saltstack/",
"upstream": "https://repo.saltproject.io/",
"file": ""
},
"slackware": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "slackware/",
"upstream": "https://mirrors.slackware.com/slackware/",
"file": ""
},
"tdf": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "tdf/",
"upstream": "https://download.documentfoundation.org/",
"file": "TIMESTAMP"
},
"trisquel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "trisquel/",
"upstream": "http://rsync.trisquel.info/trisquel/dists/",
"mirrors": [
"https://mirror.fsf.org/trisquel-images/",
"http://mirrors.ocf.berkeley.edu/trisquel-images/"
],
"file": ""
},
"ubuntu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": ""
},
"ubuntu_ports": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports/",
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
"file": "dists/"
},
"ubuntu_ports_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports-releases/",
"upstream": "https://cdimage.ubuntu.com/releases/",
"file": ""
},
"ubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release",
"file": ""
},
"vlc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "vlc/",
"upstream": "http://download.videolan.org/pub/videolan/",
"file": "trace"
},
"x_org": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "x.org/individual/",
"upstream": "https://www.x.org/releases/individual/",
"file": ""
},
"xiph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xiph/releases/",
"upstream": "https://ftp.osuosl.org/pub/xiph/releases/",
"file": ""
},
"xubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xubuntu-releases/",
"upstream": "https://cdimage.ubuntu.com/xubuntu/releases/",
"file": ""
}
}
"AlmaLinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME"
},
"Alpine": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated"
},
"Apache": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "apache/",
"upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt"
},
"Arch": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate"
},
"Artix": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "artixlinux/",
"upstream": "https://mirror1.artixlinux.org/repos/"
},
"CentOS": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME"
},
"Ceph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ceph/",
"upstream": "https://download.ceph.com/",
"file": "timestamp"
},
"CPAN": {
"out_of_sync_interval": 172800,
"out_of_sync_since": null
},
"cran": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://cran.r-project.org/mirmon_report.html",
"file": ""
},
"ctan": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://www.ctan.org/mirrors/mirmon",
"file": ""
},
"Cygwin": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "cygwin/",
"upstream": "https://cygwin.com/pub/cygwin/",
"file": "x86/sha512.sum"
},
"Debian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master"
},
"DebianCD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org"
},
"DebianMultimedia": {
"out_of_sync_since": 1659116719,
"out_of_sync_interval": 86400,
"csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org"
},
"DebianPorts": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org",
"exclude": true
},
"DebianSecurity": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-security/project/trace/master"
},
"Eclipse": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "eclipse/",
"upstream": "http://download.eclipse.org/",
"file": "TIME"
},
"Fedora": {
"out_of_sync_since": null,
"out_of_sync_interval": 259200,
"csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID"
},
"FreeBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP"
},
"GentooDistfiles": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gentoo-distfiles/",
"upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local"
},
"GentooPortage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/",
"upstream": "https://mirrorstats.gentoo.org/rsync/",
"upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest"
},
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt"
},
"Gutenberg": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs"
},
"IPFire": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800
},
"KDE": {
"out_of_sync_since": 1659116720,
"out_of_sync_interval": 86400,
"csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR"
},
"KDEApplicationData": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/",
"file": "last-updated"
},
"Kernel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc"
},
"linuxmint": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint/",
"file": ""
},
"linuxmint_packages": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "linuxmint-packages/",
"upstream": "https://mirrors.edge.kernel.org/linuxmint-packages/",
"file": "dists/"
},
"macports": {
"out_of_sync_since": 1642827723,
"out_of_sync_interval": 86400,
"csc": "MacPorts/mpdistfiles/",
"upstream": "https://distfiles.macports.org/",
"file": "ports.tar.gz"
},
"manjaro": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.manjaro.org/",
"file": ""
},
"mxlinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://rsync-mxlinux.org/mirmon/packages.html",
"file": ""
},
"mxlinux_iso": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mxlinux-iso/",
"upstream": "http://rsync-mxlinux.org/mirmon/index.html",
"mirrors": [
"http://mirror.its.dal.ca/mxlinux-cd/",
"http://mirror.umd.edu/mxlinux-iso/"
],
"file": ""
},
"mySQL": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "mysql/",
"upstream": "http://mirrors.sunsite.dk/mysql/",
"file": "last-updated.txt"
},
"netbsd": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "NetBSD/",
"upstream": "http://ftp.netbsd.org/pub/NetBSD/",
"file": ""
},
"nongnu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "nongnu/",
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
"file": "00_TIME.txt"
},
"OpenBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp",
"exclude": true
},
"opensuse": {
"out_of_sync_since": 1648699331,
"out_of_sync_interval": 86400,
"csc": "opensuse/update/",
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
"file": ""
},
"parabola": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "parabola/",
"upstream": "https://repo.parabola.nu/",
"file": "lastsync"
},
"pkgsrc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "pkgsrc/",
"upstream": "http://ftp.netbsd.org/pub/pkgsrc/",
"file": "MIRROR-TIMESTAMP",
"exclude": true
},
"puppy_linux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "puppylinux/",
"upstream": "https://distro.ibiblio.org/puppylinux/",
"file": ""
},
"qtproject": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "qtproject/",
"upstream": "https://download.qt.io/",
"file": "timestamp.txt",
"exclude": true
},
"racket": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "racket/racket-installers/",
"upstream": "https://mirror.racket-lang.org/installers/",
"file": ""
},
"raspberrypi": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "raspberrypi/debian/",
"upstream": "https://archive.raspberrypi.org/debian/",
"file": "dists/"
},
"raspbian": {
"out_of_sync_since": 1659116721,
"out_of_sync_interval": 86400,
"csc": "raspbian/",
"upstream": "http://archive.raspbian.org/",
"file": "snapshotindex.txt"
},
"sage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "",
"file": "sage/src/index.html"
},
"saltstack": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "saltstack/",
"upstream": "https://repo.saltproject.io/",
"file": ""
},
"slackware": {
"out_of_sync_since": 1642827723,
"out_of_sync_interval": 86400,
"csc": "slackware/",
"upstream": "https://mirrors.slackware.com/slackware/",
"file": ""
},
"tdf": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "tdf/",
"upstream": "https://download.documentfoundation.org/",
"file": "TIMESTAMP"
},
"trisquel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "trisquel/",
"upstream": "http://rsync.trisquel.info/trisquel/dists/",
"mirrors": [
"https://mirror.fsf.org/trisquel-images/",
"http://mirrors.ocf.berkeley.edu/trisquel-images/"
],
"file": ""
},
"ubuntu": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": ""
},
"ubuntu_ports": {
"out_of_sync_since": 1651550528,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports/",
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
"file": "dists/"
},
"ubuntu_ports_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ubuntu-ports-releases/",
"upstream": "https://cdimage.ubuntu.com/releases/",
"file": ""
},
"ubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": ""
},
"vlc": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "vlc/",
"upstream": "http://download.videolan.org/pub/videolan/",
"file": "trace"
},
"x_org": {
"out_of_sync_since": 1657512131,
"out_of_sync_interval": 86400,
"csc": "x.org/individual/",
"upstream": "https://www.x.org/releases/individual/",
"file": ""
},
"xiph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xiph/releases/",
"upstream": "https://ftp.osuosl.org/pub/xiph/releases/",
"file": ""
},
"xubuntu_releases": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "xubuntu-releases/",
"upstream": "https://cdimage.ubuntu.com/xubuntu/releases/",
"file": ""
}
}

@ -7,59 +7,88 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
import time
import sys
import requests
from multiprocessing import Pool, Manager
from typing import Optional
from time import sleep, localtime, strftime
from projects import *
import json
NUM_THREAD = 16
MAX_RETRY = 3
RETRY_TIMEOUT = 30 # In seconds
def safe_print(*args, **kwargs):
# When run with 'chronic' and 'timeout', stdout gets suppressed
# due to buffering. Make sure to always flush the output.
print(*args, **kwargs, flush=True)
# Return None if no error occurs and a string for error message otherwise
def check_project(args) -> Optional[str]:
current_time = int(time.time())
project, data = args
try:
project_class = getattr(sys.modules[__name__], project)
if __name__ == "__main__":
# Skip projects we no longer mirror
if data[project].get('exclude', False):
return None
checker_result = project_class.check(data, project, current_time)
if checker_result:
data[project]["out_of_sync_since"] = None
return None
elif (data[project]["out_of_sync_since"] is not None
and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
now_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
duration = current_time - data[project]["out_of_sync_since"]
return f"{project} out-of-sync at {now_str} for {duration}s"
else:
data[project]["out_of_sync_since"] = current_time
return None
except requests.exceptions.RequestException as err:
return f"{project}\n{err}"
def check_project_with_retry(args) -> bool:
project, _ = args
errs = []
for _ in range(MAX_RETRY):
res = check_project(args)
if res == None:
safe_print(f"Success: {project} up-to-date")
return True
else:
errs.append(res)
# Do nothing, try again later
sleep(RETRY_TIMEOUT)
# Max try reached, print errors
safe_print(f"Error: {project}")
for reason in errs:
safe_print(f" {reason}")
exit_code = 0
return False
def main():
data_file = 'data.json'
if len(sys.argv) > 1:
data_file = sys.argv[1]
manager = Manager()
data = json.load(open(data_file))
sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
with Pool(NUM_THREAD) as pool:
all_pass = all(pool.imap(check_project_with_retry, ((k, sync_data) for k in data.keys())))
current_time = int(time.time())
for project in data:
try:
project_class = getattr(sys.modules[__name__], project)
# Skip projects we no longer mirror
if data[project].get('exclude', False):
continue
checker_result = project_class.check(data, project, current_time)
if project in ["CPAN", "ubuntu_releases", "manjaro", "mxlinux", "cran", "ctan", "gentooportage", "Artix"]:
if checker_result:
safe_print(f"Success: {project} up-to-date")
else:
safe_print(f"Failure: {project} out-of-sync")
# Exit with non-zero status if any of the projects are not up-to-date
exit_code = 1
continue
if checker_result:
data[project]["out_of_sync_since"] = None
elif data[project]["out_of_sync_since"] is None:
data[project]["out_of_sync_since"] = current_time
elif current_time - data[project]["out_of_sync_since"] \
> data[project]["out_of_sync_interval"]:
safe_print(f"Failure: {project} out-of-sync")
# Exit with non-zero status if any of the projects are not up-to-date
exit_code = 1
continue
safe_print(f"Success: {project} up-to-date")
except requests.exceptions.RequestException as err:
safe_print(f"Error: {project}\n{err}")
with open(data_file, "w", encoding="utf-8") as file:
json.dump(data, file, indent='\t')
json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')
sys.exit(exit_code)
sys.exit(0 if all_pass else 1)
if __name__ == "__main__":
main()

@ -3,7 +3,35 @@ Contains Apache class
"""
from project import Project
from shared import CSC_MIRROR
import requests
class Apache(Project):
"""Apache class"""
"""Apache class"""
# Apache's time file has two segments, so we need a special function
# Example: 1648323001 rsync-he-fi
def check(data, project, current_time):
"""Check if project packages are up-to-date"""
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
try:
return get_timestamp_from_apache(upstream) - get_timestamp_from_apache(CSC) < data[project]["out_of_sync_interval"]
except ValueError:
print("failed to parse apache")
return False
def get_timestamp_from_apache(s: str) -> int:
real_time = s.split(" ")[0]
return int(real_time)

@ -39,6 +39,6 @@ class Artix(Project):
if outdated_since is not None:
data[project]['out_of_sync_since'] = int(outdated_since.timestamp())
return (datetime.now() - outdated_since).total_seconds() < data[project]['out_of_sync_interval']
return current_time - data[project]['out_of_sync_since'] < data[project]['out_of_sync_interval']
return True

@ -9,12 +9,14 @@ from shared import CSC_MIRROR
class CPAN(Project):
"""CPAN class"""
"""CPAN class"""
@staticmethod
def check(data, project, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[project]["out_of_sync_interval"]
return False
@staticmethod
def check(data, project, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/" and mirror["last_status"] == "ok":
# This is an improvised method: report we're good if CPAN think we are good
# Change this to a more precise method if you find a better way to do it
return True
return False

@ -8,7 +8,6 @@ from shared import CSC_MIRROR
import requests
import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
import re
import pandas as pd
@ -19,8 +18,9 @@ class cran(Project):
page = requests.get(data[project]["upstream"]).text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
duration = pd.to_timedelta(m.group(0))
data[project]["out_of_sync_since"] = current_time - duration.total_seconds()
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')

@ -7,8 +7,7 @@ from project import Project
from shared import CSC_MIRROR
import requests
import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
from datetime import datetime, timedelta
import re
import pandas as pd
@ -19,8 +18,10 @@ class ctan(Project):
page = requests.get(data[project]["upstream"]).text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
duration = pd.to_timedelta(m.group(0))
data[project]["out_of_sync_since"] = datetime.now() - duration
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')

@ -3,7 +3,37 @@ Contains Debian class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class Debian(Project):
"""Debian class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
CSC_date = datetime.strptime(CSC.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
CSC_utc_time = time.mktime(CSC_date.timetuple())
upstream_date = datetime.strptime(upstream.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
upstream_utc_time = time.mktime(upstream_date.timetuple())
try:
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -3,7 +3,40 @@ Contains DebianCD class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class DebianCD(Project):
"""DebianCD class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
# Date Format Example: Sun 27 Mar 00:20:12 UTC 2022
date_format = "%a %d %b %H:%M:%S UTC %Y\n"
CSC_date = datetime.strptime(CSC, date_format)
CSC_utc_time = time.mktime(CSC_date.timetuple())
upstream_date = datetime.strptime(upstream, date_format)
upstream_utc_time = time.mktime(upstream_date.timetuple())
try:
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -3,7 +3,40 @@ Contains Fedora class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class Fedora(Project):
"""Fedora class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
# Date example: Fedora-Rawhide-20220725.n.1
date_format = "%Y%m%d"
CSC_date = datetime.strptime(CSC[15:23], "%Y%m%d")
CSC_utc_time = time.mktime(CSC_date.timetuple())
upstream_date = datetime.strptime(upstream[15:23], "%Y%m%d")
upstream_utc_time = time.mktime(upstream_date.timetuple())
try:
return upstream_utc_time - CSC_utc_time < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -3,7 +3,35 @@ Contains FreeBSD class
"""
from project import Project
from shared import CSC_MIRROR
import requests
class FreeBSD(Project):
"""FreeBSD class"""
"""FreeBSD class"""
# FreeBSD's time file has two segments, so we need a special function
# Example TIMESTAMP file: 1648308600 2022-03-26T15:30:00 UTC
def check(data, project, current_time):
"""Check if project packages are up-to-date"""
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
try:
return get_timestamp_for_freebsd(upstream) - get_timestamp_for_freebsd(CSC) < data[project]["out_of_sync_interval"]
except ValueError:
print("failed to parse apache")
return False
def get_timestamp_for_freebsd(s: str) -> int:
real_time = s.split(" ")[0]
return int(real_time)

@ -3,7 +3,35 @@ Contains GentooDistfiles class
"""
from project import Project
from shared import CSC_MIRROR
import requests
from datetime import datetime
import time
class GentooDistfiles(Project):
"""GentooDistfiles class"""
@staticmethod
def check(data, project, current_time):
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
req = requests.get(csc_url)
req.raise_for_status()
CSC = req.text
req = requests.get(upstream_url)
req.raise_for_status()
upstream = req.text
if upstream == CSC:
return True
CSC_utc_time = CSC[0:11]
upstream_utc_time = upstream[0:11]
try:
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
except ValueError:
return False

@ -9,7 +9,6 @@ from project import Project
import requests
import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
import re
import pandas as pd