forked from public/mirror-checker
fixed conflict
This commit is contained in:
commit
82cf54f456
16
README.md
16
README.md
|
@ -4,7 +4,7 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
|
|||
|
||||
## How To Run
|
||||
|
||||
Run `main.py`.
|
||||
A configuration file may be provided through standard input. Without a configuration file, execute `python main.py`. By default, all of the available distributions will be checked. With a configuration file, execute `python main.py < name_of_config_file.in`, for example, `python main.py < example.in`. In this case, only the distributions listed in the configuration file will be checked.
|
||||
|
||||
## Resources
|
||||
|
||||
|
@ -21,3 +21,17 @@ to find repos of the distros to check, just search "distroName mirrors"
|
|||
ubuntu releases: https://releases.ubuntu.com/?_ga=2.251394307.425973732.1629764407-1388600952.1629764407
|
||||
ubuntu mirrors: http://mirrors.ubuntu.com/mirrors.txt
|
||||
raspbian mirror: https://mirror.ox.ac.uk/sites/archive.raspbian.org/archive/
|
||||
|
||||
mxlinux: https://sourceforge.net/projects/mx-linux/ (scrap the last day?)
|
||||
|
||||
linuxmint-packages pool: http://rsync-packages.linuxmint.com/pool/
|
||||
scientific: https://scientificlinux.org/downloads/sl-mirrors/ (CSC not listed)
|
||||
slackware: https://mirrors.slackware.com/mirrorlist/ https://mirrors.slackware.com/slackware/
|
||||
tdf: https://download.documentfoundation.org/
|
||||
trisquel: https://trisquel.info/mirmon/index.html
|
||||
ubuntu: https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive
|
||||
ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/
|
||||
ubuntu-ports-releases: https://cdimage.ubuntu.com/releases/
|
||||
ubuntu-releases: https://releases.ubuntu.com/
|
||||
vlc: http://download.videolan.org/pub/videolan/
|
||||
x.org: https://www.x.org/releases/
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains AlmaLinux class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class AlmaLinux(Distro):
|
||||
"""AlmaLinux class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains Alpine class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class Alpine(Distro):
|
||||
"""Alpine class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains Apache class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class Apache(Distro):
|
||||
"""Apache class"""
|
20
arch.py
20
arch.py
|
@ -2,27 +2,7 @@
|
|||
Contains Arch class
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class Arch(Distro):
|
||||
"""Arch class"""
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of Arch"""
|
||||
return "Arch"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if Arch packages are up-to-date"""
|
||||
arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json()
|
||||
last_check_str = arch_json["last_check"]
|
||||
last_sync_str = [url for url in arch_json["urls"] if url["url"] == \
|
||||
f"{CSC_MIRROR}archlinux/"][0]["last_sync"]
|
||||
last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ")
|
||||
# According to https://archlinux.org/mirrors/status/:
|
||||
# Due to the timing of mirror checks, any value under one hour should be viewed as ideal
|
||||
return last_check_dt < last_sync_dt + timedelta(hours = 1)
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains CentOS class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class CentOS(Distro):
|
||||
"""CentOS class"""
|
13
ceph.py
13
ceph.py
|
@ -3,19 +3,6 @@ Contains Ceph class
|
|||
"""
|
||||
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR, get_sec
|
||||
|
||||
class Ceph(Distro):
|
||||
"""Ceph class"""
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of Ceph"""
|
||||
return "Ceph"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if Ceph packages are up-to-date"""
|
||||
official_sec = get_sec("https://download.ceph.com/timestamp")
|
||||
csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp")
|
||||
# Out-of-sync by 1 day maximum
|
||||
return official_sec < csc_sec + 86400
|
||||
|
|
|
@ -0,0 +1,17 @@
|
|||
"""
|
||||
Contains CPAN class
|
||||
"""
|
||||
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class CPAN(Distro):
|
||||
"""CPAN class"""
|
||||
@staticmethod
|
||||
def check(data, distro, current_time):
|
||||
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
|
||||
for mirror in res_json:
|
||||
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
|
||||
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
|
||||
return False
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains Cygwin class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class Cygwin(Distro):
|
||||
"""Cygwin class"""
|
|
@ -0,0 +1,181 @@
|
|||
{
|
||||
"AlmaLinux": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://repo.almalinux.org/",
|
||||
"file": "almalinux/TIME"
|
||||
},
|
||||
"Alpine": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://uk.alpinelinux.org/",
|
||||
"file": "alpine/last-updated"
|
||||
},
|
||||
"Apache": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "apache/",
|
||||
"upstream": "https://downloads.apache.org/",
|
||||
"file": "zzz/time.txt"
|
||||
},
|
||||
"Arch": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "archlinux/",
|
||||
"upstream": "http://arch.mirror.constant.com/",
|
||||
"file": "lastupdate"
|
||||
},
|
||||
"CentOS": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://mirrors.edge.kernel.org/",
|
||||
"file": "centos/TIME"
|
||||
},
|
||||
"Ceph": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "ceph/",
|
||||
"upstream": "https://download.ceph.com/",
|
||||
"file": "timestamp"
|
||||
},
|
||||
"CPAN": {
|
||||
"out_of_sync_interval": 172800
|
||||
},
|
||||
"Cygwin": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "cygwin/",
|
||||
"upstream": "https://cygwin.mirror.globo.tech/",
|
||||
"file": "x86/sha512.sum"
|
||||
},
|
||||
"Debian": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://ftp-master.debian.org/",
|
||||
"file": "debian/project/trace/master"
|
||||
},
|
||||
"DebianCD": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "http://debian.mirror.estruxture.net/",
|
||||
"file": "debian-cd/project/trace/cdimage.debian.org"
|
||||
},
|
||||
"DebianMultimedia": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "debian-multimedia/",
|
||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
||||
"file": "project/trace/deb-multimedia.org"
|
||||
},
|
||||
"DebianPorts": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://deb.debian.org/",
|
||||
"file": "debian-ports/project/trace/porta.debian.org"
|
||||
},
|
||||
"DebianSecurity": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "http://debian.mirror.iweb.ca/",
|
||||
"file": "debian-security/project/trace/master"
|
||||
},
|
||||
"Eclipse": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 172800,
|
||||
"csc": "eclipse/",
|
||||
"upstream": "http://download.eclipse.org/",
|
||||
"file": "TIME"
|
||||
},
|
||||
"Fedora": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "fedora/",
|
||||
"upstream": "http://fedora.mirror.iweb.com/",
|
||||
"file": "linux/development/rawhide/COMPOSE_ID"
|
||||
},
|
||||
"FreeBSD": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "http://ftp4.freebsd.org/pub/",
|
||||
"file": "FreeBSD/TIMESTAMP"
|
||||
},
|
||||
"GentooDistfiles": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "gentoo-distfiles",
|
||||
"upstream": "http://gentoo.mirrors.tera-byte.com/",
|
||||
"file": "distfiles/timestamp.dev-local"
|
||||
},
|
||||
"GentooPortage": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "rsync://rsync4.ca.gentoo.org/",
|
||||
"upstream1": "rsync://rsync1.de.gentoo.org/",
|
||||
"upstream2": "rsync://rsync8.de.gentoo.org/",
|
||||
"file": "gentoo-portage/Manifest"
|
||||
},
|
||||
"GNOME": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "gnome/",
|
||||
"upstream1": "https://download.gnome.org/",
|
||||
"upstream2": "https://mirrors.dotsrc.org/gnome/",
|
||||
"upstream3": "https://muug.ca/mirror/gnome/",
|
||||
"file1": "core/",
|
||||
"file2": "cache.json"
|
||||
},
|
||||
"GNU": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://mirrors.kernel.org/",
|
||||
"file": "gnu/mirror-updated-timestamp.txt"
|
||||
},
|
||||
"Gutenberg": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 172800,
|
||||
"csc": "gutenberg/",
|
||||
"upstream": "https://gutenberg.pglaf.org/",
|
||||
"file": "gutenberg.dcs"
|
||||
},
|
||||
"IPFire": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 172800
|
||||
},
|
||||
"KDE": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "kde/",
|
||||
"upstream": "https://kde.c3sl.ufpr.br/",
|
||||
"file": "ls-lR"
|
||||
},
|
||||
"KDEApplicationData": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "kde-applicationdata/",
|
||||
"upstream": "https://cdn.files.kde.org/",
|
||||
"file": "last-updated"
|
||||
},
|
||||
"Kernel": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "kernel.org/",
|
||||
"upstream": "https://mirrors.edge.kernel.org/pub/",
|
||||
"file": "linux/kernel/next/sha256sums.asc"
|
||||
},
|
||||
"OpenBSD": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://ftp.openbsd.org/pub/",
|
||||
"file": "OpenBSD/timestamp"
|
||||
}
|
||||
}
|
23
debian.py
23
debian.py
|
@ -2,30 +2,7 @@
|
|||
Contains Debian class
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class Debian(Distro):
|
||||
"""Debian class"""
|
||||
@staticmethod
|
||||
def __get_dt(trace_file_url):
|
||||
"""Get Debian datetime object from trace file"""
|
||||
response_text = requests.get(trace_file_url).text
|
||||
time_str = response_text.split('\n')[0]
|
||||
return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y")
|
||||
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of Debian"""
|
||||
return "Debian"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if Debian packages are up-to-date"""
|
||||
official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master")
|
||||
csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master")
|
||||
# Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html
|
||||
# green and not yellow
|
||||
return official_dt < csc_dt + timedelta(hours = 7)
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains DebianCD class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class DebianCD(Distro):
|
||||
"""DebianCD class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains DebianMultimedia class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class DebianMultimedia(Distro):
|
||||
"""DebianMultimedia class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains DebianPorts class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class DebianPorts(Distro):
|
||||
"""DebianPorts class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains DebianSecurity class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class DebianSecurity(Distro):
|
||||
"""DebianSecurity class"""
|
23
distro.py
23
distro.py
|
@ -2,24 +2,15 @@
|
|||
Contains abstract class for a distro
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from abc import ABC
|
||||
import requests
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class Distro(ABC):
|
||||
"""Abstract class for a distro"""
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def name():
|
||||
"""Get name of distro"""
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def check():
|
||||
def check(data, distro):
|
||||
"""Check if distro packages are up-to-date"""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def print_output(cls, is_successful):
|
||||
"""Print final output of distro"""
|
||||
output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync"
|
||||
print(output.format(cls.name()))
|
||||
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
|
||||
upstream_url = data[distro]["upstream"] + data[distro]["file"]
|
||||
return requests.get(csc_url).text == requests.get(upstream_url).text
|
||||
|
|
13
eclipse.py
13
eclipse.py
|
@ -3,19 +3,6 @@ Contains Eclipse class
|
|||
"""
|
||||
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR, get_sec
|
||||
|
||||
class Eclipse(Distro):
|
||||
"""Eclipse class"""
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of Eclipse"""
|
||||
return "Eclipse"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if Eclipse packages are up-to-date"""
|
||||
official_sec = get_sec("http://download.eclipse.org/TIME")
|
||||
csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME")
|
||||
# Out-of-sync by 2 days maximum
|
||||
return official_sec < csc_sec + 172800
|
||||
|
|
|
@ -0,0 +1,26 @@
|
|||
AlmaLinux
|
||||
Alpine
|
||||
Apache
|
||||
Arch
|
||||
CentOS
|
||||
Ceph
|
||||
CPAN
|
||||
Cygwin
|
||||
Debian
|
||||
DebianCD
|
||||
DebianMultimedia
|
||||
DebianPorts
|
||||
DebianSecurity
|
||||
Eclipse
|
||||
Fedora
|
||||
FreeBSD
|
||||
GentooDistfiles
|
||||
GentooPortage
|
||||
GNOME
|
||||
GNU
|
||||
Gutenberg
|
||||
IPFire
|
||||
KDE
|
||||
KDEApplicationData
|
||||
Kernel
|
||||
OpenBSD
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains Fedora class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class Fedora(Distro):
|
||||
"""Fedora class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains FreeBSD class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class FreeBSD(Distro):
|
||||
"""FreeBSD class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains GentooDistfiles class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class GentooDistfiles(Distro):
|
||||
"""GentooDistfiles class"""
|
|
@ -0,0 +1,29 @@
|
|||
"""
|
||||
Contains GentooPortage class
|
||||
"""
|
||||
|
||||
import os
|
||||
from distro import Distro
|
||||
|
||||
class GentooPortage(Distro):
|
||||
"""GentooPortage class"""
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
rsync_command = "rsync -q {}{} {}"
|
||||
os.system(rsync_command.format(data[distro]["csc"],
|
||||
data[distro]["file"],
|
||||
"csc_manifest"))
|
||||
os.system(rsync_command.format(data[distro]["upstream1"],
|
||||
data[distro]["file"],
|
||||
"upstream_manifest1"))
|
||||
os.system(rsync_command.format(data[distro]["upstream2"],
|
||||
data[distro]["file"],
|
||||
"upstream_manifest2"))
|
||||
stream1 = os.popen("diff csc_manifest upstream_manifest1")
|
||||
output1 = stream1.read()
|
||||
stream2 = os.popen("diff csc_manifest upstream_manifest2")
|
||||
output2 = stream2.read()
|
||||
os.system("rm csc_manifest")
|
||||
os.system("rm upstream_manifest1")
|
||||
os.system("rm upstream_manifest2")
|
||||
return 0 in [len(output1), len(output2)]
|
|
@ -0,0 +1,40 @@
|
|||
"""
|
||||
Contains GNOME class
|
||||
"""
|
||||
|
||||
import re
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class GNOME(Distro):
|
||||
"""GNOME class"""
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
file = data[distro]["file1"]
|
||||
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
|
||||
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
|
||||
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
|
||||
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
|
||||
if csc_latest != upstream_latest:
|
||||
return False
|
||||
file += csc_latest + "/"
|
||||
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
|
||||
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
|
||||
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
|
||||
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
|
||||
if csc_latest != upstream_latest:
|
||||
return False
|
||||
file += csc_latest + "/"
|
||||
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
try:
|
||||
ret = csc_text == requests.get(data[distro]["upstream2"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
except requests.exceptions.RequestException:
|
||||
ret = False
|
||||
try:
|
||||
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
13
gnu.py
13
gnu.py
|
@ -3,19 +3,6 @@ Contains GNU class
|
|||
"""
|
||||
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR, get_sec
|
||||
|
||||
class GNU(Distro):
|
||||
"""GNU class"""
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of GNU"""
|
||||
return "GNU"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if GNU packages are up-to-date"""
|
||||
official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt")
|
||||
csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt")
|
||||
# Out-of-sync by 1 day maximum
|
||||
return official_sec < csc_sec + 86400
|
||||
|
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains Gutenberg class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class Gutenberg(Distro):
|
||||
"""Gutenberg class"""
|
|
@ -0,0 +1,14 @@
|
|||
"""
|
||||
Contains IPFire class
|
||||
"""
|
||||
|
||||
import requests
|
||||
from distro import Distro
|
||||
|
||||
class IPFire(Distro):
|
||||
"""IPFire class"""
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
|
||||
ipfire_text = requests.get(ipfire_url).text
|
||||
return ipfire_text.find("The mirror is up") != -1
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains KDE class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class KDE(Distro):
|
||||
"""KDE class"""
|
|
@ -0,0 +1,8 @@
|
|||
"""
|
||||
Contains KDEApplicationData class
|
||||
"""
|
||||
|
||||
from distro import Distro
|
||||
|
||||
class KDEApplicationData(Distro):
|
||||
"""KDEApplicationData class"""
|
22
kernel.py
22
kernel.py
|
@ -2,29 +2,7 @@
|
|||
Contains Kernel class
|
||||
"""
|
||||
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class Kernel(Distro):
|
||||
"""Kernel class"""
|
||||
@staticmethod
|
||||
def __get_line_count(checksum_file_url):
|
||||
"""Get Kernel line count of checksum file"""
|
||||
response_text = requests.get(checksum_file_url).text
|
||||
return len(response_text.split('\n'))
|
||||
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of Kernel"""
|
||||
return "Kernel"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if Kernel packages are up-to-date"""
|
||||
official_line_count = Kernel.__get_line_count(
|
||||
"https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc")
|
||||
csc_line_count = Kernel.__get_line_count(
|
||||
f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc")
|
||||
# A new update on a certain day adds 2 new lines
|
||||
return official_line_count <= csc_line_count + 2
|
||||
|
|
79
main.py
79
main.py
|
@ -4,14 +4,36 @@
|
|||
This mirror status checker determines whether CSC mirror is up-to-date with upstream
|
||||
"""
|
||||
|
||||
import time
|
||||
import sys
|
||||
import requests
|
||||
from almalinux import AlmaLinux
|
||||
from alpine import Alpine
|
||||
from apache import Apache
|
||||
from arch import Arch
|
||||
from centos import CentOS
|
||||
from ceph import Ceph
|
||||
from cpan import CPAN
|
||||
from cygwin import Cygwin
|
||||
from debian import Debian
|
||||
from debiancd import DebianCD
|
||||
from debianmultimedia import DebianMultimedia
|
||||
from debianports import DebianPorts
|
||||
from debiansecurity import DebianSecurity
|
||||
from eclipse import Eclipse
|
||||
from fedora import Fedora
|
||||
from freebsd import FreeBSD
|
||||
from gentoodistfiles import GentooDistfiles
|
||||
from gentooportage import GentooPortage
|
||||
from gnome import GNOME
|
||||
from gnu import GNU
|
||||
from gutenberg import Gutenberg
|
||||
from ipfire import IPFire
|
||||
from kde import KDE
|
||||
from kdeapplicationdata import KDEApplicationData
|
||||
from kernel import Kernel
|
||||
from openbsd import OpenBSD
|
||||
from shared import CSC_MIRROR
|
||||
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
|
||||
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
|
||||
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
|
||||
|
@ -47,26 +69,45 @@ def checker(directory_URL, file_name):
|
|||
return('No dates found')
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]:
|
||||
try:
|
||||
distro.print_output(distro.check())
|
||||
except requests.exceptions.RequestException as err:
|
||||
print(f"Error: {distro.name()}\n{err}")"""
|
||||
|
||||
# the implementation to check timestamps in distro mirror directory:
|
||||
distros = json.load(open('distros.json',))
|
||||
# print(distros)
|
||||
"""distros = json.load(open('distros.json',))
|
||||
print(distros)
|
||||
|
||||
for distro in distros:
|
||||
print(distro[0] + ":")
|
||||
print("CSC mirror: " + checker(distro[1], distro[3]))
|
||||
print("Official distro: " + checker(distro[2], distro[3]))
|
||||
# for eclipse, i couldn't find a directory from the official website, so i decided to temporarily cheat a little bit and peek at another person's mirror for explipse
|
||||
# i just realized that my method doesn't just work for scraping html directories but everything returned by a URL (websites, files storing a time)
|
||||
# i think i'll put checking arch, ceph, ecplipse on hold because it's more proper to check them using as.Date(17383, origin="1970-01-01") where the date is stored in
|
||||
# a file as a number, but this would require another function and me looping through different functions. i think i'll do this later
|
||||
# corresponding jsons: ["Eclipse", "https://mirror.csclub.uwaterloo.ca/eclipse/", "http://eclipse.mirror.rafal.ca/", "TIME"],
|
||||
# ["Ceph", "https://mirror.csclub.uwaterloo.ca/ceph/", "https://download.ceph.com/", "timestamp"],
|
||||
# i don't know what to do with damnsmalllinux, it seems like an abandoned project. useful link: http://distro.ibiblio.org/damnsmall/
|
||||
# the other function is probably probing mirror status sites like this: http://mirrors.cpan.org/
|
||||
# maybe i can make a column in json that specifies the method for checking the particular mirror and that identifier will find a specific function to check the update
|
||||
print("Official distro: " + checker(distro[2], distro[3]))"""
|
||||
|
||||
with open("data.json", "r", encoding = "utf-8") as file:
|
||||
data = json.load(file)
|
||||
if sys.stdin.isatty():
|
||||
distros = data
|
||||
else:
|
||||
distros = [distro.rstrip() for distro in sys.stdin.readlines()]
|
||||
current_time = int(time.time())
|
||||
for distro in distros:
|
||||
try:
|
||||
if distro not in data:
|
||||
print(f"Failure: {distro} does not exist")
|
||||
continue
|
||||
distro_class = getattr(sys.modules[__name__], distro)
|
||||
if distro == "CPAN":
|
||||
checker_result = distro_class.check(data, distro, current_time)
|
||||
if checker_result:
|
||||
print(f"Success: {distro} up-to-date")
|
||||
else:
|
||||
print(f"Failure: {distro} out-of-sync")
|
||||
continue
|
||||
checker_result = distro_class.check(data, distro)
|
||||
if checker_result:
|
||||
data[distro]["out_of_sync_since"] = None
|
||||
elif data[distro]["out_of_sync_since"] is None:
|
||||
data[distro]["out_of_sync_since"] = current_time
|
||||
elif current_time - data[distro]["out_of_sync_since"] \
|
||||
> data[distro]["out_of_sync_interval"]:
|
||||
print(f"Failure: {distro} out-of-sync")
|
||||
continue
|
||||
print(f"Success: {distro} up-to-date")
|
||||
except requests.exceptions.RequestException as err:
|
||||
print(f"Error: {distro}\n{err}")
|
||||
with open("data.json", "w", encoding = "utf-8") as file:
|
||||
json.dump(data, file, indent = 4)
|
||||
|
|
13
openbsd.py
13
openbsd.py
|
@ -3,19 +3,6 @@ Contains OpenBSD class
|
|||
"""
|
||||
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR, get_sec
|
||||
|
||||
class OpenBSD(Distro):
|
||||
"""OpenBSD class"""
|
||||
@staticmethod
|
||||
def name():
|
||||
"""Get name of OpenBSD"""
|
||||
return "OpenBSD"
|
||||
|
||||
@staticmethod
|
||||
def check():
|
||||
"""Check if OpenBSD packages are up-to-date"""
|
||||
official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp")
|
||||
csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp")
|
||||
# Out-of-sync by 1 day maximum
|
||||
return official_sec < csc_sec + 86400
|
||||
|
|
|
@ -1,10 +1,3 @@
|
|||
"""Contains shared constants and functions"""
|
||||
|
||||
import requests
|
||||
"""Contains shared constants"""
|
||||
|
||||
CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
|
||||
|
||||
def get_sec(timestamp_file_url):
|
||||
"""Get seconds since the Epoch from timestamp file"""
|
||||
sec_str = requests.get(timestamp_file_url).text
|
||||
return int(sec_str)
|
||||
|
|
Loading…
Reference in New Issue