diff --git a/almalinux.py b/almalinux.py new file mode 100644 index 0000000..201deef --- /dev/null +++ b/almalinux.py @@ -0,0 +1,8 @@ +""" +Contains AlmaLinux class +""" + +from distro import Distro + +class AlmaLinux(Distro): + """AlmaLinux class""" diff --git a/alpine.py b/alpine.py new file mode 100644 index 0000000..b47ccae --- /dev/null +++ b/alpine.py @@ -0,0 +1,8 @@ +""" +Contains Alpine class +""" + +from distro import Distro + +class Alpine(Distro): + """Alpine class""" diff --git a/apache.py b/apache.py new file mode 100644 index 0000000..0541525 --- /dev/null +++ b/apache.py @@ -0,0 +1,8 @@ +""" +Contains Apache class +""" + +from distro import Distro + +class Apache(Distro): + """Apache class""" diff --git a/arch.py b/arch.py index 2b81880..6409ef7 100644 --- a/arch.py +++ b/arch.py @@ -2,27 +2,7 @@ Contains Arch class """ -from datetime import datetime, timedelta -import requests from distro import Distro -from shared import CSC_MIRROR class Arch(Distro): """Arch class""" - @staticmethod - def name(): - """Get name of Arch""" - return "Arch" - - @staticmethod - def check(): - """Check if Arch packages are up-to-date""" - arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json() - last_check_str = arch_json["last_check"] - last_sync_str = [url for url in arch_json["urls"] if url["url"] == \ - f"{CSC_MIRROR}archlinux/"][0]["last_sync"] - last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ") - last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ") - # According to https://archlinux.org/mirrors/status/: - # Due to the timing of mirror checks, any value under one hour should be viewed as ideal - return last_check_dt < last_sync_dt + timedelta(hours = 1) diff --git a/centos.py b/centos.py new file mode 100644 index 0000000..0362130 --- /dev/null +++ b/centos.py @@ -0,0 +1,8 @@ +""" +Contains CentOS class +""" + +from distro import Distro + +class CentOS(Distro): + """CentOS class""" diff --git a/ceph.py b/ceph.py index fe8f28b..d39b81f 100644 --- a/ceph.py +++ b/ceph.py @@ -3,19 +3,6 @@ Contains Ceph class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class Ceph(Distro): """Ceph class""" - @staticmethod - def name(): - """Get name of Ceph""" - return "Ceph" - - @staticmethod - def check(): - """Check if Ceph packages are up-to-date""" - official_sec = get_sec("https://download.ceph.com/timestamp") - csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/cpan.py b/cpan.py new file mode 100644 index 0000000..d72aafc --- /dev/null +++ b/cpan.py @@ -0,0 +1,17 @@ +""" +Contains CPAN class +""" + +import requests +from distro import Distro +from shared import CSC_MIRROR + +class CPAN(Distro): + """CPAN class""" + @staticmethod + def check(data, distro, current_time): + res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() + for mirror in res_json: + if mirror["url"] == f"{CSC_MIRROR}CPAN/": + return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"] + return False diff --git a/cygwin.py b/cygwin.py new file mode 100644 index 0000000..0885224 --- /dev/null +++ b/cygwin.py @@ -0,0 +1,8 @@ +""" +Contains Cygwin class +""" + +from distro import Distro + +class Cygwin(Distro): + """Cygwin class""" diff --git a/data.json b/data.json index 0a932de..41e1c9d 100644 --- a/data.json +++ b/data.json @@ -125,9 +125,12 @@ "GNOME": { "out_of_sync_since": null, "out_of_sync_interval": 86400, - "upstream1": "https://mirrors.dotsrc.org/", - "upstream2": "https://muug.ca/mirror/", - "file": "gnome/core/41/41.beta/cache.json" + "csc": "gnome/", + "upstream1": "https://download.gnome.org/", + "upstream2": "https://mirrors.dotsrc.org/gnome/", + "upstream3": "https://muug.ca/mirror/gnome/", + "file1": "core/", + "file2": "cache.json" }, "GNU": { "out_of_sync_since": null, diff --git a/debian.py b/debian.py index 9238fa8..a720054 100644 --- a/debian.py +++ b/debian.py @@ -2,30 +2,7 @@ Contains Debian class """ -from datetime import datetime, timedelta -import requests from distro import Distro -from shared import CSC_MIRROR class Debian(Distro): """Debian class""" - @staticmethod - def __get_dt(trace_file_url): - """Get Debian datetime object from trace file""" - response_text = requests.get(trace_file_url).text - time_str = response_text.split('\n')[0] - return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y") - - @staticmethod - def name(): - """Get name of Debian""" - return "Debian" - - @staticmethod - def check(): - """Check if Debian packages are up-to-date""" - official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master") - csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master") - # Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html - # green and not yellow - return official_dt < csc_dt + timedelta(hours = 7) diff --git a/debiancd.py b/debiancd.py new file mode 100644 index 0000000..90e7f39 --- /dev/null +++ b/debiancd.py @@ -0,0 +1,8 @@ +""" +Contains DebianCD class +""" + +from distro import Distro + +class DebianCD(Distro): + """DebianCD class""" diff --git a/debianmultimedia.py b/debianmultimedia.py new file mode 100644 index 0000000..5e4fd29 --- /dev/null +++ b/debianmultimedia.py @@ -0,0 +1,8 @@ +""" +Contains DebianMultimedia class +""" + +from distro import Distro + +class DebianMultimedia(Distro): + """DebianMultimedia class""" diff --git a/debianports.py b/debianports.py new file mode 100644 index 0000000..a3f2952 --- /dev/null +++ b/debianports.py @@ -0,0 +1,8 @@ +""" +Contains DebianPorts class +""" + +from distro import Distro + +class DebianPorts(Distro): + """DebianPorts class""" diff --git a/debiansecurity.py b/debiansecurity.py new file mode 100644 index 0000000..9a50858 --- /dev/null +++ b/debiansecurity.py @@ -0,0 +1,8 @@ +""" +Contains DebianSecurity class +""" + +from distro import Distro + +class DebianSecurity(Distro): + """DebianSecurity class""" diff --git a/distro.py b/distro.py index 32b517a..7d363dc 100644 --- a/distro.py +++ b/distro.py @@ -2,24 +2,15 @@ Contains abstract class for a distro """ -from abc import ABC, abstractmethod +from abc import ABC +import requests +from shared import CSC_MIRROR class Distro(ABC): """Abstract class for a distro""" @staticmethod - @abstractmethod - def name(): - """Get name of distro""" - raise NotImplementedError - - @staticmethod - @abstractmethod - def check(): + def check(data, distro): """Check if distro packages are up-to-date""" - raise NotImplementedError - - @classmethod - def print_output(cls, is_successful): - """Print final output of distro""" - output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync" - print(output.format(cls.name())) + csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"] + upstream_url = data[distro]["upstream"] + data[distro]["file"] + return requests.get(csc_url).text == requests.get(upstream_url).text diff --git a/eclipse.py b/eclipse.py index e413e04..9945b88 100644 --- a/eclipse.py +++ b/eclipse.py @@ -3,19 +3,6 @@ Contains Eclipse class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class Eclipse(Distro): """Eclipse class""" - @staticmethod - def name(): - """Get name of Eclipse""" - return "Eclipse" - - @staticmethod - def check(): - """Check if Eclipse packages are up-to-date""" - official_sec = get_sec("http://download.eclipse.org/TIME") - csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME") - # Out-of-sync by 2 days maximum - return official_sec < csc_sec + 172800 diff --git a/fedora.py b/fedora.py new file mode 100644 index 0000000..60cc2e9 --- /dev/null +++ b/fedora.py @@ -0,0 +1,8 @@ +""" +Contains Fedora class +""" + +from distro import Distro + +class Fedora(Distro): + """Fedora class""" diff --git a/freebsd.py b/freebsd.py new file mode 100644 index 0000000..2c7eec6 --- /dev/null +++ b/freebsd.py @@ -0,0 +1,8 @@ +""" +Contains FreeBSD class +""" + +from distro import Distro + +class FreeBSD(Distro): + """FreeBSD class""" diff --git a/gentoodistfiles.py b/gentoodistfiles.py new file mode 100644 index 0000000..ae3a740 --- /dev/null +++ b/gentoodistfiles.py @@ -0,0 +1,8 @@ +""" +Contains GentooDistfiles class +""" + +from distro import Distro + +class GentooDistfiles(Distro): + """GentooDistfiles class""" diff --git a/gentooportage.py b/gentooportage.py new file mode 100644 index 0000000..29498aa --- /dev/null +++ b/gentooportage.py @@ -0,0 +1,29 @@ +""" +Contains GentooPortage class +""" + +import os +from distro import Distro + +class GentooPortage(Distro): + """GentooPortage class""" + @staticmethod + def check(data, distro): + rsync_command = "rsync -q {}{} {}" + os.system(rsync_command.format(data[distro]["csc"], + data[distro]["file"], + "csc_manifest")) + os.system(rsync_command.format(data[distro]["upstream1"], + data[distro]["file"], + "upstream_manifest1")) + os.system(rsync_command.format(data[distro]["upstream2"], + data[distro]["file"], + "upstream_manifest2")) + stream1 = os.popen("diff csc_manifest upstream_manifest1") + output1 = stream1.read() + stream2 = os.popen("diff csc_manifest upstream_manifest2") + output2 = stream2.read() + os.system("rm csc_manifest") + os.system("rm upstream_manifest1") + os.system("rm upstream_manifest2") + return 0 in [len(output1), len(output2)] diff --git a/gnome.py b/gnome.py new file mode 100644 index 0000000..8c9dfb6 --- /dev/null +++ b/gnome.py @@ -0,0 +1,40 @@ +""" +Contains GNOME class +""" + +import re +import requests +from distro import Distro +from shared import CSC_MIRROR + +class GNOME(Distro): + """GNOME class""" + @staticmethod + def check(data, distro): + file = data[distro]["file1"] + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file + + data[distro]["file2"]).text + try: + ret = csc_text == requests.get(data[distro]["upstream2"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + ret = False + try: + return ret or csc_text == requests.get(data[distro]["upstream3"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + return False diff --git a/gnu.py b/gnu.py index 2d23ade..8b7af9a 100644 --- a/gnu.py +++ b/gnu.py @@ -3,19 +3,6 @@ Contains GNU class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class GNU(Distro): """GNU class""" - @staticmethod - def name(): - """Get name of GNU""" - return "GNU" - - @staticmethod - def check(): - """Check if GNU packages are up-to-date""" - official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt") - csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/gutenberg.py b/gutenberg.py new file mode 100644 index 0000000..4ce58f9 --- /dev/null +++ b/gutenberg.py @@ -0,0 +1,8 @@ +""" +Contains Gutenberg class +""" + +from distro import Distro + +class Gutenberg(Distro): + """Gutenberg class""" diff --git a/ipfire.py b/ipfire.py new file mode 100644 index 0000000..c9b9911 --- /dev/null +++ b/ipfire.py @@ -0,0 +1,14 @@ +""" +Contains IPFire class +""" + +import requests +from distro import Distro + +class IPFire(Distro): + """IPFire class""" + @staticmethod + def check(data, distro): + ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca" + ipfire_text = requests.get(ipfire_url).text + return ipfire_text.find("The mirror is up") != -1 diff --git a/kde.py b/kde.py new file mode 100644 index 0000000..2a18445 --- /dev/null +++ b/kde.py @@ -0,0 +1,8 @@ +""" +Contains KDE class +""" + +from distro import Distro + +class KDE(Distro): + """KDE class""" diff --git a/kdeapplicationdata.py b/kdeapplicationdata.py new file mode 100644 index 0000000..6474864 --- /dev/null +++ b/kdeapplicationdata.py @@ -0,0 +1,8 @@ +""" +Contains KDEApplicationData class +""" + +from distro import Distro + +class KDEApplicationData(Distro): + """KDEApplicationData class""" diff --git a/kernel.py b/kernel.py index 30e6b83..e06dad5 100644 --- a/kernel.py +++ b/kernel.py @@ -2,29 +2,7 @@ Contains Kernel class """ -import requests from distro import Distro -from shared import CSC_MIRROR class Kernel(Distro): """Kernel class""" - @staticmethod - def __get_line_count(checksum_file_url): - """Get Kernel line count of checksum file""" - response_text = requests.get(checksum_file_url).text - return len(response_text.split('\n')) - - @staticmethod - def name(): - """Get name of Kernel""" - return "Kernel" - - @staticmethod - def check(): - """Check if Kernel packages are up-to-date""" - official_line_count = Kernel.__get_line_count( - "https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc") - csc_line_count = Kernel.__get_line_count( - f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc") - # A new update on a certain day adds 2 new lines - return official_line_count <= csc_line_count + 2 diff --git a/main.py b/main.py index 410d631..5d4e9ee 100644 --- a/main.py +++ b/main.py @@ -5,16 +5,35 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst """ import time -import os import sys import requests +from almalinux import AlmaLinux +from alpine import Alpine +from apache import Apache from arch import Arch +from centos import CentOS from ceph import Ceph +from cpan import CPAN +from cygwin import Cygwin from debian import Debian +from debiancd import DebianCD +from debianmultimedia import DebianMultimedia +from debianports import DebianPorts +from debiansecurity import DebianSecurity from eclipse import Eclipse +from fedora import Fedora +from freebsd import FreeBSD +from gentoodistfiles import GentooDistfiles +from gentooportage import GentooPortage +from gnome import GNOME from gnu import GNU +from gutenberg import Gutenberg +from ipfire import IPFire +from kde import KDE +from kdeapplicationdata import KDEApplicationData from kernel import Kernel from openbsd import OpenBSD +from shared import CSC_MIRROR from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/ import re # import regular expressions to remove stray numbers in string that might interfere with date finding @@ -22,8 +41,6 @@ import json # import json to read distro info stored in json file import datefinder # another date finding library -CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" - def checker(directory_URL, file_name): page = requests.get(directory_URL).text indexOfFile = page.find(file_name) @@ -50,53 +67,7 @@ def checker(directory_URL, file_name): else: return('No dates found') -def gentoo_portage_checker(data_json, distro_name): - """GentooPortage checker""" - rsync_command = "rsync -q {}{} {}" - os.system(rsync_command.format(data_json[distro_name]["csc"], - data_json[distro_name]["file"], - "csc_manifest")) - os.system(rsync_command.format(data_json[distro_name]["upstream1"], - data_json[distro_name]["file"], - "upstream_manifest1")) - os.system(rsync_command.format(data_json[distro_name]["upstream2"], - data_json[distro_name]["file"], - "upstream_manifest2")) - stream1 = os.popen("diff csc_manifest upstream_manifest1") - output1 = stream1.read() - stream2 = os.popen("diff csc_manifest upstream_manifest2") - output2 = stream2.read() - os.system("rm csc_manifest") - os.system("rm upstream_manifest1") - os.system("rm upstream_manifest2") - return 0 in [len(output1), len(output2)] - -def gnome_checker(data_json, distro_name): - """GNOME checker""" - csc_url = CSC_MIRROR + data_json[distro_name]["file"] - upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"] - upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"] - csc_gnome_text = requests.get(csc_url).text - return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text] - -def ipfire_checker(): - """IPFire checker""" - ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text - return ipfire_text.find("The mirror is up") != -1 - -def general_checker(data_json, distro_name): - """General distro checker""" - csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"] - upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"] - return requests.get(csc_url).text == requests.get(upstream_url).text - if __name__ == "__main__": - """for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]: - try: - distro.print_output(distro.check()) - except requests.exceptions.RequestException as err: - print(f"Error: {distro.name()}\n{err}")""" - """distros = json.load(open('distros.json',)) print(distros) @@ -114,36 +85,18 @@ if __name__ == "__main__": current_time = int(time.time()) for distro in distros: try: - if distro == "CPAN": - res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() - for mirror in res_json: - if mirror["url"] == f"{CSC_MIRROR}CPAN/": - if current_time - int(mirror["age"]) \ - > data[distro]["out_of_sync_interval"]: - print(f"Failure: {distro} out-of-sync") - else: - print(f"Success: {distro} up-to-date") - break - continue - if distro == "GentooPortage": - checker_result = gentoo_portage_checker(data, distro) - elif distro == "GNOME": - gnome_text = requests.get("https://download.gnome.org/core/").text - line_count = len(gnome_text.split('\n')) - # Latest version is currently 41, which has line count of 49 - if line_count == 49: - checker_result = gnome_checker(data, distro) - else: - data[distro]["out_of_sync_since"] = None - print(f"Failure: {distro} should check for latest version") - continue - elif distro == "IPFire": - checker_result = ipfire_checker() - elif distro not in data: + if distro not in data: print(f"Failure: {distro} does not exist") continue - else: - checker_result = general_checker(data, distro) + distro_class = getattr(sys.modules[__name__], distro) + if distro == "CPAN": + checker_result = distro_class.check(data, distro, current_time) + if checker_result: + print(f"Success: {distro} up-to-date") + else: + print(f"Failure: {distro} out-of-sync") + continue + checker_result = distro_class.check(data, distro) if checker_result: data[distro]["out_of_sync_since"] = None elif data[distro]["out_of_sync_since"] is None: diff --git a/openbsd.py b/openbsd.py index 75a3709..8b2122f 100644 --- a/openbsd.py +++ b/openbsd.py @@ -3,19 +3,6 @@ Contains OpenBSD class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class OpenBSD(Distro): """OpenBSD class""" - @staticmethod - def name(): - """Get name of OpenBSD""" - return "OpenBSD" - - @staticmethod - def check(): - """Check if OpenBSD packages are up-to-date""" - official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp") - csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/shared.py b/shared.py index 9f3ba69..9de4e7a 100644 --- a/shared.py +++ b/shared.py @@ -1,10 +1,3 @@ -"""Contains shared constants and functions""" - -import requests +"""Contains shared constants""" CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" - -def get_sec(timestamp_file_url): - """Get seconds since the Epoch from timestamp file""" - sec_str = requests.get(timestamp_file_url).text - return int(sec_str)