From 3031db966bc94b629773c103cf0067e76b560cbe Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Sat, 4 Sep 2021 12:03:18 -0400 Subject: [PATCH] Changed GNOME to automatically check for latest version and refactored code into classes --- almalinux.py | 8 ++++ alpine.py | 8 ++++ apache.py | 8 ++++ arch.py | 20 -------- centos.py | 8 ++++ ceph.py | 13 ----- cpan.py | 17 +++++++ cygwin.py | 8 ++++ data.json | 9 ++-- debian.py | 23 --------- debiancd.py | 8 ++++ debianmultimedia.py | 8 ++++ debianports.py | 8 ++++ debiansecurity.py | 8 ++++ distro.py | 23 +++------ eclipse.py | 13 ----- fedora.py | 8 ++++ freebsd.py | 8 ++++ gentoodistfiles.py | 8 ++++ gentooportage.py | 29 ++++++++++++ gnome.py | 40 ++++++++++++++++ gnu.py | 13 ----- gutenberg.py | 8 ++++ ipfire.py | 14 ++++++ kde.py | 8 ++++ kdeapplicationdata.py | 8 ++++ kernel.py | 22 --------- main.py | 107 ++++++++++++------------------------------ openbsd.py | 13 ----- shared.py | 9 +--- 30 files changed, 264 insertions(+), 221 deletions(-) create mode 100644 almalinux.py create mode 100644 alpine.py create mode 100644 apache.py create mode 100644 centos.py create mode 100644 cpan.py create mode 100644 cygwin.py create mode 100644 debiancd.py create mode 100644 debianmultimedia.py create mode 100644 debianports.py create mode 100644 debiansecurity.py create mode 100644 fedora.py create mode 100644 freebsd.py create mode 100644 gentoodistfiles.py create mode 100644 gentooportage.py create mode 100644 gnome.py create mode 100644 gutenberg.py create mode 100644 ipfire.py create mode 100644 kde.py create mode 100644 kdeapplicationdata.py diff --git a/almalinux.py b/almalinux.py new file mode 100644 index 0000000..201deef --- /dev/null +++ b/almalinux.py @@ -0,0 +1,8 @@ +""" +Contains AlmaLinux class +""" + +from distro import Distro + +class AlmaLinux(Distro): + """AlmaLinux class""" diff --git a/alpine.py b/alpine.py new file mode 100644 index 0000000..b47ccae --- /dev/null +++ b/alpine.py @@ -0,0 +1,8 @@ +""" +Contains Alpine class +""" + +from distro import Distro + +class Alpine(Distro): + """Alpine class""" diff --git a/apache.py b/apache.py new file mode 100644 index 0000000..0541525 --- /dev/null +++ b/apache.py @@ -0,0 +1,8 @@ +""" +Contains Apache class +""" + +from distro import Distro + +class Apache(Distro): + """Apache class""" diff --git a/arch.py b/arch.py index 2b81880..6409ef7 100644 --- a/arch.py +++ b/arch.py @@ -2,27 +2,7 @@ Contains Arch class """ -from datetime import datetime, timedelta -import requests from distro import Distro -from shared import CSC_MIRROR class Arch(Distro): """Arch class""" - @staticmethod - def name(): - """Get name of Arch""" - return "Arch" - - @staticmethod - def check(): - """Check if Arch packages are up-to-date""" - arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json() - last_check_str = arch_json["last_check"] - last_sync_str = [url for url in arch_json["urls"] if url["url"] == \ - f"{CSC_MIRROR}archlinux/"][0]["last_sync"] - last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ") - last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ") - # According to https://archlinux.org/mirrors/status/: - # Due to the timing of mirror checks, any value under one hour should be viewed as ideal - return last_check_dt < last_sync_dt + timedelta(hours = 1) diff --git a/centos.py b/centos.py new file mode 100644 index 0000000..0362130 --- /dev/null +++ b/centos.py @@ -0,0 +1,8 @@ +""" +Contains CentOS class +""" + +from distro import Distro + +class CentOS(Distro): + """CentOS class""" diff --git a/ceph.py b/ceph.py index fe8f28b..d39b81f 100644 --- a/ceph.py +++ b/ceph.py @@ -3,19 +3,6 @@ Contains Ceph class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class Ceph(Distro): """Ceph class""" - @staticmethod - def name(): - """Get name of Ceph""" - return "Ceph" - - @staticmethod - def check(): - """Check if Ceph packages are up-to-date""" - official_sec = get_sec("https://download.ceph.com/timestamp") - csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/cpan.py b/cpan.py new file mode 100644 index 0000000..d72aafc --- /dev/null +++ b/cpan.py @@ -0,0 +1,17 @@ +""" +Contains CPAN class +""" + +import requests +from distro import Distro +from shared import CSC_MIRROR + +class CPAN(Distro): + """CPAN class""" + @staticmethod + def check(data, distro, current_time): + res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() + for mirror in res_json: + if mirror["url"] == f"{CSC_MIRROR}CPAN/": + return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"] + return False diff --git a/cygwin.py b/cygwin.py new file mode 100644 index 0000000..0885224 --- /dev/null +++ b/cygwin.py @@ -0,0 +1,8 @@ +""" +Contains Cygwin class +""" + +from distro import Distro + +class Cygwin(Distro): + """Cygwin class""" diff --git a/data.json b/data.json index 0a932de..41e1c9d 100644 --- a/data.json +++ b/data.json @@ -125,9 +125,12 @@ "GNOME": { "out_of_sync_since": null, "out_of_sync_interval": 86400, - "upstream1": "https://mirrors.dotsrc.org/", - "upstream2": "https://muug.ca/mirror/", - "file": "gnome/core/41/41.beta/cache.json" + "csc": "gnome/", + "upstream1": "https://download.gnome.org/", + "upstream2": "https://mirrors.dotsrc.org/gnome/", + "upstream3": "https://muug.ca/mirror/gnome/", + "file1": "core/", + "file2": "cache.json" }, "GNU": { "out_of_sync_since": null, diff --git a/debian.py b/debian.py index 9238fa8..a720054 100644 --- a/debian.py +++ b/debian.py @@ -2,30 +2,7 @@ Contains Debian class """ -from datetime import datetime, timedelta -import requests from distro import Distro -from shared import CSC_MIRROR class Debian(Distro): """Debian class""" - @staticmethod - def __get_dt(trace_file_url): - """Get Debian datetime object from trace file""" - response_text = requests.get(trace_file_url).text - time_str = response_text.split('\n')[0] - return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y") - - @staticmethod - def name(): - """Get name of Debian""" - return "Debian" - - @staticmethod - def check(): - """Check if Debian packages are up-to-date""" - official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master") - csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master") - # Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html - # green and not yellow - return official_dt < csc_dt + timedelta(hours = 7) diff --git a/debiancd.py b/debiancd.py new file mode 100644 index 0000000..90e7f39 --- /dev/null +++ b/debiancd.py @@ -0,0 +1,8 @@ +""" +Contains DebianCD class +""" + +from distro import Distro + +class DebianCD(Distro): + """DebianCD class""" diff --git a/debianmultimedia.py b/debianmultimedia.py new file mode 100644 index 0000000..5e4fd29 --- /dev/null +++ b/debianmultimedia.py @@ -0,0 +1,8 @@ +""" +Contains DebianMultimedia class +""" + +from distro import Distro + +class DebianMultimedia(Distro): + """DebianMultimedia class""" diff --git a/debianports.py b/debianports.py new file mode 100644 index 0000000..a3f2952 --- /dev/null +++ b/debianports.py @@ -0,0 +1,8 @@ +""" +Contains DebianPorts class +""" + +from distro import Distro + +class DebianPorts(Distro): + """DebianPorts class""" diff --git a/debiansecurity.py b/debiansecurity.py new file mode 100644 index 0000000..9a50858 --- /dev/null +++ b/debiansecurity.py @@ -0,0 +1,8 @@ +""" +Contains DebianSecurity class +""" + +from distro import Distro + +class DebianSecurity(Distro): + """DebianSecurity class""" diff --git a/distro.py b/distro.py index 32b517a..7d363dc 100644 --- a/distro.py +++ b/distro.py @@ -2,24 +2,15 @@ Contains abstract class for a distro """ -from abc import ABC, abstractmethod +from abc import ABC +import requests +from shared import CSC_MIRROR class Distro(ABC): """Abstract class for a distro""" @staticmethod - @abstractmethod - def name(): - """Get name of distro""" - raise NotImplementedError - - @staticmethod - @abstractmethod - def check(): + def check(data, distro): """Check if distro packages are up-to-date""" - raise NotImplementedError - - @classmethod - def print_output(cls, is_successful): - """Print final output of distro""" - output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync" - print(output.format(cls.name())) + csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"] + upstream_url = data[distro]["upstream"] + data[distro]["file"] + return requests.get(csc_url).text == requests.get(upstream_url).text diff --git a/eclipse.py b/eclipse.py index e413e04..9945b88 100644 --- a/eclipse.py +++ b/eclipse.py @@ -3,19 +3,6 @@ Contains Eclipse class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class Eclipse(Distro): """Eclipse class""" - @staticmethod - def name(): - """Get name of Eclipse""" - return "Eclipse" - - @staticmethod - def check(): - """Check if Eclipse packages are up-to-date""" - official_sec = get_sec("http://download.eclipse.org/TIME") - csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME") - # Out-of-sync by 2 days maximum - return official_sec < csc_sec + 172800 diff --git a/fedora.py b/fedora.py new file mode 100644 index 0000000..60cc2e9 --- /dev/null +++ b/fedora.py @@ -0,0 +1,8 @@ +""" +Contains Fedora class +""" + +from distro import Distro + +class Fedora(Distro): + """Fedora class""" diff --git a/freebsd.py b/freebsd.py new file mode 100644 index 0000000..2c7eec6 --- /dev/null +++ b/freebsd.py @@ -0,0 +1,8 @@ +""" +Contains FreeBSD class +""" + +from distro import Distro + +class FreeBSD(Distro): + """FreeBSD class""" diff --git a/gentoodistfiles.py b/gentoodistfiles.py new file mode 100644 index 0000000..ae3a740 --- /dev/null +++ b/gentoodistfiles.py @@ -0,0 +1,8 @@ +""" +Contains GentooDistfiles class +""" + +from distro import Distro + +class GentooDistfiles(Distro): + """GentooDistfiles class""" diff --git a/gentooportage.py b/gentooportage.py new file mode 100644 index 0000000..29498aa --- /dev/null +++ b/gentooportage.py @@ -0,0 +1,29 @@ +""" +Contains GentooPortage class +""" + +import os +from distro import Distro + +class GentooPortage(Distro): + """GentooPortage class""" + @staticmethod + def check(data, distro): + rsync_command = "rsync -q {}{} {}" + os.system(rsync_command.format(data[distro]["csc"], + data[distro]["file"], + "csc_manifest")) + os.system(rsync_command.format(data[distro]["upstream1"], + data[distro]["file"], + "upstream_manifest1")) + os.system(rsync_command.format(data[distro]["upstream2"], + data[distro]["file"], + "upstream_manifest2")) + stream1 = os.popen("diff csc_manifest upstream_manifest1") + output1 = stream1.read() + stream2 = os.popen("diff csc_manifest upstream_manifest2") + output2 = stream2.read() + os.system("rm csc_manifest") + os.system("rm upstream_manifest1") + os.system("rm upstream_manifest2") + return 0 in [len(output1), len(output2)] diff --git a/gnome.py b/gnome.py new file mode 100644 index 0000000..8c9dfb6 --- /dev/null +++ b/gnome.py @@ -0,0 +1,40 @@ +""" +Contains GNOME class +""" + +import re +import requests +from distro import Distro +from shared import CSC_MIRROR + +class GNOME(Distro): + """GNOME class""" + @staticmethod + def check(data, distro): + file = data[distro]["file1"] + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file + + data[distro]["file2"]).text + try: + ret = csc_text == requests.get(data[distro]["upstream2"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + ret = False + try: + return ret or csc_text == requests.get(data[distro]["upstream3"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + return False diff --git a/gnu.py b/gnu.py index 2d23ade..8b7af9a 100644 --- a/gnu.py +++ b/gnu.py @@ -3,19 +3,6 @@ Contains GNU class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class GNU(Distro): """GNU class""" - @staticmethod - def name(): - """Get name of GNU""" - return "GNU" - - @staticmethod - def check(): - """Check if GNU packages are up-to-date""" - official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt") - csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/gutenberg.py b/gutenberg.py new file mode 100644 index 0000000..4ce58f9 --- /dev/null +++ b/gutenberg.py @@ -0,0 +1,8 @@ +""" +Contains Gutenberg class +""" + +from distro import Distro + +class Gutenberg(Distro): + """Gutenberg class""" diff --git a/ipfire.py b/ipfire.py new file mode 100644 index 0000000..c9b9911 --- /dev/null +++ b/ipfire.py @@ -0,0 +1,14 @@ +""" +Contains IPFire class +""" + +import requests +from distro import Distro + +class IPFire(Distro): + """IPFire class""" + @staticmethod + def check(data, distro): + ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca" + ipfire_text = requests.get(ipfire_url).text + return ipfire_text.find("The mirror is up") != -1 diff --git a/kde.py b/kde.py new file mode 100644 index 0000000..2a18445 --- /dev/null +++ b/kde.py @@ -0,0 +1,8 @@ +""" +Contains KDE class +""" + +from distro import Distro + +class KDE(Distro): + """KDE class""" diff --git a/kdeapplicationdata.py b/kdeapplicationdata.py new file mode 100644 index 0000000..6474864 --- /dev/null +++ b/kdeapplicationdata.py @@ -0,0 +1,8 @@ +""" +Contains KDEApplicationData class +""" + +from distro import Distro + +class KDEApplicationData(Distro): + """KDEApplicationData class""" diff --git a/kernel.py b/kernel.py index 30e6b83..e06dad5 100644 --- a/kernel.py +++ b/kernel.py @@ -2,29 +2,7 @@ Contains Kernel class """ -import requests from distro import Distro -from shared import CSC_MIRROR class Kernel(Distro): """Kernel class""" - @staticmethod - def __get_line_count(checksum_file_url): - """Get Kernel line count of checksum file""" - response_text = requests.get(checksum_file_url).text - return len(response_text.split('\n')) - - @staticmethod - def name(): - """Get name of Kernel""" - return "Kernel" - - @staticmethod - def check(): - """Check if Kernel packages are up-to-date""" - official_line_count = Kernel.__get_line_count( - "https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc") - csc_line_count = Kernel.__get_line_count( - f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc") - # A new update on a certain day adds 2 new lines - return official_line_count <= csc_line_count + 2 diff --git a/main.py b/main.py index 410d631..5d4e9ee 100644 --- a/main.py +++ b/main.py @@ -5,16 +5,35 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst """ import time -import os import sys import requests +from almalinux import AlmaLinux +from alpine import Alpine +from apache import Apache from arch import Arch +from centos import CentOS from ceph import Ceph +from cpan import CPAN +from cygwin import Cygwin from debian import Debian +from debiancd import DebianCD +from debianmultimedia import DebianMultimedia +from debianports import DebianPorts +from debiansecurity import DebianSecurity from eclipse import Eclipse +from fedora import Fedora +from freebsd import FreeBSD +from gentoodistfiles import GentooDistfiles +from gentooportage import GentooPortage +from gnome import GNOME from gnu import GNU +from gutenberg import Gutenberg +from ipfire import IPFire +from kde import KDE +from kdeapplicationdata import KDEApplicationData from kernel import Kernel from openbsd import OpenBSD +from shared import CSC_MIRROR from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/ import re # import regular expressions to remove stray numbers in string that might interfere with date finding @@ -22,8 +41,6 @@ import json # import json to read distro info stored in json file import datefinder # another date finding library -CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" - def checker(directory_URL, file_name): page = requests.get(directory_URL).text indexOfFile = page.find(file_name) @@ -50,53 +67,7 @@ def checker(directory_URL, file_name): else: return('No dates found') -def gentoo_portage_checker(data_json, distro_name): - """GentooPortage checker""" - rsync_command = "rsync -q {}{} {}" - os.system(rsync_command.format(data_json[distro_name]["csc"], - data_json[distro_name]["file"], - "csc_manifest")) - os.system(rsync_command.format(data_json[distro_name]["upstream1"], - data_json[distro_name]["file"], - "upstream_manifest1")) - os.system(rsync_command.format(data_json[distro_name]["upstream2"], - data_json[distro_name]["file"], - "upstream_manifest2")) - stream1 = os.popen("diff csc_manifest upstream_manifest1") - output1 = stream1.read() - stream2 = os.popen("diff csc_manifest upstream_manifest2") - output2 = stream2.read() - os.system("rm csc_manifest") - os.system("rm upstream_manifest1") - os.system("rm upstream_manifest2") - return 0 in [len(output1), len(output2)] - -def gnome_checker(data_json, distro_name): - """GNOME checker""" - csc_url = CSC_MIRROR + data_json[distro_name]["file"] - upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"] - upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"] - csc_gnome_text = requests.get(csc_url).text - return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text] - -def ipfire_checker(): - """IPFire checker""" - ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text - return ipfire_text.find("The mirror is up") != -1 - -def general_checker(data_json, distro_name): - """General distro checker""" - csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"] - upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"] - return requests.get(csc_url).text == requests.get(upstream_url).text - if __name__ == "__main__": - """for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]: - try: - distro.print_output(distro.check()) - except requests.exceptions.RequestException as err: - print(f"Error: {distro.name()}\n{err}")""" - """distros = json.load(open('distros.json',)) print(distros) @@ -114,36 +85,18 @@ if __name__ == "__main__": current_time = int(time.time()) for distro in distros: try: - if distro == "CPAN": - res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() - for mirror in res_json: - if mirror["url"] == f"{CSC_MIRROR}CPAN/": - if current_time - int(mirror["age"]) \ - > data[distro]["out_of_sync_interval"]: - print(f"Failure: {distro} out-of-sync") - else: - print(f"Success: {distro} up-to-date") - break - continue - if distro == "GentooPortage": - checker_result = gentoo_portage_checker(data, distro) - elif distro == "GNOME": - gnome_text = requests.get("https://download.gnome.org/core/").text - line_count = len(gnome_text.split('\n')) - # Latest version is currently 41, which has line count of 49 - if line_count == 49: - checker_result = gnome_checker(data, distro) - else: - data[distro]["out_of_sync_since"] = None - print(f"Failure: {distro} should check for latest version") - continue - elif distro == "IPFire": - checker_result = ipfire_checker() - elif distro not in data: + if distro not in data: print(f"Failure: {distro} does not exist") continue - else: - checker_result = general_checker(data, distro) + distro_class = getattr(sys.modules[__name__], distro) + if distro == "CPAN": + checker_result = distro_class.check(data, distro, current_time) + if checker_result: + print(f"Success: {distro} up-to-date") + else: + print(f"Failure: {distro} out-of-sync") + continue + checker_result = distro_class.check(data, distro) if checker_result: data[distro]["out_of_sync_since"] = None elif data[distro]["out_of_sync_since"] is None: diff --git a/openbsd.py b/openbsd.py index 75a3709..8b2122f 100644 --- a/openbsd.py +++ b/openbsd.py @@ -3,19 +3,6 @@ Contains OpenBSD class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class OpenBSD(Distro): """OpenBSD class""" - @staticmethod - def name(): - """Get name of OpenBSD""" - return "OpenBSD" - - @staticmethod - def check(): - """Check if OpenBSD packages are up-to-date""" - official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp") - csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/shared.py b/shared.py index 9f3ba69..9de4e7a 100644 --- a/shared.py +++ b/shared.py @@ -1,10 +1,3 @@ -"""Contains shared constants and functions""" - -import requests +"""Contains shared constants""" CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" - -def get_sec(timestamp_file_url): - """Get seconds since the Epoch from timestamp file""" - sec_str = requests.get(timestamp_file_url).text - return int(sec_str)