Changed GNOME to automatically check for latest version and refactored code into classes

almalinux.py

@@ -0,0 +1,8 @@
+"""
+Contains AlmaLinux class
+"""
+
+from distro import Distro
+
+class AlmaLinux(Distro):
+    """AlmaLinux class"""

alpine.py

@@ -0,0 +1,8 @@
+"""
+Contains Alpine class
+"""
+
+from distro import Distro
+
+class Alpine(Distro):
+    """Alpine class"""

apache.py

@@ -0,0 +1,8 @@
+"""
+Contains Apache class
+"""
+
+from distro import Distro
+
+class Apache(Distro):
+    """Apache class"""

arch.py

@@ -2,27 +2,7 @@
 Contains Arch class
 """
 
-from datetime import datetime, timedelta
-import requests
 from distro import Distro
-from shared import CSC_MIRROR
 
 class Arch(Distro):
     """Arch class"""
-    @staticmethod
-    def name():
-        """Get name of Arch"""
-        return "Arch"
-
-    @staticmethod
-    def check():
-        """Check if Arch packages are up-to-date"""
-        arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json()
-        last_check_str = arch_json["last_check"]
-        last_sync_str = [url for url in arch_json["urls"] if url["url"] == \
-                         f"{CSC_MIRROR}archlinux/"][0]["last_sync"]
-        last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ")
-        last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ")
-        # According to https://archlinux.org/mirrors/status/:
-        # Due to the timing of mirror checks, any value under one hour should be viewed as ideal
-        return last_check_dt < last_sync_dt + timedelta(hours = 1)

centos.py

@@ -0,0 +1,8 @@
+"""
+Contains CentOS class
+"""
+
+from distro import Distro
+
+class CentOS(Distro):
+    """CentOS class"""

ceph.py

@@ -3,19 +3,6 @@ Contains Ceph class
 """
 
 from distro import Distro
-from shared import CSC_MIRROR, get_sec
 
 class Ceph(Distro):
     """Ceph class"""
-    @staticmethod
-    def name():
-        """Get name of Ceph"""
-        return "Ceph"
-
-    @staticmethod
-    def check():
-        """Check if Ceph packages are up-to-date"""
-        official_sec = get_sec("https://download.ceph.com/timestamp")
-        csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp")
-        # Out-of-sync by 1 day maximum
-        return official_sec < csc_sec + 86400

cpan.py

@@ -0,0 +1,17 @@
+"""
+Contains CPAN class
+"""
+
+import requests
+from distro import Distro
+from shared import CSC_MIRROR
+
+class CPAN(Distro):
+    """CPAN class"""
+    @staticmethod
+    def check(data, distro, current_time):
+        res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
+        for mirror in res_json:
+            if mirror["url"] == f"{CSC_MIRROR}CPAN/":
+                return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
+        return False

cygwin.py

@@ -0,0 +1,8 @@
+"""
+Contains Cygwin class
+"""
+
+from distro import Distro
+
+class Cygwin(Distro):
+    """Cygwin class"""

data.json

@@ -125,9 +125,12 @@
     "GNOME": {
         "out_of_sync_since": null,
         "out_of_sync_interval": 86400,
-        "upstream1": "https://mirrors.dotsrc.org/",
-        "upstream2": "https://muug.ca/mirror/",
-        "file": "gnome/core/41/41.beta/cache.json"
+        "csc": "gnome/",
+        "upstream1": "https://download.gnome.org/",
+        "upstream2": "https://mirrors.dotsrc.org/gnome/",
+        "upstream3": "https://muug.ca/mirror/gnome/",
+        "file1": "core/",
+        "file2": "cache.json"
     },
     "GNU": {
         "out_of_sync_since": null,

debian.py

@@ -2,30 +2,7 @@
 Contains Debian class
 """
 
-from datetime import datetime, timedelta
-import requests
 from distro import Distro
-from shared import CSC_MIRROR
 
 class Debian(Distro):
     """Debian class"""
-    @staticmethod
-    def __get_dt(trace_file_url):
-        """Get Debian datetime object from trace file"""
-        response_text = requests.get(trace_file_url).text
-        time_str = response_text.split('\n')[0]
-        return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y")
-
-    @staticmethod
-    def name():
-        """Get name of Debian"""
-        return "Debian"
-
-    @staticmethod
-    def check():
-        """Check if Debian packages are up-to-date"""
-        official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master")
-        csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master")
-        # Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html
-        # green and not yellow
-        return official_dt < csc_dt + timedelta(hours = 7)

debiancd.py

@@ -0,0 +1,8 @@
+"""
+Contains DebianCD class
+"""
+
+from distro import Distro
+
+class DebianCD(Distro):
+    """DebianCD class"""

debianmultimedia.py

@@ -0,0 +1,8 @@
+"""
+Contains DebianMultimedia class
+"""
+
+from distro import Distro
+
+class DebianMultimedia(Distro):
+    """DebianMultimedia class"""

debianports.py

@@ -0,0 +1,8 @@
+"""
+Contains DebianPorts class
+"""
+
+from distro import Distro
+
+class DebianPorts(Distro):
+    """DebianPorts class"""

debiansecurity.py

@@ -0,0 +1,8 @@
+"""
+Contains DebianSecurity class
+"""
+
+from distro import Distro
+
+class DebianSecurity(Distro):
+    """DebianSecurity class"""

distro.py

@@ -2,24 +2,15 @@
 Contains abstract class for a distro
 """
 
-from abc import ABC, abstractmethod
+from abc import ABC
+import requests
+from shared import CSC_MIRROR
 
 class Distro(ABC):
     """Abstract class for a distro"""
     @staticmethod
-    @abstractmethod
-    def name():
-        """Get name of distro"""
-        raise NotImplementedError
-
-    @staticmethod
-    @abstractmethod
-    def check():
+    def check(data, distro):
         """Check if distro packages are up-to-date"""
-        raise NotImplementedError
-
-    @classmethod
-    def print_output(cls, is_successful):
-        """Print final output of distro"""
-        output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync"
-        print(output.format(cls.name()))
+        csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
+        upstream_url = data[distro]["upstream"] + data[distro]["file"]
+        return requests.get(csc_url).text == requests.get(upstream_url).text

eclipse.py

@@ -3,19 +3,6 @@ Contains Eclipse class
 """
 
 from distro import Distro
-from shared import CSC_MIRROR, get_sec
 
 class Eclipse(Distro):
     """Eclipse class"""
-    @staticmethod
-    def name():
-        """Get name of Eclipse"""
-        return "Eclipse"
-
-    @staticmethod
-    def check():
-        """Check if Eclipse packages are up-to-date"""
-        official_sec = get_sec("http://download.eclipse.org/TIME")
-        csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME")
-        # Out-of-sync by 2 days maximum
-        return official_sec < csc_sec + 172800

fedora.py

@@ -0,0 +1,8 @@
+"""
+Contains Fedora class
+"""
+
+from distro import Distro
+
+class Fedora(Distro):
+    """Fedora class"""

freebsd.py

@@ -0,0 +1,8 @@
+"""
+Contains FreeBSD class
+"""
+
+from distro import Distro
+
+class FreeBSD(Distro):
+    """FreeBSD class"""

gentoodistfiles.py

@@ -0,0 +1,8 @@
+"""
+Contains GentooDistfiles class
+"""
+
+from distro import Distro
+
+class GentooDistfiles(Distro):
+    """GentooDistfiles class"""

gentooportage.py

@@ -0,0 +1,29 @@
+"""
+Contains GentooPortage class
+"""
+
+import os
+from distro import Distro
+
+class GentooPortage(Distro):
+    """GentooPortage class"""
+    @staticmethod
+    def check(data, distro):
+        rsync_command = "rsync -q {}{} {}"
+        os.system(rsync_command.format(data[distro]["csc"],
+                                        data[distro]["file"],
+                                        "csc_manifest"))
+        os.system(rsync_command.format(data[distro]["upstream1"],
+                                        data[distro]["file"],
+                                        "upstream_manifest1"))
+        os.system(rsync_command.format(data[distro]["upstream2"],
+                                        data[distro]["file"],
+                                        "upstream_manifest2"))
+        stream1 = os.popen("diff csc_manifest upstream_manifest1")
+        output1 = stream1.read()
+        stream2 = os.popen("diff csc_manifest upstream_manifest2")
+        output2 = stream2.read()
+        os.system("rm csc_manifest")
+        os.system("rm upstream_manifest1")
+        os.system("rm upstream_manifest2")
+        return 0 in [len(output1), len(output2)]

gnome.py

@@ -0,0 +1,40 @@
+"""
+Contains GNOME class
+"""
+
+import re
+import requests
+from distro import Distro
+from shared import CSC_MIRROR
+
+class GNOME(Distro):
+    """GNOME class"""
+    @staticmethod
+    def check(data, distro):
+        file = data[distro]["file1"]
+        csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
+        upstream_versions = requests.get(data[distro]["upstream1"] + file).text
+        csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
+        upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
+        if csc_latest != upstream_latest:
+            return False
+        file += csc_latest + "/"
+        csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
+        upstream_versions = requests.get(data[distro]["upstream1"] + file).text
+        csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
+        upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
+        if csc_latest != upstream_latest:
+            return False
+        file += csc_latest + "/"
+        csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+                                + data[distro]["file2"]).text
+        try:
+            ret = csc_text == requests.get(data[distro]["upstream2"] + file
+                                           + data[distro]["file2"]).text
+        except requests.exceptions.RequestException:
+            ret = False
+        try:
+            return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+                                                   + data[distro]["file2"]).text
+        except requests.exceptions.RequestException:
+            return False

gnu.py

@@ -3,19 +3,6 @@ Contains GNU class
 """
 
 from distro import Distro
-from shared import CSC_MIRROR, get_sec
 
 class GNU(Distro):
     """GNU class"""
-    @staticmethod
-    def name():
-        """Get name of GNU"""
-        return "GNU"
-
-    @staticmethod
-    def check():
-        """Check if GNU packages are up-to-date"""
-        official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt")
-        csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt")
-        # Out-of-sync by 1 day maximum
-        return official_sec < csc_sec + 86400

gutenberg.py

@@ -0,0 +1,8 @@
+"""
+Contains Gutenberg class
+"""
+
+from distro import Distro
+
+class Gutenberg(Distro):
+    """Gutenberg class"""

ipfire.py

@@ -0,0 +1,14 @@
+"""
+Contains IPFire class
+"""
+
+import requests
+from distro import Distro
+
+class IPFire(Distro):
+    """IPFire class"""
+    @staticmethod
+    def check(data, distro):
+        ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
+        ipfire_text = requests.get(ipfire_url).text
+        return ipfire_text.find("The mirror is up") != -1

kde.py

@@ -0,0 +1,8 @@
+"""
+Contains KDE class
+"""
+
+from distro import Distro
+
+class KDE(Distro):
+    """KDE class"""

kdeapplicationdata.py

@@ -0,0 +1,8 @@
+"""
+Contains KDEApplicationData class
+"""
+
+from distro import Distro
+
+class KDEApplicationData(Distro):
+    """KDEApplicationData class"""

kernel.py

@@ -2,29 +2,7 @@
 Contains Kernel class
 """
 
-import requests
 from distro import Distro
-from shared import CSC_MIRROR
 
 class Kernel(Distro):
     """Kernel class"""
-    @staticmethod
-    def __get_line_count(checksum_file_url):
-        """Get Kernel line count of checksum file"""
-        response_text = requests.get(checksum_file_url).text
-        return len(response_text.split('\n'))
-
-    @staticmethod
-    def name():
-        """Get name of Kernel"""
-        return "Kernel"
-
-    @staticmethod
-    def check():
-        """Check if Kernel packages are up-to-date"""
-        official_line_count = Kernel.__get_line_count(
-            "https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc")
-        csc_line_count = Kernel.__get_line_count(
-            f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc")
-        # A new update on a certain day adds 2 new lines
-        return official_line_count <= csc_line_count + 2

main.py

@@ -5,16 +5,35 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
 """
 
 import time
-import os
 import sys
 import requests
+from almalinux import AlmaLinux
+from alpine import Alpine
+from apache import Apache
 from arch import Arch
+from centos import CentOS
 from ceph import Ceph
+from cpan import CPAN
+from cygwin import Cygwin
 from debian import Debian
+from debiancd import DebianCD
+from debianmultimedia import DebianMultimedia
+from debianports import DebianPorts
+from debiansecurity import DebianSecurity
 from eclipse import Eclipse
+from fedora import Fedora
+from freebsd import FreeBSD
+from gentoodistfiles import GentooDistfiles
+from gentooportage import GentooPortage
+from gnome import GNOME
 from gnu import GNU
+from gutenberg import Gutenberg
+from ipfire import IPFire
+from kde import KDE
+from kdeapplicationdata import KDEApplicationData
 from kernel import Kernel
 from openbsd import OpenBSD
+from shared import CSC_MIRROR
 from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
 # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
 import re # import regular expressions to remove stray numbers in string that might interfere with date finding
@@ -22,8 +41,6 @@ import json # import json to read distro info stored in json file
 
 import datefinder # another date finding library
 
-CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
-
 def checker(directory_URL, file_name):
     page = requests.get(directory_URL).text
     indexOfFile = page.find(file_name)
@@ -50,53 +67,7 @@ def checker(directory_URL, file_name):
     else:
         return('No dates found')
 
-def gentoo_portage_checker(data_json, distro_name):
-    """GentooPortage checker"""
-    rsync_command = "rsync -q {}{} {}"
-    os.system(rsync_command.format(data_json[distro_name]["csc"],
-                                   data_json[distro_name]["file"],
-                                   "csc_manifest"))
-    os.system(rsync_command.format(data_json[distro_name]["upstream1"],
-                                   data_json[distro_name]["file"],
-                                   "upstream_manifest1"))
-    os.system(rsync_command.format(data_json[distro_name]["upstream2"],
-                                   data_json[distro_name]["file"],
-                                   "upstream_manifest2"))
-    stream1 = os.popen("diff csc_manifest upstream_manifest1")
-    output1 = stream1.read()
-    stream2 = os.popen("diff csc_manifest upstream_manifest2")
-    output2 = stream2.read()
-    os.system("rm csc_manifest")
-    os.system("rm upstream_manifest1")
-    os.system("rm upstream_manifest2")
-    return 0 in [len(output1), len(output2)]
-
-def gnome_checker(data_json, distro_name):
-    """GNOME checker"""
-    csc_url = CSC_MIRROR + data_json[distro_name]["file"]
-    upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"]
-    upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"]
-    csc_gnome_text = requests.get(csc_url).text
-    return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text]
-
-def ipfire_checker():
-    """IPFire checker"""
-    ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text
-    return ipfire_text.find("The mirror is up") != -1
-
-def general_checker(data_json, distro_name):
-    """General distro checker"""
-    csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"]
-    upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"]
-    return requests.get(csc_url).text == requests.get(upstream_url).text
-
 if __name__ == "__main__":
-    """for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]:
-        try:
-            distro.print_output(distro.check())
-        except requests.exceptions.RequestException as err:
-            print(f"Error: {distro.name()}\n{err}")"""
-
     """distros = json.load(open('distros.json',))
     print(distros)
 
@@ -114,36 +85,18 @@ if __name__ == "__main__":
         current_time = int(time.time())
         for distro in distros:
             try:
-                if distro == "CPAN":
-                    res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
-                    for mirror in res_json:
-                        if mirror["url"] == f"{CSC_MIRROR}CPAN/":
-                            if current_time - int(mirror["age"]) \
-                                    > data[distro]["out_of_sync_interval"]:
-                                print(f"Failure: {distro} out-of-sync")
-                            else:
-                                print(f"Success: {distro} up-to-date")
-                            break
+                if distro not in data:
+                    print(f"Failure: {distro} does not exist")
                     continue
-                if distro == "GentooPortage":
-                    checker_result = gentoo_portage_checker(data, distro)
-                elif distro == "GNOME":
-                    gnome_text = requests.get("https://download.gnome.org/core/").text
-                    line_count = len(gnome_text.split('\n'))
-                    # Latest version is currently 41, which has line count of 49
-                    if line_count == 49:
-                        checker_result = gnome_checker(data, distro)
+                distro_class = getattr(sys.modules[__name__], distro)
+                if distro == "CPAN":
+                    checker_result = distro_class.check(data, distro, current_time)
+                    if checker_result:
+                        print(f"Success: {distro} up-to-date")
                     else:
-                        data[distro]["out_of_sync_since"] = None
-                        print(f"Failure: {distro} should check for latest version")
-                        continue
-                elif distro == "IPFire":
-                    checker_result = ipfire_checker()
-                elif distro not in data:
-                    print(f"Failure: {distro} does not exist")
+                        print(f"Failure: {distro} out-of-sync")
                     continue
-                else:
-                    checker_result = general_checker(data, distro)
+                checker_result = distro_class.check(data, distro)
                 if checker_result:
                     data[distro]["out_of_sync_since"] = None
                 elif data[distro]["out_of_sync_since"] is None:

openbsd.py

@@ -3,19 +3,6 @@ Contains OpenBSD class
 """
 
 from distro import Distro
-from shared import CSC_MIRROR, get_sec
 
 class OpenBSD(Distro):
     """OpenBSD class"""
-    @staticmethod
-    def name():
-        """Get name of OpenBSD"""
-        return "OpenBSD"
-
-    @staticmethod
-    def check():
-        """Check if OpenBSD packages are up-to-date"""
-        official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp")
-        csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp")
-        # Out-of-sync by 1 day maximum
-        return official_sec < csc_sec + 86400

shared.py

@@ -1,10 +1,3 @@
-"""Contains shared constants and functions"""
-
-import requests
+"""Contains shared constants"""
 
 CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
-
-def get_sec(timestamp_file_url):
-    """Get seconds since the Epoch from timestamp file"""
-    sec_str = requests.get(timestamp_file_url).text
-    return int(sec_str)