Refactor and use requirements file

.gitignore

@@ -139,3 +139,4 @@ cython_debug/
 
 # Idea
 .idea/
+*.txt~

cpan.py

@@ -1,17 +0,0 @@
-"""
-Contains CPAN class
-"""
-
-import requests
-from distro import Distro
-from shared import CSC_MIRROR
-
-class CPAN(Distro):
-    """CPAN class"""
-    @staticmethod
-    def check(data, distro, current_time):
-        res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
-        for mirror in res_json:
-            if mirror["url"] == f"{CSC_MIRROR}CPAN/":
-                return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
-        return False

distro.py

@@ -3,11 +3,15 @@ Contains abstract class for a distro
 """
 
 from abc import ABC
+
 import requests
+
 from shared import CSC_MIRROR
 
+
 class Distro(ABC):
 	"""Abstract class for a distro"""
+
 	@staticmethod
 	def check(data, distro):
 		"""Check if distro packages are up-to-date"""

distros.json

@@ -1,5 +1,20 @@
 [
-    ["OpenBSD", "https://mirror.csclub.uwaterloo.ca/OpenBSD/", "https://ftp.openbsd.org/pub/OpenBSD/", "timestamp"],
-    ["kernel", "http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/", "https://mirrors.edge.kernel.org/pub/linux/kernel/next/", "sha256sums.asc"],
-    ["debian", "http://mirror.csclub.uwaterloo.ca/debian/project/trace/", "https://ftp-master.debian.org/debian/project/trace/", "master"]
+  [
+	"OpenBSD",
+	"https://mirror.csclub.uwaterloo.ca/OpenBSD/",
+	"https://ftp.openbsd.org/pub/OpenBSD/",
+	"timestamp"
+  ],
+  [
+	"kernel",
+	"http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/",
+	"https://mirrors.edge.kernel.org/pub/linux/kernel/next/",
+	"sha256sums.asc"
+  ],
+  [
+	"debian",
+	"http://mirror.csclub.uwaterloo.ca/debian/project/trace/",
+	"https://ftp-master.debian.org/debian/project/trace/",
+	"master"
+  ]
 ]

gentooportage.py

@@ -1,29 +0,0 @@
-"""
-Contains GentooPortage class
-"""
-
-import os
-from distro import Distro
-
-class GentooPortage(Distro):
-    """GentooPortage class"""
-    @staticmethod
-    def check(data, distro):
-        rsync_command = "rsync -q {}{} {}"
-        os.system(rsync_command.format(data[distro]["csc"],
-                                        data[distro]["file"],
-                                        "csc_manifest"))
-        os.system(rsync_command.format(data[distro]["upstream1"],
-                                        data[distro]["file"],
-                                        "upstream_manifest1"))
-        os.system(rsync_command.format(data[distro]["upstream2"],
-                                        data[distro]["file"],
-                                        "upstream_manifest2"))
-        stream1 = os.popen("diff csc_manifest upstream_manifest1")
-        output1 = stream1.read()
-        stream2 = os.popen("diff csc_manifest upstream_manifest2")
-        output2 = stream2.read()
-        os.system("rm csc_manifest")
-        os.system("rm upstream_manifest1")
-        os.system("rm upstream_manifest2")
-        return 0 in [len(output1), len(output2)]

gnome.py

@@ -1,40 +0,0 @@
-"""
-Contains GNOME class
-"""
-
-import re
-import requests
-from distro import Distro
-from shared import CSC_MIRROR
-
-class GNOME(Distro):
-    """GNOME class"""
-    @staticmethod
-    def check(data, distro):
-        file = data[distro]["file1"]
-        csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
-        upstream_versions = requests.get(data[distro]["upstream1"] + file).text
-        csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
-        upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
-        if csc_latest != upstream_latest:
-            return False
-        file += csc_latest + "/"
-        csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
-        upstream_versions = requests.get(data[distro]["upstream1"] + file).text
-        csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
-        upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
-        if csc_latest != upstream_latest:
-            return False
-        file += csc_latest + "/"
-        csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
-                                + data[distro]["file2"]).text
-        try:
-            ret = csc_text == requests.get(data[distro]["upstream2"] + file
-                                           + data[distro]["file2"]).text
-        except requests.exceptions.RequestException:
-            ret = False
-        try:
-            return ret or csc_text == requests.get(data[distro]["upstream3"] + file
-                                                   + data[distro]["file2"]).text
-        except requests.exceptions.RequestException:
-            return False

ipfire.py

@@ -1,14 +0,0 @@
-"""
-Contains IPFire class
-"""
-
-import requests
-from distro import Distro
-
-class IPFire(Distro):
-    """IPFire class"""
-    @staticmethod
-    def check(data, distro):
-        ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
-        ipfire_text = requests.get(ipfire_url).text
-        return ipfire_text.find("The mirror is up") != -1

main.py

@@ -7,32 +7,32 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
 import time
 import sys
 import requests
-from almalinux import AlmaLinux
-from alpine import Alpine
-from apache import Apache
-from arch import Arch
-from centos import CentOS
-from ceph import Ceph
-from cpan import CPAN
-from cygwin import Cygwin
-from debian import Debian
-from debiancd import DebianCD
-from debianmultimedia import DebianMultimedia
-from debianports import DebianPorts
-from debiansecurity import DebianSecurity
-from eclipse import Eclipse
-from fedora import Fedora
-from freebsd import FreeBSD
-from gentoodistfiles import GentooDistfiles
-from gentooportage import GentooPortage
-from gnome import GNOME
-from gnu import GNU
-from gutenberg import Gutenberg
-from ipfire import IPFire
-from kde import KDE
-from kdeapplicationdata import KDEApplicationData
-from kernel import Kernel
-from openbsd import OpenBSD
+from projects.almalinux import AlmaLinux
+from projects.alpine import Alpine
+from projects.apache import Apache
+from projects.arch import Arch
+from projects.centos import CentOS
+from projects.ceph import Ceph
+from projects.cpan import CPAN
+from projects.cygwin import Cygwin
+from projects.debian import Debian
+from projects.debiancd import DebianCD
+from projects.debianmultimedia import DebianMultimedia
+from projects.debianports import DebianPorts
+from projects.debiansecurity import DebianSecurity
+from projects.eclipse import Eclipse
+from projects.fedora import Fedora
+from projects.freebsd import FreeBSD
+from projects.gentoodistfiles import GentooDistfiles
+from projects.gentooportage import GentooPortage
+from projects.gnome import GNOME
+from projects.gnu import GNU
+from projects.gutenberg import Gutenberg
+from projects.ipfire import IPFire
+from projects.kde import KDE
+from projects.kdeapplicationdata import KDEApplicationData
+from projects.kernel import Kernel
+from projects.openbsd import OpenBSD
 from shared import CSC_MIRROR
 from dateparser.search import search_dates  # this library seems to be super slow but the other library: dateutil.parser gets some errors
 # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
@@ -41,6 +41,7 @@ import json # import json to read distro info stored in json file
 
 import datefinder  # another date finding library
 
+
 def checker(directory_URL, file_name):
 	page = requests.get(directory_URL).text
 	indexOfFile = page.find(file_name)
@@ -67,6 +68,7 @@ def checker(directory_URL, file_name):
 	else:
 		return ('No dates found')
 
+
 if __name__ == "__main__":
 	"""distros = json.load(open('distros.json',))
 	print(distros)

projects/almalinux.py

@@ -4,5 +4,6 @@ Contains AlmaLinux class
 
 from distro import Distro
 
+
 class AlmaLinux(Distro):
 	"""AlmaLinux class"""

projects/alpine.py

@@ -4,5 +4,6 @@ Contains Alpine class
 
 from distro import Distro
 
+
 class Alpine(Distro):
 	"""Alpine class"""

projects/apache.py

@@ -4,5 +4,6 @@ Contains Apache class
 
 from distro import Distro
 
+
 class Apache(Distro):
 	"""Apache class"""

projects/arch.py

@@ -4,5 +4,6 @@ Contains Arch class
 
 from distro import Distro
 
+
 class Arch(Distro):
 	"""Arch class"""

projects/centos.py

@@ -4,5 +4,6 @@ Contains CentOS class
 
 from distro import Distro
 
+
 class CentOS(Distro):
 	"""CentOS class"""

projects/ceph.py

@@ -4,5 +4,6 @@ Contains Ceph class
 
 from distro import Distro
 
+
 class Ceph(Distro):
 	"""Ceph class"""

projects/cpan.py

@@ -0,0 +1,20 @@
+"""
+Contains CPAN class
+"""
+
+import requests
+
+from distro import Distro
+from shared import CSC_MIRROR
+
+
+class CPAN(Distro):
+	"""CPAN class"""
+
+	@staticmethod
+	def check(data, distro, current_time):
+		res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
+		for mirror in res_json:
+			if mirror["url"] == f"{CSC_MIRROR}CPAN/":
+				return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
+		return False

projects/cygwin.py

@@ -4,5 +4,6 @@ Contains Cygwin class
 
 from distro import Distro
 
+
 class Cygwin(Distro):
 	"""Cygwin class"""

projects/debian.py

@@ -4,5 +4,6 @@ Contains Debian class
 
 from distro import Distro
 
+
 class Debian(Distro):
 	"""Debian class"""

projects/debiancd.py

@@ -4,5 +4,6 @@ Contains DebianCD class
 
 from distro import Distro
 
+
 class DebianCD(Distro):
 	"""DebianCD class"""

projects/debianmultimedia.py

@@ -4,5 +4,6 @@ Contains DebianMultimedia class
 
 from distro import Distro
 
+
 class DebianMultimedia(Distro):
 	"""DebianMultimedia class"""

projects/debianports.py

@@ -4,5 +4,6 @@ Contains DebianPorts class
 
 from distro import Distro
 
+
 class DebianPorts(Distro):
 	"""DebianPorts class"""

projects/debiansecurity.py

@@ -4,5 +4,6 @@ Contains DebianSecurity class
 
 from distro import Distro
 
+
 class DebianSecurity(Distro):
 	"""DebianSecurity class"""

projects/eclipse.py

@@ -4,5 +4,6 @@ Contains Eclipse class
 
 from distro import Distro
 
+
 class Eclipse(Distro):
 	"""Eclipse class"""

projects/fedora.py

@@ -4,5 +4,6 @@ Contains Fedora class
 
 from distro import Distro
 
+
 class Fedora(Distro):
 	"""Fedora class"""

projects/freebsd.py

@@ -4,5 +4,6 @@ Contains FreeBSD class
 
 from distro import Distro
 
+
 class FreeBSD(Distro):
 	"""FreeBSD class"""

projects/gentoodistfiles.py

@@ -4,5 +4,6 @@ Contains GentooDistfiles class
 
 from distro import Distro
 
+
 class GentooDistfiles(Distro):
 	"""GentooDistfiles class"""

projects/gentooportage.py

@@ -0,0 +1,32 @@
+"""
+Contains GentooPortage class
+"""
+
+import os
+
+from distro import Distro
+
+
+class GentooPortage(Distro):
+	"""GentooPortage class"""
+
+	@staticmethod
+	def check(data, distro):
+		rsync_command = "rsync -q {}{} {}"
+		os.system(rsync_command.format(data[distro]["csc"],
+		                               data[distro]["file"],
+		                               "csc_manifest"))
+		os.system(rsync_command.format(data[distro]["upstream1"],
+		                               data[distro]["file"],
+		                               "upstream_manifest1"))
+		os.system(rsync_command.format(data[distro]["upstream2"],
+		                               data[distro]["file"],
+		                               "upstream_manifest2"))
+		stream1 = os.popen("diff csc_manifest upstream_manifest1")
+		output1 = stream1.read()
+		stream2 = os.popen("diff csc_manifest upstream_manifest2")
+		output2 = stream2.read()
+		os.system("rm csc_manifest")
+		os.system("rm upstream_manifest1")
+		os.system("rm upstream_manifest2")
+		return 0 in [len(output1), len(output2)]

projects/gnome.py

@@ -0,0 +1,44 @@
+"""
+Contains GNOME class
+"""
+
+import re
+
+import requests
+
+from distro import Distro
+from shared import CSC_MIRROR
+
+
+class GNOME(Distro):
+	"""GNOME class"""
+
+	@staticmethod
+	def check(data, distro):
+		file = data[distro]["file1"]
+		csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
+		upstream_versions = requests.get(data[distro]["upstream1"] + file).text
+		csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
+		upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
+		if csc_latest != upstream_latest:
+			return False
+		file += csc_latest + "/"
+		csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
+		upstream_versions = requests.get(data[distro]["upstream1"] + file).text
+		csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
+		upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
+		if csc_latest != upstream_latest:
+			return False
+		file += csc_latest + "/"
+		csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+		                        + data[distro]["file2"]).text
+		try:
+			ret = csc_text == requests.get(data[distro]["upstream2"] + file
+			                               + data[distro]["file2"]).text
+		except requests.exceptions.RequestException:
+			ret = False
+		try:
+			return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+			                                       + data[distro]["file2"]).text
+		except requests.exceptions.RequestException:
+			return False

projects/gnu.py

@@ -4,5 +4,6 @@ Contains GNU class
 
 from distro import Distro
 
+
 class GNU(Distro):
 	"""GNU class"""

projects/gutenberg.py

@@ -4,5 +4,6 @@ Contains Gutenberg class
 
 from distro import Distro
 
+
 class Gutenberg(Distro):
 	"""Gutenberg class"""

projects/ipfire.py

@@ -0,0 +1,17 @@
+"""
+Contains IPFire class
+"""
+
+import requests
+
+from distro import Distro
+
+
+class IPFire(Distro):
+	"""IPFire class"""
+
+	@staticmethod
+	def check(data, distro):
+		ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
+		ipfire_text = requests.get(ipfire_url).text
+		return ipfire_text.find("The mirror is up") != -1

projects/kde.py

@@ -4,5 +4,6 @@ Contains KDE class
 
 from distro import Distro
 
+
 class KDE(Distro):
 	"""KDE class"""

projects/kdeapplicationdata.py

@@ -4,5 +4,6 @@ Contains KDEApplicationData class
 
 from distro import Distro
 
+
 class KDEApplicationData(Distro):
 	"""KDEApplicationData class"""

projects/kernel.py

@@ -4,5 +4,6 @@ Contains Kernel class
 
 from distro import Distro
 
+
 class Kernel(Distro):
 	"""Kernel class"""

projects/openbsd.py

@@ -4,5 +4,6 @@ Contains OpenBSD class
 
 from distro import Distro
 
+
 class OpenBSD(Distro):
 	"""OpenBSD class"""

requirements.txt

@@ -0,0 +1,4 @@
+requests~=2.26.0
+datefinder~=0.7.1
+
+dateparser~=1.0.0