From 1cbb02b3fa570b149e2432d5ed5a2e470d80ec59 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:25:06 -0400 Subject: [PATCH 01/24] Store data in JSON file --- data.json | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ main.py | 29 ++++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 data.json diff --git a/data.json b/data.json new file mode 100644 index 0000000..901317d --- /dev/null +++ b/data.json @@ -0,0 +1,51 @@ +{ + "Arch": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "archlinux/", + "upstream": "http://arch.mirror.constant.com/", + "file": "lastupdate" + }, + "Ceph": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "ceph/", + "upstream": "https://download.ceph.com/", + "file": "timestamp" + }, + "Debian": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://ftp-master.debian.org/", + "file": "debian/project/trace/master" + }, + "Eclipse": { + "out_of_date_since": null, + "out_of_date_interval": 172800, + "csc": "eclipse/", + "upstream": "http://download.eclipse.org/", + "file": "TIME" + }, + "GNU": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://mirrors.kernel.org/", + "file": "gnu/mirror-updated-timestamp.txt" + }, + "Kernel": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "kernel.org/", + "upstream": "https://mirrors.edge.kernel.org/pub/", + "file": "linux/kernel/next/sha256sums.asc" + }, + "OpenBSD": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://ftp.openbsd.org/pub/", + "file": "OpenBSD/timestamp" + } +} diff --git a/main.py b/main.py index 54699db..eb32afd 100644 --- a/main.py +++ b/main.py @@ -4,6 +4,7 @@ This mirror status checker determines whether CSC mirror is up-to-date with upstream """ +import time import requests from arch import Arch from ceph import Ceph @@ -51,11 +52,33 @@ if __name__ == "__main__": distro.print_output(distro.check()) except requests.exceptions.RequestException as err: print(f"Error: {distro.name()}\n{err}")""" - - distros = json.load(open('distros.json',)) + + """distros = json.load(open('distros.json',)) print(distros) for distro in distros: print(distro[0] + ":") print("CSC mirror: " + checker(distro[1], distro[3])) - print("Official distro: " + checker(distro[2], distro[3])) \ No newline at end of file + print("Official distro: " + checker(distro[2], distro[3]))""" + + with open("data.json", "r", encoding = "utf-8") as file: + data = json.load(file) + current_time = int(time.time()) + for distro in data: + csc_url = "http://mirror.csclub.uwaterloo.ca/" + data[distro]["csc"] \ + + data[distro]["file"] + upstream_url = data[distro]["upstream"] + data[distro]["file"] + try: + if requests.get(csc_url).text == requests.get(upstream_url).text: + data[distro]["out_of_date_since"] = None + elif data[distro]["out_of_date_since"] is None: + data[distro]["out_of_date_since"] = current_time + elif current_time - data[distro]["out_of_date_since"] \ + > data[distro]["out_of_date_interval"]: + print(f"Failure: {distro} out-of-sync") + continue + print(f"Success: {distro} up-to-date") + except requests.exceptions.RequestException as err: + print(f"Error: {distro}\n{err}") + with open("data.json", "w", encoding = "utf-8") as file: + json.dump(data, file, indent = 4) From 143370f87437feee495241a8788dcdd12ccccedb Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:31:55 -0400 Subject: [PATCH 02/24] Created AlmaLinux mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 901317d..11dbafb 100644 --- a/data.json +++ b/data.json @@ -1,4 +1,11 @@ { + "AlmaLinux": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://repo.almalinux.org/", + "file": "almalinux/TIME" + }, "Arch": { "out_of_date_since": null, "out_of_date_interval": 86400, From f494e2a739927e39f140a9fe6d0e8c0db7e423d3 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:35:24 -0400 Subject: [PATCH 03/24] Created Alpine mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 11dbafb..245aebc 100644 --- a/data.json +++ b/data.json @@ -6,6 +6,13 @@ "upstream": "https://repo.almalinux.org/", "file": "almalinux/TIME" }, + "Alpine": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://uk.alpinelinux.org/", + "file": "alpine/last-updated" + }, "Arch": { "out_of_date_since": null, "out_of_date_interval": 86400, From 062fafea3b2735d0083a22c4b9160ce009c8de2f Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:41:20 -0400 Subject: [PATCH 04/24] Created Apache mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 245aebc..fc86663 100644 --- a/data.json +++ b/data.json @@ -13,6 +13,13 @@ "upstream": "https://uk.alpinelinux.org/", "file": "alpine/last-updated" }, + "Apache": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "apache/", + "upstream": "https://downloads.apache.org/", + "file": "zzz/time.txt" + }, "Arch": { "out_of_date_since": null, "out_of_date_interval": 86400, From fe5dcdc75434b2d032884f7ae1f2f778205e58ac Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:44:29 -0400 Subject: [PATCH 05/24] Created CentOS mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index fc86663..ee2feec 100644 --- a/data.json +++ b/data.json @@ -27,6 +27,13 @@ "upstream": "http://arch.mirror.constant.com/", "file": "lastupdate" }, + "CentOS": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://mirrors.edge.kernel.org/", + "file": "centos/TIME" + }, "Ceph": { "out_of_date_since": null, "out_of_date_interval": 86400, From fe0b45b5189f7843811aa38c704c1ca0f6bc3fde Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:50:09 -0400 Subject: [PATCH 06/24] Created Cygwin mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index ee2feec..2fcd034 100644 --- a/data.json +++ b/data.json @@ -41,6 +41,13 @@ "upstream": "https://download.ceph.com/", "file": "timestamp" }, + "Cygwin": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "cygwin/", + "upstream": "https://cygwin.mirror.globo.tech/", + "file": "x86/sha512.sum" + }, "Debian": { "out_of_date_since": null, "out_of_date_interval": 86400, From 00fffd5236c7f7b7f4f3546b842afadf8d5e15ee Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:53:39 -0400 Subject: [PATCH 07/24] Created DebianCD mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 2fcd034..ff34547 100644 --- a/data.json +++ b/data.json @@ -55,6 +55,13 @@ "upstream": "https://ftp-master.debian.org/", "file": "debian/project/trace/master" }, + "DebianCD": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "http://debian.mirror.estruxture.net/", + "file": "debian-cd/project/trace/cdimage.debian.org" + }, "Eclipse": { "out_of_date_since": null, "out_of_date_interval": 172800, From 2f4abdff29136490dc96285c99ae71027d7e8576 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Wed, 1 Sep 2021 23:56:41 -0400 Subject: [PATCH 08/24] Created DebianMultimedia mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index ff34547..01b5ccc 100644 --- a/data.json +++ b/data.json @@ -62,6 +62,13 @@ "upstream": "http://debian.mirror.estruxture.net/", "file": "debian-cd/project/trace/cdimage.debian.org" }, + "DebianMultimedia": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "debian-multimedia/", + "upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", + "file": "project/trace/deb-multimedia.org" + }, "Eclipse": { "out_of_date_since": null, "out_of_date_interval": 172800, From 4d9108efbe30ad1ff60161be4801eac8e41c21fa Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:00:06 -0400 Subject: [PATCH 09/24] Created DebianPorts mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 01b5ccc..bd09f2e 100644 --- a/data.json +++ b/data.json @@ -69,6 +69,13 @@ "upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", "file": "project/trace/deb-multimedia.org" }, + "DebianPorts": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "https://deb.debian.org/", + "file": "debian-ports/project/trace/porta.debian.org" + }, "Eclipse": { "out_of_date_since": null, "out_of_date_interval": 172800, From 861cee56f86598f4edab25b13fbaf7c6a3d8becf Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:02:46 -0400 Subject: [PATCH 10/24] Created DebianSecurity mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index bd09f2e..5e6210c 100644 --- a/data.json +++ b/data.json @@ -76,6 +76,13 @@ "upstream": "https://deb.debian.org/", "file": "debian-ports/project/trace/porta.debian.org" }, + "DebianSecurity": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "http://debian.mirror.iweb.ca/", + "file": "debian-security/project/trace/master" + }, "Eclipse": { "out_of_date_since": null, "out_of_date_interval": 172800, From cf0795fca6b35a2730ce470c18d2bfcf98df8562 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:07:24 -0400 Subject: [PATCH 11/24] Created Fedora mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 5e6210c..b2c0674 100644 --- a/data.json +++ b/data.json @@ -90,6 +90,13 @@ "upstream": "http://download.eclipse.org/", "file": "TIME" }, + "Fedora": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "fedora/", + "upstream": "http://fedora.mirror.iweb.com/", + "file": "linux/development/rawhide/COMPOSE_ID" + }, "GNU": { "out_of_date_since": null, "out_of_date_interval": 86400, From ace0b8193e7e792db53e67b63b28c560324bdbe8 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:14:57 -0400 Subject: [PATCH 12/24] Created FreeBSD mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index b2c0674..22e1a4a 100644 --- a/data.json +++ b/data.json @@ -97,6 +97,13 @@ "upstream": "http://fedora.mirror.iweb.com/", "file": "linux/development/rawhide/COMPOSE_ID" }, + "FreeBSD": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "", + "upstream": "http://ftp4.freebsd.org/pub/", + "file": "FreeBSD/TIMESTAMP" + }, "GNU": { "out_of_date_since": null, "out_of_date_interval": 86400, From 074cf77b6b40f6b6ebe51f4f047d0ff4d678f08e Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:18:10 -0400 Subject: [PATCH 13/24] Created GentooDistfiles mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 22e1a4a..f19f919 100644 --- a/data.json +++ b/data.json @@ -104,6 +104,13 @@ "upstream": "http://ftp4.freebsd.org/pub/", "file": "FreeBSD/TIMESTAMP" }, + "GentooDistfiles": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "gentoo-distfiles", + "upstream": "http://gentoo.mirrors.tera-byte.com/", + "file": "distfiles/timestamp.dev-local" + }, "GNU": { "out_of_date_since": null, "out_of_date_interval": 86400, From 113cddeddb36cd1a8f5ce706bbbede2a2ecf0aaf Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:25:55 -0400 Subject: [PATCH 14/24] Created Gutenberg mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index f19f919..218774c 100644 --- a/data.json +++ b/data.json @@ -118,6 +118,13 @@ "upstream": "https://mirrors.kernel.org/", "file": "gnu/mirror-updated-timestamp.txt" }, + "Gutenberg": { + "out_of_date_since": null, + "out_of_date_interval": 172800, + "csc": "gutenberg/", + "upstream": "https://gutenberg.pglaf.org/", + "file": "gutenberg.dcs" + }, "Kernel": { "out_of_date_since": null, "out_of_date_interval": 86400, From ade00673575f39095234b7dca5c49f45d7bf803c Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:32:00 -0400 Subject: [PATCH 15/24] Created KDE mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 218774c..5b330d3 100644 --- a/data.json +++ b/data.json @@ -125,6 +125,13 @@ "upstream": "https://gutenberg.pglaf.org/", "file": "gutenberg.dcs" }, + "KDE": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "kde/", + "upstream": "https://kde.c3sl.ufpr.br/", + "file": "ls-lR" + }, "Kernel": { "out_of_date_since": null, "out_of_date_interval": 86400, From ab2c3e5c6216429effc7022d614c421c7b0f9e16 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 00:39:06 -0400 Subject: [PATCH 16/24] Created KDEApplicationData mirror status checker --- data.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/data.json b/data.json index 5b330d3..bf9ccf2 100644 --- a/data.json +++ b/data.json @@ -132,6 +132,13 @@ "upstream": "https://kde.c3sl.ufpr.br/", "file": "ls-lR" }, + "KDEApplicationData": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "kde-applicationdata/", + "upstream": "https://cdn.files.kde.org/", + "file": "last-updated" + }, "Kernel": { "out_of_date_since": null, "out_of_date_interval": 86400, From 0ef22b660a87b4accd3cc5bc10246db05edac60f Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 19:23:46 -0400 Subject: [PATCH 17/24] Created IPFire mirror status checker --- data.json | 4 ++++ main.py | 21 +++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/data.json b/data.json index bf9ccf2..91a9163 100644 --- a/data.json +++ b/data.json @@ -125,6 +125,10 @@ "upstream": "https://gutenberg.pglaf.org/", "file": "gutenberg.dcs" }, + "IPFire": { + "out_of_date_since": null, + "out_of_date_interval": 172800 + }, "KDE": { "out_of_date_since": null, "out_of_date_interval": 86400, diff --git a/main.py b/main.py index eb32afd..99b2c74 100644 --- a/main.py +++ b/main.py @@ -46,6 +46,18 @@ def checker(directory_URL, file_name): else: return('No dates found') +def ipfire_checker(): + """IPFire checker""" + res_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text + return res_text.find("The mirror is up") != -1 + +def general_checker(data_json, distro_name): + """General distro checker""" + csc_url = "http://mirror.csclub.uwaterloo.ca/" + data_json[distro_name]["csc"] \ + + data_json[distro_name]["file"] + upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"] + return requests.get(csc_url).text == requests.get(upstream_url).text + if __name__ == "__main__": """for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]: try: @@ -65,11 +77,12 @@ if __name__ == "__main__": data = json.load(file) current_time = int(time.time()) for distro in data: - csc_url = "http://mirror.csclub.uwaterloo.ca/" + data[distro]["csc"] \ - + data[distro]["file"] - upstream_url = data[distro]["upstream"] + data[distro]["file"] try: - if requests.get(csc_url).text == requests.get(upstream_url).text: + if distro == "IPFire": + checker_result = ipfire_checker() + else: + checker_result = general_checker(data, distro) + if checker_result: data[distro]["out_of_date_since"] = None elif data[distro]["out_of_date_since"] is None: data[distro]["out_of_date_since"] = current_time From 273200abdf95daf60809287cb7a14d05cafc1261 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 20:53:20 -0400 Subject: [PATCH 18/24] Created CPAN mirror status checker --- data.json | 3 +++ main.py | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/data.json b/data.json index 91a9163..62c3713 100644 --- a/data.json +++ b/data.json @@ -41,6 +41,9 @@ "upstream": "https://download.ceph.com/", "file": "timestamp" }, + "CPAN": { + "out_of_date_interval": 172800 + }, "Cygwin": { "out_of_date_since": null, "out_of_date_interval": 86400, diff --git a/main.py b/main.py index 99b2c74..a4f0d59 100644 --- a/main.py +++ b/main.py @@ -20,6 +20,8 @@ import json # import json to read distro info stored in json file import datefinder # another date finding library +CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" + def checker(directory_URL, file_name): page = requests.get(directory_URL).text indexOfFile = page.find(file_name) @@ -53,8 +55,7 @@ def ipfire_checker(): def general_checker(data_json, distro_name): """General distro checker""" - csc_url = "http://mirror.csclub.uwaterloo.ca/" + data_json[distro_name]["csc"] \ - + data_json[distro_name]["file"] + csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"] upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"] return requests.get(csc_url).text == requests.get(upstream_url).text @@ -78,6 +79,17 @@ if __name__ == "__main__": current_time = int(time.time()) for distro in data: try: + if distro == "CPAN": + res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() + for mirror in res_json: + if mirror["url"] == f"{CSC_MIRROR}CPAN/": + if current_time - int(mirror["age"]) \ + > data[distro]["out_of_date_interval"]: + print(f"Failure: {distro} out-of-sync") + else: + print(f"Success: {distro} up-to-date") + break + continue if distro == "IPFire": checker_result = ipfire_checker() else: From 78dc376e8388201d67334f9cfaa52f8874953535 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Thu, 2 Sep 2021 23:44:59 -0400 Subject: [PATCH 19/24] Created GNOME mirror status checker --- data.json | 7 +++++++ main.py | 24 +++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/data.json b/data.json index 62c3713..7a27bf8 100644 --- a/data.json +++ b/data.json @@ -114,6 +114,13 @@ "upstream": "http://gentoo.mirrors.tera-byte.com/", "file": "distfiles/timestamp.dev-local" }, + "GNOME": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "upstream1": "https://mirrors.dotsrc.org/", + "upstream2": "https://muug.ca/mirror/", + "file": "gnome/core/41/41.beta/cache.json" + }, "GNU": { "out_of_date_since": null, "out_of_date_interval": 86400, diff --git a/main.py b/main.py index a4f0d59..ecf74de 100644 --- a/main.py +++ b/main.py @@ -48,10 +48,18 @@ def checker(directory_URL, file_name): else: return('No dates found') +def gnome_checker(data_json, distro_name): + """GNOME checker""" + csc_url = CSC_MIRROR + data_json[distro_name]["file"] + upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"] + upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"] + csc_gnome_text = requests.get(csc_url).text + return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text] + def ipfire_checker(): """IPFire checker""" - res_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text - return res_text.find("The mirror is up") != -1 + ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text + return ipfire_text.find("The mirror is up") != -1 def general_checker(data_json, distro_name): """General distro checker""" @@ -90,7 +98,17 @@ if __name__ == "__main__": print(f"Success: {distro} up-to-date") break continue - if distro == "IPFire": + if distro == "GNOME": + gnome_text = requests.get("https://download.gnome.org/core/").text + line_count = len(gnome_text.split('\n')) + # Latest version is currently 41, which has line count of 49 + if line_count == 49: + checker_result = gnome_checker(data, distro) + else: + data[distro]["out_of_date_since"] = None + print(f"Failure: {distro} should check for latest version") + continue + elif distro == "IPFire": checker_result = ipfire_checker() else: checker_result = general_checker(data, distro) From f448ceddc9ef2d4df57a332b6ec33a1489ab740a Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Fri, 3 Sep 2021 14:28:37 -0400 Subject: [PATCH 20/24] Created GentooPortage mirror status checker --- data.json | 8 ++++++++ main.py | 26 +++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/data.json b/data.json index 7a27bf8..9eefe62 100644 --- a/data.json +++ b/data.json @@ -114,6 +114,14 @@ "upstream": "http://gentoo.mirrors.tera-byte.com/", "file": "distfiles/timestamp.dev-local" }, + "GentooPortage": { + "out_of_date_since": null, + "out_of_date_interval": 86400, + "csc": "rsync://rsync4.ca.gentoo.org/", + "upstream1": "rsync://rsync1.de.gentoo.org/", + "upstream2": "rsync://rsync8.de.gentoo.org/", + "file": "gentoo-portage/Manifest" + }, "GNOME": { "out_of_date_since": null, "out_of_date_interval": 86400, diff --git a/main.py b/main.py index ecf74de..eb462ad 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst """ import time +import os import requests from arch import Arch from ceph import Ceph @@ -48,6 +49,27 @@ def checker(directory_URL, file_name): else: return('No dates found') +def gentoo_portage_checker(data_json, distro_name): + """GentooPortage checker""" + rsync_command = "rsync -q {}{} {}" + os.system(rsync_command.format(data_json[distro_name]["csc"], + data_json[distro_name]["file"], + "csc_manifest")) + os.system(rsync_command.format(data_json[distro_name]["upstream1"], + data_json[distro_name]["file"], + "upstream_manifest1")) + os.system(rsync_command.format(data_json[distro_name]["upstream2"], + data_json[distro_name]["file"], + "upstream_manifest2")) + stream1 = os.popen("diff csc_manifest upstream_manifest1") + output1 = stream1.read() + stream2 = os.popen("diff csc_manifest upstream_manifest2") + output2 = stream2.read() + os.system("rm csc_manifest") + os.system("rm upstream_manifest1") + os.system("rm upstream_manifest2") + return 0 in [len(output1), len(output2)] + def gnome_checker(data_json, distro_name): """GNOME checker""" csc_url = CSC_MIRROR + data_json[distro_name]["file"] @@ -98,7 +120,9 @@ if __name__ == "__main__": print(f"Success: {distro} up-to-date") break continue - if distro == "GNOME": + if distro == "GentooPortage": + checker_result = gentoo_portage_checker(data, distro) + elif distro == "GNOME": gnome_text = requests.get("https://download.gnome.org/core/").text line_count = len(gnome_text.split('\n')) # Latest version is currently 41, which has line count of 49 From d04b65928ec9dddf17a7ffc1f237628e6a08d21e Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Fri, 3 Sep 2021 14:59:39 -0400 Subject: [PATCH 21/24] Changed 'out_of_date' to 'out_of_sync' --- data.json | 102 +++++++++++++++++++++++++++--------------------------- main.py | 14 ++++---- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/data.json b/data.json index 9eefe62..0a932de 100644 --- a/data.json +++ b/data.json @@ -1,176 +1,176 @@ { "AlmaLinux": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://repo.almalinux.org/", "file": "almalinux/TIME" }, "Alpine": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://uk.alpinelinux.org/", "file": "alpine/last-updated" }, "Apache": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "apache/", "upstream": "https://downloads.apache.org/", "file": "zzz/time.txt" }, "Arch": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "archlinux/", "upstream": "http://arch.mirror.constant.com/", "file": "lastupdate" }, "CentOS": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://mirrors.edge.kernel.org/", "file": "centos/TIME" }, "Ceph": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "ceph/", "upstream": "https://download.ceph.com/", "file": "timestamp" }, "CPAN": { - "out_of_date_interval": 172800 + "out_of_sync_interval": 172800 }, "Cygwin": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "cygwin/", "upstream": "https://cygwin.mirror.globo.tech/", "file": "x86/sha512.sum" }, "Debian": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://ftp-master.debian.org/", "file": "debian/project/trace/master" }, "DebianCD": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "http://debian.mirror.estruxture.net/", "file": "debian-cd/project/trace/cdimage.debian.org" }, "DebianMultimedia": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "debian-multimedia/", "upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", "file": "project/trace/deb-multimedia.org" }, "DebianPorts": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://deb.debian.org/", "file": "debian-ports/project/trace/porta.debian.org" }, "DebianSecurity": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "http://debian.mirror.iweb.ca/", "file": "debian-security/project/trace/master" }, "Eclipse": { - "out_of_date_since": null, - "out_of_date_interval": 172800, + "out_of_sync_since": null, + "out_of_sync_interval": 172800, "csc": "eclipse/", "upstream": "http://download.eclipse.org/", "file": "TIME" }, "Fedora": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "fedora/", "upstream": "http://fedora.mirror.iweb.com/", "file": "linux/development/rawhide/COMPOSE_ID" }, "FreeBSD": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "http://ftp4.freebsd.org/pub/", "file": "FreeBSD/TIMESTAMP" }, "GentooDistfiles": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "gentoo-distfiles", "upstream": "http://gentoo.mirrors.tera-byte.com/", "file": "distfiles/timestamp.dev-local" }, "GentooPortage": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "rsync://rsync4.ca.gentoo.org/", "upstream1": "rsync://rsync1.de.gentoo.org/", "upstream2": "rsync://rsync8.de.gentoo.org/", "file": "gentoo-portage/Manifest" }, "GNOME": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "upstream1": "https://mirrors.dotsrc.org/", "upstream2": "https://muug.ca/mirror/", "file": "gnome/core/41/41.beta/cache.json" }, "GNU": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://mirrors.kernel.org/", "file": "gnu/mirror-updated-timestamp.txt" }, "Gutenberg": { - "out_of_date_since": null, - "out_of_date_interval": 172800, + "out_of_sync_since": null, + "out_of_sync_interval": 172800, "csc": "gutenberg/", "upstream": "https://gutenberg.pglaf.org/", "file": "gutenberg.dcs" }, "IPFire": { - "out_of_date_since": null, - "out_of_date_interval": 172800 + "out_of_sync_since": null, + "out_of_sync_interval": 172800 }, "KDE": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "kde/", "upstream": "https://kde.c3sl.ufpr.br/", "file": "ls-lR" }, "KDEApplicationData": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "kde-applicationdata/", "upstream": "https://cdn.files.kde.org/", "file": "last-updated" }, "Kernel": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "kernel.org/", "upstream": "https://mirrors.edge.kernel.org/pub/", "file": "linux/kernel/next/sha256sums.asc" }, "OpenBSD": { - "out_of_date_since": null, - "out_of_date_interval": 86400, + "out_of_sync_since": null, + "out_of_sync_interval": 86400, "csc": "", "upstream": "https://ftp.openbsd.org/pub/", "file": "OpenBSD/timestamp" diff --git a/main.py b/main.py index eb462ad..6de60ca 100644 --- a/main.py +++ b/main.py @@ -114,7 +114,7 @@ if __name__ == "__main__": for mirror in res_json: if mirror["url"] == f"{CSC_MIRROR}CPAN/": if current_time - int(mirror["age"]) \ - > data[distro]["out_of_date_interval"]: + > data[distro]["out_of_sync_interval"]: print(f"Failure: {distro} out-of-sync") else: print(f"Success: {distro} up-to-date") @@ -129,7 +129,7 @@ if __name__ == "__main__": if line_count == 49: checker_result = gnome_checker(data, distro) else: - data[distro]["out_of_date_since"] = None + data[distro]["out_of_sync_since"] = None print(f"Failure: {distro} should check for latest version") continue elif distro == "IPFire": @@ -137,11 +137,11 @@ if __name__ == "__main__": else: checker_result = general_checker(data, distro) if checker_result: - data[distro]["out_of_date_since"] = None - elif data[distro]["out_of_date_since"] is None: - data[distro]["out_of_date_since"] = current_time - elif current_time - data[distro]["out_of_date_since"] \ - > data[distro]["out_of_date_interval"]: + data[distro]["out_of_sync_since"] = None + elif data[distro]["out_of_sync_since"] is None: + data[distro]["out_of_sync_since"] = current_time + elif current_time - data[distro]["out_of_sync_since"] \ + > data[distro]["out_of_sync_interval"]: print(f"Failure: {distro} out-of-sync") continue print(f"Success: {distro} up-to-date") From 2ff0b0c69b5f3b9f382ae13831f28c37c8719db1 Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Fri, 3 Sep 2021 17:43:34 -0400 Subject: [PATCH 22/24] Added option of configuration file in standard input --- example.in | 26 ++++++++++++++++++++++++++ main.py | 10 +++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 example.in diff --git a/example.in b/example.in new file mode 100644 index 0000000..52bcfbd --- /dev/null +++ b/example.in @@ -0,0 +1,26 @@ +AlmaLinux +Alpine +Apache +Arch +CentOS +Ceph +CPAN +Cygwin +Debian +DebianCD +DebianMultimedia +DebianPorts +DebianSecurity +Eclipse +Fedora +FreeBSD +GentooDistfiles +GentooPortage +GNOME +GNU +Gutenberg +IPFire +KDE +KDEApplicationData +Kernel +OpenBSD diff --git a/main.py b/main.py index 6de60ca..410d631 100644 --- a/main.py +++ b/main.py @@ -6,6 +6,7 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst import time import os +import sys import requests from arch import Arch from ceph import Ceph @@ -106,8 +107,12 @@ if __name__ == "__main__": with open("data.json", "r", encoding = "utf-8") as file: data = json.load(file) + if sys.stdin.isatty(): + distros = data + else: + distros = [distro.rstrip() for distro in sys.stdin.readlines()] current_time = int(time.time()) - for distro in data: + for distro in distros: try: if distro == "CPAN": res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() @@ -134,6 +139,9 @@ if __name__ == "__main__": continue elif distro == "IPFire": checker_result = ipfire_checker() + elif distro not in data: + print(f"Failure: {distro} does not exist") + continue else: checker_result = general_checker(data, distro) if checker_result: From 4022aedc6ccac3d3a8550b19bf2605f6d3c4a84c Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Fri, 3 Sep 2021 17:59:02 -0400 Subject: [PATCH 23/24] Added configuration file to README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bb68ae1..810b712 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst ## How To Run -Run `main.py`. +A configuration file may be provided through standard input. Without a configuration file, execute `python main.py`. By default, all of the available distributions will be checked. With a configuration file, execute `python main.py < name_of_config_file.in`, for example, `python main.py < example.in`. In this case, only the distributions listed in the configuration file will be checked. ## Resources @@ -20,4 +20,4 @@ to find repos of the distros to check, just search "distroName mirrors" ubuntu releases: https://releases.ubuntu.com/?_ga=2.251394307.425973732.1629764407-1388600952.1629764407 ubuntu mirrors: http://mirrors.ubuntu.com/mirrors.txt -raspbian mirror: https://mirror.ox.ac.uk/sites/archive.raspbian.org/archive/ \ No newline at end of file +raspbian mirror: https://mirror.ox.ac.uk/sites/archive.raspbian.org/archive/ From 3031db966bc94b629773c103cf0067e76b560cbe Mon Sep 17 00:00:00 2001 From: Laura Nguyen Date: Sat, 4 Sep 2021 12:03:18 -0400 Subject: [PATCH 24/24] Changed GNOME to automatically check for latest version and refactored code into classes --- almalinux.py | 8 ++++ alpine.py | 8 ++++ apache.py | 8 ++++ arch.py | 20 -------- centos.py | 8 ++++ ceph.py | 13 ------ cpan.py | 17 +++++++ cygwin.py | 8 ++++ data.json | 9 ++-- debian.py | 23 --------- debiancd.py | 8 ++++ debianmultimedia.py | 8 ++++ debianports.py | 8 ++++ debiansecurity.py | 8 ++++ distro.py | 23 +++------ eclipse.py | 13 ------ fedora.py | 8 ++++ freebsd.py | 8 ++++ gentoodistfiles.py | 8 ++++ gentooportage.py | 29 ++++++++++++ gnome.py | 40 ++++++++++++++++ gnu.py | 13 ------ gutenberg.py | 8 ++++ ipfire.py | 14 ++++++ kde.py | 8 ++++ kdeapplicationdata.py | 8 ++++ kernel.py | 22 --------- main.py | 105 ++++++++++++------------------------------ openbsd.py | 13 ------ shared.py | 9 +--- 30 files changed, 263 insertions(+), 220 deletions(-) create mode 100644 almalinux.py create mode 100644 alpine.py create mode 100644 apache.py create mode 100644 centos.py create mode 100644 cpan.py create mode 100644 cygwin.py create mode 100644 debiancd.py create mode 100644 debianmultimedia.py create mode 100644 debianports.py create mode 100644 debiansecurity.py create mode 100644 fedora.py create mode 100644 freebsd.py create mode 100644 gentoodistfiles.py create mode 100644 gentooportage.py create mode 100644 gnome.py create mode 100644 gutenberg.py create mode 100644 ipfire.py create mode 100644 kde.py create mode 100644 kdeapplicationdata.py diff --git a/almalinux.py b/almalinux.py new file mode 100644 index 0000000..201deef --- /dev/null +++ b/almalinux.py @@ -0,0 +1,8 @@ +""" +Contains AlmaLinux class +""" + +from distro import Distro + +class AlmaLinux(Distro): + """AlmaLinux class""" diff --git a/alpine.py b/alpine.py new file mode 100644 index 0000000..b47ccae --- /dev/null +++ b/alpine.py @@ -0,0 +1,8 @@ +""" +Contains Alpine class +""" + +from distro import Distro + +class Alpine(Distro): + """Alpine class""" diff --git a/apache.py b/apache.py new file mode 100644 index 0000000..0541525 --- /dev/null +++ b/apache.py @@ -0,0 +1,8 @@ +""" +Contains Apache class +""" + +from distro import Distro + +class Apache(Distro): + """Apache class""" diff --git a/arch.py b/arch.py index 2b81880..6409ef7 100644 --- a/arch.py +++ b/arch.py @@ -2,27 +2,7 @@ Contains Arch class """ -from datetime import datetime, timedelta -import requests from distro import Distro -from shared import CSC_MIRROR class Arch(Distro): """Arch class""" - @staticmethod - def name(): - """Get name of Arch""" - return "Arch" - - @staticmethod - def check(): - """Check if Arch packages are up-to-date""" - arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json() - last_check_str = arch_json["last_check"] - last_sync_str = [url for url in arch_json["urls"] if url["url"] == \ - f"{CSC_MIRROR}archlinux/"][0]["last_sync"] - last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ") - last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ") - # According to https://archlinux.org/mirrors/status/: - # Due to the timing of mirror checks, any value under one hour should be viewed as ideal - return last_check_dt < last_sync_dt + timedelta(hours = 1) diff --git a/centos.py b/centos.py new file mode 100644 index 0000000..0362130 --- /dev/null +++ b/centos.py @@ -0,0 +1,8 @@ +""" +Contains CentOS class +""" + +from distro import Distro + +class CentOS(Distro): + """CentOS class""" diff --git a/ceph.py b/ceph.py index fe8f28b..d39b81f 100644 --- a/ceph.py +++ b/ceph.py @@ -3,19 +3,6 @@ Contains Ceph class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class Ceph(Distro): """Ceph class""" - @staticmethod - def name(): - """Get name of Ceph""" - return "Ceph" - - @staticmethod - def check(): - """Check if Ceph packages are up-to-date""" - official_sec = get_sec("https://download.ceph.com/timestamp") - csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/cpan.py b/cpan.py new file mode 100644 index 0000000..d72aafc --- /dev/null +++ b/cpan.py @@ -0,0 +1,17 @@ +""" +Contains CPAN class +""" + +import requests +from distro import Distro +from shared import CSC_MIRROR + +class CPAN(Distro): + """CPAN class""" + @staticmethod + def check(data, distro, current_time): + res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() + for mirror in res_json: + if mirror["url"] == f"{CSC_MIRROR}CPAN/": + return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"] + return False diff --git a/cygwin.py b/cygwin.py new file mode 100644 index 0000000..0885224 --- /dev/null +++ b/cygwin.py @@ -0,0 +1,8 @@ +""" +Contains Cygwin class +""" + +from distro import Distro + +class Cygwin(Distro): + """Cygwin class""" diff --git a/data.json b/data.json index 0a932de..41e1c9d 100644 --- a/data.json +++ b/data.json @@ -125,9 +125,12 @@ "GNOME": { "out_of_sync_since": null, "out_of_sync_interval": 86400, - "upstream1": "https://mirrors.dotsrc.org/", - "upstream2": "https://muug.ca/mirror/", - "file": "gnome/core/41/41.beta/cache.json" + "csc": "gnome/", + "upstream1": "https://download.gnome.org/", + "upstream2": "https://mirrors.dotsrc.org/gnome/", + "upstream3": "https://muug.ca/mirror/gnome/", + "file1": "core/", + "file2": "cache.json" }, "GNU": { "out_of_sync_since": null, diff --git a/debian.py b/debian.py index 9238fa8..a720054 100644 --- a/debian.py +++ b/debian.py @@ -2,30 +2,7 @@ Contains Debian class """ -from datetime import datetime, timedelta -import requests from distro import Distro -from shared import CSC_MIRROR class Debian(Distro): """Debian class""" - @staticmethod - def __get_dt(trace_file_url): - """Get Debian datetime object from trace file""" - response_text = requests.get(trace_file_url).text - time_str = response_text.split('\n')[0] - return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y") - - @staticmethod - def name(): - """Get name of Debian""" - return "Debian" - - @staticmethod - def check(): - """Check if Debian packages are up-to-date""" - official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master") - csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master") - # Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html - # green and not yellow - return official_dt < csc_dt + timedelta(hours = 7) diff --git a/debiancd.py b/debiancd.py new file mode 100644 index 0000000..90e7f39 --- /dev/null +++ b/debiancd.py @@ -0,0 +1,8 @@ +""" +Contains DebianCD class +""" + +from distro import Distro + +class DebianCD(Distro): + """DebianCD class""" diff --git a/debianmultimedia.py b/debianmultimedia.py new file mode 100644 index 0000000..5e4fd29 --- /dev/null +++ b/debianmultimedia.py @@ -0,0 +1,8 @@ +""" +Contains DebianMultimedia class +""" + +from distro import Distro + +class DebianMultimedia(Distro): + """DebianMultimedia class""" diff --git a/debianports.py b/debianports.py new file mode 100644 index 0000000..a3f2952 --- /dev/null +++ b/debianports.py @@ -0,0 +1,8 @@ +""" +Contains DebianPorts class +""" + +from distro import Distro + +class DebianPorts(Distro): + """DebianPorts class""" diff --git a/debiansecurity.py b/debiansecurity.py new file mode 100644 index 0000000..9a50858 --- /dev/null +++ b/debiansecurity.py @@ -0,0 +1,8 @@ +""" +Contains DebianSecurity class +""" + +from distro import Distro + +class DebianSecurity(Distro): + """DebianSecurity class""" diff --git a/distro.py b/distro.py index 32b517a..7d363dc 100644 --- a/distro.py +++ b/distro.py @@ -2,24 +2,15 @@ Contains abstract class for a distro """ -from abc import ABC, abstractmethod +from abc import ABC +import requests +from shared import CSC_MIRROR class Distro(ABC): """Abstract class for a distro""" @staticmethod - @abstractmethod - def name(): - """Get name of distro""" - raise NotImplementedError - - @staticmethod - @abstractmethod - def check(): + def check(data, distro): """Check if distro packages are up-to-date""" - raise NotImplementedError - - @classmethod - def print_output(cls, is_successful): - """Print final output of distro""" - output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync" - print(output.format(cls.name())) + csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"] + upstream_url = data[distro]["upstream"] + data[distro]["file"] + return requests.get(csc_url).text == requests.get(upstream_url).text diff --git a/eclipse.py b/eclipse.py index e413e04..9945b88 100644 --- a/eclipse.py +++ b/eclipse.py @@ -3,19 +3,6 @@ Contains Eclipse class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class Eclipse(Distro): """Eclipse class""" - @staticmethod - def name(): - """Get name of Eclipse""" - return "Eclipse" - - @staticmethod - def check(): - """Check if Eclipse packages are up-to-date""" - official_sec = get_sec("http://download.eclipse.org/TIME") - csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME") - # Out-of-sync by 2 days maximum - return official_sec < csc_sec + 172800 diff --git a/fedora.py b/fedora.py new file mode 100644 index 0000000..60cc2e9 --- /dev/null +++ b/fedora.py @@ -0,0 +1,8 @@ +""" +Contains Fedora class +""" + +from distro import Distro + +class Fedora(Distro): + """Fedora class""" diff --git a/freebsd.py b/freebsd.py new file mode 100644 index 0000000..2c7eec6 --- /dev/null +++ b/freebsd.py @@ -0,0 +1,8 @@ +""" +Contains FreeBSD class +""" + +from distro import Distro + +class FreeBSD(Distro): + """FreeBSD class""" diff --git a/gentoodistfiles.py b/gentoodistfiles.py new file mode 100644 index 0000000..ae3a740 --- /dev/null +++ b/gentoodistfiles.py @@ -0,0 +1,8 @@ +""" +Contains GentooDistfiles class +""" + +from distro import Distro + +class GentooDistfiles(Distro): + """GentooDistfiles class""" diff --git a/gentooportage.py b/gentooportage.py new file mode 100644 index 0000000..29498aa --- /dev/null +++ b/gentooportage.py @@ -0,0 +1,29 @@ +""" +Contains GentooPortage class +""" + +import os +from distro import Distro + +class GentooPortage(Distro): + """GentooPortage class""" + @staticmethod + def check(data, distro): + rsync_command = "rsync -q {}{} {}" + os.system(rsync_command.format(data[distro]["csc"], + data[distro]["file"], + "csc_manifest")) + os.system(rsync_command.format(data[distro]["upstream1"], + data[distro]["file"], + "upstream_manifest1")) + os.system(rsync_command.format(data[distro]["upstream2"], + data[distro]["file"], + "upstream_manifest2")) + stream1 = os.popen("diff csc_manifest upstream_manifest1") + output1 = stream1.read() + stream2 = os.popen("diff csc_manifest upstream_manifest2") + output2 = stream2.read() + os.system("rm csc_manifest") + os.system("rm upstream_manifest1") + os.system("rm upstream_manifest2") + return 0 in [len(output1), len(output2)] diff --git a/gnome.py b/gnome.py new file mode 100644 index 0000000..8c9dfb6 --- /dev/null +++ b/gnome.py @@ -0,0 +1,40 @@ +""" +Contains GNOME class +""" + +import re +import requests +from distro import Distro +from shared import CSC_MIRROR + +class GNOME(Distro): + """GNOME class""" + @staticmethod + def check(data, distro): + file = data[distro]["file1"] + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file + + data[distro]["file2"]).text + try: + ret = csc_text == requests.get(data[distro]["upstream2"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + ret = False + try: + return ret or csc_text == requests.get(data[distro]["upstream3"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + return False diff --git a/gnu.py b/gnu.py index 2d23ade..8b7af9a 100644 --- a/gnu.py +++ b/gnu.py @@ -3,19 +3,6 @@ Contains GNU class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class GNU(Distro): """GNU class""" - @staticmethod - def name(): - """Get name of GNU""" - return "GNU" - - @staticmethod - def check(): - """Check if GNU packages are up-to-date""" - official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt") - csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/gutenberg.py b/gutenberg.py new file mode 100644 index 0000000..4ce58f9 --- /dev/null +++ b/gutenberg.py @@ -0,0 +1,8 @@ +""" +Contains Gutenberg class +""" + +from distro import Distro + +class Gutenberg(Distro): + """Gutenberg class""" diff --git a/ipfire.py b/ipfire.py new file mode 100644 index 0000000..c9b9911 --- /dev/null +++ b/ipfire.py @@ -0,0 +1,14 @@ +""" +Contains IPFire class +""" + +import requests +from distro import Distro + +class IPFire(Distro): + """IPFire class""" + @staticmethod + def check(data, distro): + ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca" + ipfire_text = requests.get(ipfire_url).text + return ipfire_text.find("The mirror is up") != -1 diff --git a/kde.py b/kde.py new file mode 100644 index 0000000..2a18445 --- /dev/null +++ b/kde.py @@ -0,0 +1,8 @@ +""" +Contains KDE class +""" + +from distro import Distro + +class KDE(Distro): + """KDE class""" diff --git a/kdeapplicationdata.py b/kdeapplicationdata.py new file mode 100644 index 0000000..6474864 --- /dev/null +++ b/kdeapplicationdata.py @@ -0,0 +1,8 @@ +""" +Contains KDEApplicationData class +""" + +from distro import Distro + +class KDEApplicationData(Distro): + """KDEApplicationData class""" diff --git a/kernel.py b/kernel.py index 30e6b83..e06dad5 100644 --- a/kernel.py +++ b/kernel.py @@ -2,29 +2,7 @@ Contains Kernel class """ -import requests from distro import Distro -from shared import CSC_MIRROR class Kernel(Distro): """Kernel class""" - @staticmethod - def __get_line_count(checksum_file_url): - """Get Kernel line count of checksum file""" - response_text = requests.get(checksum_file_url).text - return len(response_text.split('\n')) - - @staticmethod - def name(): - """Get name of Kernel""" - return "Kernel" - - @staticmethod - def check(): - """Check if Kernel packages are up-to-date""" - official_line_count = Kernel.__get_line_count( - "https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc") - csc_line_count = Kernel.__get_line_count( - f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc") - # A new update on a certain day adds 2 new lines - return official_line_count <= csc_line_count + 2 diff --git a/main.py b/main.py index 410d631..5d4e9ee 100644 --- a/main.py +++ b/main.py @@ -5,16 +5,35 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst """ import time -import os import sys import requests +from almalinux import AlmaLinux +from alpine import Alpine +from apache import Apache from arch import Arch +from centos import CentOS from ceph import Ceph +from cpan import CPAN +from cygwin import Cygwin from debian import Debian +from debiancd import DebianCD +from debianmultimedia import DebianMultimedia +from debianports import DebianPorts +from debiansecurity import DebianSecurity from eclipse import Eclipse +from fedora import Fedora +from freebsd import FreeBSD +from gentoodistfiles import GentooDistfiles +from gentooportage import GentooPortage +from gnome import GNOME from gnu import GNU +from gutenberg import Gutenberg +from ipfire import IPFire +from kde import KDE +from kdeapplicationdata import KDEApplicationData from kernel import Kernel from openbsd import OpenBSD +from shared import CSC_MIRROR from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/ import re # import regular expressions to remove stray numbers in string that might interfere with date finding @@ -22,8 +41,6 @@ import json # import json to read distro info stored in json file import datefinder # another date finding library -CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" - def checker(directory_URL, file_name): page = requests.get(directory_URL).text indexOfFile = page.find(file_name) @@ -50,53 +67,7 @@ def checker(directory_URL, file_name): else: return('No dates found') -def gentoo_portage_checker(data_json, distro_name): - """GentooPortage checker""" - rsync_command = "rsync -q {}{} {}" - os.system(rsync_command.format(data_json[distro_name]["csc"], - data_json[distro_name]["file"], - "csc_manifest")) - os.system(rsync_command.format(data_json[distro_name]["upstream1"], - data_json[distro_name]["file"], - "upstream_manifest1")) - os.system(rsync_command.format(data_json[distro_name]["upstream2"], - data_json[distro_name]["file"], - "upstream_manifest2")) - stream1 = os.popen("diff csc_manifest upstream_manifest1") - output1 = stream1.read() - stream2 = os.popen("diff csc_manifest upstream_manifest2") - output2 = stream2.read() - os.system("rm csc_manifest") - os.system("rm upstream_manifest1") - os.system("rm upstream_manifest2") - return 0 in [len(output1), len(output2)] - -def gnome_checker(data_json, distro_name): - """GNOME checker""" - csc_url = CSC_MIRROR + data_json[distro_name]["file"] - upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"] - upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"] - csc_gnome_text = requests.get(csc_url).text - return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text] - -def ipfire_checker(): - """IPFire checker""" - ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text - return ipfire_text.find("The mirror is up") != -1 - -def general_checker(data_json, distro_name): - """General distro checker""" - csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"] - upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"] - return requests.get(csc_url).text == requests.get(upstream_url).text - if __name__ == "__main__": - """for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]: - try: - distro.print_output(distro.check()) - except requests.exceptions.RequestException as err: - print(f"Error: {distro.name()}\n{err}")""" - """distros = json.load(open('distros.json',)) print(distros) @@ -114,36 +85,18 @@ if __name__ == "__main__": current_time = int(time.time()) for distro in distros: try: - if distro == "CPAN": - res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() - for mirror in res_json: - if mirror["url"] == f"{CSC_MIRROR}CPAN/": - if current_time - int(mirror["age"]) \ - > data[distro]["out_of_sync_interval"]: - print(f"Failure: {distro} out-of-sync") - else: - print(f"Success: {distro} up-to-date") - break + if distro not in data: + print(f"Failure: {distro} does not exist") continue - if distro == "GentooPortage": - checker_result = gentoo_portage_checker(data, distro) - elif distro == "GNOME": - gnome_text = requests.get("https://download.gnome.org/core/").text - line_count = len(gnome_text.split('\n')) - # Latest version is currently 41, which has line count of 49 - if line_count == 49: - checker_result = gnome_checker(data, distro) + distro_class = getattr(sys.modules[__name__], distro) + if distro == "CPAN": + checker_result = distro_class.check(data, distro, current_time) + if checker_result: + print(f"Success: {distro} up-to-date") else: - data[distro]["out_of_sync_since"] = None - print(f"Failure: {distro} should check for latest version") - continue - elif distro == "IPFire": - checker_result = ipfire_checker() - elif distro not in data: - print(f"Failure: {distro} does not exist") + print(f"Failure: {distro} out-of-sync") continue - else: - checker_result = general_checker(data, distro) + checker_result = distro_class.check(data, distro) if checker_result: data[distro]["out_of_sync_since"] = None elif data[distro]["out_of_sync_since"] is None: diff --git a/openbsd.py b/openbsd.py index 75a3709..8b2122f 100644 --- a/openbsd.py +++ b/openbsd.py @@ -3,19 +3,6 @@ Contains OpenBSD class """ from distro import Distro -from shared import CSC_MIRROR, get_sec class OpenBSD(Distro): """OpenBSD class""" - @staticmethod - def name(): - """Get name of OpenBSD""" - return "OpenBSD" - - @staticmethod - def check(): - """Check if OpenBSD packages are up-to-date""" - official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp") - csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp") - # Out-of-sync by 1 day maximum - return official_sec < csc_sec + 86400 diff --git a/shared.py b/shared.py index 9f3ba69..9de4e7a 100644 --- a/shared.py +++ b/shared.py @@ -1,10 +1,3 @@ -"""Contains shared constants and functions""" - -import requests +"""Contains shared constants""" CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/" - -def get_sec(timestamp_file_url): - """Get seconds since the Epoch from timestamp file""" - sec_str = requests.get(timestamp_file_url).text - return int(sec_str)