Changed GNOME to automatically check for latest version and refactored code into classes

This commit is contained in:
Laura Nguyen 2021-09-04 12:03:18 -04:00
parent 4022aedc6c
commit 3031db966b
30 changed files with 264 additions and 221 deletions

8
almalinux.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains AlmaLinux class
"""
from distro import Distro
class AlmaLinux(Distro):
"""AlmaLinux class"""

8
alpine.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains Alpine class
"""
from distro import Distro
class Alpine(Distro):
"""Alpine class"""

8
apache.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains Apache class
"""
from distro import Distro
class Apache(Distro):
"""Apache class"""

20
arch.py
View File

@ -2,27 +2,7 @@
Contains Arch class
"""
from datetime import datetime, timedelta
import requests
from distro import Distro
from shared import CSC_MIRROR
class Arch(Distro):
"""Arch class"""
@staticmethod
def name():
"""Get name of Arch"""
return "Arch"
@staticmethod
def check():
"""Check if Arch packages are up-to-date"""
arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json()
last_check_str = arch_json["last_check"]
last_sync_str = [url for url in arch_json["urls"] if url["url"] == \
f"{CSC_MIRROR}archlinux/"][0]["last_sync"]
last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ")
last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ")
# According to https://archlinux.org/mirrors/status/:
# Due to the timing of mirror checks, any value under one hour should be viewed as ideal
return last_check_dt < last_sync_dt + timedelta(hours = 1)

8
centos.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains CentOS class
"""
from distro import Distro
class CentOS(Distro):
"""CentOS class"""

13
ceph.py
View File

@ -3,19 +3,6 @@ Contains Ceph class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class Ceph(Distro):
"""Ceph class"""
@staticmethod
def name():
"""Get name of Ceph"""
return "Ceph"
@staticmethod
def check():
"""Check if Ceph packages are up-to-date"""
official_sec = get_sec("https://download.ceph.com/timestamp")
csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp")
# Out-of-sync by 1 day maximum
return official_sec < csc_sec + 86400

17
cpan.py Normal file
View File

@ -0,0 +1,17 @@
"""
Contains CPAN class
"""
import requests
from distro import Distro
from shared import CSC_MIRROR
class CPAN(Distro):
"""CPAN class"""
@staticmethod
def check(data, distro, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
return False

8
cygwin.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains Cygwin class
"""
from distro import Distro
class Cygwin(Distro):
"""Cygwin class"""

View File

@ -125,9 +125,12 @@
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"upstream1": "https://mirrors.dotsrc.org/",
"upstream2": "https://muug.ca/mirror/",
"file": "gnome/core/41/41.beta/cache.json"
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,

View File

@ -2,30 +2,7 @@
Contains Debian class
"""
from datetime import datetime, timedelta
import requests
from distro import Distro
from shared import CSC_MIRROR
class Debian(Distro):
"""Debian class"""
@staticmethod
def __get_dt(trace_file_url):
"""Get Debian datetime object from trace file"""
response_text = requests.get(trace_file_url).text
time_str = response_text.split('\n')[0]
return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y")
@staticmethod
def name():
"""Get name of Debian"""
return "Debian"
@staticmethod
def check():
"""Check if Debian packages are up-to-date"""
official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master")
csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master")
# Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html
# green and not yellow
return official_dt < csc_dt + timedelta(hours = 7)

8
debiancd.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains DebianCD class
"""
from distro import Distro
class DebianCD(Distro):
"""DebianCD class"""

8
debianmultimedia.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains DebianMultimedia class
"""
from distro import Distro
class DebianMultimedia(Distro):
"""DebianMultimedia class"""

8
debianports.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains DebianPorts class
"""
from distro import Distro
class DebianPorts(Distro):
"""DebianPorts class"""

8
debiansecurity.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains DebianSecurity class
"""
from distro import Distro
class DebianSecurity(Distro):
"""DebianSecurity class"""

View File

@ -2,24 +2,15 @@
Contains abstract class for a distro
"""
from abc import ABC, abstractmethod
from abc import ABC
import requests
from shared import CSC_MIRROR
class Distro(ABC):
"""Abstract class for a distro"""
@staticmethod
@abstractmethod
def name():
"""Get name of distro"""
raise NotImplementedError
@staticmethod
@abstractmethod
def check():
def check(data, distro):
"""Check if distro packages are up-to-date"""
raise NotImplementedError
@classmethod
def print_output(cls, is_successful):
"""Print final output of distro"""
output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync"
print(output.format(cls.name()))
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

View File

@ -3,19 +3,6 @@ Contains Eclipse class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class Eclipse(Distro):
"""Eclipse class"""
@staticmethod
def name():
"""Get name of Eclipse"""
return "Eclipse"
@staticmethod
def check():
"""Check if Eclipse packages are up-to-date"""
official_sec = get_sec("http://download.eclipse.org/TIME")
csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME")
# Out-of-sync by 2 days maximum
return official_sec < csc_sec + 172800

8
fedora.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains Fedora class
"""
from distro import Distro
class Fedora(Distro):
"""Fedora class"""

8
freebsd.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains FreeBSD class
"""
from distro import Distro
class FreeBSD(Distro):
"""FreeBSD class"""

8
gentoodistfiles.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains GentooDistfiles class
"""
from distro import Distro
class GentooDistfiles(Distro):
"""GentooDistfiles class"""

29
gentooportage.py Normal file
View File

@ -0,0 +1,29 @@
"""
Contains GentooPortage class
"""
import os
from distro import Distro
class GentooPortage(Distro):
"""GentooPortage class"""
@staticmethod
def check(data, distro):
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data[distro]["csc"],
data[distro]["file"],
"csc_manifest"))
os.system(rsync_command.format(data[distro]["upstream1"],
data[distro]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data[distro]["upstream2"],
data[distro]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()
stream2 = os.popen("diff csc_manifest upstream_manifest2")
output2 = stream2.read()
os.system("rm csc_manifest")
os.system("rm upstream_manifest1")
os.system("rm upstream_manifest2")
return 0 in [len(output1), len(output2)]

40
gnome.py Normal file
View File

@ -0,0 +1,40 @@
"""
Contains GNOME class
"""
import re
import requests
from distro import Distro
from shared import CSC_MIRROR
class GNOME(Distro):
"""GNOME class"""
@staticmethod
def check(data, distro):
file = data[distro]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+ data[distro]["file2"]).text
try:
ret = csc_text == requests.get(data[distro]["upstream2"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
ret = False
try:
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
return False

13
gnu.py
View File

@ -3,19 +3,6 @@ Contains GNU class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class GNU(Distro):
"""GNU class"""
@staticmethod
def name():
"""Get name of GNU"""
return "GNU"
@staticmethod
def check():
"""Check if GNU packages are up-to-date"""
official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt")
csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt")
# Out-of-sync by 1 day maximum
return official_sec < csc_sec + 86400

8
gutenberg.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains Gutenberg class
"""
from distro import Distro
class Gutenberg(Distro):
"""Gutenberg class"""

14
ipfire.py Normal file
View File

@ -0,0 +1,14 @@
"""
Contains IPFire class
"""
import requests
from distro import Distro
class IPFire(Distro):
"""IPFire class"""
@staticmethod
def check(data, distro):
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
ipfire_text = requests.get(ipfire_url).text
return ipfire_text.find("The mirror is up") != -1

8
kde.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains KDE class
"""
from distro import Distro
class KDE(Distro):
"""KDE class"""

8
kdeapplicationdata.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains KDEApplicationData class
"""
from distro import Distro
class KDEApplicationData(Distro):
"""KDEApplicationData class"""

View File

@ -2,29 +2,7 @@
Contains Kernel class
"""
import requests
from distro import Distro
from shared import CSC_MIRROR
class Kernel(Distro):
"""Kernel class"""
@staticmethod
def __get_line_count(checksum_file_url):
"""Get Kernel line count of checksum file"""
response_text = requests.get(checksum_file_url).text
return len(response_text.split('\n'))
@staticmethod
def name():
"""Get name of Kernel"""
return "Kernel"
@staticmethod
def check():
"""Check if Kernel packages are up-to-date"""
official_line_count = Kernel.__get_line_count(
"https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc")
csc_line_count = Kernel.__get_line_count(
f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc")
# A new update on a certain day adds 2 new lines
return official_line_count <= csc_line_count + 2

107
main.py
View File

@ -5,16 +5,35 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
"""
import time
import os
import sys
import requests
from almalinux import AlmaLinux
from alpine import Alpine
from apache import Apache
from arch import Arch
from centos import CentOS
from ceph import Ceph
from cpan import CPAN
from cygwin import Cygwin
from debian import Debian
from debiancd import DebianCD
from debianmultimedia import DebianMultimedia
from debianports import DebianPorts
from debiansecurity import DebianSecurity
from eclipse import Eclipse
from fedora import Fedora
from freebsd import FreeBSD
from gentoodistfiles import GentooDistfiles
from gentooportage import GentooPortage
from gnome import GNOME
from gnu import GNU
from gutenberg import Gutenberg
from ipfire import IPFire
from kde import KDE
from kdeapplicationdata import KDEApplicationData
from kernel import Kernel
from openbsd import OpenBSD
from shared import CSC_MIRROR
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
@ -22,8 +41,6 @@ import json # import json to read distro info stored in json file
import datefinder # another date finding library
CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
def checker(directory_URL, file_name):
page = requests.get(directory_URL).text
indexOfFile = page.find(file_name)
@ -50,53 +67,7 @@ def checker(directory_URL, file_name):
else:
return('No dates found')
def gentoo_portage_checker(data_json, distro_name):
"""GentooPortage checker"""
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data_json[distro_name]["csc"],
data_json[distro_name]["file"],
"csc_manifest"))
os.system(rsync_command.format(data_json[distro_name]["upstream1"],
data_json[distro_name]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data_json[distro_name]["upstream2"],
data_json[distro_name]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()
stream2 = os.popen("diff csc_manifest upstream_manifest2")
output2 = stream2.read()
os.system("rm csc_manifest")
os.system("rm upstream_manifest1")
os.system("rm upstream_manifest2")
return 0 in [len(output1), len(output2)]
def gnome_checker(data_json, distro_name):
"""GNOME checker"""
csc_url = CSC_MIRROR + data_json[distro_name]["file"]
upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"]
upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"]
csc_gnome_text = requests.get(csc_url).text
return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text]
def ipfire_checker():
"""IPFire checker"""
ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text
return ipfire_text.find("The mirror is up") != -1
def general_checker(data_json, distro_name):
"""General distro checker"""
csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"]
upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text
if __name__ == "__main__":
"""for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]:
try:
distro.print_output(distro.check())
except requests.exceptions.RequestException as err:
print(f"Error: {distro.name()}\n{err}")"""
"""distros = json.load(open('distros.json',))
print(distros)
@ -114,36 +85,18 @@ if __name__ == "__main__":
current_time = int(time.time())
for distro in distros:
try:
if distro == "CPAN":
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
if current_time - int(mirror["age"]) \
> data[distro]["out_of_sync_interval"]:
print(f"Failure: {distro} out-of-sync")
else:
print(f"Success: {distro} up-to-date")
break
continue
if distro == "GentooPortage":
checker_result = gentoo_portage_checker(data, distro)
elif distro == "GNOME":
gnome_text = requests.get("https://download.gnome.org/core/").text
line_count = len(gnome_text.split('\n'))
# Latest version is currently 41, which has line count of 49
if line_count == 49:
checker_result = gnome_checker(data, distro)
else:
data[distro]["out_of_sync_since"] = None
print(f"Failure: {distro} should check for latest version")
continue
elif distro == "IPFire":
checker_result = ipfire_checker()
elif distro not in data:
if distro not in data:
print(f"Failure: {distro} does not exist")
continue
else:
checker_result = general_checker(data, distro)
distro_class = getattr(sys.modules[__name__], distro)
if distro == "CPAN":
checker_result = distro_class.check(data, distro, current_time)
if checker_result:
print(f"Success: {distro} up-to-date")
else:
print(f"Failure: {distro} out-of-sync")
continue
checker_result = distro_class.check(data, distro)
if checker_result:
data[distro]["out_of_sync_since"] = None
elif data[distro]["out_of_sync_since"] is None:

View File

@ -3,19 +3,6 @@ Contains OpenBSD class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class OpenBSD(Distro):
"""OpenBSD class"""
@staticmethod
def name():
"""Get name of OpenBSD"""
return "OpenBSD"
@staticmethod
def check():
"""Check if OpenBSD packages are up-to-date"""
official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp")
csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp")
# Out-of-sync by 1 day maximum
return official_sec < csc_sec + 86400

View File

@ -1,10 +1,3 @@
"""Contains shared constants and functions"""
import requests
"""Contains shared constants"""
CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
def get_sec(timestamp_file_url):
"""Get seconds since the Epoch from timestamp file"""
sec_str = requests.get(timestamp_file_url).text
return int(sec_str)