Changed GNOME to automatically check for latest version and refactored code into classes

dev
Laura Nguyen 1 year ago
parent 4022aedc6c
commit 3031db966b
  1. 8
      almalinux.py
  2. 8
      alpine.py
  3. 8
      apache.py
  4. 20
      arch.py
  5. 8
      centos.py
  6. 13
      ceph.py
  7. 17
      cpan.py
  8. 8
      cygwin.py
  9. 9
      data.json
  10. 23
      debian.py
  11. 8
      debiancd.py
  12. 8
      debianmultimedia.py
  13. 8
      debianports.py
  14. 8
      debiansecurity.py
  15. 23
      distro.py
  16. 13
      eclipse.py
  17. 8
      fedora.py
  18. 8
      freebsd.py
  19. 8
      gentoodistfiles.py
  20. 29
      gentooportage.py
  21. 40
      gnome.py
  22. 13
      gnu.py
  23. 8
      gutenberg.py
  24. 14
      ipfire.py
  25. 8
      kde.py
  26. 8
      kdeapplicationdata.py
  27. 22
      kernel.py
  28. 105
      main.py
  29. 13
      openbsd.py
  30. 9
      shared.py

@ -0,0 +1,8 @@
"""
Contains AlmaLinux class
"""
from distro import Distro
class AlmaLinux(Distro):
"""AlmaLinux class"""

@ -0,0 +1,8 @@
"""
Contains Alpine class
"""
from distro import Distro
class Alpine(Distro):
"""Alpine class"""

@ -0,0 +1,8 @@
"""
Contains Apache class
"""
from distro import Distro
class Apache(Distro):
"""Apache class"""

@ -2,27 +2,7 @@
Contains Arch class
"""
from datetime import datetime, timedelta
import requests
from distro import Distro
from shared import CSC_MIRROR
class Arch(Distro):
"""Arch class"""
@staticmethod
def name():
"""Get name of Arch"""
return "Arch"
@staticmethod
def check():
"""Check if Arch packages are up-to-date"""
arch_json = requests.get("https://archlinux.org/mirrors/status/json/").json()
last_check_str = arch_json["last_check"]
last_sync_str = [url for url in arch_json["urls"] if url["url"] == \
f"{CSC_MIRROR}archlinux/"][0]["last_sync"]
last_check_dt = datetime.strptime(last_check_str, "%Y-%m-%dT%H:%M:%S.%fZ")
last_sync_dt = datetime.strptime(last_sync_str, "%Y-%m-%dT%H:%M:%SZ")
# According to https://archlinux.org/mirrors/status/:
# Due to the timing of mirror checks, any value under one hour should be viewed as ideal
return last_check_dt < last_sync_dt + timedelta(hours = 1)

@ -0,0 +1,8 @@
"""
Contains CentOS class
"""
from distro import Distro
class CentOS(Distro):
"""CentOS class"""

@ -3,19 +3,6 @@ Contains Ceph class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class Ceph(Distro):
"""Ceph class"""
@staticmethod
def name():
"""Get name of Ceph"""
return "Ceph"
@staticmethod
def check():
"""Check if Ceph packages are up-to-date"""
official_sec = get_sec("https://download.ceph.com/timestamp")
csc_sec = get_sec(f"{CSC_MIRROR}ceph/timestamp")
# Out-of-sync by 1 day maximum
return official_sec < csc_sec + 86400

@ -0,0 +1,17 @@
"""
Contains CPAN class
"""
import requests
from distro import Distro
from shared import CSC_MIRROR
class CPAN(Distro):
"""CPAN class"""
@staticmethod
def check(data, distro, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
return False

@ -0,0 +1,8 @@
"""
Contains Cygwin class
"""
from distro import Distro
class Cygwin(Distro):
"""Cygwin class"""

@ -125,9 +125,12 @@
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"upstream1": "https://mirrors.dotsrc.org/",
"upstream2": "https://muug.ca/mirror/",
"file": "gnome/core/41/41.beta/cache.json"
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,

@ -2,30 +2,7 @@
Contains Debian class
"""
from datetime import datetime, timedelta
import requests
from distro import Distro
from shared import CSC_MIRROR
class Debian(Distro):
"""Debian class"""
@staticmethod
def __get_dt(trace_file_url):
"""Get Debian datetime object from trace file"""
response_text = requests.get(trace_file_url).text
time_str = response_text.split('\n')[0]
return datetime.strptime(time_str, "%a %b %d %H:%M:%S UTC %Y")
@staticmethod
def name():
"""Get name of Debian"""
return "Debian"
@staticmethod
def check():
"""Check if Debian packages are up-to-date"""
official_dt = Debian.__get_dt("https://ftp-master.debian.org/debian/project/trace/master")
csc_dt = Debian.__get_dt(f"{CSC_MIRROR}debian/project/trace/master")
# Keep the table cell at https://mirror-master.debian.org/status/mirror-status.html
# green and not yellow
return official_dt < csc_dt + timedelta(hours = 7)

@ -0,0 +1,8 @@
"""
Contains DebianCD class
"""
from distro import Distro
class DebianCD(Distro):
"""DebianCD class"""

@ -0,0 +1,8 @@
"""
Contains DebianMultimedia class
"""
from distro import Distro
class DebianMultimedia(Distro):
"""DebianMultimedia class"""

@ -0,0 +1,8 @@
"""
Contains DebianPorts class
"""
from distro import Distro
class DebianPorts(Distro):
"""DebianPorts class"""

@ -0,0 +1,8 @@
"""
Contains DebianSecurity class
"""
from distro import Distro
class DebianSecurity(Distro):
"""DebianSecurity class"""

@ -2,24 +2,15 @@
Contains abstract class for a distro
"""
from abc import ABC, abstractmethod
from abc import ABC
import requests
from shared import CSC_MIRROR
class Distro(ABC):
"""Abstract class for a distro"""
@staticmethod
@abstractmethod
def name():
"""Get name of distro"""
raise NotImplementedError
@staticmethod
@abstractmethod
def check():
def check(data, distro):
"""Check if distro packages are up-to-date"""
raise NotImplementedError
@classmethod
def print_output(cls, is_successful):
"""Print final output of distro"""
output = "Success: {} up-to-date" if is_successful else "Failure: {} out-of-sync"
print(output.format(cls.name()))
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

@ -3,19 +3,6 @@ Contains Eclipse class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class Eclipse(Distro):
"""Eclipse class"""
@staticmethod
def name():
"""Get name of Eclipse"""
return "Eclipse"
@staticmethod
def check():
"""Check if Eclipse packages are up-to-date"""
official_sec = get_sec("http://download.eclipse.org/TIME")
csc_sec = get_sec(f"{CSC_MIRROR}eclipse/TIME")
# Out-of-sync by 2 days maximum
return official_sec < csc_sec + 172800

@ -0,0 +1,8 @@
"""
Contains Fedora class
"""
from distro import Distro
class Fedora(Distro):
"""Fedora class"""

@ -0,0 +1,8 @@
"""
Contains FreeBSD class
"""
from distro import Distro
class FreeBSD(Distro):
"""FreeBSD class"""

@ -0,0 +1,8 @@
"""
Contains GentooDistfiles class
"""
from distro import Distro
class GentooDistfiles(Distro):
"""GentooDistfiles class"""

@ -0,0 +1,29 @@
"""
Contains GentooPortage class
"""
import os
from distro import Distro
class GentooPortage(Distro):
"""GentooPortage class"""
@staticmethod
def check(data, distro):
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data[distro]["csc"],
data[distro]["file"],
"csc_manifest"))
os.system(rsync_command.format(data[distro]["upstream1"],
data[distro]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data[distro]["upstream2"],
data[distro]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()
stream2 = os.popen("diff csc_manifest upstream_manifest2")
output2 = stream2.read()
os.system("rm csc_manifest")
os.system("rm upstream_manifest1")
os.system("rm upstream_manifest2")
return 0 in [len(output1), len(output2)]

@ -0,0 +1,40 @@
"""
Contains GNOME class
"""
import re
import requests
from distro import Distro
from shared import CSC_MIRROR
class GNOME(Distro):
"""GNOME class"""
@staticmethod
def check(data, distro):
file = data[distro]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+ data[distro]["file2"]).text
try:
ret = csc_text == requests.get(data[distro]["upstream2"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
ret = False
try:
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
return False

@ -3,19 +3,6 @@ Contains GNU class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class GNU(Distro):
"""GNU class"""
@staticmethod
def name():
"""Get name of GNU"""
return "GNU"
@staticmethod
def check():
"""Check if GNU packages are up-to-date"""
official_sec = get_sec("https://mirrors.kernel.org/gnu/mirror-updated-timestamp.txt")
csc_sec = get_sec(f"{CSC_MIRROR}gnu/mirror-updated-timestamp.txt")
# Out-of-sync by 1 day maximum
return official_sec < csc_sec + 86400

@ -0,0 +1,8 @@
"""
Contains Gutenberg class
"""
from distro import Distro
class Gutenberg(Distro):
"""Gutenberg class"""

@ -0,0 +1,14 @@
"""
Contains IPFire class
"""
import requests
from distro import Distro
class IPFire(Distro):
"""IPFire class"""
@staticmethod
def check(data, distro):
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
ipfire_text = requests.get(ipfire_url).text
return ipfire_text.find("The mirror is up") != -1

@ -0,0 +1,8 @@
"""
Contains KDE class
"""
from distro import Distro
class KDE(Distro):
"""KDE class"""

@ -0,0 +1,8 @@
"""
Contains KDEApplicationData class
"""
from distro import Distro
class KDEApplicationData(Distro):
"""KDEApplicationData class"""

@ -2,29 +2,7 @@
Contains Kernel class
"""
import requests
from distro import Distro
from shared import CSC_MIRROR
class Kernel(Distro):
"""Kernel class"""
@staticmethod
def __get_line_count(checksum_file_url):
"""Get Kernel line count of checksum file"""
response_text = requests.get(checksum_file_url).text
return len(response_text.split('\n'))
@staticmethod
def name():
"""Get name of Kernel"""
return "Kernel"
@staticmethod
def check():
"""Check if Kernel packages are up-to-date"""
official_line_count = Kernel.__get_line_count(
"https://mirrors.edge.kernel.org/pub/linux/kernel/next/sha256sums.asc")
csc_line_count = Kernel.__get_line_count(
f"{CSC_MIRROR}kernel.org/linux/kernel/next/sha256sums.asc")
# A new update on a certain day adds 2 new lines
return official_line_count <= csc_line_count + 2

@ -5,16 +5,35 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
"""
import time
import os
import sys
import requests
from almalinux import AlmaLinux
from alpine import Alpine
from apache import Apache
from arch import Arch
from centos import CentOS
from ceph import Ceph
from cpan import CPAN
from cygwin import Cygwin
from debian import Debian
from debiancd import DebianCD
from debianmultimedia import DebianMultimedia
from debianports import DebianPorts
from debiansecurity import DebianSecurity
from eclipse import Eclipse
from fedora import Fedora
from freebsd import FreeBSD
from gentoodistfiles import GentooDistfiles
from gentooportage import GentooPortage
from gnome import GNOME
from gnu import GNU
from gutenberg import Gutenberg
from ipfire import IPFire
from kde import KDE
from kdeapplicationdata import KDEApplicationData
from kernel import Kernel
from openbsd import OpenBSD
from shared import CSC_MIRROR
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
@ -22,8 +41,6 @@ import json # import json to read distro info stored in json file
import datefinder # another date finding library
CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
def checker(directory_URL, file_name):
page = requests.get(directory_URL).text
indexOfFile = page.find(file_name)
@ -50,53 +67,7 @@ def checker(directory_URL, file_name):
else:
return('No dates found')
def gentoo_portage_checker(data_json, distro_name):
"""GentooPortage checker"""
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data_json[distro_name]["csc"],
data_json[distro_name]["file"],
"csc_manifest"))
os.system(rsync_command.format(data_json[distro_name]["upstream1"],
data_json[distro_name]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data_json[distro_name]["upstream2"],
data_json[distro_name]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()
stream2 = os.popen("diff csc_manifest upstream_manifest2")
output2 = stream2.read()
os.system("rm csc_manifest")
os.system("rm upstream_manifest1")
os.system("rm upstream_manifest2")
return 0 in [len(output1), len(output2)]
def gnome_checker(data_json, distro_name):
"""GNOME checker"""
csc_url = CSC_MIRROR + data_json[distro_name]["file"]
upstream_url1 = data_json[distro_name]["upstream1"] + data_json[distro_name]["file"]
upstream_url2 = data_json[distro_name]["upstream2"] + data_json[distro_name]["file"]
csc_gnome_text = requests.get(csc_url).text
return csc_gnome_text in [requests.get(upstream_url1).text, requests.get(upstream_url2).text]
def ipfire_checker():
"""IPFire checker"""
ipfire_text = requests.get("https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca").text
return ipfire_text.find("The mirror is up") != -1
def general_checker(data_json, distro_name):
"""General distro checker"""
csc_url = CSC_MIRROR + data_json[distro_name]["csc"] + data_json[distro_name]["file"]
upstream_url = data_json[distro_name]["upstream"] + data_json[distro_name]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text
if __name__ == "__main__":
"""for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]:
try:
distro.print_output(distro.check())
except requests.exceptions.RequestException as err:
print(f"Error: {distro.name()}\n{err}")"""
"""distros = json.load(open('distros.json',))
print(distros)
@ -114,36 +85,18 @@ if __name__ == "__main__":
current_time = int(time.time())
for distro in distros:
try:
if distro == "CPAN":
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
if current_time - int(mirror["age"]) \
> data[distro]["out_of_sync_interval"]:
print(f"Failure: {distro} out-of-sync")
else:
print(f"Success: {distro} up-to-date")
break
if distro not in data:
print(f"Failure: {distro} does not exist")
continue
if distro == "GentooPortage":
checker_result = gentoo_portage_checker(data, distro)
elif distro == "GNOME":
gnome_text = requests.get("https://download.gnome.org/core/").text
line_count = len(gnome_text.split('\n'))
# Latest version is currently 41, which has line count of 49
if line_count == 49:
checker_result = gnome_checker(data, distro)
distro_class = getattr(sys.modules[__name__], distro)
if distro == "CPAN":
checker_result = distro_class.check(data, distro, current_time)
if checker_result:
print(f"Success: {distro} up-to-date")
else:
data[distro]["out_of_sync_since"] = None
print(f"Failure: {distro} should check for latest version")
continue
elif distro == "IPFire":
checker_result = ipfire_checker()
elif distro not in data:
print(f"Failure: {distro} does not exist")
print(f"Failure: {distro} out-of-sync")
continue
else:
checker_result = general_checker(data, distro)
checker_result = distro_class.check(data, distro)
if checker_result:
data[distro]["out_of_sync_since"] = None
elif data[distro]["out_of_sync_since"] is None:

@ -3,19 +3,6 @@ Contains OpenBSD class
"""
from distro import Distro
from shared import CSC_MIRROR, get_sec
class OpenBSD(Distro):
"""OpenBSD class"""
@staticmethod
def name():
"""Get name of OpenBSD"""
return "OpenBSD"
@staticmethod
def check():
"""Check if OpenBSD packages are up-to-date"""
official_sec = get_sec("https://ftp.openbsd.org/pub/OpenBSD/timestamp")
csc_sec = get_sec(f"{CSC_MIRROR}OpenBSD/timestamp")
# Out-of-sync by 1 day maximum
return official_sec < csc_sec + 86400

@ -1,10 +1,3 @@
"""Contains shared constants and functions"""
import requests
"""Contains shared constants"""
CSC_MIRROR = "http://mirror.csclub.uwaterloo.ca/"
def get_sec(timestamp_file_url):
"""Get seconds since the Epoch from timestamp file"""
sec_str = requests.get(timestamp_file_url).text
return int(sec_str)

Loading…
Cancel
Save