Refactor and use requirements file

dev
Raymond Li 1 year ago
parent 6b0d3f4de9
commit 3428e81cec
Signed by untrusted user: r389li
GPG Key ID: A014EA89B62BBB1B
  1. 1
      .gitignore
  2. 17
      cpan.py
  3. 358
      data.json
  4. 18
      distro.py
  5. 23
      distros.json
  6. 29
      gentooportage.py
  7. 40
      gnome.py
  8. 14
      ipfire.py
  9. 182
      main.py
  10. 3
      projects/almalinux.py
  11. 3
      projects/alpine.py
  12. 3
      projects/apache.py
  13. 3
      projects/arch.py
  14. 3
      projects/centos.py
  15. 3
      projects/ceph.py
  16. 20
      projects/cpan.py
  17. 3
      projects/cygwin.py
  18. 3
      projects/debian.py
  19. 3
      projects/debiancd.py
  20. 3
      projects/debianmultimedia.py
  21. 3
      projects/debianports.py
  22. 3
      projects/debiansecurity.py
  23. 3
      projects/eclipse.py
  24. 3
      projects/fedora.py
  25. 3
      projects/freebsd.py
  26. 3
      projects/gentoodistfiles.py
  27. 32
      projects/gentooportage.py
  28. 44
      projects/gnome.py
  29. 3
      projects/gnu.py
  30. 3
      projects/gutenberg.py
  31. 17
      projects/ipfire.py
  32. 3
      projects/kde.py
  33. 3
      projects/kdeapplicationdata.py
  34. 3
      projects/kernel.py
  35. 3
      projects/openbsd.py
  36. 4
      requirements.txt

1
.gitignore vendored

@ -139,3 +139,4 @@ cython_debug/
# Idea
.idea/
*.txt~

@ -1,17 +0,0 @@
"""
Contains CPAN class
"""
import requests
from distro import Distro
from shared import CSC_MIRROR
class CPAN(Distro):
"""CPAN class"""
@staticmethod
def check(data, distro, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
return False

@ -1,181 +1,181 @@
{
"AlmaLinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME"
},
"Alpine": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated"
},
"Apache": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "apache/",
"upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt"
},
"Arch": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate"
},
"CentOS": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME"
},
"Ceph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ceph/",
"upstream": "https://download.ceph.com/",
"file": "timestamp"
},
"CPAN": {
"out_of_sync_interval": 172800
},
"Cygwin": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "cygwin/",
"upstream": "https://cygwin.mirror.globo.tech/",
"file": "x86/sha512.sum"
},
"Debian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master"
},
"DebianCD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org"
},
"DebianMultimedia": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org"
},
"DebianPorts": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org"
},
"DebianSecurity": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.iweb.ca/",
"file": "debian-security/project/trace/master"
},
"Eclipse": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "eclipse/",
"upstream": "http://download.eclipse.org/",
"file": "TIME"
},
"Fedora": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID"
},
"FreeBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP"
},
"GentooDistfiles": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gentoo-distfiles",
"upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local"
},
"GentooPortage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/",
"upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest"
},
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt"
},
"Gutenberg": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs"
},
"IPFire": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800
},
"KDE": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR"
},
"KDEApplicationData": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/",
"file": "last-updated"
},
"Kernel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc"
},
"OpenBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp"
}
"AlmaLinux": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME"
},
"Alpine": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated"
},
"Apache": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "apache/",
"upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt"
},
"Arch": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate"
},
"CentOS": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME"
},
"Ceph": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "ceph/",
"upstream": "https://download.ceph.com/",
"file": "timestamp"
},
"CPAN": {
"out_of_sync_interval": 172800
},
"Cygwin": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "cygwin/",
"upstream": "https://cygwin.mirror.globo.tech/",
"file": "x86/sha512.sum"
},
"Debian": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master"
},
"DebianCD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org"
},
"DebianMultimedia": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org"
},
"DebianPorts": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org"
},
"DebianSecurity": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://debian.mirror.iweb.ca/",
"file": "debian-security/project/trace/master"
},
"Eclipse": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "eclipse/",
"upstream": "http://download.eclipse.org/",
"file": "TIME"
},
"Fedora": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID"
},
"FreeBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP"
},
"GentooDistfiles": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gentoo-distfiles",
"upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local"
},
"GentooPortage": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/",
"upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest"
},
"GNOME": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "gnome/",
"upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/",
"file2": "cache.json"
},
"GNU": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt"
},
"Gutenberg": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800,
"csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs"
},
"IPFire": {
"out_of_sync_since": null,
"out_of_sync_interval": 172800
},
"KDE": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR"
},
"KDEApplicationData": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/",
"file": "last-updated"
},
"Kernel": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc"
},
"OpenBSD": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp"
}
}

@ -3,14 +3,18 @@ Contains abstract class for a distro
"""
from abc import ABC
import requests
from shared import CSC_MIRROR
class Distro(ABC):
"""Abstract class for a distro"""
@staticmethod
def check(data, distro):
"""Check if distro packages are up-to-date"""
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text
"""Abstract class for a distro"""
@staticmethod
def check(data, distro):
"""Check if distro packages are up-to-date"""
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

@ -1,5 +1,20 @@
[
["OpenBSD", "https://mirror.csclub.uwaterloo.ca/OpenBSD/", "https://ftp.openbsd.org/pub/OpenBSD/", "timestamp"],
["kernel", "http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/", "https://mirrors.edge.kernel.org/pub/linux/kernel/next/", "sha256sums.asc"],
["debian", "http://mirror.csclub.uwaterloo.ca/debian/project/trace/", "https://ftp-master.debian.org/debian/project/trace/", "master"]
]
[
"OpenBSD",
"https://mirror.csclub.uwaterloo.ca/OpenBSD/",
"https://ftp.openbsd.org/pub/OpenBSD/",
"timestamp"
],
[
"kernel",
"http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/",
"https://mirrors.edge.kernel.org/pub/linux/kernel/next/",
"sha256sums.asc"
],
[
"debian",
"http://mirror.csclub.uwaterloo.ca/debian/project/trace/",
"https://ftp-master.debian.org/debian/project/trace/",
"master"
]
]

@ -1,29 +0,0 @@
"""
Contains GentooPortage class
"""
import os
from distro import Distro
class GentooPortage(Distro):
"""GentooPortage class"""
@staticmethod
def check(data, distro):
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data[distro]["csc"],
data[distro]["file"],
"csc_manifest"))
os.system(rsync_command.format(data[distro]["upstream1"],
data[distro]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data[distro]["upstream2"],
data[distro]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()
stream2 = os.popen("diff csc_manifest upstream_manifest2")
output2 = stream2.read()
os.system("rm csc_manifest")
os.system("rm upstream_manifest1")
os.system("rm upstream_manifest2")
return 0 in [len(output1), len(output2)]

@ -1,40 +0,0 @@
"""
Contains GNOME class
"""
import re
import requests
from distro import Distro
from shared import CSC_MIRROR
class GNOME(Distro):
"""GNOME class"""
@staticmethod
def check(data, distro):
file = data[distro]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+ data[distro]["file2"]).text
try:
ret = csc_text == requests.get(data[distro]["upstream2"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
ret = False
try:
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
return False

@ -1,14 +0,0 @@
"""
Contains IPFire class
"""
import requests
from distro import Distro
class IPFire(Distro):
"""IPFire class"""
@staticmethod
def check(data, distro):
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
ipfire_text = requests.get(ipfire_url).text
return ipfire_text.find("The mirror is up") != -1

@ -7,106 +7,108 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
import time
import sys
import requests
from almalinux import AlmaLinux
from alpine import Alpine
from apache import Apache
from arch import Arch
from centos import CentOS
from ceph import Ceph
from cpan import CPAN
from cygwin import Cygwin
from debian import Debian
from debiancd import DebianCD
from debianmultimedia import DebianMultimedia
from debianports import DebianPorts
from debiansecurity import DebianSecurity
from eclipse import Eclipse
from fedora import Fedora
from freebsd import FreeBSD
from gentoodistfiles import GentooDistfiles
from gentooportage import GentooPortage
from gnome import GNOME
from gnu import GNU
from gutenberg import Gutenberg
from ipfire import IPFire
from kde import KDE
from kdeapplicationdata import KDEApplicationData
from kernel import Kernel
from openbsd import OpenBSD
from projects.almalinux import AlmaLinux
from projects.alpine import Alpine
from projects.apache import Apache
from projects.arch import Arch
from projects.centos import CentOS
from projects.ceph import Ceph
from projects.cpan import CPAN
from projects.cygwin import Cygwin
from projects.debian import Debian
from projects.debiancd import DebianCD
from projects.debianmultimedia import DebianMultimedia
from projects.debianports import DebianPorts
from projects.debiansecurity import DebianSecurity
from projects.eclipse import Eclipse
from projects.fedora import Fedora
from projects.freebsd import FreeBSD
from projects.gentoodistfiles import GentooDistfiles
from projects.gentooportage import GentooPortage
from projects.gnome import GNOME
from projects.gnu import GNU
from projects.gutenberg import Gutenberg
from projects.ipfire import IPFire
from projects.kde import KDE
from projects.kdeapplicationdata import KDEApplicationData
from projects.kernel import Kernel
from projects.openbsd import OpenBSD
from shared import CSC_MIRROR
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
import json # import json to read distro info stored in json file
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
import json # import json to read distro info stored in json file
import datefinder # another date finding library
import datefinder # another date finding library
def checker(directory_URL, file_name):
page = requests.get(directory_URL).text
indexOfFile = page.find(file_name)
# print(page)
page = requests.get(directory_URL).text
indexOfFile = page.find(file_name)
# print(page)
# remove stray numbers (file size numbers in particular) that might interfere with date finding
segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
# print(segment_clean)
# remove stray numbers (file size numbers in particular) that might interfere with date finding
segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
# print(segment_clean)
# implementation using dateparser.search.search_dates
# notes: some dates don't parse correctly with this tool
# print(search_dates(page[indexOfFile:], languages=['en']))
# print(search_dates(page[indexOfFile:])[0])
# implementation using dateparser.search.search_dates
# notes: some dates don't parse correctly with this tool
# print(search_dates(page[indexOfFile:], languages=['en']))
# print(search_dates(page[indexOfFile:])[0])
# finds the dates in the segment after the file name
# notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom.
matches = list(datefinder.find_dates(segment_clean))
# print(matches)
# finds the dates in the segment after the file name
# notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom.
matches = list(datefinder.find_dates(segment_clean))
# print(matches)
if len(matches) > 0:
date = matches[0] # date is of type datetime.datetime
return (date.strftime("%m/%d/%Y, %H:%M:%S"))
else:
return ('No dates found')
if len(matches) > 0:
date = matches[0] # date is of type datetime.datetime
return(date.strftime("%m/%d/%Y, %H:%M:%S"))
else:
return('No dates found')
if __name__ == "__main__":
"""distros = json.load(open('distros.json',))
print(distros)
"""distros = json.load(open('distros.json',))
print(distros)
for distro in distros:
print(distro[0] + ":")
print("CSC mirror: " + checker(distro[1], distro[3]))
print("Official distro: " + checker(distro[2], distro[3]))"""
for distro in distros:
print(distro[0] + ":")
print("CSC mirror: " + checker(distro[1], distro[3]))
print("Official distro: " + checker(distro[2], distro[3]))"""
with open("data.json", "r", encoding = "utf-8") as file:
data = json.load(file)
if sys.stdin.isatty():
distros = data
else:
distros = [distro.rstrip() for distro in sys.stdin.readlines()]
current_time = int(time.time())
for distro in distros:
try:
if distro not in data:
print(f"Failure: {distro} does not exist")
continue
distro_class = getattr(sys.modules[__name__], distro)
if distro == "CPAN":
checker_result = distro_class.check(data, distro, current_time)
if checker_result:
print(f"Success: {distro} up-to-date")
else:
print(f"Failure: {distro} out-of-sync")
continue
checker_result = distro_class.check(data, distro)
if checker_result:
data[distro]["out_of_sync_since"] = None
elif data[distro]["out_of_sync_since"] is None:
data[distro]["out_of_sync_since"] = current_time
elif current_time - data[distro]["out_of_sync_since"] \
> data[distro]["out_of_sync_interval"]:
print(f"Failure: {distro} out-of-sync")
continue
print(f"Success: {distro} up-to-date")
except requests.exceptions.RequestException as err:
print(f"Error: {distro}\n{err}")
with open("data.json", "w", encoding = "utf-8") as file:
json.dump(data, file, indent = 4)
with open("data.json", "r", encoding="utf-8") as file:
data = json.load(file)
if sys.stdin.isatty():
distros = data
else:
distros = [distro.rstrip() for distro in sys.stdin.readlines()]
current_time = int(time.time())
for distro in distros:
try:
if distro not in data:
print(f"Failure: {distro} does not exist")
continue
distro_class = getattr(sys.modules[__name__], distro)
if distro == "CPAN":
checker_result = distro_class.check(data, distro, current_time)
if checker_result:
print(f"Success: {distro} up-to-date")
else:
print(f"Failure: {distro} out-of-sync")
continue
checker_result = distro_class.check(data, distro)
if checker_result:
data[distro]["out_of_sync_since"] = None
elif data[distro]["out_of_sync_since"] is None:
data[distro]["out_of_sync_since"] = current_time
elif current_time - data[distro]["out_of_sync_since"] \
> data[distro]["out_of_sync_interval"]:
print(f"Failure: {distro} out-of-sync")
continue
print(f"Success: {distro} up-to-date")
except requests.exceptions.RequestException as err:
print(f"Error: {distro}\n{err}")
with open("data.json", "w", encoding="utf-8") as file:
json.dump(data, file, indent=4)

@ -4,5 +4,6 @@ Contains AlmaLinux class
from distro import Distro
class AlmaLinux(Distro):
"""AlmaLinux class"""
"""AlmaLinux class"""

@ -4,5 +4,6 @@ Contains Alpine class
from distro import Distro
class Alpine(Distro):
"""Alpine class"""
"""Alpine class"""

@ -4,5 +4,6 @@ Contains Apache class
from distro import Distro
class Apache(Distro):
"""Apache class"""
"""Apache class"""

@ -4,5 +4,6 @@ Contains Arch class
from distro import Distro
class Arch(Distro):
"""Arch class"""
"""Arch class"""

@ -4,5 +4,6 @@ Contains CentOS class
from distro import Distro
class CentOS(Distro):
"""CentOS class"""
"""CentOS class"""

@ -4,5 +4,6 @@ Contains Ceph class
from distro import Distro
class Ceph(Distro):
"""Ceph class"""
"""Ceph class"""

@ -0,0 +1,20 @@
"""
Contains CPAN class
"""
import requests
from distro import Distro
from shared import CSC_MIRROR
class CPAN(Distro):
"""CPAN class"""
@staticmethod
def check(data, distro, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
return False

@ -4,5 +4,6 @@ Contains Cygwin class
from distro import Distro
class Cygwin(Distro):
"""Cygwin class"""
"""Cygwin class"""

@ -4,5 +4,6 @@ Contains Debian class
from distro import Distro
class Debian(Distro):
"""Debian class"""
"""Debian class"""

@ -4,5 +4,6 @@ Contains DebianCD class
from distro import Distro
class DebianCD(Distro):
"""DebianCD class"""
"""DebianCD class"""

@ -4,5 +4,6 @@ Contains DebianMultimedia class
from distro import Distro
class DebianMultimedia(Distro):
"""DebianMultimedia class"""
"""DebianMultimedia class"""

@ -4,5 +4,6 @@ Contains DebianPorts class
from distro import Distro
class DebianPorts(Distro):
"""DebianPorts class"""
"""DebianPorts class"""

@ -4,5 +4,6 @@ Contains DebianSecurity class
from distro import Distro
class DebianSecurity(Distro):
"""DebianSecurity class"""
"""DebianSecurity class"""

@ -4,5 +4,6 @@ Contains Eclipse class
from distro import Distro
class Eclipse(Distro):
"""Eclipse class"""
"""Eclipse class"""

@ -4,5 +4,6 @@ Contains Fedora class
from distro import Distro
class Fedora(Distro):
"""Fedora class"""
"""Fedora class"""

@ -4,5 +4,6 @@ Contains FreeBSD class
from distro import Distro
class FreeBSD(Distro):
"""FreeBSD class"""
"""FreeBSD class"""

@ -4,5 +4,6 @@ Contains GentooDistfiles class
from distro import Distro
class GentooDistfiles(Distro):
"""GentooDistfiles class"""
"""GentooDistfiles class"""

@ -0,0 +1,32 @@
"""
Contains GentooPortage class
"""
import os
from distro import Distro
class GentooPortage(Distro):
"""GentooPortage class"""
@staticmethod
def check(data, distro):
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data[distro]["csc"],
data[distro]["file"],
"csc_manifest"))
os.system(rsync_command.format(data[distro]["upstream1"],
data[distro]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data[distro]["upstream2"],
data[distro]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()
stream2 = os.popen("diff csc_manifest upstream_manifest2")
output2 = stream2.read()
os.system("rm csc_manifest")
os.system("rm upstream_manifest1")
os.system("rm upstream_manifest2")
return 0 in [len(output1), len(output2)]

@ -0,0 +1,44 @@
"""
Contains GNOME class
"""
import re
import requests
from distro import Distro
from shared import CSC_MIRROR
class GNOME(Distro):
"""GNOME class"""
@staticmethod
def check(data, distro):
file = data[distro]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+ data[distro]["file2"]).text
try:
ret = csc_text == requests.get(data[distro]["upstream2"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
ret = False
try:
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+ data[distro]["file2"]).text
except requests.exceptions.RequestException:
return False

@ -4,5 +4,6 @@ Contains GNU class
from distro import Distro
class GNU(Distro):
"""GNU class"""
"""GNU class"""

@ -4,5 +4,6 @@ Contains Gutenberg class
from distro import Distro
class Gutenberg(Distro):
"""Gutenberg class"""
"""Gutenberg class"""

@ -0,0 +1,17 @@
"""
Contains IPFire class
"""
import requests
from distro import Distro
class IPFire(Distro):
"""IPFire class"""
@staticmethod
def check(data, distro):
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
ipfire_text = requests.get(ipfire_url).text
return ipfire_text.find("The mirror is up") != -1

@ -4,5 +4,6 @@ Contains KDE class
from distro import Distro
class KDE(Distro):
"""KDE class"""
"""KDE class"""

@ -4,5 +4,6 @@ Contains KDEApplicationData class
from distro import Distro
class KDEApplicationData(Distro):
"""KDEApplicationData class"""
"""KDEApplicationData class"""

@ -4,5 +4,6 @@ Contains Kernel class
from distro import Distro
class Kernel(Distro):
"""Kernel class"""
"""Kernel class"""

@ -4,5 +4,6 @@ Contains OpenBSD class
from distro import Distro
class OpenBSD(Distro):
"""OpenBSD class"""
"""OpenBSD class"""

@ -0,0 +1,4 @@
requests~=2.26.0
datefinder~=0.7.1
dateparser~=1.0.0
Loading…
Cancel
Save