Rename distro to project

This commit is contained in:
Raymond Li 2021-10-03 15:44:08 -04:00
parent 3428e81cec
commit ccc11f07df
Signed by: r389li
GPG Key ID: A014EA89B62BBB1B
32 changed files with 122 additions and 122 deletions

2
.gitignore vendored
View File

@ -139,4 +139,4 @@ cython_debug/
# Idea # Idea
.idea/ .idea/
*.txt~ *~

View File

@ -16,7 +16,7 @@ if we can just view their repo online, we only have to remember the link for the
even if the date relies on a specific file in their repo, we can still find the right link for it even if the date relies on a specific file in their repo, we can still find the right link for it
to find repos of the distros to check, just search "distroName mirrors" to find repos of the mirrored projects to check, just search "projectName mirrors"
ubuntu releases: https://releases.ubuntu.com/?_ga=2.251394307.425973732.1629764407-1388600952.1629764407 ubuntu releases: https://releases.ubuntu.com/?_ga=2.251394307.425973732.1629764407-1388600952.1629764407
ubuntu mirrors: http://mirrors.ubuntu.com/mirrors.txt ubuntu mirrors: http://mirrors.ubuntu.com/mirrors.txt

View File

@ -1,20 +0,0 @@
"""
Contains abstract class for a distro
"""
from abc import ABC
import requests
from shared import CSC_MIRROR
class Distro(ABC):
"""Abstract class for a distro"""
@staticmethod
def check(data, distro):
"""Check if distro packages are up-to-date"""
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

52
main.py
View File

@ -37,7 +37,7 @@ from shared import CSC_MIRROR
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/ # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
import re # import regular expressions to remove stray numbers in string that might interfere with date finding import re # import regular expressions to remove stray numbers in string that might interfere with date finding
import json # import json to read distro info stored in json file import json # import json to read project info stored in json file
import datefinder # another date finding library import datefinder # another date finding library
@ -70,45 +70,45 @@ def checker(directory_URL, file_name):
if __name__ == "__main__": if __name__ == "__main__":
"""distros = json.load(open('distros.json',)) """projects = json.load(open('projects.json',))
print(distros) print(projects)
for distro in distros: for project in projects:
print(distro[0] + ":") print(project[0] + ":")
print("CSC mirror: " + checker(distro[1], distro[3])) print("CSC mirror: " + checker(project[1], project[3]))
print("Official distro: " + checker(distro[2], distro[3]))""" print("Official project: " + checker(project[2], project[3]))"""
with open("data.json", "r", encoding="utf-8") as file: with open("data.json", "r", encoding="utf-8") as file:
data = json.load(file) data = json.load(file)
if sys.stdin.isatty(): if sys.stdin.isatty():
distros = data projects = data
else: else:
distros = [distro.rstrip() for distro in sys.stdin.readlines()] projects = [project.rstrip() for project in sys.stdin.readlines()]
current_time = int(time.time()) current_time = int(time.time())
for distro in distros: for project in projects:
try: try:
if distro not in data: if project not in data:
print(f"Failure: {distro} does not exist") print(f"Failure: {project} does not exist")
continue continue
distro_class = getattr(sys.modules[__name__], distro) project_class = getattr(sys.modules[__name__], project)
if distro == "CPAN": if project == "CPAN":
checker_result = distro_class.check(data, distro, current_time) checker_result = project_class.check(data, project, current_time)
if checker_result: if checker_result:
print(f"Success: {distro} up-to-date") print(f"Success: {project} up-to-date")
else: else:
print(f"Failure: {distro} out-of-sync") print(f"Failure: {project} out-of-sync")
continue continue
checker_result = distro_class.check(data, distro) checker_result = project_class.check(data, project)
if checker_result: if checker_result:
data[distro]["out_of_sync_since"] = None data[project]["out_of_sync_since"] = None
elif data[distro]["out_of_sync_since"] is None: elif data[project]["out_of_sync_since"] is None:
data[distro]["out_of_sync_since"] = current_time data[project]["out_of_sync_since"] = current_time
elif current_time - data[distro]["out_of_sync_since"] \ elif current_time - data[project]["out_of_sync_since"] \
> data[distro]["out_of_sync_interval"]: > data[project]["out_of_sync_interval"]:
print(f"Failure: {distro} out-of-sync") print(f"Failure: {project} out-of-sync")
continue continue
print(f"Success: {distro} up-to-date") print(f"Success: {project} up-to-date")
except requests.exceptions.RequestException as err: except requests.exceptions.RequestException as err:
print(f"Error: {distro}\n{err}") print(f"Error: {project}\n{err}")
with open("data.json", "w", encoding="utf-8") as file: with open("data.json", "w", encoding="utf-8") as file:
json.dump(data, file, indent=4) json.dump(data, file, indent=4)

20
project.py Normal file
View File

@ -0,0 +1,20 @@
"""
Contains abstract class for a mirrored project
"""
from abc import ABC
import requests
from shared import CSC_MIRROR
class Project(ABC):
"""Abstract class for a mirrored project"""
@staticmethod
def check(data, project):
"""Check if project packages are up-to-date"""
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

View File

@ -2,8 +2,8 @@
Contains AlmaLinux class Contains AlmaLinux class
""" """
from distro import Distro from project import Project
class AlmaLinux(Distro): class AlmaLinux(Project):
"""AlmaLinux class""" """AlmaLinux class"""

View File

@ -2,8 +2,8 @@
Contains Alpine class Contains Alpine class
""" """
from distro import Distro from project import Project
class Alpine(Distro): class Alpine(Project):
"""Alpine class""" """Alpine class"""

View File

@ -2,8 +2,8 @@
Contains Apache class Contains Apache class
""" """
from distro import Distro from project import Project
class Apache(Distro): class Apache(Project):
"""Apache class""" """Apache class"""

View File

@ -2,8 +2,8 @@
Contains Arch class Contains Arch class
""" """
from distro import Distro from project import Project
class Arch(Distro): class Arch(Project):
"""Arch class""" """Arch class"""

View File

@ -2,8 +2,8 @@
Contains CentOS class Contains CentOS class
""" """
from distro import Distro from project import Project
class CentOS(Distro): class CentOS(Project):
"""CentOS class""" """CentOS class"""

View File

@ -2,8 +2,8 @@
Contains Ceph class Contains Ceph class
""" """
from distro import Distro from project import Project
class Ceph(Distro): class Ceph(Project):
"""Ceph class""" """Ceph class"""

View File

@ -4,17 +4,17 @@ Contains CPAN class
import requests import requests
from distro import Distro from project import Project
from shared import CSC_MIRROR from shared import CSC_MIRROR
class CPAN(Distro): class CPAN(Project):
"""CPAN class""" """CPAN class"""
@staticmethod @staticmethod
def check(data, distro, current_time): def check(data, project, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json: for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/": if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"] return current_time - int(mirror["age"]) <= data[project]["out_of_sync_interval"]
return False return False

View File

@ -2,8 +2,8 @@
Contains Cygwin class Contains Cygwin class
""" """
from distro import Distro from project import Project
class Cygwin(Distro): class Cygwin(Project):
"""Cygwin class""" """Cygwin class"""

View File

@ -2,8 +2,8 @@
Contains Debian class Contains Debian class
""" """
from distro import Distro from project import Project
class Debian(Distro): class Debian(Project):
"""Debian class""" """Debian class"""

View File

@ -2,8 +2,8 @@
Contains DebianCD class Contains DebianCD class
""" """
from distro import Distro from project import Project
class DebianCD(Distro): class DebianCD(Project):
"""DebianCD class""" """DebianCD class"""

View File

@ -2,8 +2,8 @@
Contains DebianMultimedia class Contains DebianMultimedia class
""" """
from distro import Distro from project import Project
class DebianMultimedia(Distro): class DebianMultimedia(Project):
"""DebianMultimedia class""" """DebianMultimedia class"""

View File

@ -2,8 +2,8 @@
Contains DebianPorts class Contains DebianPorts class
""" """
from distro import Distro from project import Project
class DebianPorts(Distro): class DebianPorts(Project):
"""DebianPorts class""" """DebianPorts class"""

View File

@ -2,8 +2,8 @@
Contains DebianSecurity class Contains DebianSecurity class
""" """
from distro import Distro from project import Project
class DebianSecurity(Distro): class DebianSecurity(Project):
"""DebianSecurity class""" """DebianSecurity class"""

View File

@ -2,8 +2,8 @@
Contains Eclipse class Contains Eclipse class
""" """
from distro import Distro from project import Project
class Eclipse(Distro): class Eclipse(Project):
"""Eclipse class""" """Eclipse class"""

View File

@ -2,8 +2,8 @@
Contains Fedora class Contains Fedora class
""" """
from distro import Distro from project import Project
class Fedora(Distro): class Fedora(Project):
"""Fedora class""" """Fedora class"""

View File

@ -2,8 +2,8 @@
Contains FreeBSD class Contains FreeBSD class
""" """
from distro import Distro from project import Project
class FreeBSD(Distro): class FreeBSD(Project):
"""FreeBSD class""" """FreeBSD class"""

View File

@ -2,8 +2,8 @@
Contains GentooDistfiles class Contains GentooDistfiles class
""" """
from distro import Distro from project import Project
class GentooDistfiles(Distro): class GentooDistfiles(Project):
"""GentooDistfiles class""" """GentooDistfiles class"""

View File

@ -4,23 +4,23 @@ Contains GentooPortage class
import os import os
from distro import Distro from project import Project
class GentooPortage(Distro): class GentooPortage(Project):
"""GentooPortage class""" """GentooPortage class"""
@staticmethod @staticmethod
def check(data, distro): def check(data, project):
rsync_command = "rsync -q {}{} {}" rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data[distro]["csc"], os.system(rsync_command.format(data[project]["csc"],
data[distro]["file"], data[project]["file"],
"csc_manifest")) "csc_manifest"))
os.system(rsync_command.format(data[distro]["upstream1"], os.system(rsync_command.format(data[project]["upstream1"],
data[distro]["file"], data[project]["file"],
"upstream_manifest1")) "upstream_manifest1"))
os.system(rsync_command.format(data[distro]["upstream2"], os.system(rsync_command.format(data[project]["upstream2"],
data[distro]["file"], data[project]["file"],
"upstream_manifest2")) "upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1") stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read() output1 = stream1.read()

View File

@ -6,39 +6,39 @@ import re
import requests import requests
from distro import Distro from project import Project
from shared import CSC_MIRROR from shared import CSC_MIRROR
class GNOME(Distro): class GNOME(Project):
"""GNOME class""" """GNOME class"""
@staticmethod @staticmethod
def check(data, distro): def check(data, project):
file = data[distro]["file1"] file = data[project]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text csc_versions = requests.get(CSC_MIRROR + data[project]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text upstream_versions = requests.get(data[project]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"') csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"') upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest: if csc_latest != upstream_latest:
return False return False
file += csc_latest + "/" file += csc_latest + "/"
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text csc_versions = requests.get(CSC_MIRROR + data[project]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text upstream_versions = requests.get(data[project]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"') csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"') upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest: if csc_latest != upstream_latest:
return False return False
file += csc_latest + "/" file += csc_latest + "/"
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file csc_text = requests.get(CSC_MIRROR + data[project]["csc"] + file
+ data[distro]["file2"]).text + data[project]["file2"]).text
try: try:
ret = csc_text == requests.get(data[distro]["upstream2"] + file ret = csc_text == requests.get(data[project]["upstream2"] + file
+ data[distro]["file2"]).text + data[project]["file2"]).text
except requests.exceptions.RequestException: except requests.exceptions.RequestException:
ret = False ret = False
try: try:
return ret or csc_text == requests.get(data[distro]["upstream3"] + file return ret or csc_text == requests.get(data[project]["upstream3"] + file
+ data[distro]["file2"]).text + data[project]["file2"]).text
except requests.exceptions.RequestException: except requests.exceptions.RequestException:
return False return False

View File

@ -2,8 +2,8 @@
Contains GNU class Contains GNU class
""" """
from distro import Distro from project import Project
class GNU(Distro): class GNU(Project):
"""GNU class""" """GNU class"""

View File

@ -2,8 +2,8 @@
Contains Gutenberg class Contains Gutenberg class
""" """
from distro import Distro from project import Project
class Gutenberg(Distro): class Gutenberg(Project):
"""Gutenberg class""" """Gutenberg class"""

View File

@ -4,14 +4,14 @@ Contains IPFire class
import requests import requests
from distro import Distro from project import Project
class IPFire(Distro): class IPFire(Project):
"""IPFire class""" """IPFire class"""
@staticmethod @staticmethod
def check(data, distro): def check(data, project):
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca" ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
ipfire_text = requests.get(ipfire_url).text ipfire_text = requests.get(ipfire_url).text
return ipfire_text.find("The mirror is up") != -1 return ipfire_text.find("The mirror is up") != -1

View File

@ -2,8 +2,8 @@
Contains KDE class Contains KDE class
""" """
from distro import Distro from project import Project
class KDE(Distro): class KDE(Project):
"""KDE class""" """KDE class"""

View File

@ -2,8 +2,8 @@
Contains KDEApplicationData class Contains KDEApplicationData class
""" """
from distro import Distro from project import Project
class KDEApplicationData(Distro): class KDEApplicationData(Project):
"""KDEApplicationData class""" """KDEApplicationData class"""

View File

@ -2,8 +2,8 @@
Contains Kernel class Contains Kernel class
""" """
from distro import Distro from project import Project
class Kernel(Distro): class Kernel(Project):
"""Kernel class""" """Kernel class"""

View File

@ -2,8 +2,8 @@
Contains OpenBSD class Contains OpenBSD class
""" """
from distro import Distro from project import Project
class OpenBSD(Distro): class OpenBSD(Project):
"""OpenBSD class""" """OpenBSD class"""