Rename distro to project

This commit is contained in:
Raymond Li 2021-10-03 15:44:08 -04:00
parent 3428e81cec
commit ccc11f07df
Signed by untrusted user: r389li
GPG Key ID: A014EA89B62BBB1B
32 changed files with 122 additions and 122 deletions

2
.gitignore vendored
View File

@ -139,4 +139,4 @@ cython_debug/
# Idea
.idea/
*.txt~
*~

View File

@ -16,7 +16,7 @@ if we can just view their repo online, we only have to remember the link for the
even if the date relies on a specific file in their repo, we can still find the right link for it
to find repos of the distros to check, just search "distroName mirrors"
to find repos of the mirrored projects to check, just search "projectName mirrors"
ubuntu releases: https://releases.ubuntu.com/?_ga=2.251394307.425973732.1629764407-1388600952.1629764407
ubuntu mirrors: http://mirrors.ubuntu.com/mirrors.txt

View File

@ -1,20 +0,0 @@
"""
Contains abstract class for a distro
"""
from abc import ABC
import requests
from shared import CSC_MIRROR
class Distro(ABC):
"""Abstract class for a distro"""
@staticmethod
def check(data, distro):
"""Check if distro packages are up-to-date"""
csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

52
main.py
View File

@ -37,7 +37,7 @@ from shared import CSC_MIRROR
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
import json # import json to read distro info stored in json file
import json # import json to read project info stored in json file
import datefinder # another date finding library
@ -70,45 +70,45 @@ def checker(directory_URL, file_name):
if __name__ == "__main__":
"""distros = json.load(open('distros.json',))
print(distros)
"""projects = json.load(open('projects.json',))
print(projects)
for distro in distros:
print(distro[0] + ":")
print("CSC mirror: " + checker(distro[1], distro[3]))
print("Official distro: " + checker(distro[2], distro[3]))"""
for project in projects:
print(project[0] + ":")
print("CSC mirror: " + checker(project[1], project[3]))
print("Official project: " + checker(project[2], project[3]))"""
with open("data.json", "r", encoding="utf-8") as file:
data = json.load(file)
if sys.stdin.isatty():
distros = data
projects = data
else:
distros = [distro.rstrip() for distro in sys.stdin.readlines()]
projects = [project.rstrip() for project in sys.stdin.readlines()]
current_time = int(time.time())
for distro in distros:
for project in projects:
try:
if distro not in data:
print(f"Failure: {distro} does not exist")
if project not in data:
print(f"Failure: {project} does not exist")
continue
distro_class = getattr(sys.modules[__name__], distro)
if distro == "CPAN":
checker_result = distro_class.check(data, distro, current_time)
project_class = getattr(sys.modules[__name__], project)
if project == "CPAN":
checker_result = project_class.check(data, project, current_time)
if checker_result:
print(f"Success: {distro} up-to-date")
print(f"Success: {project} up-to-date")
else:
print(f"Failure: {distro} out-of-sync")
print(f"Failure: {project} out-of-sync")
continue
checker_result = distro_class.check(data, distro)
checker_result = project_class.check(data, project)
if checker_result:
data[distro]["out_of_sync_since"] = None
elif data[distro]["out_of_sync_since"] is None:
data[distro]["out_of_sync_since"] = current_time
elif current_time - data[distro]["out_of_sync_since"] \
> data[distro]["out_of_sync_interval"]:
print(f"Failure: {distro} out-of-sync")
data[project]["out_of_sync_since"] = None
elif data[project]["out_of_sync_since"] is None:
data[project]["out_of_sync_since"] = current_time
elif current_time - data[project]["out_of_sync_since"] \
> data[project]["out_of_sync_interval"]:
print(f"Failure: {project} out-of-sync")
continue
print(f"Success: {distro} up-to-date")
print(f"Success: {project} up-to-date")
except requests.exceptions.RequestException as err:
print(f"Error: {distro}\n{err}")
print(f"Error: {project}\n{err}")
with open("data.json", "w", encoding="utf-8") as file:
json.dump(data, file, indent=4)

20
project.py Normal file
View File

@ -0,0 +1,20 @@
"""
Contains abstract class for a mirrored project
"""
from abc import ABC
import requests
from shared import CSC_MIRROR
class Project(ABC):
"""Abstract class for a mirrored project"""
@staticmethod
def check(data, project):
"""Check if project packages are up-to-date"""
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
upstream_url = data[project]["upstream"] + data[project]["file"]
return requests.get(csc_url).text == requests.get(upstream_url).text

View File

@ -2,8 +2,8 @@
Contains AlmaLinux class
"""
from distro import Distro
from project import Project
class AlmaLinux(Distro):
class AlmaLinux(Project):
"""AlmaLinux class"""

View File

@ -2,8 +2,8 @@
Contains Alpine class
"""
from distro import Distro
from project import Project
class Alpine(Distro):
class Alpine(Project):
"""Alpine class"""

View File

@ -2,8 +2,8 @@
Contains Apache class
"""
from distro import Distro
from project import Project
class Apache(Distro):
class Apache(Project):
"""Apache class"""

View File

@ -2,8 +2,8 @@
Contains Arch class
"""
from distro import Distro
from project import Project
class Arch(Distro):
class Arch(Project):
"""Arch class"""

View File

@ -2,8 +2,8 @@
Contains CentOS class
"""
from distro import Distro
from project import Project
class CentOS(Distro):
class CentOS(Project):
"""CentOS class"""

View File

@ -2,8 +2,8 @@
Contains Ceph class
"""
from distro import Distro
from project import Project
class Ceph(Distro):
class Ceph(Project):
"""Ceph class"""

View File

@ -4,17 +4,17 @@ Contains CPAN class
import requests
from distro import Distro
from project import Project
from shared import CSC_MIRROR
class CPAN(Distro):
class CPAN(Project):
"""CPAN class"""
@staticmethod
def check(data, distro, current_time):
def check(data, project, current_time):
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
for mirror in res_json:
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
return current_time - int(mirror["age"]) <= data[project]["out_of_sync_interval"]
return False

View File

@ -2,8 +2,8 @@
Contains Cygwin class
"""
from distro import Distro
from project import Project
class Cygwin(Distro):
class Cygwin(Project):
"""Cygwin class"""

View File

@ -2,8 +2,8 @@
Contains Debian class
"""
from distro import Distro
from project import Project
class Debian(Distro):
class Debian(Project):
"""Debian class"""

View File

@ -2,8 +2,8 @@
Contains DebianCD class
"""
from distro import Distro
from project import Project
class DebianCD(Distro):
class DebianCD(Project):
"""DebianCD class"""

View File

@ -2,8 +2,8 @@
Contains DebianMultimedia class
"""
from distro import Distro
from project import Project
class DebianMultimedia(Distro):
class DebianMultimedia(Project):
"""DebianMultimedia class"""

View File

@ -2,8 +2,8 @@
Contains DebianPorts class
"""
from distro import Distro
from project import Project
class DebianPorts(Distro):
class DebianPorts(Project):
"""DebianPorts class"""

View File

@ -2,8 +2,8 @@
Contains DebianSecurity class
"""
from distro import Distro
from project import Project
class DebianSecurity(Distro):
class DebianSecurity(Project):
"""DebianSecurity class"""

View File

@ -2,8 +2,8 @@
Contains Eclipse class
"""
from distro import Distro
from project import Project
class Eclipse(Distro):
class Eclipse(Project):
"""Eclipse class"""

View File

@ -2,8 +2,8 @@
Contains Fedora class
"""
from distro import Distro
from project import Project
class Fedora(Distro):
class Fedora(Project):
"""Fedora class"""

View File

@ -2,8 +2,8 @@
Contains FreeBSD class
"""
from distro import Distro
from project import Project
class FreeBSD(Distro):
class FreeBSD(Project):
"""FreeBSD class"""

View File

@ -2,8 +2,8 @@
Contains GentooDistfiles class
"""
from distro import Distro
from project import Project
class GentooDistfiles(Distro):
class GentooDistfiles(Project):
"""GentooDistfiles class"""

View File

@ -4,23 +4,23 @@ Contains GentooPortage class
import os
from distro import Distro
from project import Project
class GentooPortage(Distro):
class GentooPortage(Project):
"""GentooPortage class"""
@staticmethod
def check(data, distro):
def check(data, project):
rsync_command = "rsync -q {}{} {}"
os.system(rsync_command.format(data[distro]["csc"],
data[distro]["file"],
os.system(rsync_command.format(data[project]["csc"],
data[project]["file"],
"csc_manifest"))
os.system(rsync_command.format(data[distro]["upstream1"],
data[distro]["file"],
os.system(rsync_command.format(data[project]["upstream1"],
data[project]["file"],
"upstream_manifest1"))
os.system(rsync_command.format(data[distro]["upstream2"],
data[distro]["file"],
os.system(rsync_command.format(data[project]["upstream2"],
data[project]["file"],
"upstream_manifest2"))
stream1 = os.popen("diff csc_manifest upstream_manifest1")
output1 = stream1.read()

View File

@ -6,39 +6,39 @@ import re
import requests
from distro import Distro
from project import Project
from shared import CSC_MIRROR
class GNOME(Distro):
class GNOME(Project):
"""GNOME class"""
@staticmethod
def check(data, distro):
file = data[distro]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
def check(data, project):
file = data[project]["file1"]
csc_versions = requests.get(CSC_MIRROR + data[project]["csc"] + file).text
upstream_versions = requests.get(data[project]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
csc_versions = requests.get(CSC_MIRROR + data[project]["csc"] + file).text
upstream_versions = requests.get(data[project]["upstream1"] + file).text
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
if csc_latest != upstream_latest:
return False
file += csc_latest + "/"
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
+ data[distro]["file2"]).text
csc_text = requests.get(CSC_MIRROR + data[project]["csc"] + file
+ data[project]["file2"]).text
try:
ret = csc_text == requests.get(data[distro]["upstream2"] + file
+ data[distro]["file2"]).text
ret = csc_text == requests.get(data[project]["upstream2"] + file
+ data[project]["file2"]).text
except requests.exceptions.RequestException:
ret = False
try:
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
+ data[distro]["file2"]).text
return ret or csc_text == requests.get(data[project]["upstream3"] + file
+ data[project]["file2"]).text
except requests.exceptions.RequestException:
return False

View File

@ -2,8 +2,8 @@
Contains GNU class
"""
from distro import Distro
from project import Project
class GNU(Distro):
class GNU(Project):
"""GNU class"""

View File

@ -2,8 +2,8 @@
Contains Gutenberg class
"""
from distro import Distro
from project import Project
class Gutenberg(Distro):
class Gutenberg(Project):
"""Gutenberg class"""

View File

@ -4,14 +4,14 @@ Contains IPFire class
import requests
from distro import Distro
from project import Project
class IPFire(Distro):
class IPFire(Project):
"""IPFire class"""
@staticmethod
def check(data, distro):
def check(data, project):
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
ipfire_text = requests.get(ipfire_url).text
return ipfire_text.find("The mirror is up") != -1

View File

@ -2,8 +2,8 @@
Contains KDE class
"""
from distro import Distro
from project import Project
class KDE(Distro):
class KDE(Project):
"""KDE class"""

View File

@ -2,8 +2,8 @@
Contains KDEApplicationData class
"""
from distro import Distro
from project import Project
class KDEApplicationData(Distro):
class KDEApplicationData(Project):
"""KDEApplicationData class"""

View File

@ -2,8 +2,8 @@
Contains Kernel class
"""
from distro import Distro
from project import Project
class Kernel(Distro):
class Kernel(Project):
"""Kernel class"""

View File

@ -2,8 +2,8 @@
Contains OpenBSD class
"""
from distro import Distro
from project import Project
class OpenBSD(Distro):
class OpenBSD(Project):
"""OpenBSD class"""