forked from public/mirror-checker
Improvements
This commit is contained in:
parent
03106f609f
commit
a33c73a7e9
2
main.py
2
main.py
|
@ -34,7 +34,7 @@ if __name__ == "__main__":
|
||||||
else:
|
else:
|
||||||
print(f"Failure: {project} out-of-sync")
|
print(f"Failure: {project} out-of-sync")
|
||||||
continue
|
continue
|
||||||
checker_result = project_class.check(data, project)
|
checker_result = project_class.check(data, project, current_time)
|
||||||
if checker_result:
|
if checker_result:
|
||||||
data[project]["out_of_sync_since"] = None
|
data[project]["out_of_sync_since"] = None
|
||||||
elif data[project]["out_of_sync_since"] is None:
|
elif data[project]["out_of_sync_since"] is None:
|
||||||
|
|
|
@ -13,7 +13,7 @@ class Project(ABC):
|
||||||
"""Abstract class for a mirrored project"""
|
"""Abstract class for a mirrored project"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project):
|
def check(data, project, current_time):
|
||||||
"""Check if project packages are up-to-date"""
|
"""Check if project packages are up-to-date"""
|
||||||
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
upstream_url = data[project]["upstream"] + data[project]["file"]
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
|
@ -13,7 +13,7 @@ import re
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
|
||||||
class ctan(Project):
|
class ctan(Project):
|
||||||
"""cran class"""
|
"""ctan class"""
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project, current_time):
|
def check(data, project, current_time):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
|
@ -23,4 +23,4 @@ class ctan(Project):
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
|
|
||||||
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
||||||
|
|
|
@ -18,7 +18,7 @@ class GentooPortage(Project):
|
||||||
"""GentooPortage class"""
|
"""GentooPortage class"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project):
|
def check(data, project, current_time):
|
||||||
"""rsync_command = "rsync -q {}{} {}"
|
"""rsync_command = "rsync -q {}{} {}"
|
||||||
os.system(rsync_command.format(data[project]["csc"],
|
os.system(rsync_command.format(data[project]["csc"],
|
||||||
data[project]["file"],
|
data[project]["file"],
|
||||||
|
|
|
@ -14,7 +14,7 @@ class GNOME(Project):
|
||||||
"""GNOME class"""
|
"""GNOME class"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project):
|
def check(data, project, current_time):
|
||||||
file = data[project]["file1"]
|
file = data[project]["file1"]
|
||||||
csc_versions = requests.get(CSC_MIRROR + data[project]["csc"] + file).text
|
csc_versions = requests.get(CSC_MIRROR + data[project]["csc"] + file).text
|
||||||
upstream_versions = requests.get(data[project]["upstream1"] + file).text
|
upstream_versions = requests.get(data[project]["upstream1"] + file).text
|
||||||
|
|
|
@ -11,7 +11,7 @@ class IPFire(Project):
|
||||||
"""IPFire class"""
|
"""IPFire class"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project):
|
def check(data, project, current_time):
|
||||||
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
|
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
|
||||||
ipfire_text = requests.get(ipfire_url).text
|
ipfire_text = requests.get(ipfire_url).text
|
||||||
return ipfire_text.find("The mirror is up") != -1
|
return ipfire_text.find("The mirror is up") != -1
|
||||||
|
|
|
@ -5,38 +5,38 @@ from datetime import datetime
|
||||||
from project import Project
|
from project import Project
|
||||||
from shared import CSC_MIRROR
|
from shared import CSC_MIRROR
|
||||||
|
|
||||||
class sage(Project):
|
|
||||||
"""sagemath class"""
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def get_latest_date(dates):
|
|
||||||
dates = [list(datefinder.find_dates(date))[0] for date in dates]
|
|
||||||
return(max(dates))
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def check(cls, data, project):
|
|
||||||
page1 = requests.get(CSC_MIRROR + data[project]["csc"] + data[project]["file"]).text
|
|
||||||
page2 = requests.get("http://mirrors.mit.edu/sage/src/index.html").text
|
|
||||||
page3 = requests.get("https://mirror.rcg.sfu.ca/mirror/sage/src/index.html").text
|
|
||||||
|
|
||||||
CSC_dates = re.findall(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page1)
|
class sage(Project):
|
||||||
MIT_dates = re.findall(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page2)
|
"""sagemath class"""
|
||||||
SFU_dates = re.findall(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page3)
|
|
||||||
|
@staticmethod
|
||||||
# print(len(CSC_dates))
|
def get_latest_date(dates):
|
||||||
# print(len(MIT_dates))
|
dates = [list(datefinder.find_dates(date))[0] for date in dates]
|
||||||
# print(len(SFU_dates))
|
return max(dates)
|
||||||
# print(cls.get_latest_date(CSC_dates))
|
|
||||||
# print(cls.get_latest_date(MIT_dates))
|
@classmethod
|
||||||
# print(cls.get_latest_date(SFU_dates))
|
def check(cls, data, project):
|
||||||
|
page1 = requests.get(CSC_MIRROR + data[project]["csc"] + data[project]["file"]).text
|
||||||
if len(CSC_dates) < max([len(MIT_dates), len(SFU_dates)]):
|
page2 = requests.get("http://mirrors.mit.edu/sage/src/index.html").text
|
||||||
return False
|
page3 = requests.get("https://mirror.rcg.sfu.ca/mirror/sage/src/index.html").text
|
||||||
elif len(CSC_dates) > max([len(MIT_dates), len(SFU_dates)]):
|
|
||||||
# if we have more entries than their mirror, ours must be the new one
|
CSC_dates = re.findall(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page1)
|
||||||
# since distros only add new versions, and don't delete old versions
|
MIT_dates = re.findall(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page2)
|
||||||
return True
|
SFU_dates = re.findall(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page3)
|
||||||
if (cls.get_latest_date(CSC_dates) < max([cls.get_latest_date(MIT_dates),cls.get_latest_date(SFU_dates)])):
|
|
||||||
return False
|
# print(len(CSC_dates))
|
||||||
return True
|
# print(len(MIT_dates))
|
||||||
|
# print(len(SFU_dates))
|
||||||
|
# print(cls.get_latest_date(CSC_dates))
|
||||||
|
# print(cls.get_latest_date(MIT_dates))
|
||||||
|
# print(cls.get_latest_date(SFU_dates))
|
||||||
|
|
||||||
|
if len(CSC_dates) < max([len(MIT_dates), len(SFU_dates)]):
|
||||||
|
return False
|
||||||
|
elif len(CSC_dates) > max([len(MIT_dates), len(SFU_dates)]):
|
||||||
|
# if we have more entries than their mirror, ours must be the new one
|
||||||
|
# since distros only add new versions, and don't delete old versions
|
||||||
|
return True
|
||||||
|
if cls.get_latest_date(CSC_dates) < max([cls.get_latest_date(MIT_dates), cls.get_latest_date(SFU_dates)]):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
|
@ -5,21 +5,22 @@ from datetime import datetime
|
||||||
from project import Project
|
from project import Project
|
||||||
from shared import CSC_MIRROR
|
from shared import CSC_MIRROR
|
||||||
|
|
||||||
|
|
||||||
class saltstack(Project):
|
class saltstack(Project):
|
||||||
"""saltstack class"""
|
"""saltstack class"""
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def check(data, project):
|
|
||||||
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
|
||||||
upstream_url = data[project]["upstream"] + data[project]["file"]
|
|
||||||
|
|
||||||
page1 = requests.get(csc_url).text
|
@staticmethod
|
||||||
page2 = requests.get(upstream_url).text
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
CSC_release = re.search(r'Latest release: (\d)+.(\d)+ \((.+)\)', page1)
|
page1 = requests.get(csc_url).text
|
||||||
upstream_release = re.search(r'Latest release: (\d)+.(\d)+ \((.+)\)', page2)
|
page2 = requests.get(upstream_url).text
|
||||||
|
|
||||||
# print(CSC_release.group(0))
|
CSC_release = re.search(r'Latest release: (\d)+.(\d)+ \((.+)\)', page1)
|
||||||
# print(upstream_release.group(0))
|
upstream_release = re.search(r'Latest release: (\d)+.(\d)+ \((.+)\)', page2)
|
||||||
|
|
||||||
return CSC_release.group(0) == upstream_release.group(0)
|
# print(CSC_release.group(0))
|
||||||
|
# print(upstream_release.group(0))
|
||||||
|
|
||||||
|
return CSC_release.group(0) == upstream_release.group(0)
|
||||||
|
|
Loading…
Reference in New Issue