forked from public/mirror-checker
Compare commits
16 Commits
Author | SHA1 | Date |
---|---|---|
Raymond Li | 90bde5a754 | |
Justin Toft | 55f3e7bcbe | |
Justin Toft | 860d5c8e44 | |
Raymond Li | 65b06ac0a8 | |
Justin Toft | 5afebc8030 | |
Raymond Li | 54d8a47944 | |
Rio Liu | 789f0cd662 | |
Raymond Li | e1ef917af0 | |
Yiao Shen | 0571f7353a | |
Yiao Shen | 612ec9d04b | |
Raymond Li | 73bc2b5ade | |
Yiao Shen | 9f345c7b0c | |
Yiao Shen | 660b566715 | |
Yiao Shen | f408922a96 | |
Raymond Li | 4a3cedfb07 | |
Yiao Shen | 76f7863a85 |
23
data.json
23
data.json
|
@ -48,7 +48,8 @@
|
||||||
"file": "timestamp"
|
"file": "timestamp"
|
||||||
},
|
},
|
||||||
"CPAN": {
|
"CPAN": {
|
||||||
"out_of_sync_interval": 172800
|
"out_of_sync_interval": 172800,
|
||||||
|
"out_of_sync_since": null
|
||||||
},
|
},
|
||||||
"cran": {
|
"cran": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": null,
|
||||||
|
@ -86,7 +87,7 @@
|
||||||
"file": "debian-cd/project/trace/cdimage.debian.org"
|
"file": "debian-cd/project/trace/cdimage.debian.org"
|
||||||
},
|
},
|
||||||
"DebianMultimedia": {
|
"DebianMultimedia": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1659116719,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "debian-multimedia/",
|
"csc": "debian-multimedia/",
|
||||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
||||||
|
@ -116,7 +117,7 @@
|
||||||
},
|
},
|
||||||
"Fedora": {
|
"Fedora": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": null,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 259200,
|
||||||
"csc": "fedora/",
|
"csc": "fedora/",
|
||||||
"upstream": "http://fedora.mirror.iweb.com/",
|
"upstream": "http://fedora.mirror.iweb.com/",
|
||||||
"file": "linux/development/rawhide/COMPOSE_ID"
|
"file": "linux/development/rawhide/COMPOSE_ID"
|
||||||
|
@ -173,7 +174,7 @@
|
||||||
"out_of_sync_interval": 172800
|
"out_of_sync_interval": 172800
|
||||||
},
|
},
|
||||||
"KDE": {
|
"KDE": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1659116720,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "kde/",
|
"csc": "kde/",
|
||||||
"upstream": "https://kde.c3sl.ufpr.br/",
|
"upstream": "https://kde.c3sl.ufpr.br/",
|
||||||
|
@ -208,7 +209,7 @@
|
||||||
"file": "dists/"
|
"file": "dists/"
|
||||||
},
|
},
|
||||||
"macports": {
|
"macports": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1642827723,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "MacPorts/mpdistfiles/",
|
"csc": "MacPorts/mpdistfiles/",
|
||||||
"upstream": "https://distfiles.macports.org/",
|
"upstream": "https://distfiles.macports.org/",
|
||||||
|
@ -269,7 +270,7 @@
|
||||||
"exclude": true
|
"exclude": true
|
||||||
},
|
},
|
||||||
"opensuse": {
|
"opensuse": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1648699331,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "opensuse/update/",
|
"csc": "opensuse/update/",
|
||||||
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
|
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
|
||||||
|
@ -320,7 +321,7 @@
|
||||||
"file": "dists/"
|
"file": "dists/"
|
||||||
},
|
},
|
||||||
"raspbian": {
|
"raspbian": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1659116721,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "raspbian/",
|
"csc": "raspbian/",
|
||||||
"upstream": "http://archive.raspbian.org/",
|
"upstream": "http://archive.raspbian.org/",
|
||||||
|
@ -341,7 +342,7 @@
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
"slackware": {
|
"slackware": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1642827723,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "slackware/",
|
"csc": "slackware/",
|
||||||
"upstream": "https://mirrors.slackware.com/slackware/",
|
"upstream": "https://mirrors.slackware.com/slackware/",
|
||||||
|
@ -373,7 +374,7 @@
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
"ubuntu_ports": {
|
"ubuntu_ports": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1651550528,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "ubuntu-ports/",
|
"csc": "ubuntu-ports/",
|
||||||
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
|
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
|
||||||
|
@ -390,7 +391,7 @@
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": null,
|
||||||
"out_of_sync_interval": 172800,
|
"out_of_sync_interval": 172800,
|
||||||
"csc": "",
|
"csc": "",
|
||||||
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release",
|
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
"vlc": {
|
"vlc": {
|
||||||
|
@ -401,7 +402,7 @@
|
||||||
"file": "trace"
|
"file": "trace"
|
||||||
},
|
},
|
||||||
"x_org": {
|
"x_org": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1657512131,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "x.org/individual/",
|
"csc": "x.org/individual/",
|
||||||
"upstream": "https://www.x.org/releases/individual/",
|
"upstream": "https://www.x.org/releases/individual/",
|
||||||
|
|
44
main.py
44
main.py
|
@ -8,46 +8,68 @@ import time
|
||||||
import sys
|
import sys
|
||||||
import requests
|
import requests
|
||||||
from multiprocessing import Pool, Manager
|
from multiprocessing import Pool, Manager
|
||||||
|
from typing import Optional
|
||||||
|
from time import sleep, localtime, strftime
|
||||||
|
|
||||||
from projects import *
|
from projects import *
|
||||||
import json
|
import json
|
||||||
|
|
||||||
NUM_THREAD = 16
|
NUM_THREAD = 16
|
||||||
|
MAX_RETRY = 3
|
||||||
current_time = int(time.time())
|
RETRY_TIMEOUT = 30 # In seconds
|
||||||
|
|
||||||
def safe_print(*args, **kwargs):
|
def safe_print(*args, **kwargs):
|
||||||
# When run with 'chronic' and 'timeout', stdout gets suppressed
|
# When run with 'chronic' and 'timeout', stdout gets suppressed
|
||||||
# due to buffering. Make sure to always flush the output.
|
# due to buffering. Make sure to always flush the output.
|
||||||
print(*args, **kwargs, flush=True)
|
print(*args, **kwargs, flush=True)
|
||||||
|
|
||||||
def check_project(args):
|
# Return None if no error occurs and a string for error message otherwise
|
||||||
|
def check_project(args) -> Optional[str]:
|
||||||
|
current_time = int(time.time())
|
||||||
project, data = args
|
project, data = args
|
||||||
try:
|
try:
|
||||||
project_class = getattr(sys.modules[__name__], project)
|
project_class = getattr(sys.modules[__name__], project)
|
||||||
|
|
||||||
# Skip projects we no longer mirror
|
# Skip projects we no longer mirror
|
||||||
if data[project].get('exclude', False):
|
if data[project].get('exclude', False):
|
||||||
return True
|
return None
|
||||||
|
|
||||||
checker_result = project_class.check(data, project, current_time)
|
checker_result = project_class.check(data, project, current_time)
|
||||||
|
|
||||||
if checker_result:
|
if checker_result:
|
||||||
data[project]["out_of_sync_since"] = None
|
data[project]["out_of_sync_since"] = None
|
||||||
safe_print(f"Success: {project} up-to-date")
|
return None
|
||||||
return True
|
|
||||||
|
|
||||||
elif (data[project]["out_of_sync_since"] is not None
|
elif (data[project]["out_of_sync_since"] is not None
|
||||||
and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
|
and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
|
||||||
safe_print(f"Failure: {project} out-of-sync")
|
now_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
|
||||||
return False
|
duration = current_time - data[project]["out_of_sync_since"]
|
||||||
|
return f"{project} out-of-sync at {now_str} for {duration}s"
|
||||||
|
|
||||||
else:
|
else:
|
||||||
data[project]["out_of_sync_since"] = current_time
|
data[project]["out_of_sync_since"] = current_time
|
||||||
return True
|
return None
|
||||||
|
|
||||||
except requests.exceptions.RequestException as err:
|
except requests.exceptions.RequestException as err:
|
||||||
safe_print(f"Error: {project}\n{err}")
|
return f"{project}\n{err}"
|
||||||
|
|
||||||
|
def check_project_with_retry(args) -> bool:
|
||||||
|
project, _ = args
|
||||||
|
errs = []
|
||||||
|
for _ in range(MAX_RETRY):
|
||||||
|
res = check_project(args)
|
||||||
|
if res == None:
|
||||||
|
safe_print(f"Success: {project} up-to-date")
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
errs.append(res)
|
||||||
|
# Do nothing, try again later
|
||||||
|
sleep(RETRY_TIMEOUT)
|
||||||
|
|
||||||
|
# Max try reached, print errors
|
||||||
|
safe_print(f"Error: {project}")
|
||||||
|
for reason in errs:
|
||||||
|
safe_print(f" {reason}")
|
||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
|
@ -61,7 +83,7 @@ def main():
|
||||||
sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
|
sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
|
||||||
|
|
||||||
with Pool(NUM_THREAD) as pool:
|
with Pool(NUM_THREAD) as pool:
|
||||||
all_pass = all(pool.imap(check_project, ((k, sync_data) for k in data.keys())))
|
all_pass = all(pool.imap(check_project_with_retry, ((k, sync_data) for k in data.keys())))
|
||||||
|
|
||||||
with open(data_file, "w", encoding="utf-8") as file:
|
with open(data_file, "w", encoding="utf-8") as file:
|
||||||
json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')
|
json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')
|
||||||
|
|
|
@ -3,7 +3,35 @@ Contains Apache class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
|
||||||
class Apache(Project):
|
class Apache(Project):
|
||||||
"""Apache class"""
|
"""Apache class"""
|
||||||
|
|
||||||
|
# Apache's time file has two segments, so we need a special function
|
||||||
|
# Example: 1648323001 rsync-he-fi
|
||||||
|
def check(data, project, current_time):
|
||||||
|
"""Check if project packages are up-to-date"""
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
return get_timestamp_from_apache(upstream) - get_timestamp_from_apache(CSC) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
print("failed to parse apache")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_timestamp_from_apache(s: str) -> int:
|
||||||
|
real_time = s.split(" ")[0]
|
||||||
|
return int(real_time)
|
||||||
|
|
|
@ -15,7 +15,8 @@ class CPAN(Project):
|
||||||
def check(data, project, current_time):
|
def check(data, project, current_time):
|
||||||
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
|
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
|
||||||
for mirror in res_json:
|
for mirror in res_json:
|
||||||
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
|
if mirror["url"] == f"{CSC_MIRROR}CPAN/" and mirror["last_status"] == "ok":
|
||||||
data[project]["out_of_sync_since"] = int(mirror["age"])
|
# This is an improvised method: report we're good if CPAN think we are good
|
||||||
return current_time - data[project]["out_of_sync_since"] <= data[project]["out_of_sync_interval"]
|
# Change this to a more precise method if you find a better way to do it
|
||||||
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
|
@ -18,7 +18,7 @@ class cran(Project):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
data[project]["out_of_sync_since"] = current_time - duration.total_seconds()
|
data[project]["out_of_sync_since"] = current_time - duration.total_seconds()
|
||||||
|
|
|
@ -7,7 +7,7 @@ from project import Project
|
||||||
from shared import CSC_MIRROR
|
from shared import CSC_MIRROR
|
||||||
import requests
|
import requests
|
||||||
import datefinder # another date finding library
|
import datefinder # another date finding library
|
||||||
from datetime import timedelta
|
from datetime import datetime, timedelta
|
||||||
import re
|
import re
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
|
||||||
|
@ -18,9 +18,10 @@ class ctan(Project):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
|
||||||
|
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
data[project]["out_of_sync_since"] = datetime.now() - duration.total_seconds()
|
data[project]["out_of_sync_since"] = datetime.now() - duration
|
||||||
|
|
||||||
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
||||||
|
|
|
@ -3,7 +3,37 @@ Contains Debian class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class Debian(Project):
|
class Debian(Project):
|
||||||
"""Debian class"""
|
"""Debian class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
CSC_date = datetime.strptime(CSC.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
|
||||||
|
CSC_utc_time = time.mktime(CSC_date.timetuple())
|
||||||
|
|
||||||
|
upstream_date = datetime.strptime(upstream.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
|
||||||
|
upstream_utc_time = time.mktime(upstream_date.timetuple())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,7 +3,40 @@ Contains DebianCD class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class DebianCD(Project):
|
class DebianCD(Project):
|
||||||
"""DebianCD class"""
|
"""DebianCD class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Date Format Example: Sun 27 Mar 00:20:12 UTC 2022
|
||||||
|
date_format = "%a %d %b %H:%M:%S UTC %Y\n"
|
||||||
|
|
||||||
|
CSC_date = datetime.strptime(CSC, date_format)
|
||||||
|
CSC_utc_time = time.mktime(CSC_date.timetuple())
|
||||||
|
|
||||||
|
upstream_date = datetime.strptime(upstream, date_format)
|
||||||
|
upstream_utc_time = time.mktime(upstream_date.timetuple())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,7 +3,40 @@ Contains Fedora class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class Fedora(Project):
|
class Fedora(Project):
|
||||||
"""Fedora class"""
|
"""Fedora class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Date example: Fedora-Rawhide-20220725.n.1
|
||||||
|
date_format = "%Y%m%d"
|
||||||
|
|
||||||
|
CSC_date = datetime.strptime(CSC[15:23], "%Y%m%d")
|
||||||
|
CSC_utc_time = time.mktime(CSC_date.timetuple())
|
||||||
|
|
||||||
|
upstream_date = datetime.strptime(upstream[15:23], "%Y%m%d")
|
||||||
|
upstream_utc_time = time.mktime(upstream_date.timetuple())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return upstream_utc_time - CSC_utc_time < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,7 +3,35 @@ Contains FreeBSD class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
|
||||||
class FreeBSD(Project):
|
class FreeBSD(Project):
|
||||||
"""FreeBSD class"""
|
"""FreeBSD class"""
|
||||||
|
|
||||||
|
# FreeBSD's time file has two segments, so we need a special function
|
||||||
|
# Example TIMESTAMP file: 1648308600 2022-03-26T15:30:00 UTC
|
||||||
|
def check(data, project, current_time):
|
||||||
|
"""Check if project packages are up-to-date"""
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
try:
|
||||||
|
return get_timestamp_for_freebsd(upstream) - get_timestamp_for_freebsd(CSC) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
print("failed to parse apache")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_timestamp_for_freebsd(s: str) -> int:
|
||||||
|
real_time = s.split(" ")[0]
|
||||||
|
return int(real_time)
|
||||||
|
|
|
@ -3,7 +3,35 @@ Contains GentooDistfiles class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class GentooDistfiles(Project):
|
class GentooDistfiles(Project):
|
||||||
"""GentooDistfiles class"""
|
"""GentooDistfiles class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
CSC_utc_time = CSC[0:11]
|
||||||
|
|
||||||
|
upstream_utc_time = upstream[0:11]
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -18,7 +18,7 @@ class mxlinux(Project):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
data[project]["out_of_sync_since"] = current_time - duration.total_seconds()
|
data[project]["out_of_sync_since"] = current_time - duration.total_seconds()
|
||||||
|
|
|
@ -19,7 +19,7 @@ class mxlinux_iso(Project):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
m = re.search(r'(\d+ minutes?)|(\d+ hours?)|(\d+(\.)?\d+ days?)', page[indexOfFile:])
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,31 @@ Contains nongnu class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class nongnu(Project):
|
class nongnu(Project):
|
||||||
"""nongnu class"""
|
"""nongnu class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream.partition('\n')[0]) - int(CSC.partition('\n')[0]) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -23,7 +23,8 @@ class raspberrypi(Project):
|
||||||
for i in s.find_all("a"): # for a href directories
|
for i in s.find_all("a"): # for a href directories
|
||||||
href = i.attrs['href']
|
href = i.attrs['href']
|
||||||
|
|
||||||
if href.endswith("/") and href != "../" and href != "/":
|
# The raspberry pi server doesn't use a relative path to the parent directory
|
||||||
|
if href.endswith("/") and href != "../" and href != "/" and href != "/debian/":
|
||||||
site_next = site+href+"Release"
|
site_next = site+href+"Release"
|
||||||
|
|
||||||
if site_next not in urls:
|
if site_next not in urls:
|
||||||
|
@ -44,6 +45,7 @@ class raspberrypi(Project):
|
||||||
cls.scrape(urls1, csc_url)
|
cls.scrape(urls1, csc_url)
|
||||||
cls.scrape(urls2, upstream_url)
|
cls.scrape(urls2, upstream_url)
|
||||||
|
|
||||||
|
|
||||||
if (len(urls1) != len(urls2)):
|
if (len(urls1) != len(urls2)):
|
||||||
return False
|
return False
|
||||||
urls1.sort()
|
urls1.sort()
|
||||||
|
|
|
@ -34,7 +34,7 @@ class slackware(Project):
|
||||||
hrefs2 = [i.attrs['href'] for i in s2.find_all("a")]
|
hrefs2 = [i.attrs['href'] for i in s2.find_all("a")]
|
||||||
|
|
||||||
for href in hrefs1: # for a href directories
|
for href in hrefs1: # for a href directories
|
||||||
if href.endswith("/") and href != "../" and href != "/" and not href.startswith("/") and not re.match(r'slackware-([1-7]|8\.0).*', href) and href != "slackware-iso/" and href != "slackware-current/" and href != "slackware-pre-1.0-beta/" and href != "unsupported/":
|
if href.endswith("/") and href != "../" and href != "/" and not href.startswith("/") and not re.match(r'slackware-([1-7]|8\.0).*', href) and href != "slackware-iso/" and href != "slackware-current/" and href != "slackware-pre-1.0-beta/" and href != "unsupported/" and not href.startswith("http"):
|
||||||
# print(href)
|
# print(href)
|
||||||
if href not in hrefs2:
|
if href not in hrefs2:
|
||||||
return False
|
return False
|
||||||
|
|
|
@ -5,6 +5,7 @@ Contains ubuntu class
|
||||||
import os
|
import os
|
||||||
from project import Project
|
from project import Project
|
||||||
from shared import CSC_MIRROR
|
from shared import CSC_MIRROR
|
||||||
|
from shared import NUM_UBUNTU_RELEASES
|
||||||
import requests
|
import requests
|
||||||
import datefinder # another date finding library
|
import datefinder # another date finding library
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
@ -17,4 +18,4 @@ class ubuntu(Project):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project, current_time):
|
def check(data, project, current_time):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
return page.count("Up to date") == 21
|
return page.count("Up to date") == NUM_UBUNTU_RELEASES
|
||||||
|
|
|
@ -5,6 +5,7 @@ Contains ubuntu_releases class
|
||||||
import os
|
import os
|
||||||
from project import Project
|
from project import Project
|
||||||
from shared import CSC_MIRROR
|
from shared import CSC_MIRROR
|
||||||
|
from shared import NUM_UBUNTU_RELEASES
|
||||||
import requests
|
import requests
|
||||||
import datefinder # another date finding library
|
import datefinder # another date finding library
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
|
@ -17,10 +18,4 @@ class ubuntu_releases(Project):
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def check(data, project, current_time):
|
def check(data, project, current_time):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("last verified")
|
return page.count("Up to date") == NUM_UBUNTU_RELEASES
|
||||||
matches = list(datefinder.find_dates(page[indexOfFile:]))
|
|
||||||
date = matches[0].replace(tzinfo=None) # date is of type datetime.datetime
|
|
||||||
data[project]["out_of_sync_since"] = date.timestamp()
|
|
||||||
return(pd.to_datetime(current_time, unit='s') - date <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s'))
|
|
||||||
|
|
||||||
# https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release
|
|
||||||
|
|
Loading…
Reference in New Issue