forked from public/mirror-checker
115 lines
4.2 KiB
Python
115 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
|
|
"""
|
|
This mirror status checker determines whether CSC mirror is up-to-date with upstream
|
|
"""
|
|
|
|
import time
|
|
import sys
|
|
import requests
|
|
from projects.almalinux import AlmaLinux
|
|
from projects.alpine import Alpine
|
|
from projects.apache import Apache
|
|
from projects.arch import Arch
|
|
from projects.centos import CentOS
|
|
from projects.ceph import Ceph
|
|
from projects.cpan import CPAN
|
|
from projects.cygwin import Cygwin
|
|
from projects.debian import Debian
|
|
from projects.debiancd import DebianCD
|
|
from projects.debianmultimedia import DebianMultimedia
|
|
from projects.debianports import DebianPorts
|
|
from projects.debiansecurity import DebianSecurity
|
|
from projects.eclipse import Eclipse
|
|
from projects.fedora import Fedora
|
|
from projects.freebsd import FreeBSD
|
|
from projects.gentoodistfiles import GentooDistfiles
|
|
from projects.gentooportage import GentooPortage
|
|
from projects.gnome import GNOME
|
|
from projects.gnu import GNU
|
|
from projects.gutenberg import Gutenberg
|
|
from projects.ipfire import IPFire
|
|
from projects.kde import KDE
|
|
from projects.kdeapplicationdata import KDEApplicationData
|
|
from projects.kernel import Kernel
|
|
from projects.openbsd import OpenBSD
|
|
from shared import CSC_MIRROR
|
|
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
|
|
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
|
|
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
|
|
import json # import json to read project info stored in json file
|
|
|
|
import datefinder # another date finding library
|
|
|
|
|
|
def checker(directory_URL, file_name):
|
|
page = requests.get(directory_URL).text
|
|
indexOfFile = page.find(file_name)
|
|
# print(page)
|
|
|
|
# remove stray numbers (file size numbers in particular) that might interfere with date finding
|
|
segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
|
|
segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
|
|
# print(segment_clean)
|
|
|
|
# implementation using dateparser.search.search_dates
|
|
# notes: some dates don't parse correctly with this tool
|
|
# print(search_dates(page[indexOfFile:], languages=['en']))
|
|
# print(search_dates(page[indexOfFile:])[0])
|
|
|
|
# finds the dates in the segment after the file name
|
|
# notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom.
|
|
matches = list(datefinder.find_dates(segment_clean))
|
|
# print(matches)
|
|
|
|
if len(matches) > 0:
|
|
date = matches[0] # date is of type datetime.datetime
|
|
return (date.strftime("%m/%d/%Y, %H:%M:%S"))
|
|
else:
|
|
return ('No dates found')
|
|
|
|
|
|
if __name__ == "__main__":
|
|
"""projects = json.load(open('projects.json',))
|
|
print(projects)
|
|
|
|
for project in projects:
|
|
print(project[0] + ":")
|
|
print("CSC mirror: " + checker(project[1], project[3]))
|
|
print("Official project: " + checker(project[2], project[3]))"""
|
|
|
|
with open("data.json", "r", encoding="utf-8") as file:
|
|
data = json.load(file)
|
|
if sys.stdin.isatty():
|
|
projects = data
|
|
else:
|
|
projects = [project.rstrip() for project in sys.stdin.readlines()]
|
|
current_time = int(time.time())
|
|
for project in projects:
|
|
try:
|
|
if project not in data:
|
|
print(f"Failure: {project} does not exist")
|
|
continue
|
|
project_class = getattr(sys.modules[__name__], project)
|
|
if project == "CPAN":
|
|
checker_result = project_class.check(data, project, current_time)
|
|
if checker_result:
|
|
print(f"Success: {project} up-to-date")
|
|
else:
|
|
print(f"Failure: {project} out-of-sync")
|
|
continue
|
|
checker_result = project_class.check(data, project)
|
|
if checker_result:
|
|
data[project]["out_of_sync_since"] = None
|
|
elif data[project]["out_of_sync_since"] is None:
|
|
data[project]["out_of_sync_since"] = current_time
|
|
elif current_time - data[project]["out_of_sync_since"] \
|
|
> data[project]["out_of_sync_interval"]:
|
|
print(f"Failure: {project} out-of-sync")
|
|
continue
|
|
print(f"Success: {project} up-to-date")
|
|
except requests.exceptions.RequestException as err:
|
|
print(f"Error: {project}\n{err}")
|
|
with open("data.json", "w", encoding="utf-8") as file:
|
|
json.dump(data, file, indent=4)
|