Check whether our mirror packages are up to date.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

85 lines
3.6 KiB

#!/usr/bin/env python3
This mirror status checker determines whether CSC mirror is up-to-date with upstream
import time
import requests
from arch import Arch
from ceph import Ceph
from debian import Debian
from eclipse import Eclipse
from gnu import GNU
from kernel import Kernel
from openbsd import OpenBSD
from import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
import json # import json to read distro info stored in json file
import datefinder # another date finding library
def checker(directory_URL, file_name):
page = requests.get(directory_URL).text
indexOfFile = page.find(file_name)
# print(page)
# remove stray numbers (file size numbers in particular) that might interfere with date finding
segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
# print(segment_clean)
# implementation using
# notes: some dates don't parse correctly with this tool
# print(search_dates(page[indexOfFile:], languages=['en']))
# print(search_dates(page[indexOfFile:])[0])
# finds the dates in the segment after the file name
# notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom.
matches = list(datefinder.find_dates(segment_clean))
# print(matches)
if len(matches) > 0:
date = matches[0] # date is of type datetime.datetime
return(date.strftime("%m/%d/%Y, %H:%M:%S"))
return('No dates found')
if __name__ == "__main__":
"""for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]:
except requests.exceptions.RequestException as err:
print(f"Error: {}\n{err}")"""
"""distros = json.load(open('distros.json',))
for distro in distros:
print(distro[0] + ":")
print("CSC mirror: " + checker(distro[1], distro[3]))
print("Official distro: " + checker(distro[2], distro[3]))"""
with open("data.json", "r", encoding = "utf-8") as file:
data = json.load(file)
current_time = int(time.time())
for distro in data:
csc_url = "" + data[distro]["csc"] \
+ data[distro]["file"]
upstream_url = data[distro]["upstream"] + data[distro]["file"]
if requests.get(csc_url).text == requests.get(upstream_url).text:
data[distro]["out_of_date_since"] = None
elif data[distro]["out_of_date_since"] is None:
data[distro]["out_of_date_since"] = current_time
elif current_time - data[distro]["out_of_date_since"] \
> data[distro]["out_of_date_interval"]:
print(f"Failure: {distro} out-of-sync")
print(f"Success: {distro} up-to-date")
except requests.exceptions.RequestException as err:
print(f"Error: {distro}\n{err}")
with open("data.json", "w", encoding = "utf-8") as file:
json.dump(data, file, indent = 4)