forked from public/mirror-checker
72 lines
3.9 KiB
Python
72 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
|
|
"""
|
|
This mirror status checker determines whether CSC mirror is up-to-date with upstream
|
|
"""
|
|
|
|
import requests
|
|
from arch import Arch
|
|
from ceph import Ceph
|
|
from debian import Debian
|
|
from eclipse import Eclipse
|
|
from gnu import GNU
|
|
from kernel import Kernel
|
|
from openbsd import OpenBSD
|
|
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
|
|
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
|
|
import re # import regular expressions to remove stray numbers in string that might interfere with date finding
|
|
import json # import json to read distro info stored in json file
|
|
|
|
import datefinder # another date finding library
|
|
|
|
# checker: gets the timestamp of the file inside the directory at the specified URL and returns it as a string
|
|
def checker(directory_URL, file_name):
|
|
page = requests.get(directory_URL).text
|
|
indexOfFile = page.find(file_name)
|
|
# print(page)
|
|
|
|
# remove stray numbers (file size numbers in particular) that might interfere with date finding
|
|
segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
|
|
segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
|
|
# print(segment_clean)
|
|
|
|
# implementation using dateparser.search.search_dates
|
|
# notes: some dates don't parse correctly with this tool
|
|
# print(search_dates(page[indexOfFile:], languages=['en']))
|
|
# print(search_dates(page[indexOfFile:])[0])
|
|
|
|
# finds the dates in the segment after the file name
|
|
# notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom.
|
|
matches = list(datefinder.find_dates(segment_clean))
|
|
# print(matches)
|
|
|
|
if len(matches) > 0:
|
|
date = matches[0] # date is of type datetime.datetime
|
|
return(date.strftime("%m/%d/%Y, %H:%M:%S"))
|
|
else:
|
|
return('No dates found')
|
|
|
|
if __name__ == "__main__":
|
|
"""for distro in [Arch, Ceph, Debian, Eclipse, GNU, Kernel, OpenBSD]:
|
|
try:
|
|
distro.print_output(distro.check())
|
|
except requests.exceptions.RequestException as err:
|
|
print(f"Error: {distro.name()}\n{err}")"""
|
|
|
|
# the implementation to check timestamps in distro mirror directory:
|
|
distros = json.load(open('distros.json',))
|
|
# print(distros)
|
|
|
|
for distro in distros:
|
|
print(distro[0] + ":")
|
|
print("CSC mirror: " + checker(distro[1], distro[3]))
|
|
print("Official distro: " + checker(distro[2], distro[3]))
|
|
# for eclipse, i couldn't find a directory from the official website, so i decided to temporarily cheat a little bit and peek at another person's mirror for explipse
|
|
# i just realized that my method doesn't just work for scraping html directories but everything returned by a URL (websites, files storing a time)
|
|
# i think i'll put checking arch, ceph, ecplipse on hold because it's more proper to check them using as.Date(17383, origin="1970-01-01") where the date is stored in
|
|
# a file as a number, but this would require another function and me looping through different functions. i think i'll do this later
|
|
# corresponding jsons: ["Eclipse", "https://mirror.csclub.uwaterloo.ca/eclipse/", "http://eclipse.mirror.rafal.ca/", "TIME"],
|
|
# ["Ceph", "https://mirror.csclub.uwaterloo.ca/ceph/", "https://download.ceph.com/", "timestamp"],
|
|
# i don't know what to do with damnsmalllinux, it seems like an abandoned project. useful link: http://distro.ibiblio.org/damnsmall/
|
|
# the other function is probably probing mirror status sites like this: http://mirrors.cpan.org/
|
|
# maybe i can make a column in json that specifies the method for checking the particular mirror and that identifier will find a specific function to check the update |