d9hui
/
mirror-checker
forked from public/mirror-checker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

added manjaro and fixed ubuntu

This commit is contained in:
Tom 2021-10-03 22:18:07 -07:00
parent 530d22f0ab
commit 1b53f605c9
7 changed files with 326 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@ -36,5 +36,6 @@ done:
tdf: https://download.documentfoundation.org/ tdf: https://download.documentfoundation.org/
ubuntu: https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive ubuntu: https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive
vlc: http://download.videolan.org/pub/videolan/ vlc: http://download.videolan.org/pub/videolan/
manjaro
trisquel: https://trisquel.info/mirmon/index.html out of date website trisquel: https://trisquel.info/mirmon/index.html out of date website

409
data.json
View File

@ -1,202 +1,209 @@
{ {
"AlmaLinux": { "AlmaLinux": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://repo.almalinux.org/", "upstream": "https://repo.almalinux.org/",
"file": "almalinux/TIME" "file": "almalinux/TIME"
}, },
"Alpine": { "Alpine": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://uk.alpinelinux.org/", "upstream": "https://uk.alpinelinux.org/",
"file": "alpine/last-updated" "file": "alpine/last-updated"
}, },
"Apache": { "Apache": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "apache/", "csc": "apache/",
"upstream": "https://downloads.apache.org/", "upstream": "https://downloads.apache.org/",
"file": "zzz/time.txt" "file": "zzz/time.txt"
}, },
"Arch": { "Arch": {
"out_of_sync_since": 1633294718, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "archlinux/", "csc": "archlinux/",
"upstream": "http://arch.mirror.constant.com/", "upstream": "http://arch.mirror.constant.com/",
"file": "lastupdate" "file": "lastupdate"
}, },
"CentOS": { "CentOS": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://mirrors.edge.kernel.org/", "upstream": "https://mirrors.edge.kernel.org/",
"file": "centos/TIME" "file": "centos/TIME"
}, },
"Ceph": { "Ceph": {
"out_of_sync_since": 1633294718, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "ceph/", "csc": "ceph/",
"upstream": "https://download.ceph.com/", "upstream": "https://download.ceph.com/",
"file": "timestamp" "file": "timestamp"
}, },
"CPAN": { "CPAN": {
"out_of_sync_interval": 172800 "out_of_sync_interval": 172800
}, },
"Cygwin": { "Cygwin": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "cygwin/", "csc": "cygwin/",
"upstream": "https://cygwin.mirror.globo.tech/", "upstream": "https://cygwin.mirror.globo.tech/",
"file": "x86/sha512.sum" "file": "x86/sha512.sum"
}, },
"Debian": { "Debian": {
"out_of_sync_since": 1633294718, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://ftp-master.debian.org/", "upstream": "https://ftp-master.debian.org/",
"file": "debian/project/trace/master" "file": "debian/project/trace/master"
}, },
"DebianCD": { "DebianCD": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "http://debian.mirror.estruxture.net/", "upstream": "http://debian.mirror.estruxture.net/",
"file": "debian-cd/project/trace/cdimage.debian.org" "file": "debian-cd/project/trace/cdimage.debian.org"
}, },
"DebianMultimedia": { "DebianMultimedia": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "debian-multimedia/", "csc": "debian-multimedia/",
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", "upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
"file": "project/trace/deb-multimedia.org" "file": "project/trace/deb-multimedia.org"
}, },
"DebianPorts": { "DebianPorts": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://deb.debian.org/", "upstream": "https://deb.debian.org/",
"file": "debian-ports/project/trace/porta.debian.org" "file": "debian-ports/project/trace/porta.debian.org"
}, },
"DebianSecurity": { "DebianSecurity": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "http://debian.mirror.iweb.ca/", "upstream": "http://debian.mirror.iweb.ca/",
"file": "debian-security/project/trace/master" "file": "debian-security/project/trace/master"
}, },
"Eclipse": { "Eclipse": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 172800, "out_of_sync_interval": 172800,
"csc": "eclipse/", "csc": "eclipse/",
"upstream": "http://download.eclipse.org/", "upstream": "http://download.eclipse.org/",
"file": "TIME" "file": "TIME"
}, },
"Fedora": { "Fedora": {
"out_of_sync_since": 1633298732, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "fedora/", "csc": "fedora/",
"upstream": "http://fedora.mirror.iweb.com/", "upstream": "http://fedora.mirror.iweb.com/",
"file": "linux/development/rawhide/COMPOSE_ID" "file": "linux/development/rawhide/COMPOSE_ID"
}, },
"FreeBSD": { "FreeBSD": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "http://ftp4.freebsd.org/pub/", "upstream": "http://ftp4.freebsd.org/pub/",
"file": "FreeBSD/TIMESTAMP" "file": "FreeBSD/TIMESTAMP"
}, },
"GentooDistfiles": { "GentooDistfiles": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "gentoo-distfiles", "csc": "gentoo-distfiles",
"upstream": "http://gentoo.mirrors.tera-byte.com/", "upstream": "http://gentoo.mirrors.tera-byte.com/",
"file": "distfiles/timestamp.dev-local" "file": "distfiles/timestamp.dev-local"
}, },
"GentooPortage": { "GentooPortage": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "rsync://rsync4.ca.gentoo.org/", "csc": "rsync://rsync4.ca.gentoo.org/",
"upstream1": "rsync://rsync1.de.gentoo.org/", "upstream1": "rsync://rsync1.de.gentoo.org/",
"upstream2": "rsync://rsync8.de.gentoo.org/", "upstream2": "rsync://rsync8.de.gentoo.org/",
"file": "gentoo-portage/Manifest" "file": "gentoo-portage/Manifest"
}, },
"GNOME": { "GNOME": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "gnome/", "csc": "gnome/",
"upstream1": "https://download.gnome.org/", "upstream1": "https://download.gnome.org/",
"upstream2": "https://mirrors.dotsrc.org/gnome/", "upstream2": "https://mirrors.dotsrc.org/gnome/",
"upstream3": "https://muug.ca/mirror/gnome/", "upstream3": "https://muug.ca/mirror/gnome/",
"file1": "core/", "file1": "core/",
"file2": "cache.json" "file2": "cache.json"
}, },
"GNU": { "GNU": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://mirrors.kernel.org/", "upstream": "https://mirrors.kernel.org/",
"file": "gnu/mirror-updated-timestamp.txt" "file": "gnu/mirror-updated-timestamp.txt"
}, },
"Gutenberg": { "Gutenberg": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 172800, "out_of_sync_interval": 172800,
"csc": "gutenberg/", "csc": "gutenberg/",
"upstream": "https://gutenberg.pglaf.org/", "upstream": "https://gutenberg.pglaf.org/",
"file": "gutenberg.dcs" "file": "gutenberg.dcs"
}, },
"IPFire": { "IPFire": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 172800 "out_of_sync_interval": 172800
}, },
"KDE": { "KDE": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "kde/", "csc": "kde/",
"upstream": "https://kde.c3sl.ufpr.br/", "upstream": "https://kde.c3sl.ufpr.br/",
"file": "ls-lR" "file": "ls-lR"
}, },
"KDEApplicationData": { "KDEApplicationData": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "kde-applicationdata/", "csc": "kde-applicationdata/",
"upstream": "https://cdn.files.kde.org/", "upstream": "https://cdn.files.kde.org/",
"file": "last-updated" "file": "last-updated"
}, },
"Kernel": { "Kernel": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "kernel.org/", "csc": "kernel.org/",
"upstream": "https://mirrors.edge.kernel.org/pub/", "upstream": "https://mirrors.edge.kernel.org/pub/",
"file": "linux/kernel/next/sha256sums.asc" "file": "linux/kernel/next/sha256sums.asc"
}, },
"OpenBSD": { "OpenBSD": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://ftp.openbsd.org/pub/", "upstream": "https://ftp.openbsd.org/pub/",
"file": "OpenBSD/timestamp" "file": "OpenBSD/timestamp"
}, },
"tdf": { "tdf": {
"out_of_sync_since": 1633294718, "out_of_sync_since": 1633294718,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://download.documentfoundation.org/TIMESTAMP", "upstream": "https://download.documentfoundation.org/TIMESTAMP",
"file": "tdf/TIMESTAMP" "file": "tdf/TIMESTAMP"
}, },
"ubuntu": { "ubuntu": {
"out_of_sync_since": null, "out_of_sync_since": null,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "", "csc": "",
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive", "upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
"file": "" "file": ""
}, },
"vlc": { "vlc": {
"out_of_sync_since": 1633298732, "out_of_sync_since": 1633298732,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "vlc/", "csc": "vlc/",
"upstream": "http://download.videolan.org/pub/videolan/", "upstream": "http://download.videolan.org/pub/videolan/",
"file": "trace" "file": "trace"
} },
} "manjaro": {
"out_of_sync_since": null,
"out_of_sync_interval": 86400,
"csc": "",
"upstream": "https://repo.manjaro.org/",
"file": ""
}
}

2
main.py
View File

@ -67,7 +67,7 @@ if __name__ == "__main__":
print(f"Failure: {project} does not exist") print(f"Failure: {project} does not exist")
continue continue
project_class = getattr(sys.modules[__name__], project) project_class = getattr(sys.modules[__name__], project)
if project == "CPAN": if project == "CPAN" or project == "ubuntu" or project == "manjaro":
checker_result = project_class.check(data, project, current_time) checker_result = project_class.check(data, project, current_time)
if checker_result: if checker_result:
print(f"Success: {project} up-to-date") print(f"Success: {project} up-to-date")

26
projects/manjaro.py Normal file
View File

@ -0,0 +1,26 @@
"""
Contains manjaro class
"""
import os
from project import Project
from shared import CSC_MIRROR
import requests
import datefinder # another date finding library
from datetime import timedelta
import re
import pandas as pd
class manjaro(Project):
"""manjaro class"""
@staticmethod
def check(data, project, current_time):
page = requests.get(data[project]["upstream"]).text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca/manjaro")
m = re.search(r'(?P<hours>\d+):(?P<minutes>\d+)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
duration = timedelta(**{key: float(val) for key, val in m.groupdict().items()})
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
# https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive

4
projects/tdf.py
View File

@ -2,7 +2,7 @@
Contains tdf class Contains tdf class
""" """
from distro import Distro from project import Project
class tdf(Distro): class tdf(Project):
"""tdf class""" """tdf class"""

28
projects/ubuntu.py
View File

@ -3,27 +3,23 @@ Contains ubuntu class
""" """
import os import os
from distro import Distro from project import Project
from shared import CSC_MIRROR from shared import CSC_MIRROR
import requests import requests
import datefinder # another date finding library import datefinder # another date finding library
from datetime import timedelta
from datetime import datetime
import re
import pandas as pd
class ubuntu(Distro): class ubuntu(Project):
"""ubuntu class""" """ubuntu class"""
@staticmethod @staticmethod
def check(data, distro, current_time): def check(data, project, current_time):
page = requests.get(data[distro]["upstream"]).text page = requests.get(data[project]["upstream"]).text
indexOfFile = page.find("last verified on") indexOfFile = page.find("last verified")
# segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
# segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
matches = list(datefinder.find_dates(page[indexOfFile:])) matches = list(datefinder.find_dates(page[indexOfFile:]))
# print(matches) date = matches[0] # date is of type datetime.datetime
return(pd.to_datetime(current_time, unit='s') - date.replace(tzinfo=None) <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s'))
if len(matches) > 0:
date = matches[0] # date is of type datetime.datetime
return(current_time - date.strftime("%Y/%m/%d, %H:%M:%S"))
else:
return(False)
# https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive # https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive

76
test.py Normal file
View File

@ -0,0 +1,76 @@
from bs4 import BeautifulSoup
import requests
import datefinder # another date finding library
import re
from datetime import datetime
from datetime import timedelta
import time
import pandas as pd
# lists
urls=[]
home_site = "http://ports.ubuntu.com"
# function created
def scrape(site):
# getting the request from url
r = requests.get(site)
# converting the text
s = BeautifulSoup(r.text,"html.parser")
for i in s.find_all("a"): # for a href directories
href = i.attrs['href']
if href.endswith("/") and href != "../" and href != "/":
if home_site+href in urls: # avoids the link to parent directory
continue
site_next = site+href
if site_next not in urls:
urls.append(site_next)
print(site_next)
# calling it self
scrape(site_next)
def get_latest_date(web_dir):
page = requests.get(site).text
str_dates = re.findall(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}', page)
dates = [list(datefinder.find_dates(date))[0] for date in str_dates]
# for date in dates:
# print(date)
return(max(dates))
# main function
if __name__ =="__main__":
# website to be scrape
# site="http://ports.ubuntu.com/ubuntu-ports/"
# works on: https://www.x.org/releases/
# calling function
# scrape(site)
# latest_date = get_latest_date(urls[0])
# get_latest_date(urls[0])
# for dir in urls:
# latest_date2 = get_latest_date(dir)
# if (latest_date2 >= latest_date):
# latest_date = latest_date2
# print(latest_date)
page = requests.get("https://repo.manjaro.org/").text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca/manjaro")
m = re.search(r'(?P<hours>\d+):(?P<minutes>\d+)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
duration = timedelta(**{key: float(val) for key, val in m.groupdict().items()})
print(duration)
print (duration <= pd.to_timedelta(86400, unit='s'))