forked from public/mirror-checker
parent
530d22f0ab
commit
1b53f605c9
@ -1,202 +1,209 @@ |
||||
{ |
||||
"AlmaLinux": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://repo.almalinux.org/", |
||||
"file": "almalinux/TIME" |
||||
}, |
||||
"Alpine": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://uk.alpinelinux.org/", |
||||
"file": "alpine/last-updated" |
||||
}, |
||||
"Apache": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "apache/", |
||||
"upstream": "https://downloads.apache.org/", |
||||
"file": "zzz/time.txt" |
||||
}, |
||||
"Arch": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "archlinux/", |
||||
"upstream": "http://arch.mirror.constant.com/", |
||||
"file": "lastupdate" |
||||
}, |
||||
"CentOS": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://mirrors.edge.kernel.org/", |
||||
"file": "centos/TIME" |
||||
}, |
||||
"Ceph": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "ceph/", |
||||
"upstream": "https://download.ceph.com/", |
||||
"file": "timestamp" |
||||
}, |
||||
"CPAN": { |
||||
"out_of_sync_interval": 172800 |
||||
}, |
||||
"Cygwin": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "cygwin/", |
||||
"upstream": "https://cygwin.mirror.globo.tech/", |
||||
"file": "x86/sha512.sum" |
||||
}, |
||||
"Debian": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://ftp-master.debian.org/", |
||||
"file": "debian/project/trace/master" |
||||
}, |
||||
"DebianCD": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "http://debian.mirror.estruxture.net/", |
||||
"file": "debian-cd/project/trace/cdimage.debian.org" |
||||
}, |
||||
"DebianMultimedia": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "debian-multimedia/", |
||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", |
||||
"file": "project/trace/deb-multimedia.org" |
||||
}, |
||||
"DebianPorts": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://deb.debian.org/", |
||||
"file": "debian-ports/project/trace/porta.debian.org" |
||||
}, |
||||
"DebianSecurity": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "http://debian.mirror.iweb.ca/", |
||||
"file": "debian-security/project/trace/master" |
||||
}, |
||||
"Eclipse": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 172800, |
||||
"csc": "eclipse/", |
||||
"upstream": "http://download.eclipse.org/", |
||||
"file": "TIME" |
||||
}, |
||||
"Fedora": { |
||||
"out_of_sync_since": 1633298732, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "fedora/", |
||||
"upstream": "http://fedora.mirror.iweb.com/", |
||||
"file": "linux/development/rawhide/COMPOSE_ID" |
||||
}, |
||||
"FreeBSD": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "http://ftp4.freebsd.org/pub/", |
||||
"file": "FreeBSD/TIMESTAMP" |
||||
}, |
||||
"GentooDistfiles": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "gentoo-distfiles", |
||||
"upstream": "http://gentoo.mirrors.tera-byte.com/", |
||||
"file": "distfiles/timestamp.dev-local" |
||||
}, |
||||
"GentooPortage": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "rsync://rsync4.ca.gentoo.org/", |
||||
"upstream1": "rsync://rsync1.de.gentoo.org/", |
||||
"upstream2": "rsync://rsync8.de.gentoo.org/", |
||||
"file": "gentoo-portage/Manifest" |
||||
}, |
||||
"GNOME": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "gnome/", |
||||
"upstream1": "https://download.gnome.org/", |
||||
"upstream2": "https://mirrors.dotsrc.org/gnome/", |
||||
"upstream3": "https://muug.ca/mirror/gnome/", |
||||
"file1": "core/", |
||||
"file2": "cache.json" |
||||
}, |
||||
"GNU": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://mirrors.kernel.org/", |
||||
"file": "gnu/mirror-updated-timestamp.txt" |
||||
}, |
||||
"Gutenberg": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 172800, |
||||
"csc": "gutenberg/", |
||||
"upstream": "https://gutenberg.pglaf.org/", |
||||
"file": "gutenberg.dcs" |
||||
}, |
||||
"IPFire": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 172800 |
||||
}, |
||||
"KDE": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "kde/", |
||||
"upstream": "https://kde.c3sl.ufpr.br/", |
||||
"file": "ls-lR" |
||||
}, |
||||
"KDEApplicationData": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "kde-applicationdata/", |
||||
"upstream": "https://cdn.files.kde.org/", |
||||
"file": "last-updated" |
||||
}, |
||||
"Kernel": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "kernel.org/", |
||||
"upstream": "https://mirrors.edge.kernel.org/pub/", |
||||
"file": "linux/kernel/next/sha256sums.asc" |
||||
}, |
||||
"OpenBSD": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://ftp.openbsd.org/pub/", |
||||
"file": "OpenBSD/timestamp" |
||||
}, |
||||
"tdf": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://download.documentfoundation.org/TIMESTAMP", |
||||
"file": "tdf/TIMESTAMP" |
||||
}, |
||||
"ubuntu": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive", |
||||
"file": "" |
||||
}, |
||||
"vlc": { |
||||
"out_of_sync_since": 1633298732, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "vlc/", |
||||
"upstream": "http://download.videolan.org/pub/videolan/", |
||||
"file": "trace" |
||||
} |
||||
} |
||||
"AlmaLinux": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://repo.almalinux.org/", |
||||
"file": "almalinux/TIME" |
||||
}, |
||||
"Alpine": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://uk.alpinelinux.org/", |
||||
"file": "alpine/last-updated" |
||||
}, |
||||
"Apache": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "apache/", |
||||
"upstream": "https://downloads.apache.org/", |
||||
"file": "zzz/time.txt" |
||||
}, |
||||
"Arch": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "archlinux/", |
||||
"upstream": "http://arch.mirror.constant.com/", |
||||
"file": "lastupdate" |
||||
}, |
||||
"CentOS": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://mirrors.edge.kernel.org/", |
||||
"file": "centos/TIME" |
||||
}, |
||||
"Ceph": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "ceph/", |
||||
"upstream": "https://download.ceph.com/", |
||||
"file": "timestamp" |
||||
}, |
||||
"CPAN": { |
||||
"out_of_sync_interval": 172800 |
||||
}, |
||||
"Cygwin": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "cygwin/", |
||||
"upstream": "https://cygwin.mirror.globo.tech/", |
||||
"file": "x86/sha512.sum" |
||||
}, |
||||
"Debian": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://ftp-master.debian.org/", |
||||
"file": "debian/project/trace/master" |
||||
}, |
||||
"DebianCD": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "http://debian.mirror.estruxture.net/", |
||||
"file": "debian-cd/project/trace/cdimage.debian.org" |
||||
}, |
||||
"DebianMultimedia": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "debian-multimedia/", |
||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", |
||||
"file": "project/trace/deb-multimedia.org" |
||||
}, |
||||
"DebianPorts": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://deb.debian.org/", |
||||
"file": "debian-ports/project/trace/porta.debian.org" |
||||
}, |
||||
"DebianSecurity": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "http://debian.mirror.iweb.ca/", |
||||
"file": "debian-security/project/trace/master" |
||||
}, |
||||
"Eclipse": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 172800, |
||||
"csc": "eclipse/", |
||||
"upstream": "http://download.eclipse.org/", |
||||
"file": "TIME" |
||||
}, |
||||
"Fedora": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "fedora/", |
||||
"upstream": "http://fedora.mirror.iweb.com/", |
||||
"file": "linux/development/rawhide/COMPOSE_ID" |
||||
}, |
||||
"FreeBSD": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "http://ftp4.freebsd.org/pub/", |
||||
"file": "FreeBSD/TIMESTAMP" |
||||
}, |
||||
"GentooDistfiles": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "gentoo-distfiles", |
||||
"upstream": "http://gentoo.mirrors.tera-byte.com/", |
||||
"file": "distfiles/timestamp.dev-local" |
||||
}, |
||||
"GentooPortage": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "rsync://rsync4.ca.gentoo.org/", |
||||
"upstream1": "rsync://rsync1.de.gentoo.org/", |
||||
"upstream2": "rsync://rsync8.de.gentoo.org/", |
||||
"file": "gentoo-portage/Manifest" |
||||
}, |
||||
"GNOME": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "gnome/", |
||||
"upstream1": "https://download.gnome.org/", |
||||
"upstream2": "https://mirrors.dotsrc.org/gnome/", |
||||
"upstream3": "https://muug.ca/mirror/gnome/", |
||||
"file1": "core/", |
||||
"file2": "cache.json" |
||||
}, |
||||
"GNU": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://mirrors.kernel.org/", |
||||
"file": "gnu/mirror-updated-timestamp.txt" |
||||
}, |
||||
"Gutenberg": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 172800, |
||||
"csc": "gutenberg/", |
||||
"upstream": "https://gutenberg.pglaf.org/", |
||||
"file": "gutenberg.dcs" |
||||
}, |
||||
"IPFire": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 172800 |
||||
}, |
||||
"KDE": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "kde/", |
||||
"upstream": "https://kde.c3sl.ufpr.br/", |
||||
"file": "ls-lR" |
||||
}, |
||||
"KDEApplicationData": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "kde-applicationdata/", |
||||
"upstream": "https://cdn.files.kde.org/", |
||||
"file": "last-updated" |
||||
}, |
||||
"Kernel": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "kernel.org/", |
||||
"upstream": "https://mirrors.edge.kernel.org/pub/", |
||||
"file": "linux/kernel/next/sha256sums.asc" |
||||
}, |
||||
"OpenBSD": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://ftp.openbsd.org/pub/", |
||||
"file": "OpenBSD/timestamp" |
||||
}, |
||||
"tdf": { |
||||
"out_of_sync_since": 1633294718, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://download.documentfoundation.org/TIMESTAMP", |
||||
"file": "tdf/TIMESTAMP" |
||||
}, |
||||
"ubuntu": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive", |
||||
"file": "" |
||||
}, |
||||
"vlc": { |
||||
"out_of_sync_since": 1633298732, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "vlc/", |
||||
"upstream": "http://download.videolan.org/pub/videolan/", |
||||
"file": "trace" |
||||
}, |
||||
"manjaro": { |
||||
"out_of_sync_since": null, |
||||
"out_of_sync_interval": 86400, |
||||
"csc": "", |
||||
"upstream": "https://repo.manjaro.org/", |
||||
"file": "" |
||||
} |
||||
} |
@ -0,0 +1,26 @@ |
||||
""" |
||||
Contains manjaro class |
||||
""" |
||||
|
||||
import os |
||||
from project import Project |
||||
from shared import CSC_MIRROR |
||||
import requests |
||||
import datefinder # another date finding library |
||||
from datetime import timedelta |
||||
import re |
||||
import pandas as pd |
||||
|
||||
class manjaro(Project): |
||||
"""manjaro class""" |
||||
@staticmethod |
||||
def check(data, project, current_time): |
||||
page = requests.get(data[project]["upstream"]).text |
||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca/manjaro") |
||||
|
||||
m = re.search(r'(?P<hours>\d+):(?P<minutes>\d+)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460 |
||||
duration = timedelta(**{key: float(val) for key, val in m.groupdict().items()}) |
||||
|
||||
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s') |
||||
|
||||
# https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive |
@ -0,0 +1,76 @@ |
||||
from bs4 import BeautifulSoup |
||||
import requests |
||||
import datefinder # another date finding library |
||||
import re |
||||
from datetime import datetime |
||||
from datetime import timedelta |
||||
import time |
||||
import pandas as pd |
||||
|
||||
# lists |
||||
urls=[] |
||||
|
||||
home_site = "http://ports.ubuntu.com" |
||||
|
||||
# function created |
||||
def scrape(site): |
||||
|
||||
# getting the request from url |
||||
r = requests.get(site) |
||||
|
||||
# converting the text |
||||
s = BeautifulSoup(r.text,"html.parser") |
||||
|
||||
for i in s.find_all("a"): # for a href directories |
||||
href = i.attrs['href'] |
||||
|
||||
if href.endswith("/") and href != "../" and href != "/": |
||||
if home_site+href in urls: # avoids the link to parent directory |
||||
continue |
||||
site_next = site+href |
||||
|
||||
if site_next not in urls: |
||||
urls.append(site_next) |
||||
print(site_next) |
||||
# calling it self |
||||
scrape(site_next) |
||||
|
||||
def get_latest_date(web_dir): |
||||
page = requests.get(site).text |
||||
|
||||
str_dates = re.findall(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}', page) |
||||
dates = [list(datefinder.find_dates(date))[0] for date in str_dates] |
||||
|
||||
# for date in dates: |
||||
# print(date) |
||||
|
||||
return(max(dates)) |
||||
|
||||
# main function |
||||
if __name__ =="__main__": |
||||
|
||||
# website to be scrape |
||||
# site="http://ports.ubuntu.com/ubuntu-ports/" |
||||
# works on: https://www.x.org/releases/ |
||||
|
||||
# calling function |
||||
# scrape(site) |
||||
|
||||
# latest_date = get_latest_date(urls[0]) |
||||
# get_latest_date(urls[0]) |
||||
# for dir in urls: |
||||
# latest_date2 = get_latest_date(dir) |
||||
# if (latest_date2 >= latest_date): |
||||
# latest_date = latest_date2 |
||||
|
||||
# print(latest_date) |
||||
|
||||
page = requests.get("https://repo.manjaro.org/").text |
||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca/manjaro") |
||||
|
||||
m = re.search(r'(?P<hours>\d+):(?P<minutes>\d+)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460 |
||||
duration = timedelta(**{key: float(val) for key, val in m.groupdict().items()}) |
||||
print(duration) |
||||
|
||||
print (duration <= pd.to_timedelta(86400, unit='s')) |
||||
|
Loading…
Reference in new issue