forked from public/mirror-checker
added pkgsrc qtproject raspbian slackware trisquel ubuntu-ports ubuntu-releases
This commit is contained in:
parent
513bf63fc4
commit
dba5643c4e
24
README.md
24
README.md
|
@ -29,12 +29,15 @@ linuxmint-packages pool: http://rsync-packages.linuxmint.com/pool/
|
||||||
macPorts: only distfiles has public repo, no timestamp, too large to loop through
|
macPorts: only distfiles has public repo, no timestamp, too large to loop through
|
||||||
NetBSD: http://ftp.netbsd.org/pub/NetBSD/ has public repo, no timestamp, web directory hard to loop through, no mirror tracker
|
NetBSD: http://ftp.netbsd.org/pub/NetBSD/ has public repo, no timestamp, web directory hard to loop through, no mirror tracker
|
||||||
opensuse: http://download.opensuse.org/ has public repo, a possible timestamp called latest in history, our mirror doesn't have this file tho, no mirror tracker
|
opensuse: http://download.opensuse.org/ has public repo, a possible timestamp called latest in history, our mirror doesn't have this file tho, no mirror tracker
|
||||||
scientific: https://scientificlinux.org/downloads/sl-mirrors/ (CSC not listed)
|
puppylinux: https://distro.ibiblio.org/puppylinux/ has public repo, no timestamp, too hard to loop through, not likely to have a mirror tracker
|
||||||
slackware: https://mirrors.slackware.com/mirrorlist/ https://mirrors.slackware.com/slackware/
|
racket: no public repo, no timestamp, no mirror status tracker
|
||||||
ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/
|
raspberry pi: currently no public repo, no timestamp, no mirror status tracker
|
||||||
ubuntu-ports-releases: https://cdimage.ubuntu.com/releases/
|
sagemath: don't know how to deal with this, it's a website
|
||||||
ubuntu-releases: https://releases.ubuntu.com/
|
salt stack: don't know how to deal with this, it's a website
|
||||||
x.org: https://www.x.org/releases/
|
scientific: https://scientificlinux.org/downloads/sl-mirrors/ would be easy to scrape the mirror status page, except that csc is not listed here
|
||||||
|
ubuntu-ports-releases: https://cdimage.ubuntu.com/releases/ has public repo, no timestamp, no status tracker
|
||||||
|
x.org: https://www.x.org/releases/ no timestamp, but candidate for brute force looping since it has few folders, no status tracker
|
||||||
|
Xiph: no timestamp, too big to loop through, no status tracker
|
||||||
|
|
||||||
done:
|
done:
|
||||||
tdf: https://download.documentfoundation.org/
|
tdf: https://download.documentfoundation.org/
|
||||||
|
@ -46,5 +49,10 @@ mxlinux-iso: this one seems out of sync on the official tracker for 134 days, wh
|
||||||
mysql: http://mirrors.sunsite.dk/mysql/
|
mysql: http://mirrors.sunsite.dk/mysql/
|
||||||
nongnu: http://download.savannah.nongnu.org/releases/ https://savannah.gnu.org/maintenance/Mirmon/ http://download.savannah.gnu.org/mirmon/savannah/
|
nongnu: http://download.savannah.nongnu.org/releases/ https://savannah.gnu.org/maintenance/Mirmon/ http://download.savannah.gnu.org/mirmon/savannah/
|
||||||
parabola: https://repo.parabola.nu/ https://www.parabola.nu/mirrors/status/
|
parabola: https://repo.parabola.nu/ https://www.parabola.nu/mirrors/status/
|
||||||
|
pkgsrc
|
||||||
trisquel: https://trisquel.info/mirmon/index.html out of date website
|
qtproject: https://download.qt.io/
|
||||||
|
raspbian: http://archive.raspbian.org/raspbian/ snapshotindex.txt is most likely a timestamp, tho i'm not sure. also i think our mirror is completely outdated, it's not listed on official mirror list
|
||||||
|
slackware: https://mirrors.slackware.com/mirrorlist/ https://mirrors.slackware.com/slackware/ checking using the last updated date here, don't know if it's entirely accurate
|
||||||
|
trisquel: https://trisquel.info/mirmon/index.html out of date website!? please recheck this!!!
|
||||||
|
ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/ checks the file anonster.canonical.com, which appears to be a timestamp (check it to make sure!!!)
|
||||||
|
ubuntu-releases: https://releases.ubuntu.com/
|
61
data.json
61
data.json
|
@ -35,7 +35,7 @@
|
||||||
"file": "centos/TIME"
|
"file": "centos/TIME"
|
||||||
},
|
},
|
||||||
"Ceph": {
|
"Ceph": {
|
||||||
"out_of_sync_since": 1633329349,
|
"out_of_sync_since": 1633340186,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "ceph/",
|
"csc": "ceph/",
|
||||||
"upstream": "https://download.ceph.com/",
|
"upstream": "https://download.ceph.com/",
|
||||||
|
@ -52,7 +52,7 @@
|
||||||
"file": "x86/sha512.sum"
|
"file": "x86/sha512.sum"
|
||||||
},
|
},
|
||||||
"Debian": {
|
"Debian": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1633337502,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "",
|
"csc": "",
|
||||||
"upstream": "https://ftp-master.debian.org/",
|
"upstream": "https://ftp-master.debian.org/",
|
||||||
|
@ -66,7 +66,7 @@
|
||||||
"file": "debian-cd/project/trace/cdimage.debian.org"
|
"file": "debian-cd/project/trace/cdimage.debian.org"
|
||||||
},
|
},
|
||||||
"DebianMultimedia": {
|
"DebianMultimedia": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1633340186,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "debian-multimedia/",
|
"csc": "debian-multimedia/",
|
||||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
||||||
|
@ -80,7 +80,7 @@
|
||||||
"file": "debian-ports/project/trace/porta.debian.org"
|
"file": "debian-ports/project/trace/porta.debian.org"
|
||||||
},
|
},
|
||||||
"DebianSecurity": {
|
"DebianSecurity": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1633337502,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "",
|
"csc": "",
|
||||||
"upstream": "http://debian.mirror.iweb.ca/",
|
"upstream": "http://debian.mirror.iweb.ca/",
|
||||||
|
@ -234,11 +234,60 @@
|
||||||
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
|
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
|
||||||
"file": "00_TIME.txt"
|
"file": "00_TIME.txt"
|
||||||
},
|
},
|
||||||
"parabola": {
|
"parabola": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1633335556,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "parabola/",
|
"csc": "parabola/",
|
||||||
"upstream": "https://repo.parabola.nu/",
|
"upstream": "https://repo.parabola.nu/",
|
||||||
"file": "lastsync"
|
"file": "lastsync"
|
||||||
|
},
|
||||||
|
"pkgsrc": {
|
||||||
|
"out_of_sync_since": 1633335556,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "pkgsrc/",
|
||||||
|
"upstream": "http://ftp.netbsd.org/pub/pkgsrc/",
|
||||||
|
"file": "MIRROR-TIMESTAMP"
|
||||||
|
},
|
||||||
|
"qtproject": {
|
||||||
|
"out_of_sync_since": 1633340186,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "qtproject/",
|
||||||
|
"upstream": "https://download.qt.io/",
|
||||||
|
"file": "timestamp.txt"
|
||||||
|
},
|
||||||
|
"raspbian": {
|
||||||
|
"out_of_sync_since": 1633337502,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "raspbian/",
|
||||||
|
"upstream": "http://archive.raspbian.org/",
|
||||||
|
"file": "snapshotindex.txt"
|
||||||
|
},
|
||||||
|
"slackware": {
|
||||||
|
"out_of_sync_since": null,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "",
|
||||||
|
"upstream": "https://mirrors.slackware.com/mirrorlist/",
|
||||||
|
"file": ""
|
||||||
|
},
|
||||||
|
"trisquel": {
|
||||||
|
"out_of_sync_since": null,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "",
|
||||||
|
"upstream": "https://trisquel.info/mirmon/index.html",
|
||||||
|
"file": ""
|
||||||
|
},
|
||||||
|
"ubuntu_ports": {
|
||||||
|
"out_of_sync_since": 1633341982,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "ubuntu-ports/project/trace/",
|
||||||
|
"upstream": "http://ports.ubuntu.com/ubuntu-ports/project/trace/",
|
||||||
|
"file": "anonster.canonical.com"
|
||||||
|
},
|
||||||
|
"ubuntu_releases": {
|
||||||
|
"out_of_sync_since": null,
|
||||||
|
"out_of_sync_interval": 86400,
|
||||||
|
"csc": "",
|
||||||
|
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release",
|
||||||
|
"file": ""
|
||||||
}
|
}
|
||||||
}
|
}
|
2
main.py
2
main.py
|
@ -67,7 +67,7 @@ if __name__ == "__main__":
|
||||||
print(f"Failure: {project} does not exist")
|
print(f"Failure: {project} does not exist")
|
||||||
continue
|
continue
|
||||||
project_class = getattr(sys.modules[__name__], project)
|
project_class = getattr(sys.modules[__name__], project)
|
||||||
if project == "CPAN" or project == "ubuntu" or project == "manjaro" or project == "mxlinux" or project == "mxlinux_iso":
|
if project == "CPAN" or project == "ubuntu" or project == "ubuntu_releases" or project == "manjaro" or project == "mxlinux" or project == "mxlinux_iso" or project == "slackware" or project == "trisquel":
|
||||||
checker_result = project_class.check(data, project, current_time)
|
checker_result = project_class.check(data, project, current_time)
|
||||||
if checker_result:
|
if checker_result:
|
||||||
print(f"Success: {project} up-to-date")
|
print(f"Success: {project} up-to-date")
|
||||||
|
|
|
@ -19,8 +19,8 @@ class mxlinux(Project):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
m = re.search(r'(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
|
|
||||||
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
|
@ -19,7 +19,7 @@ class mxlinux_iso(Project):
|
||||||
page = requests.get(data[project]["upstream"]).text
|
page = requests.get(data[project]["upstream"]).text
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
m = re.search(r'(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
duration = pd.to_timedelta(m.group(0))
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
"""
|
||||||
|
Contains pkgsrc class
|
||||||
|
"""
|
||||||
|
|
||||||
|
from project import Project
|
||||||
|
|
||||||
|
class pkgsrc(Project):
|
||||||
|
"""pkgsrc class"""
|
|
@ -0,0 +1,8 @@
|
||||||
|
"""
|
||||||
|
Contains qtproject class
|
||||||
|
"""
|
||||||
|
|
||||||
|
from project import Project
|
||||||
|
|
||||||
|
class qtproject(Project):
|
||||||
|
"""qtproject class"""
|
|
@ -0,0 +1,8 @@
|
||||||
|
"""
|
||||||
|
Contains raspbian class
|
||||||
|
"""
|
||||||
|
|
||||||
|
from project import Project
|
||||||
|
|
||||||
|
class raspbian(Project):
|
||||||
|
"""raspbian class"""
|
|
@ -0,0 +1,26 @@
|
||||||
|
"""
|
||||||
|
Contains slackware class
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
import datefinder # another date finding library
|
||||||
|
from datetime import timedelta
|
||||||
|
from datetime import datetime
|
||||||
|
import re
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
class slackware(Project):
|
||||||
|
"""slackware class"""
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
page = requests.get(data[project]["upstream"]).text
|
||||||
|
if (page.find("mirror.csclub.uwaterloo.ca/slackware/") != -1):
|
||||||
|
indexOfFile = page.find("Last Updated:")
|
||||||
|
matches = list(datefinder.find_dates(page[indexOfFile:]))
|
||||||
|
date = matches[0] # date is of type datetime.datetime
|
||||||
|
return(pd.to_datetime(current_time, unit='s') - date.replace(tzinfo=None) <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s'))
|
||||||
|
else:
|
||||||
|
return False
|
|
@ -0,0 +1,26 @@
|
||||||
|
"""
|
||||||
|
Contains trisquel class
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
import datefinder # another date finding library
|
||||||
|
from datetime import timedelta
|
||||||
|
from datetime import datetime
|
||||||
|
import re
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
class trisquel(Project):
|
||||||
|
"""trisquel class"""
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
page = requests.get(data[project]["upstream"]).text
|
||||||
|
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
||||||
|
|
||||||
|
m = re.search(r'(\d+ hour)|(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
||||||
|
|
||||||
|
duration = pd.to_timedelta(m.group(0))
|
||||||
|
|
||||||
|
return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')
|
|
@ -0,0 +1,8 @@
|
||||||
|
"""
|
||||||
|
Contains ubuntu_ports class
|
||||||
|
"""
|
||||||
|
|
||||||
|
from project import Project
|
||||||
|
|
||||||
|
class ubuntu_ports(Project):
|
||||||
|
"""ubuntu_ports class"""
|
|
@ -0,0 +1,25 @@
|
||||||
|
"""
|
||||||
|
Contains ubuntu_releases class
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
import datefinder # another date finding library
|
||||||
|
from datetime import timedelta
|
||||||
|
from datetime import datetime
|
||||||
|
import re
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
class ubuntu_releases(Project):
|
||||||
|
"""ubuntu_releases class"""
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
page = requests.get(data[project]["upstream"]).text
|
||||||
|
indexOfFile = page.find("last verified")
|
||||||
|
matches = list(datefinder.find_dates(page[indexOfFile:]))
|
||||||
|
date = matches[0] # date is of type datetime.datetime
|
||||||
|
return(pd.to_datetime(current_time, unit='s') - date.replace(tzinfo=None) <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s'))
|
||||||
|
|
||||||
|
# https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release
|
16
test.py
16
test.py
|
@ -54,7 +54,7 @@ def get_latest_date(web_dir):
|
||||||
# main function
|
# main function
|
||||||
if __name__ =="__main__":
|
if __name__ =="__main__":
|
||||||
|
|
||||||
# website to be scrape
|
"""# website to be scrape
|
||||||
site="http://ftp.netbsd.org/pub/NetBSD/"
|
site="http://ftp.netbsd.org/pub/NetBSD/"
|
||||||
# works on: https://www.x.org/releases/
|
# works on: https://www.x.org/releases/
|
||||||
# https://mirror.csclub.uwaterloo.ca/linuxmint/ #works wonders for linuxmint
|
# https://mirror.csclub.uwaterloo.ca/linuxmint/ #works wonders for linuxmint
|
||||||
|
@ -70,14 +70,10 @@ if __name__ =="__main__":
|
||||||
if (latest_date2 >= latest_date):
|
if (latest_date2 >= latest_date):
|
||||||
latest_date = latest_date2
|
latest_date = latest_date2
|
||||||
|
|
||||||
print(latest_date)
|
print(latest_date)"""
|
||||||
|
|
||||||
"""page = requests.get("http://rsync-mxlinux.org/mirmon/index.html").text
|
csc_url = "https://mirror.csclub.uwaterloo.ca/ubuntu-ports/project/trace/anonster.canonical.com"
|
||||||
indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
|
upstream_url = "http://ports.ubuntu.com/ubuntu-ports/project/trace/anonster.canonical.com"
|
||||||
|
print(requests.get(upstream_url).text)
|
||||||
m = re.search(r'(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
|
print(requests.get(csc_url).text == requests.get(upstream_url).text)
|
||||||
|
|
||||||
duration = pd.to_timedelta(m.group(0))
|
|
||||||
|
|
||||||
print (duration <= pd.to_timedelta(86400, unit='s'))"""
|
|
||||||
|
|
Loading…
Reference in New Issue