Fixed 6 broken mirror checkers (#11)
Note that I updated the time tolerance for Fedora to 259200 seconds (3 days) and changed the ubuntu releases url Co-authored-by: Justin Toft <justintoft10@gmail.com> Reviewed-on: #11 Reviewed-by: Raymond Li <raymo@csclub.uwaterloo.ca> Co-authored-by: Justin Toft <jtoft@uwaterloo.ca> Co-committed-by: Justin Toft <jtoft@uwaterloo.ca>
This commit is contained in:
parent
65b06ac0a8
commit
860d5c8e44
23
data.json
23
data.json
|
@ -48,7 +48,8 @@
|
||||||
"file": "timestamp"
|
"file": "timestamp"
|
||||||
},
|
},
|
||||||
"CPAN": {
|
"CPAN": {
|
||||||
"out_of_sync_interval": 172800
|
"out_of_sync_interval": 172800,
|
||||||
|
"out_of_sync_since": null
|
||||||
},
|
},
|
||||||
"cran": {
|
"cran": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": null,
|
||||||
|
@ -86,7 +87,7 @@
|
||||||
"file": "debian-cd/project/trace/cdimage.debian.org"
|
"file": "debian-cd/project/trace/cdimage.debian.org"
|
||||||
},
|
},
|
||||||
"DebianMultimedia": {
|
"DebianMultimedia": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1659116719,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "debian-multimedia/",
|
"csc": "debian-multimedia/",
|
||||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
||||||
|
@ -116,7 +117,7 @@
|
||||||
},
|
},
|
||||||
"Fedora": {
|
"Fedora": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": null,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 259200,
|
||||||
"csc": "fedora/",
|
"csc": "fedora/",
|
||||||
"upstream": "http://fedora.mirror.iweb.com/",
|
"upstream": "http://fedora.mirror.iweb.com/",
|
||||||
"file": "linux/development/rawhide/COMPOSE_ID"
|
"file": "linux/development/rawhide/COMPOSE_ID"
|
||||||
|
@ -173,7 +174,7 @@
|
||||||
"out_of_sync_interval": 172800
|
"out_of_sync_interval": 172800
|
||||||
},
|
},
|
||||||
"KDE": {
|
"KDE": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1659116720,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "kde/",
|
"csc": "kde/",
|
||||||
"upstream": "https://kde.c3sl.ufpr.br/",
|
"upstream": "https://kde.c3sl.ufpr.br/",
|
||||||
|
@ -208,7 +209,7 @@
|
||||||
"file": "dists/"
|
"file": "dists/"
|
||||||
},
|
},
|
||||||
"macports": {
|
"macports": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1642827723,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "MacPorts/mpdistfiles/",
|
"csc": "MacPorts/mpdistfiles/",
|
||||||
"upstream": "https://distfiles.macports.org/",
|
"upstream": "https://distfiles.macports.org/",
|
||||||
|
@ -269,7 +270,7 @@
|
||||||
"exclude": true
|
"exclude": true
|
||||||
},
|
},
|
||||||
"opensuse": {
|
"opensuse": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1648699331,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "opensuse/update/",
|
"csc": "opensuse/update/",
|
||||||
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
|
"upstream": "http://opensuse-mirror-gce-us.opensu.se/update/",
|
||||||
|
@ -320,7 +321,7 @@
|
||||||
"file": "dists/"
|
"file": "dists/"
|
||||||
},
|
},
|
||||||
"raspbian": {
|
"raspbian": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1659116721,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "raspbian/",
|
"csc": "raspbian/",
|
||||||
"upstream": "http://archive.raspbian.org/",
|
"upstream": "http://archive.raspbian.org/",
|
||||||
|
@ -341,7 +342,7 @@
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
"slackware": {
|
"slackware": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1642827723,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "slackware/",
|
"csc": "slackware/",
|
||||||
"upstream": "https://mirrors.slackware.com/slackware/",
|
"upstream": "https://mirrors.slackware.com/slackware/",
|
||||||
|
@ -373,7 +374,7 @@
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
"ubuntu_ports": {
|
"ubuntu_ports": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1651550528,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "ubuntu-ports/",
|
"csc": "ubuntu-ports/",
|
||||||
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
|
"upstream": "http://ports.ubuntu.com/ubuntu-ports/",
|
||||||
|
@ -390,7 +391,7 @@
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": null,
|
||||||
"out_of_sync_interval": 172800,
|
"out_of_sync_interval": 172800,
|
||||||
"csc": "",
|
"csc": "",
|
||||||
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-release",
|
"upstream": "https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive",
|
||||||
"file": ""
|
"file": ""
|
||||||
},
|
},
|
||||||
"vlc": {
|
"vlc": {
|
||||||
|
@ -401,7 +402,7 @@
|
||||||
"file": "trace"
|
"file": "trace"
|
||||||
},
|
},
|
||||||
"x_org": {
|
"x_org": {
|
||||||
"out_of_sync_since": null,
|
"out_of_sync_since": 1657512131,
|
||||||
"out_of_sync_interval": 86400,
|
"out_of_sync_interval": 86400,
|
||||||
"csc": "x.org/individual/",
|
"csc": "x.org/individual/",
|
||||||
"upstream": "https://www.x.org/releases/individual/",
|
"upstream": "https://www.x.org/releases/individual/",
|
||||||
|
|
|
@ -3,7 +3,37 @@ Contains Debian class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class Debian(Project):
|
class Debian(Project):
|
||||||
"""Debian class"""
|
"""Debian class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
CSC_date = datetime.strptime(CSC.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
|
||||||
|
CSC_utc_time = time.mktime(CSC_date.timetuple())
|
||||||
|
|
||||||
|
upstream_date = datetime.strptime(upstream.partition('\n')[0], "%a %b %d %H:%M:%S UTC %Y")
|
||||||
|
upstream_utc_time = time.mktime(upstream_date.timetuple())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,7 +3,40 @@ Contains DebianCD class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class DebianCD(Project):
|
class DebianCD(Project):
|
||||||
"""DebianCD class"""
|
"""DebianCD class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Date Format Example: Sun 27 Mar 00:20:12 UTC 2022
|
||||||
|
date_format = "%a %d %b %H:%M:%S UTC %Y\n"
|
||||||
|
|
||||||
|
CSC_date = datetime.strptime(CSC, date_format)
|
||||||
|
CSC_utc_time = time.mktime(CSC_date.timetuple())
|
||||||
|
|
||||||
|
upstream_date = datetime.strptime(upstream, date_format)
|
||||||
|
upstream_utc_time = time.mktime(upstream_date.timetuple())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,7 +3,40 @@ Contains Fedora class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class Fedora(Project):
|
class Fedora(Project):
|
||||||
"""Fedora class"""
|
"""Fedora class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Date example: Fedora-Rawhide-20220725.n.1
|
||||||
|
date_format = "%Y%m%d"
|
||||||
|
|
||||||
|
CSC_date = datetime.strptime(CSC[15:23], "%Y%m%d")
|
||||||
|
CSC_utc_time = time.mktime(CSC_date.timetuple())
|
||||||
|
|
||||||
|
upstream_date = datetime.strptime(upstream[15:23], "%Y%m%d")
|
||||||
|
upstream_utc_time = time.mktime(upstream_date.timetuple())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return upstream_utc_time - CSC_utc_time < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,7 +3,35 @@ Contains GentooDistfiles class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class GentooDistfiles(Project):
|
class GentooDistfiles(Project):
|
||||||
"""GentooDistfiles class"""
|
"""GentooDistfiles class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
CSC_utc_time = CSC[0:11]
|
||||||
|
|
||||||
|
upstream_utc_time = upstream[0:11]
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream_utc_time) - int(CSC_utc_time) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -3,6 +3,31 @@ Contains nongnu class
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from project import Project
|
from project import Project
|
||||||
|
from shared import CSC_MIRROR
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
class nongnu(Project):
|
class nongnu(Project):
|
||||||
"""nongnu class"""
|
"""nongnu class"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def check(data, project, current_time):
|
||||||
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||||
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||||
|
|
||||||
|
req = requests.get(csc_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
CSC = req.text
|
||||||
|
|
||||||
|
req = requests.get(upstream_url)
|
||||||
|
req.raise_for_status()
|
||||||
|
upstream = req.text
|
||||||
|
|
||||||
|
if upstream == CSC:
|
||||||
|
return True
|
||||||
|
|
||||||
|
try:
|
||||||
|
return int(upstream.partition('\n')[0]) - int(CSC.partition('\n')[0]) < data[project]["out_of_sync_interval"]
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
|
|
@ -23,7 +23,8 @@ class raspberrypi(Project):
|
||||||
for i in s.find_all("a"): # for a href directories
|
for i in s.find_all("a"): # for a href directories
|
||||||
href = i.attrs['href']
|
href = i.attrs['href']
|
||||||
|
|
||||||
if href.endswith("/") and href != "../" and href != "/":
|
# The raspberry pi server doesn't use a relative path to the parent directory
|
||||||
|
if href.endswith("/") and href != "../" and href != "/" and href != "/debian/":
|
||||||
site_next = site+href+"Release"
|
site_next = site+href+"Release"
|
||||||
|
|
||||||
if site_next not in urls:
|
if site_next not in urls:
|
||||||
|
@ -44,6 +45,7 @@ class raspberrypi(Project):
|
||||||
cls.scrape(urls1, csc_url)
|
cls.scrape(urls1, csc_url)
|
||||||
cls.scrape(urls2, upstream_url)
|
cls.scrape(urls2, upstream_url)
|
||||||
|
|
||||||
|
|
||||||
if (len(urls1) != len(urls2)):
|
if (len(urls1) != len(urls2)):
|
||||||
return False
|
return False
|
||||||
urls1.sort()
|
urls1.sort()
|
||||||
|
|
Loading…
Reference in New Issue