forked from public/mirror-checker
Refactor and use requirements file
This commit is contained in:
parent
6b0d3f4de9
commit
3428e81cec
|
@ -139,3 +139,4 @@ cython_debug/
|
|||
|
||||
# Idea
|
||||
.idea/
|
||||
*.txt~
|
||||
|
|
17
cpan.py
17
cpan.py
|
@ -1,17 +0,0 @@
|
|||
"""
|
||||
Contains CPAN class
|
||||
"""
|
||||
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class CPAN(Distro):
|
||||
"""CPAN class"""
|
||||
@staticmethod
|
||||
def check(data, distro, current_time):
|
||||
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
|
||||
for mirror in res_json:
|
||||
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
|
||||
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
|
||||
return False
|
|
@ -3,11 +3,15 @@ Contains abstract class for a distro
|
|||
"""
|
||||
|
||||
from abc import ABC
|
||||
|
||||
import requests
|
||||
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
|
||||
class Distro(ABC):
|
||||
"""Abstract class for a distro"""
|
||||
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
"""Check if distro packages are up-to-date"""
|
||||
|
|
21
distros.json
21
distros.json
|
@ -1,5 +1,20 @@
|
|||
[
|
||||
["OpenBSD", "https://mirror.csclub.uwaterloo.ca/OpenBSD/", "https://ftp.openbsd.org/pub/OpenBSD/", "timestamp"],
|
||||
["kernel", "http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/", "https://mirrors.edge.kernel.org/pub/linux/kernel/next/", "sha256sums.asc"],
|
||||
["debian", "http://mirror.csclub.uwaterloo.ca/debian/project/trace/", "https://ftp-master.debian.org/debian/project/trace/", "master"]
|
||||
[
|
||||
"OpenBSD",
|
||||
"https://mirror.csclub.uwaterloo.ca/OpenBSD/",
|
||||
"https://ftp.openbsd.org/pub/OpenBSD/",
|
||||
"timestamp"
|
||||
],
|
||||
[
|
||||
"kernel",
|
||||
"http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/",
|
||||
"https://mirrors.edge.kernel.org/pub/linux/kernel/next/",
|
||||
"sha256sums.asc"
|
||||
],
|
||||
[
|
||||
"debian",
|
||||
"http://mirror.csclub.uwaterloo.ca/debian/project/trace/",
|
||||
"https://ftp-master.debian.org/debian/project/trace/",
|
||||
"master"
|
||||
]
|
||||
]
|
|
@ -1,29 +0,0 @@
|
|||
"""
|
||||
Contains GentooPortage class
|
||||
"""
|
||||
|
||||
import os
|
||||
from distro import Distro
|
||||
|
||||
class GentooPortage(Distro):
|
||||
"""GentooPortage class"""
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
rsync_command = "rsync -q {}{} {}"
|
||||
os.system(rsync_command.format(data[distro]["csc"],
|
||||
data[distro]["file"],
|
||||
"csc_manifest"))
|
||||
os.system(rsync_command.format(data[distro]["upstream1"],
|
||||
data[distro]["file"],
|
||||
"upstream_manifest1"))
|
||||
os.system(rsync_command.format(data[distro]["upstream2"],
|
||||
data[distro]["file"],
|
||||
"upstream_manifest2"))
|
||||
stream1 = os.popen("diff csc_manifest upstream_manifest1")
|
||||
output1 = stream1.read()
|
||||
stream2 = os.popen("diff csc_manifest upstream_manifest2")
|
||||
output2 = stream2.read()
|
||||
os.system("rm csc_manifest")
|
||||
os.system("rm upstream_manifest1")
|
||||
os.system("rm upstream_manifest2")
|
||||
return 0 in [len(output1), len(output2)]
|
40
gnome.py
40
gnome.py
|
@ -1,40 +0,0 @@
|
|||
"""
|
||||
Contains GNOME class
|
||||
"""
|
||||
|
||||
import re
|
||||
import requests
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
class GNOME(Distro):
|
||||
"""GNOME class"""
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
file = data[distro]["file1"]
|
||||
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
|
||||
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
|
||||
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
|
||||
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
|
||||
if csc_latest != upstream_latest:
|
||||
return False
|
||||
file += csc_latest + "/"
|
||||
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
|
||||
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
|
||||
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
|
||||
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
|
||||
if csc_latest != upstream_latest:
|
||||
return False
|
||||
file += csc_latest + "/"
|
||||
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
try:
|
||||
ret = csc_text == requests.get(data[distro]["upstream2"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
except requests.exceptions.RequestException:
|
||||
ret = False
|
||||
try:
|
||||
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
14
ipfire.py
14
ipfire.py
|
@ -1,14 +0,0 @@
|
|||
"""
|
||||
Contains IPFire class
|
||||
"""
|
||||
|
||||
import requests
|
||||
from distro import Distro
|
||||
|
||||
class IPFire(Distro):
|
||||
"""IPFire class"""
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
|
||||
ipfire_text = requests.get(ipfire_url).text
|
||||
return ipfire_text.find("The mirror is up") != -1
|
54
main.py
54
main.py
|
@ -7,32 +7,32 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst
|
|||
import time
|
||||
import sys
|
||||
import requests
|
||||
from almalinux import AlmaLinux
|
||||
from alpine import Alpine
|
||||
from apache import Apache
|
||||
from arch import Arch
|
||||
from centos import CentOS
|
||||
from ceph import Ceph
|
||||
from cpan import CPAN
|
||||
from cygwin import Cygwin
|
||||
from debian import Debian
|
||||
from debiancd import DebianCD
|
||||
from debianmultimedia import DebianMultimedia
|
||||
from debianports import DebianPorts
|
||||
from debiansecurity import DebianSecurity
|
||||
from eclipse import Eclipse
|
||||
from fedora import Fedora
|
||||
from freebsd import FreeBSD
|
||||
from gentoodistfiles import GentooDistfiles
|
||||
from gentooportage import GentooPortage
|
||||
from gnome import GNOME
|
||||
from gnu import GNU
|
||||
from gutenberg import Gutenberg
|
||||
from ipfire import IPFire
|
||||
from kde import KDE
|
||||
from kdeapplicationdata import KDEApplicationData
|
||||
from kernel import Kernel
|
||||
from openbsd import OpenBSD
|
||||
from projects.almalinux import AlmaLinux
|
||||
from projects.alpine import Alpine
|
||||
from projects.apache import Apache
|
||||
from projects.arch import Arch
|
||||
from projects.centos import CentOS
|
||||
from projects.ceph import Ceph
|
||||
from projects.cpan import CPAN
|
||||
from projects.cygwin import Cygwin
|
||||
from projects.debian import Debian
|
||||
from projects.debiancd import DebianCD
|
||||
from projects.debianmultimedia import DebianMultimedia
|
||||
from projects.debianports import DebianPorts
|
||||
from projects.debiansecurity import DebianSecurity
|
||||
from projects.eclipse import Eclipse
|
||||
from projects.fedora import Fedora
|
||||
from projects.freebsd import FreeBSD
|
||||
from projects.gentoodistfiles import GentooDistfiles
|
||||
from projects.gentooportage import GentooPortage
|
||||
from projects.gnome import GNOME
|
||||
from projects.gnu import GNU
|
||||
from projects.gutenberg import Gutenberg
|
||||
from projects.ipfire import IPFire
|
||||
from projects.kde import KDE
|
||||
from projects.kdeapplicationdata import KDEApplicationData
|
||||
from projects.kernel import Kernel
|
||||
from projects.openbsd import OpenBSD
|
||||
from shared import CSC_MIRROR
|
||||
from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
|
||||
# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
|
||||
|
@ -41,6 +41,7 @@ import json # import json to read distro info stored in json file
|
|||
|
||||
import datefinder # another date finding library
|
||||
|
||||
|
||||
def checker(directory_URL, file_name):
|
||||
page = requests.get(directory_URL).text
|
||||
indexOfFile = page.find(file_name)
|
||||
|
@ -67,6 +68,7 @@ def checker(directory_URL, file_name):
|
|||
else:
|
||||
return ('No dates found')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
"""distros = json.load(open('distros.json',))
|
||||
print(distros)
|
||||
|
|
|
@ -4,5 +4,6 @@ Contains AlmaLinux class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class AlmaLinux(Distro):
|
||||
"""AlmaLinux class"""
|
|
@ -4,5 +4,6 @@ Contains Alpine class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Alpine(Distro):
|
||||
"""Alpine class"""
|
|
@ -4,5 +4,6 @@ Contains Apache class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Apache(Distro):
|
||||
"""Apache class"""
|
|
@ -4,5 +4,6 @@ Contains Arch class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Arch(Distro):
|
||||
"""Arch class"""
|
|
@ -4,5 +4,6 @@ Contains CentOS class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class CentOS(Distro):
|
||||
"""CentOS class"""
|
|
@ -4,5 +4,6 @@ Contains Ceph class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Ceph(Distro):
|
||||
"""Ceph class"""
|
|
@ -0,0 +1,20 @@
|
|||
"""
|
||||
Contains CPAN class
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
|
||||
class CPAN(Distro):
|
||||
"""CPAN class"""
|
||||
|
||||
@staticmethod
|
||||
def check(data, distro, current_time):
|
||||
res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json()
|
||||
for mirror in res_json:
|
||||
if mirror["url"] == f"{CSC_MIRROR}CPAN/":
|
||||
return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"]
|
||||
return False
|
|
@ -4,5 +4,6 @@ Contains Cygwin class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Cygwin(Distro):
|
||||
"""Cygwin class"""
|
|
@ -4,5 +4,6 @@ Contains Debian class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Debian(Distro):
|
||||
"""Debian class"""
|
|
@ -4,5 +4,6 @@ Contains DebianCD class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class DebianCD(Distro):
|
||||
"""DebianCD class"""
|
|
@ -4,5 +4,6 @@ Contains DebianMultimedia class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class DebianMultimedia(Distro):
|
||||
"""DebianMultimedia class"""
|
|
@ -4,5 +4,6 @@ Contains DebianPorts class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class DebianPorts(Distro):
|
||||
"""DebianPorts class"""
|
|
@ -4,5 +4,6 @@ Contains DebianSecurity class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class DebianSecurity(Distro):
|
||||
"""DebianSecurity class"""
|
|
@ -4,5 +4,6 @@ Contains Eclipse class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Eclipse(Distro):
|
||||
"""Eclipse class"""
|
|
@ -4,5 +4,6 @@ Contains Fedora class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Fedora(Distro):
|
||||
"""Fedora class"""
|
|
@ -4,5 +4,6 @@ Contains FreeBSD class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class FreeBSD(Distro):
|
||||
"""FreeBSD class"""
|
|
@ -4,5 +4,6 @@ Contains GentooDistfiles class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class GentooDistfiles(Distro):
|
||||
"""GentooDistfiles class"""
|
|
@ -0,0 +1,32 @@
|
|||
"""
|
||||
Contains GentooPortage class
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class GentooPortage(Distro):
|
||||
"""GentooPortage class"""
|
||||
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
rsync_command = "rsync -q {}{} {}"
|
||||
os.system(rsync_command.format(data[distro]["csc"],
|
||||
data[distro]["file"],
|
||||
"csc_manifest"))
|
||||
os.system(rsync_command.format(data[distro]["upstream1"],
|
||||
data[distro]["file"],
|
||||
"upstream_manifest1"))
|
||||
os.system(rsync_command.format(data[distro]["upstream2"],
|
||||
data[distro]["file"],
|
||||
"upstream_manifest2"))
|
||||
stream1 = os.popen("diff csc_manifest upstream_manifest1")
|
||||
output1 = stream1.read()
|
||||
stream2 = os.popen("diff csc_manifest upstream_manifest2")
|
||||
output2 = stream2.read()
|
||||
os.system("rm csc_manifest")
|
||||
os.system("rm upstream_manifest1")
|
||||
os.system("rm upstream_manifest2")
|
||||
return 0 in [len(output1), len(output2)]
|
|
@ -0,0 +1,44 @@
|
|||
"""
|
||||
Contains GNOME class
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import requests
|
||||
|
||||
from distro import Distro
|
||||
from shared import CSC_MIRROR
|
||||
|
||||
|
||||
class GNOME(Distro):
|
||||
"""GNOME class"""
|
||||
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
file = data[distro]["file1"]
|
||||
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
|
||||
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
|
||||
csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"')
|
||||
upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"')
|
||||
if csc_latest != upstream_latest:
|
||||
return False
|
||||
file += csc_latest + "/"
|
||||
csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text
|
||||
upstream_versions = requests.get(data[distro]["upstream1"] + file).text
|
||||
csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"')
|
||||
upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"')
|
||||
if csc_latest != upstream_latest:
|
||||
return False
|
||||
file += csc_latest + "/"
|
||||
csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
try:
|
||||
ret = csc_text == requests.get(data[distro]["upstream2"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
except requests.exceptions.RequestException:
|
||||
ret = False
|
||||
try:
|
||||
return ret or csc_text == requests.get(data[distro]["upstream3"] + file
|
||||
+ data[distro]["file2"]).text
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
|
@ -4,5 +4,6 @@ Contains GNU class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class GNU(Distro):
|
||||
"""GNU class"""
|
|
@ -4,5 +4,6 @@ Contains Gutenberg class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Gutenberg(Distro):
|
||||
"""Gutenberg class"""
|
|
@ -0,0 +1,17 @@
|
|||
"""
|
||||
Contains IPFire class
|
||||
"""
|
||||
|
||||
import requests
|
||||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class IPFire(Distro):
|
||||
"""IPFire class"""
|
||||
|
||||
@staticmethod
|
||||
def check(data, distro):
|
||||
ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca"
|
||||
ipfire_text = requests.get(ipfire_url).text
|
||||
return ipfire_text.find("The mirror is up") != -1
|
|
@ -4,5 +4,6 @@ Contains KDE class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class KDE(Distro):
|
||||
"""KDE class"""
|
|
@ -4,5 +4,6 @@ Contains KDEApplicationData class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class KDEApplicationData(Distro):
|
||||
"""KDEApplicationData class"""
|
|
@ -4,5 +4,6 @@ Contains Kernel class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class Kernel(Distro):
|
||||
"""Kernel class"""
|
|
@ -4,5 +4,6 @@ Contains OpenBSD class
|
|||
|
||||
from distro import Distro
|
||||
|
||||
|
||||
class OpenBSD(Distro):
|
||||
"""OpenBSD class"""
|
|
@ -0,0 +1,4 @@
|
|||
requests~=2.26.0
|
||||
datefinder~=0.7.1
|
||||
|
||||
dateparser~=1.0.0
|
Loading…
Reference in New Issue