From 3428e81cec325386584c973f68e556d5bf1b6a0d Mon Sep 17 00:00:00 2001 From: Raymond Li Date: Sun, 3 Oct 2021 15:35:17 -0400 Subject: [PATCH] Refactor and use requirements file --- .gitignore | 1 + cpan.py | 17 - data.json | 358 +++++++++--------- distro.py | 18 +- distros.json | 23 +- gentooportage.py | 29 -- gnome.py | 40 -- ipfire.py | 14 - main.py | 182 ++++----- almalinux.py => projects/almalinux.py | 3 +- alpine.py => projects/alpine.py | 3 +- apache.py => projects/apache.py | 3 +- arch.py => projects/arch.py | 3 +- centos.py => projects/centos.py | 3 +- ceph.py => projects/ceph.py | 3 +- projects/cpan.py | 20 + cygwin.py => projects/cygwin.py | 3 +- debian.py => projects/debian.py | 3 +- debiancd.py => projects/debiancd.py | 3 +- .../debianmultimedia.py | 3 +- debianports.py => projects/debianports.py | 3 +- .../debiansecurity.py | 3 +- eclipse.py => projects/eclipse.py | 3 +- fedora.py => projects/fedora.py | 3 +- freebsd.py => projects/freebsd.py | 3 +- .../gentoodistfiles.py | 3 +- projects/gentooportage.py | 32 ++ projects/gnome.py | 44 +++ gnu.py => projects/gnu.py | 3 +- gutenberg.py => projects/gutenberg.py | 3 +- projects/ipfire.py | 17 + kde.py => projects/kde.py | 3 +- .../kdeapplicationdata.py | 3 +- kernel.py => projects/kernel.py | 3 +- openbsd.py => projects/openbsd.py | 3 +- requirements.txt | 4 + 36 files changed, 463 insertions(+), 402 deletions(-) delete mode 100644 cpan.py delete mode 100644 gentooportage.py delete mode 100644 gnome.py delete mode 100644 ipfire.py rename almalinux.py => projects/almalinux.py (76%) rename alpine.py => projects/alpine.py (77%) rename apache.py => projects/apache.py (77%) rename arch.py => projects/arch.py (78%) rename centos.py => projects/centos.py (77%) rename ceph.py => projects/ceph.py (78%) create mode 100644 projects/cpan.py rename cygwin.py => projects/cygwin.py (77%) rename debian.py => projects/debian.py (77%) rename debiancd.py => projects/debiancd.py (77%) rename debianmultimedia.py => projects/debianmultimedia.py (75%) rename debianports.py => projects/debianports.py (76%) rename debiansecurity.py => projects/debiansecurity.py (75%) rename eclipse.py => projects/eclipse.py (77%) rename fedora.py => projects/fedora.py (77%) rename freebsd.py => projects/freebsd.py (77%) rename gentoodistfiles.py => projects/gentoodistfiles.py (75%) create mode 100644 projects/gentooportage.py create mode 100644 projects/gnome.py rename gnu.py => projects/gnu.py (78%) rename gutenberg.py => projects/gutenberg.py (76%) create mode 100644 projects/ipfire.py rename kde.py => projects/kde.py (78%) rename kdeapplicationdata.py => projects/kdeapplicationdata.py (74%) rename kernel.py => projects/kernel.py (77%) rename openbsd.py => projects/openbsd.py (77%) create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 26280f9..4dfa1c1 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,4 @@ cython_debug/ # Idea .idea/ +*.txt~ diff --git a/cpan.py b/cpan.py deleted file mode 100644 index d72aafc..0000000 --- a/cpan.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -Contains CPAN class -""" - -import requests -from distro import Distro -from shared import CSC_MIRROR - -class CPAN(Distro): - """CPAN class""" - @staticmethod - def check(data, distro, current_time): - res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() - for mirror in res_json: - if mirror["url"] == f"{CSC_MIRROR}CPAN/": - return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"] - return False diff --git a/data.json b/data.json index 41e1c9d..0a99c59 100644 --- a/data.json +++ b/data.json @@ -1,181 +1,181 @@ { - "AlmaLinux": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://repo.almalinux.org/", - "file": "almalinux/TIME" - }, - "Alpine": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://uk.alpinelinux.org/", - "file": "alpine/last-updated" - }, - "Apache": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "apache/", - "upstream": "https://downloads.apache.org/", - "file": "zzz/time.txt" - }, - "Arch": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "archlinux/", - "upstream": "http://arch.mirror.constant.com/", - "file": "lastupdate" - }, - "CentOS": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://mirrors.edge.kernel.org/", - "file": "centos/TIME" - }, - "Ceph": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "ceph/", - "upstream": "https://download.ceph.com/", - "file": "timestamp" - }, - "CPAN": { - "out_of_sync_interval": 172800 - }, - "Cygwin": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "cygwin/", - "upstream": "https://cygwin.mirror.globo.tech/", - "file": "x86/sha512.sum" - }, - "Debian": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://ftp-master.debian.org/", - "file": "debian/project/trace/master" - }, - "DebianCD": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "http://debian.mirror.estruxture.net/", - "file": "debian-cd/project/trace/cdimage.debian.org" - }, - "DebianMultimedia": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "debian-multimedia/", - "upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", - "file": "project/trace/deb-multimedia.org" - }, - "DebianPorts": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://deb.debian.org/", - "file": "debian-ports/project/trace/porta.debian.org" - }, - "DebianSecurity": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "http://debian.mirror.iweb.ca/", - "file": "debian-security/project/trace/master" - }, - "Eclipse": { - "out_of_sync_since": null, - "out_of_sync_interval": 172800, - "csc": "eclipse/", - "upstream": "http://download.eclipse.org/", - "file": "TIME" - }, - "Fedora": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "fedora/", - "upstream": "http://fedora.mirror.iweb.com/", - "file": "linux/development/rawhide/COMPOSE_ID" - }, - "FreeBSD": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "http://ftp4.freebsd.org/pub/", - "file": "FreeBSD/TIMESTAMP" - }, - "GentooDistfiles": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "gentoo-distfiles", - "upstream": "http://gentoo.mirrors.tera-byte.com/", - "file": "distfiles/timestamp.dev-local" - }, - "GentooPortage": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "rsync://rsync4.ca.gentoo.org/", - "upstream1": "rsync://rsync1.de.gentoo.org/", - "upstream2": "rsync://rsync8.de.gentoo.org/", - "file": "gentoo-portage/Manifest" - }, - "GNOME": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "gnome/", - "upstream1": "https://download.gnome.org/", - "upstream2": "https://mirrors.dotsrc.org/gnome/", - "upstream3": "https://muug.ca/mirror/gnome/", - "file1": "core/", - "file2": "cache.json" - }, - "GNU": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://mirrors.kernel.org/", - "file": "gnu/mirror-updated-timestamp.txt" - }, - "Gutenberg": { - "out_of_sync_since": null, - "out_of_sync_interval": 172800, - "csc": "gutenberg/", - "upstream": "https://gutenberg.pglaf.org/", - "file": "gutenberg.dcs" - }, - "IPFire": { - "out_of_sync_since": null, - "out_of_sync_interval": 172800 - }, - "KDE": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "kde/", - "upstream": "https://kde.c3sl.ufpr.br/", - "file": "ls-lR" - }, - "KDEApplicationData": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "kde-applicationdata/", - "upstream": "https://cdn.files.kde.org/", - "file": "last-updated" - }, - "Kernel": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "kernel.org/", - "upstream": "https://mirrors.edge.kernel.org/pub/", - "file": "linux/kernel/next/sha256sums.asc" - }, - "OpenBSD": { - "out_of_sync_since": null, - "out_of_sync_interval": 86400, - "csc": "", - "upstream": "https://ftp.openbsd.org/pub/", - "file": "OpenBSD/timestamp" - } + "AlmaLinux": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://repo.almalinux.org/", + "file": "almalinux/TIME" + }, + "Alpine": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://uk.alpinelinux.org/", + "file": "alpine/last-updated" + }, + "Apache": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "apache/", + "upstream": "https://downloads.apache.org/", + "file": "zzz/time.txt" + }, + "Arch": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "archlinux/", + "upstream": "http://arch.mirror.constant.com/", + "file": "lastupdate" + }, + "CentOS": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://mirrors.edge.kernel.org/", + "file": "centos/TIME" + }, + "Ceph": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "ceph/", + "upstream": "https://download.ceph.com/", + "file": "timestamp" + }, + "CPAN": { + "out_of_sync_interval": 172800 + }, + "Cygwin": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "cygwin/", + "upstream": "https://cygwin.mirror.globo.tech/", + "file": "x86/sha512.sum" + }, + "Debian": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://ftp-master.debian.org/", + "file": "debian/project/trace/master" + }, + "DebianCD": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "http://debian.mirror.estruxture.net/", + "file": "debian-cd/project/trace/cdimage.debian.org" + }, + "DebianMultimedia": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "debian-multimedia/", + "upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/", + "file": "project/trace/deb-multimedia.org" + }, + "DebianPorts": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://deb.debian.org/", + "file": "debian-ports/project/trace/porta.debian.org" + }, + "DebianSecurity": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "http://debian.mirror.iweb.ca/", + "file": "debian-security/project/trace/master" + }, + "Eclipse": { + "out_of_sync_since": null, + "out_of_sync_interval": 172800, + "csc": "eclipse/", + "upstream": "http://download.eclipse.org/", + "file": "TIME" + }, + "Fedora": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "fedora/", + "upstream": "http://fedora.mirror.iweb.com/", + "file": "linux/development/rawhide/COMPOSE_ID" + }, + "FreeBSD": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "http://ftp4.freebsd.org/pub/", + "file": "FreeBSD/TIMESTAMP" + }, + "GentooDistfiles": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "gentoo-distfiles", + "upstream": "http://gentoo.mirrors.tera-byte.com/", + "file": "distfiles/timestamp.dev-local" + }, + "GentooPortage": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "rsync://rsync4.ca.gentoo.org/", + "upstream1": "rsync://rsync1.de.gentoo.org/", + "upstream2": "rsync://rsync8.de.gentoo.org/", + "file": "gentoo-portage/Manifest" + }, + "GNOME": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "gnome/", + "upstream1": "https://download.gnome.org/", + "upstream2": "https://mirrors.dotsrc.org/gnome/", + "upstream3": "https://muug.ca/mirror/gnome/", + "file1": "core/", + "file2": "cache.json" + }, + "GNU": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://mirrors.kernel.org/", + "file": "gnu/mirror-updated-timestamp.txt" + }, + "Gutenberg": { + "out_of_sync_since": null, + "out_of_sync_interval": 172800, + "csc": "gutenberg/", + "upstream": "https://gutenberg.pglaf.org/", + "file": "gutenberg.dcs" + }, + "IPFire": { + "out_of_sync_since": null, + "out_of_sync_interval": 172800 + }, + "KDE": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "kde/", + "upstream": "https://kde.c3sl.ufpr.br/", + "file": "ls-lR" + }, + "KDEApplicationData": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "kde-applicationdata/", + "upstream": "https://cdn.files.kde.org/", + "file": "last-updated" + }, + "Kernel": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "kernel.org/", + "upstream": "https://mirrors.edge.kernel.org/pub/", + "file": "linux/kernel/next/sha256sums.asc" + }, + "OpenBSD": { + "out_of_sync_since": null, + "out_of_sync_interval": 86400, + "csc": "", + "upstream": "https://ftp.openbsd.org/pub/", + "file": "OpenBSD/timestamp" + } } diff --git a/distro.py b/distro.py index 7d363dc..bc0812a 100644 --- a/distro.py +++ b/distro.py @@ -3,14 +3,18 @@ Contains abstract class for a distro """ from abc import ABC + import requests + from shared import CSC_MIRROR + class Distro(ABC): - """Abstract class for a distro""" - @staticmethod - def check(data, distro): - """Check if distro packages are up-to-date""" - csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"] - upstream_url = data[distro]["upstream"] + data[distro]["file"] - return requests.get(csc_url).text == requests.get(upstream_url).text + """Abstract class for a distro""" + + @staticmethod + def check(data, distro): + """Check if distro packages are up-to-date""" + csc_url = CSC_MIRROR + data[distro]["csc"] + data[distro]["file"] + upstream_url = data[distro]["upstream"] + data[distro]["file"] + return requests.get(csc_url).text == requests.get(upstream_url).text diff --git a/distros.json b/distros.json index 0904010..b26d55c 100644 --- a/distros.json +++ b/distros.json @@ -1,5 +1,20 @@ [ - ["OpenBSD", "https://mirror.csclub.uwaterloo.ca/OpenBSD/", "https://ftp.openbsd.org/pub/OpenBSD/", "timestamp"], - ["kernel", "http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/", "https://mirrors.edge.kernel.org/pub/linux/kernel/next/", "sha256sums.asc"], - ["debian", "http://mirror.csclub.uwaterloo.ca/debian/project/trace/", "https://ftp-master.debian.org/debian/project/trace/", "master"] -] \ No newline at end of file + [ + "OpenBSD", + "https://mirror.csclub.uwaterloo.ca/OpenBSD/", + "https://ftp.openbsd.org/pub/OpenBSD/", + "timestamp" + ], + [ + "kernel", + "http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/", + "https://mirrors.edge.kernel.org/pub/linux/kernel/next/", + "sha256sums.asc" + ], + [ + "debian", + "http://mirror.csclub.uwaterloo.ca/debian/project/trace/", + "https://ftp-master.debian.org/debian/project/trace/", + "master" + ] +] diff --git a/gentooportage.py b/gentooportage.py deleted file mode 100644 index 29498aa..0000000 --- a/gentooportage.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Contains GentooPortage class -""" - -import os -from distro import Distro - -class GentooPortage(Distro): - """GentooPortage class""" - @staticmethod - def check(data, distro): - rsync_command = "rsync -q {}{} {}" - os.system(rsync_command.format(data[distro]["csc"], - data[distro]["file"], - "csc_manifest")) - os.system(rsync_command.format(data[distro]["upstream1"], - data[distro]["file"], - "upstream_manifest1")) - os.system(rsync_command.format(data[distro]["upstream2"], - data[distro]["file"], - "upstream_manifest2")) - stream1 = os.popen("diff csc_manifest upstream_manifest1") - output1 = stream1.read() - stream2 = os.popen("diff csc_manifest upstream_manifest2") - output2 = stream2.read() - os.system("rm csc_manifest") - os.system("rm upstream_manifest1") - os.system("rm upstream_manifest2") - return 0 in [len(output1), len(output2)] diff --git a/gnome.py b/gnome.py deleted file mode 100644 index 8c9dfb6..0000000 --- a/gnome.py +++ /dev/null @@ -1,40 +0,0 @@ -""" -Contains GNOME class -""" - -import re -import requests -from distro import Distro -from shared import CSC_MIRROR - -class GNOME(Distro): - """GNOME class""" - @staticmethod - def check(data, distro): - file = data[distro]["file1"] - csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text - upstream_versions = requests.get(data[distro]["upstream1"] + file).text - csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"') - upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"') - if csc_latest != upstream_latest: - return False - file += csc_latest + "/" - csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text - upstream_versions = requests.get(data[distro]["upstream1"] + file).text - csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"') - upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"') - if csc_latest != upstream_latest: - return False - file += csc_latest + "/" - csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file - + data[distro]["file2"]).text - try: - ret = csc_text == requests.get(data[distro]["upstream2"] + file - + data[distro]["file2"]).text - except requests.exceptions.RequestException: - ret = False - try: - return ret or csc_text == requests.get(data[distro]["upstream3"] + file - + data[distro]["file2"]).text - except requests.exceptions.RequestException: - return False diff --git a/ipfire.py b/ipfire.py deleted file mode 100644 index c9b9911..0000000 --- a/ipfire.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -Contains IPFire class -""" - -import requests -from distro import Distro - -class IPFire(Distro): - """IPFire class""" - @staticmethod - def check(data, distro): - ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca" - ipfire_text = requests.get(ipfire_url).text - return ipfire_text.find("The mirror is up") != -1 diff --git a/main.py b/main.py index 5d4e9ee..707cb54 100644 --- a/main.py +++ b/main.py @@ -7,106 +7,108 @@ This mirror status checker determines whether CSC mirror is up-to-date with upst import time import sys import requests -from almalinux import AlmaLinux -from alpine import Alpine -from apache import Apache -from arch import Arch -from centos import CentOS -from ceph import Ceph -from cpan import CPAN -from cygwin import Cygwin -from debian import Debian -from debiancd import DebianCD -from debianmultimedia import DebianMultimedia -from debianports import DebianPorts -from debiansecurity import DebianSecurity -from eclipse import Eclipse -from fedora import Fedora -from freebsd import FreeBSD -from gentoodistfiles import GentooDistfiles -from gentooportage import GentooPortage -from gnome import GNOME -from gnu import GNU -from gutenberg import Gutenberg -from ipfire import IPFire -from kde import KDE -from kdeapplicationdata import KDEApplicationData -from kernel import Kernel -from openbsd import OpenBSD +from projects.almalinux import AlmaLinux +from projects.alpine import Alpine +from projects.apache import Apache +from projects.arch import Arch +from projects.centos import CentOS +from projects.ceph import Ceph +from projects.cpan import CPAN +from projects.cygwin import Cygwin +from projects.debian import Debian +from projects.debiancd import DebianCD +from projects.debianmultimedia import DebianMultimedia +from projects.debianports import DebianPorts +from projects.debiansecurity import DebianSecurity +from projects.eclipse import Eclipse +from projects.fedora import Fedora +from projects.freebsd import FreeBSD +from projects.gentoodistfiles import GentooDistfiles +from projects.gentooportage import GentooPortage +from projects.gnome import GNOME +from projects.gnu import GNU +from projects.gutenberg import Gutenberg +from projects.ipfire import IPFire +from projects.kde import KDE +from projects.kdeapplicationdata import KDEApplicationData +from projects.kernel import Kernel +from projects.openbsd import OpenBSD from shared import CSC_MIRROR -from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors +from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors # http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/ -import re # import regular expressions to remove stray numbers in string that might interfere with date finding -import json # import json to read distro info stored in json file +import re # import regular expressions to remove stray numbers in string that might interfere with date finding +import json # import json to read distro info stored in json file + +import datefinder # another date finding library -import datefinder # another date finding library def checker(directory_URL, file_name): - page = requests.get(directory_URL).text - indexOfFile = page.find(file_name) - # print(page) + page = requests.get(directory_URL).text + indexOfFile = page.find(file_name) + # print(page) - # remove stray numbers (file size numbers in particular) that might interfere with date finding - segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size - segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb - # print(segment_clean) + # remove stray numbers (file size numbers in particular) that might interfere with date finding + segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size + segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb + # print(segment_clean) - # implementation using dateparser.search.search_dates - # notes: some dates don't parse correctly with this tool - # print(search_dates(page[indexOfFile:], languages=['en'])) - # print(search_dates(page[indexOfFile:])[0]) + # implementation using dateparser.search.search_dates + # notes: some dates don't parse correctly with this tool + # print(search_dates(page[indexOfFile:], languages=['en'])) + # print(search_dates(page[indexOfFile:])[0]) - # finds the dates in the segment after the file name - # notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom. - matches = list(datefinder.find_dates(segment_clean)) - # print(matches) + # finds the dates in the segment after the file name + # notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom. + matches = list(datefinder.find_dates(segment_clean)) + # print(matches) + + if len(matches) > 0: + date = matches[0] # date is of type datetime.datetime + return (date.strftime("%m/%d/%Y, %H:%M:%S")) + else: + return ('No dates found') - if len(matches) > 0: - date = matches[0] # date is of type datetime.datetime - return(date.strftime("%m/%d/%Y, %H:%M:%S")) - else: - return('No dates found') if __name__ == "__main__": - """distros = json.load(open('distros.json',)) - print(distros) + """distros = json.load(open('distros.json',)) + print(distros) - for distro in distros: - print(distro[0] + ":") - print("CSC mirror: " + checker(distro[1], distro[3])) - print("Official distro: " + checker(distro[2], distro[3]))""" + for distro in distros: + print(distro[0] + ":") + print("CSC mirror: " + checker(distro[1], distro[3])) + print("Official distro: " + checker(distro[2], distro[3]))""" - with open("data.json", "r", encoding = "utf-8") as file: - data = json.load(file) - if sys.stdin.isatty(): - distros = data - else: - distros = [distro.rstrip() for distro in sys.stdin.readlines()] - current_time = int(time.time()) - for distro in distros: - try: - if distro not in data: - print(f"Failure: {distro} does not exist") - continue - distro_class = getattr(sys.modules[__name__], distro) - if distro == "CPAN": - checker_result = distro_class.check(data, distro, current_time) - if checker_result: - print(f"Success: {distro} up-to-date") - else: - print(f"Failure: {distro} out-of-sync") - continue - checker_result = distro_class.check(data, distro) - if checker_result: - data[distro]["out_of_sync_since"] = None - elif data[distro]["out_of_sync_since"] is None: - data[distro]["out_of_sync_since"] = current_time - elif current_time - data[distro]["out_of_sync_since"] \ - > data[distro]["out_of_sync_interval"]: - print(f"Failure: {distro} out-of-sync") - continue - print(f"Success: {distro} up-to-date") - except requests.exceptions.RequestException as err: - print(f"Error: {distro}\n{err}") - with open("data.json", "w", encoding = "utf-8") as file: - json.dump(data, file, indent = 4) + with open("data.json", "r", encoding="utf-8") as file: + data = json.load(file) + if sys.stdin.isatty(): + distros = data + else: + distros = [distro.rstrip() for distro in sys.stdin.readlines()] + current_time = int(time.time()) + for distro in distros: + try: + if distro not in data: + print(f"Failure: {distro} does not exist") + continue + distro_class = getattr(sys.modules[__name__], distro) + if distro == "CPAN": + checker_result = distro_class.check(data, distro, current_time) + if checker_result: + print(f"Success: {distro} up-to-date") + else: + print(f"Failure: {distro} out-of-sync") + continue + checker_result = distro_class.check(data, distro) + if checker_result: + data[distro]["out_of_sync_since"] = None + elif data[distro]["out_of_sync_since"] is None: + data[distro]["out_of_sync_since"] = current_time + elif current_time - data[distro]["out_of_sync_since"] \ + > data[distro]["out_of_sync_interval"]: + print(f"Failure: {distro} out-of-sync") + continue + print(f"Success: {distro} up-to-date") + except requests.exceptions.RequestException as err: + print(f"Error: {distro}\n{err}") + with open("data.json", "w", encoding="utf-8") as file: + json.dump(data, file, indent=4) diff --git a/almalinux.py b/projects/almalinux.py similarity index 76% rename from almalinux.py rename to projects/almalinux.py index 201deef..267d2e9 100644 --- a/almalinux.py +++ b/projects/almalinux.py @@ -4,5 +4,6 @@ Contains AlmaLinux class from distro import Distro + class AlmaLinux(Distro): - """AlmaLinux class""" + """AlmaLinux class""" diff --git a/alpine.py b/projects/alpine.py similarity index 77% rename from alpine.py rename to projects/alpine.py index b47ccae..53ba0c2 100644 --- a/alpine.py +++ b/projects/alpine.py @@ -4,5 +4,6 @@ Contains Alpine class from distro import Distro + class Alpine(Distro): - """Alpine class""" + """Alpine class""" diff --git a/apache.py b/projects/apache.py similarity index 77% rename from apache.py rename to projects/apache.py index 0541525..da8ea19 100644 --- a/apache.py +++ b/projects/apache.py @@ -4,5 +4,6 @@ Contains Apache class from distro import Distro + class Apache(Distro): - """Apache class""" + """Apache class""" diff --git a/arch.py b/projects/arch.py similarity index 78% rename from arch.py rename to projects/arch.py index 6409ef7..5a2cf9f 100644 --- a/arch.py +++ b/projects/arch.py @@ -4,5 +4,6 @@ Contains Arch class from distro import Distro + class Arch(Distro): - """Arch class""" + """Arch class""" diff --git a/centos.py b/projects/centos.py similarity index 77% rename from centos.py rename to projects/centos.py index 0362130..097f02a 100644 --- a/centos.py +++ b/projects/centos.py @@ -4,5 +4,6 @@ Contains CentOS class from distro import Distro + class CentOS(Distro): - """CentOS class""" + """CentOS class""" diff --git a/ceph.py b/projects/ceph.py similarity index 78% rename from ceph.py rename to projects/ceph.py index d39b81f..4253d0f 100644 --- a/ceph.py +++ b/projects/ceph.py @@ -4,5 +4,6 @@ Contains Ceph class from distro import Distro + class Ceph(Distro): - """Ceph class""" + """Ceph class""" diff --git a/projects/cpan.py b/projects/cpan.py new file mode 100644 index 0000000..6289160 --- /dev/null +++ b/projects/cpan.py @@ -0,0 +1,20 @@ +""" +Contains CPAN class +""" + +import requests + +from distro import Distro +from shared import CSC_MIRROR + + +class CPAN(Distro): + """CPAN class""" + + @staticmethod + def check(data, distro, current_time): + res_json = requests.get("http://mirrors.cpan.org/cpan-json.txt").json() + for mirror in res_json: + if mirror["url"] == f"{CSC_MIRROR}CPAN/": + return current_time - int(mirror["age"]) <= data[distro]["out_of_sync_interval"] + return False diff --git a/cygwin.py b/projects/cygwin.py similarity index 77% rename from cygwin.py rename to projects/cygwin.py index 0885224..eda63f9 100644 --- a/cygwin.py +++ b/projects/cygwin.py @@ -4,5 +4,6 @@ Contains Cygwin class from distro import Distro + class Cygwin(Distro): - """Cygwin class""" + """Cygwin class""" diff --git a/debian.py b/projects/debian.py similarity index 77% rename from debian.py rename to projects/debian.py index a720054..a98cad3 100644 --- a/debian.py +++ b/projects/debian.py @@ -4,5 +4,6 @@ Contains Debian class from distro import Distro + class Debian(Distro): - """Debian class""" + """Debian class""" diff --git a/debiancd.py b/projects/debiancd.py similarity index 77% rename from debiancd.py rename to projects/debiancd.py index 90e7f39..0b79b2c 100644 --- a/debiancd.py +++ b/projects/debiancd.py @@ -4,5 +4,6 @@ Contains DebianCD class from distro import Distro + class DebianCD(Distro): - """DebianCD class""" + """DebianCD class""" diff --git a/debianmultimedia.py b/projects/debianmultimedia.py similarity index 75% rename from debianmultimedia.py rename to projects/debianmultimedia.py index 5e4fd29..f5c5b08 100644 --- a/debianmultimedia.py +++ b/projects/debianmultimedia.py @@ -4,5 +4,6 @@ Contains DebianMultimedia class from distro import Distro + class DebianMultimedia(Distro): - """DebianMultimedia class""" + """DebianMultimedia class""" diff --git a/debianports.py b/projects/debianports.py similarity index 76% rename from debianports.py rename to projects/debianports.py index a3f2952..90b7dc3 100644 --- a/debianports.py +++ b/projects/debianports.py @@ -4,5 +4,6 @@ Contains DebianPorts class from distro import Distro + class DebianPorts(Distro): - """DebianPorts class""" + """DebianPorts class""" diff --git a/debiansecurity.py b/projects/debiansecurity.py similarity index 75% rename from debiansecurity.py rename to projects/debiansecurity.py index 9a50858..62eea7c 100644 --- a/debiansecurity.py +++ b/projects/debiansecurity.py @@ -4,5 +4,6 @@ Contains DebianSecurity class from distro import Distro + class DebianSecurity(Distro): - """DebianSecurity class""" + """DebianSecurity class""" diff --git a/eclipse.py b/projects/eclipse.py similarity index 77% rename from eclipse.py rename to projects/eclipse.py index 9945b88..8db84cb 100644 --- a/eclipse.py +++ b/projects/eclipse.py @@ -4,5 +4,6 @@ Contains Eclipse class from distro import Distro + class Eclipse(Distro): - """Eclipse class""" + """Eclipse class""" diff --git a/fedora.py b/projects/fedora.py similarity index 77% rename from fedora.py rename to projects/fedora.py index 60cc2e9..e5193a5 100644 --- a/fedora.py +++ b/projects/fedora.py @@ -4,5 +4,6 @@ Contains Fedora class from distro import Distro + class Fedora(Distro): - """Fedora class""" + """Fedora class""" diff --git a/freebsd.py b/projects/freebsd.py similarity index 77% rename from freebsd.py rename to projects/freebsd.py index 2c7eec6..20f1a43 100644 --- a/freebsd.py +++ b/projects/freebsd.py @@ -4,5 +4,6 @@ Contains FreeBSD class from distro import Distro + class FreeBSD(Distro): - """FreeBSD class""" + """FreeBSD class""" diff --git a/gentoodistfiles.py b/projects/gentoodistfiles.py similarity index 75% rename from gentoodistfiles.py rename to projects/gentoodistfiles.py index ae3a740..84cec19 100644 --- a/gentoodistfiles.py +++ b/projects/gentoodistfiles.py @@ -4,5 +4,6 @@ Contains GentooDistfiles class from distro import Distro + class GentooDistfiles(Distro): - """GentooDistfiles class""" + """GentooDistfiles class""" diff --git a/projects/gentooportage.py b/projects/gentooportage.py new file mode 100644 index 0000000..439f174 --- /dev/null +++ b/projects/gentooportage.py @@ -0,0 +1,32 @@ +""" +Contains GentooPortage class +""" + +import os + +from distro import Distro + + +class GentooPortage(Distro): + """GentooPortage class""" + + @staticmethod + def check(data, distro): + rsync_command = "rsync -q {}{} {}" + os.system(rsync_command.format(data[distro]["csc"], + data[distro]["file"], + "csc_manifest")) + os.system(rsync_command.format(data[distro]["upstream1"], + data[distro]["file"], + "upstream_manifest1")) + os.system(rsync_command.format(data[distro]["upstream2"], + data[distro]["file"], + "upstream_manifest2")) + stream1 = os.popen("diff csc_manifest upstream_manifest1") + output1 = stream1.read() + stream2 = os.popen("diff csc_manifest upstream_manifest2") + output2 = stream2.read() + os.system("rm csc_manifest") + os.system("rm upstream_manifest1") + os.system("rm upstream_manifest2") + return 0 in [len(output1), len(output2)] diff --git a/projects/gnome.py b/projects/gnome.py new file mode 100644 index 0000000..9bac6b6 --- /dev/null +++ b/projects/gnome.py @@ -0,0 +1,44 @@ +""" +Contains GNOME class +""" + +import re + +import requests + +from distro import Distro +from shared import CSC_MIRROR + + +class GNOME(Distro): + """GNOME class""" + + @staticmethod + def check(data, distro): + file = data[distro]["file1"] + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\d*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\d*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_versions = requests.get(CSC_MIRROR + data[distro]["csc"] + file).text + upstream_versions = requests.get(data[distro]["upstream1"] + file).text + csc_latest = re.findall(r"\"\d+\.?\w*\.?\w*", csc_versions)[-1].lstrip('"') + upstream_latest = re.findall(r"\"\d+\.?\w*\.?\w*", upstream_versions)[-1].lstrip('"') + if csc_latest != upstream_latest: + return False + file += csc_latest + "/" + csc_text = requests.get(CSC_MIRROR + data[distro]["csc"] + file + + data[distro]["file2"]).text + try: + ret = csc_text == requests.get(data[distro]["upstream2"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + ret = False + try: + return ret or csc_text == requests.get(data[distro]["upstream3"] + file + + data[distro]["file2"]).text + except requests.exceptions.RequestException: + return False diff --git a/gnu.py b/projects/gnu.py similarity index 78% rename from gnu.py rename to projects/gnu.py index 8b7af9a..0f4ddcb 100644 --- a/gnu.py +++ b/projects/gnu.py @@ -4,5 +4,6 @@ Contains GNU class from distro import Distro + class GNU(Distro): - """GNU class""" + """GNU class""" diff --git a/gutenberg.py b/projects/gutenberg.py similarity index 76% rename from gutenberg.py rename to projects/gutenberg.py index 4ce58f9..a31cd47 100644 --- a/gutenberg.py +++ b/projects/gutenberg.py @@ -4,5 +4,6 @@ Contains Gutenberg class from distro import Distro + class Gutenberg(Distro): - """Gutenberg class""" + """Gutenberg class""" diff --git a/projects/ipfire.py b/projects/ipfire.py new file mode 100644 index 0000000..aa28f6c --- /dev/null +++ b/projects/ipfire.py @@ -0,0 +1,17 @@ +""" +Contains IPFire class +""" + +import requests + +from distro import Distro + + +class IPFire(Distro): + """IPFire class""" + + @staticmethod + def check(data, distro): + ipfire_url = "https://mirrors.ipfire.org/mirrors/mirror.csclub.uwaterloo.ca" + ipfire_text = requests.get(ipfire_url).text + return ipfire_text.find("The mirror is up") != -1 diff --git a/kde.py b/projects/kde.py similarity index 78% rename from kde.py rename to projects/kde.py index 2a18445..a2a40c3 100644 --- a/kde.py +++ b/projects/kde.py @@ -4,5 +4,6 @@ Contains KDE class from distro import Distro + class KDE(Distro): - """KDE class""" + """KDE class""" diff --git a/kdeapplicationdata.py b/projects/kdeapplicationdata.py similarity index 74% rename from kdeapplicationdata.py rename to projects/kdeapplicationdata.py index 6474864..022c007 100644 --- a/kdeapplicationdata.py +++ b/projects/kdeapplicationdata.py @@ -4,5 +4,6 @@ Contains KDEApplicationData class from distro import Distro + class KDEApplicationData(Distro): - """KDEApplicationData class""" + """KDEApplicationData class""" diff --git a/kernel.py b/projects/kernel.py similarity index 77% rename from kernel.py rename to projects/kernel.py index e06dad5..320b444 100644 --- a/kernel.py +++ b/projects/kernel.py @@ -4,5 +4,6 @@ Contains Kernel class from distro import Distro + class Kernel(Distro): - """Kernel class""" + """Kernel class""" diff --git a/openbsd.py b/projects/openbsd.py similarity index 77% rename from openbsd.py rename to projects/openbsd.py index 8b2122f..e9ff662 100644 --- a/openbsd.py +++ b/projects/openbsd.py @@ -4,5 +4,6 @@ Contains OpenBSD class from distro import Distro + class OpenBSD(Distro): - """OpenBSD class""" + """OpenBSD class""" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..015c072 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +requests~=2.26.0 +datefinder~=0.7.1 + +dateparser~=1.0.0