From 57472390627f3fc4246b4f3542accfbee9351e42 Mon Sep 17 00:00:00 2001 From: Tom Date: Sun, 17 Oct 2021 19:28:35 -0700 Subject: [PATCH] updated mxlinux_iso --- README.md | 4 ++-- data.json | 3 ++- main.py | 7 ------- projects/mxlinux_iso.py | 40 ++++++++++++++++++++++++++++++++++++++-- test.py | 4 ++-- 5 files changed, 44 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 52dc4e0..5a92676 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ parabola: https://repo.parabola.nu/ https://www.parabola.nu/mirrors/status/ pkgsrc puppylinux: https://distro.ibiblio.org/puppylinux/ check the ISO files or htm files in the folders starting with puppy qtproject: https://download.qt.io/ -racket: https://mirror.racket-lang.org/installers/ no public repo, no timestamp, no mirror status tracker make sure that we have the latest version number under racket-installers +racket: https://mirror.racket-lang.org/installers/ make sure that we have the latest version number under racket-installers raspberry pi: https://archive.raspberrypi.org/ Checking the timestamp of either the Release file or the Packages file should suffice. raspbian: http://archive.raspbian.org/ snapshotindex.txt is most likely a timestamp, tho i'm not sure. also i think our mirror is completely outdated, it's not listed on official mirror list sagemath: same source tarballs as them (the sage-*.tar.gz files under 'Source Code') @@ -76,7 +76,7 @@ salt stack: checking the "Latest release" text under the 'About' header scientific: https://scientificlinux.org/downloads/sl-mirrors/ not checking this one since it's abandoned slackware: https://mirrors.slackware.com/slackware/ check whether we have each release and whether the timestamp for CHECKSUMS.md5 in each release is the same, for slackware-iso, just make sure that our list of directories is the same tdf: https://download.documentfoundation.org/ -trisquel: http://archive.trisquel.info/trisquel/ https://trisquel.info/mirmon/index.html out of date website!? please recheck this!!! +trisquel: http://archive.trisquel.info/trisquel/ checking Release file for all versions in packages/dist and md5sum.txt in iso/ with two other mirrors ubuntu: https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/ checking the Release files in dists ubuntu-ports-releases: https://cdimage.ubuntu.com/releases/ has public repo, no timestamp, no status tracker, brute force looped it diff --git a/data.json b/data.json index 5d10254..19f80d3 100644 --- a/data.json +++ b/data.json @@ -217,8 +217,9 @@ "mxlinux_iso": { "out_of_sync_since": null, "out_of_sync_interval": 86400, - "csc": "", + "csc": "mxlinux-iso/", "upstream": "http://rsync-mxlinux.org/mirmon/index.html", + "mirrors": ["http://mirror.its.dal.ca/mxlinux-cd/", "http://mirror.umd.edu/mxlinux-iso/"], "file": "" }, "mySQL": { diff --git a/main.py b/main.py index 10dbc56..5a3f062 100644 --- a/main.py +++ b/main.py @@ -13,13 +13,6 @@ import json # import json to read project info stored in json file if __name__ == "__main__": - """projects = json.load(open('projects.json',)) - print(projects) - - for project in projects: - print(project[0] + ":") - print("CSC mirror: " + checker(project[1], project[3])) - print("Official project: " + checker(project[2], project[3]))""" with open("data.json", "r", encoding="utf-8") as file: data = json.load(file) diff --git a/projects/mxlinux_iso.py b/projects/mxlinux_iso.py index a2a339c..2006707 100644 --- a/projects/mxlinux_iso.py +++ b/projects/mxlinux_iso.py @@ -14,7 +14,8 @@ import pandas as pd class mxlinux_iso(Project): """mxlinux_iso class""" - @staticmethod + # this method is to check the mirror tracker, but unfortunately, the mirror tracker is behaving a bit strange so we check with other mirrors + """@staticmethod def check(data, project, current_time): page = requests.get(data[project]["upstream"]).text indexOfFile = page.find("mirror.csclub.uwaterloo.ca") @@ -23,4 +24,39 @@ class mxlinux_iso(Project): duration = pd.to_timedelta(m.group(0)) - return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s') \ No newline at end of file + return duration <= pd.to_timedelta(data[project]["out_of_sync_interval"], unit='s')""" + + @staticmethod + def checker(directory_URL, file_name): + page = requests.get(directory_URL).text + file_index = page.find(file_name) + # print(page) + + if file_index == -1: + return False + + str_dates = re.findall(r'(\d{2}-\w{3}-\d{4} \d{2}:\d{2})|(\d{4}-\w{3}-\d{2} \d{2}:\d{2})|(\d{4}-\d{2}-\d{2} \d{2}:\d{2})', page[file_index:]) + + print(str_dates) + return list(datefinder.find_dates("".join(str_dates[0])))[0] + + @classmethod + def check_iso(cls, site, mirrors): + for mirror in mirrors: + # print(cls.checker(site, "md5sum.txt")) + # print(cls.checker(mirror, "md5sum.txt")) + if cls.checker(site, "TIME.txt") < cls.checker(mirror, "TIME.txt"): + return False + return True + + @classmethod + def check(cls, data, project): + """Check if project packages are up-to-date""" + + csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"] + upstream_url = data[project]["upstream"] + data[project]["file"] + + # print(cls.check_iso(upstream_url+"slackware-iso/", csc_url+"slackware-iso/")) + mirrors = data[project]["mirrors"] + + return cls.check_iso(csc_url, mirrors) \ No newline at end of file diff --git a/test.py b/test.py index adc8e47..b6f01bc 100644 --- a/test.py +++ b/test.py @@ -7,7 +7,7 @@ from datetime import timedelta import time import pandas as pd import re # for salt stack specifically -from projects import trisquel +from projects import mxlinux_iso import json # import json to read project info stored in json file # this function is brute force looping through the whole directory and checking dates @@ -65,7 +65,7 @@ def get_latest_date(web_dir): if __name__ =="__main__": with open("data.json", "r", encoding="utf-8") as file: data = json.load(file) - print(trisquel.check(data, "trisquel")) + print(mxlinux_iso.check(data, "mxlinux_iso")) """# website to be scrape site="https://cdimage.ubuntu.com/releases/"