forked from public/mirror-checker
added racket
This commit is contained in:
parent
0b3e36a8ff
commit
3d5eee45db
|
@ -23,7 +23,7 @@ macPorts: only distfiles has public repo, no timestamp, too large to loop throug
|
|||
NetBSD: http://ftp.netbsd.org/pub/NetBSD/ has public repo, no timestamp, web directory hard to loop through, no mirror tracker
|
||||
opensuse: http://download.opensuse.org/ has public repo, a possible timestamp called latest in history, our mirror doesn't have this file tho, no mirror tracker
|
||||
puppylinux: https://distro.ibiblio.org/puppylinux/ check the ISO files in the folders starting with puppy
|
||||
racket: https://mirror.racket-lang.org/installers/ no public repo, no timestamp, no mirror status tracker make sure that we have the latest version under racket-installers
|
||||
racket: https://mirror.racket-lang.org/installers/ no public repo, no timestamp, no mirror status tracker make sure that we have the latest version number under racket-installers
|
||||
x.org: https://www.x.org/releases/ no timestamp, but candidate for brute force looping since it has few folders, no status tracker
|
||||
Xiph: no timestamp, too big to loop through, no status tracker
|
||||
|
||||
|
|
19
data.json
19
data.json
|
@ -21,7 +21,7 @@
|
|||
"file": "zzz/time.txt"
|
||||
},
|
||||
"Arch": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_since": 1634334754,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "archlinux/",
|
||||
"upstream": "http://arch.mirror.constant.com/",
|
||||
|
@ -52,7 +52,7 @@
|
|||
"file": "x86/sha512.sum"
|
||||
},
|
||||
"Debian": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_since": 1634334754,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "https://ftp-master.debian.org/",
|
||||
|
@ -66,7 +66,7 @@
|
|||
"file": "debian-cd/project/trace/cdimage.debian.org"
|
||||
},
|
||||
"DebianMultimedia": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_since": 1634334754,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "debian-multimedia/",
|
||||
"upstream": "http://debian-mirrors.sdinet.de/deb-multimedia/",
|
||||
|
@ -80,7 +80,7 @@
|
|||
"file": "debian-ports/project/trace/porta.debian.org"
|
||||
},
|
||||
"DebianSecurity": {
|
||||
"out_of_sync_since": 1634275264,
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "",
|
||||
"upstream": "http://debian.mirror.iweb.ca/",
|
||||
|
@ -305,7 +305,7 @@
|
|||
"file": ""
|
||||
},
|
||||
"linuxmint_packages": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_since": 1634334754,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "linuxmint-packages/",
|
||||
"upstream": "https://mirrors.edge.kernel.org/linuxmint-packages/",
|
||||
|
@ -319,7 +319,7 @@
|
|||
"file": "dists/"
|
||||
},
|
||||
"ubuntu_ports_releases": {
|
||||
"out_of_sync_since": 1634257890,
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "ubuntu-ports-releases/",
|
||||
"upstream": "https://cdimage.ubuntu.com/releases/",
|
||||
|
@ -352,5 +352,12 @@
|
|||
"csc": "saltstack/",
|
||||
"upstream": "https://repo.saltproject.io/",
|
||||
"file": ""
|
||||
},
|
||||
"racket": {
|
||||
"out_of_sync_since": null,
|
||||
"out_of_sync_interval": 86400,
|
||||
"csc": "racket/racket-installers/",
|
||||
"upstream": "https://mirror.racket-lang.org/installers/",
|
||||
"file": ""
|
||||
}
|
||||
}
|
|
@ -0,0 +1,74 @@
|
|||
import requests
|
||||
import re
|
||||
from project import Project
|
||||
from shared import CSC_MIRROR
|
||||
import itertools
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
class racket(Project):
|
||||
"""racket class"""
|
||||
@staticmethod
|
||||
def max_version(processed_versions):
|
||||
latest_version = processed_versions[0]
|
||||
for version in processed_versions:
|
||||
if len(latest_version) > len(version):
|
||||
latest_version_portion = itertools.islice(latest_version, len(version))
|
||||
comparisons = list(map(lambda p, q: p == q, latest_version_portion, version))
|
||||
if False in comparisons:
|
||||
index = comparisons.index(False)
|
||||
if latest_version[index] < version[index]:
|
||||
latest_version = version
|
||||
if len(latest_version) < len(version):
|
||||
version_portion = itertools.islice(version, len(latest_version))
|
||||
comparisons = list(map(lambda p, q: p == q, version_portion, latest_version))
|
||||
if False in comparisons:
|
||||
index = comparisons.index(False)
|
||||
if latest_version[index] < version[index]:
|
||||
latest_version = version
|
||||
latest_version = version
|
||||
if len(latest_version) == len(version):
|
||||
comparisons = list(map(lambda p, q: p == q, version, latest_version))
|
||||
if False in comparisons:
|
||||
index = comparisons.index(False)
|
||||
if latest_version[index] < version[index]:
|
||||
latest_version = version
|
||||
return latest_version
|
||||
|
||||
@staticmethod
|
||||
def scrape(versions, site):
|
||||
# getting the request from url
|
||||
r = requests.get(site)
|
||||
|
||||
# converting the text
|
||||
s = BeautifulSoup(r.text,"html.parser")
|
||||
|
||||
for i in s.find_all("a"): # for a href directories
|
||||
href = i.attrs['href']
|
||||
|
||||
if href.endswith("/") and href != "../" and href != "/" and href != "recent/":
|
||||
if href not in versions:
|
||||
versions.append(href)
|
||||
|
||||
@classmethod
|
||||
def get_latest_version(cls, web_dir):
|
||||
page = requests.get(web_dir).text
|
||||
|
||||
versions = []
|
||||
cls.scrape(versions, web_dir)
|
||||
|
||||
processed_versions = []
|
||||
for version in versions:
|
||||
# print(version)
|
||||
processed_versions.append(version.replace("/", "").split("."))
|
||||
# print(cls.max_version(processed_versions))
|
||||
|
||||
return(cls.max_version(processed_versions))
|
||||
|
||||
|
||||
@classmethod
|
||||
def check(cls, data, project):
|
||||
"""Check if project packages are up-to-date"""
|
||||
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
||||
upstream_url = data[project]["upstream"] + data[project]["file"]
|
||||
|
||||
return cls.get_latest_version(csc_url) == cls.get_latest_version(upstream_url)
|
4
test.py
4
test.py
|
@ -7,7 +7,7 @@ from datetime import timedelta
|
|||
import time
|
||||
import pandas as pd
|
||||
import re # for salt stack specifically
|
||||
from projects import saltstack
|
||||
from projects import racket
|
||||
import json # import json to read project info stored in json file
|
||||
|
||||
# this function is brute force looping through the whole directory and checking dates
|
||||
|
@ -65,7 +65,7 @@ def get_latest_date(web_dir):
|
|||
if __name__ =="__main__":
|
||||
with open("data.json", "r", encoding="utf-8") as file:
|
||||
data = json.load(file)
|
||||
print(saltstack.check(data, "saltstack"))
|
||||
print(racket.check(data, "racket"))
|
||||
|
||||
"""# website to be scrape
|
||||
site="https://cdimage.ubuntu.com/releases/"
|
||||
|
|
Loading…
Reference in New Issue