d9hui
/
mirror-checker
forked from public/mirror-checker
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

nongnu added

This commit is contained in:
Tom 2021-10-04 00:49:24 -07:00
parent 73ff18ec67
commit f8917c9687
5 changed files with 32 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -27,6 +27,8 @@ mxlinux: https://sourceforge.net/projects/mx-linux/ (scrap the last day?)
linuxmint: no public repo linuxmint: no public repo
linuxmint-packages pool: http://rsync-packages.linuxmint.com/pool/ linuxmint-packages pool: http://rsync-packages.linuxmint.com/pool/
macPorts: only distfiles has public repo, no timestamp, too large to loop through macPorts: only distfiles has public repo, no timestamp, too large to loop through
NetBSD: http://ftp.netbsd.org/pub/NetBSD/ has public repo, no timestamp, web directory hard to loop through, no mirror tracker
nongnu: http://download.savannah.nongnu.org/releases/ https://savannah.gnu.org/maintenance/Mirmon/ http://download.savannah.gnu.org/mirmon/savannah/
scientific: https://scientificlinux.org/downloads/sl-mirrors/ (CSC not listed) scientific: https://scientificlinux.org/downloads/sl-mirrors/ (CSC not listed)
slackware: https://mirrors.slackware.com/mirrorlist/ https://mirrors.slackware.com/slackware/ slackware: https://mirrors.slackware.com/mirrorlist/ https://mirrors.slackware.com/slackware/
ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/ ubuntu-ports: http://ports.ubuntu.com/ubuntu-ports/

9
data.json
View File

@ -221,10 +221,17 @@
"file": "" "file": ""
}, },
"mySQL": { "mySQL": {
"out_of_sync_since": 1633330958, "out_of_sync_since": 1633333607,
"out_of_sync_interval": 86400, "out_of_sync_interval": 86400,
"csc": "mysql/", "csc": "mysql/",
"upstream": "http://mirrors.sunsite.dk/mysql/", "upstream": "http://mirrors.sunsite.dk/mysql/",
"file": "last-updated.txt" "file": "last-updated.txt"
},
"nongnu": {
"out_of_sync_since": 1633333607,
"out_of_sync_interval": 86400,
"csc": "nongnu/",
"upstream": "http://download-mirror.savannah.gnu.org/releases/",
"file": "00_TIME.txt"
} }
} }

2
projects/mySQL.py
View File

@ -1,5 +1,5 @@
""" """
Contains tdf class Contains mySQL class
""" """
from project import Project from project import Project

8
projects/nongnu.py Normal file
View File

@ -0,0 +1,8 @@
"""
Contains nongnu class
"""
from project import Project
class nongnu(Project):
"""nongnu class"""

26
test.py
View File

@ -13,7 +13,7 @@ import pandas as pd
# lists # lists
urls=[] urls=[]
home_site = "http://ykf.ca.distfiles.macports.org" home_site = "http://ftp.netbsd.org/pub"
# function created # function created
def scrape(site): def scrape(site):
@ -30,6 +30,8 @@ def scrape(site):
if href.endswith("/") and href != "../" and href != "/": if href.endswith("/") and href != "../" and href != "/":
"""if home_site+href in urls: # avoids the link to parent directory """if home_site+href in urls: # avoids the link to parent directory
continue""" continue"""
if href == "//ftp.netbsd.org/": # netbsd specific code
continue
site_next = site+href site_next = site+href
if site_next not in urls: if site_next not in urls:
@ -53,31 +55,29 @@ def get_latest_date(web_dir):
if __name__ =="__main__": if __name__ =="__main__":
# website to be scrape # website to be scrape
#site="http://ykf.ca.distfiles.macports.org/MacPorts/mpdistfiles/" site="http://ftp.netbsd.org/pub/NetBSD/"
# works on: https://www.x.org/releases/ # works on: https://www.x.org/releases/
# https://mirror.csclub.uwaterloo.ca/linuxmint/ #works wonders for linuxmint # https://mirror.csclub.uwaterloo.ca/linuxmint/ #works wonders for linuxmint
# unfortunately, linuxmint does not have a public repo, the worldwide mirror LayerOnline on https://linuxmint.com/mirrors.php seems like the best choice # unfortunately, linuxmint does not have a public repo, the worldwide mirror LayerOnline on https://linuxmint.com/mirrors.php seems like the best choice
# calling function # calling function
#scrape(site) scrape(site)
#latest_date = get_latest_date(urls[0]) latest_date = get_latest_date(urls[0])
# get_latest_date(urls[0]) # get_latest_date(urls[0])
#for dir in urls: for dir in urls:
# latest_date2 = get_latest_date(dir) latest_date2 = get_latest_date(dir)
# if (latest_date2 >= latest_date): if (latest_date2 >= latest_date):
# latest_date = latest_date2 latest_date = latest_date2
#print(latest_date) print(latest_date)
page = requests.get("http://rsync-mxlinux.org/mirmon/index.html").text """page = requests.get("http://rsync-mxlinux.org/mirmon/index.html").text
indexOfFile = page.find("mirror.csclub.uwaterloo.ca") indexOfFile = page.find("mirror.csclub.uwaterloo.ca")
m = re.search(r'(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460 m = re.search(r'(\d+ hours)|(\d+(\.)?\d+ days)', page[indexOfFile:]) # solution from: https://stackoverflow.com/questions/21074100/how-to-convert-standard-timedelta-string-to-timedelta-object/21074460
duration = pd.to_timedelta(m.group(0)) duration = pd.to_timedelta(m.group(0))
print (duration <= pd.to_timedelta(86400, unit='s')) print (duration <= pd.to_timedelta(86400, unit='s'))"""
# https://launchpad.net/ubuntu/+mirror/mirror.csclub.uwaterloo.ca-archive