forked from public/mirror-checker
79 lines
2.7 KiB
Python
79 lines
2.7 KiB
Python
from bs4 import BeautifulSoup
|
|
import requests
|
|
import re
|
|
import datefinder # another date finding library
|
|
from project import Project
|
|
from shared import CSC_MIRROR
|
|
|
|
class puppy_linux(Project):
|
|
"""puppy_linux class"""
|
|
@staticmethod
|
|
def checker(page, file_name):
|
|
file_index = page.find(file_name)
|
|
# print(page)
|
|
|
|
if file_index == -1:
|
|
return False
|
|
|
|
str_dates = re.findall(r'(\d{2}-\w{3}-\d{4} \d{2}:\d{2})|(\d{4}-\w{3}-\d{2} \d{2}:\d{2})', page[file_index:])
|
|
|
|
return list(datefinder.find_dates("".join(str_dates[0])))[0]
|
|
|
|
@classmethod
|
|
def scrape(cls, compare, folders, site1, site2, directory):
|
|
# getting the request from url
|
|
r = requests.get(site1 + directory)
|
|
r1 = requests.get(site2 + directory)
|
|
|
|
page1 = r.text
|
|
page2 = r1.text
|
|
|
|
# converting the text
|
|
s = BeautifulSoup(page1,"html.parser")
|
|
|
|
for i in s.find_all("a"): # for a href directories
|
|
href = i.attrs['href']
|
|
|
|
if href.endswith(".iso"):
|
|
date1 = cls.checker(page1, href)
|
|
if date1 != False:
|
|
# print (site1+directory)
|
|
# print (date1)
|
|
date2 = cls.checker(page2, href)
|
|
if date2 != False:
|
|
# print (site2+directory)
|
|
# print (date2)
|
|
compare.append(date1 <= date2)
|
|
return
|
|
compare.append(False)
|
|
return
|
|
elif href.endswith("/") and (href.startswith("puppy-") or directory != "") and href != "../" and href != "/" and not href.startswith("/"):
|
|
dir_next = directory+href
|
|
# print(dir_next)
|
|
# calling it self
|
|
if dir_next not in folders:
|
|
folders.append(dir_next)
|
|
cls.scrape(compare, folders, site1, site2, dir_next)
|
|
elif href.endswith(".htm") or href == "Packages.gz":
|
|
# print(href)
|
|
date2 = cls.checker(page2, href)
|
|
if date2 != False:
|
|
compare.append(cls.checker(page1, href) <= date2)
|
|
continue
|
|
compare.append(False)
|
|
|
|
|
|
@classmethod
|
|
def check(cls, data, project):
|
|
"""Check if project packages are up-to-date"""
|
|
# lists
|
|
compare=[]
|
|
folders=[]
|
|
|
|
csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
|
|
upstream_url = data[project]["upstream"] + data[project]["file"]
|
|
|
|
# calling function
|
|
cls.scrape(compare, folders, upstream_url, csc_url, "")
|
|
|
|
return all(compare) |