From a526e3ea2653a5329949d708eea34e98062ec5a7 Mon Sep 17 00:00:00 2001
From: Tom <shutong5s@gmail.com>
Date: Tue, 24 Aug 2021 16:14:24 -0700
Subject: [PATCH] added the implementation to check timestamps in distro mirror
 directory and store distro info in JSON files

---
 distros.json |  5 +++++
 main.py      | 44 ++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 distros.json

diff --git a/distros.json b/distros.json
new file mode 100644
index 0000000..0904010
--- /dev/null
+++ b/distros.json
@@ -0,0 +1,5 @@
+[
+    ["OpenBSD", "https://mirror.csclub.uwaterloo.ca/OpenBSD/", "https://ftp.openbsd.org/pub/OpenBSD/", "timestamp"],
+    ["kernel", "http://mirror.csclub.uwaterloo.ca/kernel.org/linux/kernel/next/", "https://mirrors.edge.kernel.org/pub/linux/kernel/next/", "sha256sums.asc"],
+    ["debian", "http://mirror.csclub.uwaterloo.ca/debian/project/trace/", "https://ftp-master.debian.org/debian/project/trace/", "master"]
+]
\ No newline at end of file
diff --git a/main.py b/main.py
index fe0a899..7c415e8 100644
--- a/main.py
+++ b/main.py
@@ -9,10 +9,50 @@ from arch import Arch
 from debian import Debian
 from kernel import Kernel
 from openbsd import OpenBSD
+from dateparser.search import search_dates # this library seems to be super slow but the other library: dateutil.parser gets some errors
+# http://theautomatic.net/2018/12/18/2-packages-for-extracting-dates-from-a-string-of-text-in-python/
+import re # import regular expressions to remove stray numbers in string that might interfere with date finding
+import json # import json to read distro info stored in json file
+
+import datefinder # another date finding library
+
+def checker(directory_URL, file_name):
+    page = requests.get(directory_URL).text
+    indexOfFile = page.find(file_name)
+    # print(page)
+
+    # remove stray numbers (file size numbers in particular) that might interfere with date finding
+    segment_clean = re.sub(r'\s\d+\s', ' ', page[indexOfFile:]) # removes numbers for size
+    segment_clean = re.sub(r'\s\d+\w*\s', ' ', page[indexOfFile:]) # removes numbers + size unit. e.x. 50kb
+    # print(segment_clean)
+
+    # implementation using dateparser.search.search_dates
+    # notes: some dates don't parse correctly with this tool
+    # print(search_dates(page[indexOfFile:], languages=['en']))
+    # print(search_dates(page[indexOfFile:])[0])
+
+    # finds the dates in the segment after the file name
+    # notes: a generator will be returned by the datefinder module. I'm typecasting it to a list. Please read the note of caution provided at the bottom.
+    matches = list(datefinder.find_dates(segment_clean))
+    # print(matches)
+
+    if len(matches) > 0:
+        date = matches[0] # date is of type datetime.datetime
+        return(date.strftime("%m/%d/%Y, %H:%M:%S"))
+    else:
+        return('No dates found')
 
 if __name__ == "__main__":
-    for distro in [Arch, Debian, Kernel, OpenBSD]:
+    """for distro in [Arch, Debian, Kernel, OpenBSD]:
         try:
             distro.print_output(distro.check())
         except requests.exceptions.RequestException as err:
-            print(f"Error: {distro.name()}\n{err}")
+            print(f"Error: {distro.name()}\n{err}")"""
+    
+    distros = json.load(open('distros.json',))
+    print(distros)
+
+    for distro in distros:
+        print(distro[0] + ":")
+        print("CSC mirror: " + checker(distro[1], distro[3]))
+        print("Official distro: " + checker(distro[2], distro[3]))
\ No newline at end of file