Merge pull request 'various mirror fixes' (#7) from y266shen/mirror-checker:feature-retry into master

Reviewed-on: #7

main.py

@@ -18,8 +18,6 @@ NUM_THREAD = 16
 MAX_RETRY = 3
 RETRY_TIMEOUT = 30 # In seconds
 
-current_time = int(time.time())
-
 def safe_print(*args, **kwargs):
     # When run with 'chronic' and 'timeout', stdout gets suppressed
     # due to buffering. Make sure to always flush the output.
@@ -27,6 +25,7 @@ def safe_print(*args, **kwargs):
 
 # Return None if no error occurs and a string for error message otherwise
 def check_project(args) -> Optional[str]:
+    current_time = int(time.time())
     project, data = args
     try:
         project_class = getattr(sys.modules[__name__], project)
@@ -43,8 +42,9 @@ def check_project(args) -> Optional[str]:
 
         elif (data[project]["out_of_sync_since"] is not None
                 and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
-            time_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
-            return f"{project} out-of-sync at {time_str}"
+            now_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
+            duration = current_time - data[project]["out_of_sync_since"]
+            return f"{project} out-of-sync at {now_str} for {duration}s"
 
         else:
             data[project]["out_of_sync_since"] = current_time

projects/apache.py

@@ -3,7 +3,35 @@ Contains Apache class
 """
 
 from project import Project
-
+from shared import CSC_MIRROR
+import requests
 
 class Apache(Project):
-	"""Apache class"""
+    """Apache class"""
+
+    # Apache's time file has two segments, so we need a special function
+    # Example: 1648323001 rsync-he-fi
+    def check(data, project, current_time):
+        """Check if project packages are up-to-date"""
+        csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
+        upstream_url = data[project]["upstream"] + data[project]["file"]
+
+        req = requests.get(csc_url)
+        req.raise_for_status()
+        CSC = req.text
+
+        req = requests.get(upstream_url)
+        req.raise_for_status()
+        upstream = req.text
+
+        if upstream == CSC:
+            return True
+        try:
+            return get_timestamp_from_apache(upstream) - get_timestamp_from_apache(CSC) < data[project]["out_of_sync_interval"]
+        except ValueError:
+            print("failed to parse apache")
+            return False
+
+def get_timestamp_from_apache(s: str) -> int:
+    real_time = s.split(" ")[0]
+    return int(real_time)

projects/freebsd.py

@@ -3,7 +3,35 @@ Contains FreeBSD class
 """
 
 from project import Project
-
+from shared import CSC_MIRROR
+import requests
 
 class FreeBSD(Project):
-	"""FreeBSD class"""
+    """FreeBSD class"""
+
+    # FreeBSD's time file has two segments, so we need a special function
+    # Example TIMESTAMP file: 1648308600 2022-03-26T15:30:00 UTC
+    def check(data, project, current_time):
+        """Check if project packages are up-to-date"""
+        csc_url = CSC_MIRROR + data[project]["csc"] + data[project]["file"]
+        upstream_url = data[project]["upstream"] + data[project]["file"]
+
+        req = requests.get(csc_url)
+        req.raise_for_status()
+        CSC = req.text
+
+        req = requests.get(upstream_url)
+        req.raise_for_status()
+        upstream = req.text
+
+        if upstream == CSC:
+            return True
+        try:
+            return get_timestamp_for_freebsd(upstream) - get_timestamp_for_freebsd(CSC) < data[project]["out_of_sync_interval"]
+        except ValueError:
+            print("failed to parse apache")
+            return False
+
+def get_timestamp_for_freebsd(s: str) -> int:
+    real_time = s.split(" ")[0]
+    return int(real_time)