main: retry 3 times before reporting errors

- modify `check_project` function so that it returns error reason
  string or returns None
- add some type annotations

main.py

@@ -8,11 +8,15 @@ import time
 import sys
 import requests
 from multiprocessing import Pool, Manager
+from typing import Optional
+from time import sleep, localtime, strftime
 
 from projects import *
 import json
 
 NUM_THREAD = 16
+MAX_RETRY = 3
+RETRY_TIMEOUT = 30 # In seconds
 
 current_time = int(time.time())
 
@@ -21,33 +25,51 @@ def safe_print(*args, **kwargs):
     # due to buffering. Make sure to always flush the output.
     print(*args, **kwargs, flush=True)
 
-def check_project(args):
+# Return None if no error occurs and a string for error message otherwise
+def check_project(args) -> Optional[str]:
     project, data = args
     try:
         project_class = getattr(sys.modules[__name__], project)
 
         # Skip projects we no longer mirror
         if data[project].get('exclude', False):
-            return True
+            return None
 
         checker_result = project_class.check(data, project, current_time)
 
         if checker_result:
             data[project]["out_of_sync_since"] = None
-            safe_print(f"Success: {project} up-to-date")
-            return True
+            return None
 
         elif (data[project]["out_of_sync_since"] is not None
                 and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
-            safe_print(f"Failure: {project} out-of-sync")
-            return False
+            time_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
+            return f"{project} out-of-sync at {time_str}"
 
         else:
             data[project]["out_of_sync_since"] = current_time
-            return True
+            return None
 
     except requests.exceptions.RequestException as err:
-        safe_print(f"Error: {project}\n{err}")
+        return f"{project}\n{err}"
+
+def check_project_with_retry(args) -> bool:
+    project, _ = args
+    errs = []
+    for _ in range(MAX_RETRY):
+        res = check_project(args)
+        if res == None:
+            safe_print(f"Success: {project} up-to-date")
+            return True
+        else:
+            errs.append(res)
+            # Do nothing, try again later
+            sleep(RETRY_TIMEOUT)
+
+    # Max try reached, print errors
+    safe_print(f"Error: {project}")
+    for reason in errs:
+        safe_print(f"       {reason}")
 
     return False
 
@@ -61,7 +83,7 @@ def main():
     sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
 
     with Pool(NUM_THREAD) as pool:
-        all_pass = all(pool.imap(check_project, ((k, sync_data) for k in data.keys())))
+        all_pass = all(pool.imap(check_project_with_retry, ((k, sync_data) for k in data.keys())))
 
     with open(data_file, "w", encoding="utf-8") as file:
         json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent='  ')