diff --git a/main.py b/main.py index 46fb640..0d2e0eb 100644 --- a/main.py +++ b/main.py @@ -8,11 +8,15 @@ import time import sys import requests from multiprocessing import Pool, Manager +from typing import Optional +from time import sleep, localtime, strftime from projects import * import json NUM_THREAD = 16 +MAX_RETRY = 3 +RETRY_TIMEOUT = 30 # In seconds current_time = int(time.time()) @@ -21,33 +25,51 @@ def safe_print(*args, **kwargs): # due to buffering. Make sure to always flush the output. print(*args, **kwargs, flush=True) -def check_project(args): +# Return None if no error occurs and a string for error message otherwise +def check_project(args) -> Optional[str]: project, data = args try: project_class = getattr(sys.modules[__name__], project) # Skip projects we no longer mirror if data[project].get('exclude', False): - return True + return None checker_result = project_class.check(data, project, current_time) if checker_result: data[project]["out_of_sync_since"] = None - safe_print(f"Success: {project} up-to-date") - return True + return None elif (data[project]["out_of_sync_since"] is not None and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]): - safe_print(f"Failure: {project} out-of-sync") - return False + time_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime()) + return f"{project} out-of-sync at {time_str}" else: data[project]["out_of_sync_since"] = current_time - return True + return None except requests.exceptions.RequestException as err: - safe_print(f"Error: {project}\n{err}") + return f"{project}\n{err}" + +def check_project_with_retry(args) -> bool: + project, _ = args + errs = [] + for _ in range(MAX_RETRY): + res = check_project(args) + if res == None: + safe_print(f"Success: {project} up-to-date") + return True + else: + errs.append(res) + # Do nothing, try again later + sleep(RETRY_TIMEOUT) + + # Max try reached, print errors + safe_print(f"Error: {project}") + for reason in errs: + safe_print(f" {reason}") return False @@ -61,7 +83,7 @@ def main(): sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()}) with Pool(NUM_THREAD) as pool: - all_pass = all(pool.imap(check_project, ((k, sync_data) for k in data.keys()))) + all_pass = all(pool.imap(check_project_with_retry, ((k, sync_data) for k in data.keys()))) with open(data_file, "w", encoding="utf-8") as file: json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')