main: retry 3 times before reporting errors
- modify `check_project` function so that it returns error reason string or returns None - add some type annotations
This commit is contained in:
parent
250f358f4e
commit
76f7863a85
40
main.py
40
main.py
|
@ -8,11 +8,15 @@ import time
|
|||
import sys
|
||||
import requests
|
||||
from multiprocessing import Pool, Manager
|
||||
from typing import Optional
|
||||
from time import sleep, localtime, strftime
|
||||
|
||||
from projects import *
|
||||
import json
|
||||
|
||||
NUM_THREAD = 16
|
||||
MAX_RETRY = 3
|
||||
RETRY_TIMEOUT = 30 # In seconds
|
||||
|
||||
current_time = int(time.time())
|
||||
|
||||
|
@ -21,33 +25,51 @@ def safe_print(*args, **kwargs):
|
|||
# due to buffering. Make sure to always flush the output.
|
||||
print(*args, **kwargs, flush=True)
|
||||
|
||||
def check_project(args):
|
||||
# Return None if no error occurs and a string for error message otherwise
|
||||
def check_project(args) -> Optional[str]:
|
||||
project, data = args
|
||||
try:
|
||||
project_class = getattr(sys.modules[__name__], project)
|
||||
|
||||
# Skip projects we no longer mirror
|
||||
if data[project].get('exclude', False):
|
||||
return True
|
||||
return None
|
||||
|
||||
checker_result = project_class.check(data, project, current_time)
|
||||
|
||||
if checker_result:
|
||||
data[project]["out_of_sync_since"] = None
|
||||
safe_print(f"Success: {project} up-to-date")
|
||||
return True
|
||||
return None
|
||||
|
||||
elif (data[project]["out_of_sync_since"] is not None
|
||||
and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
|
||||
safe_print(f"Failure: {project} out-of-sync")
|
||||
return False
|
||||
time_str = strftime("%d %b %Y %H:%M:%S (local time)", localtime())
|
||||
return f"{project} out-of-sync at {time_str}"
|
||||
|
||||
else:
|
||||
data[project]["out_of_sync_since"] = current_time
|
||||
return True
|
||||
return None
|
||||
|
||||
except requests.exceptions.RequestException as err:
|
||||
safe_print(f"Error: {project}\n{err}")
|
||||
return f"{project}\n{err}"
|
||||
|
||||
def check_project_with_retry(args) -> bool:
|
||||
project, _ = args
|
||||
errs = []
|
||||
for _ in range(MAX_RETRY):
|
||||
res = check_project(args)
|
||||
if res == None:
|
||||
safe_print(f"Success: {project} up-to-date")
|
||||
return True
|
||||
else:
|
||||
errs.append(res)
|
||||
# Do nothing, try again later
|
||||
sleep(RETRY_TIMEOUT)
|
||||
|
||||
# Max try reached, print errors
|
||||
safe_print(f"Error: {project}")
|
||||
for reason in errs:
|
||||
safe_print(f" {reason}")
|
||||
|
||||
return False
|
||||
|
||||
|
@ -61,7 +83,7 @@ def main():
|
|||
sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
|
||||
|
||||
with Pool(NUM_THREAD) as pool:
|
||||
all_pass = all(pool.imap(check_project, ((k, sync_data) for k in data.keys())))
|
||||
all_pass = all(pool.imap(check_project_with_retry, ((k, sync_data) for k in data.keys())))
|
||||
|
||||
with open(data_file, "w", encoding="utf-8") as file:
|
||||
json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')
|
||||
|
|
Loading…
Reference in New Issue