Check whether our mirror packages are up to date.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

72 lines
2.0 KiB

#!/usr/bin/env python3
"""
This mirror status checker determines whether CSC mirror is up-to-date with upstream
"""
import time
import sys
import requests
from multiprocessing import Pool, Manager
from projects import *
import json
NUM_THREAD = 16
current_time = int(time.time())
def safe_print(*args, **kwargs):
# When run with 'chronic' and 'timeout', stdout gets suppressed
# due to buffering. Make sure to always flush the output.
print(*args, **kwargs, flush=True)
def check_project(args):
project, data = args
try:
project_class = getattr(sys.modules[__name__], project)
# Skip projects we no longer mirror
if data[project].get('exclude', False):
return True
checker_result = project_class.check(data, project, current_time)
if checker_result:
data[project]["out_of_sync_since"] = None
safe_print(f"Success: {project} up-to-date")
return True
elif (data[project]["out_of_sync_since"] is not None
and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]):
safe_print(f"Failure: {project} out-of-sync")
return False
else:
data[project]["out_of_sync_since"] = current_time
return True
except requests.exceptions.RequestException as err:
safe_print(f"Error: {project}\n{err}")
return False
def main():
data_file = 'data.json'
if len(sys.argv) > 1:
data_file = sys.argv[1]
manager = Manager()
data = json.load(open(data_file))
sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()})
with Pool(NUM_THREAD) as pool:
all_pass = all(pool.imap(check_project, ((k, sync_data) for k in data.keys())))
with open(data_file, "w", encoding="utf-8") as file:
json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ')
sys.exit(0 if all_pass else 1)
if __name__ == "__main__":
main()