#!/usr/bin/env python3 """ This mirror status checker determines whether CSC mirror is up-to-date with upstream """ import time import sys import requests from multiprocessing import Pool, Manager from projects import * import json NUM_THREAD = 16 current_time = int(time.time()) def safe_print(*args, **kwargs): # When run with 'chronic' and 'timeout', stdout gets suppressed # due to buffering. Make sure to always flush the output. print(*args, **kwargs, flush=True) def check_project(args): project, data = args try: project_class = getattr(sys.modules[__name__], project) # Skip projects we no longer mirror if data[project].get('exclude', False): return True checker_result = project_class.check(data, project, current_time) if checker_result: data[project]["out_of_sync_since"] = None safe_print(f"Success: {project} up-to-date") return True elif (data[project]["out_of_sync_since"] is not None and current_time - data[project]["out_of_sync_since"] > data[project]["out_of_sync_interval"]): safe_print(f"Failure: {project} out-of-sync") return False else: data[project]["out_of_sync_since"] = current_time return True except requests.exceptions.RequestException as err: safe_print(f"Error: {project}\n{err}") return False def main(): data_file = 'data.json' if len(sys.argv) > 1: data_file = sys.argv[1] manager = Manager() data = json.load(open(data_file)) sync_data = manager.dict({k: manager.dict(v) for k, v in data.items()}) with Pool(NUM_THREAD) as pool: all_pass = all(pool.imap(check_project, ((k, sync_data) for k in data.keys()))) with open(data_file, "w", encoding="utf-8") as file: json.dump({k: dict(v) for k, v in sync_data.items()}, file, indent=' ') sys.exit(0 if all_pass else 1) if __name__ == "__main__": main()