#!/usr/bin/python import os, sys from os.path import join import pickle import re import ConfigParser import pprint import time import xmlrpclib import base64 import bz2 import socket class HostConfig(object): """Holder for config info from the configuration file""" def __init__(self): self.config = { 'version' : 0, 'global': {}, 'site': {}, 'host': {}, 'stats': {}, } _translation = [chr(_x) for _x in range(256)] def _translate(s, altchars): translation = _translation[:] for k, v in altchars.items(): translation[ord(k)] = v return s.translate(''.join(translation)) def urlsafe_b64encode(s): import binascii altchars = '-_' encoded = binascii.b2a_base64(s)[:-1] if altchars is not None: return _translate(encoded, {'+': altchars[0], '/': altchars[1]}) return encoded def gen_dirtree(path): # structure here is: # dirtree is a dict # { # dirpath : # { # filename1 : size1, # filename2 : size2, # ... # }, # ... # } # # 2009-03-09: MM's web app ignores the statfiles dict. So don't bother generating it. dirtree = {} for dirpath, dirnames, filenames in os.walk(path): statfiles = {} if path.endswith('/'): short_path = dirpath[len(path):] else: short_path = dirpath[len(path)+1:] if len(short_path) > 0: dirtree[short_path] = statfiles else: dirtree[''] = statfiles return dirtree def errorprint(error): sys.stderr.write(error+'\n') class MissingOption(Exception): pass def check_required_options(conf, section, required_options): for o in required_options: if not conf.has_option(section, o): errorprint('missing required option %s in config [%s]' % (o, section)) raise MissingOption() return True def parse_value(value): """Split multi-line values into a list""" if value.find('\n') > -1: return value.split() return value def parse_section(conf, section, item, required_options, optional_options=[]): if conf.has_option(section, 'enabled'): if conf.get(section, 'enabled') != '1' and section.lower() in item.config: print 'removing disabled section %s' % (section) del item.config[section.lower()] return False if not check_required_options(conf, section, required_options): return False if not section.lower() in item.config: item.config[section.lower()] = {} for o in required_options: item.config[section.lower()][o] = parse_value(conf.get(section, o)) for o in optional_options: if conf.has_option(section, o): item.config[section.lower()][o] = parse_value(conf.get(section, o)) return True def parse_global(conf, section, item): required_options = [ 'enabled', 'server' ] if not parse_section(conf, section, item, required_options): errorprint('missing required options (server AND enabled) in [%s] section' % (section)) return False return True def parse_site(conf, section, item): required_options = [ 'enabled', 'name', 'password' ] return parse_section(conf, section, item, required_options) def parse_host(conf, section, item): required_options = [ 'enabled', 'name' ] optional_options = [ 'user_active' ] return parse_section(conf, section, item, required_options, optional_options=optional_options) def get_stats(conf, section): if conf.has_option(section, 'enabled'): if conf.get(section, 'enabled') != '1': return None statsdata = {} for name, value in conf.items(section): if name == 'enabled': continue filenames = parse_value(conf.get(section, name)) if type(filenames) != list: filenames = [ filenames ] for fn in filenames: try: f = open(fn, 'r') contents = contents + f.readlines() statsdata[name] = pickle.dumps(contents, -1) f.close() except: pass return statsdata def parse_category(conf, section, item, crawl): required_options = [ 'enabled', 'path' ] if not parse_section(conf, section, item, required_options): return False if crawl: dirtree = gen_dirtree(conf.get(section, 'path')) item.config[section.lower()]['dirtree'] = dirtree # database doesn't need to know the disk path del item.config[section.lower()]['path'] def config(cfg, item, crawl=True): broken = False conf = ConfigParser.ConfigParser() files = conf.read(cfg) if files == []: errorprint('Configuration file %s not found' % (cfg)) return False conf.read(cfg) try: # don't grab parse_stats here for section, parsefunc in [ ('global', parse_global), ('site', parse_site), ('host', parse_host)]: if conf.has_section(section): if not parsefunc(conf, section, item): return False else: errorprint('Invalid configuration - missing section [%s]' % (section)) sys.exit(1) for section in conf.sections(): if section in [ 'global', 'site', 'host', 'stats']: continue parse_category(conf, section, item, crawl) except MissingOption: errorprint('Invalid configuration - Exiting') sys.exit(1) return True def main(): from optparse import OptionParser parser = OptionParser(usage= sys.argv[0] + " [options]") parser.add_option("-c", "--config", dest="config", default='/home/mirror/mirrormanager-client/report_mirror.conf', help='Configuration filename (required)') parser.add_option("-s", "--stats", action="store_true", dest="stats", default=False, help='Send stats') parser.add_option("-i", "--input", dest="input", default=None, help="Input filename (for debugging)") parser.add_option("-o", "--output", dest="output", default=None, help="Output filename (for debugging)") parser.add_option("-n", "--no-send", action="store_true", dest="no_send", default=False, help="Don't send data to the server.") parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False, help='Enable debugging output') (options, args) = parser.parse_args() item = HostConfig() if options.input: infile = open(options.input, 'rb') item.config = pickle.load(infile) infile.close() if not config(options.config, item, crawl=False): sys.exit(1) else: if not config(options.config, item, crawl=True): sys.exit(1) p = pickle.dumps(item.config, -1) if options.debug: pp = pprint.PrettyPrinter(indent=4) pp.pprint(item.config) if options.output is not None: outfile = open(options.output, 'wb') outfile.write(p) outfile.close() if options.stats: statdata = get_stats(conf, 'stats') # upload p and statsdata here if not item.config.has_key('global') or not item.config['global'].has_key('enabled') or item.config['global']['enabled'] != '1': sys.exit(1) if not options.no_send: # print "Connecting to %s" % item.config['global']['server'] server = xmlrpclib.ServerProxy(item.config['global']['server']) data = None try: data = base64.urlsafe_b64encode(bz2.compress(p)) except AttributeError: data = urlsafe_b64encode(bz2.compress(p)) if data is not None: try: print server.checkin(data) except socket.error, m: print "Error checking in: %s. Please try again later." % (m[1]) except xmlrpclib.Fault: print "Error checking in. Connection closed before checkin complete. Please try again later." sys.exit(1) if __name__ == '__main__': main()