273 lines
8.5 KiB
Python
Executable File
273 lines
8.5 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
import os, sys
|
|
from os.path import join
|
|
import pickle
|
|
import re
|
|
import ConfigParser
|
|
import pprint
|
|
import time
|
|
import xmlrpclib
|
|
import base64
|
|
import bz2
|
|
import socket
|
|
|
|
class HostConfig(object):
|
|
"""Holder for config info from the configuration file"""
|
|
def __init__(self):
|
|
self.config = { 'version' : 0,
|
|
'global': {},
|
|
'site': {},
|
|
'host': {},
|
|
'stats': {},
|
|
}
|
|
|
|
_translation = [chr(_x) for _x in range(256)]
|
|
def _translate(s, altchars):
|
|
translation = _translation[:]
|
|
for k, v in altchars.items():
|
|
translation[ord(k)] = v
|
|
return s.translate(''.join(translation))
|
|
|
|
def urlsafe_b64encode(s):
|
|
import binascii
|
|
altchars = '-_'
|
|
encoded = binascii.b2a_base64(s)[:-1]
|
|
if altchars is not None:
|
|
return _translate(encoded, {'+': altchars[0], '/': altchars[1]})
|
|
return encoded
|
|
|
|
def gen_dirtree(path):
|
|
# structure here is:
|
|
# dirtree is a dict
|
|
# {
|
|
# dirpath :
|
|
# {
|
|
# filename1 : size1,
|
|
# filename2 : size2,
|
|
# ...
|
|
# },
|
|
# ...
|
|
# }
|
|
#
|
|
# 2009-03-09: MM's web app ignores the statfiles dict. So don't bother generating it.
|
|
|
|
dirtree = {}
|
|
for dirpath, dirnames, filenames in os.walk(path):
|
|
statfiles = {}
|
|
if path.endswith('/'):
|
|
short_path = dirpath[len(path):]
|
|
else:
|
|
short_path = dirpath[len(path)+1:]
|
|
if len(short_path) > 0:
|
|
dirtree[short_path] = statfiles
|
|
else:
|
|
dirtree[''] = statfiles
|
|
|
|
return dirtree
|
|
|
|
def errorprint(error):
|
|
sys.stderr.write(error+'\n')
|
|
|
|
class MissingOption(Exception):
|
|
pass
|
|
|
|
def check_required_options(conf, section, required_options):
|
|
for o in required_options:
|
|
if not conf.has_option(section, o):
|
|
errorprint('missing required option %s in config [%s]' % (o, section))
|
|
raise MissingOption()
|
|
return True
|
|
|
|
def parse_value(value):
|
|
"""Split multi-line values into a list"""
|
|
if value.find('\n') > -1:
|
|
return value.split()
|
|
return value
|
|
|
|
def parse_section(conf, section, item, required_options, optional_options=[]):
|
|
if conf.has_option(section, 'enabled'):
|
|
if conf.get(section, 'enabled') != '1' and section.lower() in item.config:
|
|
print 'removing disabled section %s' % (section)
|
|
del item.config[section.lower()]
|
|
return False
|
|
|
|
if not check_required_options(conf, section, required_options):
|
|
return False
|
|
|
|
if not section.lower() in item.config:
|
|
item.config[section.lower()] = {}
|
|
|
|
for o in required_options:
|
|
item.config[section.lower()][o] = parse_value(conf.get(section, o))
|
|
for o in optional_options:
|
|
if conf.has_option(section, o):
|
|
item.config[section.lower()][o] = parse_value(conf.get(section, o))
|
|
|
|
return True
|
|
|
|
def parse_global(conf, section, item):
|
|
required_options = [ 'enabled', 'server' ]
|
|
if not parse_section(conf, section, item, required_options):
|
|
errorprint('missing required options (server AND enabled) in [%s] section' % (section))
|
|
return False
|
|
return True
|
|
|
|
def parse_site(conf, section, item):
|
|
required_options = [ 'enabled', 'name', 'password' ]
|
|
return parse_section(conf, section, item, required_options)
|
|
|
|
def parse_host(conf, section, item):
|
|
required_options = [ 'enabled', 'name' ]
|
|
optional_options = [ 'user_active' ]
|
|
return parse_section(conf, section, item, required_options, optional_options=optional_options)
|
|
|
|
def get_stats(conf, section):
|
|
if conf.has_option(section, 'enabled'):
|
|
if conf.get(section, 'enabled') != '1':
|
|
return None
|
|
statsdata = {}
|
|
for name, value in conf.items(section):
|
|
if name == 'enabled':
|
|
continue
|
|
filenames = parse_value(conf.get(section, name))
|
|
if type(filenames) != list:
|
|
filenames = [ filenames ]
|
|
for fn in filenames:
|
|
try:
|
|
f = open(fn, 'r')
|
|
contents = contents + f.readlines()
|
|
statsdata[name] = pickle.dumps(contents, -1)
|
|
f.close()
|
|
except:
|
|
pass
|
|
return statsdata
|
|
|
|
def parse_category(conf, section, item, crawl):
|
|
required_options = [ 'enabled', 'path' ]
|
|
if not parse_section(conf, section, item, required_options):
|
|
return False
|
|
|
|
if crawl:
|
|
dirtree = gen_dirtree(conf.get(section, 'path'))
|
|
item.config[section.lower()]['dirtree'] = dirtree
|
|
# database doesn't need to know the disk path
|
|
del item.config[section.lower()]['path']
|
|
|
|
def config(cfg, item, crawl=True):
|
|
broken = False
|
|
conf = ConfigParser.ConfigParser()
|
|
files = conf.read(cfg)
|
|
if files == []:
|
|
errorprint('Configuration file %s not found' % (cfg))
|
|
return False
|
|
conf.read(cfg)
|
|
|
|
try:
|
|
# don't grab parse_stats here
|
|
for section, parsefunc in [ ('global', parse_global), ('site', parse_site),
|
|
('host', parse_host)]:
|
|
if conf.has_section(section):
|
|
if not parsefunc(conf, section, item):
|
|
return False
|
|
else:
|
|
errorprint('Invalid configuration - missing section [%s]' % (section))
|
|
sys.exit(1)
|
|
|
|
for section in conf.sections():
|
|
if section in [ 'global', 'site', 'host', 'stats']:
|
|
continue
|
|
parse_category(conf, section, item, crawl)
|
|
|
|
except MissingOption:
|
|
errorprint('Invalid configuration - Exiting')
|
|
sys.exit(1)
|
|
|
|
return True
|
|
|
|
|
|
|
|
def main():
|
|
from optparse import OptionParser
|
|
parser = OptionParser(usage= sys.argv[0] + " [options]")
|
|
parser.add_option("-c", "--config",
|
|
dest="config",
|
|
default='/home/mirror/mirrormanager-client/report_mirror.conf',
|
|
help='Configuration filename (required)')
|
|
parser.add_option("-s", "--stats",
|
|
action="store_true",
|
|
dest="stats",
|
|
default=False,
|
|
help='Send stats')
|
|
parser.add_option("-i", "--input",
|
|
dest="input",
|
|
default=None,
|
|
help="Input filename (for debugging)")
|
|
parser.add_option("-o", "--output",
|
|
dest="output",
|
|
default=None,
|
|
help="Output filename (for debugging)")
|
|
parser.add_option("-n", "--no-send",
|
|
action="store_true",
|
|
dest="no_send",
|
|
default=False,
|
|
help="Don't send data to the server.")
|
|
parser.add_option("-d", "--debug",
|
|
action="store_true",
|
|
dest="debug",
|
|
default=False,
|
|
help='Enable debugging output')
|
|
|
|
|
|
(options, args) = parser.parse_args()
|
|
item = HostConfig()
|
|
if options.input:
|
|
infile = open(options.input, 'rb')
|
|
item.config = pickle.load(infile)
|
|
infile.close()
|
|
if not config(options.config, item, crawl=False):
|
|
sys.exit(1)
|
|
else:
|
|
if not config(options.config, item, crawl=True):
|
|
sys.exit(1)
|
|
|
|
p = pickle.dumps(item.config, -1)
|
|
|
|
if options.debug:
|
|
pp = pprint.PrettyPrinter(indent=4)
|
|
pp.pprint(item.config)
|
|
|
|
if options.output is not None:
|
|
outfile = open(options.output, 'wb')
|
|
outfile.write(p)
|
|
outfile.close()
|
|
|
|
if options.stats:
|
|
statdata = get_stats(conf, 'stats')
|
|
|
|
# upload p and statsdata here
|
|
if not item.config.has_key('global') or not item.config['global'].has_key('enabled') or item.config['global']['enabled'] != '1':
|
|
sys.exit(1)
|
|
|
|
if not options.no_send:
|
|
# print "Connecting to %s" % item.config['global']['server']
|
|
server = xmlrpclib.ServerProxy(item.config['global']['server'])
|
|
data = None
|
|
try:
|
|
data = base64.urlsafe_b64encode(bz2.compress(p))
|
|
except AttributeError:
|
|
data = urlsafe_b64encode(bz2.compress(p))
|
|
|
|
if data is not None:
|
|
try:
|
|
print server.checkin(data)
|
|
except socket.error, m:
|
|
print "Error checking in: %s. Please try again later." % (m[1])
|
|
except xmlrpclib.Fault:
|
|
print "Error checking in. Connection closed before checkin complete. Please try again later."
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|