a couple more recent scripts
authorJeremy Brandon Roman <jbroman@csclub.uwaterloo.ca>
Sun, 7 Mar 2010 17:28:13 +0000 (12:28 -0500)
committerJeremy Brandon Roman <jbroman@csclub.uwaterloo.ca>
Sun, 7 Mar 2010 17:28:13 +0000 (12:28 -0500)
report_mirror [new file with mode: 0755]
ubuntu-releases-sync [new file with mode: 0755]

diff --git a/report_mirror b/report_mirror
new file mode 100755 (executable)
index 0000000..2f1054f
--- /dev/null
@@ -0,0 +1,272 @@
+#!/usr/bin/python
+
+import os, sys
+from os.path import join
+import pickle
+import re
+import ConfigParser
+import pprint
+import time
+import xmlrpclib
+import base64
+import bz2
+import socket
+
+class HostConfig(object):
+    """Holder for config info from the configuration file"""
+    def __init__(self):
+        self.config = { 'version' : 0,
+                        'global': {},
+                        'site': {},
+                        'host': {},
+                        'stats': {},
+                        }
+
+_translation = [chr(_x) for _x in range(256)]
+def _translate(s, altchars):
+    translation = _translation[:]
+    for k, v in altchars.items():
+        translation[ord(k)] = v
+    return s.translate(''.join(translation))
+
+def urlsafe_b64encode(s):
+   import binascii
+   altchars = '-_'
+   encoded = binascii.b2a_base64(s)[:-1]
+   if altchars is not None:
+      return _translate(encoded, {'+': altchars[0], '/': altchars[1]})
+   return encoded
+
+def gen_dirtree(path):
+    # structure here is:
+    # dirtree is a dict
+    # {
+    #   dirpath :
+    #                {
+    #                   filename1 : size1,
+    #                   filename2 : size2,
+    #                   ...
+    #                 },
+    #   ...
+    # }
+    # 
+    # 2009-03-09: MM's web app ignores the statfiles dict.  So don't bother generating it.
+
+    dirtree = {}
+    for dirpath, dirnames, filenames in os.walk(path):
+        statfiles = {}
+        if path.endswith('/'):
+            short_path = dirpath[len(path):]
+        else:
+            short_path = dirpath[len(path)+1:]
+        if len(short_path) > 0:
+            dirtree[short_path] = statfiles
+        else:
+            dirtree[''] = statfiles
+
+    return dirtree
+
+def errorprint(error):
+    sys.stderr.write(error+'\n')
+
+class MissingOption(Exception):
+    pass
+
+def check_required_options(conf, section, required_options):
+    for o in required_options:
+        if not conf.has_option(section, o):
+            errorprint('missing required option %s in config [%s]' % (o, section))
+            raise MissingOption()
+    return True
+
+def parse_value(value):
+    """Split multi-line values into a list"""
+    if value.find('\n') > -1:
+        return value.split()
+    return value
+
+def parse_section(conf, section, item, required_options, optional_options=[]):
+    if conf.has_option(section, 'enabled'):
+        if conf.get(section, 'enabled') != '1' and section.lower() in item.config:
+            print 'removing disabled section %s' % (section)
+            del item.config[section.lower()]
+            return False
+
+    if not check_required_options(conf, section, required_options):
+        return False
+
+    if not section.lower() in item.config:
+        item.config[section.lower()] = {}
+
+    for o in required_options:
+        item.config[section.lower()][o] = parse_value(conf.get(section, o))
+    for o in optional_options:
+        if conf.has_option(section, o):
+            item.config[section.lower()][o] = parse_value(conf.get(section, o))
+
+    return True
+
+def parse_global(conf, section, item):
+    required_options = [ 'enabled', 'server' ]
+    if not parse_section(conf, section, item, required_options):
+        errorprint('missing required options (server AND enabled) in [%s] section' % (section))
+        return False
+    return True
+
+def parse_site(conf, section, item):
+    required_options = [ 'enabled', 'name', 'password' ]
+    return parse_section(conf, section, item, required_options)
+
+def parse_host(conf, section, item):
+    required_options = [ 'enabled', 'name' ]
+    optional_options = [ 'user_active' ]
+    return parse_section(conf, section, item, required_options, optional_options=optional_options)
+
+def get_stats(conf, section):
+    if conf.has_option(section, 'enabled'):
+        if conf.get(section, 'enabled') != '1':
+            return None
+    statsdata = {}
+    for name, value in conf.items(section):
+        if name == 'enabled':
+            continue
+        filenames = parse_value(conf.get(section, name))
+        if type(filenames) != list:
+            filenames = [ filenames ]
+        for fn in filenames:
+            try:
+                f = open(fn, 'r')
+                contents = contents + f.readlines()
+                statsdata[name] = pickle.dumps(contents, -1)
+                f.close()
+            except:
+                pass
+    return statsdata
+
+def parse_category(conf, section, item, crawl):
+    required_options = [ 'enabled', 'path' ]
+    if not parse_section(conf, section, item, required_options):
+        return False
+
+    if crawl:
+        dirtree = gen_dirtree(conf.get(section, 'path'))
+        item.config[section.lower()]['dirtree'] = dirtree
+    # database doesn't need to know the disk path
+    del item.config[section.lower()]['path']
+
+def config(cfg, item, crawl=True):
+    broken = False
+    conf = ConfigParser.ConfigParser()
+    files = conf.read(cfg)
+    if files == []:
+        errorprint('Configuration file %s not found' % (cfg))
+        return False
+    conf.read(cfg)
+
+    try:
+        # don't grab parse_stats here
+        for section, parsefunc in [ ('global', parse_global), ('site', parse_site),
+                                    ('host', parse_host)]:
+            if conf.has_section(section):
+                if not parsefunc(conf, section, item):
+                    return False
+            else:
+                errorprint('Invalid configuration - missing section [%s]' % (section))
+                sys.exit(1)
+
+        for section in conf.sections():
+            if section in [ 'global', 'site', 'host', 'stats']:
+                continue
+            parse_category(conf, section, item, crawl)
+
+    except MissingOption:
+        errorprint('Invalid configuration - Exiting')
+        sys.exit(1)
+
+    return True
+
+
+
+def main():
+    from optparse import OptionParser
+    parser = OptionParser(usage= sys.argv[0] + " [options]")
+    parser.add_option("-c", "--config",
+                      dest="config",
+                      default='/etc/mirrormanager-client/report_mirror.conf',
+                      help='Configuration filename (required)')
+    parser.add_option("-s", "--stats",
+                      action="store_true",
+                      dest="stats",
+                      default=False,
+                      help='Send stats')
+    parser.add_option("-i", "--input",
+                      dest="input",
+                      default=None,
+                      help="Input filename (for debugging)")
+    parser.add_option("-o", "--output",
+                      dest="output",
+                      default=None,
+                      help="Output filename (for debugging)")
+    parser.add_option("-n", "--no-send",
+                      action="store_true",
+                      dest="no_send",
+                      default=False,
+                      help="Don't send data to the server.")
+    parser.add_option("-d", "--debug",
+                      action="store_true",
+                      dest="debug",
+                      default=False,
+                      help='Enable debugging output')
+
+
+    (options, args) = parser.parse_args()
+    item = HostConfig()
+    if options.input:
+        infile = open(options.input, 'rb')
+        item.config = pickle.load(infile)
+        infile.close()
+        if not config(options.config, item, crawl=False):
+            sys.exit(1)
+    else:
+        if not config(options.config, item, crawl=True):
+            sys.exit(1)
+
+    p = pickle.dumps(item.config, -1)
+
+    if options.debug:
+        pp = pprint.PrettyPrinter(indent=4)
+        pp.pprint(item.config)
+        
+    if options.output is not None:
+        outfile = open(options.output, 'wb')
+        outfile.write(p)
+        outfile.close()
+
+    if options.stats:
+        statdata = get_stats(conf, 'stats')
+
+    # upload p and statsdata here
+    if not item.config.has_key('global') or not item.config['global'].has_key('enabled') or item.config['global']['enabled'] != '1':
+        sys.exit(1)
+
+    if not options.no_send:
+        #   print "Connecting to %s" % item.config['global']['server']
+        server = xmlrpclib.ServerProxy(item.config['global']['server'])
+        data = None
+        try:
+            data = base64.urlsafe_b64encode(bz2.compress(p))
+        except AttributeError:
+            data = urlsafe_b64encode(bz2.compress(p))
+
+        if data is not None:
+            try:
+                print server.checkin(data)
+            except socket.error, m:
+                print "Error checking in: %s.  Please try again later." % (m[1])
+            except xmlrpclib.Fault:
+                print "Error checking in.  Connection closed before checkin complete.  Please try again later."
+                sys.exit(1)                
+
+
+if __name__ == '__main__':
+    main()
diff --git a/ubuntu-releases-sync b/ubuntu-releases-sync
new file mode 100755 (executable)
index 0000000..078b559
--- /dev/null
@@ -0,0 +1,3 @@
+#!/bin/sh
+# This file is used for ubuntu push mirroring. Do not remove it!
+exec ~/bin/csc-sync-standard ubuntu-releases rsync.releases.ubuntu.com releases