fix MEF URL
[public/mirror.git] / bin / report_mirror
1 #!/usr/bin/python
2
3 import os, sys
4 from os.path import join
5 import pickle
6 import re
7 import ConfigParser
8 import pprint
9 import time
10 import xmlrpclib
11 import base64
12 import bz2
13 import socket
14
15 class HostConfig(object):
16     """Holder for config info from the configuration file"""
17     def __init__(self):
18         self.config = { 'version' : 0,
19                         'global': {},
20                         'site': {},
21                         'host': {},
22                         'stats': {},
23                         }
24
25 _translation = [chr(_x) for _x in range(256)]
26 def _translate(s, altchars):
27     translation = _translation[:]
28     for k, v in altchars.items():
29         translation[ord(k)] = v
30     return s.translate(''.join(translation))
31
32 def urlsafe_b64encode(s):
33    import binascii
34    altchars = '-_'
35    encoded = binascii.b2a_base64(s)[:-1]
36    if altchars is not None:
37       return _translate(encoded, {'+': altchars[0], '/': altchars[1]})
38    return encoded
39
40 def gen_dirtree(path):
41     # structure here is:
42     # dirtree is a dict
43     # {
44     #   dirpath :
45     #                {
46     #                   filename1 : size1,
47     #                   filename2 : size2,
48     #                   ...
49     #                 },
50     #   ...
51     # }
52     # 
53     # 2009-03-09: MM's web app ignores the statfiles dict.  So don't bother generating it.
54
55     dirtree = {}
56     for dirpath, dirnames, filenames in os.walk(path):
57         statfiles = {}
58         if path.endswith('/'):
59             short_path = dirpath[len(path):]
60         else:
61             short_path = dirpath[len(path)+1:]
62         if len(short_path) > 0:
63             dirtree[short_path] = statfiles
64         else:
65             dirtree[''] = statfiles
66
67     return dirtree
68
69 def errorprint(error):
70     sys.stderr.write(error+'\n')
71
72 class MissingOption(Exception):
73     pass
74
75 def check_required_options(conf, section, required_options):
76     for o in required_options:
77         if not conf.has_option(section, o):
78             errorprint('missing required option %s in config [%s]' % (o, section))
79             raise MissingOption()
80     return True
81
82 def parse_value(value):
83     """Split multi-line values into a list"""
84     if value.find('\n') > -1:
85         return value.split()
86     return value
87
88 def parse_section(conf, section, item, required_options, optional_options=[]):
89     if conf.has_option(section, 'enabled'):
90         if conf.get(section, 'enabled') != '1' and section.lower() in item.config:
91             print 'removing disabled section %s' % (section)
92             del item.config[section.lower()]
93             return False
94
95     if not check_required_options(conf, section, required_options):
96         return False
97
98     if not section.lower() in item.config:
99         item.config[section.lower()] = {}
100
101     for o in required_options:
102         item.config[section.lower()][o] = parse_value(conf.get(section, o))
103     for o in optional_options:
104         if conf.has_option(section, o):
105             item.config[section.lower()][o] = parse_value(conf.get(section, o))
106
107     return True
108
109 def parse_global(conf, section, item):
110     required_options = [ 'enabled', 'server' ]
111     if not parse_section(conf, section, item, required_options):
112         errorprint('missing required options (server AND enabled) in [%s] section' % (section))
113         return False
114     return True
115
116 def parse_site(conf, section, item):
117     required_options = [ 'enabled', 'name', 'password' ]
118     return parse_section(conf, section, item, required_options)
119
120 def parse_host(conf, section, item):
121     required_options = [ 'enabled', 'name' ]
122     optional_options = [ 'user_active' ]
123     return parse_section(conf, section, item, required_options, optional_options=optional_options)
124
125 def get_stats(conf, section):
126     if conf.has_option(section, 'enabled'):
127         if conf.get(section, 'enabled') != '1':
128             return None
129     statsdata = {}
130     for name, value in conf.items(section):
131         if name == 'enabled':
132             continue
133         filenames = parse_value(conf.get(section, name))
134         if type(filenames) != list:
135             filenames = [ filenames ]
136         for fn in filenames:
137             try:
138                 f = open(fn, 'r')
139                 contents = contents + f.readlines()
140                 statsdata[name] = pickle.dumps(contents, -1)
141                 f.close()
142             except:
143                 pass
144     return statsdata
145
146 def parse_category(conf, section, item, crawl):
147     required_options = [ 'enabled', 'path' ]
148     if not parse_section(conf, section, item, required_options):
149         return False
150
151     if crawl:
152         dirtree = gen_dirtree(conf.get(section, 'path'))
153         item.config[section.lower()]['dirtree'] = dirtree
154     # database doesn't need to know the disk path
155     del item.config[section.lower()]['path']
156
157 def config(cfg, item, crawl=True):
158     broken = False
159     conf = ConfigParser.ConfigParser()
160     files = conf.read(cfg)
161     if files == []:
162         errorprint('Configuration file %s not found' % (cfg))
163         return False
164     conf.read(cfg)
165
166     try:
167         # don't grab parse_stats here
168         for section, parsefunc in [ ('global', parse_global), ('site', parse_site),
169                                     ('host', parse_host)]:
170             if conf.has_section(section):
171                 if not parsefunc(conf, section, item):
172                     return False
173             else:
174                 errorprint('Invalid configuration - missing section [%s]' % (section))
175                 sys.exit(1)
176
177         for section in conf.sections():
178             if section in [ 'global', 'site', 'host', 'stats']:
179                 continue
180             parse_category(conf, section, item, crawl)
181
182     except MissingOption:
183         errorprint('Invalid configuration - Exiting')
184         sys.exit(1)
185
186     return True
187
188
189
190 def main():
191     from optparse import OptionParser
192     parser = OptionParser(usage= sys.argv[0] + " [options]")
193     parser.add_option("-c", "--config",
194                       dest="config",
195                       default='/etc/mirrormanager-client/report_mirror.conf',
196                       help='Configuration filename (required)')
197     parser.add_option("-s", "--stats",
198                       action="store_true",
199                       dest="stats",
200                       default=False,
201                       help='Send stats')
202     parser.add_option("-i", "--input",
203                       dest="input",
204                       default=None,
205                       help="Input filename (for debugging)")
206     parser.add_option("-o", "--output",
207                       dest="output",
208                       default=None,
209                       help="Output filename (for debugging)")
210     parser.add_option("-n", "--no-send",
211                       action="store_true",
212                       dest="no_send",
213                       default=False,
214                       help="Don't send data to the server.")
215     parser.add_option("-d", "--debug",
216                       action="store_true",
217                       dest="debug",
218                       default=False,
219                       help='Enable debugging output')
220
221
222     (options, args) = parser.parse_args()
223     item = HostConfig()
224     if options.input:
225         infile = open(options.input, 'rb')
226         item.config = pickle.load(infile)
227         infile.close()
228         if not config(options.config, item, crawl=False):
229             sys.exit(1)
230     else:
231         if not config(options.config, item, crawl=True):
232             sys.exit(1)
233
234     p = pickle.dumps(item.config, -1)
235
236     if options.debug:
237         pp = pprint.PrettyPrinter(indent=4)
238         pp.pprint(item.config)
239         
240     if options.output is not None:
241         outfile = open(options.output, 'wb')
242         outfile.write(p)
243         outfile.close()
244
245     if options.stats:
246         statdata = get_stats(conf, 'stats')
247
248     # upload p and statsdata here
249     if not item.config.has_key('global') or not item.config['global'].has_key('enabled') or item.config['global']['enabled'] != '1':
250         sys.exit(1)
251
252     if not options.no_send:
253         #   print "Connecting to %s" % item.config['global']['server']
254         server = xmlrpclib.ServerProxy(item.config['global']['server'])
255         data = None
256         try:
257             data = base64.urlsafe_b64encode(bz2.compress(p))
258         except AttributeError:
259             data = urlsafe_b64encode(bz2.compress(p))
260
261         if data is not None:
262             try:
263                 print server.checkin(data)
264             except socket.error, m:
265                 print "Error checking in: %s.  Please try again later." % (m[1])
266             except xmlrpclib.Fault:
267                 print "Error checking in.  Connection closed before checkin complete.  Please try again later."
268                 sys.exit(1)                
269
270
271 if __name__ == '__main__':
272     main()