mirror index
authorJeremy Brandon Roman <jbroman@csclub.uwaterloo.ca>
Sun, 7 Mar 2010 17:28:33 +0000 (12:28 -0500)
committerJeremy Brandon Roman <jbroman@csclub.uwaterloo.ca>
Sun, 7 Mar 2010 17:28:33 +0000 (12:28 -0500)
mirror-index/config.yaml [new file with mode: 0644]
mirror-index/index.css [new file with mode: 0644]
mirror-index/index.mako [new file with mode: 0644]
mirror-index/make-index.py [new file with mode: 0755]

diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml
new file mode 100644 (file)
index 0000000..74bb689
--- /dev/null
@@ -0,0 +1,172 @@
+docroot: /mirror/root
+duflags: --human-readable --max-depth=1
+output: /mirror/root/index.html
+
+directories:
+  apache:
+    site: apache.org
+    url: http://www.apache.org/
+
+  archlinux:
+    site: archlinux.org
+    url: http://www.archlinux.org/
+
+  blastwave:
+    site: blastwave.org
+    url: http://www.blastwave.org/
+
+  centos:
+    site: centos.org
+    url: http://www.centos.org/
+
+  CPAN:
+    site: cpan.org
+    url: http://www.cpan.org/
+
+  CRAN:
+    site: r-project.org
+    url: http://cran.r-project.org/
+
+  csclub:
+    site: csclub.uwaterloo.ca
+    url: http://csclub.uwaterloo.ca/media/
+
+  CTAN:
+    site: ctan.org
+    url: http://www.ctan.org/
+
+  cygwin:
+    site: cygwin.com
+    url: http://www.cygwin.com/
+
+  damnsmalllinux:
+    site: damnsmalllinux.org
+    url: http://www.damnsmalllinux.org/
+
+  debian:
+    site: debian.org
+    url: http://www.debian.org/
+
+  debian-backports:
+    site: backports.org
+    url: http://www.backports.org/
+
+  debian-cd:
+    site: debian.org
+    url: http://www.debian.org/CD/
+
+  debian-multimedia:
+    site: debian-multimedia.org
+    url: http://www.debian-multimedia.org/
+
+  debian-ports:
+    site: debian-ports.org
+    url: http://www.debian-ports.org/
+
+  debian-security:
+    site: debian.org
+    url: http://www.debian.org/security/
+
+  debian-unofficial:
+    site: debian-maintainers.org
+    url: http://unofficial.debian-maintainers.org/
+
+  debian-volatile:
+    site: debian.org
+    url: http://www.debian.org/volatile/
+
+  eclipse:
+    site: eclipse.org
+    url: http://www.eclipse.org/
+
+  emdebian:
+    site: emdebian.org
+    url: http://www.emdebian.org/
+
+  FreeBSD:
+    site: freebsd.org
+    url: http://www.freebsd.org/
+
+  gentoo-distfiles:
+    site: gentoo.org
+    url: http://www.gentoo.org/
+
+  gentoo-portage:
+    site: gentoo.org
+    url: http://www.gentoo.org/
+
+  gnome:
+    site: gnome.org
+    url: http://www.gnome.org/
+
+  gnu:
+    site: gnu.org
+    url: http://www.gnu.org/
+
+  kde:
+    site: kde.org
+    url: http://www.kde.org/
+
+  kernel.org:
+    site: kernel.org
+    url: http://www.kernel.org/
+
+  linuxmint:
+    site: linuxmint.com
+    url: http://www.linuxmint.com/
+
+  linuxmint-packages:
+    site: linuxmint.com
+    url: http://www.linuxmint.com/
+
+  mozdev:
+    site: mozdev.org
+    url: http://www.mozdev.org/
+
+  mozilla.org:
+    site: mozilla.org
+    url: http://www.mozilla.org/
+
+  mysql:
+    site: mysql.com
+    url: http://www.mysql.com/
+
+  nongnu:
+    site: nongnu.org
+    url: http://savannah.nongnu.org/
+
+  openoffice:
+    site: openoffice.org
+    url: http://www.openoffice.org/
+
+  opensuse:
+    site: opensuse.org
+    url: http://www.opensuse.org/
+
+  slackware:
+    site: slackware.com
+    url: http://www.slackware.com/
+
+  ubuntu:
+    site: ubuntu.com
+    url: http://www.ubuntu.com/
+
+  ubuntu-ports:
+    site: ports.ubuntu.com
+    url: http://ports.ubuntu.com/ubuntu-ports/
+
+  ubuntu-ports-releases:
+    site: ports.ubuntu.com
+    url: http://cdimage.ubuntu.com/ports/releases/
+
+  ubuntu-releases:
+    site: releases.ubuntu.com
+    url: http://releases.ubuntu.com/
+
+  x.org:
+    site: x.org
+    url: http://www.x.org/
+
+  xubuntu-releases:
+    site: xubuntu.org
+    url: http://www.xubuntu.org/
diff --git a/mirror-index/index.css b/mirror-index/index.css
new file mode 100644 (file)
index 0000000..ec62059
--- /dev/null
@@ -0,0 +1,43 @@
+img {
+ border: none;
+}
+html {
+ margin:0.5ex;
+ font-family: sans-serif;
+ font-size: 110%;
+}
+p {
+ margin: 1ex 0;
+}
+table {
+ border-collapse: collapse;
+ text-align: left;
+ width: 100%;
+}
+td {
+ border-top: 1px solid #aaa;
+}
+th, td {
+ padding: .4ex 2em .4ex 0;
+}
+h1 {
+ font-size: 110%;
+}
+#logo {
+ width: 100%;
+ text-align: center;
+ margin-bottom:1em;
+}
+#footer {
+ margin: 2em auto 0 auto;
+ width: 75%;
+ font-size: 70%;
+ text-align: center;
+}
+body {
+ max-width: 40em;
+ margin-top:0;
+ padding-top:0;
+}
+
+tr :last-child { text-align: right; }
diff --git a/mirror-index/index.mako b/mirror-index/index.mako
new file mode 100644 (file)
index 0000000..88f9dfb
--- /dev/null
@@ -0,0 +1,47 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="stylesheet" type="text/css" href="index.css" />
+    <title>Computer Science Club Mirror</title>
+  </head>
+  
+  <body>
+    <div id="logo">
+      <a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
+    </div>
+    
+    <div id="listing">
+      <table>
+        <tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
+       
+        % for dir in directories:
+        <tr>
+          <td>
+            ${h.link_to(dir['dir']+'/', '/'+dir['dir']+'/')}
+          </td>
+
+          <td>
+           % if 'site' in dir:
+            ${h.link_to(dir['site'], dir['url'])}
+           % endif
+          </td>
+
+          <td>${dir['size'] | h}</td>
+        </tr>
+
+       % endfor \
+       
+       <tr class="total">
+         <td>Total</td>
+         <td></td>
+         <td>${total_size}</td>
+       </tr>
+      </table>
+    </div>
+
+    <div id="footer">
+      <p>This service is run by the <a href="http://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="http://www.student.math.uwaterloo.ca/~mefcom/home">Mathematics Endowment Fund</a><br />and support from the <a href="http://www.cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
+    </div>
+  </body>
+</html>
diff --git a/mirror-index/make-index.py b/mirror-index/make-index.py
new file mode 100755 (executable)
index 0000000..204e61c
--- /dev/null
@@ -0,0 +1,166 @@
+#!/usr/bin/env python
+"""make-index.py
+
+Generates an nice index of the directories from a
+template.
+
+Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
+
+So if you don't like how I did something,
+I'm the person you get to complain to.
+Please be gentle.
+"""
+
+import os, sys, time
+from subprocess import Popen, PIPE
+from optparse import OptionParser
+import yaml, mako.exceptions, webhelpers.html.tags
+from mako.template import Template
+
+def reformat_size(size):
+    """Reformats '124M' to '124 MB', et cetera."""
+    if size[-1].isalpha():
+        return size[:-1] + " " + size[-1] + "B"
+    else:
+        return size
+
+def atomic_write(filename, body):
+    """Atomically write to a file by writing a
+    temporary file and then moving it to replace
+    the desired output file.
+    
+    This ensures that partial files are never seen
+    by clients."""
+    
+    # generate an appropriate temporary filename
+    # in the same directory
+    tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
+    
+    # open the directory so that we can fsync it
+    dir = os.open(os.path.realpath(os.path.dirname(filename)), \
+                      os.O_DIRECTORY | os.O_RDONLY)
+    
+    # write to the temporary file
+    tmp = open(tmp_filename, 'w')
+    print >>tmp, body
+    tmp.flush()
+    os.fsync(tmp.fileno())
+    tmp.close()
+    
+    # atomically replace the actual file
+    os.rename(tmp_filename, filename)
+    os.fsync(dir)
+    os.close(dir)
+
+def main():
+    # accept command-line arguments
+    parser = OptionParser()
+    parser.add_option("-c", "--config", dest="config", default="config.yaml",
+                      help="configuration file to be used", metavar="FILE")
+    parser.add_option("-D", "--docroot", dest="docroot",
+                      help="directory to be scanned", metavar="DIR")
+    parser.add_option("-F", "--duflags", dest="duflags",
+                      help="flags to be passed to du, replaces any in config")
+    parser.add_option("-o", "--output", dest="output", metavar="FILE",
+                      help="file to which index page will be written. "
+                      "Use /dev/stdout to send to standard out.")
+    parser.add_option("-t", "--template", dest="template",
+                      help="Mako template to render", metavar="FILE")
+    parser.add_option("--nonatomic", dest="nonatomic", action="store_true",
+                      default=False, help="write the output to the path "
+                      "given without creating a temporary file in between. "
+                      "This is automatically set if the output appears "
+                      "to be a character device, not a file.")
+    (options, args) = parser.parse_args()
+    
+    # load config file
+    try:
+        config = yaml.load(file(options.config,'r'))
+    except:
+        config = None
+    
+    if not config or type(config) != dict:
+        print >>sys.stderr, "Unable to load configuration '%s'." % options.config
+        sys.exit(-1)
+    
+    # determine important variables based on an appropriate order of
+    # precedence (command-line flags first, then the config file,
+    # then built-in fallbacks)
+    #
+    # fallback value for nonatomic is used so that character devices
+    # (e.g. /dev/stdout, /dev/null) are written to in the regular way
+    docroot   = options.docroot   or config.get('docroot')
+    duflags   = options.duflags   or config.get('duflags')   or "-h --max-depth=1"
+    output    = options.output    or config.get('output')
+    template  = options.template  or config.get("template")  or "index.mako"
+    nonatomic = options.nonatomic or config.get("nonatomic") or \
+        (os.path.exists(output) and not os.path.isfile(output))
+    
+    # sanity checks
+    if not docroot:
+        print >>sys.stderr, "docroot not specified."
+        print >>sys.stderr, "Define it in the config file or pass -D on the command line."
+        sys.exit(-1)
+    elif not output:
+        print >>sys.stderr, "output not specified."
+        print >>sys.stderr, "Define it in the config file or pass -o on the command line."
+    elif not config.get('directories'):
+        print >>sys.stderr, "directories not specified."
+        print >>sys.stderr, "Define it in the config file."
+        sys.exit(-1)
+    elif not os.path.isdir(docroot):
+        print >>sys.stderr, "docroot '%s' not found or not a directory." % docroot
+        sys.exit(-1)
+    elif not os.path.exists(template) or os.path.isdir(template):
+        print >>sys.stderr, "template '%s' not found or is a directory." % template
+        sys.exit(-1)
+    
+    # Call du to compute size
+    du = Popen(
+        "/usr/bin/du %s %s | /usr/bin/sort -fk2" % (docroot, duflags),
+        shell=True, stdout=PIPE, stderr=PIPE).communicate()
+    
+    # Check that du executed successfully
+    # If there's anything on stderr, send it
+    # out our own stderr and terminate.
+    if len(du[1].strip()) > 0:
+        sys.stderr.write(du[1])
+        print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
+        sys.exit(-1)
+    
+    # first one should be total, grab its size and format
+    du = du[0].splitlines() # we only care about stdout now
+    total_size = reformat_size(du[0].split(None,2)[0])
+    
+    # the rest are the sizes we want
+    directories = []
+    for line in du[1:]:
+        (size, dir) = line.split(None, 2)
+        dir = os.path.basename(dir)
+        info = {'dir':dir, 'size':reformat_size(size)}
+        
+        # use info from config.yaml, if found
+        # otherwise, skip this directory
+        if dir in config['directories']:
+            info.update(config['directories'][dir])
+        else:
+            continue
+        
+        directories.append(info)
+    
+    # render the template to a string
+    body = Template(filename=template).render(
+        total_size=total_size,
+        directories=directories,
+        config=config,
+        h=webhelpers.html.tags)
+    
+    # write the rendered output
+    if nonatomic:
+        print >>file(output,'w'), body
+    else:
+        atomic_write(output, body)
+
+
+if __name__ == "__main__":
+    main()