mirror index
This commit is contained in:
parent
a2257fb2fa
commit
b03ff809e7
|
|
@ -0,0 +1,172 @@
|
|||
docroot: /mirror/root
|
||||
duflags: --human-readable --max-depth=1
|
||||
output: /mirror/root/index.html
|
||||
|
||||
directories:
|
||||
apache:
|
||||
site: apache.org
|
||||
url: http://www.apache.org/
|
||||
|
||||
archlinux:
|
||||
site: archlinux.org
|
||||
url: http://www.archlinux.org/
|
||||
|
||||
blastwave:
|
||||
site: blastwave.org
|
||||
url: http://www.blastwave.org/
|
||||
|
||||
centos:
|
||||
site: centos.org
|
||||
url: http://www.centos.org/
|
||||
|
||||
CPAN:
|
||||
site: cpan.org
|
||||
url: http://www.cpan.org/
|
||||
|
||||
CRAN:
|
||||
site: r-project.org
|
||||
url: http://cran.r-project.org/
|
||||
|
||||
csclub:
|
||||
site: csclub.uwaterloo.ca
|
||||
url: http://csclub.uwaterloo.ca/media/
|
||||
|
||||
CTAN:
|
||||
site: ctan.org
|
||||
url: http://www.ctan.org/
|
||||
|
||||
cygwin:
|
||||
site: cygwin.com
|
||||
url: http://www.cygwin.com/
|
||||
|
||||
damnsmalllinux:
|
||||
site: damnsmalllinux.org
|
||||
url: http://www.damnsmalllinux.org/
|
||||
|
||||
debian:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/
|
||||
|
||||
debian-backports:
|
||||
site: backports.org
|
||||
url: http://www.backports.org/
|
||||
|
||||
debian-cd:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/CD/
|
||||
|
||||
debian-multimedia:
|
||||
site: debian-multimedia.org
|
||||
url: http://www.debian-multimedia.org/
|
||||
|
||||
debian-ports:
|
||||
site: debian-ports.org
|
||||
url: http://www.debian-ports.org/
|
||||
|
||||
debian-security:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/security/
|
||||
|
||||
debian-unofficial:
|
||||
site: debian-maintainers.org
|
||||
url: http://unofficial.debian-maintainers.org/
|
||||
|
||||
debian-volatile:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/volatile/
|
||||
|
||||
eclipse:
|
||||
site: eclipse.org
|
||||
url: http://www.eclipse.org/
|
||||
|
||||
emdebian:
|
||||
site: emdebian.org
|
||||
url: http://www.emdebian.org/
|
||||
|
||||
FreeBSD:
|
||||
site: freebsd.org
|
||||
url: http://www.freebsd.org/
|
||||
|
||||
gentoo-distfiles:
|
||||
site: gentoo.org
|
||||
url: http://www.gentoo.org/
|
||||
|
||||
gentoo-portage:
|
||||
site: gentoo.org
|
||||
url: http://www.gentoo.org/
|
||||
|
||||
gnome:
|
||||
site: gnome.org
|
||||
url: http://www.gnome.org/
|
||||
|
||||
gnu:
|
||||
site: gnu.org
|
||||
url: http://www.gnu.org/
|
||||
|
||||
kde:
|
||||
site: kde.org
|
||||
url: http://www.kde.org/
|
||||
|
||||
kernel.org:
|
||||
site: kernel.org
|
||||
url: http://www.kernel.org/
|
||||
|
||||
linuxmint:
|
||||
site: linuxmint.com
|
||||
url: http://www.linuxmint.com/
|
||||
|
||||
linuxmint-packages:
|
||||
site: linuxmint.com
|
||||
url: http://www.linuxmint.com/
|
||||
|
||||
mozdev:
|
||||
site: mozdev.org
|
||||
url: http://www.mozdev.org/
|
||||
|
||||
mozilla.org:
|
||||
site: mozilla.org
|
||||
url: http://www.mozilla.org/
|
||||
|
||||
mysql:
|
||||
site: mysql.com
|
||||
url: http://www.mysql.com/
|
||||
|
||||
nongnu:
|
||||
site: nongnu.org
|
||||
url: http://savannah.nongnu.org/
|
||||
|
||||
openoffice:
|
||||
site: openoffice.org
|
||||
url: http://www.openoffice.org/
|
||||
|
||||
opensuse:
|
||||
site: opensuse.org
|
||||
url: http://www.opensuse.org/
|
||||
|
||||
slackware:
|
||||
site: slackware.com
|
||||
url: http://www.slackware.com/
|
||||
|
||||
ubuntu:
|
||||
site: ubuntu.com
|
||||
url: http://www.ubuntu.com/
|
||||
|
||||
ubuntu-ports:
|
||||
site: ports.ubuntu.com
|
||||
url: http://ports.ubuntu.com/ubuntu-ports/
|
||||
|
||||
ubuntu-ports-releases:
|
||||
site: ports.ubuntu.com
|
||||
url: http://cdimage.ubuntu.com/ports/releases/
|
||||
|
||||
ubuntu-releases:
|
||||
site: releases.ubuntu.com
|
||||
url: http://releases.ubuntu.com/
|
||||
|
||||
x.org:
|
||||
site: x.org
|
||||
url: http://www.x.org/
|
||||
|
||||
xubuntu-releases:
|
||||
site: xubuntu.org
|
||||
url: http://www.xubuntu.org/
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
img {
|
||||
border: none;
|
||||
}
|
||||
html {
|
||||
margin:0.5ex;
|
||||
font-family: sans-serif;
|
||||
font-size: 110%;
|
||||
}
|
||||
p {
|
||||
margin: 1ex 0;
|
||||
}
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
text-align: left;
|
||||
width: 100%;
|
||||
}
|
||||
td {
|
||||
border-top: 1px solid #aaa;
|
||||
}
|
||||
th, td {
|
||||
padding: .4ex 2em .4ex 0;
|
||||
}
|
||||
h1 {
|
||||
font-size: 110%;
|
||||
}
|
||||
#logo {
|
||||
width: 100%;
|
||||
text-align: center;
|
||||
margin-bottom:1em;
|
||||
}
|
||||
#footer {
|
||||
margin: 2em auto 0 auto;
|
||||
width: 75%;
|
||||
font-size: 70%;
|
||||
text-align: center;
|
||||
}
|
||||
body {
|
||||
max-width: 40em;
|
||||
margin-top:0;
|
||||
padding-top:0;
|
||||
}
|
||||
|
||||
tr :last-child { text-align: right; }
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<link rel="stylesheet" type="text/css" href="index.css" />
|
||||
<title>Computer Science Club Mirror</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div id="logo">
|
||||
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
|
||||
</div>
|
||||
|
||||
<div id="listing">
|
||||
<table>
|
||||
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
|
||||
|
||||
% for dir in directories:
|
||||
<tr>
|
||||
<td>
|
||||
${h.link_to(dir['dir']+'/', '/'+dir['dir']+'/')}
|
||||
</td>
|
||||
|
||||
<td>
|
||||
% if 'site' in dir:
|
||||
${h.link_to(dir['site'], dir['url'])}
|
||||
% endif
|
||||
</td>
|
||||
|
||||
<td>${dir['size'] | h}</td>
|
||||
</tr>
|
||||
|
||||
% endfor \
|
||||
|
||||
<tr class="total">
|
||||
<td>Total</td>
|
||||
<td></td>
|
||||
<td>${total_size}</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<div id="footer">
|
||||
<p>This service is run by the <a href="http://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="http://www.student.math.uwaterloo.ca/~mefcom/home">Mathematics Endowment Fund</a><br />and support from the <a href="http://www.cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -0,0 +1,166 @@
|
|||
#!/usr/bin/env python
|
||||
"""make-index.py
|
||||
|
||||
Generates an nice index of the directories from a
|
||||
template.
|
||||
|
||||
Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
|
||||
|
||||
So if you don't like how I did something,
|
||||
I'm the person you get to complain to.
|
||||
Please be gentle.
|
||||
"""
|
||||
|
||||
import os, sys, time
|
||||
from subprocess import Popen, PIPE
|
||||
from optparse import OptionParser
|
||||
import yaml, mako.exceptions, webhelpers.html.tags
|
||||
from mako.template import Template
|
||||
|
||||
def reformat_size(size):
|
||||
"""Reformats '124M' to '124 MB', et cetera."""
|
||||
if size[-1].isalpha():
|
||||
return size[:-1] + " " + size[-1] + "B"
|
||||
else:
|
||||
return size
|
||||
|
||||
def atomic_write(filename, body):
|
||||
"""Atomically write to a file by writing a
|
||||
temporary file and then moving it to replace
|
||||
the desired output file.
|
||||
|
||||
This ensures that partial files are never seen
|
||||
by clients."""
|
||||
|
||||
# generate an appropriate temporary filename
|
||||
# in the same directory
|
||||
tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
|
||||
|
||||
# open the directory so that we can fsync it
|
||||
dir = os.open(os.path.realpath(os.path.dirname(filename)), \
|
||||
os.O_DIRECTORY | os.O_RDONLY)
|
||||
|
||||
# write to the temporary file
|
||||
tmp = open(tmp_filename, 'w')
|
||||
print >>tmp, body
|
||||
tmp.flush()
|
||||
os.fsync(tmp.fileno())
|
||||
tmp.close()
|
||||
|
||||
# atomically replace the actual file
|
||||
os.rename(tmp_filename, filename)
|
||||
os.fsync(dir)
|
||||
os.close(dir)
|
||||
|
||||
def main():
|
||||
# accept command-line arguments
|
||||
parser = OptionParser()
|
||||
parser.add_option("-c", "--config", dest="config", default="config.yaml",
|
||||
help="configuration file to be used", metavar="FILE")
|
||||
parser.add_option("-D", "--docroot", dest="docroot",
|
||||
help="directory to be scanned", metavar="DIR")
|
||||
parser.add_option("-F", "--duflags", dest="duflags",
|
||||
help="flags to be passed to du, replaces any in config")
|
||||
parser.add_option("-o", "--output", dest="output", metavar="FILE",
|
||||
help="file to which index page will be written. "
|
||||
"Use /dev/stdout to send to standard out.")
|
||||
parser.add_option("-t", "--template", dest="template",
|
||||
help="Mako template to render", metavar="FILE")
|
||||
parser.add_option("--nonatomic", dest="nonatomic", action="store_true",
|
||||
default=False, help="write the output to the path "
|
||||
"given without creating a temporary file in between. "
|
||||
"This is automatically set if the output appears "
|
||||
"to be a character device, not a file.")
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
# load config file
|
||||
try:
|
||||
config = yaml.load(file(options.config,'r'))
|
||||
except:
|
||||
config = None
|
||||
|
||||
if not config or type(config) != dict:
|
||||
print >>sys.stderr, "Unable to load configuration '%s'." % options.config
|
||||
sys.exit(-1)
|
||||
|
||||
# determine important variables based on an appropriate order of
|
||||
# precedence (command-line flags first, then the config file,
|
||||
# then built-in fallbacks)
|
||||
#
|
||||
# fallback value for nonatomic is used so that character devices
|
||||
# (e.g. /dev/stdout, /dev/null) are written to in the regular way
|
||||
docroot = options.docroot or config.get('docroot')
|
||||
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1"
|
||||
output = options.output or config.get('output')
|
||||
template = options.template or config.get("template") or "index.mako"
|
||||
nonatomic = options.nonatomic or config.get("nonatomic") or \
|
||||
(os.path.exists(output) and not os.path.isfile(output))
|
||||
|
||||
# sanity checks
|
||||
if not docroot:
|
||||
print >>sys.stderr, "docroot not specified."
|
||||
print >>sys.stderr, "Define it in the config file or pass -D on the command line."
|
||||
sys.exit(-1)
|
||||
elif not output:
|
||||
print >>sys.stderr, "output not specified."
|
||||
print >>sys.stderr, "Define it in the config file or pass -o on the command line."
|
||||
elif not config.get('directories'):
|
||||
print >>sys.stderr, "directories not specified."
|
||||
print >>sys.stderr, "Define it in the config file."
|
||||
sys.exit(-1)
|
||||
elif not os.path.isdir(docroot):
|
||||
print >>sys.stderr, "docroot '%s' not found or not a directory." % docroot
|
||||
sys.exit(-1)
|
||||
elif not os.path.exists(template) or os.path.isdir(template):
|
||||
print >>sys.stderr, "template '%s' not found or is a directory." % template
|
||||
sys.exit(-1)
|
||||
|
||||
# Call du to compute size
|
||||
du = Popen(
|
||||
"/usr/bin/du %s %s | /usr/bin/sort -fk2" % (docroot, duflags),
|
||||
shell=True, stdout=PIPE, stderr=PIPE).communicate()
|
||||
|
||||
# Check that du executed successfully
|
||||
# If there's anything on stderr, send it
|
||||
# out our own stderr and terminate.
|
||||
if len(du[1].strip()) > 0:
|
||||
sys.stderr.write(du[1])
|
||||
print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
|
||||
sys.exit(-1)
|
||||
|
||||
# first one should be total, grab its size and format
|
||||
du = du[0].splitlines() # we only care about stdout now
|
||||
total_size = reformat_size(du[0].split(None,2)[0])
|
||||
|
||||
# the rest are the sizes we want
|
||||
directories = []
|
||||
for line in du[1:]:
|
||||
(size, dir) = line.split(None, 2)
|
||||
dir = os.path.basename(dir)
|
||||
info = {'dir':dir, 'size':reformat_size(size)}
|
||||
|
||||
# use info from config.yaml, if found
|
||||
# otherwise, skip this directory
|
||||
if dir in config['directories']:
|
||||
info.update(config['directories'][dir])
|
||||
else:
|
||||
continue
|
||||
|
||||
directories.append(info)
|
||||
|
||||
# render the template to a string
|
||||
body = Template(filename=template).render(
|
||||
total_size=total_size,
|
||||
directories=directories,
|
||||
config=config,
|
||||
h=webhelpers.html.tags)
|
||||
|
||||
# write the rendered output
|
||||
if nonatomic:
|
||||
print >>file(output,'w'), body
|
||||
else:
|
||||
atomic_write(output, body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue