parent
d5b90baecf
commit
07153e8ecc
@ -0,0 +1 @@ |
||||
- name: create |
||||
|
Before Width: | Height: | Size: 1.1 KiB After Width: | Height: | Size: 1.1 KiB |
|
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 14 KiB |
@ -1,299 +0,0 @@ |
||||
docroot: /mirror/root |
||||
duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub |
||||
output: /mirror/root/index.html |
||||
|
||||
exclude: |
||||
- include |
||||
- lost+found |
||||
- pub |
||||
- uw-courseware |
||||
- wics |
||||
- mirror-rootfs |
||||
- mirror-highpriority |
||||
- ups |
||||
|
||||
directories: |
||||
alpine: |
||||
site: alpinelinux.org |
||||
url: https://alpinelinux.org/ |
||||
|
||||
almalinux: |
||||
site: almalinux.org |
||||
url: https://almalinux.org |
||||
|
||||
apache: |
||||
site: apache.org |
||||
url: https://www.apache.org/ |
||||
|
||||
archlinux: |
||||
site: archlinux.org |
||||
url: https://www.archlinux.org/ |
||||
|
||||
centos: |
||||
site: centos.org |
||||
url: https://www.centos.org/ |
||||
|
||||
ceph: |
||||
site: ceph.io |
||||
url: https://ceph.io |
||||
|
||||
CPAN: |
||||
site: cpan.org |
||||
url: https://www.cpan.org/ |
||||
|
||||
CRAN: |
||||
site: r-project.org |
||||
url: https://cran.r-project.org/ |
||||
|
||||
csclub: |
||||
site: csclub.uwaterloo.ca |
||||
url: https://csclub.uwaterloo.ca/media/ |
||||
|
||||
CTAN: |
||||
site: ctan.org |
||||
url: https://www.ctan.org/ |
||||
|
||||
cygwin: |
||||
site: cygwin.com |
||||
url: https://www.cygwin.com/ |
||||
|
||||
damnsmalllinux: |
||||
site: damnsmalllinux.org |
||||
url: https://www.damnsmalllinux.org/ |
||||
|
||||
debian: |
||||
site: debian.org |
||||
url: https://www.debian.org/ |
||||
|
||||
debian-backports: |
||||
site: backports.debian.org |
||||
url: https://backports.debian.org/ |
||||
|
||||
debian-cd: |
||||
site: debian.org |
||||
url: https://www.debian.org/CD/ |
||||
|
||||
debian-multimedia: |
||||
site: debian-multimedia.org |
||||
url: https://www.debian-multimedia.org/ |
||||
|
||||
debian-ports: |
||||
site: debian-ports.org |
||||
url: https://www.debian-ports.org/ |
||||
|
||||
debian-security: |
||||
site: debian.org |
||||
url: https://www.debian.org/security/ |
||||
|
||||
debian-volatile: |
||||
site: debian.org |
||||
url: https://www.debian.org/volatile/ |
||||
|
||||
eclipse: |
||||
site: eclipse.org |
||||
url: https://www.eclipse.org/ |
||||
|
||||
emacsconf: |
||||
site: emacsconf.org |
||||
url: https://emacsconf.org |
||||
|
||||
# this is no longer a project |
||||
# emdebian: |
||||
# site: emdebian.org |
||||
# url: https://www.emdebian.org/ |
||||
|
||||
fedora: |
||||
site: fedoraproject.org |
||||
url: https://www.fedoraproject.org/ |
||||
|
||||
FreeBSD: |
||||
site: freebsd.org |
||||
url: https://www.freebsd.org/ |
||||
|
||||
gentoo-distfiles: |
||||
site: gentoo.org |
||||
url: https://www.gentoo.org/ |
||||
|
||||
gentoo-portage: |
||||
site: gentoo.org |
||||
url: https://www.gentoo.org/ |
||||
|
||||
gnome: |
||||
site: gnome.org |
||||
url: https://www.gnome.org/ |
||||
|
||||
gnu: |
||||
site: gnu.org |
||||
url: https://www.gnu.org/ |
||||
|
||||
gutenberg: |
||||
site: gutenberg.org |
||||
url: https://www.gutenberg.org/ |
||||
|
||||
kali: |
||||
site: kali.org |
||||
url: https://www.kali.org/ |
||||
|
||||
kali-images: |
||||
site: kali.org |
||||
url: https://www.kali.org/ |
||||
|
||||
kde: |
||||
site: kde.org |
||||
url: https://www.kde.org/ |
||||
|
||||
kde-applicationdata: |
||||
site: kde.org |
||||
url: https://www.kde.org/ |
||||
|
||||
kernel.org: |
||||
site: kernel.org |
||||
url: https://www.kernel.org/ |
||||
|
||||
linuxmint: |
||||
site: linuxmint.com |
||||
url: https://www.linuxmint.com/ |
||||
|
||||
linuxmint-packages: |
||||
site: linuxmint.com |
||||
url: https://www.linuxmint.com/ |
||||
|
||||
MacPorts: |
||||
site: macports.org |
||||
url: https://www.macports.org/ |
||||
|
||||
mozdev: |
||||
site: mozdev.org |
||||
url: https://www.mozdev.org/ |
||||
|
||||
mozilla.org: |
||||
site: mozilla.org |
||||
url: https://www.mozilla.org/ |
||||
|
||||
mysql: |
||||
site: mysql.com |
||||
url: https://www.mysql.com/ |
||||
|
||||
nongnu: |
||||
site: nongnu.org |
||||
url: https://savannah.nongnu.org/ |
||||
|
||||
openoffice: |
||||
site: openoffice.org |
||||
url: https://www.openoffice.org/ |
||||
|
||||
opensuse: |
||||
site: opensuse.org |
||||
url: https://www.opensuse.org/ |
||||
|
||||
OpenBSD: |
||||
site: openbsd.org |
||||
url: https://www.openbsd.org/ |
||||
|
||||
puppylinux: |
||||
site: puppylinux.org |
||||
url: https://puppylinux.org/ |
||||
|
||||
qtproject: |
||||
site: qt-project.org |
||||
url: https://qt-project.org/ |
||||
|
||||
racket: |
||||
site: racket-lang.org |
||||
url: https://racket-lang.org/ |
||||
|
||||
sage: |
||||
site: sagemath.org |
||||
url: https://www.sagemath.org/ |
||||
|
||||
saltstack: |
||||
site: saltstack.com |
||||
url: https://saltstack.com/ |
||||
|
||||
slackware: |
||||
site: slackware.com |
||||
url: https://www.slackware.com/ |
||||
|
||||
tdf: |
||||
site: documentfoundation.org |
||||
url: https://www.documentfoundation.org/ |
||||
|
||||
ubuntu: |
||||
site: ubuntu.com |
||||
url: https://www.ubuntu.com/ |
||||
|
||||
ubuntu-ports: |
||||
site: ports.ubuntu.com |
||||
url: http://ports.ubuntu.com/ubuntu-ports/ |
||||
|
||||
ubuntu-ports-releases: |
||||
site: ports.ubuntu.com |
||||
url: https://cdimage.ubuntu.com/releases/ |
||||
|
||||
ubuntu-releases: |
||||
site: releases.ubuntu.com |
||||
url: https://releases.ubuntu.com/ |
||||
|
||||
vlc: |
||||
site: videolan.org |
||||
url: https://videolan.org/ |
||||
|
||||
xiph: |
||||
site: xiph.org |
||||
url: https://xiph.org/ |
||||
|
||||
x.org: |
||||
site: x.org |
||||
url: https://www.x.org/ |
||||
|
||||
xubuntu-releases: |
||||
site: xubuntu.org |
||||
url: https://www.xubuntu.org/ |
||||
|
||||
NetBSD: |
||||
site: netbsd.org |
||||
url: https://www.netbsd.org/ |
||||
|
||||
pkgsrc: |
||||
site: pkgsrc.org |
||||
url: https://pkgsrc.org/ |
||||
|
||||
raspberrypi: |
||||
site: raspberrypi.org |
||||
url: https://www.raspberrypi.org/ |
||||
|
||||
raspbian: |
||||
site: raspbian.org |
||||
url: https://www.raspbian.org/ |
||||
|
||||
ipfire: |
||||
site: ipfire.org |
||||
url: https://www.ipfire.org/ |
||||
|
||||
manjaro: |
||||
site: manjaro.org |
||||
url: https://manjaro.org/ |
||||
|
||||
scientific: |
||||
site: scientificlinux.org |
||||
url: https://www.scientificlinux.org/ |
||||
|
||||
mxlinux: |
||||
site: mxlinux.org |
||||
url: https://mxlinux.org |
||||
|
||||
mxlinux-iso: |
||||
site: mxlinux.org |
||||
url: https://mxlinux.org |
||||
|
||||
parabola: |
||||
site: parabola.nu |
||||
url: https://www.parabola.nu/ |
||||
|
||||
hyperbola: |
||||
site: hyperbola.info |
||||
url: https://www.hyperbola.info/ |
||||
|
||||
trisquel: |
||||
site: trisquel.info |
||||
url: https://trisquel.info/ |
||||
@ -1,43 +0,0 @@ |
||||
img { |
||||
border: none; |
||||
} |
||||
html { |
||||
margin:0.5ex; |
||||
font-family: sans-serif; |
||||
font-size: 110%; |
||||
} |
||||
p { |
||||
margin: 1ex 0; |
||||
} |
||||
table { |
||||
border-collapse: collapse; |
||||
text-align: left; |
||||
width: 100%; |
||||
} |
||||
td { |
||||
border-top: 1px solid #aaa; |
||||
} |
||||
th, td { |
||||
padding: .4ex 2em .4ex 0; |
||||
} |
||||
h1 { |
||||
font-size: 110%; |
||||
} |
||||
#logo { |
||||
width: 100%; |
||||
text-align: center; |
||||
margin-bottom:1em; |
||||
} |
||||
#footer { |
||||
margin: 2em auto 0 auto; |
||||
width: 75%; |
||||
font-size: 70%; |
||||
text-align: center; |
||||
} |
||||
body { |
||||
max-width: 40em; |
||||
margin-top:0; |
||||
padding-top:0; |
||||
} |
||||
|
||||
tr :last-child { text-align: right; } |
||||
@ -1,49 +0,0 @@ |
||||
<!DOCTYPE html> |
||||
<html lang="en"> |
||||
<head> |
||||
<meta charset="UTF-8" /> |
||||
<link rel="stylesheet" type="text/css" href="index.css" /> |
||||
<title>Computer Science Club Mirror</title> |
||||
</head> |
||||
|
||||
<body> |
||||
<div id="logo"> |
||||
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a> |
||||
</div> |
||||
|
||||
<div id="listing"> |
||||
<table> |
||||
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr> |
||||
|
||||
% for dir in directories: |
||||
<tr> |
||||
<td> |
||||
${h.link_to(dir['dir']+'/', '/'+dir['dir']+'/')} |
||||
</td> |
||||
|
||||
<td> |
||||
% if 'site' in dir: |
||||
${h.link_to(dir['site'], dir['url'])} |
||||
% endif |
||||
</td> |
||||
|
||||
<td>${dir['size'] | h}</td> |
||||
</tr> |
||||
|
||||
% endfor \ |
||||
|
||||
<tr class="total"> |
||||
<td>Total</td> |
||||
<td></td> |
||||
<td>${total_size}</td> |
||||
</tr> |
||||
</table> |
||||
</div> |
||||
|
||||
<div id="footer"> |
||||
<p>This service is run by the <a href="https://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="https://uwaterloo.ca/math-endowment-fund/">Mathematics Endowment Fund</a><br />and support from the <a href="https://cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p> |
||||
<p>Report an issue or request we mirror a project by contacting <a href="mailto:systems-committee@csclub.uwaterloo.ca">systems-committee@csclub.uwaterloo.ca</a>.</p> |
||||
<p>Generated (local time): ${generated.strftime("%Y-%m-%d %-I:%M%p")}</p> |
||||
</div> |
||||
</body> |
||||
</html> |
||||
@ -1,231 +0,0 @@ |
||||
#!/usr/bin/env python2 |
||||
"""make-index.py |
||||
|
||||
Generates an nice index of the directories from a |
||||
template. |
||||
|
||||
Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca> |
||||
Modified by: Zachary Seguin <ztseguin@csclub.uwaterloo.ca> |
||||
|
||||
So if you don't like how I did something, |
||||
I'm the person you get to complain to. |
||||
Please be gentle. |
||||
""" |
||||
|
||||
import os, sys, time |
||||
from subprocess import Popen, PIPE |
||||
from optparse import OptionParser |
||||
import yaml, mako.exceptions, webhelpers.html.tags |
||||
from mako.template import Template |
||||
import humanize |
||||
import datetime |
||||
|
||||
def parse_size(s): |
||||
if s.endswith("T"): |
||||
return float(s[:-1])*1000*1000*1000*1000 |
||||
elif s.endswith("G"): |
||||
return float(s[:-1])*1000*1000*1000 |
||||
elif s.endswith("M"): |
||||
return float(s[:-1])*1000*1000 |
||||
elif s.endswith("K"): |
||||
return float(s[:-1])*1000 |
||||
|
||||
return float(size) |
||||
|
||||
def reformat_size(size): |
||||
return humanize.naturalsize(size) |
||||
|
||||
def atomic_write(filename, body): |
||||
"""Atomically write to a file by writing a |
||||
temporary file and then moving it to replace |
||||
the desired output file. |
||||
|
||||
This ensures that partial files are never seen |
||||
by clients.""" |
||||
|
||||
# generate an appropriate temporary filename |
||||
# in the same directory |
||||
tmp_filename = "%s.%d.tmp" % (filename, os.getpid()) |
||||
|
||||
# open the directory so that we can fsync it |
||||
dir = os.open(os.path.realpath(os.path.dirname(filename)), \ |
||||
os.O_DIRECTORY | os.O_RDONLY) |
||||
|
||||
# write to the temporary file |
||||
tmp = open(tmp_filename, 'w') |
||||
print >>tmp, body |
||||
tmp.flush() |
||||
os.fsync(tmp.fileno()) |
||||
tmp.close() |
||||
|
||||
# atomically replace the actual file |
||||
os.rename(tmp_filename, filename) |
||||
os.fsync(dir) |
||||
os.close(dir) |
||||
|
||||
def main(): |
||||
# accept command-line arguments |
||||
parser = OptionParser() |
||||
parser.add_option("-c", "--config", dest="config", default="config.yaml", |
||||
help="configuration file to be used", metavar="FILE") |
||||
parser.add_option("-D", "--docroot", dest="docroot", |
||||
help="directory to be scanned", metavar="DIR") |
||||
parser.add_option("-F", "--duflags", dest="duflags", |
||||
help="flags to be passed to du, replaces any in config") |
||||
parser.add_option("-o", "--output", dest="output", metavar="FILE", |
||||
help="file to which index page will be written. " |
||||
"Use /dev/stdout to send to standard out.") |
||||
parser.add_option("-t", "--template", dest="template", |
||||
help="Mako template to render", metavar="FILE") |
||||
parser.add_option("--nonatomic", dest="nonatomic", action="store_true", |
||||
default=False, help="write the output to the path " |
||||
"given without creating a temporary file in between. " |
||||
"This is automatically set if the output appears " |
||||
"to be a character device, not a file.") |
||||
(options, args) = parser.parse_args() |
||||
|
||||
# load config file |
||||
try: |
||||
config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader) |
||||
except: |
||||
config = None |
||||
|
||||
if not config or type(config) != dict: |
||||
print >>sys.stderr, "Unable to load configuration '%s'." % options.config |
||||
sys.exit(-1) |
||||
|
||||
# determine important variables based on an appropriate order of |
||||
# precedence (command-line flags first, then the config file, |
||||
# then built-in fallbacks) |
||||
# |
||||
# fallback value for nonatomic is used so that character devices |
||||
# (e.g. /dev/stdout, /dev/null) are written to in the regular way |
||||
docroot = options.docroot or config.get('docroot') |
||||
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1 --exclude='.~tmp~'" |
||||
output = options.output or config.get('output') |
||||
template = options.template or config.get("template") or "index.mako" |
||||
nonatomic = options.nonatomic or config.get("nonatomic") or \ |
||||
(os.path.exists(output) and not os.path.isfile(output)) |
||||
|
||||
# sanity checks |
||||
if not docroot: |
||||
print >>sys.stderr, "docroot not specified." |
||||
print >>sys.stderr, "Define it in the config file or pass -D on the command line." |
||||
sys.exit(-1) |
||||
elif not output: |
||||
print >>sys.stderr, "output not specified." |
||||
print >>sys.stderr, "Define it in the config file or pass -o on the command line." |
||||
elif not config.get('directories'): |
||||
print >>sys.stderr, "directories not specified." |
||||
print >>sys.stderr, "Define it in the config file." |
||||
sys.exit(-1) |
||||
elif not os.path.isdir(docroot): |
||||
print >>sys.stderr, "docroot '%s' not found or not a directory." % docroot |
||||
sys.exit(-1) |
||||
elif not os.path.exists(template) or os.path.isdir(template): |
||||
print >>sys.stderr, "template '%s' not found or is a directory." % template |
||||
sys.exit(-1) |
||||
|
||||
generated = datetime.datetime.now() |
||||
|
||||
directories = [] |
||||
total_size = 0.0 |
||||
|
||||
ls = os.listdir(docroot) |
||||
|
||||
for item in ls: |
||||
if item == "pub": |
||||
continue |
||||
|
||||
path = os.path.join(docroot, item) |
||||
|
||||
# If symlink, then this is a project |
||||
if os.path.islink(path): |
||||
dataset = os.readlink(path)[1:] |
||||
|
||||
# Get the disk space used by that dataset |
||||
du = Popen( |
||||
"/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset, |
||||
shell=True, stdout=PIPE, stderr=PIPE).communicate() |
||||
|
||||
size = "-" |
||||
if len(du[1].strip()) > 0: |
||||
sys.stderr.write(du[1]) |
||||
print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item |
||||
continue |
||||
else: |
||||
size = parse_size(du[0].strip()) |
||||
total_size += size |
||||
|
||||
info = {'dir':item, 'size':reformat_size(size)} |
||||
|
||||
# use info from config.yaml, if found |
||||
# otherwise, skip this directory |
||||
if item in config['exclude']: |
||||
continue |
||||
elif item in config['directories']: |
||||
info.update(config['directories'][item]) |
||||
else: |
||||
info.update({ 'site': '', 'url': '' }) |
||||
|
||||
directories.append(info) |
||||
|
||||
# Sort the directories |
||||
directories.sort(key=lambda d : d['dir'].lower()) |
||||
|
||||
# Call du to compute size |
||||
#du = Popen( |
||||
# "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags), |
||||
# shell=True, stdout=PIPE, stderr=PIPE).communicate() |
||||
# du = [] |
||||
|
||||
# Check that du executed successfully |
||||
# If there's anything on stderr, send it |
||||
# out our own stderr and terminate. |
||||
#if len(du[1].strip()) > 0: |
||||
# sys.stderr.write(du[1]) |
||||
# print >>sys.stderr, "du terminated unsuccessfully. Not generating index." |
||||
# sys.exit(-1) |
||||
|
||||
# first one should be total, grab its size and format |
||||
#du = du[0].splitlines() # we only care about stdout now |
||||
#total_size = reformat_size(du[0].split(None,2)[0]) |
||||
|
||||
# the rest are the sizes we want |
||||
# directories = [] |
||||
# total_size = 0 |
||||
|
||||
# for line in du: |
||||
# (size, path) = line.split(None, 2) |
||||
# dir = os.path.basename(path) |
||||
# total_size += int(size) |
||||
# info = {'dir':dir, 'size':reformat_size(int(size))} |
||||
|
||||
# use info from config.yaml, if found |
||||
# otherwise, skip this directory |
||||
# if dir in config['exclude'] or not os.path.isdir(path): |
||||
# continue |
||||
# elif dir in config['directories']: |
||||
# info.update(config['directories'][dir]) |
||||
# else: |
||||
# info.update({ 'site': '', 'url': '' }) |
||||
|
||||
# directories.append(info) |
||||
|
||||
# render the template to a string |
||||
body = Template(filename=template).render( |
||||
total_size=reformat_size(total_size), |
||||
directories=directories, |
||||
generated=generated, |
||||
config=config, |
||||
h=webhelpers.html.tags) |
||||
|
||||
# write the rendered output |
||||
if nonatomic: |
||||
print >>file(output,'w'), body |
||||
else: |
||||
atomic_write(output, body) |
||||
|
||||
|
||||
if __name__ == "__main__": |
||||
main() |
||||
Loading…
Reference in new issue