feat: sync with updated scripts (rebase 2)
This commit is contained in:
root
parent
e84c1e71bb
commit
df3502e7e5
|
|
@ -1,171 +1,226 @@
|
|||
docroot: /mirror/root
|
||||
duflags: --human-readable --max-depth=1
|
||||
duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub
|
||||
output: /mirror/root/index.html
|
||||
|
||||
exclude:
|
||||
- include
|
||||
- lost+found
|
||||
- pub
|
||||
- uw-courseware
|
||||
- wics
|
||||
- mirror-rootfs
|
||||
- mirror-highpriority
|
||||
- ups
|
||||
|
||||
directories:
|
||||
alpine:
|
||||
site: alpinelinux.org
|
||||
url: https://alpinelinux.org/
|
||||
|
||||
almalinux:
|
||||
site: almalinux.org
|
||||
url: https://almalinux.org
|
||||
|
||||
apache:
|
||||
site: apache.org
|
||||
url: http://www.apache.org/
|
||||
url: https://www.apache.org/
|
||||
|
||||
archlinux:
|
||||
site: archlinux.org
|
||||
url: http://www.archlinux.org/
|
||||
url: https://www.archlinux.org/
|
||||
|
||||
centos:
|
||||
site: centos.org
|
||||
url: http://www.centos.org/
|
||||
url: https://www.centos.org/
|
||||
|
||||
ceph:
|
||||
site: ceph.io
|
||||
url: https://ceph.io
|
||||
|
||||
CPAN:
|
||||
site: cpan.org
|
||||
url: http://www.cpan.org/
|
||||
url: https://www.cpan.org/
|
||||
|
||||
CRAN:
|
||||
site: r-project.org
|
||||
url: http://cran.r-project.org/
|
||||
url: https://cran.r-project.org/
|
||||
|
||||
csclub:
|
||||
site: csclub.uwaterloo.ca
|
||||
url: http://csclub.uwaterloo.ca/media/
|
||||
url: https://csclub.uwaterloo.ca/media/
|
||||
|
||||
CTAN:
|
||||
site: ctan.org
|
||||
url: http://www.ctan.org/
|
||||
url: https://www.ctan.org/
|
||||
|
||||
cygwin:
|
||||
site: cygwin.com
|
||||
url: http://www.cygwin.com/
|
||||
url: https://www.cygwin.com/
|
||||
|
||||
damnsmalllinux:
|
||||
site: damnsmalllinux.org
|
||||
url: https://www.damnsmalllinux.org/
|
||||
|
||||
debian:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/
|
||||
url: https://www.debian.org/
|
||||
|
||||
debian-backports:
|
||||
site: backports.debian.org
|
||||
url: http://backports.debian.org/
|
||||
url: https://backports.debian.org/
|
||||
|
||||
debian-cd:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/CD/
|
||||
url: https://www.debian.org/CD/
|
||||
|
||||
debian-multimedia:
|
||||
site: debian-multimedia.org
|
||||
url: http://www.debian-multimedia.org/
|
||||
url: https://www.debian-multimedia.org/
|
||||
|
||||
debian-ports:
|
||||
site: debian-ports.org
|
||||
url: http://www.debian-ports.org/
|
||||
url: https://www.debian-ports.org/
|
||||
|
||||
debian-security:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/security/
|
||||
url: https://www.debian.org/security/
|
||||
|
||||
debian-volatile:
|
||||
site: debian.org
|
||||
url: http://www.debian.org/volatile/
|
||||
url: https://www.debian.org/volatile/
|
||||
|
||||
eclipse:
|
||||
site: eclipse.org
|
||||
url: http://www.eclipse.org/
|
||||
url: https://www.eclipse.org/
|
||||
|
||||
emdebian:
|
||||
site: emdebian.org
|
||||
url: http://www.emdebian.org/
|
||||
emacsconf:
|
||||
site: emacsconf.org
|
||||
url: https://emacsconf.org
|
||||
|
||||
# this is no longer a project
|
||||
# emdebian:
|
||||
# site: emdebian.org
|
||||
# url: https://www.emdebian.org/
|
||||
|
||||
fedora:
|
||||
site: fedoraproject.org
|
||||
url: http://www.fedoraproject.org/
|
||||
url: https://www.fedoraproject.org/
|
||||
|
||||
FreeBSD:
|
||||
site: freebsd.org
|
||||
url: http://www.freebsd.org/
|
||||
url: https://www.freebsd.org/
|
||||
|
||||
gentoo-distfiles:
|
||||
site: gentoo.org
|
||||
url: http://www.gentoo.org/
|
||||
url: https://www.gentoo.org/
|
||||
|
||||
gentoo-portage:
|
||||
site: gentoo.org
|
||||
url: http://www.gentoo.org/
|
||||
url: https://www.gentoo.org/
|
||||
|
||||
gnome:
|
||||
site: gnome.org
|
||||
url: http://www.gnome.org/
|
||||
url: https://www.gnome.org/
|
||||
|
||||
gnu:
|
||||
site: gnu.org
|
||||
url: http://www.gnu.org/
|
||||
url: https://www.gnu.org/
|
||||
|
||||
gutenberg:
|
||||
site: gutenberg.org
|
||||
url: http://www.gutenberg.org
|
||||
url: https://www.gutenberg.org/
|
||||
|
||||
kali:
|
||||
site: kali.org
|
||||
url: https://www.kali.org/
|
||||
|
||||
kali-images:
|
||||
site: kali.org
|
||||
url: https://www.kali.org/
|
||||
|
||||
kde:
|
||||
site: kde.org
|
||||
url: http://www.kde.org/
|
||||
url: https://www.kde.org/
|
||||
|
||||
kde-applicationdata:
|
||||
site: kde.org
|
||||
url: https://www.kde.org/
|
||||
|
||||
kernel.org:
|
||||
site: kernel.org
|
||||
url: http://www.kernel.org/
|
||||
url: https://www.kernel.org/
|
||||
|
||||
linuxmint:
|
||||
site: linuxmint.com
|
||||
url: http://www.linuxmint.com/
|
||||
url: https://www.linuxmint.com/
|
||||
|
||||
linuxmint-packages:
|
||||
site: linuxmint.com
|
||||
url: http://www.linuxmint.com/
|
||||
url: https://www.linuxmint.com/
|
||||
|
||||
MacPorts:
|
||||
site: macports.org
|
||||
url: http://www.macports.org/
|
||||
url: https://www.macports.org/
|
||||
|
||||
mozdev:
|
||||
site: mozdev.org
|
||||
url: http://www.mozdev.org/
|
||||
url: https://www.mozdev.org/
|
||||
|
||||
mozilla.org:
|
||||
site: mozilla.org
|
||||
url: http://www.mozilla.org/
|
||||
url: https://www.mozilla.org/
|
||||
|
||||
mysql:
|
||||
site: mysql.com
|
||||
url: http://www.mysql.com/
|
||||
url: https://www.mysql.com/
|
||||
|
||||
nongnu:
|
||||
site: nongnu.org
|
||||
url: http://savannah.nongnu.org/
|
||||
url: https://savannah.nongnu.org/
|
||||
|
||||
openoffice:
|
||||
site: openoffice.org
|
||||
url: http://www.openoffice.org/
|
||||
url: https://www.openoffice.org/
|
||||
|
||||
opensuse:
|
||||
site: opensuse.org
|
||||
url: http://www.opensuse.org/
|
||||
url: https://www.opensuse.org/
|
||||
|
||||
OpenBSD:
|
||||
site: openbsd.org
|
||||
url: http://www.openbsd.org/
|
||||
url: https://www.openbsd.org/
|
||||
|
||||
puppylinux:
|
||||
site: puppylinux.org
|
||||
url: http://puppylinux.org/
|
||||
url: https://puppylinux.org/
|
||||
|
||||
qtproject:
|
||||
site: qt-project.org
|
||||
url: http://qt-project.org/
|
||||
url: https://qt-project.org/
|
||||
|
||||
racket:
|
||||
site: racket-lang.org
|
||||
url: http://racket-lang.org/
|
||||
url: https://racket-lang.org/
|
||||
|
||||
sage:
|
||||
site: sagemath.org
|
||||
url: https://www.sagemath.org/
|
||||
|
||||
saltstack:
|
||||
site: saltstack.com
|
||||
url: https://saltstack.com/
|
||||
|
||||
slackware:
|
||||
site: slackware.com
|
||||
url: http://www.slackware.com/
|
||||
url: https://www.slackware.com/
|
||||
|
||||
tdf:
|
||||
site: documentfoundation.org
|
||||
url: https://www.documentfoundation.org/
|
||||
|
||||
ubuntu:
|
||||
site: ubuntu.com
|
||||
url: http://www.ubuntu.com/
|
||||
url: https://www.ubuntu.com/
|
||||
|
||||
ubuntu-ports:
|
||||
site: ports.ubuntu.com
|
||||
|
|
@ -173,33 +228,72 @@ directories:
|
|||
|
||||
ubuntu-ports-releases:
|
||||
site: ports.ubuntu.com
|
||||
url: http://cdimage.ubuntu.com/ports/releases/
|
||||
url: https://cdimage.ubuntu.com/releases/
|
||||
|
||||
ubuntu-releases:
|
||||
site: releases.ubuntu.com
|
||||
url: http://releases.ubuntu.com/
|
||||
url: https://releases.ubuntu.com/
|
||||
|
||||
vlc:
|
||||
site: videolan.org
|
||||
url: http://videolan.org/
|
||||
url: https://videolan.org/
|
||||
|
||||
xiph:
|
||||
site: xiph.org
|
||||
url: http://xiph.org/
|
||||
url: https://xiph.org/
|
||||
|
||||
x.org:
|
||||
site: x.org
|
||||
url: http://www.x.org/
|
||||
url: https://www.x.org/
|
||||
|
||||
xubuntu-releases:
|
||||
site: xubuntu.org
|
||||
url: http://www.xubuntu.org/
|
||||
url: https://www.xubuntu.org/
|
||||
|
||||
NetBSD:
|
||||
site: netbsd.org
|
||||
url: http://www.netbsd.org/
|
||||
url: https://www.netbsd.org/
|
||||
|
||||
pkgsrc:
|
||||
site: pkgsrc.org
|
||||
url: https://pkgsrc.org/
|
||||
|
||||
raspberrypi:
|
||||
site: raspberrypi.org
|
||||
url: http://www.raspberrypi.org/
|
||||
url: https://www.raspberrypi.org/
|
||||
|
||||
raspbian:
|
||||
site: raspbian.org
|
||||
url: https://www.raspbian.org/
|
||||
|
||||
ipfire:
|
||||
site: ipfire.org
|
||||
url: https://www.ipfire.org/
|
||||
|
||||
manjaro:
|
||||
site: manjaro.org
|
||||
url: https://manjaro.org/
|
||||
|
||||
scientific:
|
||||
site: scientificlinux.org
|
||||
url: https://www.scientificlinux.org/
|
||||
|
||||
mxlinux:
|
||||
site: mxlinux.org
|
||||
url: https://mxlinux.org
|
||||
|
||||
mxlinux-iso:
|
||||
site: mxlinux.org
|
||||
url: https://mxlinux.org
|
||||
|
||||
parabola:
|
||||
site: parabola.nu
|
||||
url: https://www.parabola.nu/
|
||||
|
||||
hyperbola:
|
||||
site: hyperbola.info
|
||||
url: https://www.hyperbola.info/
|
||||
|
||||
trisquel:
|
||||
site: trisquel.info
|
||||
url: https://trisquel.info/
|
||||
|
|
|
|||
|
|
@ -5,16 +5,16 @@
|
|||
<link rel="stylesheet" type="text/css" href="index.css" />
|
||||
<title>Computer Science Club Mirror</title>
|
||||
</head>
|
||||
|
||||
|
||||
<body>
|
||||
<div id="logo">
|
||||
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
|
||||
</div>
|
||||
|
||||
|
||||
<div id="listing">
|
||||
<table>
|
||||
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
|
||||
|
||||
|
||||
% for dir in directories:
|
||||
<tr>
|
||||
<td>
|
||||
|
|
@ -31,7 +31,7 @@
|
|||
</tr>
|
||||
|
||||
% endfor \
|
||||
|
||||
|
||||
<tr class="total">
|
||||
<td>Total</td>
|
||||
<td></td>
|
||||
|
|
@ -41,7 +41,9 @@
|
|||
</div>
|
||||
|
||||
<div id="footer">
|
||||
<p>This service is run by the <a href="http://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="http://www.student.math.uwaterloo.ca/~mefcom/">Mathematics Endowment Fund</a><br />and support from the <a href="http://www.cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
|
||||
<p>This service is run by the <a href="https://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="https://uwaterloo.ca/math-endowment-fund/">Mathematics Endowment Fund</a><br />and support from the <a href="https://cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
|
||||
<p>Report an issue or request we mirror a project by contacting <a href="mailto:systems-committee@csclub.uwaterloo.ca">systems-committee@csclub.uwaterloo.ca</a>.</p>
|
||||
<p>Generated (local time): ${generated.strftime("%Y-%m-%d %-I:%M%p")}</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
#!/usr/bin/env python
|
||||
#!/usr/bin/env python2
|
||||
"""make-index.py
|
||||
|
||||
Generates an nice index of the directories from a
|
||||
template.
|
||||
|
||||
Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
|
||||
Modified by: Zachary Seguin <ztseguin@csclub.uwaterloo.ca>
|
||||
|
||||
So if you don't like how I did something,
|
||||
I'm the person you get to complain to.
|
||||
|
|
@ -16,37 +17,47 @@ from subprocess import Popen, PIPE
|
|||
from optparse import OptionParser
|
||||
import yaml, mako.exceptions, webhelpers.html.tags
|
||||
from mako.template import Template
|
||||
import humanize
|
||||
import datetime
|
||||
|
||||
def parse_size(s):
|
||||
if s.endswith("T"):
|
||||
return float(s[:-1])*1000*1000*1000*1000
|
||||
elif s.endswith("G"):
|
||||
return float(s[:-1])*1000*1000*1000
|
||||
elif s.endswith("M"):
|
||||
return float(s[:-1])*1000*1000
|
||||
elif s.endswith("K"):
|
||||
return float(s[:-1])*1000
|
||||
|
||||
return float(size)
|
||||
|
||||
def reformat_size(size):
|
||||
"""Reformats '124M' to '124 MB', et cetera."""
|
||||
if size[-1].isalpha():
|
||||
return size[:-1] + " " + size[-1] + "B"
|
||||
else:
|
||||
return size
|
||||
return humanize.naturalsize(size)
|
||||
|
||||
def atomic_write(filename, body):
|
||||
"""Atomically write to a file by writing a
|
||||
temporary file and then moving it to replace
|
||||
the desired output file.
|
||||
|
||||
|
||||
This ensures that partial files are never seen
|
||||
by clients."""
|
||||
|
||||
|
||||
# generate an appropriate temporary filename
|
||||
# in the same directory
|
||||
tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
|
||||
|
||||
|
||||
# open the directory so that we can fsync it
|
||||
dir = os.open(os.path.realpath(os.path.dirname(filename)), \
|
||||
os.O_DIRECTORY | os.O_RDONLY)
|
||||
|
||||
|
||||
# write to the temporary file
|
||||
tmp = open(tmp_filename, 'w')
|
||||
print >>tmp, body
|
||||
tmp.flush()
|
||||
os.fsync(tmp.fileno())
|
||||
tmp.close()
|
||||
|
||||
|
||||
# atomically replace the actual file
|
||||
os.rename(tmp_filename, filename)
|
||||
os.fsync(dir)
|
||||
|
|
@ -72,17 +83,17 @@ def main():
|
|||
"This is automatically set if the output appears "
|
||||
"to be a character device, not a file.")
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
|
||||
# load config file
|
||||
try:
|
||||
config = yaml.load(file(options.config,'r'))
|
||||
config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader)
|
||||
except:
|
||||
config = None
|
||||
|
||||
|
||||
if not config or type(config) != dict:
|
||||
print >>sys.stderr, "Unable to load configuration '%s'." % options.config
|
||||
sys.exit(-1)
|
||||
|
||||
|
||||
# determine important variables based on an appropriate order of
|
||||
# precedence (command-line flags first, then the config file,
|
||||
# then built-in fallbacks)
|
||||
|
|
@ -90,12 +101,12 @@ def main():
|
|||
# fallback value for nonatomic is used so that character devices
|
||||
# (e.g. /dev/stdout, /dev/null) are written to in the regular way
|
||||
docroot = options.docroot or config.get('docroot')
|
||||
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1"
|
||||
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1 --exclude='.~tmp~'"
|
||||
output = options.output or config.get('output')
|
||||
template = options.template or config.get("template") or "index.mako"
|
||||
nonatomic = options.nonatomic or config.get("nonatomic") or \
|
||||
(os.path.exists(output) and not os.path.isfile(output))
|
||||
|
||||
|
||||
# sanity checks
|
||||
if not docroot:
|
||||
print >>sys.stderr, "docroot not specified."
|
||||
|
|
@ -114,47 +125,101 @@ def main():
|
|||
elif not os.path.exists(template) or os.path.isdir(template):
|
||||
print >>sys.stderr, "template '%s' not found or is a directory." % template
|
||||
sys.exit(-1)
|
||||
|
||||
|
||||
generated = datetime.datetime.now()
|
||||
|
||||
directories = []
|
||||
total_size = 0.0
|
||||
|
||||
ls = os.listdir(docroot)
|
||||
|
||||
for item in ls:
|
||||
if item == "pub":
|
||||
continue
|
||||
|
||||
path = os.path.join(docroot, item)
|
||||
|
||||
# If symlink, then this is a project
|
||||
if os.path.islink(path):
|
||||
dataset = os.readlink(path)[1:]
|
||||
|
||||
# Get the disk space used by that dataset
|
||||
du = Popen(
|
||||
"/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset,
|
||||
shell=True, stdout=PIPE, stderr=PIPE).communicate()
|
||||
|
||||
size = "-"
|
||||
if len(du[1].strip()) > 0:
|
||||
sys.stderr.write(du[1])
|
||||
print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item
|
||||
continue
|
||||
else:
|
||||
size = parse_size(du[0].strip())
|
||||
total_size += size
|
||||
|
||||
info = {'dir':item, 'size':reformat_size(size)}
|
||||
|
||||
# use info from config.yaml, if found
|
||||
# otherwise, skip this directory
|
||||
if item in config['exclude']:
|
||||
continue
|
||||
elif item in config['directories']:
|
||||
info.update(config['directories'][item])
|
||||
else:
|
||||
info.update({ 'site': '', 'url': '' })
|
||||
|
||||
directories.append(info)
|
||||
|
||||
# Sort the directories
|
||||
directories.sort(key=lambda d : d['dir'].lower())
|
||||
|
||||
# Call du to compute size
|
||||
du = Popen(
|
||||
"/usr/bin/du %s %s | /usr/bin/sort -fk2" % (docroot, duflags),
|
||||
shell=True, stdout=PIPE, stderr=PIPE).communicate()
|
||||
|
||||
#du = Popen(
|
||||
# "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags),
|
||||
# shell=True, stdout=PIPE, stderr=PIPE).communicate()
|
||||
# du = []
|
||||
|
||||
# Check that du executed successfully
|
||||
# If there's anything on stderr, send it
|
||||
# out our own stderr and terminate.
|
||||
if len(du[1].strip()) > 0:
|
||||
sys.stderr.write(du[1])
|
||||
print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
|
||||
sys.exit(-1)
|
||||
|
||||
#if len(du[1].strip()) > 0:
|
||||
# sys.stderr.write(du[1])
|
||||
# print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
|
||||
# sys.exit(-1)
|
||||
|
||||
# first one should be total, grab its size and format
|
||||
du = du[0].splitlines() # we only care about stdout now
|
||||
total_size = reformat_size(du[0].split(None,2)[0])
|
||||
|
||||
#du = du[0].splitlines() # we only care about stdout now
|
||||
#total_size = reformat_size(du[0].split(None,2)[0])
|
||||
|
||||
# the rest are the sizes we want
|
||||
directories = []
|
||||
for line in du[1:]:
|
||||
(size, dir) = line.split(None, 2)
|
||||
dir = os.path.basename(dir)
|
||||
info = {'dir':dir, 'size':reformat_size(size)}
|
||||
|
||||
# directories = []
|
||||
# total_size = 0
|
||||
|
||||
# for line in du:
|
||||
# (size, path) = line.split(None, 2)
|
||||
# dir = os.path.basename(path)
|
||||
# total_size += int(size)
|
||||
# info = {'dir':dir, 'size':reformat_size(int(size))}
|
||||
|
||||
# use info from config.yaml, if found
|
||||
# otherwise, skip this directory
|
||||
if dir in config['directories']:
|
||||
info.update(config['directories'][dir])
|
||||
else:
|
||||
continue
|
||||
|
||||
directories.append(info)
|
||||
|
||||
# if dir in config['exclude'] or not os.path.isdir(path):
|
||||
# continue
|
||||
# elif dir in config['directories']:
|
||||
# info.update(config['directories'][dir])
|
||||
# else:
|
||||
# info.update({ 'site': '', 'url': '' })
|
||||
|
||||
# directories.append(info)
|
||||
|
||||
# render the template to a string
|
||||
body = Template(filename=template).render(
|
||||
total_size=total_size,
|
||||
total_size=reformat_size(total_size),
|
||||
directories=directories,
|
||||
generated=generated,
|
||||
config=config,
|
||||
h=webhelpers.html.tags)
|
||||
|
||||
|
||||
# write the rendered output
|
||||
if nonatomic:
|
||||
print >>file(output,'w'), body
|
||||
|
|
|
|||
Loading…
Reference in New Issue