feat: sync with updated scripts (rebase 2)

master
Zachary Seguin 1 year ago committed by root
parent e84c1e71bb
commit df3502e7e5
  1. 198
      mirror-index/config.yaml
  2. 12
      mirror-index/index.mako
  3. 155
      mirror-index/make-index.py

@ -1,171 +1,226 @@
docroot: /mirror/root
duflags: --human-readable --max-depth=1
duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub
output: /mirror/root/index.html
exclude:
- include
- lost+found
- pub
- uw-courseware
- wics
- mirror-rootfs
- mirror-highpriority
- ups
directories:
alpine:
site: alpinelinux.org
url: https://alpinelinux.org/
almalinux:
site: almalinux.org
url: https://almalinux.org
apache:
site: apache.org
url: http://www.apache.org/
url: https://www.apache.org/
archlinux:
site: archlinux.org
url: http://www.archlinux.org/
url: https://www.archlinux.org/
centos:
site: centos.org
url: http://www.centos.org/
url: https://www.centos.org/
ceph:
site: ceph.io
url: https://ceph.io
CPAN:
site: cpan.org
url: http://www.cpan.org/
url: https://www.cpan.org/
CRAN:
site: r-project.org
url: http://cran.r-project.org/
url: https://cran.r-project.org/
csclub:
site: csclub.uwaterloo.ca
url: http://csclub.uwaterloo.ca/media/
url: https://csclub.uwaterloo.ca/media/
CTAN:
site: ctan.org
url: http://www.ctan.org/
url: https://www.ctan.org/
cygwin:
site: cygwin.com
url: http://www.cygwin.com/
url: https://www.cygwin.com/
damnsmalllinux:
site: damnsmalllinux.org
url: https://www.damnsmalllinux.org/
debian:
site: debian.org
url: http://www.debian.org/
url: https://www.debian.org/
debian-backports:
site: backports.debian.org
url: http://backports.debian.org/
url: https://backports.debian.org/
debian-cd:
site: debian.org
url: http://www.debian.org/CD/
url: https://www.debian.org/CD/
debian-multimedia:
site: debian-multimedia.org
url: http://www.debian-multimedia.org/
url: https://www.debian-multimedia.org/
debian-ports:
site: debian-ports.org
url: http://www.debian-ports.org/
url: https://www.debian-ports.org/
debian-security:
site: debian.org
url: http://www.debian.org/security/
url: https://www.debian.org/security/
debian-volatile:
site: debian.org
url: http://www.debian.org/volatile/
url: https://www.debian.org/volatile/
eclipse:
site: eclipse.org
url: http://www.eclipse.org/
url: https://www.eclipse.org/
emacsconf:
site: emacsconf.org
url: https://emacsconf.org
emdebian:
site: emdebian.org
url: http://www.emdebian.org/
# this is no longer a project
# emdebian:
# site: emdebian.org
# url: https://www.emdebian.org/
fedora:
site: fedoraproject.org
url: http://www.fedoraproject.org/
url: https://www.fedoraproject.org/
FreeBSD:
site: freebsd.org
url: http://www.freebsd.org/
url: https://www.freebsd.org/
gentoo-distfiles:
site: gentoo.org
url: http://www.gentoo.org/
url: https://www.gentoo.org/
gentoo-portage:
site: gentoo.org
url: http://www.gentoo.org/
url: https://www.gentoo.org/
gnome:
site: gnome.org
url: http://www.gnome.org/
url: https://www.gnome.org/
gnu:
site: gnu.org
url: http://www.gnu.org/
url: https://www.gnu.org/
gutenberg:
site: gutenberg.org
url: http://www.gutenberg.org
url: https://www.gutenberg.org/
kali:
site: kali.org
url: https://www.kali.org/
kali-images:
site: kali.org
url: https://www.kali.org/
kde:
site: kde.org
url: http://www.kde.org/
url: https://www.kde.org/
kde-applicationdata:
site: kde.org
url: https://www.kde.org/
kernel.org:
site: kernel.org
url: http://www.kernel.org/
url: https://www.kernel.org/
linuxmint:
site: linuxmint.com
url: http://www.linuxmint.com/
url: https://www.linuxmint.com/
linuxmint-packages:
site: linuxmint.com
url: http://www.linuxmint.com/
url: https://www.linuxmint.com/
MacPorts:
site: macports.org
url: http://www.macports.org/
url: https://www.macports.org/
mozdev:
site: mozdev.org
url: http://www.mozdev.org/
url: https://www.mozdev.org/
mozilla.org:
site: mozilla.org
url: http://www.mozilla.org/
url: https://www.mozilla.org/
mysql:
site: mysql.com
url: http://www.mysql.com/
url: https://www.mysql.com/
nongnu:
site: nongnu.org
url: http://savannah.nongnu.org/
url: https://savannah.nongnu.org/
openoffice:
site: openoffice.org
url: http://www.openoffice.org/
url: https://www.openoffice.org/
opensuse:
site: opensuse.org
url: http://www.opensuse.org/
url: https://www.opensuse.org/
OpenBSD:
site: openbsd.org
url: http://www.openbsd.org/
url: https://www.openbsd.org/
puppylinux:
site: puppylinux.org
url: http://puppylinux.org/
url: https://puppylinux.org/
qtproject:
site: qt-project.org
url: http://qt-project.org/
url: https://qt-project.org/
racket:
site: racket-lang.org
url: http://racket-lang.org/
url: https://racket-lang.org/
sage:
site: sagemath.org
url: https://www.sagemath.org/
saltstack:
site: saltstack.com
url: https://saltstack.com/
slackware:
site: slackware.com
url: http://www.slackware.com/
url: https://www.slackware.com/
tdf:
site: documentfoundation.org
url: https://www.documentfoundation.org/
ubuntu:
site: ubuntu.com
url: http://www.ubuntu.com/
url: https://www.ubuntu.com/
ubuntu-ports:
site: ports.ubuntu.com
@ -173,33 +228,72 @@ directories:
ubuntu-ports-releases:
site: ports.ubuntu.com
url: http://cdimage.ubuntu.com/ports/releases/
url: https://cdimage.ubuntu.com/releases/
ubuntu-releases:
site: releases.ubuntu.com
url: http://releases.ubuntu.com/
url: https://releases.ubuntu.com/
vlc:
site: videolan.org
url: http://videolan.org/
url: https://videolan.org/
xiph:
site: xiph.org
url: http://xiph.org/
url: https://xiph.org/
x.org:
site: x.org
url: http://www.x.org/
url: https://www.x.org/
xubuntu-releases:
site: xubuntu.org
url: http://www.xubuntu.org/
url: https://www.xubuntu.org/
NetBSD:
site: netbsd.org
url: http://www.netbsd.org/
url: https://www.netbsd.org/
pkgsrc:
site: pkgsrc.org
url: https://pkgsrc.org/
raspberrypi:
site: raspberrypi.org
url: http://www.raspberrypi.org/
url: https://www.raspberrypi.org/
raspbian:
site: raspbian.org
url: https://www.raspbian.org/
ipfire:
site: ipfire.org
url: https://www.ipfire.org/
manjaro:
site: manjaro.org
url: https://manjaro.org/
scientific:
site: scientificlinux.org
url: https://www.scientificlinux.org/
mxlinux:
site: mxlinux.org
url: https://mxlinux.org
mxlinux-iso:
site: mxlinux.org
url: https://mxlinux.org
parabola:
site: parabola.nu
url: https://www.parabola.nu/
hyperbola:
site: hyperbola.info
url: https://www.hyperbola.info/
trisquel:
site: trisquel.info
url: https://trisquel.info/

@ -5,16 +5,16 @@
<link rel="stylesheet" type="text/css" href="index.css" />
<title>Computer Science Club Mirror</title>
</head>
<body>
<div id="logo">
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
</div>
<div id="listing">
<table>
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
% for dir in directories:
<tr>
<td>
@ -31,7 +31,7 @@
</tr>
% endfor \
<tr class="total">
<td>Total</td>
<td></td>
@ -41,7 +41,9 @@
</div>
<div id="footer">
<p>This service is run by the <a href="http://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="http://www.student.math.uwaterloo.ca/~mefcom/">Mathematics Endowment Fund</a><br />and support from the <a href="http://www.cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
<p>This service is run by the <a href="https://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="https://uwaterloo.ca/math-endowment-fund/">Mathematics Endowment Fund</a><br />and support from the <a href="https://cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
<p>Report an issue or request we mirror a project by contacting <a href="mailto:systems-committee@csclub.uwaterloo.ca">systems-committee@csclub.uwaterloo.ca</a>.</p>
<p>Generated (local time): ${generated.strftime("%Y-%m-%d %-I:%M%p")}</p>
</div>
</body>
</html>

@ -1,10 +1,11 @@
#!/usr/bin/env python
#!/usr/bin/env python2
"""make-index.py
Generates an nice index of the directories from a
template.
Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
Modified by: Zachary Seguin <ztseguin@csclub.uwaterloo.ca>
So if you don't like how I did something,
I'm the person you get to complain to.
@ -16,37 +17,47 @@ from subprocess import Popen, PIPE
from optparse import OptionParser
import yaml, mako.exceptions, webhelpers.html.tags
from mako.template import Template
import humanize
import datetime
def parse_size(s):
if s.endswith("T"):
return float(s[:-1])*1000*1000*1000*1000
elif s.endswith("G"):
return float(s[:-1])*1000*1000*1000
elif s.endswith("M"):
return float(s[:-1])*1000*1000
elif s.endswith("K"):
return float(s[:-1])*1000
return float(size)
def reformat_size(size):
"""Reformats '124M' to '124 MB', et cetera."""
if size[-1].isalpha():
return size[:-1] + " " + size[-1] + "B"
else:
return size
return humanize.naturalsize(size)
def atomic_write(filename, body):
"""Atomically write to a file by writing a
temporary file and then moving it to replace
the desired output file.
This ensures that partial files are never seen
by clients."""
# generate an appropriate temporary filename
# in the same directory
tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
# open the directory so that we can fsync it
dir = os.open(os.path.realpath(os.path.dirname(filename)), \
os.O_DIRECTORY | os.O_RDONLY)
# write to the temporary file
tmp = open(tmp_filename, 'w')
print >>tmp, body
tmp.flush()
os.fsync(tmp.fileno())
tmp.close()
# atomically replace the actual file
os.rename(tmp_filename, filename)
os.fsync(dir)
@ -72,17 +83,17 @@ def main():
"This is automatically set if the output appears "
"to be a character device, not a file.")
(options, args) = parser.parse_args()
# load config file
try:
config = yaml.load(file(options.config,'r'))
config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader)
except:
config = None
if not config or type(config) != dict:
print >>sys.stderr, "Unable to load configuration '%s'." % options.config
sys.exit(-1)
# determine important variables based on an appropriate order of
# precedence (command-line flags first, then the config file,
# then built-in fallbacks)
@ -90,12 +101,12 @@ def main():
# fallback value for nonatomic is used so that character devices
# (e.g. /dev/stdout, /dev/null) are written to in the regular way
docroot = options.docroot or config.get('docroot')
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1"
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1 --exclude='.~tmp~'"
output = options.output or config.get('output')
template = options.template or config.get("template") or "index.mako"
nonatomic = options.nonatomic or config.get("nonatomic") or \
(os.path.exists(output) and not os.path.isfile(output))
# sanity checks
if not docroot:
print >>sys.stderr, "docroot not specified."
@ -114,47 +125,101 @@ def main():
elif not os.path.exists(template) or os.path.isdir(template):
print >>sys.stderr, "template '%s' not found or is a directory." % template
sys.exit(-1)
generated = datetime.datetime.now()
directories = []
total_size = 0.0
ls = os.listdir(docroot)
for item in ls:
if item == "pub":
continue
path = os.path.join(docroot, item)
# If symlink, then this is a project
if os.path.islink(path):
dataset = os.readlink(path)[1:]
# Get the disk space used by that dataset
du = Popen(
"/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset,
shell=True, stdout=PIPE, stderr=PIPE).communicate()
size = "-"
if len(du[1].strip()) > 0:
sys.stderr.write(du[1])
print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item
continue
else:
size = parse_size(du[0].strip())
total_size += size
info = {'dir':item, 'size':reformat_size(size)}
# use info from config.yaml, if found
# otherwise, skip this directory
if item in config['exclude']:
continue
elif item in config['directories']:
info.update(config['directories'][item])
else:
info.update({ 'site': '', 'url': '' })
directories.append(info)
# Sort the directories
directories.sort(key=lambda d : d['dir'].lower())
# Call du to compute size
du = Popen(
"/usr/bin/du %s %s | /usr/bin/sort -fk2" % (docroot, duflags),
shell=True, stdout=PIPE, stderr=PIPE).communicate()
#du = Popen(
# "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags),
# shell=True, stdout=PIPE, stderr=PIPE).communicate()
# du = []
# Check that du executed successfully
# If there's anything on stderr, send it
# out our own stderr and terminate.
if len(du[1].strip()) > 0:
sys.stderr.write(du[1])
print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
sys.exit(-1)
#if len(du[1].strip()) > 0:
# sys.stderr.write(du[1])
# print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
# sys.exit(-1)
# first one should be total, grab its size and format
du = du[0].splitlines() # we only care about stdout now
total_size = reformat_size(du[0].split(None,2)[0])
#du = du[0].splitlines() # we only care about stdout now
#total_size = reformat_size(du[0].split(None,2)[0])
# the rest are the sizes we want
directories = []
for line in du[1:]:
(size, dir) = line.split(None, 2)
dir = os.path.basename(dir)
info = {'dir':dir, 'size':reformat_size(size)}
# directories = []
# total_size = 0
# for line in du:
# (size, path) = line.split(None, 2)
# dir = os.path.basename(path)
# total_size += int(size)
# info = {'dir':dir, 'size':reformat_size(int(size))}
# use info from config.yaml, if found
# otherwise, skip this directory
if dir in config['directories']:
info.update(config['directories'][dir])
else:
continue
directories.append(info)
# if dir in config['exclude'] or not os.path.isdir(path):
# continue
# elif dir in config['directories']:
# info.update(config['directories'][dir])
# else:
# info.update({ 'site': '', 'url': '' })
# directories.append(info)
# render the template to a string
body = Template(filename=template).render(
total_size=total_size,
total_size=reformat_size(total_size),
directories=directories,
generated=generated,
config=config,
h=webhelpers.html.tags)
# write the rendered output
if nonatomic:
print >>file(output,'w'), body

Loading…
Cancel
Save