From df3502e7e50afa476207d0fe230eb7fd799487c6 Mon Sep 17 00:00:00 2001 From: Zachary Seguin Date: Tue, 17 Aug 2021 19:23:45 -0400 Subject: [PATCH] feat: sync with updated scripts (rebase 2) --- mirror-index/config.yaml | 198 +++++++++++++++++++++++++++---------- mirror-index/index.mako | 12 ++- mirror-index/make-index.py | 155 ++++++++++++++++++++--------- 3 files changed, 263 insertions(+), 102 deletions(-) diff --git a/mirror-index/config.yaml b/mirror-index/config.yaml index bcd3395..edd06d2 100644 --- a/mirror-index/config.yaml +++ b/mirror-index/config.yaml @@ -1,171 +1,226 @@ docroot: /mirror/root -duflags: --human-readable --max-depth=1 +duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub output: /mirror/root/index.html +exclude: + - include + - lost+found + - pub + - uw-courseware + - wics + - mirror-rootfs + - mirror-highpriority + - ups + directories: + alpine: + site: alpinelinux.org + url: https://alpinelinux.org/ + + almalinux: + site: almalinux.org + url: https://almalinux.org + apache: site: apache.org - url: http://www.apache.org/ + url: https://www.apache.org/ archlinux: site: archlinux.org - url: http://www.archlinux.org/ + url: https://www.archlinux.org/ centos: site: centos.org - url: http://www.centos.org/ + url: https://www.centos.org/ + + ceph: + site: ceph.io + url: https://ceph.io CPAN: site: cpan.org - url: http://www.cpan.org/ + url: https://www.cpan.org/ CRAN: site: r-project.org - url: http://cran.r-project.org/ + url: https://cran.r-project.org/ csclub: site: csclub.uwaterloo.ca - url: http://csclub.uwaterloo.ca/media/ + url: https://csclub.uwaterloo.ca/media/ CTAN: site: ctan.org - url: http://www.ctan.org/ + url: https://www.ctan.org/ cygwin: site: cygwin.com - url: http://www.cygwin.com/ + url: https://www.cygwin.com/ + + damnsmalllinux: + site: damnsmalllinux.org + url: https://www.damnsmalllinux.org/ debian: site: debian.org - url: http://www.debian.org/ + url: https://www.debian.org/ debian-backports: site: backports.debian.org - url: http://backports.debian.org/ + url: https://backports.debian.org/ debian-cd: site: debian.org - url: http://www.debian.org/CD/ + url: https://www.debian.org/CD/ debian-multimedia: site: debian-multimedia.org - url: http://www.debian-multimedia.org/ + url: https://www.debian-multimedia.org/ debian-ports: site: debian-ports.org - url: http://www.debian-ports.org/ + url: https://www.debian-ports.org/ debian-security: site: debian.org - url: http://www.debian.org/security/ + url: https://www.debian.org/security/ debian-volatile: site: debian.org - url: http://www.debian.org/volatile/ + url: https://www.debian.org/volatile/ eclipse: site: eclipse.org - url: http://www.eclipse.org/ + url: https://www.eclipse.org/ - emdebian: - site: emdebian.org - url: http://www.emdebian.org/ + emacsconf: + site: emacsconf.org + url: https://emacsconf.org + + # this is no longer a project + # emdebian: + # site: emdebian.org + # url: https://www.emdebian.org/ fedora: site: fedoraproject.org - url: http://www.fedoraproject.org/ + url: https://www.fedoraproject.org/ FreeBSD: site: freebsd.org - url: http://www.freebsd.org/ + url: https://www.freebsd.org/ gentoo-distfiles: site: gentoo.org - url: http://www.gentoo.org/ + url: https://www.gentoo.org/ gentoo-portage: site: gentoo.org - url: http://www.gentoo.org/ + url: https://www.gentoo.org/ gnome: site: gnome.org - url: http://www.gnome.org/ + url: https://www.gnome.org/ gnu: site: gnu.org - url: http://www.gnu.org/ + url: https://www.gnu.org/ gutenberg: site: gutenberg.org - url: http://www.gutenberg.org + url: https://www.gutenberg.org/ + + kali: + site: kali.org + url: https://www.kali.org/ + + kali-images: + site: kali.org + url: https://www.kali.org/ kde: site: kde.org - url: http://www.kde.org/ + url: https://www.kde.org/ + + kde-applicationdata: + site: kde.org + url: https://www.kde.org/ kernel.org: site: kernel.org - url: http://www.kernel.org/ + url: https://www.kernel.org/ linuxmint: site: linuxmint.com - url: http://www.linuxmint.com/ + url: https://www.linuxmint.com/ linuxmint-packages: site: linuxmint.com - url: http://www.linuxmint.com/ + url: https://www.linuxmint.com/ MacPorts: site: macports.org - url: http://www.macports.org/ + url: https://www.macports.org/ mozdev: site: mozdev.org - url: http://www.mozdev.org/ + url: https://www.mozdev.org/ mozilla.org: site: mozilla.org - url: http://www.mozilla.org/ + url: https://www.mozilla.org/ mysql: site: mysql.com - url: http://www.mysql.com/ + url: https://www.mysql.com/ nongnu: site: nongnu.org - url: http://savannah.nongnu.org/ + url: https://savannah.nongnu.org/ openoffice: site: openoffice.org - url: http://www.openoffice.org/ + url: https://www.openoffice.org/ opensuse: site: opensuse.org - url: http://www.opensuse.org/ + url: https://www.opensuse.org/ OpenBSD: site: openbsd.org - url: http://www.openbsd.org/ + url: https://www.openbsd.org/ puppylinux: site: puppylinux.org - url: http://puppylinux.org/ + url: https://puppylinux.org/ qtproject: site: qt-project.org - url: http://qt-project.org/ + url: https://qt-project.org/ racket: site: racket-lang.org - url: http://racket-lang.org/ + url: https://racket-lang.org/ + + sage: + site: sagemath.org + url: https://www.sagemath.org/ + + saltstack: + site: saltstack.com + url: https://saltstack.com/ slackware: site: slackware.com - url: http://www.slackware.com/ + url: https://www.slackware.com/ + + tdf: + site: documentfoundation.org + url: https://www.documentfoundation.org/ ubuntu: site: ubuntu.com - url: http://www.ubuntu.com/ + url: https://www.ubuntu.com/ ubuntu-ports: site: ports.ubuntu.com @@ -173,33 +228,72 @@ directories: ubuntu-ports-releases: site: ports.ubuntu.com - url: http://cdimage.ubuntu.com/ports/releases/ + url: https://cdimage.ubuntu.com/releases/ ubuntu-releases: site: releases.ubuntu.com - url: http://releases.ubuntu.com/ + url: https://releases.ubuntu.com/ vlc: site: videolan.org - url: http://videolan.org/ + url: https://videolan.org/ xiph: site: xiph.org - url: http://xiph.org/ + url: https://xiph.org/ x.org: site: x.org - url: http://www.x.org/ + url: https://www.x.org/ xubuntu-releases: site: xubuntu.org - url: http://www.xubuntu.org/ + url: https://www.xubuntu.org/ NetBSD: site: netbsd.org - url: http://www.netbsd.org/ + url: https://www.netbsd.org/ + + pkgsrc: + site: pkgsrc.org + url: https://pkgsrc.org/ raspberrypi: site: raspberrypi.org - url: http://www.raspberrypi.org/ + url: https://www.raspberrypi.org/ + raspbian: + site: raspbian.org + url: https://www.raspbian.org/ + + ipfire: + site: ipfire.org + url: https://www.ipfire.org/ + + manjaro: + site: manjaro.org + url: https://manjaro.org/ + + scientific: + site: scientificlinux.org + url: https://www.scientificlinux.org/ + + mxlinux: + site: mxlinux.org + url: https://mxlinux.org + + mxlinux-iso: + site: mxlinux.org + url: https://mxlinux.org + + parabola: + site: parabola.nu + url: https://www.parabola.nu/ + + hyperbola: + site: hyperbola.info + url: https://www.hyperbola.info/ + + trisquel: + site: trisquel.info + url: https://trisquel.info/ diff --git a/mirror-index/index.mako b/mirror-index/index.mako index b9fa68c..c7f0ee4 100644 --- a/mirror-index/index.mako +++ b/mirror-index/index.mako @@ -5,16 +5,16 @@ Computer Science Club Mirror - + - +
- + % for dir in directories: % endfor \ - + @@ -41,7 +41,9 @@ diff --git a/mirror-index/make-index.py b/mirror-index/make-index.py index 204e61c..6a5d745 100755 --- a/mirror-index/make-index.py +++ b/mirror-index/make-index.py @@ -1,10 +1,11 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 """make-index.py Generates an nice index of the directories from a template. Original Author: Jeremy Roman +Modified by: Zachary Seguin So if you don't like how I did something, I'm the person you get to complain to. @@ -16,37 +17,47 @@ from subprocess import Popen, PIPE from optparse import OptionParser import yaml, mako.exceptions, webhelpers.html.tags from mako.template import Template +import humanize +import datetime + +def parse_size(s): + if s.endswith("T"): + return float(s[:-1])*1000*1000*1000*1000 + elif s.endswith("G"): + return float(s[:-1])*1000*1000*1000 + elif s.endswith("M"): + return float(s[:-1])*1000*1000 + elif s.endswith("K"): + return float(s[:-1])*1000 + + return float(size) def reformat_size(size): - """Reformats '124M' to '124 MB', et cetera.""" - if size[-1].isalpha(): - return size[:-1] + " " + size[-1] + "B" - else: - return size + return humanize.naturalsize(size) def atomic_write(filename, body): """Atomically write to a file by writing a temporary file and then moving it to replace the desired output file. - + This ensures that partial files are never seen by clients.""" - + # generate an appropriate temporary filename # in the same directory tmp_filename = "%s.%d.tmp" % (filename, os.getpid()) - + # open the directory so that we can fsync it dir = os.open(os.path.realpath(os.path.dirname(filename)), \ os.O_DIRECTORY | os.O_RDONLY) - + # write to the temporary file tmp = open(tmp_filename, 'w') print >>tmp, body tmp.flush() os.fsync(tmp.fileno()) tmp.close() - + # atomically replace the actual file os.rename(tmp_filename, filename) os.fsync(dir) @@ -72,17 +83,17 @@ def main(): "This is automatically set if the output appears " "to be a character device, not a file.") (options, args) = parser.parse_args() - + # load config file try: - config = yaml.load(file(options.config,'r')) + config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader) except: config = None - + if not config or type(config) != dict: print >>sys.stderr, "Unable to load configuration '%s'." % options.config sys.exit(-1) - + # determine important variables based on an appropriate order of # precedence (command-line flags first, then the config file, # then built-in fallbacks) @@ -90,12 +101,12 @@ def main(): # fallback value for nonatomic is used so that character devices # (e.g. /dev/stdout, /dev/null) are written to in the regular way docroot = options.docroot or config.get('docroot') - duflags = options.duflags or config.get('duflags') or "-h --max-depth=1" + duflags = options.duflags or config.get('duflags') or "-h --max-depth=1 --exclude='.~tmp~'" output = options.output or config.get('output') template = options.template or config.get("template") or "index.mako" nonatomic = options.nonatomic or config.get("nonatomic") or \ (os.path.exists(output) and not os.path.isfile(output)) - + # sanity checks if not docroot: print >>sys.stderr, "docroot not specified." @@ -114,47 +125,101 @@ def main(): elif not os.path.exists(template) or os.path.isdir(template): print >>sys.stderr, "template '%s' not found or is a directory." % template sys.exit(-1) - + + generated = datetime.datetime.now() + + directories = [] + total_size = 0.0 + + ls = os.listdir(docroot) + + for item in ls: + if item == "pub": + continue + + path = os.path.join(docroot, item) + + # If symlink, then this is a project + if os.path.islink(path): + dataset = os.readlink(path)[1:] + + # Get the disk space used by that dataset + du = Popen( + "/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset, + shell=True, stdout=PIPE, stderr=PIPE).communicate() + + size = "-" + if len(du[1].strip()) > 0: + sys.stderr.write(du[1]) + print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item + continue + else: + size = parse_size(du[0].strip()) + total_size += size + + info = {'dir':item, 'size':reformat_size(size)} + + # use info from config.yaml, if found + # otherwise, skip this directory + if item in config['exclude']: + continue + elif item in config['directories']: + info.update(config['directories'][item]) + else: + info.update({ 'site': '', 'url': '' }) + + directories.append(info) + + # Sort the directories + directories.sort(key=lambda d : d['dir'].lower()) + # Call du to compute size - du = Popen( - "/usr/bin/du %s %s | /usr/bin/sort -fk2" % (docroot, duflags), - shell=True, stdout=PIPE, stderr=PIPE).communicate() - + #du = Popen( + # "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags), + # shell=True, stdout=PIPE, stderr=PIPE).communicate() + # du = [] + # Check that du executed successfully # If there's anything on stderr, send it # out our own stderr and terminate. - if len(du[1].strip()) > 0: - sys.stderr.write(du[1]) - print >>sys.stderr, "du terminated unsuccessfully. Not generating index." - sys.exit(-1) - + #if len(du[1].strip()) > 0: + # sys.stderr.write(du[1]) + # print >>sys.stderr, "du terminated unsuccessfully. Not generating index." + # sys.exit(-1) + # first one should be total, grab its size and format - du = du[0].splitlines() # we only care about stdout now - total_size = reformat_size(du[0].split(None,2)[0]) - + #du = du[0].splitlines() # we only care about stdout now + #total_size = reformat_size(du[0].split(None,2)[0]) + # the rest are the sizes we want - directories = [] - for line in du[1:]: - (size, dir) = line.split(None, 2) - dir = os.path.basename(dir) - info = {'dir':dir, 'size':reformat_size(size)} - + # directories = [] + # total_size = 0 + + # for line in du: + # (size, path) = line.split(None, 2) + # dir = os.path.basename(path) + # total_size += int(size) + # info = {'dir':dir, 'size':reformat_size(int(size))} + # use info from config.yaml, if found # otherwise, skip this directory - if dir in config['directories']: - info.update(config['directories'][dir]) - else: - continue - - directories.append(info) - + # if dir in config['exclude'] or not os.path.isdir(path): + # continue + # elif dir in config['directories']: + # info.update(config['directories'][dir]) + # else: + # info.update({ 'site': '', 'url': '' }) + + # directories.append(info) + # render the template to a string body = Template(filename=template).render( - total_size=total_size, + total_size=reformat_size(total_size), directories=directories, + generated=generated, config=config, h=webhelpers.html.tags) - + # write the rendered output if nonatomic: print >>file(output,'w'), body
DirectoryProject SiteSize
@@ -31,7 +31,7 @@
Total