feat: sync with updated scripts (rebase 2)

mirror-index/config.yaml

@@ -1,171 +1,226 @@
 docroot: /mirror/root
-duflags: --human-readable --max-depth=1
+duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub
 output: /mirror/root/index.html
 
+exclude:
+   - include
+   - lost+found
+   - pub
+   - uw-courseware
+   - wics
+   - mirror-rootfs
+   - mirror-highpriority
+   - ups
+
 directories:
+  alpine:
+     site: alpinelinux.org
+     url: https://alpinelinux.org/
+
+  almalinux:
+    site: almalinux.org
+    url: https://almalinux.org
+
   apache:
     site: apache.org
-    url: http://www.apache.org/
+    url: https://www.apache.org/
 
   archlinux:
     site: archlinux.org
-    url: http://www.archlinux.org/
+    url: https://www.archlinux.org/
 
   centos:
     site: centos.org
-    url: http://www.centos.org/
+    url: https://www.centos.org/
+
+  ceph:
+    site: ceph.io
+    url: https://ceph.io
 
   CPAN:
     site: cpan.org
-    url: http://www.cpan.org/
+    url: https://www.cpan.org/
 
   CRAN:
     site: r-project.org
-    url: http://cran.r-project.org/
+    url: https://cran.r-project.org/
 
   csclub:
     site: csclub.uwaterloo.ca
-    url: http://csclub.uwaterloo.ca/media/
+    url: https://csclub.uwaterloo.ca/media/
 
   CTAN:
     site: ctan.org
-    url: http://www.ctan.org/
+    url: https://www.ctan.org/
 
   cygwin:
     site: cygwin.com
-    url: http://www.cygwin.com/
+    url: https://www.cygwin.com/
+
+  damnsmalllinux:
+    site: damnsmalllinux.org
+    url: https://www.damnsmalllinux.org/
 
   debian:
     site: debian.org
-    url: http://www.debian.org/
+    url: https://www.debian.org/
 
   debian-backports:
     site: backports.debian.org
-    url: http://backports.debian.org/
+    url: https://backports.debian.org/
 
   debian-cd:
     site: debian.org
-    url: http://www.debian.org/CD/
+    url: https://www.debian.org/CD/
 
   debian-multimedia:
     site: debian-multimedia.org
-    url: http://www.debian-multimedia.org/
+    url: https://www.debian-multimedia.org/
 
   debian-ports:
     site: debian-ports.org
-    url: http://www.debian-ports.org/
+    url: https://www.debian-ports.org/
 
   debian-security:
     site: debian.org
-    url: http://www.debian.org/security/
+    url: https://www.debian.org/security/
 
   debian-volatile:
     site: debian.org
-    url: http://www.debian.org/volatile/
+    url: https://www.debian.org/volatile/
 
   eclipse:
     site: eclipse.org
-    url: http://www.eclipse.org/
+    url: https://www.eclipse.org/
 
-  emdebian:
+  emacsconf:
-    site: emdebian.org
+    site: emacsconf.org
-    url: http://www.emdebian.org/
+    url: https://emacsconf.org
+
+ # this is no longer a project
+ # emdebian:
+ #   site: emdebian.org
+ #   url: https://www.emdebian.org/
 
   fedora:
     site: fedoraproject.org
-    url: http://www.fedoraproject.org/
+    url: https://www.fedoraproject.org/
 
   FreeBSD:
     site: freebsd.org
-    url: http://www.freebsd.org/
+    url: https://www.freebsd.org/
 
   gentoo-distfiles:
     site: gentoo.org
-    url: http://www.gentoo.org/
+    url: https://www.gentoo.org/
 
   gentoo-portage:
     site: gentoo.org
-    url: http://www.gentoo.org/
+    url: https://www.gentoo.org/
 
   gnome:
     site: gnome.org
-    url: http://www.gnome.org/
+    url: https://www.gnome.org/
 
   gnu:
     site: gnu.org
-    url: http://www.gnu.org/
+    url: https://www.gnu.org/
 
   gutenberg:
     site: gutenberg.org
-    url: http://www.gutenberg.org
+    url: https://www.gutenberg.org/
+
+  kali:
+    site: kali.org
+    url: https://www.kali.org/
+
+  kali-images:
+    site: kali.org
+    url: https://www.kali.org/
 
   kde:
     site: kde.org
-    url: http://www.kde.org/
+    url: https://www.kde.org/
+
+  kde-applicationdata:
+    site: kde.org
+    url: https://www.kde.org/
 
   kernel.org:
     site: kernel.org
-    url: http://www.kernel.org/
+    url: https://www.kernel.org/
 
   linuxmint:
     site: linuxmint.com
-    url: http://www.linuxmint.com/
+    url: https://www.linuxmint.com/
 
   linuxmint-packages:
     site: linuxmint.com
-    url: http://www.linuxmint.com/
+    url: https://www.linuxmint.com/
 
   MacPorts:
     site: macports.org
-    url: http://www.macports.org/
+    url: https://www.macports.org/
 
   mozdev:
     site: mozdev.org
-    url: http://www.mozdev.org/
+    url: https://www.mozdev.org/
 
   mozilla.org:
     site: mozilla.org
-    url: http://www.mozilla.org/
+    url: https://www.mozilla.org/
 
   mysql:
     site: mysql.com
-    url: http://www.mysql.com/
+    url: https://www.mysql.com/
 
   nongnu:
     site: nongnu.org
-    url: http://savannah.nongnu.org/
+    url: https://savannah.nongnu.org/
 
   openoffice:
     site: openoffice.org
-    url: http://www.openoffice.org/
+    url: https://www.openoffice.org/
 
   opensuse:
     site: opensuse.org
-    url: http://www.opensuse.org/
+    url: https://www.opensuse.org/
 
   OpenBSD:
     site: openbsd.org
-    url: http://www.openbsd.org/
+    url: https://www.openbsd.org/
 
   puppylinux:
     site: puppylinux.org
-    url: http://puppylinux.org/
+    url: https://puppylinux.org/
 
   qtproject:
     site: qt-project.org
-    url: http://qt-project.org/
+    url: https://qt-project.org/
 
   racket:
     site: racket-lang.org
-    url: http://racket-lang.org/
+    url: https://racket-lang.org/
+
+  sage:
+    site: sagemath.org
+    url: https://www.sagemath.org/
+
+  saltstack:
+    site: saltstack.com
+    url: https://saltstack.com/
 
   slackware:
     site: slackware.com
-    url: http://www.slackware.com/
+    url: https://www.slackware.com/
+
+  tdf:
+    site: documentfoundation.org
+    url: https://www.documentfoundation.org/
 
   ubuntu:
     site: ubuntu.com
-    url: http://www.ubuntu.com/
+    url: https://www.ubuntu.com/
 
   ubuntu-ports:
     site: ports.ubuntu.com
@@ -173,33 +228,72 @@ directories:
 
   ubuntu-ports-releases:
     site: ports.ubuntu.com
-    url: http://cdimage.ubuntu.com/ports/releases/
+    url: https://cdimage.ubuntu.com/releases/
 
   ubuntu-releases:
     site: releases.ubuntu.com
-    url: http://releases.ubuntu.com/
+    url: https://releases.ubuntu.com/
 
   vlc:
     site: videolan.org
-    url: http://videolan.org/
+    url: https://videolan.org/
 
   xiph:
     site: xiph.org
-    url: http://xiph.org/
+    url: https://xiph.org/
 
   x.org:
     site: x.org
-    url: http://www.x.org/
+    url: https://www.x.org/
 
   xubuntu-releases:
     site: xubuntu.org
-    url: http://www.xubuntu.org/
+    url: https://www.xubuntu.org/
 
   NetBSD:
     site: netbsd.org
-    url: http://www.netbsd.org/
+    url: https://www.netbsd.org/
+
+  pkgsrc:
+    site: pkgsrc.org
+    url: https://pkgsrc.org/
 
   raspberrypi:
     site: raspberrypi.org
-    url: http://www.raspberrypi.org/
+    url: https://www.raspberrypi.org/
 
+  raspbian:
+    site: raspbian.org
+    url: https://www.raspbian.org/
+
+  ipfire:
+    site: ipfire.org
+    url: https://www.ipfire.org/
+
+  manjaro:
+    site: manjaro.org
+    url: https://manjaro.org/
+
+  scientific:
+    site: scientificlinux.org
+    url: https://www.scientificlinux.org/
+
+  mxlinux:
+    site: mxlinux.org
+    url: https://mxlinux.org
+
+  mxlinux-iso:
+    site: mxlinux.org
+    url: https://mxlinux.org
+
+  parabola:
+    site: parabola.nu
+    url: https://www.parabola.nu/
+
+  hyperbola:
+    site: hyperbola.info
+    url: https://www.hyperbola.info/
+
+  trisquel:
+    site: trisquel.info
+    url: https://trisquel.info/

mirror-index/index.mako

@@ -5,16 +5,16 @@
     <link rel="stylesheet" type="text/css" href="index.css" />
     <title>Computer Science Club Mirror</title>
   </head>
-  
+
   <body>
     <div id="logo">
       <a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
     </div>
-    
+
     <div id="listing">
       <table>
         <tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
-	
+
         % for dir in directories:
         <tr>
           <td>
@@ -31,7 +31,7 @@
         </tr>
 
 	% endfor \
-	
+
 	<tr class="total">
 	  <td>Total</td>
 	  <td></td>
@@ -41,7 +41,9 @@
     </div>
 
     <div id="footer">
-      <p>This service is run by the <a href="http://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="http://www.student.math.uwaterloo.ca/~mefcom/">Mathematics Endowment Fund</a><br />and support from the <a href="http://www.cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
+      <p>This service is run by the <a href="https://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="https://uwaterloo.ca/math-endowment-fund/">Mathematics Endowment Fund</a><br />and support from the <a href="https://cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
+      <p>Report an issue or request we mirror a project by contacting <a href="mailto:systems-committee@csclub.uwaterloo.ca">systems-committee@csclub.uwaterloo.ca</a>.</p>
+      <p>Generated (local time): ${generated.strftime("%Y-%m-%d %-I:%M%p")}</p>
     </div>
   </body>
 </html>

mirror-index/make-index.py

@@ -1,10 +1,11 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 """make-index.py
 
 Generates an nice index of the directories from a
 template.
 
 Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
+Modified by: Zachary Seguin <ztseguin@csclub.uwaterloo.ca>
 
 So if you don't like how I did something,
 I'm the person you get to complain to.
@@ -16,37 +17,47 @@ from subprocess import Popen, PIPE
 from optparse import OptionParser
 import yaml, mako.exceptions, webhelpers.html.tags
 from mako.template import Template
+import humanize
+import datetime
+
+def parse_size(s):
+    if s.endswith("T"):
+        return float(s[:-1])*1000*1000*1000*1000
+    elif s.endswith("G"):
+        return float(s[:-1])*1000*1000*1000
+    elif s.endswith("M"):
+        return float(s[:-1])*1000*1000
+    elif s.endswith("K"):
+        return float(s[:-1])*1000
+
+    return float(size)
 
 def reformat_size(size):
-    """Reformats '124M' to '124 MB', et cetera."""
+    return humanize.naturalsize(size)
-    if size[-1].isalpha():
-        return size[:-1] + " " + size[-1] + "B"
-    else:
-        return size
 
 def atomic_write(filename, body):
     """Atomically write to a file by writing a
     temporary file and then moving it to replace
     the desired output file.
-    
+
     This ensures that partial files are never seen
     by clients."""
-    
+
     # generate an appropriate temporary filename
     # in the same directory
     tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
-    
+
     # open the directory so that we can fsync it
     dir = os.open(os.path.realpath(os.path.dirname(filename)), \
                       os.O_DIRECTORY | os.O_RDONLY)
-    
+
     # write to the temporary file
     tmp = open(tmp_filename, 'w')
     print >>tmp, body
     tmp.flush()
     os.fsync(tmp.fileno())
     tmp.close()
-    
+
     # atomically replace the actual file
     os.rename(tmp_filename, filename)
     os.fsync(dir)
@@ -72,17 +83,17 @@ def main():
                       "This is automatically set if the output appears "
                       "to be a character device, not a file.")
     (options, args) = parser.parse_args()
-    
+
     # load config file
     try:
-        config = yaml.load(file(options.config,'r'))
+        config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader)
     except:
         config = None
-    
+
     if not config or type(config) != dict:
         print >>sys.stderr, "Unable to load configuration '%s'." % options.config
         sys.exit(-1)
-    
+
     # determine important variables based on an appropriate order of
     # precedence (command-line flags first, then the config file,
     # then built-in fallbacks)
@@ -90,12 +101,12 @@ def main():
     # fallback value for nonatomic is used so that character devices
     # (e.g. /dev/stdout, /dev/null) are written to in the regular way
     docroot   = options.docroot   or config.get('docroot')
-    duflags   = options.duflags   or config.get('duflags')   or "-h --max-depth=1"
+    duflags   = options.duflags   or config.get('duflags')   or "-h --max-depth=1 --exclude='.~tmp~'"
     output    = options.output    or config.get('output')
     template  = options.template  or config.get("template")  or "index.mako"
     nonatomic = options.nonatomic or config.get("nonatomic") or \
         (os.path.exists(output) and not os.path.isfile(output))
-    
+
     # sanity checks
     if not docroot:
         print >>sys.stderr, "docroot not specified."
@@ -114,47 +125,101 @@ def main():
     elif not os.path.exists(template) or os.path.isdir(template):
         print >>sys.stderr, "template '%s' not found or is a directory." % template
         sys.exit(-1)
-    
+
+    generated = datetime.datetime.now()
+
+    directories = []
+    total_size = 0.0
+
+    ls = os.listdir(docroot)
+
+    for item in ls:
+        if item == "pub":
+            continue
+
+        path = os.path.join(docroot, item)
+
+        # If symlink, then this is a project
+        if os.path.islink(path):
+            dataset = os.readlink(path)[1:]
+
+            # Get the disk space used by that dataset
+            du = Popen(
+                    "/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset,
+                    shell=True, stdout=PIPE, stderr=PIPE).communicate()
+
+            size = "-"
+            if len(du[1].strip()) > 0:
+                sys.stderr.write(du[1])
+                print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item
+                continue
+            else:
+                size = parse_size(du[0].strip())
+                total_size += size
+
+            info = {'dir':item, 'size':reformat_size(size)}
+
+            # use info from config.yaml, if found
+            # otherwise, skip this directory
+            if item in config['exclude']:
+                continue
+            elif item in config['directories']:
+                info.update(config['directories'][item])
+            else:
+                info.update({ 'site': '', 'url': ''  })
+
+            directories.append(info)
+
+    # Sort the directories
+    directories.sort(key=lambda d : d['dir'].lower())
+
     # Call du to compute size
-    du = Popen(
+    #du = Popen(
-        "/usr/bin/du %s %s | /usr/bin/sort -fk2" % (docroot, duflags),
+    #    "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags),
-        shell=True, stdout=PIPE, stderr=PIPE).communicate()
+    #    shell=True, stdout=PIPE, stderr=PIPE).communicate()
-    
+    # du = []
+
     # Check that du executed successfully
     # If there's anything on stderr, send it
     # out our own stderr and terminate.
-    if len(du[1].strip()) > 0:
+    #if len(du[1].strip()) > 0:
-        sys.stderr.write(du[1])
+    #    sys.stderr.write(du[1])
-        print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
+    #    print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
-        sys.exit(-1)
+    #    sys.exit(-1)
-    
+
     # first one should be total, grab its size and format
-    du = du[0].splitlines() # we only care about stdout now
+    #du = du[0].splitlines() # we only care about stdout now
-    total_size = reformat_size(du[0].split(None,2)[0])
+    #total_size = reformat_size(du[0].split(None,2)[0])
-    
+
     # the rest are the sizes we want
-    directories = []
+    # directories = []
-    for line in du[1:]:
+    # total_size = 0
-        (size, dir) = line.split(None, 2)
+
-        dir = os.path.basename(dir)
+    # for line in du:
-        info = {'dir':dir, 'size':reformat_size(size)}
+    #    (size, path) = line.split(None, 2)
-        
+    #	dir = os.path.basename(path)
+    #    total_size += int(size)
+    #    info = {'dir':dir, 'size':reformat_size(int(size))}
+
         # use info from config.yaml, if found
         # otherwise, skip this directory
-        if dir in config['directories']:
+    #    if dir in config['exclude'] or not os.path.isdir(path):
-            info.update(config['directories'][dir])
+    #        continue
-        else:
+    #    elif dir in config['directories']:
-            continue
+    #        info.update(config['directories'][dir])
-        
+    #    else:
-        directories.append(info)
+    #        info.update({ 'site': '', 'url': ''  })
-    
+
+    #    directories.append(info)
+
     # render the template to a string
     body = Template(filename=template).render(
-        total_size=total_size,
+        total_size=reformat_size(total_size),
         directories=directories,
+        generated=generated,
         config=config,
         h=webhelpers.html.tags)
-    
+
     # write the rendered output
     if nonatomic:
         print >>file(output,'w'), body