reorginize ftp and mirror-index

autoautoinstall
Andrew Wang 1 year ago
parent d5b90baecf
commit 07153e8ecc
  1. 6
      todo/README.md
  2. 1
      todo/index/main.yml
  3. 5
      todo/index/templates/README.md
  4. 0
      todo/index/templates/include/favicon.ico
  5. 0
      todo/index/templates/include/header.png
  6. 299
      todo/mirror/mirror-index/config.yaml
  7. 43
      todo/mirror/mirror-index/index.css
  8. 49
      todo/mirror/mirror-index/index.mako
  9. 231
      todo/mirror/mirror-index/make-index.py

@ -1,10 +1,16 @@
## TODO
- modify configs when appropriate
- net.ipv4.ip_forward=1 for both host and vm
- create 3 users
- local (does nothing?)
- mirror (does something)
- push (ssh authorized_keys for pushing)
### Storage
symlink projects from `/mirror/root/.cscmirror` to `/mirror/root`
### dirs of importance
### Merlin
check wiki for more details

@ -0,0 +1 @@
- name: create

@ -7,3 +7,8 @@ make the `/home/mirror` dir
mirror-index/ ->
/home/mirror/mirror-index/
```
```
include/ ->
/mirror/root/include/
```

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

@ -1,299 +0,0 @@
docroot: /mirror/root
duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub
output: /mirror/root/index.html
exclude:
- include
- lost+found
- pub
- uw-courseware
- wics
- mirror-rootfs
- mirror-highpriority
- ups
directories:
alpine:
site: alpinelinux.org
url: https://alpinelinux.org/
almalinux:
site: almalinux.org
url: https://almalinux.org
apache:
site: apache.org
url: https://www.apache.org/
archlinux:
site: archlinux.org
url: https://www.archlinux.org/
centos:
site: centos.org
url: https://www.centos.org/
ceph:
site: ceph.io
url: https://ceph.io
CPAN:
site: cpan.org
url: https://www.cpan.org/
CRAN:
site: r-project.org
url: https://cran.r-project.org/
csclub:
site: csclub.uwaterloo.ca
url: https://csclub.uwaterloo.ca/media/
CTAN:
site: ctan.org
url: https://www.ctan.org/
cygwin:
site: cygwin.com
url: https://www.cygwin.com/
damnsmalllinux:
site: damnsmalllinux.org
url: https://www.damnsmalllinux.org/
debian:
site: debian.org
url: https://www.debian.org/
debian-backports:
site: backports.debian.org
url: https://backports.debian.org/
debian-cd:
site: debian.org
url: https://www.debian.org/CD/
debian-multimedia:
site: debian-multimedia.org
url: https://www.debian-multimedia.org/
debian-ports:
site: debian-ports.org
url: https://www.debian-ports.org/
debian-security:
site: debian.org
url: https://www.debian.org/security/
debian-volatile:
site: debian.org
url: https://www.debian.org/volatile/
eclipse:
site: eclipse.org
url: https://www.eclipse.org/
emacsconf:
site: emacsconf.org
url: https://emacsconf.org
# this is no longer a project
# emdebian:
# site: emdebian.org
# url: https://www.emdebian.org/
fedora:
site: fedoraproject.org
url: https://www.fedoraproject.org/
FreeBSD:
site: freebsd.org
url: https://www.freebsd.org/
gentoo-distfiles:
site: gentoo.org
url: https://www.gentoo.org/
gentoo-portage:
site: gentoo.org
url: https://www.gentoo.org/
gnome:
site: gnome.org
url: https://www.gnome.org/
gnu:
site: gnu.org
url: https://www.gnu.org/
gutenberg:
site: gutenberg.org
url: https://www.gutenberg.org/
kali:
site: kali.org
url: https://www.kali.org/
kali-images:
site: kali.org
url: https://www.kali.org/
kde:
site: kde.org
url: https://www.kde.org/
kde-applicationdata:
site: kde.org
url: https://www.kde.org/
kernel.org:
site: kernel.org
url: https://www.kernel.org/
linuxmint:
site: linuxmint.com
url: https://www.linuxmint.com/
linuxmint-packages:
site: linuxmint.com
url: https://www.linuxmint.com/
MacPorts:
site: macports.org
url: https://www.macports.org/
mozdev:
site: mozdev.org
url: https://www.mozdev.org/
mozilla.org:
site: mozilla.org
url: https://www.mozilla.org/
mysql:
site: mysql.com
url: https://www.mysql.com/
nongnu:
site: nongnu.org
url: https://savannah.nongnu.org/
openoffice:
site: openoffice.org
url: https://www.openoffice.org/
opensuse:
site: opensuse.org
url: https://www.opensuse.org/
OpenBSD:
site: openbsd.org
url: https://www.openbsd.org/
puppylinux:
site: puppylinux.org
url: https://puppylinux.org/
qtproject:
site: qt-project.org
url: https://qt-project.org/
racket:
site: racket-lang.org
url: https://racket-lang.org/
sage:
site: sagemath.org
url: https://www.sagemath.org/
saltstack:
site: saltstack.com
url: https://saltstack.com/
slackware:
site: slackware.com
url: https://www.slackware.com/
tdf:
site: documentfoundation.org
url: https://www.documentfoundation.org/
ubuntu:
site: ubuntu.com
url: https://www.ubuntu.com/
ubuntu-ports:
site: ports.ubuntu.com
url: http://ports.ubuntu.com/ubuntu-ports/
ubuntu-ports-releases:
site: ports.ubuntu.com
url: https://cdimage.ubuntu.com/releases/
ubuntu-releases:
site: releases.ubuntu.com
url: https://releases.ubuntu.com/
vlc:
site: videolan.org
url: https://videolan.org/
xiph:
site: xiph.org
url: https://xiph.org/
x.org:
site: x.org
url: https://www.x.org/
xubuntu-releases:
site: xubuntu.org
url: https://www.xubuntu.org/
NetBSD:
site: netbsd.org
url: https://www.netbsd.org/
pkgsrc:
site: pkgsrc.org
url: https://pkgsrc.org/
raspberrypi:
site: raspberrypi.org
url: https://www.raspberrypi.org/
raspbian:
site: raspbian.org
url: https://www.raspbian.org/
ipfire:
site: ipfire.org
url: https://www.ipfire.org/
manjaro:
site: manjaro.org
url: https://manjaro.org/
scientific:
site: scientificlinux.org
url: https://www.scientificlinux.org/
mxlinux:
site: mxlinux.org
url: https://mxlinux.org
mxlinux-iso:
site: mxlinux.org
url: https://mxlinux.org
parabola:
site: parabola.nu
url: https://www.parabola.nu/
hyperbola:
site: hyperbola.info
url: https://www.hyperbola.info/
trisquel:
site: trisquel.info
url: https://trisquel.info/

@ -1,43 +0,0 @@
img {
border: none;
}
html {
margin:0.5ex;
font-family: sans-serif;
font-size: 110%;
}
p {
margin: 1ex 0;
}
table {
border-collapse: collapse;
text-align: left;
width: 100%;
}
td {
border-top: 1px solid #aaa;
}
th, td {
padding: .4ex 2em .4ex 0;
}
h1 {
font-size: 110%;
}
#logo {
width: 100%;
text-align: center;
margin-bottom:1em;
}
#footer {
margin: 2em auto 0 auto;
width: 75%;
font-size: 70%;
text-align: center;
}
body {
max-width: 40em;
margin-top:0;
padding-top:0;
}
tr :last-child { text-align: right; }

@ -1,49 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="stylesheet" type="text/css" href="index.css" />
<title>Computer Science Club Mirror</title>
</head>
<body>
<div id="logo">
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
</div>
<div id="listing">
<table>
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
% for dir in directories:
<tr>
<td>
${h.link_to(dir['dir']+'/', '/'+dir['dir']+'/')}
</td>
<td>
% if 'site' in dir:
${h.link_to(dir['site'], dir['url'])}
% endif
</td>
<td>${dir['size'] | h}</td>
</tr>
% endfor \
<tr class="total">
<td>Total</td>
<td></td>
<td>${total_size}</td>
</tr>
</table>
</div>
<div id="footer">
<p>This service is run by the <a href="https://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="https://uwaterloo.ca/math-endowment-fund/">Mathematics Endowment Fund</a><br />and support from the <a href="https://cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
<p>Report an issue or request we mirror a project by contacting <a href="mailto:systems-committee@csclub.uwaterloo.ca">systems-committee@csclub.uwaterloo.ca</a>.</p>
<p>Generated (local time): ${generated.strftime("%Y-%m-%d %-I:%M%p")}</p>
</div>
</body>
</html>

@ -1,231 +0,0 @@
#!/usr/bin/env python2
"""make-index.py
Generates an nice index of the directories from a
template.
Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
Modified by: Zachary Seguin <ztseguin@csclub.uwaterloo.ca>
So if you don't like how I did something,
I'm the person you get to complain to.
Please be gentle.
"""
import os, sys, time
from subprocess import Popen, PIPE
from optparse import OptionParser
import yaml, mako.exceptions, webhelpers.html.tags
from mako.template import Template
import humanize
import datetime
def parse_size(s):
if s.endswith("T"):
return float(s[:-1])*1000*1000*1000*1000
elif s.endswith("G"):
return float(s[:-1])*1000*1000*1000
elif s.endswith("M"):
return float(s[:-1])*1000*1000
elif s.endswith("K"):
return float(s[:-1])*1000
return float(size)
def reformat_size(size):
return humanize.naturalsize(size)
def atomic_write(filename, body):
"""Atomically write to a file by writing a
temporary file and then moving it to replace
the desired output file.
This ensures that partial files are never seen
by clients."""
# generate an appropriate temporary filename
# in the same directory
tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
# open the directory so that we can fsync it
dir = os.open(os.path.realpath(os.path.dirname(filename)), \
os.O_DIRECTORY | os.O_RDONLY)
# write to the temporary file
tmp = open(tmp_filename, 'w')
print >>tmp, body
tmp.flush()
os.fsync(tmp.fileno())
tmp.close()
# atomically replace the actual file
os.rename(tmp_filename, filename)
os.fsync(dir)
os.close(dir)
def main():
# accept command-line arguments
parser = OptionParser()
parser.add_option("-c", "--config", dest="config", default="config.yaml",
help="configuration file to be used", metavar="FILE")
parser.add_option("-D", "--docroot", dest="docroot",
help="directory to be scanned", metavar="DIR")
parser.add_option("-F", "--duflags", dest="duflags",
help="flags to be passed to du, replaces any in config")
parser.add_option("-o", "--output", dest="output", metavar="FILE",
help="file to which index page will be written. "
"Use /dev/stdout to send to standard out.")
parser.add_option("-t", "--template", dest="template",
help="Mako template to render", metavar="FILE")
parser.add_option("--nonatomic", dest="nonatomic", action="store_true",
default=False, help="write the output to the path "
"given without creating a temporary file in between. "
"This is automatically set if the output appears "
"to be a character device, not a file.")
(options, args) = parser.parse_args()
# load config file
try:
config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader)
except:
config = None
if not config or type(config) != dict:
print >>sys.stderr, "Unable to load configuration '%s'." % options.config
sys.exit(-1)
# determine important variables based on an appropriate order of
# precedence (command-line flags first, then the config file,
# then built-in fallbacks)
#
# fallback value for nonatomic is used so that character devices
# (e.g. /dev/stdout, /dev/null) are written to in the regular way
docroot = options.docroot or config.get('docroot')
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1 --exclude='.~tmp~'"
output = options.output or config.get('output')
template = options.template or config.get("template") or "index.mako"
nonatomic = options.nonatomic or config.get("nonatomic") or \
(os.path.exists(output) and not os.path.isfile(output))
# sanity checks
if not docroot:
print >>sys.stderr, "docroot not specified."
print >>sys.stderr, "Define it in the config file or pass -D on the command line."
sys.exit(-1)
elif not output:
print >>sys.stderr, "output not specified."
print >>sys.stderr, "Define it in the config file or pass -o on the command line."
elif not config.get('directories'):
print >>sys.stderr, "directories not specified."
print >>sys.stderr, "Define it in the config file."
sys.exit(-1)
elif not os.path.isdir(docroot):
print >>sys.stderr, "docroot '%s' not found or not a directory." % docroot
sys.exit(-1)
elif not os.path.exists(template) or os.path.isdir(template):
print >>sys.stderr, "template '%s' not found or is a directory." % template
sys.exit(-1)
generated = datetime.datetime.now()
directories = []
total_size = 0.0
ls = os.listdir(docroot)
for item in ls:
if item == "pub":
continue
path = os.path.join(docroot, item)
# If symlink, then this is a project
if os.path.islink(path):
dataset = os.readlink(path)[1:]
# Get the disk space used by that dataset
du = Popen(
"/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset,
shell=True, stdout=PIPE, stderr=PIPE).communicate()
size = "-"
if len(du[1].strip()) > 0:
sys.stderr.write(du[1])
print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item
continue
else:
size = parse_size(du[0].strip())
total_size += size
info = {'dir':item, 'size':reformat_size(size)}
# use info from config.yaml, if found
# otherwise, skip this directory
if item in config['exclude']:
continue
elif item in config['directories']:
info.update(config['directories'][item])
else:
info.update({ 'site': '', 'url': '' })
directories.append(info)
# Sort the directories
directories.sort(key=lambda d : d['dir'].lower())
# Call du to compute size
#du = Popen(
# "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags),
# shell=True, stdout=PIPE, stderr=PIPE).communicate()
# du = []
# Check that du executed successfully
# If there's anything on stderr, send it
# out our own stderr and terminate.
#if len(du[1].strip()) > 0:
# sys.stderr.write(du[1])
# print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
# sys.exit(-1)
# first one should be total, grab its size and format
#du = du[0].splitlines() # we only care about stdout now
#total_size = reformat_size(du[0].split(None,2)[0])
# the rest are the sizes we want
# directories = []
# total_size = 0
# for line in du:
# (size, path) = line.split(None, 2)
# dir = os.path.basename(path)
# total_size += int(size)
# info = {'dir':dir, 'size':reformat_size(int(size))}
# use info from config.yaml, if found
# otherwise, skip this directory
# if dir in config['exclude'] or not os.path.isdir(path):
# continue
# elif dir in config['directories']:
# info.update(config['directories'][dir])
# else:
# info.update({ 'site': '', 'url': '' })
# directories.append(info)
# render the template to a string
body = Template(filename=template).render(
total_size=reformat_size(total_size),
directories=directories,
generated=generated,
config=config,
h=webhelpers.html.tags)
# write the rendered output
if nonatomic:
print >>file(output,'w'), body
else:
atomic_write(output, body)
if __name__ == "__main__":
main()
Loading…
Cancel
Save