mirror-index: a rewrite

- now supports showing sync status, dark mode, and a new layout
This commit is contained in:
Yiao Shen 2023-06-20 23:56:39 -04:00 committed by Mirror
parent c048fc5e04
commit 688a8f44c4
35 changed files with 1497 additions and 611 deletions

4
mirror-index/.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
.hugo_build.lock
resources
public
data

View File

@ -0,0 +1,44 @@
$sansSerifFonts : 'IBM Plex Sans', 'Hiragino Sans GB', 'Noto Sans CJK SC', 'Microsoft YaHei', sans-serif
$serifFonts : 'Lora', 'Noto Serif CJK SC', 'Songti SC', 'Microsoft YaHei', serif
$monospaceFonts : 'IBM Plex Mono', 'Microsoft YaHei', monospace
$singleColWidth: 65rem
$smallScreenWidth: 45rem
$reallySmallScreenWidth: 35rem
// Colors
$grey10: #faf9f8
$grey20: #f3f2f1
$grey30: #edebe9
$grey40: #e1dfdd
$grey50: #d2d0ce
$grey60: #c8c6c4
$grey80: #b3b0ad
$grey100: #979593
$grey130: #605e5c
$grey150: #3b3a39
$grey160: #323130
$grey200: #1b1a19
$grey220: #11100f
$blue: #0078d4
$blueLight: #00bcf2
$blueBright: #c7e0f4
$blueBrighter: #eff6fc
$blueDark: #005a9e
$blueDarker: #004578
$blueDarkest: #003b6b
$yellow: #fce100
$yellowLight: #fffbd6
$yellowDark: #997825
$yellowDarker: #4d3c12
$orange: #ca5010
$orangeLight: #d06228
$red: #a4262c
$redLight: #c50f1f
$redOrange: #da3b01
@import "partials/normalize"
@import "partials/main.sass"
@import "partials/news.sass"
@import "partials/color.sass"
@import "partials/color-dark.sass"

View File

@ -0,0 +1,31 @@
@media (prefers-color-scheme: dark)
html
background-color: $grey220
color: $grey10
a
color: $blueLight
#nav
.title
.mef
color: $grey50
.links
a
color: $grey10
.link:hover
color: $blueLight
.current
color: $blueLight
&:hover
color: $blueLight
main
.projects
th, td
border-color: $grey130
.button:hover
background-color: $grey150
.news-article
border-color: $grey100
footer
color: $grey80

View File

@ -0,0 +1,45 @@
a
color: $blue
#nav
.title
.site-name
background-color: #181818
color: #18f018
.mef
color: $grey130
.links
a
// Reset global rule
color: black
.link
transition: 0.1s color
.link:hover
color: $blue
.current
color: $blue
&:hover
color: $blue
main
.projects
th, td
border-color: $grey50
.table-header
th, td
border-color: $blueLight
#refresh-indicator
color: $blue
.syncing.badge
color: white
background-color: $blue
.error.badge
color: white
background-color: $redOrange
.button:hover
background-color: $grey50
.news-article
border-color: $grey50
footer
color: $grey200

View File

@ -0,0 +1,222 @@
body
font-family: $sansSerifFonts
font-size: 16px
line-height: 1.45
hyphens: auto
text-rendering: optimizelegibility
text-size-adjust: none
html
scroll-behavior: smooth
a
text-decoration: none
#center
margin: 0 auto
margin-top: 1.5em
padding: 0 .7em
max-width: 75em
min-height: 90vh
position: relative
display: flex
flex-direction: column
#left
width: 65%
display: block
float: left
overflow-x: scroll
#right
width: 32.5%
margin-left: 2.5%
display: block
float: left
@media screen and (max-width: $singleColWidth)
max-width: $singleColWidth
padding: 0 0.25em
#left
width: 100%
display: reset
#right
width: 100%
display: reset
margin-left: 0
float: reset
// And set a top margin
margin-top: 2em
#offcenter
margin: 3em auto 0 auto
max-width: 60em
display: flex
flex-direction: column
position: relative
min-height: 90vh
.container-content
min-width: 40em
padding: 0 0 0 15em
.left
position: absolute
right: 100%
padding: 0 1em
width: 14em
@media screen and (max-width: $singleColWidth)
//margin: 0 auto
padding: 0 .7em
max-width: 50em
.container-content
min-width: 5em
padding: 0
.left
position: relative
right: 0
padding: 0
width: 100%
.hide-on-single-col
@media screen and (max-width: $singleColWidth)
display: none
#nav
position: relative
margin-bottom: 2em
.logo
display: inline-block
vertical-align: middle
height: 4.5em
.title
display: inline-block
margin-left: 0.75em
// Align with the logo
// Set the height to be the same
height: 4em
font-size: 1.15em
line-height: 1.4
// And then apply absolute position
position: absolute
bottom: 0
.club-name, .site-name, .mef
font-size: 1em
line-height: 1.4
margin: 0
display: inline-block
font-weight: 500
.site-name
font-family: $monospaceFonts
font-weight: 400
padding: 0 0.25em
.mef
font-size: 0.9em
font-weight: 400
img
height: 1em
margin-bottom: 0.22em
vertical-align: bottom
displya: inline-block
// Optimize effect on phones
@media screen and (max-width: $smallScreenWidth)
margin-bottom: 2em
.logo
height: 3em
.title
height: 3em
font-size: 1.05em
padding-bottom: 1em
.site-name
font-size: 0.95em
@media screen and (max-width: $reallySmallScreenWidth)
margin-bottom: 1em
.logo
height: 3em
display: block
margin-bottom: 0.5em
.title
display: block
position: relative
margin: 0
padding: 0
height: auto
br
display: none
.club-name, .site-name, .mef
display: inline-block
.links
position: absolute
height: 4.5em
right: 0
bottom: 0
margin: 0
vertical-align: bottom
list-style-type: none
li
float: left
.link
text-align: center
display: block
padding: em
line-height: 1
font-size: 1.1em
padding: 1.2em 1em
margin: 0.4em 0
text-decoration: none
.current
font-weight: 600
border-bottom: 2px solid
main
.projects
text-align: left
// Fill width
width: 100%
// Table styles
border-collapse: collapse
tr
line-height: 1.75
th, td
border: 0
border-top: 1px
border-style: solid
.table-header
th, td
border-top: 0
border-bottom: 2px solid
border-collapse: collapse
// #refresh-indicator
#refresh-indicator
transition: all .2s
.summary
font-weight: 500
.project-size
text-align: right
min-width: 5em
.news
h2
margin: 0.1em 0 0.2em 0
font-size: 1.2em
font-weight: 500
p
margin: 0.1em 0 0.2em 0
ul, dl
list-style-type: none
padding: 0
margin: 0.5em 0 1em 0
.news-date
font-weight: 500
user-select: none
dt
font-weight: 500
dd
margin-left: 1em
footer
font-size: 0.9em
margin-top: 1em
.badge
font-size: 0.8em
font-weight: 400
opacity: 0.8
padding: 0.05em 0.2em
border-radius: 3px
white-space: nowrap

View File

@ -0,0 +1,27 @@
main
// Single
.left
.button
padding: 0.25em
transition: 0.1s background-color
line-height: 2em
display: block
margin: 0 5%
width: 90%
font-size: 1.05em
tt
margin-right: .5em
article
h1
margin-top: 0
// List
.news-list
p
margin: 0.5em 0
.news-article
padding-bottom: .5em
border-bottom: 1px solid $grey100
.news-title
margin: 0 0 0.2em 0
font-size: 1.35em

View File

@ -0,0 +1,349 @@
/*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */
/* Document
========================================================================== */
/**
* 1. Correct the line height in all browsers.
* 2. Prevent adjustments of font size after orientation changes in iOS.
*/
html {
line-height: 1.15; /* 1 */
-webkit-text-size-adjust: 100%; /* 2 */
}
/* Sections
========================================================================== */
/**
* Remove the margin in all browsers.
*/
body {
margin: 0;
}
/**
* Render the `main` element consistently in IE.
*/
main {
display: block;
}
/**
* Correct the font size and margin on `h1` elements within `section` and
* `article` contexts in Chrome, Firefox, and Safari.
*/
h1 {
font-size: 2em;
margin: 0.67em 0;
}
/* Grouping content
========================================================================== */
/**
* 1. Add the correct box sizing in Firefox.
* 2. Show the overflow in Edge and IE.
*/
hr {
box-sizing: content-box; /* 1 */
height: 0; /* 1 */
overflow: visible; /* 2 */
}
/**
* 1. Correct the inheritance and scaling of font size in all browsers.
* 2. Correct the odd `em` font sizing in all browsers.
*/
pre {
font-family: monospace, monospace; /* 1 */
font-size: 1em; /* 2 */
}
/* Text-level semantics
========================================================================== */
/**
* Remove the gray background on active links in IE 10.
*/
a {
background-color: transparent;
}
/**
* 1. Remove the bottom border in Chrome 57-
* 2. Add the correct text decoration in Chrome, Edge, IE, Opera, and Safari.
*/
abbr[title] {
border-bottom: none; /* 1 */
text-decoration: underline; /* 2 */
text-decoration: underline dotted; /* 2 */
}
/**
* Add the correct font weight in Chrome, Edge, and Safari.
*/
b,
strong {
font-weight: bolder;
}
/**
* 1. Correct the inheritance and scaling of font size in all browsers.
* 2. Correct the odd `em` font sizing in all browsers.
*/
code,
kbd,
samp {
font-family: monospace, monospace; /* 1 */
font-size: 1em; /* 2 */
}
/**
* Add the correct font size in all browsers.
*/
small {
font-size: 80%;
}
/**
* Prevent `sub` and `sup` elements from affecting the line height in
* all browsers.
*/
sub,
sup {
font-size: 75%;
line-height: 0;
position: relative;
vertical-align: baseline;
}
sub {
bottom: -0.25em;
}
sup {
top: -0.5em;
}
/* Embedded content
========================================================================== */
/**
* Remove the border on images inside links in IE 10.
*/
img {
border-style: none;
}
/* Forms
========================================================================== */
/**
* 1. Change the font styles in all browsers.
* 2. Remove the margin in Firefox and Safari.
*/
button,
input,
optgroup,
select,
textarea {
font-family: inherit; /* 1 */
font-size: 100%; /* 1 */
line-height: 1.15; /* 1 */
margin: 0; /* 2 */
}
/**
* Show the overflow in IE.
* 1. Show the overflow in Edge.
*/
button,
input { /* 1 */
overflow: visible;
}
/**
* Remove the inheritance of text transform in Edge, Firefox, and IE.
* 1. Remove the inheritance of text transform in Firefox.
*/
button,
select { /* 1 */
text-transform: none;
}
/**
* Correct the inability to style clickable types in iOS and Safari.
*/
button,
[type="button"],
[type="reset"],
[type="submit"] {
-webkit-appearance: button;
}
/**
* Remove the inner border and padding in Firefox.
*/
button::-moz-focus-inner,
[type="button"]::-moz-focus-inner,
[type="reset"]::-moz-focus-inner,
[type="submit"]::-moz-focus-inner {
border-style: none;
padding: 0;
}
/**
* Restore the focus styles unset by the previous rule.
*/
button:-moz-focusring,
[type="button"]:-moz-focusring,
[type="reset"]:-moz-focusring,
[type="submit"]:-moz-focusring {
outline: 1px dotted ButtonText;
}
/**
* Correct the padding in Firefox.
*/
fieldset {
padding: 0.35em 0.75em 0.625em;
}
/**
* 1. Correct the text wrapping in Edge and IE.
* 2. Correct the color inheritance from `fieldset` elements in IE.
* 3. Remove the padding so developers are not caught out when they zero out
* `fieldset` elements in all browsers.
*/
legend {
box-sizing: border-box; /* 1 */
color: inherit; /* 2 */
display: table; /* 1 */
max-width: 100%; /* 1 */
padding: 0; /* 3 */
white-space: normal; /* 1 */
}
/**
* Add the correct vertical alignment in Chrome, Firefox, and Opera.
*/
progress {
vertical-align: baseline;
}
/**
* Remove the default vertical scrollbar in IE 10+.
*/
textarea {
overflow: auto;
}
/**
* 1. Add the correct box sizing in IE 10.
* 2. Remove the padding in IE 10.
*/
[type="checkbox"],
[type="radio"] {
box-sizing: border-box; /* 1 */
padding: 0; /* 2 */
}
/**
* Correct the cursor style of increment and decrement buttons in Chrome.
*/
[type="number"]::-webkit-inner-spin-button,
[type="number"]::-webkit-outer-spin-button {
height: auto;
}
/**
* 1. Correct the odd appearance in Chrome and Safari.
* 2. Correct the outline style in Safari.
*/
[type="search"] {
-webkit-appearance: textfield; /* 1 */
outline-offset: -2px; /* 2 */
}
/**
* Remove the inner padding in Chrome and Safari on macOS.
*/
[type="search"]::-webkit-search-decoration {
-webkit-appearance: none;
}
/**
* 1. Correct the inability to style clickable types in iOS and Safari.
* 2. Change font properties to `inherit` in Safari.
*/
::-webkit-file-upload-button {
-webkit-appearance: button; /* 1 */
font: inherit; /* 2 */
}
/* Interactive
========================================================================== */
/*
* Add the correct display in Edge, IE 10+, and Firefox.
*/
details {
display: block;
}
/*
* Add the correct display in all browsers.
*/
summary {
display: list-item;
}
/* Misc
========================================================================== */
/**
* Add the correct display in IE 10+.
*/
template {
display: none;
}
/**
* Add the correct display in IE 10.
*/
[hidden] {
display: none;
}

13
mirror-index/build.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
# Build static dependencies
# Requires: postcss-cli, autoprefixer, minify (all from npm)
# I recommend yarn global but do whatever you want
# Build auto-prefix-ed, minified CSS
sass assets/csc-mirror.sass | postcss --use autoprefixer | minify --css > ./static/include/mirror-index-ng.css
# Compress CSS & JS assets
# To be used alongside nginx's gzip-static module
gzip -c ./static/include/htmx.min.js > ./static/include/htmx.min.js.gz
gzip -c ./static/include/mirror-index-ng.css > ./static/include/mirror-index-ng.css.gz
gzip -c ./static/include/csc-logo.svg > ./static/include/csc-logo.svg.gz

3
mirror-index/config.toml Normal file
View File

@ -0,0 +1,3 @@
baseURL = 'https://mirror.csclub.uwaterloo.ca/'
languageCode = 'en-us'
title = 'Computer Science Club Open Source Software Mirror'

View File

@ -1,288 +0,0 @@
docroot: /mirror/root
duflags: --max-depth=0 --dereference-args --exclude=/mirror/root/pub
output: /mirror/root/index.html
exclude:
- archlinux-test
- include
- lost+found
- pub
- uw-courseware
- wics
- mirror-rootfs
- mirror-highpriority
- ups
directories:
alpine:
site: alpinelinux.org
url: https://alpinelinux.org/
almalinux:
site: almalinux.org
url: https://almalinux.org
apache:
site: apache.org
url: https://www.apache.org/
archlinux:
site: archlinux.org
url: https://www.archlinux.org/
archlinuxarm:
site: archlinuxarm.org
url: https://archlinuxarm.org/
artixlinux:
site: artixlinux.org
url: https://artixlinux.org/
centos:
site: centos.org
url: https://www.centos.org/
ceph:
site: ceph.io
url: https://ceph.io
CPAN:
site: cpan.org
url: https://www.cpan.org/
CRAN:
site: r-project.org
url: https://cran.r-project.org/
csclub:
site: csclub.uwaterloo.ca
url: https://csclub.uwaterloo.ca/media/
CTAN:
site: ctan.org
url: https://www.ctan.org/
cygwin:
site: cygwin.com
url: https://www.cygwin.com/
damnsmalllinux:
site: damnsmalllinux.org
url: https://www.damnsmalllinux.org/
debian:
site: debian.org
url: https://www.debian.org/
debian-cd:
site: debian.org
url: https://www.debian.org/CD/
debian-multimedia:
site: debian-multimedia.org
url: https://www.debian-multimedia.org/
debian-security:
site: debian.org
url: https://www.debian.org/security/
debian-volatile:
site: debian.org
url: https://www.debian.org/volatile/
eclipse:
site: eclipse.org
url: https://www.eclipse.org/
emacsconf:
site: emacsconf.org
url: https://emacsconf.org
# this is no longer a project
# emdebian:
# site: emdebian.org
# url: https://www.emdebian.org/
fedora:
site: fedoraproject.org
url: https://www.fedoraproject.org/
FreeBSD:
site: freebsd.org
url: https://www.freebsd.org/
gentoo-distfiles:
site: gentoo.org
url: https://www.gentoo.org/
gentoo-portage:
site: gentoo.org
url: https://www.gentoo.org/
gnome:
site: gnome.org
url: https://www.gnome.org/
gnu:
site: gnu.org
url: https://www.gnu.org/
gutenberg:
site: gutenberg.org
url: https://www.gutenberg.org/
kali:
site: kali.org
url: https://www.kali.org/
kali-images:
site: kali.org
url: https://www.kali.org/
kde:
site: kde.org
url: https://www.kde.org/
kde-applicationdata:
site: kde.org
url: https://www.kde.org/
kernel.org:
site: kernel.org
url: https://www.kernel.org/
linuxmint:
site: linuxmint.com
url: https://www.linuxmint.com/
linuxmint-packages:
site: linuxmint.com
url: https://www.linuxmint.com/
MacPorts:
site: macports.org
url: https://www.macports.org/
mozdev:
site: mozdev.org
url: https://www.mozdev.org/
mozilla.org:
site: mozilla.org
url: https://www.mozilla.org/
mysql:
site: mysql.com
url: https://www.mysql.com/
nongnu:
site: nongnu.org
url: https://savannah.nongnu.org/
openoffice:
site: openoffice.org
url: https://www.openoffice.org/
opensuse:
site: opensuse.org
url: https://www.opensuse.org/
puppylinux:
site: puppylinux.org
url: https://puppylinux.org/
racket:
site: racket-lang.org
url: https://racket-lang.org/
sage:
site: sagemath.org
url: https://www.sagemath.org/
saltstack:
site: saltstack.com
url: https://saltstack.com/
slackware:
site: slackware.com
url: https://www.slackware.com/
tdf:
site: documentfoundation.org
url: https://www.documentfoundation.org/
ubuntu:
site: ubuntu.com
url: https://www.ubuntu.com/
ubuntu-ports:
site: ports.ubuntu.com
url: http://ports.ubuntu.com/ubuntu-ports/
ubuntu-ports-releases:
site: ports.ubuntu.com
url: https://cdimage.ubuntu.com/releases/
ubuntu-releases:
site: releases.ubuntu.com
url: https://releases.ubuntu.com/
vlc:
site: videolan.org
url: https://videolan.org/
xiph:
site: xiph.org
url: https://xiph.org/
x.org:
site: x.org
url: https://www.x.org/
xubuntu-releases:
site: xubuntu.org
url: https://www.xubuntu.org/
NetBSD:
site: netbsd.org
url: https://www.netbsd.org/
raspberrypi:
site: raspberrypi.org
url: https://www.raspberrypi.org/
raspbian:
site: raspbian.org
url: https://www.raspbian.org/
ipfire:
site: ipfire.org
url: https://www.ipfire.org/
manjaro:
site: manjaro.org
url: https://manjaro.org/
mxlinux:
site: mxlinux.org
url: https://mxlinux.org
mxlinux-iso:
site: mxlinux.org
url: https://mxlinux.org
parabola:
site: parabola.nu
url: https://www.parabola.nu/
hyperbola:
site: hyperbola.info
url: https://www.hyperbola.info/
trisquel:
site: trisquel.info
url: https://trisquel.info/
termux:
site: termux.dev
url: https://termux.dev/

View File

@ -0,0 +1,8 @@
+++
title = "New mirror index page"
date = "2023-05-04"
+++
We've updated the mirror index page to include more detailed synchronization status information.
If you experienced any usability issues due to browser compatibility, please let us know on [syscom@csclub.uwaterloo.ca](syscom@csclub.uwaterloo.ca).

View File

@ -0,0 +1,6 @@
+++
# This is only a placeholder file to force Hugo to render projects_table.html
# Don't remove me
type = "page"
layout = "project_table"
+++

20
mirror-index/deploy.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
# Deploy mirror index page
# Note: This script is only used for deployment (commonly used in cron)
# To build dependencies and do initial setup, run setup.sh
set -euo pipefail
# Generate json
cd ./synctask2project/
./synctask2project /mirror/root/sync.json
# Run page generation
cd ../
cp /mirror/root/sync.json ./data/
hugo > /dev/null
# Copy to mirror root
cp public/index.html /mirror/root/index.html
cp public/project_table/index.html /mirror/root/include/project_table.html
cp -r public/news /mirror/root/

View File

@ -1,43 +0,0 @@
img {
border: none;
}
html {
margin:0.5ex;
font-family: sans-serif;
font-size: 110%;
}
p {
margin: 1ex 0;
}
table {
border-collapse: collapse;
text-align: left;
width: 100%;
}
td {
border-top: 1px solid #aaa;
}
th, td {
padding: .4ex 2em .4ex 0;
}
h1 {
font-size: 110%;
}
#logo {
width: 100%;
text-align: center;
margin-bottom:1em;
}
#footer {
margin: 2em auto 0 auto;
width: 75%;
font-size: 70%;
text-align: center;
}
body {
max-width: 40em;
margin-top:0;
padding-top:0;
}
tr :last-child { text-align: right; }

View File

@ -1,49 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="stylesheet" type="text/css" href="index.css" />
<title>Computer Science Club Mirror</title>
</head>
<body>
<div id="logo">
<a href="/"><img src="/include/header.png" alt="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" title="Computer Science Club Mirror - The University of Waterloo - Funded by MEF" /></a>
</div>
<div id="listing">
<table>
<tr><th>Directory</th><th>Project Site</th><th>Size</th></tr>
% for dir in directories:
<tr>
<td>
${h.link_to(dir['dir']+'/', '/'+dir['dir']+'/')}
</td>
<td>
% if 'site' in dir:
${h.link_to(dir['site'], dir['url'])}
% endif
</td>
<td>${dir['size'] | h}</td>
</tr>
% endfor \
<tr class="total">
<td>Total</td>
<td></td>
<td>${total_size}</td>
</tr>
</table>
</div>
<div id="footer">
<p>This service is run by the <a href="https://csclub.uwaterloo.ca/">Computer Science Club of the University of Waterloo</a>.<br />It is made possible by funding from the <a href="https://uwaterloo.ca/math-endowment-fund/">Mathematics Endowment Fund</a><br />and support from the <a href="https://cs.uwaterloo.ca">David R. Cheriton School of Computer Science</a>.</p>
<p>Report an issue or request we mirror a project by contacting <a href="mailto:systems-committee@csclub.uwaterloo.ca">systems-committee@csclub.uwaterloo.ca</a>.</p>
<p>Generated (local time): ${generated.strftime("%Y-%m-%d %-I:%M%p")}</p>
</div>
</body>
</html>

View File

@ -0,0 +1,40 @@
<!DOCTYPE html>
<html lang="{{ .Site.Language.Lang }}">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>
{{- block "title" . -}}
{{ .Site.Title }}
{{- end -}}
</title>
{{/* Some common tags */}}
<meta name="renderer" content="webkit">
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
<link rel="canonical" href="{{ .Permalink }}" />
<meta property="og:site_name" content="{{ .Site.Title }}" />
<meta property="og:image" content="/include/csc-logo.svg" />
<meta property="og:url" content="{{ .Permalink }}" />
<link rel="icon" href="/include/csc-logo.png">
<link rel="mask-icon" href="/include/csc-logo.svg">
<link name="theme-color" content="#00bcf2">
{{- block "meta" . -}}
<meta property="og:title" content="{{ .Title }} - {{ .Site.Title }}" />
<meta property="og:type" content="website" />
<meta property="og:description" content="{{ .Description }}" />
<meta name="description" content="{{ .Description }}" />
{{- end -}}
<link rel ="stylesheet" href="/include/mirror-index-ng.css">
{{- block "stylesheet" . -}}
{{- end -}}
</head>
<body>
{{- block "main" . -}}
{{- end -}}
</body>
</html>

View File

@ -0,0 +1,42 @@
{{- define "title" -}}
{{ .Title }} - {{ .Site.Title }}
{{- end -}}
{{- define "main" -}}
<div id="center">
<header id="nav">
<img class="logo" src="/include/csc-logo.svg" alt="CSC Logo">
<div class="title">
<h1 class="club-name">University of Waterloo <br> Computer Science Club</h1>
<h2 class="site-name">Open Source Software Mirror</h2>
<br>
<h2 class="mef">
Funded by
<img src="/include/mef-logo.png" alt="MEF Logo">
<a href="https://uwaterloo.ca/math-endowment-fund">Mathematics Endowment Fund</a>
</h2>
</div>
<ul class="links hide-on-single-col">
<li><a class="link" href="/">Mirror Home</a></li>
<li><span class="link current">News</span></li>
</ul>
</header>
<div class="container-content">
<main style="min-height: 72vh; position:relative" class="news-list">
{{ .Content }}
{{- range .Pages -}}
<div class="news-article">
<span class="item-preinfo">{{ .Date.Format "2006-01-02" }}</span>
<a href="{{.Permalink}}">
<h1 class="news-title">{{.Title}}</h1>
</a>
{{ .Content }}
</div>
{{- end -}}
</main>
</div>
{{ partial "footer" }}
</div>
{{ end }}

View File

@ -0,0 +1,41 @@
{{- define "title" -}}
{{ .Title }} - {{ .Site.Title }}
{{- end -}}
{{- define "main" -}}
<div id="offcenter">
<header id="nav">
<img class="logo" src="/include/csc-logo.svg" alt="CSC Logo">
<div class="title">
<h1 class="club-name">University of Waterloo <br> Computer Science Club</h1>
<h2 class="site-name">Open Source Software Mirror</h2>
<br>
<h2 class="mef">
Funded by
<img src="/include/mef-logo.png" alt="MEF Logo">
<a href="https://uwaterloo.ca/math-endowment-fund">Mathematics Endowment Fund</a>
</h2>
</div>
<ul class="links hide-on-single-col">
<li><a class="link" href="/">Mirror Home</a></li>
<li><span class="link current">News</span></li>
</ul>
</header>
<div class="container-content">
<main style="min-height: 72vh; position:relative">
<div class="left">
<a href="/news" class="button"><tt><-</tt>All Mirror News</a>
</div>
<article>
<h1>{{ .Title }}</h1>
<span class="date">Published on {{ .Date.Format "2006-01-02" }}</span>
{{- .Content -}}
</article>
</main>
</div>
{{ partial "footer" }}
</div>
{{- end -}}

View File

@ -0,0 +1,105 @@
{{ define "meta" }}
<meta property="og:title" content="{{ .Site.Title }}" />
<meta property="og:type" content="website" />
<meta property="og:description" content="{{ .Site.Params.Description | markdownify | plainify }}" />
<meta name="description" content="{{ .Site.Title }}" />
{{ end }}
{{ define "main" }}
<div id="center">
<header id="nav">
<img class="logo" src="/include/csc-logo.svg" alt="CSC Logo">
<div class="title">
<h1 class="club-name">University of Waterloo</h1>
<br>
<h1 class="club-name">Computer Science Club</h1>
<h2 class="site-name">Open Source Software Mirror</h2>
<br>
<h2 class="mef">
Funded by
<img src="/include/mef-logo.png" alt="MEF Logo">
<a href="https://uwaterloo.ca/math-endowment-fund" class="mef-link">Mathematics Endowment Fund</a>
</h2>
</div>
<ul class="links hide-on-single-col">
<li><a class="link" href="https://csclub.uwaterloo.ca">Home</a></li>
<li><a class="link" href="https://wiki.csclub.uwaterloo.ca">Wiki</a></li>
<li><span class="link current" href="/">Mirror</span></li>
</ul>
</header>
<main>
<div id="left">
{{ partial "project_table" . }}
</div>
<div id="right" class="news">
<h2>Mirror News</h2>
<ul>
{{ range (where .Site.Pages "Section" "news") }}
{{ range .Pages }}
<li>
<span class="news-date">{{ .Date.Format "2006-01-02" }}</span>
<a href="{{ .Permalink }}">{{ .Title }}</a>
</li>
{{ end }}
{{ end }}
</ul>
<a href="/news" style="display: block; margin-top: .8em; text-align: end">All News</a>
<h2>Auto Refresh<span id="auto-refresh-indicator" class="badge">disabled</span></h2>
<p id="auto-refresh">Auto refresh disabled because JavaScript is disabled.</p>
<p style="margin-bottom: 1em">
Last update:
<span id="last-update">
{{ time.Format "2006-01-02 15:04:05" (int .Site.Data.sync.generate_time) }}
</span>
</p>
<h2>Contact Us</h2>
<dl>
<dt>Contact Syscom by Email</dt>
<dd><a href="mailto:syscom@csclub.uwaterloo.ca">syscom@csclub.uwaterloo.ca</a></dd>
<dt>IRC</dt>
<dd><a href="https://web.libera.chat/#chat-with-syscom">#chat-with-syscom on Libera.Chat</a></dd>
<dt>Discord</dt>
<dd><a href="https://discord.gg/pHfYBCg">#chat-with-syscom on CSC Discord</a></dd>
</dl>
</div>
</main>
{{ partial "footer" }}
</div>
<script src="/include/htmx.min.js"></script>
<script>
const sleep = ms => new Promise(r => setTimeout(r, ms));
const auto_refresh_indicator = document.getElementById('auto-refresh-indicator');
const project_table = document.getElementById('project-table');
const last_update = document.getElementById('last-update');
const INTERVAL = 60;
async function update_refresh_indicator() {
while (true) {
auto_refresh_indicator.innerHTML = "waiting";
await sleep(1000 * (INTERVAL - 10));
auto_refresh_indicator.innerHTML = "syncing";
await htmx.ajax('GET', '/include/project_table.html', {target:'#project-table', swap:'outerHTML'});
await sleep(1000 * 10);
auto_refresh_indicator.innerHTML = "waiting";
}
}
// Logic starts here
document.getElementById('auto-refresh').innerHTML = "Mirror status will refresh every 60 seconds.";
// Update auto refresh indicator
document.body.addEventListener("htmx:responseError", function(e) {
auto_refresh_indicator.innerHTML = "error!";
console.log("Error refreshing mirror status: " + e.detail.xhr.status);
});
document.body.addEventListener("htmx:afterSwap", () => {
auto_refresh_indicator.innerHTML = "success!";
last_update.innerHTML = document.getElementById('project-table').dataset.generateTime;
});
update_refresh_indicator().then();
</script>
{{ end }}

View File

@ -0,0 +1 @@
{{ partial "project_table" . }}

View File

@ -0,0 +1,7 @@
<footer>
<p>
This service is run by the Systems Committee (Syscom) of the <a href="https://csclub.uwaterloo.ca">Computer Science Club of University of Waterloo</a>.
<br>
It is made possible by the funding from the the <a href="https://uwaterloo.ca/math-endowment-fund">Mathematics Endowment Fund</a> and support from the <a href="https://cs.uwaterloo.ca/">David R. Cheriton School of Computer Science.</a>
</p>
</footer>

View File

@ -0,0 +1,48 @@
<table class="projects" id="project-table" data-generate-time="{{ time.Format "2006-01-02 15:04:05" (int .Site.Data.sync.generate_time) }}">
<tr class="table-header">
<th>Name</th>
<th>Project Site</th>
<th style="min-width: 8em">Last Sync</th>
<th style="min-width: 3em">Status</th>
<th class="project-size">Size</th>
</tr>
{{ $list := slice }}
{{ range $name, $status := $.Site.Data.sync.projects }}
{{ $nameLower := lower $name }}
{{ $list = $list | append (dict "nameLower" $nameLower "name" $name "status" $status) }}
{{ end }}
{{ range sort $list ".nameLower" }}
<tr>
<td><a href="/{{ .name }}" style="white-space: nowrap">{{ .name }}</a></td>
<td><a href="{{ .status.url }}">{{ .status.url_short }}</a></td>
<td>
{{ if .status.upstream }}
N/A
{{ else if .status.last_successful_sync_time }}
{{ time.Format "2006-01-02 15:04:05" (int .status.last_successful_sync_time)}}
{{ else }}
Never
{{ end }}
</td>
<td>
{{ if .status.is_running }}
<span class="badge syncing">syncing</span>
{{ else if (and (not .status.last_sync_success) (not .status.upstream)) }}
<span class="badge error">error</span>
{{ end }}
</td>
<td class="project-size">{{ .status.size_human }}</td>
</tr>
{{ end }}
<tr class="summary">
<td>Total</td>
<td></td>
<td></td>
<td></td>
<td class="project-size">{{ .Site.Data.sync.total_size_human }}</td>
</tr>
</table>

View File

@ -1,231 +0,0 @@
#!/usr/bin/env python2
"""make-index.py
Generates an nice index of the directories from a
template.
Original Author: Jeremy Roman <jbroman@csclub.uwaterloo.ca>
Modified by: Zachary Seguin <ztseguin@csclub.uwaterloo.ca>
So if you don't like how I did something,
I'm the person you get to complain to.
Please be gentle.
"""
import os, sys, time
from subprocess import Popen, PIPE
from optparse import OptionParser
import yaml, mako.exceptions, webhelpers.html.tags
from mako.template import Template
import humanize
import datetime
def parse_size(s):
if s.endswith("T"):
return float(s[:-1])*1000*1000*1000*1000
elif s.endswith("G"):
return float(s[:-1])*1000*1000*1000
elif s.endswith("M"):
return float(s[:-1])*1000*1000
elif s.endswith("K"):
return float(s[:-1])*1000
return float(size)
def reformat_size(size):
return humanize.naturalsize(size)
def atomic_write(filename, body):
"""Atomically write to a file by writing a
temporary file and then moving it to replace
the desired output file.
This ensures that partial files are never seen
by clients."""
# generate an appropriate temporary filename
# in the same directory
tmp_filename = "%s.%d.tmp" % (filename, os.getpid())
# open the directory so that we can fsync it
dir = os.open(os.path.realpath(os.path.dirname(filename)), \
os.O_DIRECTORY | os.O_RDONLY)
# write to the temporary file
tmp = open(tmp_filename, 'w')
print >>tmp, body
tmp.flush()
os.fsync(tmp.fileno())
tmp.close()
# atomically replace the actual file
os.rename(tmp_filename, filename)
os.fsync(dir)
os.close(dir)
def main():
# accept command-line arguments
parser = OptionParser()
parser.add_option("-c", "--config", dest="config", default="config.yaml",
help="configuration file to be used", metavar="FILE")
parser.add_option("-D", "--docroot", dest="docroot",
help="directory to be scanned", metavar="DIR")
parser.add_option("-F", "--duflags", dest="duflags",
help="flags to be passed to du, replaces any in config")
parser.add_option("-o", "--output", dest="output", metavar="FILE",
help="file to which index page will be written. "
"Use /dev/stdout to send to standard out.")
parser.add_option("-t", "--template", dest="template",
help="Mako template to render", metavar="FILE")
parser.add_option("--nonatomic", dest="nonatomic", action="store_true",
default=False, help="write the output to the path "
"given without creating a temporary file in between. "
"This is automatically set if the output appears "
"to be a character device, not a file.")
(options, args) = parser.parse_args()
# load config file
try:
config = yaml.load(file(options.config,'r'), Loader=yaml.FullLoader)
except:
config = None
if not config or type(config) != dict:
print >>sys.stderr, "Unable to load configuration '%s'." % options.config
sys.exit(-1)
# determine important variables based on an appropriate order of
# precedence (command-line flags first, then the config file,
# then built-in fallbacks)
#
# fallback value for nonatomic is used so that character devices
# (e.g. /dev/stdout, /dev/null) are written to in the regular way
docroot = options.docroot or config.get('docroot')
duflags = options.duflags or config.get('duflags') or "-h --max-depth=1 --exclude='.~tmp~'"
output = options.output or config.get('output')
template = options.template or config.get("template") or "index.mako"
nonatomic = options.nonatomic or config.get("nonatomic") or \
(os.path.exists(output) and not os.path.isfile(output))
# sanity checks
if not docroot:
print >>sys.stderr, "docroot not specified."
print >>sys.stderr, "Define it in the config file or pass -D on the command line."
sys.exit(-1)
elif not output:
print >>sys.stderr, "output not specified."
print >>sys.stderr, "Define it in the config file or pass -o on the command line."
elif not config.get('directories'):
print >>sys.stderr, "directories not specified."
print >>sys.stderr, "Define it in the config file."
sys.exit(-1)
elif not os.path.isdir(docroot):
print >>sys.stderr, "docroot '%s' not found or not a directory." % docroot
sys.exit(-1)
elif not os.path.exists(template) or os.path.isdir(template):
print >>sys.stderr, "template '%s' not found or is a directory." % template
sys.exit(-1)
generated = datetime.datetime.now()
directories = []
total_size = 0.0
ls = os.listdir(docroot)
for item in ls:
if item == "pub":
continue
path = os.path.join(docroot, item)
# If symlink, then this is a project
if os.path.islink(path):
dataset = os.readlink(path)[1:]
# Get the disk space used by that dataset
du = Popen(
"/usr/bin/sudo /sbin/zfs get -H -o value used %s" % dataset,
shell=True, stdout=PIPE, stderr=PIPE).communicate()
size = "-"
if len(du[1].strip()) > 0:
sys.stderr.write(du[1])
print >>sys.stderr, "zfs get terminated unsuccessfully for %s" % item
continue
else:
size = parse_size(du[0].strip())
total_size += size
info = {'dir':item, 'size':reformat_size(size)}
# use info from config.yaml, if found
# otherwise, skip this directory
if item in config['exclude']:
continue
elif item in config['directories']:
info.update(config['directories'][item])
else:
info.update({ 'site': '', 'url': '' })
directories.append(info)
# Sort the directories
directories.sort(key=lambda d : d['dir'].lower())
# Call du to compute size
#du = Popen(
# "/usr/bin/du %s/* %s | /usr/bin/sort -fk2" % (docroot, duflags),
# shell=True, stdout=PIPE, stderr=PIPE).communicate()
# du = []
# Check that du executed successfully
# If there's anything on stderr, send it
# out our own stderr and terminate.
#if len(du[1].strip()) > 0:
# sys.stderr.write(du[1])
# print >>sys.stderr, "du terminated unsuccessfully. Not generating index."
# sys.exit(-1)
# first one should be total, grab its size and format
#du = du[0].splitlines() # we only care about stdout now
#total_size = reformat_size(du[0].split(None,2)[0])
# the rest are the sizes we want
# directories = []
# total_size = 0
# for line in du:
# (size, path) = line.split(None, 2)
# dir = os.path.basename(path)
# total_size += int(size)
# info = {'dir':dir, 'size':reformat_size(int(size))}
# use info from config.yaml, if found
# otherwise, skip this directory
# if dir in config['exclude'] or not os.path.isdir(path):
# continue
# elif dir in config['directories']:
# info.update(config['directories'][dir])
# else:
# info.update({ 'site': '', 'url': '' })
# directories.append(info)
# render the template to a string
body = Template(filename=template).render(
total_size=reformat_size(total_size),
directories=directories,
generated=generated,
config=config,
h=webhelpers.html.tags)
# write the rendered output
if nonatomic:
print >>file(output,'w'), body
else:
atomic_write(output, body)
if __name__ == "__main__":
main()

13
mirror-index/setup.sh Executable file
View File

@ -0,0 +1,13 @@
#!/bin/bash
# Setup mirror-index
set -euo pipefail
# Create directories
mkdir -p data
# Run deploy once
./deploy.sh
# Copy assets to destination
cp public/include/* /mirror/root/include/

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.6 KiB

View File

@ -0,0 +1,9 @@
<svg width="175" height="76" viewBox="0 0 175 76" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M31.3226 0.526582C17.4935 3.13665 6.39763 12.7205 2.0327 25.8116C-1.14921 35.3954 -0.578098 45.3055 3.62366 54.1145C5.86731 58.8044 8.23335 61.9447 12.2719 65.819C22.7967 75.7291 38.5023 78.7062 52.0866 73.3637L54.2079 72.5073L58.3689 73.9347C72.4427 78.5838 86.8837 75.2805 97.2861 64.9626C104.67 57.6625 107.974 49.9547 108.831 37.8424C109.035 34.8245 109.524 31.3172 109.891 30.0937C111.074 26.1786 113.4 22.3043 116.5 19.1233L119.396 16.1054L121.069 17.4104C124.944 20.4691 128.738 26.9535 129.921 32.5407C131.145 38.3317 130.084 45.3871 127.106 51.0966C125.475 54.1553 119.682 60.0279 116.663 61.6592C115.439 62.3117 113.155 63.2497 111.605 63.7391L108.831 64.6363L111.278 66.8793C114.501 69.7749 119.56 72.6296 124.169 74.0978C137.917 78.5431 152.317 75.0766 162.556 64.9626C169.246 58.3558 172.959 50.4033 173.938 40.6155C174.549 34.2943 172.51 25.363 168.961 19.1233C166.717 15.1674 161.781 9.74337 158.028 7.05174C147.87 -0.207499 134.612 -2.00192 122.782 2.321L119.519 3.50368L117.112 2.52491C112.543 0.689709 109.443 0.159538 103.364 0.159538C97.1229 0.159538 94.4713 0.648929 88.9642 2.76961C80.3159 6.11375 72.0348 14.1479 68.4041 22.7937C66.5684 27.1574 65.6302 31.4395 65.2222 37.4345C65.0183 40.4524 64.5695 43.9597 64.2024 45.1832C63.1009 49.3429 60.8165 53.258 57.6346 56.5614L54.6566 59.6609L53.5144 59.0083C51.6787 57.948 47.8849 53.1765 46.2939 49.9547C44.4174 46.1211 43.6015 42.5731 43.6015 38.0463C43.6015 33.3563 44.4174 29.9714 46.5387 25.6077C49.5166 19.5719 56.0028 14.0255 62.1627 12.1495C63.0602 11.9048 64.0392 11.5786 64.3656 11.497C64.8143 11.3747 64.6511 10.9668 63.7536 10.0696C60.4901 6.60314 53.1881 2.60648 47.4769 1.09753C43.1936 -0.00358955 35.5652 -0.289063 31.3226 0.526582ZM42.5409 11.1707C43.0304 11.3747 42.7041 12.0272 40.9499 14.2702C35.0348 21.8557 32.1385 30.9501 32.7096 40.2485C33.2399 48.6089 36.6666 57.01 41.8882 62.8011C42.6633 63.6575 43.1936 64.4732 43.0712 64.5955C42.9488 64.7586 41.1947 64.9626 39.2366 65.0441C26.4682 65.7374 14.5156 56.7653 11.6192 44.3675C10.7218 40.6155 10.8034 34.9876 11.7824 31.1541C14.1484 21.8149 21.7361 14.1886 31.0371 11.7417C33.9334 11.0076 41.1131 10.6406 42.5409 11.1707ZM107.811 11.1707C108.3 11.3747 107.974 12.0272 106.22 14.2702C100.917 21.0808 97.8572 29.6043 97.8572 37.72C97.8572 41.6351 96.7558 46.2027 94.9201 50.077C93.5739 52.891 92.5948 54.196 89.7393 57.0508C86.8837 59.9056 85.5783 60.8843 82.7635 62.2301C77.6643 64.6771 71.5453 65.6966 66.7724 64.9218L65.263 64.6363L67.0171 62.5564C72.7691 55.7865 75.4207 48.6497 76.1957 37.8424C76.8484 28.8295 78.7658 24.3026 84.1913 18.8786C89.6169 13.4546 95.4912 11.0076 103.242 10.9668C105.404 10.9261 107.444 11.0484 107.811 11.1707ZM142.2 11.5378C150.155 13.4138 156.6 18.6339 160.312 26.2194C164.065 33.8865 164.025 42.0837 160.19 50.077C158.844 52.891 157.865 54.196 155.009 57.0508C152.154 59.9056 150.848 60.8843 148.034 62.2301C142.934 64.6771 136.815 65.6966 132.042 64.9218L130.533 64.6363L132.287 62.5564C139.018 54.6446 142.282 44.0005 141.099 33.8865C140.201 26.3418 136.897 18.5931 132.491 13.6177C130.819 11.7825 130.615 11.3747 131.186 11.2115C132.858 10.7629 139.875 10.9668 142.2 11.5378Z" fill="url(#paint0_linear)"/>
<defs>
<linearGradient id="paint0_linear" x1="-19.8935" y1="117.914" x2="316.186" y2="-16.9617" gradientUnits="userSpaceOnUse">
<stop stop-color="#1481E3"/>
<stop offset="1" stop-color="#4ED4B2"/>
</linearGradient>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

File diff suppressed because one or more lines are too long

Binary file not shown.

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.8 KiB

File diff suppressed because one or more lines are too long

Binary file not shown.

View File

@ -0,0 +1,258 @@
# The configuration file for synctask2project
# By default, sync tasks will be mapped to project based on name
# But in case of different naming or multiple sync task per project,
# use merlin-tasks and specify a list
# If we are the upstream (so sync is not requires), set upstream = true
merlin_sock = "/home/mirror/merlin/merlin.sock"
zfs_pools = ["cscmirror0"]
[alpine]
site = "alpinelinux.org"
url = "https://alpinelinux.org/"
[almalinux]
site = "almalinux.org"
url = "https://almalinux.org"
[apache]
site = "apache.org"
url = "https://www.apache.org/"
[archlinux]
site = "archlinux.org"
url = "https://www.archlinux.org/"
[archlinuxarm]
site = "archlinuxarm.org"
url = "https://archlinuxarm.org/"
[artixlinux]
site = "artixlinux.org"
url = "https://artixlinux.org/"
[centos]
site = "centos.org"
url = "https://www.centos.org/"
[ceph]
site = "ceph.io"
url = "https://ceph.io"
[CPAN]
site = "cpan.org"
url = "https://www.cpan.org/"
[CRAN]
site = "r-project.org"
url = "https://cran.r-project.org/"
[csclub]
site = "csclub.uwaterloo.ca"
url = "https://csclub.uwaterloo.ca/media/"
upstream = true
[CTAN]
site = "ctan.org"
url = "https://www.ctan.org/"
[cygwin]
site = "cygwin.com"
url = "https://www.cygwin.com/"
[damnsmalllinux]
site = "damnsmalllinux.org"
url = "https://www.damnsmalllinux.org/"
upstream = true
[debian]
site = "debian.org"
url = "https://www.debian.org/"
[debian-cd]
site = "debian.org"
url = "https://www.debian.org/CD/"
[deb-multimedia]
site = "deb-multimedia.org"
url = "https://www.deb-multimedia.org/"
[debian-security]
site = "debian.org"
url = "https://www.debian.org/security/"
[eclipse]
site = "eclipse.org"
url = "https://www.eclipse.org/"
[emacsconf]
site = "emacsconf.org"
url = "https://emacsconf.org"
# Temp, debugging only
upstream = true
[fedora]
site = "fedoraproject.org"
url = "https://www.fedoraproject.org/"
[FreeBSD]
site = "freebsd.org"
url = "https://www.freebsd.org/"
[gentoo-distfiles]
site = "gentoo.org"
url = "https://www.gentoo.org/"
[gentoo-portage]
site = "gentoo.org"
url = "https://www.gentoo.org/"
[gnome]
site = "gnome.org"
url = "https://www.gnome.org/"
[gnu]
site = "gnu.org"
url = "https://www.gnu.org/"
[gutenberg]
site = "gutenberg.org"
url = "https://www.gutenberg.org/"
[kde]
site = "kde.org"
url = "https://www.kde.org/"
[kde-applicationdata]
site = "kde.org"
url = "https://www.kde.org/"
["kernel.org"]
site = "kernel.org"
url = "https://www.kernel.org/"
merlin-tasks = ["kernel.org-linux", "kernel.org-software"]
[linuxmint]
site = "linuxmint.com"
url = "https://www.linuxmint.com/"
[linuxmint-packages]
site = "linuxmint.com"
url = "https://www.linuxmint.com/"
[MacPorts]
site = "macports.org"
url = "https://www.macports.org/"
merlin-tasks = ["macports-distfiles", "macports-release"]
[mysql]
site = "mysql.com"
url = "https://www.mysql.com/"
[nongnu]
site = "nongnu.org"
url = "https://savannah.nongnu.org/"
[opensuse]
site = "opensuse.org"
url = "https://www.opensuse.org/"
[puppylinux]
site = "puppylinux.org"
url = "https://puppylinux.org/"
[racket]
site = "racket-lang.org"
url = "https://racket-lang.org/"
merlin-tasks = ["plt-bundles", "racket-installers"]
[sage]
site = "sagemath.org"
url = "https://www.sagemath.org/"
merlin-tasks = ["sagemath"]
[saltstack]
site = "saltstack.com"
url = "https://saltstack.com/"
[slackware]
site = "slackware.com"
url = "https://www.slackware.com/"
[tdf]
site = "documentfoundation.org"
url = "https://www.documentfoundation.org/"
[ubuntu]
site = "ubuntu.com"
url = "https://www.ubuntu.com/"
[ubuntu-ports]
site = "ports.ubuntu.com"
url = "http://ports.ubuntu.com/ubuntu-ports/"
[ubuntu-ports-releases]
site = "ports.ubuntu.com"
url = "https://cdimage.ubuntu.com/releases/"
[ubuntu-releases]
site = "releases.ubuntu.com"
url = "https://releases.ubuntu.com/"
[vlc]
site = "videolan.org"
url = "https://videolan.org/"
[xiph]
site = "xiph.org"
url = "https://xiph.org/"
["x.org"]
site = "x.org"
url = "https://www.x.org/"
[xubuntu-releases]
site = "xubuntu.org"
url = "https://www.xubuntu.org/"
[NetBSD]
site = "netbsd.org"
url = "https://www.netbsd.org/"
merlin-tasks = ["netbsd"]
[raspberrypi]
site = "raspberrypi.org"
url = "https://www.raspberrypi.org/"
[raspbian]
site = "raspbian.org"
url = "https://www.raspbian.org/"
[ipfire]
site = "ipfire.org"
url = "https://www.ipfire.org/"
[manjaro]
site = "manjaro.org"
url = "https://manjaro.org/"
[mxlinux]
site = "mxlinux.org"
url = "https://mxlinux.org"
[mxlinux-iso]
site = "mxlinux.org"
url = "https://mxlinux.org"
[parabola]
site = "parabola.nu"
url = "https://www.parabola.nu/"
[trisquel]
site = "trisquel.info"
url = "https://trisquel.info/"
merlin-tasks = ["trisquel-iso", "trisquel-packages"]
[termux]
site = "termux.dev"
url = "https://termux.dev/"

View File

@ -0,0 +1,23 @@
#!/usr/bin/env python3
import socket, sys, time
def recvall(sock):
BUFF_SIZE = 4096 # 4 KiB
data = b''
while True:
part = sock.recv(BUFF_SIZE)
data += part
if len(part) < BUFF_SIZE:
# either 0 or end of data
break
return data
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
merlin_socket = "/home/mirror/merlin/merlin.sock"
sock.connect(merlin_socket)
command = "status"
sock.sendall(str.encode(command))
sock.shutdown(socket.SHUT_WR)
result = recvall(sock).decode("utf-8")
print(result)

View File

@ -0,0 +1,136 @@
#!/usr/bin/env python3
# Read sync task status generated by arthur and convert them to project to show on website
# Note: Will invoke `zfs list` command so run me on the actual mirror server with appropriate permissions
import json, toml
import humanize
import time
import copy, subprocess, re
import socket
from typing import Dict
import argparse
def get_project_sizes_from_zfs(prefix: str) -> Dict[str, int]:
real_prefix = prefix + "/"
stdout = subprocess.check_output(['/sbin/zfs', 'list', '-Hp']).decode("utf-8")
result = {}
for line in stdout.splitlines():
segments = line.split()
if len(segments) < 5:
raise Exception("Malformed zfs list output (should have 5 columns)")
if segments[0].startswith(real_prefix):
name = segments[0][len(real_prefix):]
size = int(segments[1])
result[name] = size
return result
def recvall(sock):
BUFF_SIZE = 4096 # 4 KiB
data = b''
while True:
part = sock.recv(BUFF_SIZE)
data += part
if len(part) < BUFF_SIZE:
# either 0 or end of data
break
return data
def get_sync_task_status(merlin_sock_path: str):
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.connect(merlin_sock_path)
command = "status"
sock.sendall(str.encode(command))
sock.shutdown(socket.SHUT_WR)
result_str = recvall(sock).decode("utf-8")
return json.loads(result_str)["repos"]
# Main logic starts here
argparser = argparse.ArgumentParser(prog="synctask2project",
description="Convert merlin sync task status to project for frontend display")
argparser.add_argument("output", type=str, help="output file to be written to")
argparser.add_argument("--config", dest="config_toml", type=str, help="location of config.toml", default="config.toml")
args = argparser.parse_args()
config_toml = open(args.config_toml)
config = toml.load(config_toml)
status = get_sync_task_status(config["merlin_sock"])
project_sizes = {}
for pool in config["zfs_pools"]:
project_sizes.update(get_project_sizes_from_zfs(pool))
# The result we are gonna to present
projects = {}
total_size = 0
for (name, config) in config.items():
if name == "zfs_pools" or name == "merlin_sock":
continue
# The project data we're gonna to present
data = copy.deepcopy(config)
if name in project_sizes:
data["size"] = project_sizes[name]
data["size_human"] = humanize.naturalsize(project_sizes[name])
total_size += project_sizes[name]
# We are not the upstream in most cases
data["upstream"] = False
# short URL for display
data["url_short"] = re.sub("^https://www\.", "", data["url"])
data["url_short"] = re.sub("^https://", "", data["url_short"])
data["url_short"] = re.sub("^http://", "", data["url_short"])
data["url_short"] = re.sub("/.*$", "", data["url_short"])
if "upstream" in config and config["upstream"]:
data["upstream"] = True
elif "merlin-tasks" not in config:
# Use the project name to search in status
found = False
for task in status:
if task["name"] == name:
found = True
data["is_running"] = task["is_running"]
data["last_sync_success"] = task["last_attempt_exit"] == "completed"
if "last_successful_attempt_time" in task:
data["last_successful_sync_time"] = task["last_successful_attempt_time"]
break
else:
del data["merlin-tasks"]
# Gather information from all the sub-tasks
# First set the initial state
data["is_running"] = False
data["last_sync_success"] = True
for merlin_task in config["merlin-tasks"]:
found = False
for task in status:
if task["name"] == merlin_task:
found = True
data["is_running"] |= task["is_running"]
data["last_sync_success"] &= task["last_attempt_exit"] == "completed"
if "last_successful_attempt_time" not in task:
break
if "last_successful_attempt_time" not in data:
data["last_successful_sync_time"] = task["last_successful_attempt_time"]
elif task["last_successful_attempt_time"] < data["last_successful_sync_time"]:
data["last_successful_sync_time"] = task["last_successful_attempt_time"]
break
if not found:
print(f"WARNING: {merlin_task} (subtask for {name}) not found in status")
projects[name] = data
result = { "total_size": total_size,
"total_size_human": humanize.naturalsize(total_size),
"generate_time": time.time(),
"projects": projects }
output_str = json.dumps(result, indent=4)
# Write to output file
with open(args.output, "w") as f:
f.write(output_str)