diff --git a/.gitignore b/.gitignore index 0842e6c..def1152 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /src/extra_mailman_archivers.egg-info/ __pycache__/ +*.swp diff --git a/README.md b/README.md index 7f5ffc5..4e511bc 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,28 @@ # Extra Mailman Archivers This contains some extra archivers to be used with Mailman 3. -Currently there is only one archiver, `MonthlyArchiver`, which simply gzips +Currently there are two archivers, `MonthlyArchiver`, and `Pipermail`. + +## Installation +Make sure to have the following packages installed first: +```sh +apt install python3-pip python3-setuptools python3-wheel +``` + +Then: +```sh +pip3 install . +``` + +## MonthlyArchiver +MonthlyArchiver simply gzips each message and stores it in a folder named by the year and month. Each message's file name is a Unix epoch timestamp. +The motivation behind `MonthlyArchiver` was to avoid storing thousands of +messages in a single directory, which the `Prototype` archiver currently +does since it uses a maildir. + Example directory structure: ``` /var/lib/mailman3/archives @@ -17,16 +35,43 @@ Example directory structure: \_ ... ``` -The motivation behind `MonthlyArchiver` was to avoid storing thousands of -messages in a single directory, which the `Prototype` archiver currently -does since it uses a maildir. +To enable this archiver, add the following to your mailman.cfg: +``` +[archiver.monthly_archiver] +class: extra_mailman_archivers.monthly_archiver.MonthlyArchiver +enable: yes +``` -## Installation -Make sure to have the following packages installed first: -```sh -apt install python3-pip python3-setuptools python3-wheel +## Pipermail +This archiver attempts to add messages to Pipermail, the archiver for +Mailman 2. An existing Mailman 2 installation is required. + +If the mailing list does not exist in the Mailman 2 directory, you +must create it first. Use the `newlist-pipermail` script in the `pipermail` +directory after copying it to the `bin` directory where Mailman 2 +is installed. It works the same way as the traditional `newlist` script +except that it does not communicate with the MTA. **Note**: by default, +this will create a public archive; to make it private, unlink the appropriate +folder in the `archives/public` directory where Mailman 2 is installed. + +To enable this archiver, add the following to your mailman.cfg: ``` -Then: -```sh -pip3 install . +[archiver.pipermail] +class: extra_mailman_archivers.pipermail.Pipermail +enable: yes +``` + +The default list URL for this archiver is `http://$domain/pipermail/$short_listname`. +The default Mailman 2 directory is `/var/lib/mailman`. +To change either of these values, add the following to the pipermail +section in mailman.cfg: +``` +configuration: /etc/mailman3/mailman-pipermail.cfg +``` + +Then, in `/etc/mailman3/mailman-pipermail.cfg`, set `base_url` and/or `mailman2_dir` +appropriately, e.g. +``` +[general] +base_url: http://$domain/old/pipermail/$short_listname ``` diff --git a/pipermail/newlist-pipermail b/pipermail/newlist-pipermail new file mode 100755 index 0000000..6d50b1e --- /dev/null +++ b/pipermail/newlist-pipermail @@ -0,0 +1,239 @@ +#! /usr/bin/python +# +# Copyright (C) 1998-2018 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, +# USA. + +"""Create a new, unpopulated mailing list. + +Usage: %(PROGRAM)s [options] [listname [listadmin-addr [admin-password]]] + +Options: + + -l language + --language=language + Make the list's preferred language `language', which must be a two + letter language code. + + -u urlhost + --urlhost=urlhost + Gives the list's web interface host name. + + -e emailhost + --emailhost=emailhost + Gives the list's email domain name. + + -q/--quiet + Normally the administrator is notified by email (after a prompt) that + their list has been created. This option suppresses the prompt and + notification. + + -a/--automate + This option suppresses the prompt prior to administrator notification + but still sends the notification. It can be used to make newlist + totally non-interactive but still send the notification, assuming + listname, listadmin-addr and admin-password are all specified on the + command line. + + -h/--help + Print this help text and exit. + +You can specify as many of the arguments as you want on the command line: +you will be prompted for the missing ones. + +Every Mailman list has two parameters which define the default host name for +outgoing email, and the default URL for all web interfaces. When you +configured Mailman, certain defaults were calculated, but if you are running +multiple virtual Mailman sites, then the defaults may not be appropriate for +the list you are creating. + +You also specify the domain to create your new list in by typing the command +like so: + + newlist --urlhost=www.mydom.ain mylist + +where `www.mydom.ain' should be the base hostname for the URL to this virtual +hosts's lists. E.g. with this setting people will view the general list +overviews at http://www.mydom.ain/mailman/listinfo. Also, www.mydom.ain +should be a key in the VIRTUAL_HOSTS mapping in mm_cfg.py/Defaults.py if +the email hostname to be automatically determined. + +If you want the email hostname to be different from the one looked up by the +VIRTUAL_HOSTS or if urlhost is not registered in VIRTUAL_HOSTS, you can specify +`emailhost' like so: + + newlist --urlhost=www.mydom.ain --emailhost=mydom.ain mylist + +where `mydom.ain' is the mail domain name. If you don't specify emailhost but +urlhost is not in the virtual host list, then mm_cfg.DEFAULT_EMAIL_HOST will +be used for the email interface. + +For backward compatibility, you can also specify the domain to create your +new list in by spelling the listname like so: + + mylist@www.mydom.ain + +where www.mydom.ain is used for `urlhost' but it will also be used for +`emailhost' if it is not found in the virtual host table. Note that +'--urlhost' and '--emailhost' have precedence to this notation. + +If you spell the list name as just `mylist', then the email hostname will be +taken from DEFAULT_EMAIL_HOST and the url will be taken from DEFAULT_URL_HOST +interpolated into DEFAULT_URL_PATTERN (as defined in your Defaults.py file or +overridden by settings in mm_cfg.py). + +Note that listnames are forced to lowercase. +""" + +import sys +import os +import getpass +import getopt + +import paths +from Mailman import mm_cfg +from Mailman import MailList +from Mailman import Utils +from Mailman import Errors +from Mailman import Message +from Mailman import i18n + +_ = i18n._ +C_ = i18n.C_ + +PROGRAM = sys.argv[0] + + + +def usage(code, msg=''): + if code: + fd = sys.stderr + else: + fd = sys.stdout + print >> fd, C_(__doc__) + if msg: + print >> fd, msg + sys.exit(code) + + + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'hqal:u:e:', + ['help', 'quiet', 'automate', 'language=', + 'urlhost=', 'emailhost=']) + except getopt.error, msg: + usage(1, msg) + + lang = mm_cfg.DEFAULT_SERVER_LANGUAGE + quiet = False + automate = False + urlhost = None + emailhost = None + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + if opt in ('-q', '--quiet'): + quiet = True + if opt in ('-a', '--automate'): + automate = True + if opt in ('-l', '--language'): + lang = arg + if opt in ('-u', '--urlhost'): + urlhost = arg + if opt in ('-e', '--emailhost'): + emailhost = arg + + # Is the language known? + if lang not in mm_cfg.LC_DESCRIPTIONS.keys(): + usage(1, C_('Unknown language: %(lang)s')) + + if len(args) > 0: + listname = args[0] + else: + listname = raw_input(C_('Enter the name of the list: ')) + listname = listname.lower() + + if '@' in listname: + # note that --urlhost and --emailhost have precedence + listname, domain = listname.split('@', 1) + urlhost = urlhost or domain + emailhost = emailhost or mm_cfg.VIRTUAL_HOSTS.get(domain, domain) + + urlhost = urlhost or mm_cfg.DEFAULT_URL_HOST + host_name = emailhost or \ + mm_cfg.VIRTUAL_HOSTS.get(urlhost, mm_cfg.DEFAULT_EMAIL_HOST) + web_page_url = mm_cfg.DEFAULT_URL_PATTERN % urlhost + + if Utils.list_exists(listname): + usage(1, C_('List already exists: %(listname)s')) + + if len(args) > 1: + owner_mail = args[1] + else: + owner_mail = raw_input( + C_('Enter the email of the person running the list: ')) + + if len(args) > 2: + listpasswd = args[2] + else: + listpasswd = getpass.getpass(C_('Initial %(listname)s password: ')) + # List passwords cannot be empty + listpasswd = listpasswd.strip() + if not listpasswd: + usage(1, C_('The list password cannot be empty')) + + mlist = MailList.MailList() + try: + pw = Utils.sha_new(listpasswd).hexdigest() + # Guarantee that all newly created files have the proper permission. + # proper group ownership should be assured by the autoconf script + # enforcing that all directories have the group sticky bit set + oldmask = os.umask(002) + try: + try: + if lang == mm_cfg.DEFAULT_SERVER_LANGUAGE: + langs = [lang] + else: + langs = [lang, mm_cfg.DEFAULT_SERVER_LANGUAGE] + mlist.Create(listname, owner_mail, pw, langs=langs, + emailhost=host_name, urlhost=urlhost) + finally: + os.umask(oldmask) + except Errors.BadListNameError, s: + usage(1, C_('Illegal list name: %(s)s')) + except Errors.EmailAddressError, s: + usage(1, C_('Bad owner email address: %(s)s') + + C_(' - owner addresses need to be fully-qualified names' + ' like "owner@example.com", not just "owner".')) + except Errors.MMListAlreadyExistsError: + usage(1, C_('List already exists: %(listname)s')) + + # Assign domain-specific attributes + mlist.host_name = host_name + mlist.web_page_url = web_page_url + + # And assign the preferred language + mlist.preferred_language = lang + + mlist.Save() + finally: + mlist.Unlock() + + + +if __name__ == '__main__': + main() diff --git a/setup.cfg b/setup.cfg index a9085fe..5f9df96 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = extra-mailman-archivers -version = 0.0.1 +version = 0.0.2 author = Max Erenberg author_email = merenber@csclub.uwaterloo.ca description = Some extra archivers for Mailman 3 diff --git a/src/extra_mailman_archivers/monthly_archiver.py b/src/extra_mailman_archivers/monthly_archiver.py index 0a63513..a3156a8 100644 --- a/src/extra_mailman_archivers/monthly_archiver.py +++ b/src/extra_mailman_archivers/monthly_archiver.py @@ -37,7 +37,7 @@ from public import public from zope.interface import implementer -log = logging.getLogger('mailman.error') +log = logging.getLogger('mailman.archiver') @public diff --git a/src/extra_mailman_archivers/pipermail.py b/src/extra_mailman_archivers/pipermail.py new file mode 100644 index 0000000..674a4ab --- /dev/null +++ b/src/extra_mailman_archivers/pipermail.py @@ -0,0 +1,131 @@ +# Copyright (C) 2021 Max Erenberg +# This file is adapted from GNU Mailman. The original copyright notice +# is preserved below. +# +# Copyright (C) 2008-2019 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +""" +This archiver passes messages to Pipermail. Mailman 2 must already be +installed. +""" + +from contextlib import suppress +from datetime import timedelta +from email.generator import BytesGenerator +from io import BytesIO +import logging +import os +import tempfile + +from flufl.lock import Lock +from mailman.config import config +from mailman.config.config import external_configuration +from mailman.interfaces.archiver import IArchiver +from mailman.interfaces.configuration import MissingConfigurationFileError +from mailman.utilities.string import expand +from public import public +from subprocess import PIPE, DEVNULL, Popen +from urllib.parse import urljoin +from zope.interface import implementer + + +log = logging.getLogger('mailman.archiver') + + +@public +@implementer(IArchiver) +class Pipermail: + """Local archiver which passes messages to Pipermail.""" + + name = 'pipermail' + is_enabled = False + + def __init__(self): + default_base_url = 'http://$domain/pipermail/$short_listname' + default_mailman2_dir = '/var/lib/mailman' + # Read our specific configuration file + try: + archiver_config = external_configuration( + config.archiver.pipermail.configuration) + self.base_url = archiver_config.get( + 'general', 'base_url', fallback=default_base_url) + self.mailman2_dir = archiver_config.get( + 'general', 'mailman2_dir', fallback=default_mailman2_dir) + except MissingConfigurationFileError: + self.base_url = default_base_url + self.mailman2_dir = default_mailman2_dir + + def list_url(self, mlist): + """See `IArchiver`.""" + return expand(self.base_url, mlist) + + def permalink(self, mlist, msg): + """See `IArchiver`.""" + # Unfortunately Pipermail URLs are not guaranteed to be stable + # since the mbox file can be modified and the archives regenerated. + return None + + def archive_message(self, mlist, msg): + """See `IArchiver`.""" + # Mangle the 'From ' lines and add the Unix envelope header. + fp = BytesIO() + g = BytesGenerator(fp, mangle_from_=True) + g.flatten(msg, unixfrom=True) + msg_bytes = fp.getvalue() + + mbox_path = os.path.join( + self.mailman2_dir, 'archives', 'private', + mlist.list_name + '.mbox', mlist.list_name + '.mbox') + lock_path = os.path.join( + config.LOCK_DIR, '{0}-pipermail.lock'.format(mlist.list_name)) + lock = Lock(lock_path) + + # Unfortunately the `bin/arch` script needs to seek/tell the mbox + # file, so we need to write the message to a real file. + fd, tempfile_name = tempfile.mkstemp() + fo = os.fdopen(fd, 'wb') + fo.write(msg_bytes) + fo.close() + + try: + lock.lock(timeout=timedelta(seconds=3)) + + # Append the message to the mbox file. + with open(mbox_path, 'ab') as fo: + # Add a newline before and after to be absolutely sure we don't + # corrupt the mbox file. + fo.write(b'\n') + fo.write(msg_bytes) + fo.write(b'\n') + + # Send the message to Pipermail. + args = ['bin/arch', mlist.list_name, tempfile_name] + proc = Popen( + args, cwd=self.mailman2_dir, + stdin=DEVNULL, stdout=PIPE, stderr=PIPE) + stdout, stderr = proc.communicate() + if proc.returncode != 0: + log.error('%s: bin/arch subprocess had non-zero exit code: %s' % + (msg.get('message-id'), proc.returncode)) + log.info(stdout.decode()) + log.error(stderr.decode()) + finally: + lock.unlock(unconditionally=True) + os.unlink(tempfile_name) + + return None