extra-mailman-archivers/src/extra_mailman_archivers/pipermail.py

# Copyright (C) 2021 Max Erenberg
# This file is adapted from GNU Mailman. The original copyright notice
# is preserved below.
#
# Copyright (C) 2008-2019 by the Free Software Foundation, Inc.
#
# This file is part of GNU Mailman.
#
# GNU Mailman is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free
# Software Foundation, either version 3 of the License, or (at your option)
# any later version.
#
# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.

"""
This archiver passes messages to Pipermail. Mailman 2 must already be
installed.
"""

from contextlib import suppress
from datetime import timedelta
from email.generator import BytesGenerator
from io import BytesIO
import logging
import os
import tempfile
import time

from flufl.lock import Lock
from mailman.config import config
from mailman.config.config import external_configuration
from mailman.interfaces.archiver import IArchiver
from mailman.interfaces.configuration import MissingConfigurationFileError
from mailman.utilities.string import expand
from public import public
from subprocess import PIPE, DEVNULL, Popen
from urllib.parse import urljoin
from zope.interface import implementer


log = logging.getLogger('mailman.archiver')


@public
@implementer(IArchiver)
class Pipermail:
    """Local archiver which passes messages to Pipermail."""

    name = 'pipermail'
    is_enabled = False

    def __init__(self):
        default_base_url = 'http://$domain/pipermail/$short_listname'
        default_mailman2_dir = '/var/lib/mailman'
        # Read our specific configuration file
        try:
            archiver_config = external_configuration(
                config.archiver.pipermail.configuration)
            self.base_url = archiver_config.get(
                'general', 'base_url', fallback=default_base_url)
            self.mailman2_dir = archiver_config.get(
                'general', 'mailman2_dir', fallback=default_mailman2_dir)
        except MissingConfigurationFileError:
            self.base_url = default_base_url
            self.mailman2_dir = default_mailman2_dir

    def list_url(self, mlist):
        """See `IArchiver`."""
        if self.base_url == '':
            # admin explicitly doesn't want list URL to be advertised
            return None
        return expand(self.base_url, mlist)

    def permalink(self, mlist, msg):
        """See `IArchiver`."""
        # Unfortunately Pipermail URLs are not guaranteed to be stable
        # since the mbox file can be modified and the archives regenerated.
        return None

    def archive_message(self, mlist, msg):
        """
        See `IArchiver`.

        :type mlist: mailman.interfaces.mailinglist.IMailingList
        :type message: mailman.email.message.Message
        """
        
        # Workaround for a bug in Mailman which sets unixfrom to an email
        # address instead of an mbox header
        # https://gitlab.com/mailman/mailman/-/blob/master/src/mailman/runners/lmtp.py#L139
        old_unixfrom = msg.get_unixfrom()
        if old_unixfrom is not None and not old_unixfrom.startswith('From '):
            # Adapted from https://github.com/python/cpython/blob/3.9/Lib/email/generator.py
            msg.set_unixfrom('From ' + old_unixfrom + ' ' + time.ctime(time.time()))

        # Mangle the 'From ' lines and add the Unix envelope header.
        fp = BytesIO()
        g = BytesGenerator(fp, mangle_from_=True)
        g.flatten(msg, unixfrom=True)
        msg_bytes = fp.getvalue()

        mbox_path = os.path.join(
            self.mailman2_dir, 'archives', 'private',
            mlist.list_name + '.mbox', mlist.list_name + '.mbox')
        lock_path = os.path.join(
            config.LOCK_DIR, '{0}-pipermail.lock'.format(mlist.list_name))
        lock = Lock(lock_path)

        # Unfortunately the `bin/arch` script needs to seek/tell the mbox
        # file, so we need to write the message to a real file.
        fd, tempfile_name = tempfile.mkstemp()
        fo = os.fdopen(fd, 'wb')
        fo.write(msg_bytes)
        fo.close()

        try:
            lock.lock(timeout=timedelta(seconds=3))

            # Append the message to the mbox file.
            with open(mbox_path, 'ab') as fo:
                # Add a newline before and after to be absolutely sure we don't
                # corrupt the mbox file.
                fo.write(b'\n')
                fo.write(msg_bytes)
                fo.write(b'\n')

            # Send the message to Pipermail.
            args = ['bin/arch', mlist.list_name, tempfile_name]
            proc = Popen(
                args, cwd=self.mailman2_dir,
                stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
            stdout, stderr = proc.communicate()
            if proc.returncode != 0:
                log.error('%s: bin/arch subprocess had non-zero exit code: %s' %
                          (msg.get('message-id'), proc.returncode))
            log.info(stdout.decode())
            log.error(stderr.decode())
        finally:
            lock.unlock(unconditionally=True)
            os.unlink(tempfile_name)
            msg.set_unixfrom(old_unixfrom)

        return None