From a534a91985ba8e6a0f3578ef979690e19922d858 Mon Sep 17 00:00:00 2001 From: Max Erenberg Date: Thu, 29 Apr 2021 21:13:38 +0000 Subject: [PATCH] first commit --- README.md | 82 ++++ ansible.cfg | 2 + coffee/README.md | 7 + coffee/main.yml | 27 ++ dns/README.md | 9 + dns/main.yml | 28 ++ dns/templates/dnsmasq.conf.j2 | 10 + hosts | 18 + mail/README.md | 108 +++++ mail/common.yml | 10 + mail/dovecot/dovecot.conf | 115 ++++++ mail/mailman2/mailman.conf.j2 | 65 +++ mail/mailman2/mailman2.yml | 64 +++ mail/mailman2/mm_cfg.py.j2 | 112 +++++ mail/mailman3/mailman-hyperkitty.cfg.j2 | 20 + mail/mailman3/mailman.cfg.j2 | 282 +++++++++++++ mail/mailman3/mailman.conf.j2 | 82 ++++ mail/mailman3/mailman3-web.cron | 7 + mail/mailman3/mailman3-web.service | 16 + mail/mailman3/mailman3.cron | 4 + mail/mailman3/mailman3.service | 18 + mail/mailman3/mailman3.yml | 173 ++++++++ mail/mailman3/master.cf | 131 ++++++ mail/mailman3/settings.py.j2 | 429 ++++++++++++++++++++ mail/mailman3/urls.py | 35 ++ mail/mailman3/uwsgi.ini | 55 +++ mail/main.yml | 74 ++++ mail/postfix/login_maps.pcre.j2 | 1 + mail/postfix/main.cf.j2 | 100 +++++ mail/postfix/master.cf | 131 ++++++ mail/procmail/procmailrc | 1 + mail/spamassassin/local.cf.j2 | 109 +++++ mail/spamassassin/spamc.conf | 2 + outsider/README.md | 37 ++ outsider/main.yml | 10 + roles/local_users/tasks/main.yml | 29 ++ roles/local_users/templates/muttrc.j2 | 28 ++ roles/local_users/vars/main.yml | 4 + roles/network_setup/handlers/main.yml | 4 + roles/network_setup/tasks/main.yml | 17 + roles/network_setup/templates/hosts.j2 | 6 + roles/network_setup/templates/interfaces.j2 | 10 + roles/systemd_workarounds/handlers/main.yml | 8 + roles/systemd_workarounds/tasks/main.yml | 20 + 44 files changed, 2500 insertions(+) create mode 100644 README.md create mode 100644 ansible.cfg create mode 100644 coffee/README.md create mode 100644 coffee/main.yml create mode 100644 dns/README.md create mode 100644 dns/main.yml create mode 100644 dns/templates/dnsmasq.conf.j2 create mode 100644 hosts create mode 100644 mail/README.md create mode 100644 mail/common.yml create mode 100644 mail/dovecot/dovecot.conf create mode 100644 mail/mailman2/mailman.conf.j2 create mode 100644 mail/mailman2/mailman2.yml create mode 100644 mail/mailman2/mm_cfg.py.j2 create mode 100644 mail/mailman3/mailman-hyperkitty.cfg.j2 create mode 100644 mail/mailman3/mailman.cfg.j2 create mode 100644 mail/mailman3/mailman.conf.j2 create mode 100644 mail/mailman3/mailman3-web.cron create mode 100644 mail/mailman3/mailman3-web.service create mode 100644 mail/mailman3/mailman3.cron create mode 100644 mail/mailman3/mailman3.service create mode 100644 mail/mailman3/mailman3.yml create mode 100644 mail/mailman3/master.cf create mode 100644 mail/mailman3/settings.py.j2 create mode 100644 mail/mailman3/urls.py create mode 100644 mail/mailman3/uwsgi.ini create mode 100644 mail/main.yml create mode 100644 mail/postfix/login_maps.pcre.j2 create mode 100644 mail/postfix/main.cf.j2 create mode 100644 mail/postfix/master.cf create mode 100644 mail/procmail/procmailrc create mode 100644 mail/spamassassin/local.cf.j2 create mode 100644 mail/spamassassin/spamc.conf create mode 100644 outsider/README.md create mode 100644 outsider/main.yml create mode 100644 roles/local_users/tasks/main.yml create mode 100644 roles/local_users/templates/muttrc.j2 create mode 100644 roles/local_users/vars/main.yml create mode 100644 roles/network_setup/handlers/main.yml create mode 100644 roles/network_setup/tasks/main.yml create mode 100644 roles/network_setup/templates/hosts.j2 create mode 100644 roles/network_setup/templates/interfaces.j2 create mode 100644 roles/systemd_workarounds/handlers/main.yml create mode 100644 roles/systemd_workarounds/tasks/main.yml diff --git a/README.md b/README.md new file mode 100644 index 0000000..7d1075d --- /dev/null +++ b/README.md @@ -0,0 +1,82 @@ +# syscom-playground +The objective of this repo is to allow syscom members to create a local +development environment which is reasonably close to the services which +run on the CSC servers. The idea is to encourage experimentation without +breaking the real services and causing outages. + +## Prerequisites +This repo consists of several Ansible playbooks which will automate tasks +in LXC containers. I strongly recommend creating a VM and running the +containers inside the VM to avoid screwing up the network interfaces +on your real computer. I am using KVM + QEMU, but VirtualBox should +theoretically also work. The VM should be running some reasonably +recent version of Debian or Ubuntu. 2 CPU cores and 2 GB of RAM +should be sufficient. + +The VM should be attached to a bridge interface with NAT forwarding. +QEMU should create a default interface like this called 'virbr0'. +For this tutorial, I am assuming that the interface subnet is +192.168.122.0/24, and the bridge IP address on the host is 192.168.122.1. +If you decide to use a different subnet, make sure to update the `hosts` +file accordingly. If you need to edit the subnet which QEMU uses, +do this via virsh or virt-manager; do not modify the subnet manually +using iproute2. The reason for this is because libvirt needs to know +what the subnet is to setup dnsmasq and iptables properly. + +Once the VM is up and running, you will need to create a shared bridge +interface. First, disable the default bridge: +``` +systemctl stop lxc-net +systemctl mask lxc-net +``` +Then paste the following into /etc/network/interfaces: +``` +iface enp1s0 inet manual + +auto lxcbr0 +iface lxcbr0 inet dhcp + bridge_ports enp1s0 + bridge_fd 0 + bridge_maxwait 0 +``` +Replace enp1s0 by the name of the default interface in the VM. +Then, restart the VM. + +Once you have restarted the VM, take note of its IP address on lxcbr0, +and write it to the variable `host_ipv4_addr` in the `hosts` file. + +## Creating the LXC containers +Install the lxc-utils package if you have not done so already: +``` +apt install lxc-utils +``` +For the time being, it is necessary to manually create each container and to +install python3 in it before running the corresponding playbooks. For example, +to setup the DNS container: +``` +lxc-create -t download -n dns -- -d debian -r buster -a amd64 +lxc-start dns +lxc-attach dns +apt update +apt install python3 +exit +``` +The containers should be privileged since the CSC currently uses privileged +LXC containers. If we switch to unprivileged containers in the future, this +repo should be correspondingly updated. + +It is also necessary to have Ansible and the Python LXC driver installed +on the host where the LXC containers are running. e.g. for Debian: +``` +apt install ansible python3-lxc +`` +Now we are ready to run the playbook: +``` +ansible-playbook dns/main.yml +``` +If you see a whole bunch of errors like +``` +RuntimeError: cannot release un-acquired lock +``` +it is safe to ignore those. [Here](https://github.com/lxc/python3-lxc/issues/11) +is the GitHub issue if you are interested. diff --git a/ansible.cfg b/ansible.cfg new file mode 100644 index 0000000..ed865bf --- /dev/null +++ b/ansible.cfg @@ -0,0 +1,2 @@ +[defaults] +inventory = hosts diff --git a/coffee/README.md b/coffee/README.md new file mode 100644 index 0000000..a7af0a7 --- /dev/null +++ b/coffee/README.md @@ -0,0 +1,7 @@ +# Coffee container +This container just hosts the databases for other apps to use. + +## Installation +``` +ansible-playbook main.yml +``` diff --git a/coffee/main.yml b/coffee/main.yml new file mode 100644 index 0000000..9835bde --- /dev/null +++ b/coffee/main.yml @@ -0,0 +1,27 @@ +--- +- hosts: coffee + roles: + - role: ../roles/network_setup + vars: + ipv4_addr: "{{ coffee_ipv4_addr }}" + tasks: + - name: install MariaDB + apt: + name: default-mysql-server + state: present + - name: override systemd services + import_role: + name: ../../roles/systemd_workarounds + vars: + services: [ "mariadb" ] + - name: allow remote connections to MariaDB + lineinfile: + path: /etc/mysql/mariadb.conf.d/50-server.cnf + regexp: "^bind-address\\s+=\\s+127.0.0.1$" + line: "#bind-address = 127.0.0.1" + notify: restart MariaDB + handlers: + - name: restart MariaDB + systemd: + name: mariadb + state: restarted diff --git a/dns/README.md b/dns/README.md new file mode 100644 index 0000000..5f55120 --- /dev/null +++ b/dns/README.md @@ -0,0 +1,9 @@ +## DNS container setup +This doesn't actually reflect the way the CSC manages its nameservers, +but the other containers in this repo need some sort of DNS server +to work properly. + +## Instructions +``` +ansible-playbook main.yml +``` diff --git a/dns/main.yml b/dns/main.yml new file mode 100644 index 0000000..77818d3 --- /dev/null +++ b/dns/main.yml @@ -0,0 +1,28 @@ +--- +- hosts: dns + roles: + - role: ../roles/network_setup + vars: + ipv4_addr: "{{ dns_ipv4_addr }}" + tasks: + - name: install dnsmasq + apt: + name: dnsmasq + state: present + update_cache: true + - name: add hosts file for CNAME + copy: + content: | + {{ mail_ipv4_addr }} mail.{{ base_domain }} + dest: /etc/dnsmasq_hosts + notify: restart dnsmasq + - name: add dnsmasq config + template: + src: templates/dnsmasq.conf.j2 + dest: /etc/dnsmasq.d/dnsmasq.conf + notify: restart dnsmasq + handlers: + - name: restart dnsmasq + systemd: + name: dnsmasq + state: restarted diff --git a/dns/templates/dnsmasq.conf.j2 b/dns/templates/dnsmasq.conf.j2 new file mode 100644 index 0000000..7541d5d --- /dev/null +++ b/dns/templates/dnsmasq.conf.j2 @@ -0,0 +1,10 @@ +no-hosts +no-resolv +server={{ upstream_dns }} +interface=eth0 +addn-hosts=/etc/dnsmasq_hosts +address=/dns.{{ base_domain }}/{{ dns_ipv4_addr }} +address=/mail.{{ base_domain }}/{{ mail_ipv4_addr }} +cname=mailman.{{ base_domain }},mail.{{ base_domain }} +address=/coffee.{{ base_domain }}/{{ coffee_ipv4_addr }} +mx-host={{ base_domain }},mail.{{ base_domain }},50 diff --git a/hosts b/hosts new file mode 100644 index 0000000..a5e73fb --- /dev/null +++ b/hosts @@ -0,0 +1,18 @@ +[containers] +dns ansible_lxc_host=dns +mail ansible_lxc_host=mail +coffee ansible_lxc_host=coffee +outsider ansible_lxc_host=outsider + +[containers:vars] +ansible_connection = lxc +ansible_python_interpreter = python3 +base_domain = csclub.internal +ipv4_subnet = 192.168.122.0/24 +ipv4_gateway = 192.168.122.1 +upstream_dns = 192.168.122.1 +host_ipv4_addr = 192.168.122.226 +outsider_ipv4_addr = 192.168.125.2 +dns_ipv4_addr = 192.168.122.4 +mail_ipv4_addr = 192.168.122.52 +coffee_ipv4_addr = 192.168.122.20 diff --git a/mail/README.md b/mail/README.md new file mode 100644 index 0000000..f574189 --- /dev/null +++ b/mail/README.md @@ -0,0 +1,108 @@ +# Mail container setup +This is one of the trickier ones. + +## Instructions +First, get the email server up and running: +``` +ansible-playbook main.yml +``` +At this point, it would be a good idea to send a few emails back +and forth between the dummy users to make sure everything's working +(their usernames are alice, bob, and eve). [Mutt](http://www.mutt.org/) +has already been setup for each user. +``` +lxc-attach mail +su - alice +mutt +(send an email to bob@csclub.internal) +exit +su - bob +mutt +(check that alice's email got sent) +``` + +## Installing Mailman 2 +``` +ansible-playbook mailman2/mailman2.yml +``` +Attach to the mail container and create a new list, e.g. syscom: +``` +cd /var/lib/mailman +bin/newlist -a syscom root@csclub.internal mailman +``` +Now on your **real** computer (i.e. not the VM), you are going to visit +the web interface for Mailman to adjust some settings and subscribe +some new users. + +First, open `/etc/hosts` on your computer and add the following entry: +``` +192.168.122.52 mailman.csclub.internal +``` + +Now visit http://mailman.csclub.internal/admin/syscom in your browser. +The admin password is 'mailman' (no quotes). + +I suggest going over each setting in the Privacy section and reading it +carefully. Under 'Sender filters', I suggest setting 'generic_nonmember_action' +to 'Accept' rather than 'Hold'. Under 'Recipient filters', I suggest setting +'require_explicit_destination' to 'No'. + +If you are feeling adventurous, add the following to the `/etc/aliases` file +in the mail container: +``` +www-data: root +root: syscom +``` +Then run `postalias /etc/aliases` and `postfix reload`. +This is actually what the CSC uses on the real mail container (on xylitol). +Make sure not to create mailing loops, though, especially when you make one +mailing list the owner of another mailing list. + +You should also subscribe some members to the list and make sure that +messages get sent to them properly. Go to http://mailman.csclub.internal/listinfo/syscom +and subscribe both alice@csclub.internal and bob@csclub.internal. They will +get confirmation messages; go back to the mail container and use Mutt +as each user to confirm their subscriptions. Then, as either bob or alice, +send a message to the syscom list, and make sure that they both received +the message. You should also be able to see the message in Pipermail by +going to http://mailman.csclub.internal/pipermail/syscom/. + +## Installing Mailman 3 +Make sure you have installed Mailman 2 first, since we are going to need +Pipermail. + +We are also going to need the MariaDB database on coffee, so make sure the +coffee container has been setup. + +Run the playbook for Mailman 3: +``` +ansible-playbook mailman3/mailman3.yml +``` + +Run the database migration and collect static files: +``` +cd /opt/mailman3 +su -s /bin/bash www-data +source bin/activate +mailman-web migrate +mailman-web collectstatic +mailman-web compress +exit +systemctl restart mailman3-web +``` + +You will also want to create a superuser account: +``` +cd /opt/mailman3 +su -s /bin/bash www-data +source bin/activate +mailman-web createsuperuser --username bob --email bob@csclub.internal +exit +``` + +Now open http://mailman.csclub.internal in your browser, login as bob, +and start doing things. See the official [Mailman 3 documentation](https://docs.mailman3.org) +and [our wiki](https://wiki.csclub.uwaterloo.ca/Mailing_Lists#Mailman_3) +for information on how to create lists, import lists from Mailman 2, +etc. Make sure to send some messages to the lists which you create +to verify that everything's working. diff --git a/mail/common.yml b/mail/common.yml new file mode 100644 index 0000000..a3ccd4c --- /dev/null +++ b/mail/common.yml @@ -0,0 +1,10 @@ +- name: reload Postfix + command: postfix reload +- name: reload Apache + systemd: + name: apache2 + state: reloaded +- name: restart Apache + systemd: + name: apache2 + state: restarted diff --git a/mail/dovecot/dovecot.conf b/mail/dovecot/dovecot.conf new file mode 100644 index 0000000..868bb0a --- /dev/null +++ b/mail/dovecot/dovecot.conf @@ -0,0 +1,115 @@ +## Dovecot configuration file + +# If you're in a hurry, see http://wiki2.dovecot.org/QuickConfiguration + +# "doveconf -n" command gives a clean output of the changed settings. Use it +# instead of copy&pasting files when posting to the Dovecot mailing list. + +# '#' character and everything after it is treated as comments. Extra spaces +# and tabs are ignored. If you want to use either of these explicitly, put the +# value inside quotes, eg.: key = "# char and trailing whitespace " + +# Most (but not all) settings can be overridden by different protocols and/or +# source/destination IPs by placing the settings inside sections, for example: +# protocol imap { }, local 127.0.0.1 { }, remote 10.0.0.0/8 { } + +# Default values are shown for each setting, it's not required to uncomment +# those. These are exceptions to this though: No sections (e.g. namespace {}) +# or plugin settings are added by default, they're listed only as examples. +# Paths are also just examples with the real defaults being based on configure +# options. The paths listed here are for configure --prefix=/usr +# --sysconfdir=/etc --localstatedir=/var + +# Enable installed protocols +!include_try /usr/share/dovecot/protocols.d/*.protocol + +# A comma separated list of IPs or hosts where to listen in for connections. +# "*" listens in all IPv4 interfaces, "::" listens in all IPv6 interfaces. +# If you want to specify non-default ports or anything more complex, +# edit conf.d/master.conf. +#listen = *, :: + +# Base directory where to store runtime data. +#base_dir = /var/run/dovecot/ + +# Name of this instance. In multi-instance setup doveadm and other commands +# can use -i to select which instance is used (an alternative +# to -c ). The instance name is also added to Dovecot processes +# in ps output. +#instance_name = dovecot + +# Greeting message for clients. +#login_greeting = Dovecot ready. + +# Space separated list of trusted network ranges. Connections from these +# IPs are allowed to override their IP addresses and ports (for logging and +# for authentication checks). disable_plaintext_auth is also ignored for +# these networks. Typically you'd specify your IMAP proxy servers here. +#login_trusted_networks = + +# Space separated list of login access check sockets (e.g. tcpwrap) +#login_access_sockets = + +# With proxy_maybe=yes if proxy destination matches any of these IPs, don't do +# proxying. This isn't necessary normally, but may be useful if the destination +# IP is e.g. a load balancer's IP. +#auth_proxy_self = + +# Show more verbose process titles (in ps). Currently shows user name and +# IP address. Useful for seeing who are actually using the IMAP processes +# (eg. shared mailboxes or if same uid is used for multiple accounts). +#verbose_proctitle = no + +# Should all processes be killed when Dovecot master process shuts down. +# Setting this to "no" means that Dovecot can be upgraded without +# forcing existing client connections to close (although that could also be +# a problem if the upgrade is e.g. because of a security fix). +#shutdown_clients = yes + +# If non-zero, run mail commands via this many connections to doveadm server, +# instead of running them directly in the same process. +#doveadm_worker_count = 0 +# UNIX socket or host:port used for connecting to doveadm server +#doveadm_socket_path = doveadm-server + +# Space separated list of environment variables that are preserved on Dovecot +# startup and passed down to all of its child processes. You can also give +# key=value pairs to always set specific settings. +#import_environment = TZ + +## +## Dictionary server settings +## + +# Dictionary can be used to store key=value lists. This is used by several +# plugins. The dictionary can be accessed either directly or though a +# dictionary server. The following dict block maps dictionary names to URIs +# when the server is used. These can then be referenced using URIs in format +# "proxy::". + +dict { + #quota = mysql:/etc/dovecot/dovecot-dict-sql.conf.ext + #expire = sqlite:/etc/dovecot/dovecot-dict-sql.conf.ext +} + +# Most of the actual configuration gets included below. The filenames are +# first sorted by their ASCII value and parsed in that order. The 00-prefixes +# in filenames are intended to make it easier to understand the ordering. +!include conf.d/*.conf + +# A config file can also tried to be included without giving an error if +# it's not found: +!include_try local.conf + + +mail_location = maildir:~/.maildir +disable_plaintext_auth = no + +# Authentication service for Postfix +service auth { + unix_listener /var/spool/postfix/private/auth { + mode = 0666 + user = postfix + group = postfix + } +} diff --git a/mail/mailman2/mailman.conf.j2 b/mail/mailman2/mailman.conf.j2 new file mode 100644 index 0000000..677f2a4 --- /dev/null +++ b/mail/mailman2/mailman.conf.j2 @@ -0,0 +1,65 @@ + + ServerName mailman.{{ base_domain }} + ServerAlias mailman + ServerAdmin root@{{ base_domain }} + + DocumentRoot /usr/lib/cgi-bin/mailman + + Options FollowSymLinks + AllowOverride None + + + RedirectMatch ^/$ /listinfo + + Alias /pipermail/ /var/lib/mailman/archives/public/ + Alias /images/mailman/ /usr/share/images/mailman/ + ScriptAlias /admin /usr/lib/cgi-bin/mailman/admin + ScriptAlias /admindb /usr/lib/cgi-bin/mailman/admindb + ScriptAlias /confirm /usr/lib/cgi-bin/mailman/confirm + ScriptAlias /create /usr/lib/cgi-bin/mailman/create + ScriptAlias /edithtml /usr/lib/cgi-bin/mailman/edithtml + ScriptAlias /listinfo /usr/lib/cgi-bin/mailman/listinfo + ScriptAlias /options /usr/lib/cgi-bin/mailman/options + ScriptAlias /private /usr/lib/cgi-bin/mailman/private + ScriptAlias /rmlist /usr/lib/cgi-bin/mailman/rmlist + ScriptAlias /roster /usr/lib/cgi-bin/mailman/roster + ScriptAlias /subscribe /usr/lib/cgi-bin/mailman/subscribe + ScriptAlias /mailman/ /usr/lib/cgi-bin/mailman/ + ScriptAlias /cgi-bin/mailman/ /usr/lib/cgi-bin/mailman/ + + + Options ExecCGI + SetHandler cgi-script + AllowOverride None + Order allow,deny + Allow from all + + + Options FollowSymlinks + AllowOverride None + #Order allow,deny + #Allow from all + Require all granted + + + Options FollowSymlinks + AllowOverride None + Order allow,deny + Allow from all + + + Options Indexes FollowSymlinks + AllowOverride None + #Order allow,deny + #Allow from all + Require all granted + + + ErrorLog ${APACHE_LOG_DIR}/mailman-error.log + + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/mailman-access.log combined + diff --git a/mail/mailman2/mailman2.yml b/mail/mailman2/mailman2.yml new file mode 100644 index 0000000..be58991 --- /dev/null +++ b/mail/mailman2/mailman2.yml @@ -0,0 +1,64 @@ +--- +- hosts: mail + vars: + # use this password for all mailing lists + list_password: mailman + tasks: + - name: install packages for Mailman 2 + apt: + name: "{{ item }}" + state: present + loop: + - mailman + - apache2 + - name: override systemd services + import_role: + name: ../../roles/systemd_workarounds + vars: + services: [ "apache2" ] + - name: add Mailman config + template: + src: mm_cfg.py.j2 + dest: /etc/mailman/mm_cfg.py + - name: create Mailman aliases file + command: + chdir: /var/lib/mailman + cmd: bin/genaliases + creates: /var/lib/mailman/data/aliases + - name: create initial list + shell: + chdir: /var/lib/mailman + cmd: "bin/newlist -a mailman root@{{ base_domain }} {{ list_password }} || true" + - name: restart Mailman + systemd: + name: mailman + state: restarted + - name: add Mailman aliases to Postfix config + lineinfile: + path: /etc/postfix/main.cf + regexp: "^alias_maps = .*$" + line: "alias_maps = hash:/etc/aliases, hash:/var/lib/mailman/data/aliases" + notify: reload Postfix + - name: add Apache config + template: + src: mailman.conf.j2 + dest: /etc/apache2/sites-available/mailman.conf + notify: reload Apache + - name: enable Mailman site + command: + cmd: a2ensite mailman.conf + creates: /etc/apache2/sites-enabled/mailman.conf + notify: reload Apache + - name: disable default site + command: + cmd: a2dissite 000-default.conf + removes: /etc/apache2/sites-enabled/000-default.conf + notify: reload Apache + - name: enable CGI on Apache + command: + cmd: a2enmod cgid + creates: /etc/apache2/mods-enabled/cgid.load + notify: restart Apache + handlers: + - name: _imports + import_tasks: ../common.yml diff --git a/mail/mailman2/mm_cfg.py.j2 b/mail/mailman2/mm_cfg.py.j2 new file mode 100644 index 0000000..8621b37 --- /dev/null +++ b/mail/mailman2/mm_cfg.py.j2 @@ -0,0 +1,112 @@ +# -*- python -*- + +# Copyright (C) 1998,1999,2000 by the Free Software Foundation, Inc. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA +# 02110-1301 USA + + +"""This is the module which takes your site-specific settings. + +From a raw distribution it should be copied to mm_cfg.py. If you +already have an mm_cfg.py, be careful to add in only the new settings +you want. The complete set of distributed defaults, with annotation, +are in ./Defaults. In mm_cfg, override only those you want to +change, after the + + from Defaults import * + +line (see below). + +Note that these are just default settings - many can be overridden via the +admin and user interfaces on a per-list or per-user basis. + +Note also that some of the settings are resolved against the active list +setting by using the value as a format string against the +list-instance-object's dictionary - see the distributed value of +DEFAULT_MSG_FOOTER for an example.""" + + +####################################################### +# Here's where we get the distributed defaults. # + +from Defaults import * + +############################################################## +# Put YOUR site-specific configuration below, in mm_cfg.py . # +# See Defaults.py for explanations of the values. # + +#------------------------------------------------------------- +# The name of the list Mailman uses to send password reminders +# and similar. Don't change if you want mailman-owner to be +# a valid local part. +MAILMAN_SITE_LIST = 'mailman' + +#------------------------------------------------------------- +# If you change these, you have to configure your http server +# accordingly (Alias and ScriptAlias directives in most httpds) +DEFAULT_URL_PATTERN = 'http://%s/' +PRIVATE_ARCHIVE_URL = '/private' +IMAGE_LOGOS = '/images/mailman/' + +#------------------------------------------------------------- +# Default domain for email addresses of newly created MLs +DEFAULT_EMAIL_HOST = '{{ base_domain }}' +#------------------------------------------------------------- +# Default host for web interface of newly created MLs +DEFAULT_URL_HOST = 'mailman.{{ base_domain }}' +#------------------------------------------------------------- +# Required when setting any of its arguments. +add_virtualhost(DEFAULT_URL_HOST, DEFAULT_EMAIL_HOST) + +#------------------------------------------------------------- +# The default language for this server. +DEFAULT_SERVER_LANGUAGE = 'en' + +#------------------------------------------------------------- +# Iirc this was used in pre 2.1, leave it for now +USE_ENVELOPE_SENDER = 0 # Still used? + +#------------------------------------------------------------- +# Unset send_reminders on newly created lists +DEFAULT_SEND_REMINDERS = 0 + +#------------------------------------------------------------- +# Uncomment this if you configured your MTA such that it +# automatically recognizes newly created lists. +# (see /usr/share/doc/mailman/README.Exim4.Debian or +# /usr/share/mailman/postfix-to-mailman.py) +# MTA=None # Misnomer, suppresses alias output on newlist + +#------------------------------------------------------------- +# Uncomment if you use Postfix virtual domains (but not +# postfix-to-mailman.py), but be sure to see +# /usr/share/doc/mailman/README.Debian first. +MTA='Postfix' + +DELIVERY_MODULE = 'SMTPDirect' +SMTPHOST = '127.0.0.1' +SMTPPORT = 10025 + +#------------------------------------------------------------- +# Uncomment if you want to filter mail with SpamAssassin. For +# more information please visit this website: +# http://www.jamesh.id.au/articles/mailman-spamassassin/ +# GLOBAL_PIPELINE.insert(1, 'SpamAssassin') + +# Note - if you're looking for something that is imported from mm_cfg, but you +# didn't find it above, it's probably in /usr/lib/mailman/Mailman/Defaults.py. + +POSTFIX_STYLE_VIRTUAL_DOMAINS = [] diff --git a/mail/mailman3/mailman-hyperkitty.cfg.j2 b/mail/mailman3/mailman-hyperkitty.cfg.j2 new file mode 100644 index 0000000..76e2b55 --- /dev/null +++ b/mail/mailman3/mailman-hyperkitty.cfg.j2 @@ -0,0 +1,20 @@ +# This is the mailman extension configuration file to enable HyperKitty as an +# archiver. Remember to add the following lines in the mailman.cfg file: +# +# [archiver.hyperkitty] +# class: mailman_hyperkitty.Archiver +# enable: yes +# configuration: /etc/mailman3/mailman-hyperkitty.cfg +# + +[general] + +# This is your HyperKitty installation, preferably on the localhost. This +# address will be used by Mailman to forward incoming emails to HyperKitty +# for archiving. It does not need to be publicly available, in fact it's +# better if it is not. +base_url: http://mailman.{{ base_domain }}/hyperkitty/ + +# Shared API key, must be the identical to the value in HyperKitty's +# settings. +api_key: mailman3 diff --git a/mail/mailman3/mailman.cfg.j2 b/mail/mailman3/mailman.cfg.j2 new file mode 100644 index 0000000..1b0ba11 --- /dev/null +++ b/mail/mailman3/mailman.cfg.j2 @@ -0,0 +1,282 @@ +# Copyright (C) 2008-2017 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see . + +# This file contains the Debian configuration for mailman. It uses ini-style +# formats under the lazr.config regime to define all system configuration +# options. See for details. + + +[mailman] +# This address is the "site owner" address. Certain messages which must be +# delivered to a human, but which can't be delivered to a list owner (e.g. a +# bounce from a list owner), will be sent to this address. It should point to +# a human. +site_owner: root@{{ base_domain }} + +# This is the local-part of an email address used in the From field whenever a +# message comes from some entity to which there is no natural reply recipient. +# Mailman will append '@' and the host name of the list involved. This +# address must not bounce and it must not point to a Mailman process. +noreply_address: noreply + +# The default language for this server. +default_language: en + +# Membership tests for posting purposes are usually performed by looking at a +# set of headers, passing the test if any of their values match a member of +# the list. Headers are checked in the order given in this variable. The +# value From_ means to use the envelope sender. Field names are case +# insensitive. This is a space separate list of headers. +sender_headers: from from_ reply-to sender + +# Mail command processor will ignore mail command lines after designated max. +email_commands_max_lines: 10 + +# Default length of time a pending request is live before it is evicted from +# the pending database. +pending_request_life: 3d + +# How long should files be saved before they are evicted from the cache? +cache_life: 7d + +# A callable to run with no arguments early in the initialization process. +# This runs before database initialization. +pre_hook: + +# A callable to run with no arguments late in the initialization process. +# This runs after adapters are initialized. +post_hook: + +# Which paths.* file system layout to use. +# You should not change this variable. +layout: venv + +# Can MIME filtered messages be preserved by list owners? +filtered_messages_are_preservable: no + +# How should text/html parts be converted to text/plain when the mailing list +# is set to convert HTML to plaintext? This names a command to be called, +# where the substitution variable $filename is filled in by Mailman, and +# contains the path to the temporary file that the command should read from. +# The command should print the converted text to stdout. +html_to_plain_text_command: /usr/bin/lynx -dump $filename + +# Specify what characters are allowed in list names. Characters outside of +# the class [-_.+=!$*{}~0-9a-z] matched case insensitively are never allowed, +# but this specifies a subset as the only allowable characters. This must be +# a valid character class regexp or the effect on list creation is +# unpredictable. +listname_chars: [-_.0-9a-z] + + +[shell] +# `mailman shell` (also `withlist`) gives you an interactive prompt that you +# can use to interact with an initialized and configured Mailman system. Use +# --help for more information. This section allows you to configure certain +# aspects of this interactive shell. + +# Customize the interpreter prompt. +prompt: >>> + +# Banner to show on startup. +banner: Welcome to the GNU Mailman shell + +# Use IPython as the shell, which must be found on the system. Valid values +# are `no`, `yes`, and `debug` where the latter is equivalent to `yes` except +# that any import errors will be displayed to stderr. +use_ipython: no + +# Set this to allow for command line history if readline is available. This +# can be as simple as $var_dir/history.py to put the file in the var directory. +history_file: + + +[paths.venv] +# Important directories for Mailman operation. These are defined here so that +# different layouts can be supported. For example, a developer layout would +# be different from a FHS layout. Most paths are based off the var_dir, and +# often just setting that will do the right thing for all the other paths. +# You might also have to set spool_dir though. +# +# Substitutions are allowed, but must be of the form $var where 'var' names a +# configuration variable in the paths.* section. Substitutions are expanded +# recursively until no more $-variables are present. Beware of infinite +# expansion loops! +# +# This is the root of the directory structure that Mailman will use to store +# its run-time data. +var_dir: /opt/mailman3 +# This is where the Mailman queue files directories will be created. +queue_dir: $var_dir/queue +# This is the directory containing the Mailman 'runner' and 'master' commands +# if set to the string '$argv', it will be taken as the directory containing +# the 'mailman' command. +bin_dir: /opt/mailman3/bin +# All list-specific data. +list_data_dir: $var_dir/lists +# Directory where log files go. +log_dir: /var/log/mailman3 +# Directory for system-wide locks. +lock_dir: $var_dir/locks +# Directory for system-wide data. +data_dir: $var_dir/data +# Cache files. +cache_dir: $var_dir/cache +# Directory for configuration files and such. +etc_dir: /etc/mailman3 +# Directory containing Mailman plugins. +ext_dir: $var_dir/ext +# Directory where the default IMessageStore puts its messages. +messages_dir: $var_dir/messages +# Directory for archive backends to store their messages in. Archivers should +# create a subdirectory in here to store their files. +archive_dir: $var_dir/archives +# Root directory for site-specific template override files. +template_dir: $var_dir/templates +# There are also a number of paths to specific file locations that can be +# defined. For these, the directory containing the file must already exist, +# or be one of the directories created by Mailman as per above. +# +# This is where PID file for the master runner is stored. +pid_file: /run/mailman3/master.pid +# Lock file. +lock_file: $lock_dir/master.lck + + +[database] +# The class implementing the IDatabase. +#class: mailman.database.sqlite.SQLiteDatabase +class: mailman.database.mysql.MySQLDatabase +#class: mailman.database.postgresql.PostgreSQLDatabase + +# Use this to set the Storm database engine URL. You generally have one +# primary database connection for all of Mailman. List data and most rosters +# will store their data in this database, although external rosters may access +# other databases in their own way. This string supports standard +# 'configuration' substitutions. +#url: sqlite:///$DATA_DIR/mailman.db +url: mysql+mysqldb://mailman3:mailman3@coffee/mailman3?charset=utf8&use_unicode=1 +#url: postgres://mailman3:mmpass@localhost/mailman3 + +debug: no + + +[logging.debian] +# This defines various log settings. The options available are: +# +# - level -- Overrides the default level; this may be any of the +# standard Python logging levels, case insensitive. +# - format -- Overrides the default format string +# - datefmt -- Overrides the default date format string +# - path -- Overrides the default logger path. This may be a relative +# path name, in which case it is relative to Mailman's LOG_DIR, +# or it may be an absolute path name. You cannot change the +# handler class that will be used. +# - propagate -- Boolean specifying whether to propagate log message from this +# logger to the root "mailman" logger. You cannot override +# settings for the root logger. +# +# In this section, you can define defaults for all loggers, which will be +# prefixed by 'mailman.'. Use subsections to override settings for specific +# loggers. The names of the available loggers are: +# +# - archiver -- All archiver output +# - bounce -- All bounce processing logs go here +# - config -- Configuration issues +# - database -- Database logging (SQLAlchemy and Alembic) +# - debug -- Only used for development +# - error -- All exceptions go to this log +# - fromusenet -- Information related to the Usenet to Mailman gateway +# - http -- Internal wsgi-based web interface +# - locks -- Lock state changes +# - mischief -- Various types of hostile activity +# - runner -- Runner process start/stops +# - smtp -- Successful SMTP activity +# - smtp-failure -- Unsuccessful SMTP activity +# - subscribe -- Information about leaves/joins +# - vette -- Message vetting information +format: %(asctime)s (%(process)d) %(message)s +datefmt: %b %d %H:%M:%S %Y +propagate: no +level: info +path: mailman.log + +[webservice] +# The hostname at which admin web service resources are exposed. +hostname: localhost + +# The port at which the admin web service resources are exposed. +port: 8001 + +# Whether or not requests to the web service are secured through SSL. +use_https: no + +# Whether or not to show tracebacks in an HTTP response for a request that +# raised an exception. +show_tracebacks: yes + +# The API version number for the current (highest) API. +api_version: 3.1 + +# The administrative username. +admin_user: restadmin + +# The administrative password. +admin_pass: mailman3 + +[mta] +# The class defining the interface to the incoming mail transport agent. +#incoming: mailman.mta.exim4.LMTP +incoming: mailman.mta.postfix.LMTP + +# The callable implementing delivery to the outgoing mail transport agent. +# This must accept three arguments, the mailing list, the message, and the +# message metadata dictionary. +outgoing: mailman.mta.deliver.deliver + +# How to connect to the outgoing MTA. If smtp_user and smtp_pass is given, +# then Mailman will attempt to log into the MTA when making a new connection. +smtp_host: localhost +smtp_port: 10025 +smtp_user: +smtp_pass: + +# Where the LMTP server listens for connections. Use 127.0.0.1 instead of +# localhost for Postfix integration, because Postfix only consults DNS +# (e.g. not /etc/hosts). +lmtp_host: 127.0.0.1 +lmtp_port: 8024 + +# Where can we find the mail server specific configuration file? The path can +# be either a file system path or a Python import path. If the value starts +# with python: then it is a Python import path, otherwise it is a file system +# path. File system paths must be absolute since no guarantees are made about +# the current working directory. Python paths should not include the trailing +# .cfg, which the file must end with. +#configuration: python:mailman.config.exim4 +configuration: python:mailman.config.postfix + +# The following lines are specific to mailing lists archiving using +# HyperKitty. They require 'python3-mailman-hyperkitty' to be installed +# and will produce errors otherwise. +# +# If you don't want to use HyperKitty, please comment them out. + +[archiver.hyperkitty] +class: mailman_hyperkitty.Archiver +enable: yes +configuration: /etc/mailman3/mailman-hyperkitty.cfg diff --git a/mail/mailman3/mailman.conf.j2 b/mail/mailman3/mailman.conf.j2 new file mode 100644 index 0000000..e1f49e4 --- /dev/null +++ b/mail/mailman3/mailman.conf.j2 @@ -0,0 +1,82 @@ + + ServerName mailman.{{ base_domain }} + ServerAlias mailman + ServerAdmin root@{{ base_domain }} + + # MAIMLAN 3 CONFIG + Alias /favicon.ico /opt/mailman3/web/static/postorius/img/favicon.ico + Alias /static /opt/mailman3/web/static + + + Require all granted + + + + ProxyPass /pipermail ! + ProxyPass /images/mailman ! + ProxyPass /favicon.ico ! + ProxyPass /static ! + ProxyPass / unix:/run/mailman3-web/uwsgi.sock|uwsgi://localhost/ + + + # MAILMAN 2 CONFIG + #DocumentRoot /usr/lib/cgi-bin/mailman + # + # Options FollowSymLinks + # AllowOverride None + # + # + #RedirectMatch ^/$ /listinfo + # + Alias /pipermail/ /var/lib/mailman/archives/public/ + Alias /images/mailman/ /usr/share/images/mailman/ + #ScriptAlias /admin /usr/lib/cgi-bin/mailman/admin + #ScriptAlias /admindb /usr/lib/cgi-bin/mailman/admindb + #ScriptAlias /confirm /usr/lib/cgi-bin/mailman/confirm + #ScriptAlias /create /usr/lib/cgi-bin/mailman/create + #ScriptAlias /edithtml /usr/lib/cgi-bin/mailman/edithtml + #ScriptAlias /listinfo /usr/lib/cgi-bin/mailman/listinfo + #ScriptAlias /options /usr/lib/cgi-bin/mailman/options + #ScriptAlias /private /usr/lib/cgi-bin/mailman/private + #ScriptAlias /rmlist /usr/lib/cgi-bin/mailman/rmlist + #ScriptAlias /roster /usr/lib/cgi-bin/mailman/roster + #ScriptAlias /subscribe /usr/lib/cgi-bin/mailman/subscribe + #ScriptAlias /mailman/ /usr/lib/cgi-bin/mailman/ + #ScriptAlias /cgi-bin/mailman/ /usr/lib/cgi-bin/mailman/ + # + # + # Options ExecCGI + # SetHandler cgi-script + # AllowOverride None + # Order allow,deny + # Allow from all + # + # + # Options FollowSymlinks + # AllowOverride None + # #Order allow,deny + # #Allow from all + # Require all granted + # + + Options FollowSymlinks + AllowOverride None + Order allow,deny + Allow from all + + + Options Indexes FollowSymlinks + AllowOverride None + #Order allow,deny + #Allow from all + Require all granted + + + ErrorLog ${APACHE_LOG_DIR}/mailman-error.log + + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + + CustomLog ${APACHE_LOG_DIR}/mailman-access.log combined + diff --git a/mail/mailman3/mailman3-web.cron b/mail/mailman3/mailman3-web.cron new file mode 100644 index 0000000..042961e --- /dev/null +++ b/mail/mailman3/mailman3-web.cron @@ -0,0 +1,7 @@ +*/15 * * * * www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.minutely /opt/mailman3/bin/mailman-web runjobs minutely +2,17,32,47 * * * * www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.quarter_hourly /opt/mailman3/bin/mailman-web runjobs quarter_hourly +@hourly www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.hourly /opt/mailman3/bin/mailman-web runjobs hourly +@daily www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.daily /opt/mailman3/bin/mailman-web runjobs daily +@weekly www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.weekly /opt/mailman3/bin/mailman-web runjobs weekly +@monthly www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.monthly /opt/mailman3/bin/mailman-web runjobs monthly +@yearly www-data [ -f /opt/mailman3/bin/django-admin ] && flock -n /run/mailman3-web/cron.yearly /opt/mailman3/bin/mailman-web runjobs yearly diff --git a/mail/mailman3/mailman3-web.service b/mail/mailman3/mailman3-web.service new file mode 100644 index 0000000..9499d23 --- /dev/null +++ b/mail/mailman3/mailman3-web.service @@ -0,0 +1,16 @@ +[Unit] +Description=Mailman3-web uWSGI service +After=network.target + +[Service] +ExecStart=/usr/bin/uwsgi --ini /etc/mailman3/uwsgi.ini +Restart=on-failure +KillSignal=SIGQUIT +Type=notify +StandardError=syslog +NotifyAccess=all +User=root +Group=root + +[Install] +WantedBy=multi-user.target diff --git a/mail/mailman3/mailman3.cron b/mail/mailman3/mailman3.cron new file mode 100644 index 0000000..b567945 --- /dev/null +++ b/mail/mailman3/mailman3.cron @@ -0,0 +1,4 @@ +SHELL=/bin/sh +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +0 0 * * * list if [ -x /opt/mailman3/bin/mailman ]; then /opt/mailman3/bin/mailman digests --send; fi diff --git a/mail/mailman3/mailman3.service b/mail/mailman3/mailman3.service new file mode 100644 index 0000000..2d8693b --- /dev/null +++ b/mail/mailman3/mailman3.service @@ -0,0 +1,18 @@ +[Unit] +Description=Mailman3 server +Documentation=man:mailman(1) +Documentation=https://mailman.readthedocs.io/ +ConditionPathExists=/etc/mailman3/mailman.cfg + +[Service] +ExecStart=/opt/mailman3/bin/mailman -C /etc/mailman3/mailman.cfg start --force +ExecReload=/opt/mailman3/bin/mailman -C /etc/mailman3/mailman.cfg restart +ExecStop=/opt/mailman3/bin/mailman -C /etc/mailman3/mailman.cfg stop +Type=forking +PIDFile=/run/mailman3/master.pid +SyslogIdentifier=mailman3 +User=list +Group=list + +[Install] +WantedBy=multi-user.target diff --git a/mail/mailman3/mailman3.yml b/mail/mailman3/mailman3.yml new file mode 100644 index 0000000..7c56502 --- /dev/null +++ b/mail/mailman3/mailman3.yml @@ -0,0 +1,173 @@ +- hosts: coffee + tasks: + - name: setup the database on coffee + command: + cmd: mysql + stdin: | + CREATE DATABASE IF NOT EXISTS {{ item }}; + CREATE USER IF NOT EXISTS {{ item }} IDENTIFIED BY '{{ item }}'; + GRANT ALL PRIVILEGES ON {{ item }}.* TO {{ item }}; + loop: + - mailman3 + - mailman3web +- hosts: mail + tasks: + - name: install Mailman 3 prerequisites + apt: + name: "{{ item }}" + loop: + - python3-pip + - python3-dev + - python3-xapian + - virtualenv + - uwsgi + - uwsgi-plugin-python3 + - default-libmysqlclient-dev + - sassc + - lynx + - git + - memcached + - name: override systemd services + import_role: + name: ../../roles/systemd_workarounds + vars: + services: [ "memcached" ] + - name: upgrade pip + pip: + executable: pip3 + name: pip + extra_args: --upgrade + - name: create mailman3 directory + file: + path: /opt/mailman3 + state: directory + owner: list + group: list + mode: '2755' + - name: create mailman3-web directory + file: + path: /opt/mailman3/web + state: directory + owner: www-data + group: www-data + - name: create mailman3-web run directory + file: + path: /run/mailman3-web + state: directory + owner: www-data + group: www-data + - name: install pip packages + become_user: list + pip: + virtualenv: /opt/mailman3 + virtualenv_python: python3 + name: "{{ item }}" + loop: + - mysqlclient + - pylibmc + - git+https://github.com/notanumber/xapian-haystack.git + - mailman + - mailman-web + - mailman-hyperkitty + - name: create mailman3 folder + file: + path: /etc/mailman3 + state: directory + mode: 0755 + - name: add Mailman 3 configs + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + group: "{{ item.group }}" + mode: 0640 + loop: + - src: mailman.cfg.j2 + dest: /etc/mailman3/mailman.cfg + group: list + - src: mailman-hyperkitty.cfg.j2 + dest: /etc/mailman3/mailman-hyperkitty.cfg + group: list + - src: settings.py.j2 + dest: /etc/mailman3/settings.py + group: www-data + - src: urls.py + dest: /etc/mailman3/urls.py + group: www-data + - src: uwsgi.ini + dest: /etc/mailman3/uwsgi.ini + group: www-data + - name: update cron log level + lineinfile: + path: /etc/default/cron + line: 'EXTRA_OPTS="-L 4"' + notify: restart cron + - name: add new services + copy: + src: "{{ item }}.service" + dest: "/etc/systemd/system/{{ item }}.service" + loop: + - mailman3 + - mailman3-web + notify: reload systemd + - name: enable and start new services + systemd: + name: "{{ item }}" + enabled: true + state: started + loop: + - mailman3 + - mailman3-web + - name: add cron jobs + copy: + src: "{{ item }}.cron" + dest: "/etc/cron.d/{{ item }}" + loop: + - mailman3 + - mailman3-web + - name: enable mod_proxy_uwsgi + command: + cmd: a2enmod proxy_uwsgi + creates: /etc/apache2/mods-enabled/proxy_uwsgi.load + notify: restart Apache + - name: update Apache config + template: + src: mailman.conf.j2 + dest: /etc/apache2/sites-available/mailman.conf + notify: reload Apache + - name: update Postfix config + blockinfile: + path: /etc/postfix/main.cf + block: | + owner_request_special = no + transport_maps = hash:/opt/mailman3/data/postfix_lmtp + local_recipient_maps = + proxy:unix:passwd.byname, + $alias_maps, + hash:/opt/mailman3/data/postfix_lmtp + notify: reload Postfix + - name: disable Mailman 2 in Postfix main.cf + lineinfile: + path: /etc/postfix/main.cf + regexp: "^alias_maps = .*$" + line: "alias_maps = hash:/etc/aliases" + notify: reload Postfix + - name: disable Mailman 2 in Postfix master.cf + copy: + src: master.cf + dest: /etc/postfix/master.cf + notify: reload Postfix + - name: disable Mailman 2 cron jobs + replace: + path: /etc/cron.d/mailman + regexp: "^([*\\d@].*)$" + replace: "### \\1" + handlers: + - name: _imports + import_tasks: ../common.yml + - name: reload systemd + systemd: + daemon_reload: true + - name: restart cron + systemd: + name: cron + state: restarted diff --git a/mail/mailman3/master.cf b/mail/mailman3/master.cf new file mode 100644 index 0000000..fa1e48b --- /dev/null +++ b/mail/mailman3/master.cf @@ -0,0 +1,131 @@ +# +# Postfix master process configuration file. For details on the format +# of the file, see the master(5) manual page (command: "man 5 master" or +# on-line: http://www.postfix.org/master.5.html). +# +# Do not forget to execute "postfix reload" after editing this file. +# +# ========================================================================== +# service type private unpriv chroot wakeup maxproc command + args +# (yes) (yes) (no) (never) (100) +# ========================================================================== +smtp inet n - n - - smtpd +#smtp inet n - y - 1 postscreen +#smtpd pass - - y - - smtpd +#dnsblog unix - - y - 0 dnsblog +#tlsproxy unix - - y - 0 tlsproxy +submission inet n - n - - smtpd + -o smtpd_sasl_auth_enable=yes + -o smtpd_client_restrictions=permit_sasl_authenticated,reject + -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject + -o smtpd_helo_restrictions=reject_invalid_helo_hostname,permit + -o milter_macro_daemon_name=ORIGINATING +#smtps inet n - y - - smtpd +# -o syslog_name=postfix/smtps +# -o smtpd_tls_wrappermode=yes +# -o smtpd_sasl_auth_enable=yes +# -o smtpd_reject_unlisted_recipient=no +# -o smtpd_client_restrictions=$mua_client_restrictions +# -o smtpd_helo_restrictions=$mua_helo_restrictions +# -o smtpd_sender_restrictions=$mua_sender_restrictions +# -o smtpd_recipient_restrictions= +# -o smtpd_relay_restrictions=permit_sasl_authenticated,reject +# -o milter_macro_daemon_name=ORIGINATING +#628 inet n - y - - qmqpd +pickup unix n - y 60 1 pickup +cleanup unix n - y - 0 cleanup +qmgr unix n - n 300 1 qmgr +#qmgr unix n - n 300 1 oqmgr +tlsmgr unix - - y 1000? 1 tlsmgr +rewrite unix - - y - - trivial-rewrite +bounce unix - - y - 0 bounce +defer unix - - y - 0 bounce +trace unix - - y - 0 bounce +verify unix - - y - 1 verify +flush unix n - y 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - y - - smtp +relay unix - - y - - smtp + -o syslog_name=postfix/$service_name +# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 +showq unix n - y - - showq +error unix - - y - - error +retry unix - - y - - error +discard unix - - y - - discard +local unix - n n - - local +virtual unix - n n - - virtual +lmtp unix - - y - - lmtp +anvil unix - - y - 1 anvil +scache unix - - y - 1 scache +postlog unix-dgram n - n - 1 postlogd +# +# ==================================================================== +# Interfaces to non-Postfix software. Be sure to examine the manual +# pages of the non-Postfix software to find out what options it wants. +# +# Many of the following services use the Postfix pipe(8) delivery +# agent. See the pipe(8) man page for information about ${recipient} +# and other message envelope options. +# ==================================================================== +# +# maildrop. See the Postfix MAILDROP_README file for details. +# Also specify in main.cf: maildrop_destination_recipient_limit=1 +# +maildrop unix - n n - - pipe + flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} +# +# ==================================================================== +# +# Recent Cyrus versions can use the existing "lmtp" master.cf entry. +# +# Specify in cyrus.conf: +# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 +# +# Specify in main.cf one or more of the following: +# mailbox_transport = lmtp:inet:localhost +# virtual_transport = lmtp:inet:localhost +# +# ==================================================================== +# +# Cyrus 2.1.5 (Amos Gouaux) +# Also specify in main.cf: cyrus_destination_recipient_limit=1 +# +#cyrus unix - n n - - pipe +# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} +# +# ==================================================================== +# Old example of delivery via Cyrus. +# +#old-cyrus unix - n n - - pipe +# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} +# +# ==================================================================== +# +# See the Postfix UUCP_README file for configuration details. +# +uucp unix - n n - - pipe + flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) +# +# Other external delivery methods. +# +ifmail unix - n n - - pipe + flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) +bsmtp unix - n n - - pipe + flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient +scalemail-backend unix - n n - 2 pipe + flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} +#mailman unix - n n - - pipe +# flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py +# ${nexthop} ${user} + +# Mailman +127.0.0.1:10025 inet n - n - 16 smtpd + -o content_filter= + -o receive_override_options=no_unknown_recipient_checks,no_header_body_checks + -o smtpd_helo_restrictions=permit_mynetworks,reject + -o smtpd_client_restrictions=permit_mynetworks,reject + -o smtpd_sender_restrictions=permit_mynetworks,reject + -o smtpd_recipient_restrictions=permit_mynetworks,reject + -o mynetworks_style=host + -o smtpd_authorized_xforward_hosts=127.0.0.0/8 diff --git a/mail/mailman3/settings.py.j2 b/mail/mailman3/settings.py.j2 new file mode 100644 index 0000000..0f9db8e --- /dev/null +++ b/mail/mailman3/settings.py.j2 @@ -0,0 +1,429 @@ +""" +Django Settings for Mailman Suite (hyperkitty + postorius) + +For more information on this file, see +https://docs.djangoproject.com/en/1.8/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.8/ref/settings/ +""" +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = False + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'very_very_secret' + +ADMINS = ( + ('Mailman Suite Admin', 'root@localhost'), +) + +SITE_ID = 1 + +# Hosts/domain names that are valid for this site; required if DEBUG is False +# See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts +# Set to '*' per default in the Deian package to allow all hostnames. Mailman3 +# is meant to run behind a webserver reverse proxy anyway. +ALLOWED_HOSTS = [ + #"localhost", # Archiving API from Mailman, keep it. + # "lists.your-domain.org", + # Add here all production URLs you may have. + '*' +] + +# Mailman API credentials +MAILMAN_REST_API_URL = 'http://localhost:8001' +MAILMAN_REST_API_USER = 'restadmin' +MAILMAN_REST_API_PASS = 'mailman3' +MAILMAN_ARCHIVER_KEY = 'mailman3' +MAILMAN_ARCHIVER_FROM = ( + '127.0.0.1', + '::1', + # IP addresses for this machine + '{{ mail_ipv4_addr }}', +) + +# Application definition + +INSTALLED_APPS = ( + 'hyperkitty', + 'postorius', + 'django_mailman3', + # Uncomment the next line to enable the admin: + 'django.contrib.admin', + # Uncomment the next line to enable admin documentation: + # 'django.contrib.admindocs', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.sites', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'django_gravatar', + 'compressor', + 'haystack', + 'django_extensions', + 'django_q', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', + #'django_mailman3.lib.auth.fedora', + #'allauth.socialaccount.providers.openid', + #'allauth.socialaccount.providers.github', + #'allauth.socialaccount.providers.gitlab', + #'allauth.socialaccount.providers.google', + #'allauth.socialaccount.providers.facebook', + #'allauth.socialaccount.providers.twitter', + #'allauth.socialaccount.providers.stackexchange', +) + +MIDDLEWARE = ( + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.middleware.locale.LocaleMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.security.SecurityMiddleware', + 'django_mailman3.middleware.TimezoneMiddleware', + 'postorius.middleware.PostoriusMiddleware', +) + +ROOT_URLCONF = 'urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.i18n', + 'django.template.context_processors.media', + 'django.template.context_processors.static', + 'django.template.context_processors.tz', + 'django.template.context_processors.csrf', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + 'django_mailman3.context_processors.common', + 'hyperkitty.context_processors.common', + 'postorius.context_processors.postorius', + ], + }, + }, +] + +WSGI_APPLICATION = 'wsgi.application' + +# Database +# https://docs.djangoproject.com/en/1.8/ref/settings/#databases + +DATABASES = { + 'default': { + # Use 'sqlite3', 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. + #'ENGINE': 'django.db.backends.sqlite3', + #'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'ENGINE': 'django.db.backends.mysql', + # DB name or path to database file if using sqlite3. + 'NAME': 'mailman3web', + # The following settings are not used with sqlite3: + 'USER': 'mailman3web', + 'PASSWORD': 'mailman3web', + # HOST: empty for localhost through domain sockets or '127.0.0.1' for + # localhost through TCP. + 'HOST': 'coffee', + # PORT: set to empty string for default. + 'PORT': '', + # OPTIONS: Extra parameters to use when connecting to the database. + 'OPTIONS': { + # Set sql_mode to 'STRICT_TRANS_TABLES' for MySQL. See + # https://docs.djangoproject.com/en/1.11/ref/ + # databases/#setting-sql-mode + 'init_command': "SET sql_mode='STRICT_TRANS_TABLES'", + 'charset': 'utf8mb4', + }, + } +} + +# Full-text search engine +HAYSTACK_CONNECTIONS = { + 'default': { + 'ENGINE': 'xapian_backend.XapianEngine', + 'PATH': '/opt/mailman3/web/xapian_index', + } +} + +# Django Q +Q_CLUSTER = { + 'timeout': 300, + 'retry': 305, + 'save_limit': 100, + 'orm': 'default', + 'poll': 5, + 'workers': 2, +} + +# Cache +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', + 'LOCATION': '127.0.0.1:11211', + } +} + +# If you're behind a proxy, use the X-Forwarded-Host header +# See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host +USE_X_FORWARDED_HOST = True + +# And if your proxy does your SSL encoding for you, set SECURE_PROXY_SSL_HEADER +# https://docs.djangoproject.com/en/1.8/ref/settings/#secure-proxy-ssl-header +# SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +# SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_SCHEME', 'https') + +# Other security settings +# SECURE_SSL_REDIRECT = True +# If you set SECURE_SSL_REDIRECT to True, make sure the SECURE_REDIRECT_EXEMPT +# contains at least this line: +# SECURE_REDIRECT_EXEMPT = [ +# "archives/api/mailman/.*", # Request from Mailman. +# ] +# SESSION_COOKIE_SECURE = True +# SECURE_CONTENT_TYPE_NOSNIFF = True +# SECURE_BROWSER_XSS_FILTER = True +# CSRF_COOKIE_SECURE = True +# CSRF_COOKIE_HTTPONLY = True +# X_FRAME_OPTIONS = 'DENY' + +# Password validation +# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': +'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': +'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': +'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': +'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +# Internationalization +# https://docs.djangoproject.com/en/1.8/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'America/Toronto' + +USE_I18N = True +USE_L10N = True +USE_TZ = True + +# Django 1.6+ defaults to a JSON serializer, but it won't work with +# django-openid, see +# https://bugs.launchpad.net/django-openid-auth/+bug/1252826 +SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer' + +LOGIN_URL = 'account_login' +LOGIN_REDIRECT_URL = 'list_index' +LOGOUT_URL = 'account_logout' + +# List of finder classes that know how to find static files in +# various locations. +STATICFILES_FINDERS = ( + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + # 'django.contrib.staticfiles.finders.DefaultStorageFinder', + 'compressor.finders.CompressorFinder', +) + +STATIC_ROOT = '/opt/mailman3/web/static' +STATIC_URL = '/static/' +HOSTNAME = 'mailman.{{ base_domain }}' +# Set default domain for email addresses. +EMAILNAME = '{{ base_domain }}' + +# If you enable internal authentication, this is the address that the emails +# will appear to be coming from. Make sure you set a valid domain name, +# otherwise the emails may get rejected. +# https://docs.djangoproject.com/en/1.8/ref/settings/#default-from-email +# DEFAULT_FROM_EMAIL = "mailing-lists@you-domain.org" +DEFAULT_FROM_EMAIL = 'postorius@{}'.format(EMAILNAME) + +# If you enable email reporting for error messages, this is where those emails +# will appear to be coming from. Make sure you set a valid domain name, +# otherwise the emails may get rejected. +# https://docs.djangoproject.com/en/1.8/ref/settings/#std:setting-SERVER_EMAIL +# SERVER_EMAIL = 'root@your-domain.org' +SERVER_EMAIL = 'root@{}'.format(EMAILNAME) + +# Change this when you have a real email backend +EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' + +# Compatibility with Bootstrap 3 +from django.contrib.messages import constants as messages # flake8: noqa +MESSAGE_TAGS = { + messages.ERROR: 'danger' +} + + +# +# Social auth +# +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'allauth.account.auth_backends.AuthenticationBackend', +) + +# Django Allauth +ACCOUNT_AUTHENTICATION_METHOD = "username_email" +ACCOUNT_EMAIL_REQUIRED = True +ACCOUNT_EMAIL_VERIFICATION = "mandatory" +# You probably want https in production, but this is a dev setup file +ACCOUNT_DEFAULT_HTTP_PROTOCOL = "http" +ACCOUNT_UNIQUE_EMAIL = True + +SOCIALACCOUNT_PROVIDERS = { + #'openid': { + # 'SERVERS': [ + # dict(id='yahoo', + # name='Yahoo', + # openid_url='http://me.yahoo.com'), + # ], + #}, + #'google': { + # 'SCOPE': ['profile', 'email'], + # 'AUTH_PARAMS': {'access_type': 'online'}, + #}, + #'facebook': { + # 'METHOD': 'oauth2', + # 'SCOPE': ['email'], + # 'FIELDS': [ + # 'email', + # 'name', + # 'first_name', + # 'last_name', + # 'locale', + # 'timezone', + # ], + # 'VERSION': 'v2.4', + #}, +} + +# +# Gravatar +# https://github.com/twaddington/django-gravatar +# +# Gravatar base url. +# GRAVATAR_URL = 'http://cdn.libravatar.org/' +# Gravatar base secure https url. +# GRAVATAR_SECURE_URL = 'https://seccdn.libravatar.org/' +# Gravatar size in pixels. +# GRAVATAR_DEFAULT_SIZE = '80' +# An image url or one of the following: 'mm', 'identicon', 'monsterid', +# 'wavatar', 'retro'. +# GRAVATAR_DEFAULT_IMAGE = 'mm' +# One of the following: 'g', 'pg', 'r', 'x'. +# GRAVATAR_DEFAULT_RATING = 'g' +# True to use https by default, False for plain http. +# GRAVATAR_DEFAULT_SECURE = True + +# +# django-compressor +# https://pypi.python.org/pypi/django_compressor +# +COMPRESS_PRECOMPILERS = ( + ('text/less', 'lessc {infile} {outfile}'), + ('text/x-scss', 'sassc -t compressed {infile} {outfile}'), + ('text/x-sass', 'sassc -t compressed {infile} {outfile}'), +) + +# On a production setup, setting COMPRESS_OFFLINE to True will bring a +# significant performance improvement, as CSS files will not need to be +# recompiled on each requests. It means running an additional "compress" +# management command after each code upgrade. +# http://django-compressor.readthedocs.io/en/latest/usage/#offline-compression +COMPRESS_OFFLINE = True + +POSTORIUS_TEMPLATE_BASE_URL = 'http://mailman.{{ base_domain }}/' + +# Custom logging. Disable sending log emails because this floods our inbox. +# See https://lincolnloop.com/blog/disabling-error-emails-django/. +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'filters': { + 'require_debug_false': { + '()': 'django.utils.log.RequireDebugFalse' + } + }, + 'handlers': { + 'file':{ + 'level': 'WARNING', + 'class': 'logging.handlers.RotatingFileHandler', + #'class': 'logging.handlers.WatchedFileHandler', + 'filename': '/var/log/mailman3/web/mailman-web.log', + 'formatter': 'verbose', + }, + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'simple', + }, + }, + 'loggers': { + 'django.request': { + 'handlers': ['file'], + 'level': 'WARNING', + 'propagate': True, + }, + 'django': { + 'handlers': ['file'], + 'level': 'WARNING', + 'propagate': True, + }, + 'hyperkitty': { + 'handlers': ['file'], + 'level': 'WARNING', + 'propagate': True, + }, + 'postorius': { + 'handlers': ['file'], + 'level': 'WARNING', + 'propagate': True, + }, + }, + 'formatters': { + 'verbose': { + 'format': '%(levelname)s %(asctime)s %(process)d %(name)s %(message)s' + }, + 'simple': { + 'format': '%(levelname)s %(message)s' + }, + }, + #'root': { + # 'handlers': ['file'], + # 'level': 'INFO', + #}, +} + +# +# HyperKitty-specific +# + +# Only display mailing-lists from the same virtual host as the webserver +FILTER_VHOST = False diff --git a/mail/mailman3/urls.py b/mail/mailman3/urls.py new file mode 100644 index 0000000..5c2a7f9 --- /dev/null +++ b/mail/mailman3/urls.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# Copyright (C) 1998-2016 by the Free Software Foundation, Inc. +# +# This file is part of Postorius. +# +# Postorius is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# Postorius is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# Postorius. If not, see . + + +from django.conf.urls import include, url +from django.contrib import admin +from django.urls import reverse_lazy +from django.views.generic import RedirectView + +urlpatterns = [ + url(r'^$', RedirectView.as_view( + url=reverse_lazy('list_index'), + permanent=True)), + url(r'^postorius/', include('postorius.urls')), + url(r'^hyperkitty/', include('hyperkitty.urls')), + url(r'', include('django_mailman3.urls')), + url(r'^accounts/', include('allauth.urls')), + # Django admin + url(r'^admin/', admin.site.urls), +] diff --git a/mail/mailman3/uwsgi.ini b/mail/mailman3/uwsgi.ini new file mode 100644 index 0000000..b31be07 --- /dev/null +++ b/mail/mailman3/uwsgi.ini @@ -0,0 +1,55 @@ +[uwsgi] +# Port on which uwsgi will be listening. +uwsgi-socket = /run/mailman3-web/uwsgi.sock +virtualenv = /opt/mailman3 + +#Enable threading for python +enable-threads = true + +# Move to the directory wher the django files are. +chdir = /opt/mailman3/web + +module = mailman_web.wsgi:application +env = PYTHONPATH=/etc/mailman3 +env = DJANGO_SETTINGS_MODULE=settings + +# Setup default number of processes and threads per process. +master = true +process = 2 +threads = 2 + +# Drop privielges and don't run as root. +uid = www-data +gid = www-data + +plugins = python3 + +# Setup the django_q related worker processes. +attach-daemon = /opt/mailman3/bin/mailman-web qcluster + +# Increase the buffer size limit (default is 4096) +buffer-size = 32768 + +# Setup hyperkitty's cron jobs. +#unique-cron = -1 -1 -1 -1 -1 ./manage.py runjobs minutely +#unique-cron = -15 -1 -1 -1 -1 ./manage.py runjobs quarter_hourly +#unique-cron = 0 -1 -1 -1 -1 ./manage.py runjobs hourly +#unique-cron = 0 0 -1 -1 -1 ./manage.py runjobs daily +#unique-cron = 0 0 1 -1 -1 ./manage.py runjobs monthly +#unique-cron = 0 0 -1 -1 0 ./manage.py runjobs weekly +#unique-cron = 0 0 1 1 -1 ./manage.py runjobs yearly + +# Setup the request log. +#req-logger = file:/var/log/mailman3/web/mailman-web.log + +# Log cron seperately. +#logger = cron file:/var/log/mailman3/web/mailman-web-cron.log +#log-route = cron uwsgi-cron + +# Log qcluster commands seperately. +#logger = qcluster file:/var/log/mailman3/web/mailman-web-qcluster.log +#log-route = qcluster uwsgi-daemons + +# Last log and it logs the rest of the stuff. +#logger = file:/var/log/mailman3/web/mailman-web-error.log +#logto = /var/log/mailman3/web/mailman-web.log diff --git a/mail/main.yml b/mail/main.yml new file mode 100644 index 0000000..214c472 --- /dev/null +++ b/mail/main.yml @@ -0,0 +1,74 @@ +--- +- hosts: mail + roles: + - role: ../roles/network_setup + vars: + ipv4_addr: "{{ mail_ipv4_addr }}" + tasks: + - name: install packages for email server + apt: + name: "{{ item }}" + state: present + update_cache: true + loop: + - postfix + - postfix-pcre + - dovecot-imapd + - spamassassin + - spamass-milter + - procmail + - name: override systemd services + import_role: + name: ../roles/systemd_workarounds + vars: + services: [ "dovecot" ] + - name: enable and start SpamAssassin + systemd: + name: spamassassin + enabled: true + state: started + - name: add Dovecot config + copy: + src: dovecot/dovecot.conf + dest: /etc/dovecot/dovecot.conf + notify: restart Dovecot + - name: add Postfix config + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + loop: + - src: postfix/main.cf.j2 + dest: /etc/postfix/main.cf + - src: postfix/master.cf + dest: /etc/postfix/master.cf + - src: postfix/login_maps.pcre.j2 + dest: /etc/postfix/login_maps.pcre + notify: reload Postfix + - name: add Procmail config + copy: + src: procmail/procmailrc + dest: /etc/procmailrc + - name: add SpamAssassin config + template: + src: "{{ item.src }}" + dest: "{{ item.dest }}" + loop: + - src: spamassassin/local.cf.j2 + dest: /etc/spamassassin/local.cf + - src: spamassassin/spamc.conf + dest: /etc/spamassassin/spamc.conf + notify: restart SpamAssassin + - name: add local users + import_role: + name: ../roles/local_users + handlers: + - name: _imports + import_tasks: common.yml + - name: restart Dovecot + systemd: + name: dovecot + state: restarted + - name: restart SpamAssassin + systemd: + name: spamassassin + state: restarted diff --git a/mail/postfix/login_maps.pcre.j2 b/mail/postfix/login_maps.pcre.j2 new file mode 100644 index 0000000..d1e96dd --- /dev/null +++ b/mail/postfix/login_maps.pcre.j2 @@ -0,0 +1 @@ +/^(.*)@{{ base_domain | replace('.', '\\.') }}$/ ${1} diff --git a/mail/postfix/main.cf.j2 b/mail/postfix/main.cf.j2 new file mode 100644 index 0000000..8c11839 --- /dev/null +++ b/mail/postfix/main.cf.j2 @@ -0,0 +1,100 @@ +# See /usr/share/postfix/main.cf.dist for a commented, more complete version + + +# Debian specific: Specifying a file name will cause the first +# line of that file to be used as the name. The Debian default +# is /etc/mailname. +#myorigin = /etc/mailname + +smtpd_banner = $myhostname ESMTP $mail_name (Debian/GNU) +biff = no + +# appending .domain is the MUA's job. +append_dot_mydomain = no + +# Uncomment the next line to generate "delayed mail" warnings +#delay_warning_time = 4h + +readme_directory = no + +# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 2 on +# fresh installs. +compatibility_level = 2 + +mailbox_size_limit = 0 + +myhostname = mail.{{ base_domain }} +mydomain = {{ base_domain }} +alias_maps = hash:/etc/aliases +myorigin = $mydomain +mydestination = $myhostname, $mydomain, localhost.$mydomain, localhost +relayhost = +relay_domains = +mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128 {{ ipv4_subnet }} +mailbox_size_limit = 0 +recipient_delimiter = + +inet_interfaces = {{ mail_ipv4_addr }}, 127.0.0.1 +inet_protocols = ipv4, ipv6 + +require_home_directory = yes + +# Procmail +mailbox_command = procmail -a "$EXTENSION" + +# rewrite @foo.csclub.internal to @csclub.internal for locally generated mail +masquerade_domains = $mydomain +masquerade_classes = envelope_sender, envelope_recipient, header_sender, header_recipient +local_header_rewrite_clients = permit_inet_interfaces, permit_mynetworks + +# SpamAssassin +milter_default_action = accept +milter_connect_macros = j {daemon_name} v {if_name} _ +non_smtpd_milters = $smtpd_milters +smtpd_milters = unix:/var/spool/postfix/spamass/spamass.sock + +# TLS parameters +#smtpd_tls_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem +#smtpd_tls_key_file=/etc/ssl/private/ssl-cert-snakeoil.key +#smtpd_use_tls=yes +#smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache +#smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache + +# See /usr/share/doc/postfix/TLS_README.gz in the postfix-doc package for +# information on enabling SSL in the smtp client. + +# HELO restrictions +smtpd_helo_required = yes +smtpd_helo_restrictions = + permit_mynetworks, + reject_invalid_helo_hostname, + #reject_non_fqdn_helo_hostname, + permit + +# Sender (MAIL FROM) restrictions +smtpd_sender_login_maps = pcre:/etc/postfix/login_maps.pcre +smtpd_sender_restrictions = + permit_mynetworks, + reject_sender_login_mismatch, + reject_non_fqdn_sender + #reject_unknown_sender_domain + +# Relay restrictions +smtpd_relay_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_unauth_destination + +# Recipient (RCPT TO) restrictions +smtpd_recipient_restrictions = + permit_mynetworks, + permit_sasl_authenticated, + reject_unauth_destination, + reject_non_fqdn_recipient, + #reject_unknown_reverse_client_hostname, + reject_unknown_recipient_domain + +# SASL +smtpd_sasl_auth_enable = yes +smtpd_sasl_type = dovecot +smtpd_sasl_path = private/auth +broken_sasl_auth_clients = yes diff --git a/mail/postfix/master.cf b/mail/postfix/master.cf new file mode 100644 index 0000000..d649afa --- /dev/null +++ b/mail/postfix/master.cf @@ -0,0 +1,131 @@ +# +# Postfix master process configuration file. For details on the format +# of the file, see the master(5) manual page (command: "man 5 master" or +# on-line: http://www.postfix.org/master.5.html). +# +# Do not forget to execute "postfix reload" after editing this file. +# +# ========================================================================== +# service type private unpriv chroot wakeup maxproc command + args +# (yes) (yes) (no) (never) (100) +# ========================================================================== +smtp inet n - n - - smtpd +#smtp inet n - y - 1 postscreen +#smtpd pass - - y - - smtpd +#dnsblog unix - - y - 0 dnsblog +#tlsproxy unix - - y - 0 tlsproxy +submission inet n - n - - smtpd + -o smtpd_sasl_auth_enable=yes + -o smtpd_client_restrictions=permit_sasl_authenticated,reject + -o smtpd_recipient_restrictions=permit_sasl_authenticated,reject + -o smtpd_helo_restrictions=reject_invalid_helo_hostname,permit + -o milter_macro_daemon_name=ORIGINATING +#smtps inet n - y - - smtpd +# -o syslog_name=postfix/smtps +# -o smtpd_tls_wrappermode=yes +# -o smtpd_sasl_auth_enable=yes +# -o smtpd_reject_unlisted_recipient=no +# -o smtpd_client_restrictions=$mua_client_restrictions +# -o smtpd_helo_restrictions=$mua_helo_restrictions +# -o smtpd_sender_restrictions=$mua_sender_restrictions +# -o smtpd_recipient_restrictions= +# -o smtpd_relay_restrictions=permit_sasl_authenticated,reject +# -o milter_macro_daemon_name=ORIGINATING +#628 inet n - y - - qmqpd +pickup unix n - y 60 1 pickup +cleanup unix n - y - 0 cleanup +qmgr unix n - n 300 1 qmgr +#qmgr unix n - n 300 1 oqmgr +tlsmgr unix - - y 1000? 1 tlsmgr +rewrite unix - - y - - trivial-rewrite +bounce unix - - y - 0 bounce +defer unix - - y - 0 bounce +trace unix - - y - 0 bounce +verify unix - - y - 1 verify +flush unix n - y 1000? 0 flush +proxymap unix - - n - - proxymap +proxywrite unix - - n - 1 proxymap +smtp unix - - y - - smtp +relay unix - - y - - smtp + -o syslog_name=postfix/$service_name +# -o smtp_helo_timeout=5 -o smtp_connect_timeout=5 +showq unix n - y - - showq +error unix - - y - - error +retry unix - - y - - error +discard unix - - y - - discard +local unix - n n - - local +virtual unix - n n - - virtual +lmtp unix - - y - - lmtp +anvil unix - - y - 1 anvil +scache unix - - y - 1 scache +postlog unix-dgram n - n - 1 postlogd +# +# ==================================================================== +# Interfaces to non-Postfix software. Be sure to examine the manual +# pages of the non-Postfix software to find out what options it wants. +# +# Many of the following services use the Postfix pipe(8) delivery +# agent. See the pipe(8) man page for information about ${recipient} +# and other message envelope options. +# ==================================================================== +# +# maildrop. See the Postfix MAILDROP_README file for details. +# Also specify in main.cf: maildrop_destination_recipient_limit=1 +# +maildrop unix - n n - - pipe + flags=DRhu user=vmail argv=/usr/bin/maildrop -d ${recipient} +# +# ==================================================================== +# +# Recent Cyrus versions can use the existing "lmtp" master.cf entry. +# +# Specify in cyrus.conf: +# lmtp cmd="lmtpd -a" listen="localhost:lmtp" proto=tcp4 +# +# Specify in main.cf one or more of the following: +# mailbox_transport = lmtp:inet:localhost +# virtual_transport = lmtp:inet:localhost +# +# ==================================================================== +# +# Cyrus 2.1.5 (Amos Gouaux) +# Also specify in main.cf: cyrus_destination_recipient_limit=1 +# +#cyrus unix - n n - - pipe +# user=cyrus argv=/cyrus/bin/deliver -e -r ${sender} -m ${extension} ${user} +# +# ==================================================================== +# Old example of delivery via Cyrus. +# +#old-cyrus unix - n n - - pipe +# flags=R user=cyrus argv=/cyrus/bin/deliver -e -m ${extension} ${user} +# +# ==================================================================== +# +# See the Postfix UUCP_README file for configuration details. +# +uucp unix - n n - - pipe + flags=Fqhu user=uucp argv=uux -r -n -z -a$sender - $nexthop!rmail ($recipient) +# +# Other external delivery methods. +# +ifmail unix - n n - - pipe + flags=F user=ftn argv=/usr/lib/ifmail/ifmail -r $nexthop ($recipient) +bsmtp unix - n n - - pipe + flags=Fq. user=bsmtp argv=/usr/lib/bsmtp/bsmtp -t$nexthop -f$sender $recipient +scalemail-backend unix - n n - 2 pipe + flags=R user=scalemail argv=/usr/lib/scalemail/bin/scalemail-store ${nexthop} ${user} ${extension} +mailman unix - n n - - pipe + flags=FR user=list argv=/usr/lib/mailman/bin/postfix-to-mailman.py + ${nexthop} ${user} + +# Mailman +127.0.0.1:10025 inet n - n - 16 smtpd + -o content_filter= + -o receive_override_options=no_unknown_recipient_checks,no_header_body_checks + -o smtpd_helo_restrictions=permit_mynetworks,reject + -o smtpd_client_restrictions=permit_mynetworks,reject + -o smtpd_sender_restrictions=permit_mynetworks,reject + -o smtpd_recipient_restrictions=permit_mynetworks,reject + -o mynetworks_style=host + -o smtpd_authorized_xforward_hosts=127.0.0.0/8 diff --git a/mail/procmail/procmailrc b/mail/procmail/procmailrc new file mode 100644 index 0000000..f15453a --- /dev/null +++ b/mail/procmail/procmailrc @@ -0,0 +1 @@ +DEFAULT=$HOME/.maildir/ diff --git a/mail/spamassassin/local.cf.j2 b/mail/spamassassin/local.cf.j2 new file mode 100644 index 0000000..827d7de --- /dev/null +++ b/mail/spamassassin/local.cf.j2 @@ -0,0 +1,109 @@ +# This is the right place to customize your installation of SpamAssassin. +# +# See 'perldoc Mail::SpamAssassin::Conf' for details of what can be +# tweaked. +# +# Only a small subset of options are listed below +# +########################################################################### + +# Add *****SPAM***** to the Subject header of spam e-mails +# +# rewrite_header Subject *****SPAM***** + +# Add X-Spam-Flag header to all emails, not just spam ones +add_header all Flag _YESNOCAPS_ + +# Save spam messages as a message/rfc822 MIME attachment instead of +# modifying the original message (0: off, 2: use text/plain instead) +# +# report_safe 1 +report_safe 0 + +# Use site-wide Bayesian learner instead of per-user +bayes_path /var/lib/spamassassin/.spamassassin/bayes +bayes_file_mode 0775 + +# Don't use autolearning; we are using supervised learning +bayes_auto_learn 0 + +# Default DB file size is 150000 tokens, which is roughly equivalent to 8Mb +bayes_expiry_max_db_size 600000 + +# Whitelist ourselves +whitelist_from_rcvd *@{{ base_domain }} {{ base_domain }} +whitelist_from_rcvd *@*.{{ base_domain }} {{ base_domain }} +whitelist_from_rcvd *@{{ base_domain }} localhost + +# Custom Bayes scores +score BAYES_95 0 0 4.0 4.0 +score BAYES_99 0 0 7.0 7.0 + +# Custom rules +header TO_UNDISCLOSED_RECIPIENTS To =~ /\bundisclosed-recipients\b/i +describe TO_UNDISCLOSED_RECIPIENTS undisclosed-recipients in To: header +score TO_UNDISCLOSED_RECIPIENTS 4.0 + +# Set which networks or hosts are considered 'trusted' by your mail +# server (i.e. not spammers) +# +# trusted_networks 212.17.35. + + +# Set file-locking method (flock is not safe over NFS, but is faster) +# +# lock_method flock + + +# Set the threshold at which a message is considered spam (default: 5.0) +# +# required_score 5.0 + + +# Use Bayesian classifier (default: 1) +# +# use_bayes 1 + +# Set headers which may provide inappropriate cues to the Bayesian +# classifier +# +# bayes_ignore_header X-Bogosity +# bayes_ignore_header X-Spam-Flag +# bayes_ignore_header X-Spam-Status + + +# Whether to decode non- UTF-8 and non-ASCII textual parts and recode +# them to UTF-8 before the text is given over to rules processing. +# +# normalize_charset 1 + +# Some shortcircuiting, if the plugin is enabled +# +ifplugin Mail::SpamAssassin::Plugin::Shortcircuit +# +# default: strongly-whitelisted mails are *really* whitelisted now, if the +# shortcircuiting plugin is active, causing early exit to save CPU load. +# Uncomment to turn this on +# +# shortcircuit USER_IN_WHITELIST on +# shortcircuit USER_IN_DEF_WHITELIST on +# shortcircuit USER_IN_ALL_SPAM_TO on +# shortcircuit SUBJECT_IN_WHITELIST on + +# the opposite; blacklisted mails can also save CPU +# +# shortcircuit USER_IN_BLACKLIST on +# shortcircuit USER_IN_BLACKLIST_TO on +# shortcircuit SUBJECT_IN_BLACKLIST on + +# if you have taken the time to correctly specify your "trusted_networks", +# this is another good way to save CPU +# +# shortcircuit ALL_TRUSTED on + +# and a well-trained bayes DB can save running rules, too +# +# shortcircuit BAYES_99 spam +# shortcircuit BAYES_00 ham + +endif # Mail::SpamAssassin::Plugin::Shortcircuit diff --git a/mail/spamassassin/spamc.conf b/mail/spamassassin/spamc.conf new file mode 100644 index 0000000..d7fb3ab --- /dev/null +++ b/mail/spamassassin/spamc.conf @@ -0,0 +1,2 @@ +# same value as message_size_limit in /etc/postfix/main.cf +-s 20971520 diff --git a/outsider/README.md b/outsider/README.md new file mode 100644 index 0000000..c621b76 --- /dev/null +++ b/outsider/README.md @@ -0,0 +1,37 @@ +# Outsider container +So this container's a bit special - it represents a host which is **not** +on the UW network. The motivation is to test software which have different +privilege settings for people outside of the local network, e.g. Postfix. + +The idea is to route packets from the 'outsider' container to the LXC host +(i.e. the VM), and the VM will then route them to the other containers. +We could've also created an extra container to act as the router, but +that seemed kind of wasteful. + +## Installation +Once you have created the container, add the following iptables rules on +the VM: +``` +iptables -t nat -A POSTROUTING -s 192.168.125.0/24 -d 192.168.122.1 -j MASQUERADE +iptables -t nat -A POSTROUTING -s 192.168.125.0/24 ! -d 192.168.122.0/24 -j MASQUERADE +``` +I also strongly suggest installing iptables-persistent so that these rules +persist on the next reboot: +``` +apt install iptables-persistent +``` +The idea here is that packets from the 'outsider' container should only be +**forwarded**, not masqueraded, to the other containers (to preserve its IP +address), unless if it needs to communicate with the outside world (e.g. to +download Debian packages), in which case we need to use NAT because the +iptables rules which libvirt created on your real computer don't take that +subnet into account (run `iptables -t nat -L -v` on your real computer +to see what I mean). 192.168.122.1, which is your real computer, is a special +case because your host does not have a routing table entry for that +subnet, so it wouldn't be able to reply. + +As usual, create the container, start it, and install python3. +Now detach and run the playbook: +``` +ansible-playbook main.yml +``` diff --git a/outsider/main.yml b/outsider/main.yml new file mode 100644 index 0000000..c676e8c --- /dev/null +++ b/outsider/main.yml @@ -0,0 +1,10 @@ +--- +- hosts: outsider + roles: + - role: ../roles/network_setup + vars: + ipv4_addr: "{{ outsider_ipv4_addr }}" + tasks: + - name: add local users + import_role: + name: ../roles/local_users diff --git a/roles/local_users/tasks/main.yml b/roles/local_users/tasks/main.yml new file mode 100644 index 0000000..dca1d8c --- /dev/null +++ b/roles/local_users/tasks/main.yml @@ -0,0 +1,29 @@ +- name: create users + user: + name: "{{ item }}" + # password is dovecotpostfix + # This hash was generated with the following command: + # python3 -c "import crypt; import getpass; print(crypt.crypt(getpass.getpass()))" + password: $6$StYWEDOVhv1TnjaM$xq9mmGWY8VDGNGavODuVbCYs56keLivi9GUB3W.NCTNusQaFZsYllSVwCMGXef0.P3vlHL.GsHuD1LCngv2G7/ + shell: /bin/bash + loop: "{{ local_users }}" +- name: install Mutt + apt: + name: mutt + state: present +- name: create Mutt cache directory + file: + path: "/home/{{ item }}/.mutt/cache" + state: directory + owner: "{{ item }}" + group: "{{ item }}" + loop: "{{ local_users }}" +- name: add muttrc for each user + template: + src: templates/muttrc.j2 + dest: "/home/{{ item }}/.muttrc" + owner: "{{ item }}" + group: "{{ item }}" + vars: + username: "{{ item }}" + loop: "{{ local_users }}" diff --git a/roles/local_users/templates/muttrc.j2 b/roles/local_users/templates/muttrc.j2 new file mode 100644 index 0000000..6ae9854 --- /dev/null +++ b/roles/local_users/templates/muttrc.j2 @@ -0,0 +1,28 @@ +# SMTP +set realname = "{{ username.capitalize() }} Smith" +set from = "{{ username }}@{{ base_domain }}" +set smtp_url = "smtp://{{ username }}@mail.{{ base_domain }}:587" +set smtp_pass = "dovecotpostfix" +unset record + +# IMAP +set folder = "imap://mail.{{ base_domain }}" +set imap_user = "{{ username }}" +set imap_pass = "dovecotpostfix" +set imap_authenticators = "plain" +set spoolfile = "+Inbox" +set sort = reverse-date-received +set header_cache = "~/.mutt/cache/headers" +set message_cachedir = "~/.mutt/cache/bodies" + +# TLS +set ssl_starttls = no +set ssl_force_tls = no + +# Misc +bind index G imap-fetch-mail +set pager_stop=yes +bind pager previous-line +bind pager next-line +set mail_check=60 +set imap_keepalive=900 diff --git a/roles/local_users/vars/main.yml b/roles/local_users/vars/main.yml new file mode 100644 index 0000000..5e2303e --- /dev/null +++ b/roles/local_users/vars/main.yml @@ -0,0 +1,4 @@ +local_users: + - alice + - bob + - eve diff --git a/roles/network_setup/handlers/main.yml b/roles/network_setup/handlers/main.yml new file mode 100644 index 0000000..9bfcc94 --- /dev/null +++ b/roles/network_setup/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart networking + systemd: + name: networking + state: restarted diff --git a/roles/network_setup/tasks/main.yml b/roles/network_setup/tasks/main.yml new file mode 100644 index 0000000..aa6eb91 --- /dev/null +++ b/roles/network_setup/tasks/main.yml @@ -0,0 +1,17 @@ +- name: setup /etc/network/interfaces + template: + src: templates/interfaces.j2 + dest: /etc/network/interfaces + notify: + - restart networking +- name: setup /etc/hosts + template: + src: templates/hosts.j2 + dest: /etc/hosts +- name: setup /etc/resolv.conf + copy: + content: | + search {{ base_domain }} + nameserver {{ dns_ipv4_addr }} + dest: /etc/resolv.conf + when: ansible_host != 'dns' diff --git a/roles/network_setup/templates/hosts.j2 b/roles/network_setup/templates/hosts.j2 new file mode 100644 index 0000000..7394dfa --- /dev/null +++ b/roles/network_setup/templates/hosts.j2 @@ -0,0 +1,6 @@ +127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters + +{{ ipv4_addr }} {{ ansible_hostname }}.{{ base_domain }} {{ ansible_hostname }} diff --git a/roles/network_setup/templates/interfaces.j2 b/roles/network_setup/templates/interfaces.j2 new file mode 100644 index 0000000..27f6a46 --- /dev/null +++ b/roles/network_setup/templates/interfaces.j2 @@ -0,0 +1,10 @@ +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet static + address {{ ipv4_addr }} + netmask 255.255.255.0 + gateway {{ host_ipv4_addr }} + +source /etc/network/interfaces.d/* diff --git a/roles/systemd_workarounds/handlers/main.yml b/roles/systemd_workarounds/handlers/main.yml new file mode 100644 index 0000000..50c8015 --- /dev/null +++ b/roles/systemd_workarounds/handlers/main.yml @@ -0,0 +1,8 @@ +- name: reload systemd + systemd: + daemon_reload: true +- name: restart service + systemd: + name: "{{ item.dest | regex_replace('^/etc/systemd/system/(.*)\\.service\\.d/override\\.conf$', '\\1') }}" + state: restarted + loop: "{{ service_overrides.results }}" diff --git a/roles/systemd_workarounds/tasks/main.yml b/roles/systemd_workarounds/tasks/main.yml new file mode 100644 index 0000000..c788ca2 --- /dev/null +++ b/roles/systemd_workarounds/tasks/main.yml @@ -0,0 +1,20 @@ +- name: create override directory + file: + path: "/etc/systemd/system/{{ item }}.service.d" + mode: 0755 + state: directory + loop: "{{ services }}" +- name: disable mount namespaces + copy: + content: | + [Service] + ProtectSystem=false + PrivateTmp=false + PrivateDevices=false + ProtectHome=false + dest: "/etc/systemd/system/{{ item }}.service.d/override.conf" + loop: "{{ services }}" + register: service_overrides + notify: + - reload systemd + - restart service