Merge pull request #4855 from rmol/4783-supervisor-to-systemd

Manage rq services with systemd instead of supervisor

MANIFEST.in

@@ -1,5 +1,6 @@
 graft debian
 graft etc
+graft lib
 graft var
 prune var/www/securedrop/tests
 include requirements.txt

docs/backup_and_restore.rst

@@ -41,7 +41,7 @@ currently on the *Application Server*: log in over SSH and run
           queued deletion jobs by logging in to the *Application
           Server* over SSH and running::
 
-            sudo tail -f /var/log/securedrop_worker/rqworker.err
+            sudo journalctl -u securedrop_rqworker
 
 
 If you find you cannot perform a backup or restore due to this constraint,
@@ -85,8 +85,8 @@ to debug your connectivity before proceeding further. Make sure:
 * Ansible is installed
 * the *Admin Workstation* is connected to the Internet
 * Tor starts successfully
-* The appropriate onion service configuration files are present in 
-  ``~/Persistent/securedrop/install_files/ansible-base`` and the 
+* The appropriate onion service configuration files are present in
+  ``~/Persistent/securedrop/install_files/ansible-base`` and the
   ``./securedrop-admin tailsconfig`` command completes successfully
 
 If Ansible is not installed, or ``./securedrop-admin tailsconfig`` fails, see

install_files/ansible-base/roles/app/handlers/main.yml

@@ -14,16 +14,6 @@
     name: apache2
     state: restarted
 
-- name: reload supervisor
-  supervisorctl:
-    name: securedrop_worker
-    state: present
-
-- name: reload supervisor
-  supervisorctl:
-    name: securedrop_rqrequeue
-    state: present
-
 ## Here, we list apparmor before haveged to ensure the correct AppArmor
 ## profile is loaded prior to restarting haveged
 - name: restart apparmor

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml

@@ -16,10 +16,14 @@ securedrop_venv_site_packages: "{{ securedrop_venv }}/lib/python3.5/site-package
 # SecureDrop code installation directory
 securedrop_code: /var/www/securedrop
 
-# Configuration files for SecureDrop programs run under supervisord
-supervisor_programs:
-  - securedrop_rqrequeue.conf
-  - securedrop_worker.conf
+# Location of the application storage on disk, including database.
+securedrop_data: /var/lib/securedrop
+
+# Configuration files for SecureDrop systemd services
+systemd_services:
+  - securedrop_rqrequeue.service
+  - securedrop_rqworker.service
+  - securedrop_shredder.service
 
 # SecureDrop rq worker log directory
 securedrop_worker_log_dir: /var/log/securedrop_worker

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2

@@ -102,7 +102,6 @@
   /usr/lib{,32,64}/** mr,
   /usr/share/file/magic r,
   /usr/share/file/magic.mgc r,
-  /usr/share/pyshared/supervisor-*-nspkg.pth r,
   /opt/venvs/securedrop-app-code/**/__pycache__/ rw,
   /opt/venvs/securedrop-app-code/**/__pycache__/* rw,
   /opt/venvs/securedrop-app-code/bin/python3 r,
@@ -132,6 +131,8 @@
   /var/lib/securedrop/keys/secring.gpg.tmp rw,
   /var/lib/securedrop/keys/trustdb.gpg rw,
   /var/lib/securedrop/keys/trustdb.gpg.lock rwl,
+  /var/lib/securedrop/shredder/** rw,
+  /var/lib/securedrop/shredder/*/ w,
   /var/lib/securedrop/store/** rw,
   /var/lib/securedrop/store/*/ w,
   /var/lib/securedrop/tmp/** rw,

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml

@@ -57,19 +57,19 @@
     src: "changelog-{{ securedrop_package_dist }}"
     dest: "{{ securedrop_app_code_prep_dir }}/debian/changelog"
 
-- name: Create supervisor conf.d directory in prep directory
+- name: Create lib/systemd/services directory in prep directory
   file:
     state: directory
-    dest: "{{ securedrop_app_code_prep_dir }}/etc/supervisor/conf.d"
-  tags: supervisor
+    dest: "{{ securedrop_app_code_prep_dir }}/lib/systemd/system"
+  tags: systemd
 
-- name: Copy supervisor program configurations to prep path
+- name: Copy systemd service configurations to prep path
   template:
     src: "{{ item }}"
-    dest: "{{ securedrop_app_code_prep_dir }}/etc/supervisor/conf.d/{{ item }}"
+    dest: "{{ securedrop_app_code_prep_dir }}/lib/systemd/system/{{ item }}"
     mode: 0644
-  with_items: "{{ supervisor_programs }}"
-  tags: supervisor
+  with_items: "{{ systemd_services }}"
+  tags: systemd
 
 - name: Create sdist in prep dir
   command: python3 setup.py sdist

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqrequeue.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=SecureDrop rqrequeue process
+After=redis-server.service
+Wants=redis-server.service
+
+[Service]
+Environment=PYTHONPATH="{{ securedrop_code }}:{{ securedrop_venv_site_packages }}"
+ExecStart={{ securedrop_venv_bin }}/python {{ securedrop_code }}/scripts/rqrequeue --interval 60
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectSystem=full
+ReadOnlyDirectories=/
+ReadWriteDirectories={{ securedrop_data }}
+Restart=always
+RestartSec=10s
+UMask=077
+User=www-data
+WorkingDirectory={{ securedrop_code }}
+
+[Install]
+WantedBy=multi-user.target

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqworker.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=SecureDrop rq worker
+After=redis-server.service
+Wants=redis-server.service
+
+[Service]
+Environment=PYTHONPATH="{{ securedrop_code }}:{{ securedrop_venv_site_packages }}"
+ExecStart={{ securedrop_venv_bin }}/rqworker
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectSystem=full
+ReadOnlyDirectories=/
+ReadWriteDirectories={{ securedrop_data }}
+Restart=always
+RestartSec=10s
+UMask=077
+User=www-data
+WorkingDirectory={{ securedrop_code }}
+
+[Install]
+WantedBy=multi-user.target

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_shredder.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=SecureDrop shredder
+
+[Service]
+Environment=PYTHONPATH="{{ securedrop_code }}:{{ securedrop_venv_site_packages }}"
+ExecStart={{ securedrop_venv_bin }}/python {{ securedrop_code }}/scripts/shredder --interval 60
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectSystem=full
+ReadOnlyDirectories=/
+ReadWriteDirectories={{ securedrop_data }}
+Restart=always
+RestartSec=10s
+UMask=077
+User=www-data
+WorkingDirectory={{ securedrop_code }}
+
+[Install]
+WantedBy=multi-user.target

install_files/securedrop-app-code/debian/control

@@ -3,13 +3,13 @@ Section: web
 Priority: optional
 Maintainer: SecureDrop Team <[email protected]>
 Homepage: https://securedrop.org
-Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-setuptools, dh-virtualenv
+Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-setuptools, dh-systemd, dh-virtualenv
 Standards-Version: 3.9.8
 X-Python3-Version: >= 3.5
 
 Package: securedrop-app-code
 Architecture: amd64
-Conflicts: libapache2-mod-wsgi
-Replaces: libapache2-mod-wsgi
-Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.5, paxctld, python3 (>= 3.5), python3 (<< 3.6), redis-server, securedrop-config, securedrop-keyring, sqlite3, supervisor
-Description: Packages the SecureDrop application code pip dependencies and apparmor profiles. This package will put the apparmor profiles in enforce mode. This package does use pip to install the pip wheelhouse
+Conflicts: libapache2-mod-wsgi,supervisor
+Replaces: libapache2-mod-wsgi,supervisor
+Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.5, paxctld, python3 (>= 3.5), python3 (<< 3.6), redis-server, securedrop-config, securedrop-keyring, sqlite3
+Description: SecureDrop application code, dependencies, Apache configuration, systemd services, and AppArmor profiles. This package will put the AppArmor profiles in enforce mode.

install_files/securedrop-app-code/debian/postinst

@@ -95,27 +95,18 @@ set_paxctld_config() {
     fi
 }
 
-mkdir_rqworker_logs() {
-    worker_logs_dir="/var/log/securedrop_worker"
-    mkdir -p "$worker_logs_dir"
-    chown root:root "$worker_logs_dir"
-    chmod 0700 "$worker_logs_dir"
-}
-
 case "$1" in
     configure)
 
     # Ensure PaX flags are set appropriately
     set_paxctld_config
 
     # Ensure SecureDrop's necessary directories are created
-    for dir in /var/lib/securedrop/{,tmp,store,keys,/keys/private-keys-v1.d,/keys/openpgp-revocs.d,backups} /var/www/securedrop; do
+    for dir in /var/lib/securedrop/{,tmp,shredder,store,keys,/keys/private-keys-v1.d,/keys/openpgp-revocs.d,backups} /var/www/securedrop; do
       mkdir -p "$dir"
       chmod 0700 "$dir"
     done
 
-    mkdir_rqworker_logs
-
     # Ensure required gpg-agent.conf is in place, see #4013.
     if [ -e "/var/lib/securedrop/keys/gpg-agent.conf" ]; then
 
@@ -186,16 +177,8 @@ case "$1" in
     # in versions prior to 0.5.1 a custom logo was installed with u-w
     chmod u+w /var/www/securedrop/static/i/logo.png
 
-    # This removes the MAC "hmac-sha1" from sshd_config.
-    # Ansible was updated, so future instances will not have this line present.
-    # This if-block may be removed from this script on 2019-01-01.
-    if grep -qE 'MACs\s.*hmac-sha1' /etc/ssh/sshd_config; then
-        sed -i 's/^\s*MACs\s.*/MACs hmac-sha2-256,hmac-sha2-512/' /etc/ssh/sshd_config;
-    fi
-
-    service supervisor restart
-
     database_migration
+
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

install_files/securedrop-app-code/debian/rules

@@ -5,7 +5,7 @@ DEB_DH_INSTALL_ARGS=-X .git
 SECUREDROP_BUILD_PLATFORM=$(shell lsb_release -sc)
 
 %:
-	dh $@ --with python-virtualenv
+	dh $@ --with python-virtualenv --with systemd
 
 override_dh_gencontrol:
 	dh_gencontrol -- $(SUBSTVARS)
@@ -32,3 +32,20 @@ override_dh_virtualenv:
 		--extra-pip-arg "--ignore-installed" \
 		--extra-pip-arg "--no-binary=:all:" \
 		--extra-pip-arg "--no-cache-dir"
+
+#
+# Have to override the automatic service handling since we have more
+# than one.
+#
+override_dh_installinit:
+	dh_installinit --noscripts
+
+override_dh_systemd_enable:
+	dh_systemd_enable --name=securedrop_rqrequeue
+	dh_systemd_enable --name=securedrop_rqworker
+	dh_systemd_enable --name=securedrop_shredder
+
+override_dh_systemd_start:
+	dh_systemd_start --name=securedrop_rqrequeue
+	dh_systemd_start --name=securedrop_rqworker
+	dh_systemd_start --name=securedrop_shredder

install_files/securedrop-app-code/debian/securedrop-app-code.install

@@ -1,5 +1,6 @@
 var/www /var/
 etc/apparmor.d/usr.sbin.apache2 /etc/apparmor.d
 etc/apparmor.d/usr.sbin.tor /etc/apparmor.d
-etc/supervisor/conf.d/securedrop_rqrequeue.conf /etc/supervisor/conf.d
-etc/supervisor/conf.d/securedrop_worker.conf /etc/supervisor/conf.d
+lib/systemd/system/securedrop_rqrequeue.service /lib/systemd/system
+lib/systemd/system/securedrop_rqworker.service /lib/systemd/system
+lib/systemd/system/securedrop_shredder.service /lib/systemd/system

molecule/builder-xenial/Dockerfile

@@ -12,6 +12,7 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         debhelper \
         devscripts \
         dh-python \
+        dh-systemd \
         dh-virtualenv \
         gdb \
         git \

molecule/builder-xenial/image_hash

@@ -1,2 +1,2 @@
-# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2019_10_21
-0288d35d316047302e6e15887eb34fb5440415054835cf0c0f25f5cc8ab80279
+# sha256 digest quay.io/freedomofpress/sd-docker-builder-xenial:2019_10_24
+981d2190f643964aee7d1ea8e4fe6a99fd3ab8ebab7179886cd8d558b20d044d