Manage rq services with systemd instead of supervisor

This change makes securedrop-app-code Conflict/Replace supervisor, to ensure that supervisor is purged from the system. The rq worker and requeuing processes are now managed as systemd services, with configuration files in /lib/systemd/system. Submission hashing and deletion works as before. Requeuing a job interrupted by a reboot is less likely to work, as the worker seems to be shut down such that the running jobs are terminated and moved by rq to the failed queue. Files surviving that will be reported in the next daily check, and can be deleted with manage.py. The services now log to the systemd journal, which in our current configuration is stored only in memory, so will not survive reboots. This could be changed by creating /var/log/journal, but if we do that we'll need to audit what other systemd services are logging.
freedomofpress · Oct 23, 2019 · 5e00c90 · 5e00c90
1 parent 43cbd72
commit 5e00c90
Show file tree

Hide file tree

Showing 26 changed files with 192 additions and 201 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1,5 +1,6 @@
 graft debian
 graft etc
+graft lib
 graft var
 prune var/www/securedrop/tests
 include requirements.txt

diff --git a/docs/backup_and_restore.rst b/docs/backup_and_restore.rst
@@ -41,7 +41,7 @@ currently on the *Application Server*: log in over SSH and run
           queued deletion jobs by logging in to the *Application
           Server* over SSH and running::
 
-            sudo tail -f /var/log/securedrop_worker/rqworker.err
+            sudo journalctl -u securedrop_rqworker
 
 
 If you find you cannot perform a backup or restore due to this constraint,
@@ -85,8 +85,8 @@ to debug your connectivity before proceeding further. Make sure:
 * Ansible is installed
 * the *Admin Workstation* is connected to the Internet
 * Tor starts successfully
-* The appropriate onion service configuration files are present in 
-  ``~/Persistent/securedrop/install_files/ansible-base`` and the 
+* The appropriate onion service configuration files are present in
+  ``~/Persistent/securedrop/install_files/ansible-base`` and the
   ``./securedrop-admin tailsconfig`` command completes successfully
 
 If Ansible is not installed, or ``./securedrop-admin tailsconfig`` fails, see

diff --git a/install_files/ansible-base/roles/app/handlers/main.yml b/install_files/ansible-base/roles/app/handlers/main.yml
@@ -14,16 +14,6 @@
     name: apache2
     state: restarted
 
-- name: reload supervisor
-  supervisorctl:
-    name: securedrop_worker
-    state: present
-
-- name: reload supervisor
-  supervisorctl:
-    name: securedrop_rqrequeue
-    state: present
-
 ## Here, we list apparmor before haveged to ensure the correct AppArmor
 ## profile is loaded prior to restarting haveged
 - name: restart apparmor

diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/defaults/main.yml
@@ -16,10 +16,10 @@ securedrop_venv_site_packages: "{{ securedrop_venv }}/lib/python3.5/site-package
 # SecureDrop code installation directory
 securedrop_code: /var/www/securedrop
 
-# Configuration files for SecureDrop programs run under supervisord
-supervisor_programs:
-  - securedrop_rqrequeue.conf
-  - securedrop_worker.conf
+# Configuration files for SecureDrop systemd services
+systemd_services:
+  - securedrop_rqrequeue.service
+  - securedrop_rqworker.service
 
 # SecureDrop rq worker log directory
 securedrop_worker_log_dir: /var/log/securedrop_worker

diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2 b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/usr.sbin.apache2
@@ -102,7 +102,6 @@
   /usr/lib{,32,64}/** mr,
   /usr/share/file/magic r,
   /usr/share/file/magic.mgc r,
-  /usr/share/pyshared/supervisor-*-nspkg.pth r,
   /opt/venvs/securedrop-app-code/**/__pycache__/ rw,
   /opt/venvs/securedrop-app-code/**/__pycache__/* rw,
   /opt/venvs/securedrop-app-code/bin/python3 r,

diff --git a/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml b/install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/main.yml
@@ -57,19 +57,19 @@
     src: "changelog-{{ securedrop_package_dist }}"
     dest: "{{ securedrop_app_code_prep_dir }}/debian/changelog"
 
-- name: Create supervisor conf.d directory in prep directory
+- name: Create lib/systemd/services directory in prep directory
   file:
     state: directory
-    dest: "{{ securedrop_app_code_prep_dir }}/etc/supervisor/conf.d"
-  tags: supervisor
+    dest: "{{ securedrop_app_code_prep_dir }}/lib/systemd/system"
+  tags: systemd
 
-- name: Copy supervisor program configurations to prep path
+- name: Copy systemd service configurations to prep path
   template:
     src: "{{ item }}"
-    dest: "{{ securedrop_app_code_prep_dir }}/etc/supervisor/conf.d/{{ item }}"
+    dest: "{{ securedrop_app_code_prep_dir }}/lib/systemd/system/{{ item }}"
     mode: 0644
-  with_items: "{{ supervisor_programs }}"
-  tags: supervisor
+  with_items: "{{ systemd_services }}"
+  tags: systemd
 
 - name: Create sdist in prep dir
   command: python3 setup.py sdist

diff --git a/.../ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqrequeue.conf b/.../ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqrequeue.conf
diff --git a/...sible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqrequeue.service b/...sible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqrequeue.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=SecureDrop rqrequeue process
+After=redis-server.service
+Wants=redis-server.service
+
+[Service]
+Environment=PYTHONPATH="{{ securedrop_code }}:{{ securedrop_venv_site_packages }}"
+ExecStart={{ securedrop_venv_bin }}/python /var/www/securedrop/scripts/rqrequeue --interval 60
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectSystem=full
+ReadOnlyDirectories=/
+ReadWriteDirectories=/var/lib/securedrop
+Restart=always
+RestartSec=10s
+UMask=077
+User=www-data
+WorkingDirectory={{ securedrop_code }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/...nsible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqworker.service b/...nsible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_rqworker.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=SecureDrop rq worker
+After=redis-server.service
+Wants=redis-server.service
+
+[Service]
+Environment=PYTHONPATH="{{ securedrop_code }}:{{ securedrop_venv_site_packages }}"
+ExecStart={{ securedrop_venv_bin }}/rqworker
+PrivateDevices=yes
+PrivateTmp=yes
+ProtectSystem=full
+ReadOnlyDirectories=/
+ReadWriteDirectories=/var/lib/securedrop
+Restart=always
+RestartSec=10s
+UMask=077
+User=www-data
+WorkingDirectory={{ securedrop_code }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/...les/ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_worker.conf b/...les/ansible-base/roles/build-securedrop-app-code-deb-pkg/templates/securedrop_worker.conf
diff --git a/install_files/securedrop-app-code/debian/control b/install_files/securedrop-app-code/debian/control
@@ -3,13 +3,13 @@ Section: web
 Priority: optional
 Maintainer: SecureDrop Team <[email protected]>
 Homepage: https://securedrop.org
-Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-setuptools, dh-virtualenv
+Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-setuptools, dh-systemd, dh-virtualenv
 Standards-Version: 3.9.8
 X-Python3-Version: >= 3.5
 
 Package: securedrop-app-code
 Architecture: amd64
-Conflicts: libapache2-mod-wsgi
-Replaces: libapache2-mod-wsgi
-Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.5, paxctld, python3 (>= 3.5), python3 (<< 3.6), redis-server, securedrop-config, securedrop-keyring, sqlite3, supervisor
-Description: Packages the SecureDrop application code pip dependencies and apparmor profiles. This package will put the apparmor profiles in enforce mode. This package does use pip to install the pip wheelhouse
+Conflicts: libapache2-mod-wsgi,supervisor
+Replaces: libapache2-mod-wsgi,supervisor
+Depends: ${dist:Depends}, ${misc:Depends}, ${python3:Depends}, apache2, apparmor-utils, coreutils, gnupg2, haveged, libapache2-mod-xsendfile, libpython3.5, paxctld, python3 (>= 3.5), python3 (<< 3.6), redis-server, securedrop-config, securedrop-keyring, sqlite3
+Description: SecureDrop application code, dependencies, Apache configuration, systemd services, and AppArmor profiles. This package will put the AppArmor profiles in enforce mode.
diff --git a/install_files/securedrop-app-code/debian/postinst b/install_files/securedrop-app-code/debian/postinst
@@ -95,13 +95,6 @@ set_paxctld_config() {
     fi
 }
 
-mkdir_rqworker_logs() {
-    worker_logs_dir="/var/log/securedrop_worker"
-    mkdir -p "$worker_logs_dir"
-    chown root:root "$worker_logs_dir"
-    chmod 0700 "$worker_logs_dir"
-}
-
 case "$1" in
     configure)
 
@@ -114,8 +107,6 @@ case "$1" in
       chmod 0700 "$dir"
     done
 
-    mkdir_rqworker_logs
-
     # Ensure required gpg-agent.conf is in place, see #4013.
     if [ -e "/var/lib/securedrop/keys/gpg-agent.conf" ]; then
 
@@ -186,16 +177,8 @@ case "$1" in
     # in versions prior to 0.5.1 a custom logo was installed with u-w
     chmod u+w /var/www/securedrop/static/i/logo.png
 
-    # This removes the MAC "hmac-sha1" from sshd_config.
-    # Ansible was updated, so future instances will not have this line present.
-    # This if-block may be removed from this script on 2019-01-01.
-    if grep -qE 'MACs\s.*hmac-sha1' /etc/ssh/sshd_config; then
-        sed -i 's/^\s*MACs\s.*/MACs hmac-sha2-256,hmac-sha2-512/' /etc/ssh/sshd_config;
-    fi
-
-    service supervisor restart
-
     database_migration
+
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure)

diff --git a/install_files/securedrop-app-code/debian/rules b/install_files/securedrop-app-code/debian/rules
@@ -5,7 +5,7 @@ DEB_DH_INSTALL_ARGS=-X .git
 SECUREDROP_BUILD_PLATFORM=$(shell lsb_release -sc)
 
 %:
-	dh $@ --with python-virtualenv
+	dh $@ --with python-virtualenv --with systemd
 
 override_dh_gencontrol:
 	dh_gencontrol -- $(SUBSTVARS)
@@ -32,3 +32,18 @@ override_dh_virtualenv:
 		--extra-pip-arg "--ignore-installed" \
 		--extra-pip-arg "--no-binary=:all:" \
 		--extra-pip-arg "--no-cache-dir"
+
+#
+# Have to override the automatic service handling since we have more
+# than one.
+#
+override_dh_installinit:
+	dh_installinit --noscripts
+
+override_dh_systemd_enable:
+	dh_systemd_enable --name=securedrop_rqrequeue
+	dh_systemd_enable --name=securedrop_rqworker
+
+override_dh_systemd_start:
+	dh_systemd_start --name=securedrop_rqrequeue
+	dh_systemd_start --name=securedrop_rqworker
diff --git a/install_files/securedrop-app-code/debian/securedrop-app-code.install b/install_files/securedrop-app-code/debian/securedrop-app-code.install
@@ -1,5 +1,5 @@
 var/www /var/
 etc/apparmor.d/usr.sbin.apache2 /etc/apparmor.d
 etc/apparmor.d/usr.sbin.tor /etc/apparmor.d
-etc/supervisor/conf.d/securedrop_rqrequeue.conf /etc/supervisor/conf.d
-etc/supervisor/conf.d/securedrop_worker.conf /etc/supervisor/conf.d
+lib/systemd/system/securedrop_rqrequeue.service /lib/systemd/system
+lib/systemd/system/securedrop_rqworker.service /lib/systemd/system
diff --git a/molecule/builder-xenial/Dockerfile b/molecule/builder-xenial/Dockerfile
@@ -12,6 +12,7 @@ RUN apt-get -y update && apt-get upgrade -y && apt-get install -y \
         debhelper \
         devscripts \
         dh-python \
+        dh-systemd \
         dh-virtualenv \
         gdb \
         git \

diff --git a/molecule/testinfra/staging/app-code/test_redis_worker.py b/molecule/testinfra/staging/app-code/test_redis_worker.py
diff --git a/molecule/testinfra/staging/app-code/test_rqrequeue_conf.py b/molecule/testinfra/staging/app-code/test_rqrequeue_conf.py
diff --git a/molecule/testinfra/staging/app-code/test_securedrop_app_code.py b/molecule/testinfra/staging/app-code/test_securedrop_app_code.py
@@ -28,7 +28,6 @@ def test_apache_default_docroot_is_absent(host):
     'securedrop-config',
     'securedrop-keyring',
     'sqlite3',
-    'supervisor',
 ])
 def test_securedrop_application_apt_dependencies(host, package):
     """