Adds reprotest for .deb files

More pytest-based reprotest invocations, this time focusing on .deb files. Replaces the CircleCI repro tests that manually compared hashes. Also modifies build script to support commit hash The logic assumed we were always building from a prod release tag. As a result, the CI logic was reimplementing the tarball mangling. Let's make the script more flexible, so we can run the script in CI and thereby get a bit more test coverage for it. Modifies CI env for reprotest support When building .deb packages, we need the python version for the packaging environment to match that of the target platform, i.e. python3.7 for buster. In CI, our platform options are: * VM, Ubuntu 20.04 * Container, Debian 10 The container driver in CircleCI does not permit "setarch" calls, erroring out immediately. The setarch calls are not optional in reprotest, unfortunately, so let's hack the file and remove it entirely, only in CI.
freedomofpress · Jan 25, 2021 · 18770bd · 18770bd
1 parent 1a69e39
commit 18770bd
Show file tree

Hide file tree

Showing 8 changed files with 108 additions and 80 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -138,30 +138,8 @@ common-steps:
         export PKG_PATH=~/packaging/$PKG_NAME/dist/$PKG_NAME-$VERSION_TO_BUILD.tar.gz
         export PKG_VERSION=$VERSION_TO_BUILD
         make $PKG_NAME
-        ls ~/debbuild/packaging/*.deb
+        ls ~/project/build/debbuild/packaging/*.deb
 
-  - &builddebianpackagefromexistingtarball
-    run:
-      name: Build debian package from committed tarball
-      command: |
-        export PKG_PATH=~/project/tarballs/$PKG_NAME-$PKG_VERSION.tar.gz
-
-        # Every tarball should be signed
-        gpg --import ~/project/pubkeys/release_key.pub
-        gpg --verify $PKG_PATH.asc
-
-        # Build debian package
-        make $PKG_NAME
-        export PKG_HASH_1=$(shasum -a 256 ~/debbuild/packaging/$PKG_NAME*.deb | awk '{print $1}')
-        echo $PKG_HASH_1
-
-        # Build debian package again
-        make $PKG_NAME
-        export PKG_HASH_2=$(shasum -a 256 ~/debbuild/packaging/$PKG_NAME*.deb | awk '{print $1}')
-        echo $PKG_HASH_2
-
-        # Fail build if hashes aren’t equal
-        python -c "import os, sys; sys.exit(os.environ['PKG_HASH_1'] != os.environ['PKG_HASH_2'])"
 
   - &addsshkeys
     add_ssh_keys:
@@ -294,6 +272,38 @@ jobs:
             pip install -r test-requirements.txt
             make test
 
+  reprotest-wheels:
+    machine:
+      image: ubuntu-2004:202010-01
+    steps:
+      - checkout
+      - run:
+          name: install test requirements and run tests
+          command: |
+            make install-deps
+            virtualenv -p python3 .venv
+            source .venv/bin/activate
+            pip install -r test-requirements.txt
+            pytest -vvs tests/test_reproducible_wheels.py
+
+  reprotest-debs:
+    docker:
+      - image: circleci/python:3.7-buster
+    steps:
+      - checkout
+      - run:
+          name: install test requirements and run tests
+          command: |
+            make install-deps
+            virtualenv -p python3 .venv
+            source .venv/bin/activate
+            pip install -r test-requirements.txt
+            # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers.
+            # We cannot use Ubuntu 20.04 python3.8 to build Debian 10 python3.7 packages.
+            sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
+            pytest -vvs tests/test_reproducible_debian_packages.py
+
+
   build-buster-securedrop-log:
     docker:
       - image: circleci/python:3.7-buster
@@ -302,7 +312,7 @@ jobs:
       - *removevirtualenv
       - *installdeps
       - *clonesecuredroplog
-      - *getlatestreleasedversion
+      - *getnightlyversion
       - *makesourcetarball
       - *builddebianpackage
 
@@ -329,7 +339,7 @@ jobs:
       - *removevirtualenv
       - *installdeps
       - *clonesecuredropclient
-      - *getlatestreleasedversion
+      - *getnightlyversion
       - *makesourcetarball
       - *builddebianpackage
 
@@ -356,7 +366,7 @@ jobs:
       - *removevirtualenv
       - *installdeps
       - *clonesecuredropproxy
-      - *getlatestreleasedversion
+      - *getnightlyversion
       - *makesourcetarball
       - *builddebianpackage
 
@@ -481,51 +491,12 @@ jobs:
       - *setmetapackageversion
       - *builddebianpackage
 
-  reproducibility-checks:
-    docker:
-      - image: circleci/python:3.7-buster
-    steps:
-      - checkout
-      - *removevirtualenv
-      - *installdeps
-      - run: git lfs pull
-      - run:
-          name: Test build process reproducibility on latest securedrop-client tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-client-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-client' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-      - run:
-          name: Test build process reproducibility on latest securedrop-proxy tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-proxy-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-proxy' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-      - run:
-          name: Test build process reproducibility on latest securedrop-log tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-log-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-log' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-      - run:
-          name: Test build process reproducibility on latest securedrop-export tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-export-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-export' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-
 workflows:
   build-packages:
     jobs:
       - tests
+      - reprotest-wheels
+      - reprotest-debs
       - build-buster-securedrop-client
       - build-buster-securedrop-proxy
       - build-buster-securedrop-workstation-svs-disp
@@ -535,7 +506,6 @@ workflows:
       - build-buster-securedrop-workstation-config
       - build-buster-securedrop-keyring
       - make-dom0-rpm
-      - reproducibility-checks
 
   # Nightly jobs for each package are run in series to ensure there are no
   # conflicts or race conditions when committing deb packages to git-lfs.

diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,4 @@
 tests/__pycache__/
 debhelper-build-stamp
 *.debhelper.log
+build/
diff --git a/Makefile b/Makefile
@@ -54,16 +54,16 @@ build-wheels: ## Builds the wheels and adds them to the localwheels directory
 	@printf "to push these changes to the FPF PyPI index\n"
 
 .PHONY: test
-test: ## Run test suite
-	pytest -v tests/
+test: ## Run simple test suite (skips reproducibility checks)
+	pytest -v tests/test_update_requirements.py
 
 .PHONY: clean
 clean: ## Removes all non-version controlled packaging artifacts
 	rm -rf localwheels/*
 
 .PHONY: reprotest
-reprotest: ## Reproducibility test, currently only for wheels
-	pytest -vvs tests/test_reproducible_wheels.py
+reprotest: ## Runs only reproducibility tests, for .deb and .whl files
+	pytest -vvs tests/test_reproducible_*.py
 
 .PHONY: help
 help: ## Prints this message and exits

diff --git a/build/.gitkeep b/build/.gitkeep
diff --git a/scripts/build-debianpackage b/scripts/build-debianpackage
@@ -23,12 +23,12 @@ export DH_PIP_EXTRA_ARGS="--no-cache-dir --require-hashes"
 
 # Declare general packaging building workspace; subdirs will
 # be created within, to build specific packages.
-TOP_BUILDDIR="$HOME/debbuild/packaging"
+TOP_BUILDDIR="$PWD/build/debbuild/packaging"
 mkdir -p "$TOP_BUILDDIR"
 rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}"
 mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}"
 # Move changelog into place (we have separate changelogs for each platform)
-PLATFORM="$(lsb_release -sc)"
+PLATFORM="${PKG_PLATFORM:-buster}"
 
 # Validate required args.
 if [[ -z "${PKG_NAME:-}" ]]; then
@@ -83,11 +83,16 @@ function build_source_tarball() {
     rm -rf "$build_dir"
     git clone "$repo_url" "$build_dir"
 
-    # Verify tag, using only the prod key
-    verify_git_tag "$build_dir" "$PKG_VERSION"
+    if [[ -n "$PKG_GITREF" ]]; then
+        # Can't expect a prod sig on a gitref, likely a feature branch
+        git -C "$build_dir" checkout "$PKG_GITREF"
+    else
+        # Verify tag, using only the prod key
+        verify_git_tag "$build_dir" "$PKG_VERSION"
+        # Tag is verified, proceed with checkout
+        git -C "$build_dir" checkout "$PKG_VERSION"
+    fi
 
-    # Tag is verified, proceed with checkout
-    git -C "$build_dir" checkout "$PKG_VERSION"
     (cd "$build_dir" && LC_ALL="C.UTF-8" python setup.py sdist)
 
     # Initial tarball will contain timestamps from NOW, let's repack

diff --git a/test-requirements.txt b/test-requirements.txt
@@ -1,2 +1,3 @@
 pytest
-pytest-mock
+pytest-mock
+virtualenv<16
diff --git a/tests/test_reproducible_debian_packages.py b/tests/test_reproducible_debian_packages.py
@@ -0,0 +1,48 @@
+import pytest
+import subprocess
+import os
+
+
+PACKAGE_BUILD_TARGETS = {
+    "securedrop-client": "main",
+    "securedrop-log": "main",
+    "securedrop-proxy": "main",
+    "securedrop-export": "main",
+}
+
+# These are the package names we want to test reproducibility for
+PACKAGE_NAMES = PACKAGE_BUILD_TARGETS.keys()
+
+
+def get_repo_root():
+    cmd = "git rev-parse --show-toplevel".split()
+    top_level = subprocess.check_output(cmd).decode("utf-8").rstrip()
+    return top_level
+
+repo_root = get_repo_root()
+
+
+@pytest.mark.parametrize("pkg_name", PACKAGE_NAMES)
+def test_deb_builds_are_reproducible(pkg_name):
+    """
+    Uses 'reprotest' to confirm that the Debian package build process
+    is deterministic, i.e. all .deb files are created with the same checksum
+    across multiple builds.
+
+    We're not testing many variations, only exec_path, as a simple test
+    for deterministic builds with most aspects controlled.
+    """
+
+    cmd_env = os.environ.copy()
+    cmd_env["PKG_GITREF"] = os.environ.get("PKG_GITREF", PACKAGE_BUILD_TARGETS[pkg_name])
+    cmd_env["TERM"] = "xterm-256color"
+    cmd = [
+        "reprotest",
+        "-c",
+        f"make {pkg_name}",
+        "--variations",
+        "-all, -kernel, +exec_path",
+        ".",
+        f"build/debbuild/packaging/{pkg_name}*.deb",
+    ]
+    subprocess.check_call(cmd, env=cmd_env, cwd=repo_root)
diff --git a/tests/test_reproducible_wheels.py b/tests/test_reproducible_wheels.py
@@ -1,5 +1,6 @@
 import pytest
 import subprocess
+import os
 
 
 # These are the SDW repositories that we build wheels for.
@@ -30,6 +31,8 @@ def test_wheel_builds_are_reproducible(repo_name):
       * kernel: x86_64 is the supported architecure, we don't ship others
     """
     repo_url = f"https://github.com/freedomofpress/{repo_name}"
+    cmd_env = os.environ.copy()
+    cmd_env["TERM"] = "xterm-256color"
     cmd = [
         "reprotest",
         "-c",
@@ -40,4 +43,4 @@ def test_wheel_builds_are_reproducible(repo_name):
         "localwheels/*.whl",
     ]
     repo_root = get_repo_root()
-    subprocess.check_call(cmd, cwd=repo_root)
+    subprocess.check_call(cmd, env=cmd_env, cwd=repo_root)