C locking callback (#3226)

* Remove Python OpenSSL locking callback and replace it with one in C

The Python OpenSSL locking callback is unsafe; if GC is triggered
during the callback's invocation, it can result in the callback being
invoked reentrantly, which can lead to deadlocks. This patch replaces
it with one in C that gets built at compile time via cffi along with
the rest of the OpenSSL binding.

* fixes for some issues

* unused

* revert these changes

* these two for good measure

* missing param

* sigh, syntax

* delete tests that assumed an ability to mess with locks

* style fixes

* licensing stuff

* utf8

* Unicode. Huh. What it isn't good for, absolutely nothing.

LICENSE

@@ -1,3 +1,6 @@
 This software is made available under the terms of *either* of the licenses
 found in LICENSE.APACHE or LICENSE.BSD. Contributions to cryptography are made
 under the terms of *both* these licenses.
+
+The code used in the OpenSSL locking callback is derived from the same in
+Python itself, and is licensed under the terms of the PSF License Agreement.

LICENSE.PSF

@@ -0,0 +1,41 @@
+1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and
+   the Individual or Organization ("Licensee") accessing and otherwise using Python
+   2.7.12 software in source or binary form and its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+   grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+   analyze, test, perform and/or display publicly, prepare derivative works,
+   distribute, and otherwise use Python 2.7.12 alone or in any derivative
+   version, provided, however, that PSF's License Agreement and PSF's notice of
+   copyright, i.e., "Copyright © 2001-2016 Python Software Foundation; All Rights
+   Reserved" are retained in Python 2.7.12 alone or in any derivative version
+   prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on or
+   incorporates Python 2.7.12 or any part thereof, and wants to make the
+   derivative work available to others as provided herein, then Licensee hereby
+   agrees to include in any such work a brief summary of the changes made to Python
+   2.7.12.
+
+4. PSF is making Python 2.7.12 available to Licensee on an "AS IS" basis.
+   PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED.  BY WAY OF
+   EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR
+   WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE
+   USE OF PYTHON 2.7.12 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 2.7.12
+   FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF
+   MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 2.7.12, OR ANY DERIVATIVE
+   THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material breach of
+   its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any relationship
+   of agency, partnership, or joint venture between PSF and Licensee.  This License
+   Agreement does not grant permission to use PSF trademarks or trade name in a
+   trademark sense to endorse or promote products or services of Licensee, or any
+   third party.
+
+8. By copying, installing or otherwise using Python 2.7.12, Licensee agrees
+   to be bound by the terms and conditions of this License Agreement.

src/_cffi_src/openssl/callbacks.py

@@ -12,6 +12,9 @@
 #include <openssl/ssl.h>
 #include <openssl/x509.h>
 #include <openssl/x509_vfy.h>
+#include <openssl/crypto.h>
+
+#include <pythread.h>
 """
 
 TYPES = """
@@ -37,6 +40,7 @@
 """
 
 FUNCTIONS = """
+int _setup_ssl_threads(void);
 """
 
 MACROS = """
@@ -50,4 +54,71 @@
     # backwards compatibility for old cffi version on PyPy
     # and Python >=3.5 (https://github.com/pyca/cryptography/issues/2970)
     TYPES = "static const long Cryptography_STATIC_CALLBACKS;"
-    CUSTOMIZATIONS = "static const long Cryptography_STATIC_CALLBACKS = 0;"
+    CUSTOMIZATIONS = """static const long Cryptography_STATIC_CALLBACKS = 0;
+"""
+
+CUSTOMIZATIONS += """
+/* This code is derived from the locking code found in the Python _ssl module's
+   locking callback for OpenSSL.
+
+   Copyright 2001-2016 Python Software Foundation; All Rights Reserved.
+*/
+
+static unsigned int _ssl_locks_count = 0;
+static PyThread_type_lock *_ssl_locks = NULL;
+
+static void _ssl_thread_locking_function(int mode, int n, const char *file,
+                                         int line) {
+    /* this function is needed to perform locking on shared data
+       structures. (Note that OpenSSL uses a number of global data
+       structures that will be implicitly shared whenever multiple
+       threads use OpenSSL.) Multi-threaded applications will
+       crash at random if it is not set.
+
+       locking_function() must be able to handle up to
+       CRYPTO_num_locks() different mutex locks. It sets the n-th
+       lock if mode & CRYPTO_LOCK, and releases it otherwise.
+
+       file and line are the file number of the function setting the
+       lock. They can be useful for debugging.
+    */
+
+    if ((_ssl_locks == NULL) ||
+        (n < 0) || ((unsigned)n >= _ssl_locks_count)) {
+        return;
+    }
+
+    if (mode & CRYPTO_LOCK) {
+        PyThread_acquire_lock(_ssl_locks[n], 1);
+    } else {
+        PyThread_release_lock(_ssl_locks[n]);
+    }
+}
+
+int _setup_ssl_threads(void) {
+    unsigned int i;
+
+    if (_ssl_locks == NULL) {
+        _ssl_locks_count = CRYPTO_num_locks();
+        _ssl_locks = PyMem_New(PyThread_type_lock, _ssl_locks_count);
+        if (_ssl_locks == NULL) {
+            PyErr_NoMemory();
+            return 0;
+        }
+        memset(_ssl_locks, 0, sizeof(PyThread_type_lock) * _ssl_locks_count);
+        for (i = 0;  i < _ssl_locks_count;  i++) {
+            _ssl_locks[i] = PyThread_allocate_lock();
+            if (_ssl_locks[i] == NULL) {
+                unsigned int j;
+                for (j = 0;  j < i;  j++) {
+                    PyThread_free_lock(_ssl_locks[j]);
+                }
+                PyMem_Free(_ssl_locks);
+                return 0;
+            }
+        }
+        CRYPTO_set_locking_callback(_ssl_thread_locking_function);
+    }
+    return 1;
+}
+"""

src/cryptography/hazmat/bindings/openssl/binding.py

@@ -118,8 +118,6 @@ class Binding(object):
     lib = None
     ffi = ffi
     _lib_loaded = False
-    _locks = None
-    _lock_cb_handle = None
     _init_lock = threading.Lock()
     _lock_init_lock = threading.Lock()
 
@@ -178,14 +176,6 @@ def _ensure_ffi_initialized(cls):
     def init_static_locks(cls):
         with cls._lock_init_lock:
             cls._ensure_ffi_initialized()
-
-            if not cls._lock_cb_handle:
-                wrapper = ffi_callback(
-                    "void(int, int, const char *, int)",
-                    name="Cryptography_locking_cb",
-                )
-                cls._lock_cb_handle = wrapper(cls._lock_cb)
-
             # Use Python's implementation if available, importing _ssl triggers
             # the setup for this.
             __import__("_ssl")
@@ -195,25 +185,8 @@ def init_static_locks(cls):
 
             # If nothing else has setup a locking callback already, we set up
             # our own
-            num_locks = cls.lib.CRYPTO_num_locks()
-            cls._locks = [threading.Lock() for n in range(num_locks)]
-
-            cls.lib.CRYPTO_set_locking_callback(cls._lock_cb_handle)
-
-    @classmethod
-    def _lock_cb(cls, mode, n, file, line):
-        lock = cls._locks[n]
-
-        if mode & cls.lib.CRYPTO_LOCK:
-            lock.acquire()
-        elif mode & cls.lib.CRYPTO_UNLOCK:
-            lock.release()
-        else:
-            raise RuntimeError(
-                "Unknown lock mode {0}: lock={1}, file={2}, line={3}.".format(
-                    mode, n, file, line
-                )
-            )
+            res = lib._setup_ssl_threads()
+            _openssl_assert(cls.lib, res == 1)
 
 
 def _verify_openssl_version(version):

tests/hazmat/bindings/test_openssl.py

@@ -31,71 +31,6 @@ def test_crypto_lock_init(self):
         lock_cb = b.lib.CRYPTO_get_locking_callback()
         assert lock_cb != b.ffi.NULL
 
-    def _skip_if_not_fallback_lock(self, b):
-        # only run this test if we are using our locking cb
-        original_cb = b.lib.CRYPTO_get_locking_callback()
-        if original_cb != b._lock_cb_handle:
-            pytest.skip(
-                "Not using the fallback Python locking callback "
-                "implementation. Probably because import _ssl set one"
-            )
-
-    def test_fallback_crypto_lock_via_openssl_api(self):
-        b = Binding()
-        b.init_static_locks()
-
-        self._skip_if_not_fallback_lock(b)
-
-        # check that the lock state changes appropriately
-        lock = b._locks[b.lib.CRYPTO_LOCK_SSL]
-
-        # starts out unlocked
-        assert lock.acquire(False)
-        lock.release()
-
-        b.lib.CRYPTO_lock(
-            b.lib.CRYPTO_LOCK | b.lib.CRYPTO_READ,
-            b.lib.CRYPTO_LOCK_SSL, b.ffi.NULL, 0
-        )
-
-        # becomes locked
-        assert not lock.acquire(False)
-
-        b.lib.CRYPTO_lock(
-            b.lib.CRYPTO_UNLOCK | b.lib.CRYPTO_READ,
-            b.lib.CRYPTO_LOCK_SSL, b.ffi.NULL, 0
-        )
-
-        # then unlocked
-        assert lock.acquire(False)
-        lock.release()
-
-    def test_fallback_crypto_lock_via_binding_api(self):
-        b = Binding()
-        b.init_static_locks()
-
-        self._skip_if_not_fallback_lock(b)
-
-        lock = b._locks[b.lib.CRYPTO_LOCK_SSL]
-
-        with pytest.raises(RuntimeError):
-            b._lock_cb(0, b.lib.CRYPTO_LOCK_SSL, "<test>", 1)
-
-        # errors shouldn't cause locking
-        assert lock.acquire(False)
-        lock.release()
-
-        b._lock_cb(b.lib.CRYPTO_LOCK | b.lib.CRYPTO_READ,
-                   b.lib.CRYPTO_LOCK_SSL, "<test>", 1)
-        # locked
-        assert not lock.acquire(False)
-
-        b._lock_cb(b.lib.CRYPTO_UNLOCK | b.lib.CRYPTO_READ,
-                   b.lib.CRYPTO_LOCK_SSL, "<test>", 1)
-        # unlocked
-        assert lock.acquire(False)
-        lock.release()
-
     def test_add_engine_more_than_once(self):
         b = Binding()
         b._register_osrandom_engine()