x86_64 assembly implementation of cysetjmp/cylongjmp

configure.ac

@@ -107,6 +107,20 @@ if test x$sigsetjmp = xyes; then
     AC_DEFINE(CYSIGNALS_USE_SIGSETJMP, 1, [Define to 1 to use sigsetjmp() in sig_on(), as opposed to setjmp().])
 fi
 
+AC_MSG_CHECKING([for assembly implementation of cysetjmp])
+AC_COMPILE_IFELSE([AC_LANG_SOURCE(
+    [
+    #define CYSIGNALS_ASM_CYSETJMP 1
+    #include "src/cysignals/cysetjmp.h"
+    ])],
+    dnl YES
+    [AC_MSG_RESULT([yes])]
+    AC_DEFINE(CYSIGNALS_ASM_CYSETJMP, 1, [Define to 1 to use an assembly implementation of cysetjmp().])
+    ,
+    dnl NO
+    [AC_MSG_RESULT([no])]
+)
+
 dnl Check for atomic operations
 AC_MSG_CHECKING([for _Atomic in C code])
 AC_COMPILE_IFELSE([AC_LANG_SOURCE(

src/cysignals/cysetjmp.h

@@ -0,0 +1,116 @@
+/*****************************************************************************
+ *       Copyright (C) 2006 William Stein <[email protected]>
+ *                     2006 Martin Albrecht <[email protected]>
+ *                     2010-2019 Jeroen Demeyer <[email protected]>
+ *
+ * cysignals is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * cysignals is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with cysignals.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ ****************************************************************************/
+
+#ifndef CYSIGNALS_CYSETJMP_H
+#define CYSIGNALS_CYSETJMP_H
+
+
+#include <stddef.h>
+#include <setjmp.h>
+
+
+#if CYSIGNALS_ASM_CYSETJMP
+#ifdef __x86_64__
+/*
+ * x86_64 assembly implementation of cysetjmp(): we store the registers
+ * rsp and rbp and the instruction pointer.
+ *
+ * rbx is used to pass a pointer to a cyjmp_struct,
+ * rdx is the value passed to cylongjmp().
+ */
+struct cyjmp_struct
+{
+    size_t rsp;
+    size_t rbp;
+    size_t rip;
+};
+
+typedef struct cyjmp_struct cyjmp_buf[1];
+
+static inline int __attribute__((always_inline))
+cysetjmp(struct cyjmp_struct* env)
+{
+    int res;
+    __asm__ goto("\n"
+        "\tleaq %l1(%%rip), %%rcx\n"
+        "\tmovq %%rsp, 0(%0)\n"
+        "\tmovq %%rbp, 8(%0)\n"
+        "\tmovq %%rcx, 16(%0)\n"
+    :
+    : "b" (env)
+    : /* Clobber all registers except for rbx, rsp, rbp */
+      "%rax", "%rcx", "%rdx", "%rsi", "%rdi",
+      "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
+      "%mm0", "%mm1", "%mm2", "%mm3", "%mm4", "%mm5", "%mm6", "%mm7",
+      "%xmm0", "%xmm1", "%xmm2", "%xmm3", "%xmm4", "%xmm5", "%xmm6", "%xmm7",
+      "%xmm8", "%xmm9", "%xmm10", "%xmm11", "%xmm12", "%xmm13", "%xmm14", "%xmm15",
+#ifdef __AVX__
+      "%ymm0", "%ymm1", "%ymm2", "%ymm3", "%ymm4", "%ymm5", "%ymm6", "%ymm7",
+      "%ymm8", "%ymm9", "%ymm10", "%ymm11", "%ymm12", "%ymm13", "%ymm14", "%ymm15",
+#endif
+#ifdef __AVX512F__
+      "%xmm16", "%xmm17", "%xmm18", "%xmm19", "%xmm20", "%xmm21", "%xmm22", "%xmm23",
+      "%xmm24", "%xmm25", "%xmm26", "%xmm27", "%xmm28", "%xmm29", "%xmm30", "%xmm31",
+      "%ymm16", "%ymm17", "%ymm18", "%ymm19", "%ymm20", "%ymm21", "%ymm22", "%ymm23",
+      "%ymm24", "%ymm25", "%ymm26", "%ymm27", "%ymm28", "%ymm29", "%ymm30", "%ymm31",
+      "%zmm0", "%zmm1", "%zmm2", "%zmm3", "%zmm4", "%zmm5", "%zmm6", "%zmm7",
+      "%zmm8", "%zmm9", "%zmm10", "%zmm11", "%zmm12", "%zmm13", "%zmm14", "%zmm15",
+      "%zmm16", "%zmm17", "%zmm18", "%zmm19", "%zmm20", "%zmm21", "%zmm22", "%zmm23",
+      "%zmm24", "%zmm25", "%zmm26", "%zmm27", "%zmm28", "%zmm29", "%zmm30", "%zmm31",
+#endif
+      "cc", "memory"
+    : res_in_rdx);
+
+    return 0;
+
+res_in_rdx:
+    __attribute__((cold));
+    __asm__ volatile("": "=d" (res));
+    return res;
+}
+
+static void __attribute__((noreturn))
+cylongjmp(const struct cyjmp_struct* env, int val)
+{
+    if (val == 0) val = 1;
+
+    __asm__ volatile("\n"
+        "\tmovq 16(%0), %%rcx\n"
+        "\tmovq 8(%0), %%rbp\n"
+        "\tmovq 0(%0), %%rsp\n"
+        "\tjmp *%%rcx\n"
+    :
+    : "b" (env), "d" (val));
+    __builtin_unreachable();
+}
+#else
+#error "assembly implementation of cysetjmp requires x86_64"
+#endif
+#elif CYSIGNALS_USE_SIGSETJMP
+#define cyjmp_buf sigjmp_buf
+#define cysetjmp(env) sigsetjmp(env, 0)
+#define cylongjmp(env, val) siglongjmp(env, val)
+#else
+#define cyjmp_buf jmp_buf
+#define cysetjmp(env) setjmp(env)
+#define cylongjmp(env, val) longjmp(env, val)
+#endif
+
+#endif

src/cysignals/cysignals_config.h.in

@@ -9,6 +9,7 @@
 #undef ENABLE_DEBUG_CYSIGNALS
 #endif
 
+
 /*
  * Should sig_on() use sigsetjmp(env, 0) instead of setjmp(env)? This
  * is needed on BSD and OS X because setjmp() saves the signal mask.
@@ -19,6 +20,13 @@
 #endif
 
 
+/* Should we use an assembly implementation of cysetjmp()? */
+#ifndef CYSIGNALS_ASM_CYSETJMP
+#undef CYSIGNALS_ASM_CYSETJMP
+#endif
+
+
+/* Which implementation of atomic variables (if any) to use? */
 #ifndef __cplusplus
 #undef CYSIGNALS_C_ATOMIC
 #else

src/cysignals/struct_signals.h

@@ -1,7 +1,7 @@
 /*****************************************************************************
  *       Copyright (C) 2006 William Stein <[email protected]>
  *                     2006 Martin Albrecht <[email protected]>
- *                     2010-2016 Jeroen Demeyer <[email protected]>
+ *                     2010-2019 Jeroen Demeyer <[email protected]>
  *
  * cysignals is free software: you can redistribute it and/or modify it
  * under the terms of the GNU Lesser General Public License as published
@@ -23,23 +23,11 @@
 
 
 #include "cysignals_config.h"
-#include <setjmp.h>
+#include "cysetjmp.h"
 #include <signal.h>
 #include <Python.h>
 
 
-/* Choose sigjmp/longjmp variant */
-#if CYSIGNALS_USE_SIGSETJMP
-#define cyjmp_buf sigjmp_buf
-#define cysetjmp(env) sigsetjmp(env, 0)
-#define cylongjmp(env, val) siglongjmp(env, val)
-#else
-#define cyjmp_buf jmp_buf
-#define cysetjmp(env) setjmp(env)
-#define cylongjmp(env, val) longjmp(env, val)
-#endif
-
-
 /* Define a cy_atomic_int type for atomic operations */
 #if CYSIGNALS_C_ATOMIC || CYSIGNALS_CXX_ATOMIC
 typedef volatile _Atomic int cy_atomic_int;
@@ -97,4 +85,5 @@ typedef struct
 #endif
 } cysigs_t;
 
+
 #endif  /* ifndef CYSIGNALS_STRUCT_SIGNALS_H */

src/setjmp_bench.c

@@ -8,6 +8,42 @@ static jmp_buf env;
 static sigjmp_buf sigenv;
 
 
+#if __x86_64__
+struct cyjmp_struct
+{
+    size_t rsp;
+    size_t rbp;
+    size_t rip;
+};
+
+static inline int
+cysetjmp(struct cyjmp_struct* env)
+{
+    int res;
+    asm goto("\n"
+        "\tleaq %l1(%%rip), %%rcx\n"
+        "\tmovq %%rsp, 0(%0)\n"
+        "\tmovq %%rbp, 8(%0)\n"
+        "\tmovq %%rcx, 16(%0)\n"
+    :
+    : "b" (env)
+    : /* Clobber all registers except for rdx, rbx, rsp, rbp */
+      "%rax", "%rcx", "%rdx", "%rsi", "%rdi",
+      "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15",
+      "cc", "memory"
+    : res_in_rdx);
+
+    return 0;
+
+res_in_rdx:
+    asm volatile("": "=d" (res));
+    return res;
+}
+
+static struct cyjmp_struct cyenv;
+#endif
+
+
 #define BENCH(CODE) \
     gettimeofday(&tv0, NULL); \
     for (i = 0; i < N; i++) {CODE; asm("");} \
@@ -27,9 +63,14 @@ int main(int argc, char** argv)
     BENCH(if (setjmp(env)) return 0)
     printf("Time for setjmp(env):      %8.2fns\n", ns);
 
-    BENCH(if (sigsetjmp(env, 0)) return 0)
+    BENCH(if (sigsetjmp(sigenv, 0)) return 0)
     printf("Time for sigsetjmp(env, 0):%8.2fns\n", ns);
 
-    BENCH(if (sigsetjmp(env, 1)) return 0)
+    BENCH(if (sigsetjmp(sigenv, 1)) return 0)
     printf("Time for sigsetjmp(env, 1):%8.2fns\n", ns);
+
+#if __x86_64__
+    BENCH(if (cysetjmp(&cyenv)) return 0)
+    printf("Time for asm implementation:%7.2fns\n", ns);
+#endif
 }