From 08a8b245c378b0a7b1d786f3582e8f2098322056 Mon Sep 17 00:00:00 2001 From: stepan Date: Thu, 22 Sep 2022 14:30:19 +0200 Subject: [PATCH 1/5] Emit HPy ABI version into the binary and check it when loading a module --- hpy/devel/include/hpy.h | 8 ++++++ hpy/devel/include/hpy/hpymodule.h | 32 ++++++++++++---------- hpy/universal/src/hpymodule.c | 45 +++++++++++++++++++++++++++++++ test/test_00_basic.py | 18 +++++++++++++ 4 files changed, 89 insertions(+), 14 deletions(-) diff --git a/hpy/devel/include/hpy.h b/hpy/devel/include/hpy.h index 32e66605e..9fcccf110 100644 --- a/hpy/devel/include/hpy.h +++ b/hpy/devel/include/hpy.h @@ -6,7 +6,15 @@ extern "C" { /* ~~~~~~~~~~~~~~~~ HPy ABI version ~~~~~~~~~~~~~~~ */ // NOTE: these must be kept on sync with the equivalent variables in hpy/devel/abitag.py +/** + * The ABI version. + * + * Minor version N+1 is binary compatible to minor version N. Major versions + * are not binary compatible (note: HPy can run several binary incompatible + * versions in one process). + */ #define HPY_ABI_VERSION 0 +#define HPY_ABI_VERSION_MINOR 0 #define HPY_ABI_TAG "hpy0" diff --git a/hpy/devel/include/hpy/hpymodule.h b/hpy/devel/include/hpy/hpymodule.h index c5cf19b23..ba6079eda 100644 --- a/hpy/devel/include/hpy/hpymodule.h +++ b/hpy/devel/include/hpy/hpymodule.h @@ -40,25 +40,29 @@ typedef struct { #ifdef HPY_ABI_CPYTHON // module initialization in the CPython case -#define HPy_MODINIT(modname) \ - static HPy init_##modname##_impl(HPyContext *ctx); \ - PyMODINIT_FUNC \ - PyInit_##modname(void) \ - { \ - return _h2py(init_##modname##_impl(_HPyGetContext())); \ +#define HPy_MODINIT(modname) \ + static HPy init_##modname##_impl(HPyContext *ctx); \ + PyMODINIT_FUNC \ + PyInit_##modname(void) \ + { \ + return _h2py(init_##modname##_impl(_HPyGetContext())); \ } #else // HPY_ABI_CPYTHON // module initialization in the universal and hybrid case -#define HPy_MODINIT(modname) \ - _HPy_CTX_MODIFIER HPyContext *_ctx_for_trampolines; \ - static HPy init_##modname##_impl(HPyContext *ctx); \ - HPyMODINIT_FUNC \ - HPyInit_##modname(HPyContext *ctx) \ - { \ - _ctx_for_trampolines = ctx; \ - return init_##modname##_impl(ctx); \ +#define HPy_MODINIT(modname) \ + HPy_EXPORTED_SYMBOL uint32_t \ + required_hpy_major_version_##modname = HPY_ABI_VERSION; \ + HPy_EXPORTED_SYMBOL uint32_t \ + required_hpy_minor_version_##modname = HPY_ABI_VERSION_MINOR; \ + _HPy_CTX_MODIFIER HPyContext *_ctx_for_trampolines; \ + static HPy init_##modname##_impl(HPyContext *ctx); \ + HPyMODINIT_FUNC \ + HPyInit_##modname(HPyContext *ctx) \ + { \ + _ctx_for_trampolines = ctx; \ + return init_##modname##_impl(ctx); \ } #endif // HPY_ABI_CPYTHON diff --git a/hpy/universal/src/hpymodule.c b/hpy/universal/src/hpymodule.c index f596ccbec..73728174d 100644 --- a/hpy/universal/src/hpymodule.c +++ b/hpy/universal/src/hpymodule.c @@ -9,6 +9,7 @@ #include #endif #include +#include #include "api.h" @@ -150,6 +151,50 @@ static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode) goto error; } + char minor_version_symbol_name[258]; + char major_version_symbol_name[258]; + PyOS_snprintf(minor_version_symbol_name, sizeof(init_name), + "required_hpy_minor_version_%.200s", shortname); + PyOS_snprintf(major_version_symbol_name, sizeof(init_name), + "required_hpy_major_version_%.200s", shortname); + void *minor_version_ptr = dlsym(mylib, minor_version_symbol_name); + void *major_version_ptr = dlsym(mylib, major_version_symbol_name); + if (minor_version_ptr == NULL || major_version_ptr == NULL) { + const char *error = dlerror(); + if (error == NULL) + error = "no error message provided by the system"; + PyErr_Format(PyExc_RuntimeError, + "Error during loading of the HPy extension module at path " + "'%s'. Cannot locate the required minimal HPy versions as symbols '%s' and `%s`. " + "Error message from dlopen/WinAPI: %s", + soname, minor_version_symbol_name, major_version_symbol_name, error); + goto error; + } + uint32_t required_minor_version = *((uint32_t*) minor_version_ptr); + uint32_t required_major_version = *((uint32_t*) major_version_ptr); + if (required_major_version != HPY_ABI_VERSION || required_minor_version > HPY_ABI_VERSION_MINOR) { + // For now, we have only one major version, but in the future at this + // point we would decide which HPyContext to create + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' requires unsupported version of the HPy runtime. " + "Requested version: %" PRIu32 ".%" PRIu32 ". Current HPy version: %" PRIu32 ".%" PRIu32 ".", + shortname, required_major_version, required_minor_version, + HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR); + goto error; + } + + char expected_tag[16]; + PyOS_snprintf(expected_tag, sizeof(expected_tag), ".hpy%" PRIu32 ".", required_major_version); + // Sanity check of the tag in the shared object filename + if (strstr(soname, expected_tag) == NULL) { + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': mismatch between the " + "HPy ABI tag encoded in the filename and the major version requested " + "by the HPy extension itself. Requested version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, required_major_version, required_minor_version); + goto error; + } + void *initfn = dlsym(mylib, init_name); if (initfn == NULL) { const char *error = dlerror(); diff --git a/test/test_00_basic.py b/test/test_00_basic.py index 84d489985..0fbdc5544 100644 --- a/test/test_00_basic.py +++ b/test/test_00_basic.py @@ -7,6 +7,7 @@ "fake pytest module") """ from .support import HPyTest +from hpy.devel.abitag import HPY_ABI_VERSION class TestBasic(HPyTest): @@ -28,6 +29,23 @@ def test_empty_module(self): """) assert type(mod) is type(sys) + def test_abi_version_check(self): + if self.compiler.hpy_abi != 'universal': + return + try: + self.make_module(""" + // hack: we redefine the version + #undef HPY_ABI_VERSION + #define HPY_ABI_VERSION 999 + @INIT + """) + except RuntimeError as ex: + assert str(ex) == "HPy extension module 'mytest' requires unsupported " \ + "version of the HPy runtime. Requested version: 999.0. " \ + "Current HPy version: {}.0.".format(HPY_ABI_VERSION) + else: + assert False, "Expected exception" + def test_different_name(self): mod = self.make_module(""" @INIT From 5d10484725ae739b161f9432f62852a87a191c00 Mon Sep 17 00:00:00 2001 From: stepan Date: Fri, 2 Dec 2022 21:55:05 +0100 Subject: [PATCH 2/5] More thorough ABI tag check in loader, unit tests --- hpy/devel/abitag.py | 1 + hpy/universal/src/hpymodule.c | 37 ++++++++++++++++++++++++++------- test/test_00_basic.py | 39 +++++++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 10 deletions(-) diff --git a/hpy/devel/abitag.py b/hpy/devel/abitag.py index 32d1c0973..2d31668d0 100644 --- a/hpy/devel/abitag.py +++ b/hpy/devel/abitag.py @@ -3,6 +3,7 @@ # NOTE: these must be kept on sync with the equivalent defines in hpy.h HPY_ABI_VERSION = 0 +HPY_ABI_VERSION_MINOR = 0 HPY_ABI_TAG = 'hpy%d' % HPY_ABI_VERSION def parse_ext_suffix(ext_suffix=None): diff --git a/hpy/universal/src/hpymodule.c b/hpy/universal/src/hpymodule.c index 73728174d..d673c7ba7 100644 --- a/hpy/universal/src/hpymodule.c +++ b/hpy/universal/src/hpymodule.c @@ -121,6 +121,34 @@ get_encoded_name(PyObject *name) { return NULL; } +static bool validate_abi_tag(const char *shortname, const char *soname, + uint32_t required_major_version, uint32_t required_minor_version) { + char *substr = strstr(soname, ".hpy"); + if (substr != NULL) { + substr += strlen(".hpy"); + if (*substr >= '0' && *substr <= '9') { + // It is a number w/o sign and whitespace, we can now parse it with atoi + uint32_t abi_tag = (uint32_t) atoi(substr); + if (abi_tag == required_major_version) { + return true; + } + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': mismatch between the " + "HPy ABI tag encoded in the filename and the major version requested " + "by the HPy extension itself. Major version tag parsed from " + "filename: %" PRIu32 ". Requested version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, abi_tag, required_major_version, required_minor_version); + return false; + } + } + PyErr_Format(PyExc_RuntimeError, + "HPy extension module '%s' at path '%s': could not find " + "HPy ABI tag encoded in the filename. The extension claims to be compiled with " + "HPy ABI version: %" PRIu32 ".%" PRIu32 ".", + shortname, soname, required_major_version, required_minor_version); + return false; +} + static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode) { PyObject *name = NULL; @@ -183,15 +211,8 @@ static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode) goto error; } - char expected_tag[16]; - PyOS_snprintf(expected_tag, sizeof(expected_tag), ".hpy%" PRIu32 ".", required_major_version); // Sanity check of the tag in the shared object filename - if (strstr(soname, expected_tag) == NULL) { - PyErr_Format(PyExc_RuntimeError, - "HPy extension module '%s' at path '%s': mismatch between the " - "HPy ABI tag encoded in the filename and the major version requested " - "by the HPy extension itself. Requested version: %" PRIu32 ".%" PRIu32 ".", - shortname, soname, required_major_version, required_minor_version); + if (!validate_abi_tag(shortname, soname, required_major_version, required_minor_version)) { goto error; } diff --git a/test/test_00_basic.py b/test/test_00_basic.py index 0fbdc5544..1028d5da5 100644 --- a/test/test_00_basic.py +++ b/test/test_00_basic.py @@ -7,7 +7,9 @@ "fake pytest module") """ from .support import HPyTest -from hpy.devel.abitag import HPY_ABI_VERSION +from hpy.universal import MODE_UNIVERSAL +from hpy.devel.abitag import HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR +import shutil class TestBasic(HPyTest): @@ -42,10 +44,43 @@ def test_abi_version_check(self): except RuntimeError as ex: assert str(ex) == "HPy extension module 'mytest' requires unsupported " \ "version of the HPy runtime. Requested version: 999.0. " \ - "Current HPy version: {}.0.".format(HPY_ABI_VERSION) + "Current HPy version: {}.{}.".format(HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR) else: assert False, "Expected exception" + def test_abi_tag_check(self): + if self.compiler.hpy_abi != 'universal': + return + + def assert_load_raises(filename, message): + try: + self.compiler.load_universal_module('mytest', filename, mode=MODE_UNIVERSAL) + except RuntimeError as ex: + assert str(ex) == message + else: + assert False, "Expected exception" + + module = self.compile_module("@INIT") + filename = module.so_filename + hpy_tag = ".hpy{}".format(HPY_ABI_VERSION) + + filename_wrong_tag = filename.replace(hpy_tag, ".hpy999") + shutil.move(filename, filename_wrong_tag) + assert_load_raises(filename_wrong_tag, + "HPy extension module 'mytest' at path '{}': mismatch " + "between the HPy ABI tag encoded in the filename and " + "the major version requested by the HPy extension itself. " + "Major version tag parsed from filename: 999. " + "Requested version: {}.{}.".format(filename_wrong_tag, HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR)) + + filename_no_tag = filename.replace(hpy_tag, "") + shutil.move(filename_wrong_tag, filename_no_tag) + assert_load_raises(filename_no_tag, + "HPy extension module 'mytest' at path '{}': " + "could not find HPy ABI tag encoded in the filename. " + "The extension claims to be compiled with HPy ABI version: " + "{}.{}.".format(filename_no_tag, HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR)) + def test_different_name(self): mod = self.make_module(""" @INIT From 09bb06c607132806ade649ad2f4b77e6b0a240c8 Mon Sep 17 00:00:00 2001 From: stepan Date: Mon, 5 Dec 2022 21:00:53 +0100 Subject: [PATCH 3/5] Change the exported required HPy version symbols from constants to functions --- hpy/devel/include/hpy/hpymodule.h | 8 ++++++-- hpy/universal/src/hpymodule.c | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/hpy/devel/include/hpy/hpymodule.h b/hpy/devel/include/hpy/hpymodule.h index ba6079eda..cb88cd275 100644 --- a/hpy/devel/include/hpy/hpymodule.h +++ b/hpy/devel/include/hpy/hpymodule.h @@ -53,9 +53,13 @@ typedef struct { // module initialization in the universal and hybrid case #define HPy_MODINIT(modname) \ HPy_EXPORTED_SYMBOL uint32_t \ - required_hpy_major_version_##modname = HPY_ABI_VERSION; \ + get_required_hpy_major_version_##modname() { \ + return HPY_ABI_VERSION; \ + } \ HPy_EXPORTED_SYMBOL uint32_t \ - required_hpy_minor_version_##modname = HPY_ABI_VERSION_MINOR; \ + get_required_hpy_minor_version_##modname() { \ + return HPY_ABI_VERSION_MINOR; \ + } \ _HPy_CTX_MODIFIER HPyContext *_ctx_for_trampolines; \ static HPy init_##modname##_impl(HPyContext *ctx); \ HPyMODINIT_FUNC \ diff --git a/hpy/universal/src/hpymodule.c b/hpy/universal/src/hpymodule.c index d673c7ba7..f2798725a 100644 --- a/hpy/universal/src/hpymodule.c +++ b/hpy/universal/src/hpymodule.c @@ -149,6 +149,8 @@ static bool validate_abi_tag(const char *shortname, const char *soname, return false; } +typedef uint32_t (*version_getter)(void); + static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode) { PyObject *name = NULL; @@ -182,9 +184,9 @@ static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode) char minor_version_symbol_name[258]; char major_version_symbol_name[258]; PyOS_snprintf(minor_version_symbol_name, sizeof(init_name), - "required_hpy_minor_version_%.200s", shortname); + "get_required_hpy_minor_version_%.200s", shortname); PyOS_snprintf(major_version_symbol_name, sizeof(init_name), - "required_hpy_major_version_%.200s", shortname); + "get_required_hpy_major_version_%.200s", shortname); void *minor_version_ptr = dlsym(mylib, minor_version_symbol_name); void *major_version_ptr = dlsym(mylib, major_version_symbol_name); if (minor_version_ptr == NULL || major_version_ptr == NULL) { @@ -198,8 +200,8 @@ static PyObject *do_load(PyObject *name_unicode, PyObject *path, HPyMode mode) soname, minor_version_symbol_name, major_version_symbol_name, error); goto error; } - uint32_t required_minor_version = *((uint32_t*) minor_version_ptr); - uint32_t required_major_version = *((uint32_t*) major_version_ptr); + uint32_t required_minor_version = ((version_getter) minor_version_ptr)(); + uint32_t required_major_version = ((version_getter) major_version_ptr)(); if (required_major_version != HPY_ABI_VERSION || required_minor_version > HPY_ABI_VERSION_MINOR) { // For now, we have only one major version, but in the future at this // point we would decide which HPyContext to create From 76ec78eef54d46cfaf2aa3a9629a55800dd6fb58 Mon Sep 17 00:00:00 2001 From: stepan Date: Tue, 6 Dec 2022 11:46:38 +0100 Subject: [PATCH 4/5] Use extern "C" if C++ for the new exported symbols --- hpy/devel/include/hpy/hpymodule.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/hpy/devel/include/hpy/hpymodule.h b/hpy/devel/include/hpy/hpymodule.h index cb88cd275..27f601603 100644 --- a/hpy/devel/include/hpy/hpymodule.h +++ b/hpy/devel/include/hpy/hpymodule.h @@ -32,8 +32,10 @@ typedef struct { #if defined(__cplusplus) +# define HPyVERSION_FUNC extern "C" HPy_EXPORTED_SYMBOL uint32_t # define HPyMODINIT_FUNC extern "C" HPy_EXPORTED_SYMBOL HPy #else /* __cplusplus */ +# define HPyVERSION_FUNC HPy_EXPORTED_SYMBOL uint32_t # define HPyMODINIT_FUNC HPy_EXPORTED_SYMBOL HPy #endif /* __cplusplus */ @@ -52,12 +54,14 @@ typedef struct { // module initialization in the universal and hybrid case #define HPy_MODINIT(modname) \ - HPy_EXPORTED_SYMBOL uint32_t \ - get_required_hpy_major_version_##modname() { \ + HPyVERSION_FUNC \ + get_required_hpy_major_version_##modname() \ + { \ return HPY_ABI_VERSION; \ } \ - HPy_EXPORTED_SYMBOL uint32_t \ - get_required_hpy_minor_version_##modname() { \ + HPyVERSION_FUNC \ + get_required_hpy_minor_version_##modname() \ + { \ return HPY_ABI_VERSION_MINOR; \ } \ _HPy_CTX_MODIFIER HPyContext *_ctx_for_trampolines; \ From e70eff8660c6ae46c1f06f35b71ca6dd03e92898 Mon Sep 17 00:00:00 2001 From: stepan Date: Tue, 6 Dec 2022 13:21:21 +0100 Subject: [PATCH 5/5] Make test_00_basic Python 2.7 compatible again --- test/test_00_basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_00_basic.py b/test/test_00_basic.py index 1028d5da5..8fa390411 100644 --- a/test/test_00_basic.py +++ b/test/test_00_basic.py @@ -7,7 +7,6 @@ "fake pytest module") """ from .support import HPyTest -from hpy.universal import MODE_UNIVERSAL from hpy.devel.abitag import HPY_ABI_VERSION, HPY_ABI_VERSION_MINOR import shutil @@ -52,6 +51,7 @@ def test_abi_tag_check(self): if self.compiler.hpy_abi != 'universal': return + from hpy.universal import MODE_UNIVERSAL def assert_load_raises(filename, message): try: self.compiler.load_universal_module('mytest', filename, mode=MODE_UNIVERSAL)