Multiphase module initialization

Make the multiphase module initialization the only way to initialize HPy
modules, abandons the previous single phase initialization.

HPy_MODINIT takes name of the extesion, so that it can generate
PyInit_XXX for CPython, and HPyModuleDef. There is no user provided init
function anymore. The runtime initializes the module and if the user wants
to populate the module, or do any other initialization, they can define one
or more HPy_mod_exec slots that are called by the runtime right after
the module is created. The slots are defined in the same way as slots
for types: by using HPyDef_SLOT macro.

As a consequence, `HPy_MODINIT(myextension, hpydef)` is all that is necessary
if the module only exposes module functions.

Users can also define `HPy_mod_create` slot, similar to CPython's
`Py_mod_create`, but in HPy this slot can return only objects that are not
of the builtin module type.

The reasoning is that builtin module objects can be created by the runtime
from `HPyModuleDef` and, for the time being, there is no use-case for
manually creating builtin module objects in the `HPy_mod_create` slot or
anywhere else. If such a use-case emerges, support for it can be easily
added to HPy.

If user specifies the `HPy_mod_create slot`, specifying anything else
(setting it to a non-default value) in `HPyModuleDef` is an error.

Removes `HPyModule_Create` API/ABI function. The CPython equivalent
`PyModule_Create` works only for single phase module initialization.
We may add equivalents of `PyModule_FromDefAndSpec` to allow manual
module creation if there is a use-case.

Additionally, removes the CPython semantics of negative `m_size`. HPy
will add other more explicit means to communicate subinterpreters
support.

docs/examples/mixed-example/mixed.c

@@ -39,20 +39,11 @@ static PyMethodDef py_defines[] = {
 };
 
 static HPyModuleDef moduledef = {
-    .name = "mixed",
     .doc = "HPy Example of mixing CPython API and HPy API",
-    .size = -1,
+    .size = 0,
     .defines = hpy_defines,
     .legacy_methods = py_defines
 };
 
 
-HPy_MODINIT(mixed)
-static HPy init_mixed_impl(HPyContext *ctx)
-{
-    HPy m;
-    m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-    return m;
-}
+HPy_MODINIT(mixed, moduledef)

docs/examples/simple-example/simple.c

@@ -31,17 +31,13 @@ static HPyDef *SimpleMethods[] = {
 };
 
 static HPyModuleDef simple = {
-        .name = "simple",
         .doc = "HPy Example",
-        .size = -1,
+        .size = 0,
         .defines = SimpleMethods,
         .legacy_methods = NULL
 };
 // END: methodsdef
 
 // BEGIN: moduledef
-HPy_MODINIT(simple)
-HPy init_simple_impl(HPyContext *ctx) {
-    return HPyModule_Create(ctx, &simple);
-}
+HPy_MODINIT(simple, simple)
 // END: moduledef

docs/examples/snippets/hpyvarargs.c

@@ -36,14 +36,10 @@ static HPyDef *SimpleMethods[] = {
 };
 // END: methodsdef
 
-static HPyModuleDef simple = {
-        .name = "hpyvarargs",
+static HPyModuleDef def = {
         .doc = "HPy Example of varargs calling convention",
-        .size = -1,
+        .size = 0,
         .defines = SimpleMethods
 };
 
-HPy_MODINIT(hpyvarargs)
-HPy init_hpyvarargs_impl(HPyContext *ctx) {
-    return HPyModule_Create(ctx, &simple);
-}
+HPy_MODINIT(hpyvarargs, def)

docs/examples/snippets/snippets.c

@@ -73,13 +73,9 @@ static HPyDef *Methods[] = {
 };
 
 static HPyModuleDef snippets = {
-        .name = "snippets",
         .doc = "Various HPy code snippets for the docs",
-        .size = -1,
+        .size = 0,
         .defines = Methods
 };
 
-HPy_MODINIT(snippets)
-HPy init_snippets_impl(HPyContext *ctx) {
-    return HPyModule_Create(ctx, &snippets);
-}
+HPy_MODINIT(snippets, snippets)

docs/porting-example/steps/step_01_hpy_legacy.c

@@ -149,30 +149,40 @@ static PyMethodDef PointModuleLegacyMethods[] = {
     {NULL, NULL, 0, NULL}
 };
 
-// HPy module methods (no methods have been ported yet)
+// HPy supports only multiphase module initialization, so we must migrate the
+// single phase initialization by extracting the code that populates the module
+// object with attributes into a separate 'exec' slot. The module is not
+// created manually by calling API like PyModule_Create, but the runtime creates
+// the module for us from the specification in HPyModuleDef, and we can provide
+// additional slots to populate the module before its initialization is finalized
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *ctx, HPy mod)
+{
+    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
+    if (HPy_IsNull(point_type))
+        return -1;
+    HPy_SetAttr_s(ctx, mod, "Point", point_type);
+    return 0;
+}
+
+// HPy module methods: no regular methods have been ported yet,
+// but we add the module execute slot
 static HPyDef *module_defines[] = {
+    &module_exec,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "step_01_hpy_legacy",
+    // .name = "step_01_hpy_legacy",
+    // ^-- .name is not needed for multiphase module initialization,
+    // it is always taken from the ModuleSpec
     .doc = "Point module (Step 1; All legacy methods)",
-    .size = -1,
+    .size = 0,
     .legacy_methods = PointModuleLegacyMethods,
     .defines = module_defines,
 };
 
-HPy_MODINIT(step_01_hpy_legacy)
-static HPy init_step_01_hpy_legacy_impl(HPyContext *ctx)
-{
-    HPy m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
-    if (HPy_IsNull(point_type))
-      return HPy_NULL;
-    HPy_SetAttr_s(ctx, m, "Point", point_type);
-
-    return m;
-}
+// HPy_MODINIT takes the extension name, i.e., what would be XXX in PyInit_XXX,
+// and the module definition. The module will be created by the runtime and
+// passed to the HPy_mod_exec slots if any are defined
+HPy_MODINIT(step_01_hpy_legacy, moduledef)

docs/porting-example/steps/step_02_hpy_legacy.c

@@ -140,30 +140,28 @@ static PyMethodDef PointModuleLegacyMethods[] = {
     {NULL, NULL, 0, NULL}
 };
 
-// HPy module methods (no methods have been ported yet)
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *ctx, HPy mod)
+{
+    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
+    if (HPy_IsNull(point_type))
+        return -1;
+    HPy_SetAttr_s(ctx, mod, "Point", point_type);
+    return 0;
+}
+
+// HPy module methods: no regular methods have been ported yet,
+// but we add the module execute slot
 static HPyDef *module_defines[] = {
+    &module_exec,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "step_02_hpy_legacy",
     .doc = "Point module (Step 2; Porting some methods)",
-    .size = -1,
+    .size = 0,
     .legacy_methods = PointModuleLegacyMethods,
     .defines = module_defines,
 };
 
-HPy_MODINIT(step_02_hpy_legacy)
-static HPy init_step_02_hpy_legacy_impl(HPyContext *ctx)
-{
-    HPy m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
-    if (HPy_IsNull(point_type))
-      return HPy_NULL;
-    HPy_SetAttr_s(ctx, m, "Point", point_type);
-
-    return m;
-}
+HPy_MODINIT(step_02_hpy_legacy, moduledef)

docs/porting-example/steps/step_03_hpy_final.c

@@ -125,30 +125,27 @@ static HPyType_Spec Point_Type_spec = {
     .defines = point_defines
 };
 
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *ctx, HPy mod)
+{
+    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
+    if (HPy_IsNull(point_type))
+        return -1;
+    HPy_SetAttr_s(ctx, mod, "Point", point_type);
+    return 0;
+}
+
 // HPy module methods
 static HPyDef *module_defines[] = {
+        &module_exec,
     &dot,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "step_03_hpy_final",
     .doc = "Point module (Step 3; Porting complete)",
-    .size = -1,
+    .size = 0,
     .defines = module_defines,
 };
 
-HPy_MODINIT(step_03_hpy_final)
-static HPy init_step_03_hpy_final_impl(HPyContext *ctx)
-{
-    HPy m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
-    if (HPy_IsNull(point_type))
-      return HPy_NULL;
-    HPy_SetAttr_s(ctx, m, "Point", point_type);
-
-    return m;
-}
+HPy_MODINIT(step_03_hpy_final, moduledef)

gdb-py.test

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-gdb --eval-command run --args python3 -m pytest -s "$@"
+gdb -ex "set breakpoint pending on" -ex "b create_impl" --eval-command run --args python3 -m pytest -s "$@"

hpy/debug/src/_debugmod.c

@@ -377,6 +377,17 @@ static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType,
 
 /* ~~~~~~ definition of the module hpy.debug._debug ~~~~~~~ */
 
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *uctx, HPy m)
+{
+    UHPy h_DebugHandleType = HPyType_FromSpec(uctx, &DebugHandleType_spec, NULL);
+    if (HPy_IsNull(h_DebugHandleType))
+        return -1;
+    HPy_SetAttr_s(uctx, m, "DebugHandle", h_DebugHandleType);
+    HPy_Close(uctx, h_DebugHandleType);
+    return 0;
+}
+
 static HPyDef *module_defines[] = {
     &new_generation,
     &get_open_handles,
@@ -387,28 +398,14 @@ static HPyDef *module_defines[] = {
     &set_protected_raw_data_max_size,
     &set_on_invalid_handle,
     &set_handle_stack_trace_limit,
+    &module_exec,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "hpy.debug._debug",
     .doc = "HPy debug mode",
-    .size = -1,
+    .size = 0,
     .defines = module_defines
 };
 
-
-HPy_MODINIT(_debug)
-static UHPy init__debug_impl(HPyContext *uctx)
-{
-    UHPy m = HPyModule_Create(uctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    UHPy h_DebugHandleType = HPyType_FromSpec(uctx, &DebugHandleType_spec, NULL);
-    if (HPy_IsNull(h_DebugHandleType))
-        return HPy_NULL;
-    HPy_SetAttr_s(uctx, m, "DebugHandle", h_DebugHandleType);
-    HPy_Close(uctx, h_DebugHandleType);
-    return m;
-}
+HPy_MODINIT(_debug, moduledef)

hpy/debug/src/debug_ctx_cpython.c

@@ -24,7 +24,9 @@
 #include <Python.h>
 #include "debug_internal.h"
 #include "hpy/runtime/ctx_type.h" // for call_traverseproc_from_trampoline
+#include "hpy/runtime/ctx_module.h"
 #include "handles.h" // for _py2h and _h2py
+
 #if defined(_MSC_VER)
 # include <malloc.h>   /* for alloca() */
 #endif
@@ -193,6 +195,17 @@ void debug_ctx_CallRealFunctionFromTrampoline(HPyContext *dctx,
                 PyCapsule_GetContext(capsule));
         return;
     }
+    case HPyFunc_MOD_CREATE: {
+        HPyFunc_unaryfunc f = (HPyFunc_unaryfunc)func;
+        _HPyFunc_args_UNARYFUNC *a = (_HPyFunc_args_UNARYFUNC*)args;
+        DHPy dh_arg0 = _py2dh(dctx, a->arg0);
+        DHPy dh_result = f(dctx, dh_arg0);
+        DHPy_close_and_check(dctx, dh_arg0);
+        a->result = _dh2py(dctx, dh_result);
+        _HPyModule_CheckCreateSlotResult(&a->result);
+        DHPy_close(dctx, dh_result);
+        return;
+    }
 #include "autogen_debug_ctx_call.i"
     default:
         Py_FatalError("Unsupported HPyFunc_Signature in debug_ctx_cpython.c");

hpy/debug/src/include/hpy_debug.h

@@ -46,6 +46,12 @@ void hpy_debug_close_handle(HPyContext *dctx, HPy dh);
 extern "C"
 #endif
 HPy_EXPORTED_SYMBOL
-HPy HPyInit__debug(HPyContext *uctx);
+HPyModuleDef* HPyInit__debug();
+
+#ifdef ___cplusplus
+extern "C"
+#endif
+HPy_EXPORTED_SYMBOL
+void HPyInitGlobalContext__debug(HPyContext *ctx);
 
 #endif /* HPY_DEBUG_H */

hpy/devel/include/hpy/cpy_types.h

@@ -19,6 +19,7 @@
 
 typedef struct FORBIDDEN_cpy_PyObject cpy_PyObject;
 typedef struct FORBIDDEN_PyMethodDef cpy_PyMethodDef;
+typedef struct FORBIDDEN_PyModuleDef cpy_PyModuleDef;
 typedef struct FORBIDDEN_bufferinfo cpy_Py_buffer;
 
 // declare the following API functions as _HPY_LEGACY, which triggers an
@@ -31,6 +32,7 @@ HPyAPI_FUNC _HPY_LEGACY HPy HPy_FromPyObject(HPyContext *ctx, cpy_PyObject *obj)
 // Python.h has already been included by the main hpy.h
 typedef PyObject cpy_PyObject;
 typedef PyMethodDef cpy_PyMethodDef;
+typedef PyModuleDef cpy_PyModuleDef;
 typedef Py_buffer cpy_Py_buffer;
 
 #endif /* HPY_ABI_UNIVERSAL */