Multiphase module initialization

Make the multiphase module initialization the only way to initialize HPy
modules, abandons the previous single phase initialization.

HPy_MODINIT takes name of the extesion, so that it can generate
PyInit_XXX for CPython, and HPyModuleDef. There is no user provided init
function anymore. The runtime initializes the module and if the user wants
to populate the module, or do any other initialization, they can define one
or more HPy_mod_exec slots that are called by the runtime right after
the module is created. The slots are defined in the same way as slots
for types: by using HPyDef_SLOT macro.

As a consequence, `HPy_MODINIT(myextension, hpydef)` is all that is necessary
if the module only exposes module functions.

Users can also define `HPy_mod_create` slot, similar to CPython's
`Py_mod_create`, but in HPy this slot can return only objects that are not
of the builtin module type.

The reasoning is that builtin module objects can be created by the runtime
from `HPyModuleDef` and, for the time being, there is no use-case for
manually creating builtin module objects in the `HPy_mod_create` slot or
anywhere else. If such a use-case emerges, support for it can be easily
added to HPy.

If user specifies the `HPy_mod_create slot`, specifying anything else
(setting it to a non-default value) in `HPyModuleDef` is an error.

Removes `HPyModule_Create` API/ABI function. The CPython equivalent
`PyModule_Create` works only for single phase module initialization.
We may add equivalents of `PyModule_FromDefAndSpec` to allow manual
module creation if there is a use-case.

Additionally, removes the CPython semantics of negative `m_size`. HPy
will add other more explicit means to communicate subinterpreters
support.

docs/api-reference/function-index.rst

@@ -9,7 +9,6 @@
 HPy Core API Function Index
 ###########################
 
-* :c:func:`HPyModule_Create`
 * :c:func:`HPy_Dup`
 * :c:func:`HPy_Close`
 * :c:func:`HPyLong_FromInt32_t`

docs/examples/mixed-example/mixed.c

@@ -39,20 +39,11 @@ static PyMethodDef py_defines[] = {
 };
 
 static HPyModuleDef moduledef = {
-    .name = "mixed",
     .doc = "HPy Example of mixing CPython API and HPy API",
-    .size = -1,
+    .size = 0,
     .defines = hpy_defines,
     .legacy_methods = py_defines
 };
 
 
-HPy_MODINIT(mixed)
-static HPy init_mixed_impl(HPyContext *ctx)
-{
-    HPy m;
-    m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-    return m;
-}
+HPy_MODINIT(mixed, moduledef)

docs/examples/simple-example/simple.c

@@ -31,17 +31,13 @@ static HPyDef *SimpleMethods[] = {
 };
 
 static HPyModuleDef simple = {
-        .name = "simple",
         .doc = "HPy Example",
-        .size = -1,
+        .size = 0,
         .defines = SimpleMethods,
         .legacy_methods = NULL
 };
 // END: methodsdef
 
 // BEGIN: moduledef
-HPy_MODINIT(simple)
-HPy init_simple_impl(HPyContext *ctx) {
-    return HPyModule_Create(ctx, &simple);
-}
+HPy_MODINIT(simple, simple)
 // END: moduledef

docs/examples/snippets/hpyvarargs.c

@@ -36,14 +36,10 @@ static HPyDef *SimpleMethods[] = {
 };
 // END: methodsdef
 
-static HPyModuleDef simple = {
-        .name = "hpyvarargs",
+static HPyModuleDef def = {
         .doc = "HPy Example of varargs calling convention",
-        .size = -1,
+        .size = 0,
         .defines = SimpleMethods
 };
 
-HPy_MODINIT(hpyvarargs)
-HPy init_hpyvarargs_impl(HPyContext *ctx) {
-    return HPyModule_Create(ctx, &simple);
-}
+HPy_MODINIT(hpyvarargs, def)

docs/examples/snippets/snippets.c

@@ -73,13 +73,9 @@ static HPyDef *Methods[] = {
 };
 
 static HPyModuleDef snippets = {
-        .name = "snippets",
         .doc = "Various HPy code snippets for the docs",
-        .size = -1,
+        .size = 0,
         .defines = Methods
 };
 
-HPy_MODINIT(snippets)
-HPy init_snippets_impl(HPyContext *ctx) {
-    return HPyModule_Create(ctx, &snippets);
-}
+HPy_MODINIT(snippets, snippets)

docs/misc/embedding.rst

@@ -43,8 +43,4 @@ Also refer to the API reference :ref:`api-reference/hpy-type:hpy module`.
         // ...
     }
 
-    HPy_MODINIT(hpymodA)
-    static HPy init_hpymodA_impl(HPyContext *ctx)
-    {
-        // ...
-    }
+    HPy_MODINIT(extension_name, hpymodA)

docs/porting-example/index.rst

@@ -107,31 +107,42 @@ Similarly we replace ``PyModuleDef`` with ``HPyModuleDef``:
 .. literalinclude:: steps/step_01_hpy_legacy.c
     :lineno-match:
     :start-at: // Legacy module methods (the "dot" method is still a PyCFunction)
-    :end-before: HPy_MODINIT(step_01_hpy_legacy)
+    :end-before: // END-OF: HPyModuleDef
 
 Like the type, the list of ported methods in ``module_defines`` is initially
-empty and all the methods are still in ``PointModuleMethods`` which has
-been renamed to ``PointModuleLegacyMethods``.
+almost empty: all the regular methods are still in ``PointModuleMethods`` which has
+been renamed to ``PointModuleLegacyMethods``. However, because HPy supports only
+multiphase module initialization, we must convert our module initialization code
+to an "exec" slot on the module and add that slot to ``module_defines``.
 
 Now all that is left is to replace the module initialization function with
-one that uses ``HPy_MODINIT``:
+one that uses ``HPy_MODINIT``. The first argument is the name of the extension,
+i.e., what was ``XXX`` in ``PyInit_XXX``, and the second argument
+is the ``HPyModuleDef``.
 
 .. literalinclude:: steps/step_01_hpy_legacy.c
     :lineno-match:
-    :start-at: HPy_MODINIT(step_01_hpy_legacy)
+    :start-at: HPy_MODINIT(step_01_hpy_legacy, moduledef)
 
 And we're done!
 
-Note that the initialization function now takes an ``HPyContext *`` as an
-argument and that this ``ctx`` is passed as the first argument to calls to
-HPy API methods.
+Instead of the ``PyInit_XXX``, we now have an "exec" slot on the module.
+We implement it with a C function that that takes an ``HPyContext *ctx`` and ``HPy mod``
+as arguments. The ``ctx`` must be forwarded as the first argument to calls to
+HPy API methods. The ``mod`` argument is a handle for the module object. The runtime
+creates the module for us from the provided ``HPyModuleDef``. There is no need to
+call API like ``PyModule_Create`` explicitly.
 
-``PyModule_Create`` is replaced with ``HPyModule_Create`` and ``PyType_FromSpec``
-is replaced by ``HPyType_FromSpec``.
+Next step is to replace ``PyType_FromSpec`` by ``HPyType_FromSpec``.
 
 ``HPy_SetAttr_s`` is used to add the ``Point`` class to the module. HPy requires no
 special ``PyModule_AddObject`` method.
 
+.. literalinclude:: steps/step_01_hpy_legacy.c
+    :lineno-match:
+    :start-at: HPyDef_SLOT(module_exec, HPy_mod_exec)
+    :end-at: }
+
 
 Step 02: Transition some methods to HPy
 ---------------------------------------
@@ -338,7 +349,7 @@ and the module definition is simpler too:
 .. literalinclude:: steps/step_03_hpy_final.c
     :lineno-match:
     :start-at: static HPyDef *module_defines[] = {
-    :end-before: HPy_MODINIT(step_03_hpy_final)
+    :end-before: HPy_MODINIT(step_03_hpy_final, moduledef)
 
 Now that the port is complete, when we compile our extension in HPy
 universal mode, we obtain a built extension that depends only on the HPy ABI

docs/porting-example/steps/step_01_hpy_legacy.c

@@ -143,36 +143,47 @@ static HPyType_Spec Point_Type_spec = {
     .defines = point_defines,
 };
 
+// HPy supports only multiphase module initialization, so we must migrate the
+// single phase initialization by extracting the code that populates the module
+// object with attributes into a separate 'exec' slot. The module is not
+// created manually by calling API like PyModule_Create, but the runtime creates
+// the module for us from the specification in HPyModuleDef, and we can provide
+// additional slots to populate the module before its initialization is finalized
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *ctx, HPy mod)
+{
+    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
+    if (HPy_IsNull(point_type))
+        return -1;
+    HPy_SetAttr_s(ctx, mod, "Point", point_type);
+    return 0;
+}
+
 // Legacy module methods (the "dot" method is still a PyCFunction)
 static PyMethodDef PointModuleLegacyMethods[] = {
     {"dot", (PyCFunction)dot, METH_VARARGS, "Dot product."},
     {NULL, NULL, 0, NULL}
 };
 
-// HPy module methods (no methods have been ported yet)
+// HPy module methods: no regular methods have been ported yet,
+// but we add the module execute slot
 static HPyDef *module_defines[] = {
+    &module_exec,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "step_01_hpy_legacy",
+    // .name = "step_01_hpy_legacy",
+    // ^-- .name is not needed for multiphase module initialization,
+    // it is always taken from the ModuleSpec
     .doc = "Point module (Step 1; All legacy methods)",
-    .size = -1,
+    .size = 0,
     .legacy_methods = PointModuleLegacyMethods,
     .defines = module_defines,
 };
+// END-OF: HPyModuleDef
 
-HPy_MODINIT(step_01_hpy_legacy)
-static HPy init_step_01_hpy_legacy_impl(HPyContext *ctx)
-{
-    HPy m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
-    if (HPy_IsNull(point_type))
-      return HPy_NULL;
-    HPy_SetAttr_s(ctx, m, "Point", point_type);
-
-    return m;
-}
+// HPy_MODINIT takes the extension name, i.e., what would be XXX in PyInit_XXX,
+// and the module definition. The module will be created by the runtime and
+// passed to the HPy_mod_exec slots if any are defined
+HPy_MODINIT(step_01_hpy_legacy, moduledef)

docs/porting-example/steps/step_02_hpy_legacy.c

@@ -140,30 +140,28 @@ static PyMethodDef PointModuleLegacyMethods[] = {
     {NULL, NULL, 0, NULL}
 };
 
-// HPy module methods (no methods have been ported yet)
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *ctx, HPy mod)
+{
+    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
+    if (HPy_IsNull(point_type))
+        return -1;
+    HPy_SetAttr_s(ctx, mod, "Point", point_type);
+    return 0;
+}
+
+// HPy module methods: no regular methods have been ported yet,
+// but we add the module execute slot
 static HPyDef *module_defines[] = {
+    &module_exec,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "step_02_hpy_legacy",
     .doc = "Point module (Step 2; Porting some methods)",
-    .size = -1,
+    .size = 0,
     .legacy_methods = PointModuleLegacyMethods,
     .defines = module_defines,
 };
 
-HPy_MODINIT(step_02_hpy_legacy)
-static HPy init_step_02_hpy_legacy_impl(HPyContext *ctx)
-{
-    HPy m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
-    if (HPy_IsNull(point_type))
-      return HPy_NULL;
-    HPy_SetAttr_s(ctx, m, "Point", point_type);
-
-    return m;
-}
+HPy_MODINIT(step_02_hpy_legacy, moduledef)

docs/porting-example/steps/step_03_hpy_final.c

@@ -125,30 +125,27 @@ static HPyType_Spec Point_Type_spec = {
     .defines = point_defines
 };
 
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *ctx, HPy mod)
+{
+    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
+    if (HPy_IsNull(point_type))
+        return -1;
+    HPy_SetAttr_s(ctx, mod, "Point", point_type);
+    return 0;
+}
+
 // HPy module methods
 static HPyDef *module_defines[] = {
+        &module_exec,
     &dot,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "step_03_hpy_final",
     .doc = "Point module (Step 3; Porting complete)",
-    .size = -1,
+    .size = 0,
     .defines = module_defines,
 };
 
-HPy_MODINIT(step_03_hpy_final)
-static HPy init_step_03_hpy_final_impl(HPyContext *ctx)
-{
-    HPy m = HPyModule_Create(ctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    HPy point_type = HPyType_FromSpec(ctx, &Point_Type_spec, NULL);
-    if (HPy_IsNull(point_type))
-      return HPy_NULL;
-    HPy_SetAttr_s(ctx, m, "Point", point_type);
-
-    return m;
-}
+HPy_MODINIT(step_03_hpy_final, moduledef)

hpy/debug/src/_debugmod.c

@@ -397,6 +397,17 @@ static UHPy new_DebugHandleObj(HPyContext *uctx, UHPy u_DebugHandleType,
 
 /* ~~~~~~ definition of the module hpy.debug._debug ~~~~~~~ */
 
+HPyDef_SLOT(module_exec, HPy_mod_exec)
+static int module_exec_impl(HPyContext *uctx, HPy m)
+{
+    UHPy h_DebugHandleType = HPyType_FromSpec(uctx, &DebugHandleType_spec, NULL);
+    if (HPy_IsNull(h_DebugHandleType))
+        return -1;
+    HPy_SetAttr_s(uctx, m, "DebugHandle", h_DebugHandleType);
+    HPy_Close(uctx, h_DebugHandleType);
+    return 0;
+}
+
 static HPyDef *module_defines[] = {
     &new_generation,
     &get_open_handles,
@@ -408,28 +419,14 @@ static HPyDef *module_defines[] = {
     &set_on_invalid_handle,
     &set_on_invalid_builder_handle,
     &set_handle_stack_trace_limit,
+    &module_exec,
     NULL
 };
 
 static HPyModuleDef moduledef = {
-    .name = "hpy.debug._debug",
     .doc = "HPy debug mode",
-    .size = -1,
+    .size = 0,
     .defines = module_defines
 };
 
-
-HPy_MODINIT(_debug)
-static UHPy init__debug_impl(HPyContext *uctx)
-{
-    UHPy m = HPyModule_Create(uctx, &moduledef);
-    if (HPy_IsNull(m))
-        return HPy_NULL;
-
-    UHPy h_DebugHandleType = HPyType_FromSpec(uctx, &DebugHandleType_spec, NULL);
-    if (HPy_IsNull(h_DebugHandleType))
-        return HPy_NULL;
-    HPy_SetAttr_s(uctx, m, "DebugHandle", h_DebugHandleType);
-    HPy_Close(uctx, h_DebugHandleType);
-    return m;
-}
+HPy_MODINIT(_debug, moduledef)