Immutable strings

docs/guide/_5_memory.md

@@ -0,0 +1,32 @@
+# Memory Management
+
+Pydust, like Zig, doesn't perform any implicit memory management. Pydust is designed to be a relatively
+thin layer around the CPython API, and therefore the same semantics apply.
+
+All Pydust Python types (such as `py.PyObject` and `py.Py<Name>`) have `incref()` and `decref()` member
+functions. These correspond to `ffi.Py_INCREF` and `ffi.Py_DECREF` respectively.
+
+For example, if we take a Zig string `right` and wish to append it to a Python string, we first need
+to convert it to a `py.PyString`. We will no longer need this new string at the end of the function,
+so we should defer a call to `decref()`.
+
+``` zig
+--8<-- "example/memory.zig:append"
+```
+
+The left-hand-side does not need be decref'd because `PyString.append` _steals_ a reference to itself.
+This is rare in the CPython API, but exists to create more performant and ergonomic code around
+string building. For example, chained appends don't need to decref all the intermediate strings.
+
+``` zig 
+const s = py.PyString.fromSlice("Hello ");
+s = s.appendSlice("1, ");
+s = s.appendSlice("2, ");
+s = s.appendSlice("3");
+return s;
+```
+
+!!! tip "Upcoming Feature!"
+
+    Work is underway to provide a test harness that uses Zig's `GeneralPurposeAllocator` to 
+    catch memory leaks within your Pydust extension code and surface them to pytest.

docs/guide/index.md

@@ -53,23 +53,3 @@ For native Zig types however, the following conversions apply:
 | `u32`, `u64`  | `int`        |
 | `f32`, `f64`  | `float`      |
 | `struct`      | `dict`       |
-
-## Memory Management
-
-Pydust, like Zig, doesn't perform any implicit memory management. Pydust is designed to be a relatively
-thin layer around the CPython API, and therefore the same semantics apply.
-
-All Pydust Python types (such as `py.PyObject` and `py.Py<Name>`) have `incref()` and `decref()` member
-functions. These correspond to `ffi.Py_INCREF` and `ffi.Py_DECREF` respectively.
-
-For example, if we take a Zig string `right` and wish to append it to a Python string, we first need
-to convert it to a `py.PyString`.
-
-``` zig
---8<-- "example/memory.zig:append"
-```
-
-!!! tip "Upcoming Feature!"
-
-    Work is underway to provide a test harness that uses Zig's `GeneralPurposeAllocator` to 
-    catch memory leaks within your Pydust extension code and surface them to pytest.

example/memory.zig

@@ -1,18 +1,22 @@
 const py = @import("pydust");
 
 // --8<-- [start:append]
-fn append(left: py.PyString, right: []const u8) !void {
-    // Since we create the PyString, and no longer need it after
-    // this function, we are responsible for calling decref on it.
+fn append(left: *py.PyString, right: []const u8) !py.PyString {
     const rightPy = try py.PyString.fromSlice(right);
     defer rightPy.decref();
 
-    _ = try left.append(rightPy);
+    return left.append(rightPy);
 }
 // --8<-- [end:append]
 
-pub fn appendFoo(args: *const struct { left: py.PyString }) !void {
-    try append(args.left, "foo");
+// --8<-- [start:append2]
+fn append2(left: *py.PyString, right: []const u8) !py.PyString {
+    return left.appendSlice(right);
+}
+// --8<-- [end:append2]
+
+pub fn appendFoo(args: *const struct { left: py.PyString }) !py.PyString {
+    return append(@constCast(&args.left), "foo");
 }
 
 comptime {

example/modules.zig

@@ -35,9 +35,9 @@ pub fn hello(
     args: *const struct { name: py.PyString }, // (5)!
 ) !py.PyString {
     var str = try py.PyString.fromSlice("Hello, ");
-    try str.append(args.name);
-    try str.appendSlice(". It's ");
-    try str.append(self.name);
+    str = try str.append(args.name);
+    str = try str.appendSlice(". It's ");
+    str = try str.append(self.name);
     return str;
 }
 

mkdocs.yml

@@ -18,6 +18,7 @@ nav:
   - 'User Guide':
     - "guide/index.md"
     - 'Python Modules': "guide/_1_modules.md"
+    - 'Memory Management': "guide/_5_memory.md"
 
 extra:
   social:

pydust/src/types/str.zig

@@ -21,29 +21,37 @@ pub const PyString = extern struct {
         return .{ .obj = .{ .py = unicode } };
     }
 
-    pub fn append(self: *PyString, other: PyString) !void {
-        try self.appendObj(other.obj);
+    /// Append other to self. A reference to self is stolen so there's no need to decref.
+    pub fn append(self: PyString, other: PyString) !PyString {
+        return self.appendObj(other.obj);
     }
 
-    pub fn appendSlice(self: *PyString, str: [:0]const u8) !void {
+    /// Append the slice to self. A reference to self is stolen so there's no need to decref.
+    pub fn appendSlice(self: PyString, str: [:0]const u8) !PyString {
         const other = try fromSlice(str);
         defer other.decref();
-        try self.appendObj(other.obj);
+        return self.appendObj(other.obj);
     }
 
-    fn appendObj(self: *PyString, other: PyObject) !void {
+    fn appendObj(self: PyString, other: PyObject) !PyString {
         // This function effectively decref's the left-hand side.
         // The semantics therefore sort of imply mutation, and so we expose the same in our API.
+        // FIXME(ngates): this comment
         var self_ptr: ?*ffi.PyObject = self.obj.py;
         ffi.PyUnicode_Append(&self_ptr, other.py);
         if (self_ptr) |ptr| {
-            self.obj.py = ptr;
+            return of(.{ .py = ptr });
         } else {
             // If set to null, then it failed.
             return PyError.Propagate;
         }
     }
 
+    /// Return the length of the Unicode object, in code points.
+    pub fn length(self: PyString) !usize {
+        return @intCast(ffi.PyUnicode_GetLength(self.obj.py));
+    }
+
     pub fn asOwnedSlice(self: PyString) ![:0]const u8 {
         defer self.decref();
         return try self.asSlice();
@@ -74,10 +82,9 @@ test "PyString" {
     const b = ", world!";
 
     var ps = try PyString.fromSlice(a);
+    ps = try ps.appendSlice(b);
     defer ps.decref();
 
-    try ps.appendSlice(b);
-
     var ps_slice = try ps.asSlice();
 
     // Null-terminated strings have len == non-null bytes, but are guaranteed to have a null byte

pyproject.toml

@@ -9,6 +9,9 @@ packages = [{ include = "pydust" }]
 include = ["src"]
 exclude = ["example"]
 
+[tool.poetry.plugins."pytest11"]
+pydust = "pydust.pytest_plugin"
+
 [tool.poetry.dependencies]
 python = "^3.11"
 ziglang = "^0.11.0"
@@ -38,6 +41,10 @@ target-version = "py310"
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.pytest.ini_options]
+# Disable our own pytest plugin to avoid conflicting test runs
+addopts = "-p no:pydust"
+testpaths = ["test"]
 
 # Out test modules
 
@@ -48,9 +55,9 @@ root = "example/"
 name = "example.hello"
 root = "example/hello.zig"
 
-# [[tool.pydust.ext_module]]
-# name = "example.memory"
-# root = "example/memory.zig"
+[[tool.pydust.ext_module]]
+name = "example.memory"
+root = "example/memory.zig"
 
 [[tool.pydust.ext_module]]
 name = "example.modules"

test/test_memory.py

@@ -0,0 +1,5 @@
+from example import memory
+
+
+def test_memory_append():
+    assert memory.appendFoo("hello ") == "hello foo"