Support Buffer Protocol #8

delta003 · 2023-09-01T09:47:13Z

https://docs.python.org/3/c-api/buffer.html

and a few other useful link https://peps.python.org/pep-3118/ & https://peps.python.org/pep-0688 & https://jakevdp.github.io/blog/2014/05/05/introduction-to-the-python-buffer-protocol/

…ust into mb/buffers

…buffers

gatesn · 2023-09-02T06:59:34Z

pydust/src/types/buffer.zig

+        return getWithFlag(obj, ffi.PyBUF_FULL_RO);
+    }
+
+    pub fn getWithFlag(obj: py.PyObject, flag: c_int) !Self {


Non-pub I think

gatesn · 2023-09-02T07:02:07Z

pydust/src/types/buffer.zig

+        var out: Self = undefined;
+        if (ffi.PyObject_GetBuffer(obj.py, @ptrCast(&out), flag) != 0) {
+            // TODO(marko): This should be an error once we figure out how to do it
+            @panic("unable to get buffer");


I think just return PyError.Propagate since the protocol says the exporter must have raised a BufferError

gatesn · 2023-09-05T07:51:54Z

pydust/src/pytypes.zig

+                }};
+            }
+
+            if (@hasDecl(definition, "__release_buffer__")) {


Maybe we should fold this inside the it block above and compileError if you have one and not the other?

This is an optional

The __release_buffer__ method should be called when a caller no longer needs the buffer returned by __buffer__. It corresponds to the bf_releasebuffer C slot. This is an optional part of the buffer protocol.

gatesn · 2023-09-05T07:52:12Z

pydust/src/pytypes.zig

@@ -119,6 +133,18 @@ fn Slots(comptime name: [:0]const u8, comptime definition: type, comptime Instan

            ffi.PyErr_Restore(error_type, error_value, error_tb);
        }
+
+        pub fn bf_getbuffer(self: *ffi.PyObject, view: *ffi.Py_buffer, flags: c_int) callconv(.C) c_int {


No need to be pub

delta003 · 2023-09-05T10:40:09Z

example/buffers.zig

+
+// Accept a buffer protocol object.
+pub fn sum(args: *const extern struct { buf: py.PyObject }) !py.PyLong {
+    var view = try py.PyBuffer.of(args.buf, py.ffi.PyBUF_C_CONTIGUOUS);


We should make this less primitive in the future. The returned PyBuffer could have a lot more useful methods

delta003 · 2023-09-05T10:41:45Z

example/buffers.zig

+        @memset(self.values, elem);
+    }
+
+    // TODO(marko): Get obj from self.


This is blocking I think

…buffers

delta003 · 2023-09-05T11:49:59Z

example/buffers.zig

+            return py.BufferError.raise("buffer is not writable");
+        }
+
+        const pyObj = try py.self(@constCast(self));


@robert3005 this looks right?

I think we should constCast inside py.self instead:

pub fn self(selfInstance: anytype) !types.PyObject { const selfState = @fieldParentPtr(pytypes.State(@typeInfo(@TypeOf(selfInstance)).Pointer.child), "state", selfInstance); return .{ .py = @constCast(&selfState.obj) }; <--- here }

delta003 · 2023-09-05T11:52:39Z

pydust/src/types/buffer.zig

+    pub fn initFromSlice(self: *Self, comptime value_type: type, values: []value_type, shape: []isize, obj: py.PyObject) void {
+        self.* = .{
+            .buf = std.mem.sliceAsBytes(values).ptr,
+            .obj = obj.py,
+            .len = @intCast(values.len * @sizeOf(value_type)),
+            .itemsize = @sizeOf(value_type),
+            .readonly = 1,
+            .ndim = 1,
+            .format = getFormat(value_type).ptr,
+            .shape = shape.ptr,
+        };
+    }
+
+    // asSlice returns buf property as Zig slice. The view must have been created with ND flag.
+    pub fn asSlice(self: *const Self, comptime value_type: type) []value_type {
+        return @alignCast(std.mem.bytesAsSlice(value_type, self.buf.?[0..@intCast(self.len)]));
+    }


This is a start #29

gatesn · 2023-09-05T11:50:38Z

example/buffers.zig

+    const Self = @This();
+
+    values: []i64,
+    shape: []isize,


Since the shape is only ever 1-dimensional, this can just be pylength: isize right? And leave a comment that it's an isize to be compatible with Python API.

Then just take the address of it, possibly needing a ptr cast.

gatesn · 2023-09-05T11:52:03Z

example/buffers.zig

+    pub fn __buffer__(self: *const Self, view: *py.PyBuffer, flags: c_int) !void {
+        // For more details on request types, see https://docs.python.org/3/c-api/buffer.html#buffer-request-types
+        if (flags & py.PyBuffer.WRITABLE != 0) {
+            return py.BufferError.raise("buffer is not writable");


Doesn't this mean that the caller didn't request a writable buffer? I'm sure you can still give them a writable one

If a caller requested writable, this will fail. I was going to show an example of using flags - reject a request for a writable buffer.

gatesn · 2023-09-05T11:53:21Z

example/buffers.zig

+            return py.BufferError.raise("buffer is not writable");
+        }
+
+        const pyObj = try py.self(@constCast(self));


@constcast is weird here. But I don't think we've extracted out the logic to correctly construct the self argument for functions that are not class methods.

gatesn · 2023-09-05T11:53:42Z

example/buffers.zig

+        view.initFromSlice(i64, self.values, self.shape, pyObj);
+
+        // We need to incref the self object because it's being used by the view.
+        pyObj.incref();


Seems like initFromSlice should do this?

gatesn · 2023-09-05T11:53:56Z

example/buffers.zig

+    pub fn __release_buffer__(self: *const Self, view: *py.PyBuffer) void {
+        py.allocator.free(self.values);
+        py.allocator.free(self.shape);
+        // It might be necessary to clean up the view here. Depends on the implementation.


In the case of more complex / custom buffer implementation. That's what I was trying to say

gatesn · 2023-09-05T11:55:04Z

example/buffers.zig

+
+// A function that accepts an object implementing the buffer protocol.
+pub fn sum(args: *const extern struct { buf: py.PyObject }) !i64 {
+    // ND is required by asSlice.


Why not use PyBUF_SIMPLE?

We can but I thought it's useful to get the slice of bytes of specific type, even if you don't get back the right shape. Happy to switch for this example.

gatesn · 2023-09-05T11:56:49Z

example/buffers.zig

+// A function that accepts an object implementing the buffer protocol.
+pub fn sum(args: *const extern struct { buf: py.PyObject }) !i64 {
+    // ND is required by asSlice.
+    var view = try py.PyBuffer.of(args.buf, py.PyBuffer.ND);


I think this should actually be a function on PyObject? It aligns with C API of PyObject_GetBuffer and it's maybe clearer what's happening?

So it would be:

var buffer: py.PyBuffer = undefined; args.buf.getBuffer(&buffer)

gatesn · 2023-09-05T11:58:15Z

pydust/src/types/buffer.zig

+pub const PyBuffer = extern struct {
+    const Self = @This();
+
+    pub const SIMPLE: c_int = 0;


Maybe wrap these in a pub const Flags = struct { ... } just so it's obvious what they are. I was initially confused by py.PyBuffer.ND

gatesn · 2023-09-05T11:59:06Z

pydust/src/types/buffer.zig

+                        8 => return "B",
+                        16 => return "H",
+                        32 => return "I",
+                        64 => return "L",


Since you pushed the return inside the switch, you can just have else => {} and then raise a single compileError at the end of this function

delta003 added 20 commits September 1, 2023 10:43

init work for buffer protocol

94714e9

buffers module

1da4948

This commit will be squashed.

4120a6f

inplace, don't decref

8960e60

This commit will be squashed.

632af3c

This commit will be squashed.

8ac93a3

better doc

b44f30d

some progress on the array to buffer

1ba0eb2

iterate in zig

3f14aed

merge develop

c55dc78

merge develop

2fcebc4

Use pytest.build.zig in debug example

fb866b8

Merge branch 'mb/use-pytest-build' of github.com:fulcrum-so/ziggy-pyd…

0c80134

…ust into mb/buffers

This commit will be squashed.

b4820f8

This commit will be squashed.

39950fb

__buffer__, __release_buffer__

1dc7506

__buffer__

ba2ce8b

Merge branch 'develop' of github.com:fulcrum-so/ziggy-pydust into mb/…

fc32d01

…buffers

revert lock

96c5fe7

This commit will be squashed.

34c0a85

delta003 changed the title ~~Support Buffer Protocol types~~ Support Buffer Protocol Sep 4, 2023

delta003 added 7 commits September 5, 2023 08:12

merge, refactor

11a91e2

release buffer

3b4355b

optional ref

52cbd40

add some useful methods

0fd2326

union with allocator oom

6a90e96

raise buffer

4b0e423

This commit will be squashed.

142dc8e

gatesn reviewed Sep 5, 2023

View reviewed changes

add sum, handle errors

449604d

delta003 added 3 commits September 5, 2023 12:33

This commit will be squashed.

8a221b0

This commit will be squashed.

77e1cf1

This commit will be squashed.

123e06e

delta003 commented Sep 5, 2023

View reviewed changes

example/buffers.zig Outdated

@memset(self.values, elem);

}

// TODO(marko): Get obj from self.

Copy link

Member Author

delta003 Sep 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is blocking I think

delta003 added 6 commits September 5, 2023 13:00

flags

56b1ec6

Merge branch 'develop' of github.com:fulcrum-so/ziggy-pydust into mb/…

ea73657

…buffers

remove self obj

6e0af6d

comments

c4bf1e2

more improvements

b1377d8

Merge branch 'develop' of github.com:fulcrum-so/ziggy-pydust into mb/…

f2680b3

…buffers

delta003 marked this pull request as ready for review September 5, 2023 11:47

delta003 commented Sep 5, 2023

View reviewed changes

gatesn reviewed Sep 5, 2023

View reviewed changes

delta003 added 3 commits September 5, 2023 14:13

comments

60e6881

better error message

f7d69e2

types

54f4817

delta003 enabled auto-merge (squash) September 5, 2023 13:19

gatesn approved these changes Sep 5, 2023

View reviewed changes

delta003 merged commit 843b38a into develop Sep 5, 2023
1 check passed

delta003 deleted the mb/buffers branch September 5, 2023 14:19

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Support Buffer Protocol #8

Support Buffer Protocol #8

delta003 commented Sep 1, 2023 •

edited

Loading

gatesn Sep 2, 2023

gatesn Sep 2, 2023

gatesn Sep 5, 2023

delta003 Sep 5, 2023

gatesn Sep 5, 2023

delta003 Sep 5, 2023

delta003 Sep 5, 2023

delta003 Sep 5, 2023

gatesn Sep 5, 2023

delta003 Sep 5, 2023

gatesn Sep 5, 2023

gatesn Sep 5, 2023

delta003 Sep 5, 2023

gatesn Sep 5, 2023

gatesn Sep 5, 2023

gatesn Sep 5, 2023

delta003 Sep 5, 2023

gatesn Sep 5, 2023

delta003 Sep 5, 2023

gatesn Sep 5, 2023

gatesn Sep 5, 2023

gatesn Sep 5, 2023

Support Buffer Protocol #8

Support Buffer Protocol #8

Conversation

delta003 commented Sep 1, 2023 • edited Loading

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

delta003 commented Sep 1, 2023 •

edited

Loading