feat: add byteorder argument to to_buffers (#2095)

src/awkward/_connect/rdataframe/from_rdataframe.py

@@ -230,9 +230,7 @@ def cpp_fill_function(depth):
             cpp_buffers_self.to_char_buffers[builder_type](builder)
 
             array = ak.from_buffers(
-                form,
-                builder.length(),
-                buffers,
+                form, builder.length(), buffers, byteorder=ak._util.native_byteorder
             )
 
             if col == "awkward_index_":

src/awkward/_do.py

@@ -10,7 +10,7 @@
 from awkward.contents.content import ActionType, Content
 from awkward.forms import form
 from awkward.record import Record
-from awkward.typing import Any, AxisMaybeNone
+from awkward.typing import Any, AxisMaybeNone, Literal
 
 np = ak._nplikes.NumpyMetadata.instance()
 
@@ -76,6 +76,7 @@ def to_buffers(
     form_key: str | None = "node{id}",
     id_start: Integral = 0,
     backend: Backend = None,
+    byteorder: Literal["<", ">"] = "<",
 ) -> tuple[form.Form, int, Mapping[str, Any]]:
     if container is None:
         container = {}
@@ -119,7 +120,7 @@ def getkey(layout, form, attribute):
 
     form = content.form_with_key(form_key=form_key, id_start=id_start)
 
-    content._to_buffers(form, getkey, container, backend)
+    content._to_buffers(form, getkey, container, backend, byteorder)
 
     return form, len(content), container
 

src/awkward/_util.py

@@ -6,6 +6,7 @@
 import numbers
 import os
 import re
+import sys
 from collections.abc import Iterable, Mapping, Sized
 
 import packaging.version
@@ -69,8 +70,24 @@ def tobytes(array):
         return array.tostring()
 
 
-def little_endian(array):
-    return array.astype(array.dtype.newbyteorder("<"), copy=False)
+native_byteorder = "<" if sys.byteorder == "little" else ">"
+
+
+def native_to_byteorder(array, byteorder: str):
+    """
+    Args:
+        array: nplike array
+        byteorder (`"<"` or `">"`): desired byteorder
+
+    Return a copy of array. Swap the byteorder if `byteorder` does not match
+    `ak._util.native_byteorder`. This function is _not_ idempotent; no metadata
+    from `array` exists to determine its current byteorder.
+    """
+    assert byteorder in "<>"
+    if byteorder != native_byteorder:
+        return array.byteswap(inplace=False)
+    else:
+        return array
 
 
 def identifier_hash(str):

src/awkward/contents/bitmaskedarray.py

@@ -200,11 +200,13 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key = getkey(self, form, "mask")
-        container[key] = ak._util.little_endian(self._mask.raw(backend.index_nplike))
-        self._content._to_buffers(form.content, getkey, container, backend)
+        container[key] = ak._util.native_to_byteorder(
+            self._mask.raw(backend.index_nplike), byteorder
+        )
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         tt = ak._typetracer.TypeTracer.instance()

src/awkward/contents/bytemaskedarray.py

@@ -148,11 +148,13 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key = getkey(self, form, "mask")
-        container[key] = ak._util.little_endian(self._mask.raw(backend.index_nplike))
-        self._content._to_buffers(form.content, getkey, container, backend)
+        container[key] = ak._util.native_to_byteorder(
+            self._mask.raw(backend.index_nplike), byteorder
+        )
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         tt = ak._typetracer.TypeTracer.instance()

src/awkward/contents/content.py

@@ -9,7 +9,7 @@
 import awkward as ak
 from awkward._backends import Backend
 from awkward.forms.form import Form, _parameters_equal
-from awkward.typing import Any, AxisMaybeNone, Self, TypeAlias, TypedDict
+from awkward.typing import Any, AxisMaybeNone, Literal, Self, TypeAlias, TypedDict
 
 np = ak._nplikes.NumpyMetadata.instance()
 numpy = ak._nplikes.Numpy.instance()
@@ -197,6 +197,7 @@ def _to_buffers(
         getkey: Callable[[Content, Form, str], str],
         container: MutableMapping[str, Any] | None,
         backend: Backend,
+        byteorder: Literal["<", ">"],
     ) -> tuple[Form, int, Mapping[str, Any]]:
         raise ak._errors.wrap_error(NotImplementedError)
 

src/awkward/contents/emptyarray.py

@@ -49,7 +49,7 @@ def simplified(cls, *, parameters=None, backend=None):
     def _form_with_key(self, getkey):
         return self.form_cls(parameters=self._parameters, form_key=getkey(self))
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
 
     def _to_typetracer(self, forget_length: bool) -> Self:

src/awkward/contents/indexedarray.py

@@ -145,11 +145,13 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key = getkey(self, form, "index")
-        container[key] = ak._util.little_endian(self._index.raw(backend.index_nplike))
-        self._content._to_buffers(form.content, getkey, container, backend)
+        container[key] = ak._util.native_to_byteorder(
+            self._index.raw(backend.index_nplike), byteorder
+        )
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         index = self._index.to_nplike(ak._typetracer.TypeTracer.instance())

src/awkward/contents/indexedoptionarray.py

@@ -132,11 +132,13 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key = getkey(self, form, "index")
-        container[key] = ak._util.little_endian(self._index.raw(backend.index_nplike))
-        self._content._to_buffers(form.content, getkey, container, backend)
+        container[key] = ak._util.native_to_byteorder(
+            self._index.raw(backend.index_nplike), byteorder
+        )
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         index = self._index.to_nplike(ak._typetracer.TypeTracer.instance())

src/awkward/contents/listarray.py

@@ -134,13 +134,17 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key1 = getkey(self, form, "starts")
         key2 = getkey(self, form, "stops")
-        container[key1] = ak._util.little_endian(self._starts.raw(backend.index_nplike))
-        container[key2] = ak._util.little_endian(self._stops.raw(backend.index_nplike))
-        self._content._to_buffers(form.content, getkey, container, backend)
+        container[key1] = ak._util.native_to_byteorder(
+            self._starts.raw(backend.index_nplike), byteorder
+        )
+        container[key2] = ak._util.native_to_byteorder(
+            self._stops.raw(backend.index_nplike), byteorder
+        )
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         tt = ak._typetracer.TypeTracer.instance()

src/awkward/contents/listoffsetarray.py

@@ -120,11 +120,13 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key = getkey(self, form, "offsets")
-        container[key] = ak._util.little_endian(self._offsets.raw(backend.index_nplike))
-        self._content._to_buffers(form.content, getkey, container, backend)
+        container[key] = ak._util.native_to_byteorder(
+            self._offsets.raw(backend.index_nplike), byteorder
+        )
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         offsets = self._offsets.to_nplike(ak._typetracer.TypeTracer.instance())

src/awkward/contents/numpyarray.py

@@ -111,10 +111,12 @@ def _form_with_key(self, getkey):
             form_key=getkey(self),
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key = getkey(self, form, "data")
-        container[key] = ak._util.little_endian(self._raw(backend.nplike))
+        container[key] = ak._util.native_to_byteorder(
+            self._raw(backend.nplike), byteorder
+        )
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         backend = ak._backends.TypeTracerBackend.instance()

src/awkward/contents/recordarray.py

@@ -204,14 +204,18 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         if self._fields is None:
             for i, content in enumerate(self._contents):
-                content._to_buffers(form.content(i), getkey, container, backend)
+                content._to_buffers(
+                    form.content(i), getkey, container, backend, byteorder
+                )
         else:
             for field, content in zip(self._fields, self._contents):
-                content._to_buffers(form.content(field), getkey, container, backend)
+                content._to_buffers(
+                    form.content(field), getkey, container, backend, byteorder
+                )
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         backend = ak._backends.TypeTracerBackend.instance()

src/awkward/contents/regulararray.py

@@ -127,9 +127,9 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
-        self._content._to_buffers(form.content, getkey, container, backend)
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         return RegularArray(

src/awkward/contents/unionarray.py

@@ -391,14 +391,18 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
         key1 = getkey(self, form, "tags")
         key2 = getkey(self, form, "index")
-        container[key1] = ak._util.little_endian(self._tags.raw(backend.index_nplike))
-        container[key2] = ak._util.little_endian(self._index.raw(backend.index_nplike))
+        container[key1] = ak._util.native_to_byteorder(
+            self._tags.raw(backend.index_nplike), byteorder
+        )
+        container[key2] = ak._util.native_to_byteorder(
+            self._index.raw(backend.index_nplike), byteorder
+        )
         for i, content in enumerate(self._contents):
-            content._to_buffers(form.content(i), getkey, container, backend)
+            content._to_buffers(form.content(i), getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         tt = ak._typetracer.TypeTracer.instance()

src/awkward/contents/unmaskedarray.py

@@ -80,9 +80,9 @@ def _form_with_key(self, getkey):
             form_key=form_key,
         )
 
-    def _to_buffers(self, form, getkey, container, backend):
+    def _to_buffers(self, form, getkey, container, backend, byteorder):
         assert isinstance(form, self.form_cls)
-        self._content._to_buffers(form.content, getkey, container, backend)
+        self._content._to_buffers(form.content, getkey, container, backend, byteorder)
 
     def _to_typetracer(self, forget_length: bool) -> Self:
         return UnmaskedArray(

src/awkward/forms/form.py

@@ -426,6 +426,7 @@ def length_zero_array(
             container={"": b"\x00\x00\x00\x00\x00\x00\x00\x00"},
             buffer_key="",
             backend=backend,
+            byteorder=ak._util.native_byteorder,
             highlevel=highlevel,
             behavior=behavior,
             simplify=False,
@@ -446,6 +447,7 @@ def length_one_array(self, *, backend=numpy_backend, highlevel=True, behavior=No
             },
             buffer_key="",
             backend=backend,
+            byteorder=ak._util.native_byteorder,
             highlevel=highlevel,
             behavior=behavior,
             simplify=False,

src/awkward/highlevel.py

@@ -1433,7 +1433,10 @@ def numba_type(self):
     def __getstate__(self):
         packed = ak.operations.to_packed(self._layout, highlevel=False)
         form, length, container = ak.operations.to_buffers(
-            packed, buffer_key="{form_key}-{attribute}", form_key="node{id}"
+            packed,
+            buffer_key="{form_key}-{attribute}",
+            form_key="node{id}",
+            byteorder="<",
         )
         if self._behavior is ak.behavior:
             behavior = None
@@ -1449,6 +1452,7 @@ def __setstate__(self, state):
             container,
             highlevel=False,
             buffer_key="{form_key}-{attribute}",
+            byteorder="<",
         )
         self.layout = layout
         self.behavior = behavior
@@ -2067,7 +2071,10 @@ def numba_type(self):
     def __getstate__(self):
         packed = ak.operations.to_packed(self._layout, highlevel=False)
         form, length, container = ak.operations.to_buffers(
-            packed.array, buffer_key="{form_key}-{attribute}", form_key="node{id}"
+            packed.array,
+            buffer_key="{form_key}-{attribute}",
+            form_key="node{id}",
+            byteorder="<",
         )
         if self._behavior is ak.behavior:
             behavior = None
@@ -2083,6 +2090,7 @@ def __setstate__(self, state):
             container,
             highlevel=False,
             buffer_key="{form_key}-{attribute}",
+            byteorder="<",
         )
         layout = ak.record.Record(layout, at)
         self.layout = layout
@@ -2435,6 +2443,7 @@ def snapshot(self):
                 container,
                 buffer_key="{form_key}-{attribute}",
                 backend="cpu",
+                byteorder=ak._util.native_byteorder,
                 highlevel=True,
                 behavior=self._behavior,
                 simplify=True,

src/awkward/operations/ak_from_avro_file.py

@@ -71,6 +71,7 @@ def _impl(form, length, container, highlevel, behavior):
         container=container,
         buffer_key="{form_key}-{attribute}",
         backend="cpu",
+        byteorder=ak._util.native_byteorder,
         highlevel=highlevel,
         behavior=behavior,
         simplify=True,