feat: use None for unknown lengths (1 of 2) (#2168)

* wip: none shape items

* fix: remaining `ensure_known_scalar`

* refactor: use new shape aPI

* test: remove skips

* fix: use `unset` for length in `RecordArray`

* fix: use `unionarray.length`

* fix: uses of `length=None` for RecordArray

* refactor: cleanup shape handling

* fix:bugs in form association

* refactor: small type hint

* refactor: cleanup typing, shape handling

* refactor: remove `UnknownLength`

* feat: update compatability shim

* fix: type alias

* refactor: rename `as_shape_item` to `scalar_as_shape_item`

* refactor: remove redundant line

* fix: warn if `to_list` called with no data

* refactor: add return type, ensure `x-y >= 0`

* fix: deprecate `length=None` for `RecordArray`

* style: pre-commit fixes

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

src/awkward/_broadcasting.py

@@ -12,7 +12,8 @@
 from awkward._nplikes import nplike_of
 from awkward._nplikes.numpy import Numpy
 from awkward._nplikes.numpylike import NumpyMetadata
-from awkward._nplikes.typetracer import TypeTracerArray, is_unknown_length
+from awkward._nplikes.typetracer import TypeTracerArray
+from awkward._util import unset
 from awkward.contents.bitmaskedarray import BitMaskedArray
 from awkward.contents.bytemaskedarray import ByteMaskedArray
 from awkward.contents.content import Content
@@ -46,7 +47,7 @@ def length_of_broadcast(inputs: Sequence) -> int | TypeTracerArray:
 
     for x in inputs:
         if isinstance(x, Content):
-            if is_unknown_length(x.length):
+            if x.length is None:
                 return x.length
 
             maxlen = max(maxlen, x.length)
@@ -688,14 +689,14 @@ def continuation():
                 for x in inputs
             ):
                 # Ensure all layouts have same length
-                length = None
+                length = unset
                 for x in inputs:
                     if isinstance(x, Content):
-                        if length is None:
+                        if length is unset:
                             length = x.length
                         elif backend.nplike.known_shape:
                             assert length == x.length
-                assert length is not None
+                assert length is not unset
 
                 if any(x.size == 0 for x in inputs if isinstance(x, RegularArray)):
                     dimsize = 0
@@ -951,10 +952,10 @@ def continuation():
                     ValueError(f"cannot broadcast records {in_function(options)}")
                 )
 
-            fields, length, istuple = None, None, True
+            fields, length, istuple = unset, unset, unset
             for x in inputs:
                 if isinstance(x, RecordArray):
-                    if fields is None:
+                    if fields is unset:
                         fields = x.fields
                     elif set(fields) != set(x.fields):
                         raise ak._errors.wrap_error(
@@ -967,7 +968,7 @@ def continuation():
                                 )
                             )
                         )
-                    if length is None:
+                    if length is unset:
                         length = x.length
                     elif length != x.length:
                         raise ak._errors.wrap_error(
@@ -978,8 +979,10 @@ def continuation():
                                 )
                             )
                         )
-                    if not x.is_tuple:
+                    # Records win over tuples
+                    if istuple is unset or not x.is_tuple:
                         istuple = False
+
             outcontents, numoutputs = [], None
             for field in fields:
                 outcontents.append(

src/awkward/_nplikes/array_module.py

@@ -4,7 +4,7 @@
 import numpy
 
 import awkward as ak
-from awkward._nplikes.numpylike import ArrayLike, NumpyLike, NumpyMetadata
+from awkward._nplikes.numpylike import ArrayLike, NumpyLike, NumpyMetadata, ShapeItem
 from awkward.typing import Final, Literal, SupportsInt
 
 np = NumpyMetadata.instance()
@@ -115,6 +115,35 @@ def searchsorted(
     def broadcast_arrays(self, *arrays: ArrayLike) -> list[ArrayLike]:
         return self._module.broadcast_arrays(*arrays)
 
+    def shape_item_as_scalar(self, x1: ShapeItem):
+        if x1 is None:
+            raise ak._errors.wrap_error(
+                TypeError("array module nplikes do not support unknown lengths")
+            )
+        elif isinstance(x1, int):
+            return self._module.asarray(x1, dtype=np.int64)
+        else:
+            raise ak._errors.wrap_error(
+                TypeError(f"expected None or int type, received {x1}")
+            )
+
+    def scalar_as_shape_item(self, x1) -> ShapeItem:
+        if x1 is None:
+            return None
+        else:
+            return int(x1)
+
+    def add_shape_item(self, x1: ShapeItem, x2: ShapeItem) -> ShapeItem:
+        return x1 + x2
+
+    def sub_shape_item(self, x1: ShapeItem, x2: ShapeItem) -> ShapeItem:
+        result = x1 - x2
+        assert result >= 0
+        return result
+
+    def mul_shape_item(self, x1: ShapeItem, x2: ShapeItem) -> ShapeItem:
+        return x1 * x2
+
     def nonzero(self, x: ArrayLike) -> tuple[ArrayLike, ...]:
         return self._module.nonzero(x)
 

src/awkward/_nplikes/numpylike.py

@@ -6,7 +6,17 @@
 import numpy
 
 from awkward._singleton import Singleton
-from awkward.typing import Literal, Protocol, Self, SupportsIndex, SupportsInt, overload
+from awkward.typing import (
+    Literal,
+    Protocol,
+    Self,
+    SupportsIndex,
+    SupportsInt,  # noqa: F401
+    TypeAlias,
+    overload,
+)
+
+ShapeItem: TypeAlias = "SupportsInt | None"
 
 
 class ArrayLike(Protocol):
@@ -22,12 +32,12 @@ def ndim(self) -> int:
 
     @property
     @abstractmethod
-    def shape(self) -> tuple[SupportsInt, ...]:
+    def shape(self) -> tuple[ShapeItem, ...]:
         ...
 
     @property
     @abstractmethod
-    def size(self) -> SupportsInt:
+    def size(self) -> ShapeItem:
         ...
 
     @property
@@ -287,7 +297,27 @@ def broadcast_arrays(self, *arrays: ArrayLike) -> list[ArrayLike]:
         ...
 
     @abstractmethod
-    def broadcast_to(self, x: ArrayLike, shape: tuple[SupportsInt, ...]) -> ArrayLike:
+    def broadcast_to(self, x: ArrayLike, shape: tuple[ShapeItem, ...]) -> ArrayLike:
+        ...
+
+    @abstractmethod
+    def shape_item_as_scalar(self, x1: ShapeItem):
+        ...
+
+    @abstractmethod
+    def scalar_as_shape_item(self, x1) -> ShapeItem:
+        ...
+
+    @abstractmethod
+    def sub_shape_item(self, x1: ShapeItem, x2: ShapeItem) -> ShapeItem:
+        ...
+
+    @abstractmethod
+    def add_shape_item(self, x1: ShapeItem, x2: ShapeItem) -> ShapeItem:
+        ...
+
+    @abstractmethod
+    def mul_shape_item(self, x1: ShapeItem, x2: ShapeItem) -> ShapeItem:
         ...
 
     @abstractmethod