Error-handling as a struct, rather than just a string. (#13)

The `Error` struct can carry a lot of information into error messages for C++, but Numba can only raise serialized and globally registered exceptions. Therefore, the error messages from Numba have to be known at compile-time, which severely limits their specificity. Users with unhelpful error messages will have to re-run their code outside of Numba.

* Foot-in-the-door to start implementing error-handling as a struct with feedback about the index position of the error (and therefore the Identity, too, if that's available).

* Try to initiate a build from the new repository owner (scikit-hep).

* Make sure I can still trigger builds.

* Add classname() as a virtual method on all Content classes

* Changed Error from a 'const char*' to a struct.

* The 'failure' function didn't return an error (thanks, Windows compiler).

* Start methodically testing error messages.

* NumpyArray error messages are correct.

* Have to carry knowledge of fake dimensions to give the right error message.

* All cpu-kernels should return Error objects, to ensure consistency in the future.

* Locations can only be used if there are Identities.

* The 'fake' argument isn't necessary because fake dimensions will never have Identities.

* These are good error messages.

* Update the Numba side in preparation for handling errors.

* Numba can raise errors in response to the Error.str value.

* Raise errors in Numba, if the function we call returns them (too bad we can't have more specific messages, though).

* Put the 'str' member first so that success vs failure can be determined without an offset.

* Change order of 'failure' function options to highlight the leading role of the static string.

VERSION_INFO

@@ -1 +1 @@
-0.1.9
+0.1.10

awkward1/_numba/content.py

@@ -8,7 +8,9 @@
 from .._numba import cpu, util, identity
 
 class ContentType(numba.types.Type):
-    pass
+    @property
+    def shortname(self):
+        return self.name[:self.name.index("Type")]
 
 @numba.typing.templates.infer_global(len)
 class type_len(numba.typing.templates.AbstractTemplate):

awkward1/_numba/cpu.py

@@ -19,6 +19,26 @@
 
 kernels = ctypes.cdll.LoadLibrary(libpath)
 
+class ErrorType(numba.types.Type):
+    def __init__(self):
+        super(ErrorType, self).__init__(name="awkward._numba.cpu.ErrorType")
+
+class Error(ctypes.Structure):
+    _fields_ = [("str", ctypes.c_char_p),
+                ("location", ctypes.c_int64),
+                ("attempt", ctypes.c_int64),
+                ("extra", ctypes.c_int64)]
+    numbatpe = ErrorType()
+
+@numba.extending.register_model(ErrorType)
+class ErrorModel(numba.datamodel.models.StructModel):
+    def __init__(self, dmm, fe_type):
+        members = [("str", numba.intp),
+                   ("location", numba.int64),
+                   ("attempt", numba.int64),
+                   ("extra", numba.int64)]
+        super(ErrorModel, self).__init__(dmm, fe_type, members)
+
 h2ctypes = {
     "bool": ctypes.c_uint8,
     "uint8_t *": ctypes.POINTER(ctypes.c_uint8),
@@ -29,7 +49,7 @@
     "int64_t": ctypes.c_int64,
     "int64_t *": ctypes.POINTER(ctypes.c_int64),
     "const int64_t *": ctypes.POINTER(ctypes.c_int64),
-    "Error": ctypes.c_void_p,
+    "Error": Error,
     "void": None,
     }
 
@@ -41,7 +61,13 @@
             if hasattr(kernels, name):
                 rettype = xfcn.find("./type").text
                 params = [(x.find("./declname").text, x.find("./type").text) for x in xfcn.findall("./param")]
-                getattr(kernels, name).argtypes = [h2ctypes[t] for n, t in params]
-                getattr(kernels, name).restype = h2ctypes[rettype]
                 getattr(kernels, name).name = name
-                getattr(kernels, name).numbatpe = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name))
+                getattr(kernels, name).argtypes = [h2ctypes[t] for n, t in params]
+                if rettype == "Error":
+                    getattr(kernels, name).restype = None
+                    tmp = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name))
+                    getattr(kernels, name).numbatpe = numba.types.functions.ExternalFunctionPointer(Error.numbatpe(*tmp.sig.args), tmp.get_pointer, cconv=tmp.cconv)
+                    getattr(kernels, name).restype = h2ctypes[rettype]
+                else:
+                    getattr(kernels, name).restype = h2ctypes[rettype]
+                    getattr(kernels, name).numbatpe = numba.typing.ctypes_utils.make_function_type(getattr(kernels, name))

awkward1/_numba/listarray.py

@@ -293,7 +293,8 @@ def lower_getitem_next(context, builder, arraytpe, wheretpe, arrayval, whereval,
              context.get_constant(numba.int64, 0),
              lenstarts,
              lenflathead,
-             lencontent))
+             lencontent),
+            "in {}, indexing error".format(arraytpe.shortname))
 
         nexttpe = arraytpe.contenttpe.carry()
         nextval = arraytpe.contenttpe.lower_carry(context, builder, arraytpe.contenttpe, util.index64tpe, proxyin.content, nextcarry)

awkward1/_numba/util.py

@@ -2,6 +2,9 @@
 
 import numpy
 import numba
+import llvmlite.ir.types
+
+from .._numba import cpu
 
 RefType = numba.int64
 
@@ -28,12 +31,30 @@ def arraylen(context, builder, tpe, val, totpe=None):
     else:
         return cast(context, builder, numba.intp, totpe, out)
 
-def call(context, builder, fcn, args):
+def call(context, builder, fcn, args, errormessage=None):
     fcntpe = context.get_function_pointer_type(fcn.numbatpe)
     fcnval = context.add_dynamic_addr(builder, fcn.numbatpe.get_pointer(fcn), info=fcn.name)
     fcnptr = builder.bitcast(fcnval, fcntpe)
+
     err = context.call_function_pointer(builder, fcnptr, args)
-    # FIXME: handle error
+
+    if fcn.restype is cpu.Error:
+        assert errormessage is not None, "this function can return an error"
+        proxyerr = numba.cgutils.create_struct_proxy(cpu.Error.numbatpe)(context, builder, value=err)
+        with builder.if_then(builder.icmp_signed("!=", proxyerr.str, context.get_constant(numba.intp, 0)), likely=False):
+            context.call_conv.return_user_exc(builder, ValueError, (errormessage,))
+
+            # pyapi = context.get_python_api(builder)
+            # exc = pyapi.serialize_object(ValueError(errormessage))
+            # excptr = context.call_conv._get_excinfo_argument(builder.function)
+            # if excptr.name == "excinfo" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.struct([llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)), llvmlite.llvmpy.core.Type.int(32)]))):
+            #     builder.store(exc, excptr)
+            #     builder.ret(numba.targets.callconv.RETCODE_USEREXC)
+            # elif excptr.name == "py_args" and excptr.type == llvmlite.llvmpy.core.Type.pointer(llvmlite.llvmpy.core.Type.int(8)):
+            #     pyapi.raise_object(exc)
+            #     builder.ret(llvmlite.llvmpy.core.Constant.null(context.get_value_type(numba.types.pyobject)))
+            # else:
+            #     raise AssertionError("unrecognized exception calling convention: {}".format(excptr))
 
 def newindex64(context, builder, lentpe, lenval):
     return numba.targets.arrayobj.numpy_empty_nd(context, builder, index64tpe(lentpe), (lenval,))

include/awkward/Content.h

@@ -10,6 +10,7 @@
 namespace awkward {
   class Content {
   public:
+    virtual const std::string classname() const = 0;
     virtual const std::shared_ptr<Identity> id() const = 0;
     virtual void setid() = 0;
     virtual void setid(const std::shared_ptr<Identity> id) = 0;

include/awkward/Identity.h

@@ -32,6 +32,8 @@ namespace awkward {
     const int64_t width() const { return width_; }
     const int64_t length() const { return length_; }
 
+    virtual const std::string classname() const = 0;
+    virtual const std::string location(int64_t where) const = 0;
     virtual const std::shared_ptr<Identity> to64() const = 0;
     virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const = 0;
     virtual const std::shared_ptr<Identity> getitem_range(int64_t start, int64_t stop) const = 0;
@@ -58,6 +60,8 @@ namespace awkward {
 
     const std::shared_ptr<T> ptr() const { return ptr_; }
 
+    virtual const std::string classname() const;
+    virtual const std::string location(int64_t where) const;
     virtual const std::shared_ptr<Identity> to64() const;
     virtual const std::string tostring_part(const std::string indent, const std::string pre, const std::string post) const;
     virtual const std::shared_ptr<Identity> getitem_range(int64_t start, int64_t stop) const;

include/awkward/ListArray.h

@@ -24,6 +24,7 @@ namespace awkward {
     const IndexOf<T> stops() const { return stops_; }
     const std::shared_ptr<Content> content() const { return content_.get()->shallow_copy(); }
 
+    virtual const std::string classname() const;
     virtual const std::shared_ptr<Identity> id() const { return id_; }
     virtual void setid();
     virtual void setid(const std::shared_ptr<Identity> id);

include/awkward/ListOffsetArray.h

@@ -22,6 +22,7 @@ namespace awkward {
     const IndexOf<T> offsets() const { return offsets_; }
     const std::shared_ptr<Content> content() const { return content_.get()->shallow_copy(); }
 
+    virtual const std::string classname() const;
     virtual const std::shared_ptr<Identity> id() const { return id_; }
     virtual void setid();
     virtual void setid(const std::shared_ptr<Identity> id);

include/awkward/NumpyArray.h

@@ -40,6 +40,7 @@ namespace awkward {
     ssize_t bytelength() const;
     uint8_t getbyte(ssize_t at) const;
 
+    virtual const std::string classname() const;
     virtual const std::shared_ptr<Identity> id() const { return id_; }
     virtual void setid();
     virtual void setid(const std::shared_ptr<Identity> id);

include/awkward/RawArray.h

@@ -58,6 +58,7 @@ namespace awkward {
     T* borrow() const { return borrow(0); }
     T* borrow(int64_t at) const { return reinterpret_cast<T*>(reinterpret_cast<ssize_t>(ptr_.get()) + (ssize_t)itemsize_*(ssize_t)(offset_ + at)); }
 
+    virtual const std::string classname() const { return std::string("RawArrayOf<") + std::string(typeid(T).name()) + std::string(">"); }
     virtual const std::shared_ptr<Identity> id() const { return id_; }
     virtual void setid() {
       if (length() <= kMaxInt32) {
@@ -138,7 +139,7 @@ namespace awkward {
       std::shared_ptr<SliceItem> nexthead = where.head();
       Slice nexttail = where.tail();
       Index64 nextadvanced(0);
-      return getitem_next(nexthead, nexttail, nextadvanced);
+      return getitem_next(nexthead, nexttail, nextadvanced, false);
     }
     const std::shared_ptr<Content> getitem_next(const std::shared_ptr<SliceItem> head, const Slice& tail, const Index64& advanced) const {
       if (tail.length() != 0) {

include/awkward/cpu-kernels/getitem.h

@@ -9,46 +9,47 @@ extern "C" {
   void awkward_regularize_rangeslice(int64_t* start, int64_t* stop, bool posstep, bool hasstart, bool hasstop, int64_t length);
   Error awkward_regularize_arrayslice_64(int64_t* flatheadptr, int64_t lenflathead, int64_t length);
 
-  void awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides);
+  Error awkward_slicearray_ravel_64(int64_t* toptr, const int64_t* fromptr, int64_t ndim, const int64_t* shape, const int64_t* strides);
 
-  void awkward_carry_arange_64(int64_t* toptr, int64_t length);
+  Error awkward_carry_arange_64(int64_t* toptr, int64_t length);
 
   Error awkward_identity32_getitem_carry_64(int32_t* newidentityptr, const int32_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length);
   Error awkward_identity64_getitem_carry_64(int64_t* newidentityptr, const int64_t* identityptr, const int64_t* carryptr, int64_t lencarry, int64_t offset, int64_t width, int64_t length);
 
-  void awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride);
-  void awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
-  void awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride);
-  void awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
-  void awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at);
-  void awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
-  void awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
-  void awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip);
-  void awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip);
+  Error awkward_numpyarray_contiguous_init_64(int64_t* toptr, int64_t skip, int64_t stride);
+  Error awkward_numpyarray_contiguous_copy_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
+  Error awkward_numpyarray_contiguous_next_64(int64_t* topos, const int64_t* frompos, int64_t len, int64_t skip, int64_t stride);
+
+  Error awkward_numpyarray_getitem_next_null_64(uint8_t* toptr, const uint8_t* fromptr, int64_t len, int64_t stride, int64_t offset, const int64_t* pos);
+  Error awkward_numpyarray_getitem_next_at_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t skip, int64_t at);
+  Error awkward_numpyarray_getitem_next_range_64(int64_t* nextcarryptr, const int64_t* carryptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
+  Error awkward_numpyarray_getitem_next_range_advanced_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* advancedptr, int64_t lencarry, int64_t lenhead, int64_t skip, int64_t start, int64_t step);
+  Error awkward_numpyarray_getitem_next_array_64(int64_t* nextcarryptr, int64_t* nextadvancedptr, const int64_t* carryptr, const int64_t* flatheadptr, int64_t lencarry, int64_t lenflathead, int64_t skip);
+  Error awkward_numpyarray_getitem_next_array_advanced_64(int64_t* nextcarryptr, const int64_t* carryptr, const int64_t* advancedptr, const int64_t* flatheadptr, int64_t lencarry, int64_t skip);
 
   Error awkward_listarray32_getitem_next_at_64(int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at);
   Error awkward_listarray64_getitem_next_at_64(int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t at);
 
-  void awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
-  void awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+  Error awkward_listarray32_getitem_next_range_carrylength(int64_t* carrylength, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+  Error awkward_listarray64_getitem_next_range_carrylength(int64_t* carrylength, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
 
-  void awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
-  void awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+  Error awkward_listarray32_getitem_next_range_64(int32_t* tooffsets, int64_t* tocarry, const int32_t* fromstarts, const int32_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
+  Error awkward_listarray64_getitem_next_range_64(int64_t* tooffsets, int64_t* tocarry, const int64_t* fromstarts, const int64_t* fromstops, int64_t lenstarts, int64_t startsoffset, int64_t stopsoffset, int64_t start, int64_t stop, int64_t step);
 
-  void awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts);
-  void awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts);
+  Error awkward_listarray32_getitem_next_range_counts_64(int64_t* total, const int32_t* fromoffsets, int64_t lenstarts);
+  Error awkward_listarray64_getitem_next_range_counts_64(int64_t* total, const int64_t* fromoffsets, int64_t lenstarts);
 
-  void awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts);
-  void awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts);
+  Error awkward_listarray32_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int32_t* fromoffsets, int64_t lenstarts);
+  Error awkward_listarray64_getitem_next_range_spreadadvanced_64(int64_t* toadvanced, const int64_t* fromadvanced, const int64_t* fromoffsets, int64_t lenstarts);
 
   Error awkward_listarray32_getitem_next_array_64(int32_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
   Error awkward_listarray64_getitem_next_array_64(int64_t* tooffsets, int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
 
   Error awkward_listarray32_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
   Error awkward_listarray64_getitem_next_array_advanced_64(int64_t* tocarry, int64_t* toadvanced, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromarray, const int64_t* fromadvanced, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lenarray, int64_t lencontent);
 
-  void awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry);
-  void awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lencarry);
+  Error awkward_listarray32_getitem_carry_64(int32_t* tostarts, int32_t* tostops, const int32_t* fromstarts, const int32_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry);
+  Error awkward_listarray64_getitem_carry_64(int64_t* tostarts, int64_t* tostops, const int64_t* fromstarts, const int64_t* fromstops, const int64_t* fromcarry, int64_t startsoffset, int64_t stopsoffset, int64_t lenstarts, int64_t lencarry);
 }
 
 #endif // AWKWARDCPU_GETITEM_H_

include/awkward/cpu-kernels/identity.h

@@ -6,10 +6,10 @@
 #include "awkward/cpu-kernels/util.h"
 
 extern "C" {
-  void awkward_new_identity32(int32_t* toptr, int64_t length);
-  void awkward_new_identity64(int64_t* toptr, int64_t length);
+  Error awkward_new_identity32(int32_t* toptr, int64_t length);
+  Error awkward_new_identity64(int64_t* toptr, int64_t length);
 
-  void awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length);
+  Error awkward_identity32_to_identity64(int64_t* toptr, const int32_t* fromptr, int64_t length);
 
   Error awkward_identity32_from_listarray32(int32_t* toptr, const int32_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth);
   Error awkward_identity64_from_listarray32(int64_t* toptr, const int64_t* fromptr, const int32_t* fromstarts, const int32_t* fromstops, int64_t fromptroffset, int64_t startsoffset, int64_t stopsoffset, int64_t tolength, int64_t fromlength, int64_t fromwidth);

include/awkward/cpu-kernels/util.h

@@ -20,9 +20,17 @@
   #include <cstdint>
 #endif
 
+#include <iostream>
+
 extern "C" {
-  typedef const char* Error;
-  const Error kNoError = nullptr;
+  struct Error {
+    const char* str;
+    int64_t location;
+    int64_t attempt;
+    int64_t extra;
+  };
+  Error success();
+  Error failure(const char* str, int64_t location, int64_t attempt);
 
   const int8_t  kMaxInt8   =                 127;   // 2**7  - 1
   const uint8_t kMaxUInt8  =                 255;   // 2**8  - 1