-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
fc7d4a3
commit 0affe5c
Showing
10 changed files
with
185 additions
and
82 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -122,5 +122,6 @@ rever/ | |
|
||
# Cython byproducts | ||
versioned_hdf5/*.c | ||
versioned_hdf5/*.cpp | ||
versioned_hdf5/*.html | ||
versioned_hdf5/*.so |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
# This file allows cimport'ing the functions and types declared below from other | ||
# Cython modules | ||
|
||
from cython cimport Py_ssize_t | ||
from libc.stdint cimport uint64_t | ||
|
||
# Centralized definition of hsize_t, in accordance with libhd5: | ||
# https://github.com/HDFGroup/hdf5/blob/6b43197b0817596f47670c6b55d26ff7f86d6bd9/src/H5public.h#L301 | ||
# | ||
# versioned_hdf5 uses the same datatype for indexing as libhdf5. Notably, this differs | ||
# from numpy's npy_intp (same as Py_ssize_t, ssize_t, and signed long), which is only 32 | ||
# bit wide on 32 bit platforms, to allow indexing datasets with >=2**31 points per axis | ||
# on disk, as long as you don't load them in memory all at once. | ||
# | ||
# Note that hsize_t is unsigned, which can lead to integer underflows. | ||
# | ||
# The definition of hsize_t has changed over time in the libhdf5 headers, as noted | ||
# below. | ||
# C99 dictates that long long is *at least* 64 bit wide, while uint64_t is *exactly* 64 | ||
# bit wide. So the two definitions are de facto equivalent. | ||
# The below definition is inconsequential, as it is overridden by whatever version of | ||
# hdf5.h is installed thanks to the 'cdef extern from "hdf5.h"' block. | ||
# | ||
# h5py gets this wildly wrong, as it defines hsize_t as long long, which is signed, and | ||
# confusingly also defines hssize_t as signed long long - which is an alias for | ||
# long long: | ||
# https://github.com/h5py/h5py/blob/eaa9d93cc7620f3e7d8cff6f3a739b7d9834aef7/h5py/api_types_hdf5.pxd#L21-L22 | ||
|
||
cdef extern from "hdf5.h": | ||
# ctypedef unsigned long long hsize_t # hdf5 <=1.12 | ||
ctypedef uint64_t hsize_t # hdf5 >=1.14 | ||
|
||
cpdef hsize_t stop2count(hsize_t start, hsize_t stop, hsize_t step) noexcept nogil | ||
cpdef hsize_t count2stop(hsize_t start, hsize_t count, hsize_t step) noexcept nogil | ||
cpdef Py_ssize_t ceil_a_over_b(Py_ssize_t a, Py_ssize_t b) noexcept nogil | ||
cpdef Py_ssize_t smallest_step_after( | ||
Py_ssize_t x, Py_ssize_t a, Py_ssize_t m | ||
) noexcept nogil |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,73 @@ | ||
from __future__ import annotations | ||
|
||
import cython | ||
from cython import Py_ssize_t | ||
|
||
if cython.compiled: # pragma: nocover | ||
# This is repeated here from the header in order to silence mypy, which now treats | ||
# hsize_t as Any, without breaking cython. | ||
from cython.cimports.versioned_hdf5.cytools import hsize_t | ||
|
||
# numpy equivalent dtype for hsize_t | ||
from numpy import uint64 as np_hsize_t # noqa: F401 | ||
|
||
|
||
@cython.ccall | ||
@cython.nogil | ||
@cython.exceptval(check=False) | ||
def stop2count(start: hsize_t, stop: hsize_t, step: hsize_t) -> hsize_t: | ||
"""Given a start:stop:step slice or range, return the number of elements yielded. | ||
This is functionally identical to:: | ||
len(range(start, stop, step)) | ||
Doesn't assume that stop >= start. Assumes that step >= 1. | ||
""" | ||
# Note that hsize_t is unsigned so stop - start could underflow. | ||
if stop <= start: | ||
return 0 | ||
return (stop - start - 1) // step + 1 | ||
|
||
|
||
@cython.ccall | ||
@cython.nogil | ||
@cython.exceptval(check=False) | ||
def count2stop(start: hsize_t, count: hsize_t, step: hsize_t) -> hsize_t: | ||
"""Inverse of stop2count. | ||
When count == 0 or when step>1, multiple stops can yield the same count. | ||
This function returns the smallest stop >= start. | ||
""" | ||
if count == 0: | ||
return start | ||
return start + (count - 1) * step + 1 | ||
|
||
|
||
@cython.ccall | ||
@cython.nogil | ||
@cython.exceptval(check=False) | ||
def ceil_a_over_b(a: Py_ssize_t, b: Py_ssize_t) -> Py_ssize_t: | ||
"""Returns ceil(a/b). Assumes a >= 0 and b > 0. | ||
Note | ||
---- | ||
This module is compiled with the cython.cdivision flag. This causes behaviour to | ||
change if a and b have opposite signs and you try debugging the module in pure | ||
python, without compiling it. This function blindly assumes that a and b are always | ||
the same sign. | ||
""" | ||
return a // b + (a % b > 0) | ||
|
||
|
||
@cython.ccall | ||
@cython.nogil | ||
@cython.exceptval(check=False) | ||
def smallest_step_after(x: Py_ssize_t, a: Py_ssize_t, m: Py_ssize_t) -> Py_ssize_t: | ||
"""Find the smallest integer y >= x where y = a + k*m for whole k's | ||
Assumes 0 <= a <= x and m >= 1. | ||
a x y | ||
| <-- m --> | <-- m --> | | ||
""" | ||
return a + ceil_a_over_b(x - a, m) * m |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1 @@ | ||
cytools.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
import math | ||
|
||
from hypothesis import given | ||
from hypothesis import strategies as st | ||
|
||
from ..cytools import ceil_a_over_b, count2stop, smallest_step_after, stop2count | ||
|
||
|
||
def free_slices_st(size: int): | ||
"""Hypothesis draw of a slice object to slice an array of up to size elements""" | ||
start_st = st.integers(0, size) | ||
stop_st = st.integers(0, size) | ||
# only non-negative steps are allowed | ||
step_st = st.integers(1, size) | ||
return st.builds(slice, start_st, stop_st, step_st) | ||
|
||
|
||
@given(free_slices_st(5)) | ||
def test_stop2count_count2stop(s): | ||
count = stop2count(s.start, s.stop, s.step) | ||
assert count == len(range(s.start, s.stop, s.step)) | ||
|
||
stop = count2stop(s.start, count, s.step) | ||
# When count == 0 or when step>1, multiple stops yield the same count, | ||
# so stop won't necessarily be equal to s.stop | ||
assert count == len(range(s.start, stop, s.step)) | ||
|
||
|
||
@given(st.integers(0, 10), st.integers(1, 10)) | ||
def test_ceil_a_over_b(a, b): | ||
expect = math.ceil(a / b) | ||
actual = ceil_a_over_b(a, b) | ||
assert actual == expect | ||
|
||
|
||
@given(st.data(), st.integers(0, 10), st.integers(1, 4)) | ||
def test_smallest_step_after(data, x, m): | ||
a = data.draw(st.integers(0, x)) | ||
expect = a | ||
while expect < x: | ||
expect += m | ||
actual = smallest_step_after(x, a, m) | ||
assert actual == expect |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters