From 68d3d73d95c7d7fef3324f627d15cbacfd18e759 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Mon, 12 Feb 2024 15:58:24 +0200 Subject: [PATCH 1/4] refactor deserializer + tests --- osiris/cairo/serde/deserialize.py | 31 ++++++++ tests/test_deserialize2.py | 115 ++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 tests/test_deserialize2.py diff --git a/osiris/cairo/serde/deserialize.py b/osiris/cairo/serde/deserialize.py index f54c21b..2bb58c1 100644 --- a/osiris/cairo/serde/deserialize.py +++ b/osiris/cairo/serde/deserialize.py @@ -1,10 +1,41 @@ import json +import re import numpy as np from .utils import felt_to_int, from_fp + +def deserializer2(serialized: str, dtype: int): + serialized = convert_data(serialized) + + if dtype in ("u8", "u16", "u32", "u64", "u128", "i8", "i16", "i32", "i64", "i128"): + return deserialize_int(serialized) + elif dtype == "FP16x16": + return deserialize_fixed_point(serialized, "FP16x16") + elif dtype == "FP8x23": + return deserialize_fixed_point(serialized, "FP8x23") + elif dtype == "FP32x32": + return deserialize_fixed_point(serialized, "FP32x32") + elif dtype == "FP64x64": + return deserialize_fixed_point(serialized, "FP64x64") + elif dtype.startswith("Span<") and dtype.endswith(">"): + # Extract the inner type from the Span + inner_type = dtype[5:-1] + if inner_type.startswith("FP"): + return deserialize_arr_fixed_point(serialized, inner_type) + else: + return deserialize_arr_int(serialized) + elif dtype.startswith("Tensor<") and dtype.endswith(">"): + # Extract the inner type from the Tensor + inner_type = dtype[7:-1] + if inner_type.startswith("FP"): + return deserialize_tensor_fixed_point(serialized) + else: + return deserialize_tensor_int(serialized) + pass + def deserializer(serialized: str, data_type: str, fp_impl='FP16x16'): """ Main deserialization function that handles various data types. diff --git a/tests/test_deserialize2.py b/tests/test_deserialize2.py new file mode 100644 index 0000000..0ffbeac --- /dev/null +++ b/tests/test_deserialize2.py @@ -0,0 +1,115 @@ +import numpy as np +import pytest +from math import isclose + +from osiris.cairo.serde.deserialize import * + + +def test_deserialize_int(): + serialized = '[{"Int":"2A"}]' + deserialized = deserializer2(serialized, 'u32') + assert deserialized == 42 + + serialized = '[{"Int":"800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]' + deserialized = deserializer2(serialized, 'i32') + assert deserialized == -42 + + +def test_deserialize_fp(): + serialized = '[{"Int":"2A6B85"}, {"Int":"0"}]' + deserialized = deserializer2(serialized, 'FP16x16') + assert isclose(deserialized, 42.42, rel_tol=1e-7) + + serialized = '[{"Int":"2A6B85"}, {"Int":"1"}]' + deserialized = deserializer2(serialized, 'FP16x16') + assert isclose(deserialized, -42.42, rel_tol=1e-7) + + +def test_deserialize_array_int(): + serialized = '[{"Array": [{"Int": "0x1"}, {"Int": "0x2"}]}]' + deserialized = deserializer2(serialized, 'Span') + assert np.array_equal(deserialized, np.array([1, 2], dtype=np.int64)) + + serialized = '[{"Array": [{"Int": "2A"}, {"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]}]' + deserialized = deserializer2(serialized, 'Span') + assert np.array_equal(deserialized, np.array([42, -42], dtype=np.int64)) + + +def test_deserialize_arr_fixed_point(): + serialized = '[{"Array": [{"Int": "2A6B85"}, {"Int": "0"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' + deserialized = deserializer2(serialized, 'Span') + expected = np.array([42.42, -42.42], dtype=np.float64) + assert np.all(np.isclose(deserialized, expected, atol=1e-7)) + + +def test_deserialize_tensor_int(): + serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "0x1"}, {"Int": "0x2"}, {"Int": "0x3"}, {"Int": "0x4"}]}]' + deserialized = deserializer2(serialized, 'Tensor') + assert np.array_equal(deserialized, np.array( + ([1, 2], [3, 4]), dtype=np.int64)) + + serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A"}, {"Int": "2A"},{"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}, {"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]}]' + deserialized = deserializer2(serialized, 'Tensor') + assert np.array_equal(deserialized, np.array([[42, 42], [-42, -42]])) + + +def test_deserialize_tensor_fixed_point(): + serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x1"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' + expected_array = np.array([[42.42, 42.42], [-42.42, -42.42]]) + deserialized = deserializer2(serialized, 'Tensor') + assert np.allclose(deserialized, expected_array, atol=1e-7) + +# def test_deserialize_tuple_int(): +# serialized = '[{"Int":"0x1"},{"Int":"0x3"}]' +# deserialized = deserializer2(serialized, '(u32, u32)') +# assert deserialized == (1,3) + +# def test_deserialize_tuple_uint(): +# serialized = [1, 2] +# deserialized = deserialize_tuple_uint(serialized) +# assert np.array_equal(deserialized, np.array([1, 2], dtype=np.int64)) + + +# def test_deserialize_tuple_signed_int(): +# serialized = [42, 0, 42, 1, 42, 0] +# deserialized = deserialize_tuple_signed_int(serialized) +# assert np.array_equal(deserialized, np.array( +# [42, -42, 42], dtype=np.int64)) + + +# def test_deserialize_tuple_fixed_point(): +# serialized = [2780037, 0, 2780037, 1, 2780037, 0] +# deserialized = deserialize_tuple_fixed_point(serialized) +# expected = np.array([42.42, -42.42, 42.42], dtype=np.float64) +# assert np.all(np.isclose(deserialized, expected, atol=1e-7)) + +# def test_deserialize_tensor_tuple_tensor_uint(): +# serialized = [2, 2, 2, 4, 1, 2, 3, 4, 2, 2, 2, 4, 5, 6, 7, 8] +# deserialized = deserialize_tuple_tensor_uint(serialized) + +# assert np.array_equal(deserialized[0], np.array( +# [[1, 2], [3, 4]], dtype=np.int64)) +# assert np.array_equal(deserialized[1], np.array( +# [[5, 6], [7, 8]], dtype=np.int64)) + + +# def test_deserialize_tensor_tuple_tensor_signed_int(): +# serialized = [2, 2, 2, 8, 42, +# 0, 42, 0, 42, 1, 42, 1, 2, 2, 2, 8, 42, +# 0, 42, 0, 42, 1, 42, 1] +# deserialized = deserialize_tuple_tensor_signed_int(serialized) + +# expected_array = np.array([[42, 42], [-42, -42]]) +# assert np.allclose(deserialized[0], expected_array, atol=1e-7) +# assert np.allclose(deserialized[1], expected_array, atol=1e-7) + + +# def test_deserialize_tensor_tuple_tensor_fixed_point(): +# serialized = [2, 2, 2, 8, 2780037, +# 0, 2780037, 0, 2780037, 1, 2780037, 1, 2, 2, 2, 8, 2780037, +# 0, 2780037, 0, 2780037, 1, 2780037, 1] +# deserialized = deserialize_tuple_tensor_fixed_point(serialized) + +# expected_array = np.array([[42.42, 42.42], [-42.42, -42.42]]) +# assert np.allclose(deserialized[0], expected_array, atol=1e-7) +# assert np.allclose(deserialized[1], expected_array, atol=1e-7) From b444ae97050f399c4f1a81fd6a665d6707893517 Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Tue, 13 Feb 2024 15:02:50 +0200 Subject: [PATCH 2/4] implement tuple --- osiris/cairo/serde/deserialize.py | 155 ++++++++---------------------- tests/test_deserialize2.py | 27 ++++-- 2 files changed, 61 insertions(+), 121 deletions(-) diff --git a/osiris/cairo/serde/deserialize.py b/osiris/cairo/serde/deserialize.py index 2bb58c1..5b9c073 100644 --- a/osiris/cairo/serde/deserialize.py +++ b/osiris/cairo/serde/deserialize.py @@ -7,34 +7,49 @@ -def deserializer2(serialized: str, dtype: int): - serialized = convert_data(serialized) - - if dtype in ("u8", "u16", "u32", "u64", "u128", "i8", "i16", "i32", "i64", "i128"): - return deserialize_int(serialized) - elif dtype == "FP16x16": - return deserialize_fixed_point(serialized, "FP16x16") - elif dtype == "FP8x23": - return deserialize_fixed_point(serialized, "FP8x23") - elif dtype == "FP32x32": - return deserialize_fixed_point(serialized, "FP32x32") - elif dtype == "FP64x64": - return deserialize_fixed_point(serialized, "FP64x64") - elif dtype.startswith("Span<") and dtype.endswith(">"): - # Extract the inner type from the Span - inner_type = dtype[5:-1] - if inner_type.startswith("FP"): - return deserialize_arr_fixed_point(serialized, inner_type) +def deserializer2(serialized: str, dtype: str): + # Check if the serialized data is a string and needs conversion + if isinstance(serialized, str): + serialized = convert_data(serialized) + + # Function to deserialize individual elements within a tuple + def deserialize_element(element, element_type): + if element_type in ("u8", "u16", "u32", "u64", "u128", "i8", "i16", "i32", "i64", "i128"): + return deserialize_int(element) + elif element_type.startswith("FP"): + return deserialize_fixed_point(element, element_type) + elif element_type.startswith("Span<") and element_type.endswith(">"): + inner_type = element_type[5:-1] + if inner_type.startswith("FP"): + return deserialize_arr_fixed_point(element, inner_type) + else: + return deserialize_arr_int(element) + elif element_type.startswith("Tensor<") and element_type.endswith(">"): + inner_type = element_type[7:-1] + if inner_type.startswith("FP"): + return deserialize_tensor_fixed_point(element, inner_type) + else: + return deserialize_tensor_int(element) + elif element_type.startswith("(") and element_type.endswith(")"): + return deserializer2(element, element_type) # Recursive call for nested tuples else: - return deserialize_arr_int(serialized) - elif dtype.startswith("Tensor<") and dtype.endswith(">"): - # Extract the inner type from the Tensor - inner_type = dtype[7:-1] - if inner_type.startswith("FP"): - return deserialize_tensor_fixed_point(serialized) - else: - return deserialize_tensor_int(serialized) - pass + raise ValueError(f"Unsupported data type: {element_type}") + + # Handle tuple data type + if dtype.startswith("(") and dtype.endswith(")"): + types = dtype[1:-1].split(", ") + if len(serialized) != len(types): + raise ValueError("Serialized data length does not match tuple length") + + # Deserialize each element in the tuple according to its type + deserialized_elements = [] + for i in range(len(serialized)): + ele = serialized[i] + ele_type = types[i] + deserialized_elements.append(deserialize_element([ele], ele_type)) + return tuple(deserialized_elements) + else: + return deserialize_element(serialized, dtype) def deserializer(serialized: str, data_type: str, fp_impl='FP16x16'): """ @@ -182,89 +197,3 @@ def deserialize_tensor_fixed_point(serialized: list, impl='FP16x16') -> np.array return np.array(data, dtype=np.float64).reshape(shape) -# ================= TUPLE UINT ================= - - -# def deserialize_tuple_uint(serialized: list): -# return np.array(serialized[0], dtype=np.int64) - - -# # ================= TUPLE SIGNED INT ================= - - -# def deserialize_tuple_signed_int(serialized: list): -# num_ele = (len(serialized)) // 2 - -# deserialized_array = np.empty(num_ele, dtype=np.int64) - -# for i in range(num_ele): -# deserialized_array[i] = deserialize_signed_int( -# serialized[i*2: 3 + i*2]) - -# return deserialized_array - -# # ================= TUPLE FIXED POINT ================= - - -# def deserialize_tuple_fixed_point(serialized: list, impl='FP16x16'): -# num_ele = (len(serialized)) // 2 - -# deserialized_array = np.empty(num_ele, dtype=np.float64) - -# for i in range(num_ele): -# deserialized_array[i] = deserialize_fixed_point( -# serialized[i*2: 3 + i*2], impl) - -# return deserialized_array - - -# # ================= TUPLE TENSOR UINT ================= - -# def deserialize_tuple_tensor_uint(serialized: list): -# return deserialize_tuple_tensor(serialized, deserialize_arr_uint) - -# # ================= TUPLE TENSOR SIGNED INT ================= - - -# def deserialize_tuple_tensor_signed_int(serialized: list): -# return deserialize_tuple_tensor(serialized, deserialize_arr_signed_int) - -# # ================= TUPLE TENSOR FIXED POINT ================= - - -# def deserialize_tuple_tensor_fixed_point(serialized: list, impl='FP16x16'): -# return deserialize_tuple_tensor(serialized, deserialize_arr_fixed_point, impl) - - -# # ================= HELPERS ================= - - -# def extract_shape(serialized, start_index): -# """ Extracts the shape part of a tensor from a serialized list. """ -# num_shape_elements = serialized[start_index] -# shape = serialized[start_index + 1: start_index + 1 + num_shape_elements] -# return shape, start_index + 1 + num_shape_elements - - -# def extract_data(serialized, start_index, deserialization_func, impl=None): -# """ Extracts and deserializes the data part of a tensor from a serialized list. """ -# num_data_elements = serialized[start_index] -# end_index = start_index + 1 + num_data_elements -# data_serialized = serialized[start_index: end_index] -# if impl: -# data = deserialization_func(data_serialized, impl) -# else: -# data = deserialization_func(data_serialized) -# return data, end_index - - -# def deserialize_tuple_tensor(serialized, deserialization_func, impl=None): -# """ Generic deserialization function for a tuple of tensors. """ -# deserialized_tensors = [] -# i = 0 -# while i < len(serialized): -# shape, i = extract_shape(serialized, i) -# data, i = extract_data(serialized, i, deserialization_func, impl) -# tensor = data.reshape(shape) -# deserialized_tensors.append(tensor) -# return tuple(deserialized_tensors) diff --git a/tests/test_deserialize2.py b/tests/test_deserialize2.py index 0ffbeac..f56a2aa 100644 --- a/tests/test_deserialize2.py +++ b/tests/test_deserialize2.py @@ -1,4 +1,5 @@ import numpy as np +import numpy.testing as npt import pytest from math import isclose @@ -59,15 +60,25 @@ def test_deserialize_tensor_fixed_point(): deserialized = deserializer2(serialized, 'Tensor') assert np.allclose(deserialized, expected_array, atol=1e-7) -# def test_deserialize_tuple_int(): -# serialized = '[{"Int":"0x1"},{"Int":"0x3"}]' -# deserialized = deserializer2(serialized, '(u32, u32)') -# assert deserialized == (1,3) -# def test_deserialize_tuple_uint(): -# serialized = [1, 2] -# deserialized = deserialize_tuple_uint(serialized) -# assert np.array_equal(deserialized, np.array([1, 2], dtype=np.int64)) +def test_deserialize_tuple_int(): + serialized = '[{"Int":"0x1"},{"Int":"0x3"}]' + deserialized = deserializer2(serialized, '(u32, u32)') + assert deserialized == (1, 3) + + +def test_deserialize_tuple_span(): + serialized = '[{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Int":"0x3"}]' + deserialized = deserializer2(serialized, '(Span, u32)') + expected = (np.array([1, 2]), 3) + npt.assert_array_equal(deserialized[0], expected[0]) + assert deserialized[1] == expected[1] + + +def test_deserialize_tuple_span_tensor_fp(): + serialized = '[{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Array":[{"Int":"0x2a0000"},{"Int":"0x0"},{"Int":"0x2a0000"},{"Int":"0x1"}]}]' + deserialized = deserializer2(serialized, '(Span, Tensor)') + # def test_deserialize_tuple_signed_int(): From 1ab270b52ed1982eac04ae0c8c01aa6dd26ddbba Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Tue, 13 Feb 2024 16:14:20 +0200 Subject: [PATCH 3/4] implement tensor tuple deserialization --- osiris/cairo/serde/deserialize.py | 82 +++++++------------ tests/test_deserialize.py | 86 ++++++++------------ tests/test_deserialize2.py | 126 ------------------------------ 3 files changed, 60 insertions(+), 234 deletions(-) delete mode 100644 tests/test_deserialize2.py diff --git a/osiris/cairo/serde/deserialize.py b/osiris/cairo/serde/deserialize.py index 5b9c073..0193e6c 100644 --- a/osiris/cairo/serde/deserialize.py +++ b/osiris/cairo/serde/deserialize.py @@ -6,8 +6,7 @@ from .utils import felt_to_int, from_fp - -def deserializer2(serialized: str, dtype: str): +def deserializer(serialized: str, dtype: str): # Check if the serialized data is a string and needs conversion if isinstance(serialized, str): serialized = convert_data(serialized) @@ -31,67 +30,44 @@ def deserialize_element(element, element_type): else: return deserialize_tensor_int(element) elif element_type.startswith("(") and element_type.endswith(")"): - return deserializer2(element, element_type) # Recursive call for nested tuples + # Recursive call for nested tuples + return deserializer(element, element_type) else: raise ValueError(f"Unsupported data type: {element_type}") # Handle tuple data type if dtype.startswith("(") and dtype.endswith(")"): types = dtype[1:-1].split(", ") - if len(serialized) != len(types): - raise ValueError("Serialized data length does not match tuple length") - - # Deserialize each element in the tuple according to its type deserialized_elements = [] - for i in range(len(serialized)): - ele = serialized[i] - ele_type = types[i] - deserialized_elements.append(deserialize_element([ele], ele_type)) - return tuple(deserialized_elements) - else: - return deserialize_element(serialized, dtype) + i = 0 # Initialize loop counter -def deserializer(serialized: str, data_type: str, fp_impl='FP16x16'): - """ - Main deserialization function that handles various data types. + while i < len(serialized): + ele_type = types[len(deserialized_elements)] - :param serialized: The serialized list of data. - :param data_type: The type of data to deserialize ('uint', 'signed_int', 'fixed_point', etc.). - :param fp_impl: The implementation detail, used for fixed-point deserialization. - :return: The deserialized data. - """ + if ele_type.startswith("Tensor<"): + # For Tensors, take two elements from serialized (shape and data) + ele = serialized[i:i+2] + i += 2 + else: + # For other types, take one element + ele = serialized[i] + i += 1 + + if ele_type.startswith("Tensor<"): + deserialized_elements.append( + deserialize_element(ele, ele_type)) + else: + deserialized_elements.append( + deserialize_element([ele], ele_type)) + + if len(deserialized_elements) != len(types): + raise ValueError( + "Serialized data length does not match tuple length") + + return tuple(deserialized_elements) - serialized = convert_data(serialized) - - if data_type == 'int': - return deserialize_int(serialized) - elif data_type == 'fixed_point': - return deserialize_fixed_point(serialized, fp_impl) - elif data_type == 'arr_int': - return deserialize_arr_int(serialized) - elif data_type == 'arr_fixed_point': - return deserialize_arr_fixed_point(serialized, fp_impl) - elif data_type == 'tensor_int': - return deserialize_tensor_int(serialized) - elif data_type == 'tensor_fixed_point': - return deserialize_tensor_fixed_point(serialized) - # TODO: Support Tuples - # elif data_type == 'tensor_fixed_point': - # return deserialize_tensor_fixed_point(serialized, fp_impl) - # elif data_type == 'tuple_uint': - # return deserialize_tuple_uint(serialized) - # elif data_type == 'tuple_signed_int': - # return deserialize_tuple_signed_int(serialized) - # elif data_type == 'tuple_fixed_point': - # return deserialize_tuple_fixed_point(serialized, fp_impl) - # elif data_type == 'tuple_tensor_uint': - # return deserialize_tuple_tensor_uint(serialized) - # elif data_type == 'tuple_tensor_signed_int': - # return deserialize_tuple_tensor_signed_int(serialized) - # elif data_type == 'tuple_tensor_fixed_point': - # return deserialize_tuple_tensor_fixed_point(serialized, fp_impl) else: - raise ValueError(f"Unknown data type: {data_type}") + return deserialize_element(serialized, dtype) def parse_return_value(return_value): @@ -195,5 +171,3 @@ def deserialize_tensor_fixed_point(serialized: list, impl='FP16x16') -> np.array data = deserialize_arr_fixed_point([serialized[1]], impl) return np.array(data, dtype=np.float64).reshape(shape) - - diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py index a867ef9..e4f14c5 100644 --- a/tests/test_deserialize.py +++ b/tests/test_deserialize.py @@ -1,4 +1,5 @@ import numpy as np +import numpy.testing as npt import pytest from math import isclose @@ -7,105 +8,82 @@ def test_deserialize_int(): serialized = '[{"Int":"2A"}]' - deserialized = deserializer(serialized, 'int') + deserialized = deserializer(serialized, 'u32') assert deserialized == 42 serialized = '[{"Int":"800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]' - deserialized = deserializer(serialized, 'int') + deserialized = deserializer(serialized, 'i32') assert deserialized == -42 def test_deserialize_fp(): serialized = '[{"Int":"2A6B85"}, {"Int":"0"}]' - deserialized = deserializer(serialized, 'fixed_point', 'FP16x16') + deserialized = deserializer(serialized, 'FP16x16') assert isclose(deserialized, 42.42, rel_tol=1e-7) serialized = '[{"Int":"2A6B85"}, {"Int":"1"}]' - deserialized = deserializer(serialized, 'fixed_point', 'FP16x16') + deserialized = deserializer(serialized, 'FP16x16') assert isclose(deserialized, -42.42, rel_tol=1e-7) def test_deserialize_array_int(): serialized = '[{"Array": [{"Int": "0x1"}, {"Int": "0x2"}]}]' - deserialized = deserializer(serialized, 'arr_int') + deserialized = deserializer(serialized, 'Span') assert np.array_equal(deserialized, np.array([1, 2], dtype=np.int64)) serialized = '[{"Array": [{"Int": "2A"}, {"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]}]' - deserialized = deserializer(serialized, 'arr_int') + deserialized = deserializer(serialized, 'Span') assert np.array_equal(deserialized, np.array([42, -42], dtype=np.int64)) def test_deserialize_arr_fixed_point(): serialized = '[{"Array": [{"Int": "2A6B85"}, {"Int": "0"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' - deserialized = deserializer(serialized, 'arr_fixed_point') + deserialized = deserializer(serialized, 'Span') expected = np.array([42.42, -42.42], dtype=np.float64) assert np.all(np.isclose(deserialized, expected, atol=1e-7)) def test_deserialize_tensor_int(): serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "0x1"}, {"Int": "0x2"}, {"Int": "0x3"}, {"Int": "0x4"}]}]' - deserialized = deserializer(serialized, 'tensor_int') + deserialized = deserializer(serialized, 'Tensor') assert np.array_equal(deserialized, np.array( ([1, 2], [3, 4]), dtype=np.int64)) serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A"}, {"Int": "2A"},{"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}, {"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]}]' - deserialized = deserializer(serialized, 'tensor_int') + deserialized = deserializer(serialized, 'Tensor') assert np.array_equal(deserialized, np.array([[42, 42], [-42, -42]])) def test_deserialize_tensor_fixed_point(): serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x1"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' expected_array = np.array([[42.42, 42.42], [-42.42, -42.42]]) - deserialized = deserializer(serialized, 'tensor_fixed_point') + deserialized = deserializer(serialized, 'Tensor') assert np.allclose(deserialized, expected_array, atol=1e-7) -# def test_deserialize_tuple_uint(): -# serialized = [1, 2] -# deserialized = deserialize_tuple_uint(serialized) -# assert np.array_equal(deserialized, np.array([1, 2], dtype=np.int64)) +def test_deserialize_tuple_int(): + serialized = '[{"Int":"0x1"},{"Int":"0x3"}]' + deserialized = deserializer(serialized, '(u32, u32)') + assert deserialized == (1, 3) -# def test_deserialize_tuple_signed_int(): -# serialized = [42, 0, 42, 1, 42, 0] -# deserialized = deserialize_tuple_signed_int(serialized) -# assert np.array_equal(deserialized, np.array( -# [42, -42, 42], dtype=np.int64)) +def test_deserialize_tuple_span(): + serialized = '[{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Int":"0x3"}]' + deserialized = deserializer(serialized, '(Span, u32)') + expected = (np.array([1, 2]), 3) + npt.assert_array_equal(deserialized[0], expected[0]) + assert deserialized[1] == expected[1] -# def test_deserialize_tuple_fixed_point(): -# serialized = [2780037, 0, 2780037, 1, 2780037, 0] -# deserialized = deserialize_tuple_fixed_point(serialized) -# expected = np.array([42.42, -42.42, 42.42], dtype=np.float64) -# assert np.all(np.isclose(deserialized, expected, atol=1e-7)) +def test_deserialize_tuple_span_tensor_fp(): + serialized = '[{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x1"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' + deserialized = deserializer(serialized, '(Span, Tensor)') + expected = (np.array([1, 2]), np.array([[42.42, 42.42], [-42.42, -42.42]])) + npt.assert_array_equal(deserialized[0], expected[0]) + assert np.allclose(deserialized[1], expected[1], atol=1e-7) -# def test_deserialize_tensor_tuple_tensor_uint(): -# serialized = [2, 2, 2, 4, 1, 2, 3, 4, 2, 2, 2, 4, 5, 6, 7, 8] -# deserialized = deserialize_tuple_tensor_uint(serialized) - -# assert np.array_equal(deserialized[0], np.array( -# [[1, 2], [3, 4]], dtype=np.int64)) -# assert np.array_equal(deserialized[1], np.array( -# [[5, 6], [7, 8]], dtype=np.int64)) - - -# def test_deserialize_tensor_tuple_tensor_signed_int(): -# serialized = [2, 2, 2, 8, 42, -# 0, 42, 0, 42, 1, 42, 1, 2, 2, 2, 8, 42, -# 0, 42, 0, 42, 1, 42, 1] -# deserialized = deserialize_tuple_tensor_signed_int(serialized) - -# expected_array = np.array([[42, 42], [-42, -42]]) -# assert np.allclose(deserialized[0], expected_array, atol=1e-7) -# assert np.allclose(deserialized[1], expected_array, atol=1e-7) - - -# def test_deserialize_tensor_tuple_tensor_fixed_point(): -# serialized = [2, 2, 2, 8, 2780037, -# 0, 2780037, 0, 2780037, 1, 2780037, 1, 2, 2, 2, 8, 2780037, -# 0, 2780037, 0, 2780037, 1, 2780037, 1] -# deserialized = deserialize_tuple_tensor_fixed_point(serialized) - -# expected_array = np.array([[42.42, 42.42], [-42.42, -42.42]]) -# assert np.allclose(deserialized[0], expected_array, atol=1e-7) -# assert np.allclose(deserialized[1], expected_array, atol=1e-7) + serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x1"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}, {"Array":[{"Int":"0x1"},{"Int":"0x2"}]}]' + deserialized = deserializer(serialized, '(Tensor, Span)') + expected = (np.array([[42.42, 42.42], [-42.42, -42.42]]), np.array([1, 2])) + assert np.allclose(deserialized[0], expected[0], atol=1e-7) + npt.assert_array_equal(deserialized[1], expected[1]) diff --git a/tests/test_deserialize2.py b/tests/test_deserialize2.py deleted file mode 100644 index f56a2aa..0000000 --- a/tests/test_deserialize2.py +++ /dev/null @@ -1,126 +0,0 @@ -import numpy as np -import numpy.testing as npt -import pytest -from math import isclose - -from osiris.cairo.serde.deserialize import * - - -def test_deserialize_int(): - serialized = '[{"Int":"2A"}]' - deserialized = deserializer2(serialized, 'u32') - assert deserialized == 42 - - serialized = '[{"Int":"800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]' - deserialized = deserializer2(serialized, 'i32') - assert deserialized == -42 - - -def test_deserialize_fp(): - serialized = '[{"Int":"2A6B85"}, {"Int":"0"}]' - deserialized = deserializer2(serialized, 'FP16x16') - assert isclose(deserialized, 42.42, rel_tol=1e-7) - - serialized = '[{"Int":"2A6B85"}, {"Int":"1"}]' - deserialized = deserializer2(serialized, 'FP16x16') - assert isclose(deserialized, -42.42, rel_tol=1e-7) - - -def test_deserialize_array_int(): - serialized = '[{"Array": [{"Int": "0x1"}, {"Int": "0x2"}]}]' - deserialized = deserializer2(serialized, 'Span') - assert np.array_equal(deserialized, np.array([1, 2], dtype=np.int64)) - - serialized = '[{"Array": [{"Int": "2A"}, {"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]}]' - deserialized = deserializer2(serialized, 'Span') - assert np.array_equal(deserialized, np.array([42, -42], dtype=np.int64)) - - -def test_deserialize_arr_fixed_point(): - serialized = '[{"Array": [{"Int": "2A6B85"}, {"Int": "0"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' - deserialized = deserializer2(serialized, 'Span') - expected = np.array([42.42, -42.42], dtype=np.float64) - assert np.all(np.isclose(deserialized, expected, atol=1e-7)) - - -def test_deserialize_tensor_int(): - serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "0x1"}, {"Int": "0x2"}, {"Int": "0x3"}, {"Int": "0x4"}]}]' - deserialized = deserializer2(serialized, 'Tensor') - assert np.array_equal(deserialized, np.array( - ([1, 2], [3, 4]), dtype=np.int64)) - - serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A"}, {"Int": "2A"},{"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}, {"Int": "800000000000010FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD7"}]}]' - deserialized = deserializer2(serialized, 'Tensor') - assert np.array_equal(deserialized, np.array([[42, 42], [-42, -42]])) - - -def test_deserialize_tensor_fixed_point(): - serialized = '[{"Array": [{"Int": "0x2"}, {"Int": "0x2"}]}, {"Array": [{"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x0"}, {"Int": "2A6B85"}, {"Int": "0x1"}, {"Int": "2A6B85"}, {"Int": "0x1"}]}]' - expected_array = np.array([[42.42, 42.42], [-42.42, -42.42]]) - deserialized = deserializer2(serialized, 'Tensor') - assert np.allclose(deserialized, expected_array, atol=1e-7) - - -def test_deserialize_tuple_int(): - serialized = '[{"Int":"0x1"},{"Int":"0x3"}]' - deserialized = deserializer2(serialized, '(u32, u32)') - assert deserialized == (1, 3) - - -def test_deserialize_tuple_span(): - serialized = '[{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Int":"0x3"}]' - deserialized = deserializer2(serialized, '(Span, u32)') - expected = (np.array([1, 2]), 3) - npt.assert_array_equal(deserialized[0], expected[0]) - assert deserialized[1] == expected[1] - - -def test_deserialize_tuple_span_tensor_fp(): - serialized = '[{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Array":[{"Int":"0x1"},{"Int":"0x2"}]},{"Array":[{"Int":"0x2a0000"},{"Int":"0x0"},{"Int":"0x2a0000"},{"Int":"0x1"}]}]' - deserialized = deserializer2(serialized, '(Span, Tensor)') - - - -# def test_deserialize_tuple_signed_int(): -# serialized = [42, 0, 42, 1, 42, 0] -# deserialized = deserialize_tuple_signed_int(serialized) -# assert np.array_equal(deserialized, np.array( -# [42, -42, 42], dtype=np.int64)) - - -# def test_deserialize_tuple_fixed_point(): -# serialized = [2780037, 0, 2780037, 1, 2780037, 0] -# deserialized = deserialize_tuple_fixed_point(serialized) -# expected = np.array([42.42, -42.42, 42.42], dtype=np.float64) -# assert np.all(np.isclose(deserialized, expected, atol=1e-7)) - -# def test_deserialize_tensor_tuple_tensor_uint(): -# serialized = [2, 2, 2, 4, 1, 2, 3, 4, 2, 2, 2, 4, 5, 6, 7, 8] -# deserialized = deserialize_tuple_tensor_uint(serialized) - -# assert np.array_equal(deserialized[0], np.array( -# [[1, 2], [3, 4]], dtype=np.int64)) -# assert np.array_equal(deserialized[1], np.array( -# [[5, 6], [7, 8]], dtype=np.int64)) - - -# def test_deserialize_tensor_tuple_tensor_signed_int(): -# serialized = [2, 2, 2, 8, 42, -# 0, 42, 0, 42, 1, 42, 1, 2, 2, 2, 8, 42, -# 0, 42, 0, 42, 1, 42, 1] -# deserialized = deserialize_tuple_tensor_signed_int(serialized) - -# expected_array = np.array([[42, 42], [-42, -42]]) -# assert np.allclose(deserialized[0], expected_array, atol=1e-7) -# assert np.allclose(deserialized[1], expected_array, atol=1e-7) - - -# def test_deserialize_tensor_tuple_tensor_fixed_point(): -# serialized = [2, 2, 2, 8, 2780037, -# 0, 2780037, 0, 2780037, 1, 2780037, 1, 2, 2, 2, 8, 2780037, -# 0, 2780037, 0, 2780037, 1, 2780037, 1] -# deserialized = deserialize_tuple_tensor_fixed_point(serialized) - -# expected_array = np.array([[42.42, 42.42], [-42.42, -42.42]]) -# assert np.allclose(deserialized[0], expected_array, atol=1e-7) -# assert np.allclose(deserialized[1], expected_array, atol=1e-7) From d341ba1548e454a92ae30cf2ddd24a97d863841d Mon Sep 17 00:00:00 2001 From: raphaelDkhn Date: Tue, 13 Feb 2024 16:16:14 +0200 Subject: [PATCH 4/4] bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b4a2672..d3e6d79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "giza-osiris" -version = "0.2.3" +version = "0.2.4" description = "Osiris is a Python library designed for efficient data conversion and management, primarily transforming data into Cairo programs" authors = ["Fran Algaba "] readme = "README.md"