Merge branch 'main' of https://github.com/Point72/csp into ag/expose-…

…engine-shutdown

.github/CODEOWNERS

@@ -2,18 +2,18 @@
 * @robambalu @AdamGlustein @svatasoiu @alexddobkin
 
 # Packaging code
-.bumpversion.cfg @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-conda @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-CMakeLists.txt @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-Makefile @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-MANIFEST.in @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-pyproject.toml @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-setup.py @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-vcpkg.json @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
+.bumpversion.cfg @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+conda @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+CMakeLists.txt @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+Makefile @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+MANIFEST.in @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+pyproject.toml @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+setup.py @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+vcpkg.json @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
 
 # CI/CD
-.github @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
-ci @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine @czgdp1807
+.github @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
+ci @robambalu @AdamGlustein @svatasoiu @alexddobkin @ptomecek @timkpaine
 
 # Administrative
 .github/CODEOWNERS @robambalu

.github/workflows/build.yml

@@ -522,7 +522,9 @@ jobs:
           if: ${{ runner.os == 'Linux' }}
 
         - name: Install wheel (Linux)
-          run: python -m pip install -U *manylinux*.whl --target .
+          run: |
+            python -m pip install -U *manylinux*.whl
+            python -m pip install -U --no-deps *manylinux*.whl --target .
           if: ${{ runner.os == 'Linux' }}
 
         ########
@@ -533,11 +535,15 @@ jobs:
           if: ${{ runner.os == 'macOS' }}
 
         - name: Install wheel (OSX x86)
-          run: python -m pip install -U *x86*.whl --target .
+          run: |
+            python -m pip install -U *x86*.whl
+            python -m pip install -U --no-deps *x86*.whl --target .
           if: ${{ runner.os == 'macOS' && runner.arch == 'X64' }}
 
         - name: Install wheel (OSX arm)
-          run: python -m pip install -U *arm64*.whl --target .
+          run: |
+            python -m pip install -U *arm64*.whl
+            python -m pip install -U --no-deps *arm64*.whl --target .
           if: ${{ runner.os == 'macOS' && runner.arch == 'ARM64' }}
 
         ########
@@ -547,7 +553,9 @@ jobs:
           if: ${{ runner.os == 'Windows' }}
 
         - name: Install wheel (windows)
-          run: python -m pip install -U (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target .
+          run: |
+            python -m pip install -U (Get-ChildItem .\*.whl | Select-Object -Expand FullName)
+            python -m pip install -U --no-deps (Get-ChildItem .\*.whl | Select-Object -Expand FullName) --target .
           if: ${{ runner.os == 'Windows' }}
 
         ##########
@@ -615,7 +623,9 @@ jobs:
             path: dist/
 
         - name: Install sdist
-          run: python -m pip install -U -vvv dist/csp*.tar.gz --target .
+          run: |
+            python -m pip install -U -vvv dist/csp*.tar.gz
+            python -m pip install -U --no-deps -vvv dist/csp*.tar.gz --target .
           env:
             CCACHE_DIR: /home/runner/work/csp/csp/.ccache
             VCPKG_DEFAULT_BINARY_CACHE: /home/runner/vcpkg_cache
@@ -690,7 +700,9 @@ jobs:
             name: csp-dist-${{ runner.os }}-${{ runner.arch }}-${{ matrix.python-version }}
 
         - name: Install wheel
-          run: python -m pip install -U *manylinux*.whl --target .
+          run: |
+            python -m pip install -U *manylinux*.whl
+            python -m pip install -U --no-deps *manylinux*.whl --target .
 
         - name: Install package - ${{ matrix.package }}
           run: python -m pip install -U "${{ matrix.package }}"

CMakeLists.txt

@@ -194,6 +194,7 @@ else()
         -O3 \
         -g0 \
         -Wall \
+        -Werror \
         -Wno-deprecated-declarations \
         -Wno-deprecated \
         ")

conda/dev-environment-unix.yml

@@ -11,6 +11,7 @@ dependencies:
  - codespell>=2.2.6,<2.3
  - compilers
  - cyrus-sasl
+ - deprecated
  - exprtk
  - flex
  - graphviz

conda/dev-environment-win.yml

@@ -10,6 +10,7 @@ dependencies:
  - codespell>=2.2.6,<2.3
  - compilers
  - cyrus-sasl
+ - deprecated
  - exprtk
  - graphviz
  - gtest

cpp/csp/adapters/parquet/ParquetOutputAdapter.cpp

@@ -140,9 +140,10 @@ template< typename A, typename V = typename A::value_type >
 inline std::shared_ptr<::arrow::ArrayBuilder> makeArrayAndAttachToWriter( DialectGenericListWriterInterface::Ptr &listWriterInterface )
 {
     auto&& typedWriter = std::dynamic_pointer_cast<TypedDialectGenericListWriterInterface<V>>( listWriterInterface );
+    auto& listWriterInterfaceRef = *listWriterInterface;
     CSP_TRUE_OR_THROW( typedWriter != nullptr, TypeError,
                        "Expected " << typeid( TypedDialectGenericListWriterInterface<V> ).name() << " " << " got " <<
-                                   typeid( *listWriterInterface ).name() );
+                                   typeid( listWriterInterfaceRef ).name() );
 
     auto res = std::make_shared<A>();
     typedWriter -> setWriteFunction(

cpp/csp/engine/Struct.h

@@ -756,11 +756,15 @@ class Struct
     void decref()
     {
 //Work around GCC12 bug mis-identifying this code as use-after-free
-//#pragma GCC diagnostic push
-//#pragma GCC diagnostic ignored "-Wuse-after-free"
+#ifdef __linux__
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wuse-after-free"
+#endif
         if( --hidden() -> refcount == 0 )
             delete this;
-//#pragma GCC diagnostic pop
+#ifdef __linux__
+#pragma GCC diagnostic pop
+#endif
     }
 
 

cpp/csp/python/PyStructFastList_impl.h

@@ -516,7 +516,6 @@ static PyObject * PyStructFastList_reduce( PyStructFastList<StorageT> * self, Py
 {
     CSP_BEGIN_METHOD;
 
-    typename VectorWrapper<StorageT>::IndexType sz = self -> vector.size();
     PyObjectPtr list = PyObjectPtr::own( toPython( self -> vector.getVector(), self -> arrayType ) );
     PyObject * result = Py_BuildValue( "O(O)", &PyList_Type, list.ptr() );
     return result;

cpp/csp/python/PyStructList_impl.h

@@ -302,7 +302,6 @@ static PyObject * PyStructList_reduce( PyStructList<StorageT> * self, PyObject *
 {
     CSP_BEGIN_METHOD;
 
-    typename VectorWrapper<StorageT>::IndexType sz = self -> vector.size();
     PyObjectPtr list = PyObjectPtr::own( toPython( self -> vector.getVector(), self -> arrayType ) );
     PyObject * result = Py_BuildValue( "O(O)", &PyList_Type, list.ptr() );
     return result;

cpp/csp/python/PyStructToJson.cpp

@@ -72,7 +72,7 @@ inline rapidjson::Value toJson( const TimeDelta& val, const CspType& typ, rapidj
     // Convert TimeDelta to <sign><seconds>.<microseconds>
     // sign( 1 ) + seconds ( 18 ) + '.'( 1 ) + microseconds( 9 ) + '\0'( 1 )
     char buf[32] = {};
-    auto seconds = val.abs().asSeconds();
+    long seconds = val.abs().asSeconds();
     auto microseconds = static_cast<unsigned>( val.abs().nanoseconds() / NANOS_PER_MICROSECOND );
     auto len = sprintf( buf, "%c%ld.%06u", ( val.sign() >= 0 ) ? '+' : '-', seconds, microseconds );
     rapidjson::Value res;

cpp/csp/python/adapters/CMakeLists.txt

@@ -29,6 +29,10 @@ if(CSP_BUILD_PARQUET_ADAPTER)
         ${VENDORED_PYARROW_ROOT}/arrow/python/csv.cc
         ${VENDORED_PYARROW_ROOT}/arrow/python/filesystem.cc)
     add_library(parquetadapterimpl SHARED parquetadapterimpl.cpp ${ARROW_PYTHON_SRCS})
+    # Ignore warning regarding static datetime API initialization coming from vendored Arrow code
+    if(CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+        set_target_properties(parquetadapterimpl PROPERTIES COMPILE_OPTIONS "-Wno-unused-variable")
+    endif()
     target_link_libraries(parquetadapterimpl csp_core csp_engine cspimpl csp_parquet_adapter)
     target_include_directories(parquetadapterimpl PUBLIC ${ARROW_INCLUDE_DIR} ${PARQUET_INCLUDE_DIR} "${VENDORED_PYARROW_ROOT}")
     target_compile_definitions(parquetadapterimpl PUBLIC ARROW_PYTHON_STATIC)

cpp/tests/CMakeLists.txt

@@ -1,2 +1,14 @@
 add_subdirectory(core)
 add_subdirectory(engine)
+
+# Ignore bogus gcc warning which is coming from gtest headers
+# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105651
+if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+    set(SUBDIRS core engine)
+    foreach(SUBDIR ${SUBDIRS})
+        get_directory_property(SUBDIR_TARGETS DIRECTORY ${SUBDIR} BUILDSYSTEM_TARGETS)
+        foreach(TARGET ${SUBDIR_TARGETS})
+            set_target_properties(${TARGET} PROPERTIES COMPILE_OPTIONS "-Wno-restrict")
+        endforeach()
+    endforeach()
+endif()

csp/adapters/db.py

@@ -267,9 +267,9 @@ def schema_struct(self):
             table = db.Table(self._table_name, db_metadata, autoload_with=self._connection)
             struct_metadata = {col: col_obj.type.python_type for col, col_obj in table.columns.items()}
 
-            from csp.impl.struct import defineStruct
+            from csp.impl.struct import define_struct
 
-            typ = defineStruct(name, struct_metadata)
+            typ = define_struct(name, struct_metadata)
             globals()[name] = typ
         return globals()[name]
 

csp/build/csp_autogen.py

@@ -498,7 +498,7 @@ class Derived(Test):
     flt: float
 
 
-# Test2 = csp.impl.struct.defineStruct( 'Test2', { 'A' + str(i) : bool for i in range(25 )})
+# Test2 = csp.impl.struct.define_struct( 'Test2', { 'A' + str(i) : bool for i in range(25 )})
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()

csp/impl/pandas_accessor.py

@@ -8,7 +8,7 @@
 import csp
 from csp import ts
 from csp.impl.pandas_ext_type import TsDtype, is_csp_type
-from csp.impl.struct import defineNestedStruct
+from csp.impl.struct import define_nested_struct
 from csp.impl.wiring.edge import Edge
 
 T = TypeVar("T")
@@ -615,7 +615,7 @@ def collect(self, columns=None, struct_type=None, delim=" "):
             datatree[parts[-1]] = self._obj[col]
 
         if not struct_type:
-            struct_type = defineNestedStruct("_C", metadata, defaults)
+            struct_type = define_nested_struct("_C", metadata, defaults)
 
         if not data:
             return csp.null_ts(struct_type)

csp/impl/struct.py

@@ -2,6 +2,7 @@
 import ruamel.yaml
 import typing
 from copy import deepcopy
+from deprecated import deprecated
 
 import csp
 from csp.impl.__csptypesimpl import _csptypesimpl
@@ -226,7 +227,7 @@ def __dir__(self):
         return self.__full_metadata_typed__.keys()
 
 
-def defineStruct(name, metadata: dict, defaults: dict = {}, base=Struct):
+def define_struct(name, metadata: dict, defaults: dict = {}, base=Struct):
     """Helper method to dynamically create struct types"""
 
     dct = deepcopy(defaults)
@@ -235,18 +236,28 @@ def defineStruct(name, metadata: dict, defaults: dict = {}, base=Struct):
     return clazz
 
 
-def defineNestedStruct(name, metadata: dict, defaults: dict = {}, base=Struct):
+def define_nested_struct(name, metadata: dict, defaults: dict = {}, base=Struct):
     """Helper method to dynamically create nested struct types.
     metadata and defaults can be a nested dictionaries"""
     metadata = deepcopy(metadata)
     defaults = deepcopy(defaults)
     child_structs = {
-        field: defineNestedStruct(f"{name}_{field}", submeta, defaults.get(field, {}))
+        field: define_nested_struct(f"{name}_{field}", submeta, defaults.get(field, {}))
         for field, submeta in metadata.items()
         if isinstance(submeta, dict)
     }
     for fld, struct in child_structs.items():
         if fld in defaults:
             defaults[fld] = struct()
     metadata.update(child_structs)
-    return defineStruct(name, metadata, defaults, base)
+    return define_struct(name, metadata, defaults, base)
+
+
+@deprecated(version="0.0.6", reason="Replaced by define_struct")
+def defineStruct(name, metadata: dict, defaults: dict = {}, base=Struct):
+    return define_struct(name, metadata, defaults, base)
+
+
+@deprecated(version="0.0.6", reason="Replaced by define_nested_struct")
+def defineNestedStruct(name, metadata: dict, defaults: dict = {}, base=Struct):
+    return define_nested_struct(name, metadata, defaults, base)

csp/impl/types/instantiation_type_resolver.py

@@ -557,10 +557,7 @@ def _resolve_tvar_container_internal_types(self, tvar, container_typ, arg, raise
             else:
                 return False
         if len(arg) == 0:
-            if raise_on_error:
-                raise ContainerTypeVarResolutionError(self._function_name, tvar, arg)
-            else:
-                return None
+            return container_typ
         res = None
         if isinstance(arg, set):
             first_val = arg.__iter__().__next__()