From fddb0e393ee09965b56912c89320671c1b1b6cd0 Mon Sep 17 00:00:00 2001 From: Phoebus Mak Date: Thu, 14 Sep 2023 12:00:03 +0000 Subject: [PATCH] -Set ca path and directory automatically --- cpp/CMake/AzureVcpkg.cmake | 169 ------------------ cpp/CMakeLists.txt | 4 +- cpp/arcticdb/storage/azure/azure_storage.cpp | 15 ++ cpp/arcticdb/storage/python_bindings.cpp | 3 +- cpp/arcticdb/storage/storage_override.hpp | 10 ++ cpp/proto/arcticc/pb2/azure_storage.proto | 1 + .../azure-core-cpp/capath.patch | 54 ++++++ .../azure-core-cpp/portfile.cmake | 28 +++ .../vcpkg_overlays/azure-core-cpp/vcpkg.json | 77 ++++++++ .../azure-identity-cpp/capath.patch | 54 ++++++ .../azure-identity-cpp/portfile.cmake | 20 +++ .../azure-identity-cpp/vcpkg.json | 26 +++ .../azure-storage-blobs-cpp/capath.patch | 54 ++++++ .../azure-storage-blobs-cpp/portfile.cmake | 20 +++ .../azure-storage-blobs-cpp/vcpkg.json | 25 +++ .../azure-storage-common-cpp/capath.patch | 54 ++++++ .../azure-storage-common-cpp/portfile.cmake | 20 +++ .../azure-storage-common-cpp/vcpkg.json | 33 ++++ .../adapters/azure_library_adapter.py | 19 +- python/arcticdb/version_store/helper.py | 4 + .../tests/integration/arcticdb/test_arctic.py | 15 ++ python/tests/scripts/test_update_storage.py | 1 + 22 files changed, 532 insertions(+), 174 deletions(-) delete mode 100644 cpp/CMake/AzureVcpkg.cmake create mode 100644 cpp/third_party/vcpkg_overlays/azure-core-cpp/capath.patch create mode 100644 cpp/third_party/vcpkg_overlays/azure-core-cpp/portfile.cmake create mode 100644 cpp/third_party/vcpkg_overlays/azure-core-cpp/vcpkg.json create mode 100644 cpp/third_party/vcpkg_overlays/azure-identity-cpp/capath.patch create mode 100644 cpp/third_party/vcpkg_overlays/azure-identity-cpp/portfile.cmake create mode 100644 cpp/third_party/vcpkg_overlays/azure-identity-cpp/vcpkg.json create mode 100644 cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/capath.patch create mode 100644 cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/portfile.cmake create mode 100644 cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/vcpkg.json create mode 100644 cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/capath.patch create mode 100644 cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/portfile.cmake create mode 100644 cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/vcpkg.json diff --git a/cpp/CMake/AzureVcpkg.cmake b/cpp/CMake/AzureVcpkg.cmake deleted file mode 100644 index 15388bda2b..0000000000 --- a/cpp/CMake/AzureVcpkg.cmake +++ /dev/null @@ -1,169 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# SPDX-License-Identifier: MIT - -# We need to know an absolute path to our repo root to do things like referencing ./LICENSE.txt file. -set(AZ_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/..") - -macro(az_vcpkg_integrate) - message("Vcpkg integrate step.") - # AUTO CMAKE_TOOLCHAIN_FILE: - # User can call `cmake -DCMAKE_TOOLCHAIN_FILE="path_to_the_toolchain"` as the most specific scenario. - # As the last alternative (default case), Azure SDK will automatically clone VCPKG folder and set toolchain from there. - if(NOT DEFINED CMAKE_TOOLCHAIN_FILE) - message("CMAKE_TOOLCHAIN_FILE is not defined. Define it for the user.") - # Set AZURE_SDK_DISABLE_AUTO_VCPKG env var to avoid Azure SDK from cloning and setting VCPKG automatically - # This option delegate package's dependencies installation to user. - if(NOT DEFINED ENV{AZURE_SDK_DISABLE_AUTO_VCPKG}) - message("AZURE_SDK_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.") - # GET VCPKG FROM SOURCE - # User can set env var AZURE_SDK_VCPKG_COMMIT to pick the VCPKG commit to fetch - set(VCPKG_COMMIT_STRING 94ce0dab56f4d8ba6bd631ba59ed682b02d45c46) # default SDK tested commit - if(DEFINED ENV{AZURE_SDK_VCPKG_COMMIT}) - message("AZURE_SDK_VCPKG_COMMIT is defined. Using that instead of the default.") - set(VCPKG_COMMIT_STRING "$ENV{AZURE_SDK_VCPKG_COMMIT}") # default SDK tested commit - endif() - message("Vcpkg commit string used: ${VCPKG_COMMIT_STRING}") - include(FetchContent) - FetchContent_Declare( - vcpkg - GIT_REPOSITORY https://github.com/microsoft/vcpkg.git - GIT_TAG ${VCPKG_COMMIT_STRING} - ) - FetchContent_GetProperties(vcpkg) - # make sure to pull vcpkg only once. - if(NOT vcpkg_POPULATED) - FetchContent_Populate(vcpkg) - endif() - # use the vcpkg source path - set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "") - endif() - endif() - - # enable triplet customization - if(DEFINED ENV{VCPKG_DEFAULT_TRIPLET} AND NOT DEFINED VCPKG_TARGET_TRIPLET) - set(VCPKG_TARGET_TRIPLET "$ENV{VCPKG_DEFAULT_TRIPLET}" CACHE STRING "") - endif() -endmacro() - -macro(az_vcpkg_portfile_prep targetName fileName contentToRemove) - # with sdk//vcpkg/ - file(READ "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/${fileName}" fileContents) - - # Windows -> Unix line endings - string(FIND fileContents "\r\n" crLfPos) - - if (crLfPos GREATER -1) - string(REPLACE "\r\n" "\n" fileContents ${fileContents}) - endif() - - # remove comment header - string(REPLACE "${contentToRemove}" "" fileContents ${fileContents}) - - # undo Windows -> Unix line endings (if applicable) - if (crLfPos GREATER -1) - string(REPLACE "\n" "\r\n" fileContents ${fileContents}) - endif() - unset(crLfPos) - - # output to an intermediate location - file (WRITE "${CMAKE_BINARY_DIR}/vcpkg_prep/${targetName}/${fileName}" ${fileContents}) - unset(fileContents) - - # Produce the files to help with the vcpkg release. - # Go to the /out/build//vcpkg directory, and copy (merge) "ports" folder to the vcpkg repo. - # Then, update the portfile.cmake file SHA512 from "1" to the actual hash (a good way to do it is to uninstall a package, - # clean vcpkg/downloads, vcpkg/buildtrees, run "vcpkg install ", and get the SHA from the error message). - configure_file( - "${CMAKE_BINARY_DIR}/vcpkg_prep/${targetName}/${fileName}" - "${CMAKE_BINARY_DIR}/vcpkg/ports/${targetName}-cpp/${fileName}" - @ONLY - ) -endmacro() - -macro(az_vcpkg_export targetName macroNamePart dllImportExportHeaderPath) - foreach(vcpkgFile "vcpkg.json" "portfile.cmake") - az_vcpkg_portfile_prep( - "${targetName}" - "${vcpkgFile}" - "# Copyright (c) Microsoft Corporation. All rights reserved.\n# SPDX-License-Identifier: MIT\n\n" - ) - endforeach() - - # Standard names for folders such as "bin", "lib", "include". We could hardcode, but some other libs use it too (curl). - include(GNUInstallDirs) - - # When installing, copy our "inc" directory (headers) to "include" directory at the install location. - install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/inc/azure/" DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/azure") - - # Copy license as "copyright" (vcpkg dictates naming and location). - install(FILES "${AZ_ROOT_DIR}/LICENSE.txt" DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/${targetName}-cpp" RENAME "copyright") - - # Indicate where to install targets. Mirrors what other ports do. - install( - TARGETS "${targetName}" - EXPORT "${targetName}-cppTargets" - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} # DLLs (if produced by build) go to "/bin" - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} # static .lib files - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} # .lib files for DLL build - INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} # headers - ) - - # If building a Windows DLL, patch the dll_import_export.hpp - if(WIN32 AND BUILD_SHARED_LIBS) - add_compile_definitions(AZ_${macroNamePart}_BEING_BUILT) - target_compile_definitions(${targetName} PUBLIC AZ_${macroNamePart}_DLL) - - set(AZ_${macroNamePart}_DLL_INSTALLED_AS_PACKAGE "*/ + 1 /*") - configure_file( - "${CMAKE_CURRENT_SOURCE_DIR}/inc/${dllImportExportHeaderPath}" - "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${dllImportExportHeaderPath}" - @ONLY - ) - unset(AZ_${macroNamePart}_DLL_INSTALLED_AS_PACKAGE) - - get_filename_component(dllImportExportHeaderDir ${dllImportExportHeaderPath} DIRECTORY) - install( - FILES "${CMAKE_BINARY_DIR}/${CMAKE_INSTALL_INCLUDEDIR}/${dllImportExportHeaderPath}" - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/${dllImportExportHeaderDir}" - ) - unset(dllImportExportHeaderDir) - endif() - - # Export the targets file itself. - install( - EXPORT "${targetName}-cppTargets" - DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/${targetName}-cpp" - NAMESPACE Azure:: # Not the C++ namespace, but a namespace in terms of cmake. - FILE "${targetName}-cppTargets.cmake" - ) - - # configure_package_config_file(), write_basic_package_version_file() - include(CMakePackageConfigHelpers) - - # Produce package config file. - configure_package_config_file( - "${CMAKE_CURRENT_SOURCE_DIR}/vcpkg/Config.cmake.in" - "${targetName}-cppConfig.cmake" - INSTALL_DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/${targetName}-cpp" - PATH_VARS - CMAKE_INSTALL_LIBDIR) - - # Produce version file. - write_basic_package_version_file( - "${targetName}-cppConfigVersion.cmake" - VERSION ${AZ_LIBRARY_VERSION} # the version that we extracted from package_version.hpp - COMPATIBILITY SameMajorVersion - ) - - # Install package config and version files. - install( - FILES - "${CMAKE_CURRENT_BINARY_DIR}/${targetName}-cppConfig.cmake" - "${CMAKE_CURRENT_BINARY_DIR}/${targetName}-cppConfigVersion.cmake" - DESTINATION - "${CMAKE_INSTALL_DATAROOTDIR}/${targetName}-cpp" # to shares/ - ) - - # Export all the installs above as package. - export(PACKAGE "${targetName}-cpp") -endmacro() \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index be04efd6f4..d97a9b7dc5 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -100,9 +100,7 @@ include(PythonUtils) # Must be called before Pybind (third_party) to override it add_subdirectory(third_party) add_subdirectory(proto) -if(NOT ${ARCTICDB_USING_CONDA}) - include(AzureVcpkg) #AzureVcpkg.cmake is from https://github.com/Azure/azure-sdk-for-cpp/blob/main/cmake-modules/AzureVcpkg.cmake, commit ada77b3 -endif() + python_utils_dump_vars_if_enabled("After Pybind") python_utils_check_include_dirs("accepted by pybind") diff --git a/cpp/arcticdb/storage/azure/azure_storage.cpp b/cpp/arcticdb/storage/azure/azure_storage.cpp index a448a902ac..200c1875d4 100644 --- a/cpp/arcticdb/storage/azure/azure_storage.cpp +++ b/cpp/arcticdb/storage/azure/azure_storage.cpp @@ -353,6 +353,10 @@ AzureStorage::AzureStorage(const LibraryPath &library_path, OpenMode mode, const ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using default CA cert path"); else ARCTICDB_RUNTIME_DEBUG(log::storage(), "CA cert path: {}", conf.ca_cert_path()); + if (conf.ca_cert_dir().empty()) + ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using default CA cert directory"); + else + ARCTICDB_RUNTIME_DEBUG(log::storage(), "CA cert directory: {}", conf.ca_cert_dir()); ARCTICDB_RUNTIME_DEBUG(log::storage(), "Connecting to Azure Blob Storage: {} Container: {}", conf.endpoint(), conf.container_name()); if (!conf.prefix().empty()) { @@ -366,4 +370,15 @@ AzureStorage::AzureStorage(const LibraryPath &library_path, OpenMode mode, const download_option_.TransferOptions.Concurrency = max_connections; } +Azure::Storage::Blobs::BlobClientOptions AzureStorage::get_client_options(const Config &conf) { + BlobClientOptions client_options; + if (!conf.ca_cert_path().empty() || !conf.ca_cert_dir().empty()) {//WARNING: Setting ca_cert_path will force Azure sdk uses libcurl as backend support, instead of winhttp + Azure::Core::Http::CurlTransportOptions curl_transport_options; + curl_transport_options.CAInfo = conf.ca_cert_path(); + curl_transport_options.CAPath = conf.ca_cert_dir(); + client_options.Transport.Transport = std::make_shared(curl_transport_options); + } + return client_options; +} + } // namespace arcticdb::storage::azure diff --git a/cpp/arcticdb/storage/python_bindings.cpp b/cpp/arcticdb/storage/python_bindings.cpp index c573852554..f4b5ee192e 100644 --- a/cpp/arcticdb/storage/python_bindings.cpp +++ b/cpp/arcticdb/storage/python_bindings.cpp @@ -124,7 +124,8 @@ void register_bindings(py::module& storage) { .def(py::init<>()) .def_property("container_name", &AzureOverride::container_name, &AzureOverride::set_container_name) .def_property("endpoint", &AzureOverride::endpoint, &AzureOverride::set_endpoint) - .def_property("ca_cert_path", &AzureOverride::ca_cert_path, &AzureOverride::set_ca_cert_path); + .def_property("ca_cert_path", &AzureOverride::ca_cert_path, &AzureOverride::set_ca_cert_path) + .def_property("ca_cert_dir", &AzureOverride::ca_cert_dir, &AzureOverride::set_ca_cert_dir); py::class_(storage, "LmdbOverride") .def(py::init<>()) diff --git a/cpp/arcticdb/storage/storage_override.hpp b/cpp/arcticdb/storage/storage_override.hpp index bb4d2ccb92..40f42d2305 100644 --- a/cpp/arcticdb/storage/storage_override.hpp +++ b/cpp/arcticdb/storage/storage_override.hpp @@ -127,6 +127,7 @@ class AzureOverride { std::string container_name_; std::string endpoint_; std::string ca_cert_path_; + std::string ca_cert_dir_; public: std::string container_name() const { @@ -153,6 +154,14 @@ class AzureOverride { ca_cert_path_ = ca_cert_path; } + std::string ca_cert_dir() const { + return ca_cert_dir_; + } + + void set_ca_cert_dir(std::string_view ca_cert_dir){ + ca_cert_dir_ = ca_cert_dir; + } + void modify_storage_config(arcticdb::proto::storage::VariantStorage& storage) const { if(storage.config().Is()) { arcticdb::proto::azure_storage::Config azure_storage; @@ -161,6 +170,7 @@ class AzureOverride { azure_storage.set_container_name(container_name_); azure_storage.set_endpoint(endpoint_); azure_storage.set_ca_cert_path(ca_cert_path_); + azure_storage.set_ca_cert_dir(ca_cert_dir_); util::pack_to_any(azure_storage, *storage.mutable_config()); } diff --git a/cpp/proto/arcticc/pb2/azure_storage.proto b/cpp/proto/arcticc/pb2/azure_storage.proto index 9d47677e03..84abb3ab2f 100644 --- a/cpp/proto/arcticc/pb2/azure_storage.proto +++ b/cpp/proto/arcticc/pb2/azure_storage.proto @@ -18,4 +18,5 @@ message Config { string prefix = 5; string ca_cert_path = 6; bool use_mock_storage_for_testing = 7; + string ca_cert_dir = 8; } diff --git a/cpp/third_party/vcpkg_overlays/azure-core-cpp/capath.patch b/cpp/third_party/vcpkg_overlays/azure-core-cpp/capath.patch new file mode 100644 index 0000000000..288d539154 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-core-cpp/capath.patch @@ -0,0 +1,54 @@ +diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +index a8e28364..4beebde6 100644 +--- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp ++++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +@@ -120,6 +120,18 @@ namespace Azure { namespace Core { namespace Http { + */ + std::string CAInfo; + ++ /** ++ * @brief Path to a directory which holds PEM encoded file, containing the certificate authorities ++ * sent to libcurl handle directly. ++ * ++ * @remark The Azure SDK will not check if the path is valid or not. ++ * ++ * @remark The default is the built-in system specific path. More about this option: ++ * https://curl.se/libcurl/c/CURLOPT_CAPATH.html ++ * ++ */ ++ std::string CAPath; ++ + /** + * @brief All HTTP requests will keep the connection channel open to the service. + * +diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp +index 89c0ade9..c307aa68 100644 +--- a/sdk/core/azure-core/src/http/curl/curl.cpp ++++ b/sdk/core/azure-core/src/http/curl/curl.cpp +@@ -1281,6 +1281,8 @@ inline std::string GetConnectionKey(std::string const& host, CurlTransportOption + key.append(","); + key.append(!options.CAInfo.empty() ? options.CAInfo : "0"); + key.append(","); ++ key.append(!options.CAPath.empty() ? options.CAPath : "0"); ++ key.append(","); + key.append( + options.Proxy.HasValue() ? (options.Proxy.Value().empty() ? "NoProxy" : options.Proxy.Value()) + : "0"); +@@ -2308,6 +2310,17 @@ CurlConnection::CurlConnection( + } + } + ++ if (!options.CAPath.empty()) ++ { ++ if (!SetLibcurlOption(m_handle, CURLOPT_CAPATH, options.CAPath.c_str(), &result)) ++ { ++ throw Azure::Core::Http::TransportException( ++ _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName ++ + ". Failed to set CA path to:" + options.CAPath + ". " ++ + std::string(curl_easy_strerror(result))); ++ } ++ } ++ + if (!options.SslOptions.PemEncodedExpectedRootCertificates.empty()) + { + curl_blob rootCertBlob diff --git a/cpp/third_party/vcpkg_overlays/azure-core-cpp/portfile.cmake b/cpp/third_party/vcpkg_overlays/azure-core-cpp/portfile.cmake new file mode 100644 index 0000000000..c10ba07177 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-core-cpp/portfile.cmake @@ -0,0 +1,28 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Azure/azure-sdk-for-cpp + REF azure-core_1.10.0 + SHA512 8917d5a3934a743bd8b44dacc12c3cd410cd59f1fa596c36a50f78562a1c7fe7a189c762e4099f3b24d23fcf5dcb0243ac3f139f1d345e6283ff4b5db418ecda + PATCHES + capath.patch +) + +vcpkg_check_features( + OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + curl BUILD_TRANSPORT_CURL + winhttp BUILD_TRANSPORT_WINHTTP +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}/sdk/core/azure-core/" + OPTIONS + ${FEATURE_OPTIONS} + -DWARNINGS_AS_ERRORS=OFF +) + +vcpkg_cmake_install() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +vcpkg_cmake_config_fixup() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_copy_pdbs() diff --git a/cpp/third_party/vcpkg_overlays/azure-core-cpp/vcpkg.json b/cpp/third_party/vcpkg_overlays/azure-core-cpp/vcpkg.json new file mode 100644 index 0000000000..1b14cfbc5a --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-core-cpp/vcpkg.json @@ -0,0 +1,77 @@ +{ + "name": "azure-core-cpp", + "version-semver": "1.10.0", + "description": [ + "Microsoft Azure Core SDK for C++", + "This library provides shared primitives, abstractions, and helpers for modern Azure SDK client libraries written in the C++." + ], + "homepage": "https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/core/azure-core", + "license": "MIT", + "dependencies": [ + { + "name": "openssl", + "platform": "!windows & !uwp" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "http" + ], + "features": { + "curl": { + "description": "Libcurl HTTP transport implementation", + "dependencies": [ + { + "name": "azure-core-cpp", + "default-features": false + }, + { + "name": "curl", + "default-features": false, + "features": [ + "ssl" + ] + } + ] + }, + "http": { + "description": "All HTTP transport implementations available on the platform", + "dependencies": [ + { + "name": "azure-core-cpp", + "default-features": false, + "features": [ + "curl" + ] + }, + { + "name": "azure-core-cpp", + "default-features": false, + "features": [ + "curl", + "winhttp" + ], + "platform": "windows & !uwp" + } + ] + }, + "winhttp": { + "description": "WinHTTP HTTP transport implementation", + "supports": "windows", + "dependencies": [ + { + "name": "azure-core-cpp", + "default-features": false + }, + "wil" + ] + } + } +} diff --git a/cpp/third_party/vcpkg_overlays/azure-identity-cpp/capath.patch b/cpp/third_party/vcpkg_overlays/azure-identity-cpp/capath.patch new file mode 100644 index 0000000000..288d539154 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-identity-cpp/capath.patch @@ -0,0 +1,54 @@ +diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +index a8e28364..4beebde6 100644 +--- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp ++++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +@@ -120,6 +120,18 @@ namespace Azure { namespace Core { namespace Http { + */ + std::string CAInfo; + ++ /** ++ * @brief Path to a directory which holds PEM encoded file, containing the certificate authorities ++ * sent to libcurl handle directly. ++ * ++ * @remark The Azure SDK will not check if the path is valid or not. ++ * ++ * @remark The default is the built-in system specific path. More about this option: ++ * https://curl.se/libcurl/c/CURLOPT_CAPATH.html ++ * ++ */ ++ std::string CAPath; ++ + /** + * @brief All HTTP requests will keep the connection channel open to the service. + * +diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp +index 89c0ade9..c307aa68 100644 +--- a/sdk/core/azure-core/src/http/curl/curl.cpp ++++ b/sdk/core/azure-core/src/http/curl/curl.cpp +@@ -1281,6 +1281,8 @@ inline std::string GetConnectionKey(std::string const& host, CurlTransportOption + key.append(","); + key.append(!options.CAInfo.empty() ? options.CAInfo : "0"); + key.append(","); ++ key.append(!options.CAPath.empty() ? options.CAPath : "0"); ++ key.append(","); + key.append( + options.Proxy.HasValue() ? (options.Proxy.Value().empty() ? "NoProxy" : options.Proxy.Value()) + : "0"); +@@ -2308,6 +2310,17 @@ CurlConnection::CurlConnection( + } + } + ++ if (!options.CAPath.empty()) ++ { ++ if (!SetLibcurlOption(m_handle, CURLOPT_CAPATH, options.CAPath.c_str(), &result)) ++ { ++ throw Azure::Core::Http::TransportException( ++ _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName ++ + ". Failed to set CA path to:" + options.CAPath + ". " ++ + std::string(curl_easy_strerror(result))); ++ } ++ } ++ + if (!options.SslOptions.PemEncodedExpectedRootCertificates.empty()) + { + curl_blob rootCertBlob diff --git a/cpp/third_party/vcpkg_overlays/azure-identity-cpp/portfile.cmake b/cpp/third_party/vcpkg_overlays/azure-identity-cpp/portfile.cmake new file mode 100644 index 0000000000..6b2243fa27 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-identity-cpp/portfile.cmake @@ -0,0 +1,20 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Azure/azure-sdk-for-cpp + REF azure-identity_1.5.0 + SHA512 1f96c2c9a056e83e083d9d8bc6c55fce1bc84b1ccee0a980c4b09d13b364551a1760c8a44232e5786b179fd60bea4b5b05c6cadd6b7fb270979a606a895bb7b6 + PATCHES + capath.patch +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}/sdk/identity/azure-identity/" + OPTIONS + -DWARNINGS_AS_ERRORS=OFF +) + +vcpkg_cmake_install() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +vcpkg_cmake_config_fixup() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_copy_pdbs() diff --git a/cpp/third_party/vcpkg_overlays/azure-identity-cpp/vcpkg.json b/cpp/third_party/vcpkg_overlays/azure-identity-cpp/vcpkg.json new file mode 100644 index 0000000000..057682c678 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-identity-cpp/vcpkg.json @@ -0,0 +1,26 @@ +{ + "name": "azure-identity-cpp", + "version-semver": "1.5.0", + "description": [ + "Microsoft Azure Identity SDK for C++", + "This library provides common authentication-related abstractions for Azure SDK." + ], + "homepage": "https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity", + "license": "MIT", + "dependencies": [ + { + "name": "azure-core-cpp", + "default-features": false, + "version>=": "1.9.0" + }, + "openssl", + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/capath.patch b/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/capath.patch new file mode 100644 index 0000000000..d16376f0fa --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/capath.patch @@ -0,0 +1,54 @@ +diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +index a8e28364..4beebde6 100644 +--- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp ++++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +@@ -120,6 +120,18 @@ namespace Azure { namespace Core { namespace Http { + */ + std::string CAInfo; + ++ /** ++ * @brief Path to a directory which holds PEM encoded file, containing the certificate authorities ++ * sent to libcurl handle directly. ++ * ++ * @remark The Azure SDK will not check if the path is valid or not. ++ * ++ * @remark The default is the built-in system specific path. More about this option: ++ * https://curl.se/libcurl/c/CURLOPT_CAPATH.html ++ * ++ */ ++ std::string CAPath; ++ + /** + * @brief All HTTP requests will keep the connection channel open to the service. + * +diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp +index 01edbcc1..b34eb199 100644 +--- a/sdk/core/azure-core/src/http/curl/curl.cpp ++++ b/sdk/core/azure-core/src/http/curl/curl.cpp +@@ -1280,6 +1280,8 @@ inline std::string GetConnectionKey(std::string const& host, CurlTransportOption + key.append(","); + key.append(!options.CAInfo.empty() ? options.CAInfo : "0"); + key.append(","); ++ key.append(!options.CAPath.empty() ? options.CAPath : "0"); ++ key.append(","); + key.append( + options.Proxy.HasValue() ? (options.Proxy.Value().empty() ? "NoProxy" : options.Proxy.Value()) + : "0"); +@@ -2307,6 +2309,17 @@ CurlConnection::CurlConnection( + } + } + ++ if (!options.CAPath.empty()) ++ { ++ if (!SetLibcurlOption(m_handle, CURLOPT_CAPATH, options.CAPath.c_str(), &result)) ++ { ++ throw Azure::Core::Http::TransportException( ++ _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName ++ + ". Failed to set CA path to:" + options.CAPath + ". " ++ + std::string(curl_easy_strerror(result))); ++ } ++ } ++ + if (!options.SslOptions.PemEncodedExpectedRootCertificates.empty()) + { + curl_blob rootCertBlob diff --git a/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/portfile.cmake b/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/portfile.cmake new file mode 100644 index 0000000000..b60407297a --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/portfile.cmake @@ -0,0 +1,20 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Azure/azure-sdk-for-cpp + REF azure-storage-blobs_12.7.0 + SHA512 7750035f42c7cc120b294d0449885446d5d06c0393a24d957f1484525967fd609dda0319574e9d95888441d749bbca820fccaf7741ac1321bc071adcfb48f3a9 + PATCHES + capath.patch +) + +vcpkg_cmake_configure( + SOURCE_PATH ${SOURCE_PATH}/sdk/storage/azure-storage-blobs/ + OPTIONS + -DWARNINGS_AS_ERRORS=OFF +) + +vcpkg_cmake_install() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +vcpkg_cmake_config_fixup() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_copy_pdbs() diff --git a/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/vcpkg.json b/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/vcpkg.json new file mode 100644 index 0000000000..14a527dfe0 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-storage-blobs-cpp/vcpkg.json @@ -0,0 +1,25 @@ +{ + "name": "azure-storage-blobs-cpp", + "version-semver": "12.7.0", + "description": [ + "Microsoft Azure Storage Blobs SDK for C++", + "This library provides Azure Storage Blobs SDK." + ], + "homepage": "https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/storage/azure-storage-blobs", + "license": "MIT", + "dependencies": [ + { + "name": "azure-storage-common-cpp", + "default-features": false, + "version>=": "12.3.1" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/capath.patch b/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/capath.patch new file mode 100644 index 0000000000..aabc99f5d2 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/capath.patch @@ -0,0 +1,54 @@ +diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +index a8e28364..4beebde6 100644 +--- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp ++++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp +@@ -120,6 +120,18 @@ namespace Azure { namespace Core { namespace Http { + */ + std::string CAInfo; + ++ /** ++ * @brief Path to a directory which holds PEM encoded file, containing the certificate authorities ++ * sent to libcurl handle directly. ++ * ++ * @remark The Azure SDK will not check if the path is valid or not. ++ * ++ * @remark The default is the built-in system specific path. More about this option: ++ * https://curl.se/libcurl/c/CURLOPT_CAPATH.html ++ * ++ */ ++ std::string CAPath; ++ + /** + * @brief All HTTP requests will keep the connection channel open to the service. + * +diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp +index 80309f89..c1236456 100644 +--- a/sdk/core/azure-core/src/http/curl/curl.cpp ++++ b/sdk/core/azure-core/src/http/curl/curl.cpp +@@ -1280,6 +1280,8 @@ inline std::string GetConnectionKey(std::string const& host, CurlTransportOption + key.append(","); + key.append(!options.CAInfo.empty() ? options.CAInfo : "0"); + key.append(","); ++ key.append(!options.CAPath.empty() ? options.CAPath : "0"); ++ key.append(","); + key.append( + options.Proxy.HasValue() ? (options.Proxy.Value().empty() ? "NoProxy" : options.Proxy.Value()) + : "0"); +@@ -2307,6 +2309,17 @@ CurlConnection::CurlConnection( + } + } + ++ if (!options.CAPath.empty()) ++ { ++ if (!SetLibcurlOption(m_handle, CURLOPT_CAPATH, options.CAPath.c_str(), &result)) ++ { ++ throw Azure::Core::Http::TransportException( ++ _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName ++ + ". Failed to set CA path to:" + options.CAPath + ". " ++ + std::string(curl_easy_strerror(result))); ++ } ++ } ++ + if (!options.SslOptions.PemEncodedExpectedRootCertificates.empty()) + { + curl_blob rootCertBlob diff --git a/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/portfile.cmake b/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/portfile.cmake new file mode 100644 index 0000000000..9a6f43278a --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/portfile.cmake @@ -0,0 +1,20 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Azure/azure-sdk-for-cpp + REF azure-storage-common_12.3.2 + SHA512 e437c12ba1056838a1d68257522f412f709b4bcbd3e58d4f7fc47768f125a61258667e0f4fcb93927522da9ab2a9495c3d8c53d81f01a86feb142a7025b1ee4c + PATCHES + capath.patch +) + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}/sdk/storage/azure-storage-common/" + OPTIONS + -DWARNINGS_AS_ERRORS=OFF +) + +vcpkg_cmake_install() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +vcpkg_cmake_config_fixup() +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") +vcpkg_copy_pdbs() diff --git a/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/vcpkg.json b/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/vcpkg.json new file mode 100644 index 0000000000..24dfb910a9 --- /dev/null +++ b/cpp/third_party/vcpkg_overlays/azure-storage-common-cpp/vcpkg.json @@ -0,0 +1,33 @@ +{ + "name": "azure-storage-common-cpp", + "version-semver": "12.3.2", + "description": [ + "Microsoft Azure Common Storage SDK for C++", + "This library provides common Azure Storage-related abstractions for Azure SDK." + ], + "homepage": "https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/storage/azure-storage-common", + "license": "MIT", + "dependencies": [ + { + "name": "azure-core-cpp", + "default-features": false, + "version>=": "1.9.0" + }, + { + "name": "libxml2", + "platform": "!windows" + }, + { + "name": "openssl", + "platform": "!windows" + }, + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ] +} diff --git a/python/arcticdb/adapters/azure_library_adapter.py b/python/arcticdb/adapters/azure_library_adapter.py index 1f14717953..2faa993758 100644 --- a/python/arcticdb/adapters/azure_library_adapter.py +++ b/python/arcticdb/adapters/azure_library_adapter.py @@ -8,6 +8,7 @@ import re import time from typing import Optional +import ssl import platform from arcticc.pb2.storage_pb2 import EnvironmentConfigsMap, LibraryConfig @@ -26,7 +27,18 @@ @dataclass class ParsedQuery: Path_prefix: Optional[str] = None - CA_cert_path: str = "" + # winhttp is used as Azure backend support on Winodws by default; winhttp itself mainatains ca cert. + # The options should be left empty else libcurl will be used on Windows + CA_cert_path: str = ( + "" + if platform.system() is "Windows" or ssl.get_default_verify_paths().cafile == None + else ssl.get_default_verify_paths().cafile + ) # CURLOPT_CAINFO in curl + CA_cert_dir: str = ( + "" + if platform.system() is "Windows" or ssl.get_default_verify_paths().capath == None + else ssl.get_default_verify_paths().capath + ) # CURLOPT_CAPATH in curl Container: Optional[str] = None @@ -57,6 +69,7 @@ def __init__(self, uri: str, encoding_version: EncodingVersion, *args, **kwargs) if platform.system() == "Windows" and self._query_params.CA_cert_path: raise ValueError(f"CA_cert_path cannot be set on Windows platform") self._ca_cert_path = self._query_params.CA_cert_path + self._ca_cert_dir = self._query_params.CA_cert_dir self._encoding_version = encoding_version super().__init__(uri, self._encoding_version) @@ -79,6 +92,7 @@ def config_library(self): endpoint=self._endpoint, with_prefix=with_prefix, ca_cert_path=self._ca_cert_path, + ca_cert_dir=self._ca_cert_dir, ) lib = NativeVersionStore.create_store_from_config( @@ -110,6 +124,8 @@ def get_storage_override(self) -> AzureOverride: azure_override.endpoint = self._endpoint if self._ca_cert_path: azure_override.ca_cert_path = self._ca_cert_path + if self._ca_cert_dir: + azure_override.ca_cert_dir = self._ca_cert_dir storage_override = StorageOverride() storage_override.set_azure_override(azure_override) @@ -137,6 +153,7 @@ def add_library_to_env(self, env_cfg: EnvironmentConfigsMap, name: str): endpoint=self._endpoint, with_prefix=with_prefix, ca_cert_path=self._ca_cert_path, + ca_cert_dir=self._ca_cert_dir, ) @property diff --git a/python/arcticdb/version_store/helper.py b/python/arcticdb/version_store/helper.py index fe674f1821..bcf3ca1d87 100644 --- a/python/arcticdb/version_store/helper.py +++ b/python/arcticdb/version_store/helper.py @@ -323,6 +323,7 @@ def get_azure_proto( endpoint, with_prefix: Optional[Union[bool, str]] = True, ca_cert_path: str = "", + ca_cert_dir: str = "", ): env = cfg.env_by_id[env_name] azure = AzureConfig() @@ -338,6 +339,7 @@ def get_azure_proto( else: azure.prefix = lib_name azure.ca_cert_path = ca_cert_path + azure.ca_cert_dir = ca_cert_dir sid, storage = get_storage_for_lib_name(azure.prefix, env) storage.config.Pack(azure, type_url_prefix="cxx.arctic.org") @@ -353,6 +355,7 @@ def add_azure_library_to_env( description: Optional[bool] = None, with_prefix: Optional[Union[bool, str]] = True, ca_cert_path: str = "", + ca_cert_dir: str = "", ): env = cfg.env_by_id[env_name] sid, _ = get_azure_proto( @@ -363,6 +366,7 @@ def add_azure_library_to_env( endpoint=endpoint, with_prefix=with_prefix, ca_cert_path=ca_cert_path, + ca_cert_dir=ca_cert_dir, ) _add_lib_desc_to_env(env, lib_name, sid, description) diff --git a/python/tests/integration/arcticdb/test_arctic.py b/python/tests/integration/arcticdb/test_arctic.py index b5a9703409..fbfb0c464c 100644 --- a/python/tests/integration/arcticdb/test_arctic.py +++ b/python/tests/integration/arcticdb/test_arctic.py @@ -887,6 +887,21 @@ def test_azure_no_ca_path(azurite_storage: StorageFixture): ac.create_library("x") +@AZURE_TESTS_MARK +def test_azure_ca_dir_only(azurite_azure_test_connection_setting, azure_client_and_create_container): + (endpoint, container, credential_name, credential_key, _, ca_cert_dir) = azurite_azure_test_connection_setting + ac = Arctic( + f"azure://DefaultEndpointsProtocol=https;AccountName={credential_name};AccountKey={credential_key};BlobEndpoint={endpoint}/{credential_name};Container={container};CA_cert_dir={ca_cert_dir}" + ) + LIB_NAME = "abc" + expected = pd.DataFrame({"col1": [1, 2, 3], "col2": [4, 5, 6]}) + sym = "test" + ac.create_library(LIB_NAME) + ac[LIB_NAME].write(sym, expected) + + assert_frame_equal(expected, ac[LIB_NAME].read(sym).data) + + @AZURE_TESTS_MARK def test_azure_sas_token(azurite_storage_factory: StorageFixtureFactory): with azurite_storage_factory.enforcing_permissions_context(): diff --git a/python/tests/scripts/test_update_storage.py b/python/tests/scripts/test_update_storage.py index 919a6bb9d0..600499b3f2 100644 --- a/python/tests/scripts/test_update_storage.py +++ b/python/tests/scripts/test_update_storage.py @@ -133,6 +133,7 @@ def test_upgrade_script_azure(azurite_storage: AzureContainer): config = ac._library_manager.get_library_config(LIB_NAME) azure_storage = _get_azure_storage_config(config) assert azure_storage.ca_cert_path == "" + assert azure_storage.ca_cert_dir == "" assert azure_storage.container_name == "" assert azure_storage.endpoint == "" assert azure_storage.prefix.startswith(LIB_NAME)