-Set ca path and directory automatically

cpp/CMakeLists.txt

@@ -100,9 +100,7 @@ include(PythonUtils) # Must be called before Pybind (third_party) to override it
 
 add_subdirectory(third_party)
 add_subdirectory(proto)
-if(NOT ${ARCTICDB_USING_CONDA})
-    include(AzureVcpkg) #AzureVcpkg.cmake is from https://github.com/Azure/azure-sdk-for-cpp/blob/main/cmake-modules/AzureVcpkg.cmake, commit ada77b3
-endif()
+
 
 python_utils_dump_vars_if_enabled("After Pybind")
 python_utils_check_include_dirs("accepted by pybind")

cpp/arcticdb/storage/azure/azure_storage.cpp

@@ -353,6 +353,10 @@ AzureStorage::AzureStorage(const LibraryPath &library_path, OpenMode mode, const
             ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using default CA cert path");
         else
             ARCTICDB_RUNTIME_DEBUG(log::storage(), "CA cert path: {}", conf.ca_cert_path());
+        if (conf.ca_cert_dir().empty())
+            ARCTICDB_RUNTIME_DEBUG(log::storage(), "Using default CA cert directory");
+        else
+            ARCTICDB_RUNTIME_DEBUG(log::storage(), "CA cert directory: {}", conf.ca_cert_dir());
         ARCTICDB_RUNTIME_DEBUG(log::storage(), "Connecting to Azure Blob Storage: {} Container: {}", conf.endpoint(), conf.container_name());
 
         if (!conf.prefix().empty()) {
@@ -366,4 +370,15 @@ AzureStorage::AzureStorage(const LibraryPath &library_path, OpenMode mode, const
         download_option_.TransferOptions.Concurrency = max_connections;
 }
 
+Azure::Storage::Blobs::BlobClientOptions AzureStorage::get_client_options(const Config &conf) {
+    BlobClientOptions client_options;
+    if (!conf.ca_cert_path().empty() || !conf.ca_cert_dir().empty()) {//WARNING: Setting ca_cert_path will force Azure sdk uses libcurl as backend support, instead of winhttp
+        Azure::Core::Http::CurlTransportOptions curl_transport_options;
+        curl_transport_options.CAInfo = conf.ca_cert_path();
+        curl_transport_options.CAPath = conf.ca_cert_dir();
+        client_options.Transport.Transport = std::make_shared<Azure::Core::Http::CurlTransport>(curl_transport_options);
+    }
+    return client_options;
+}
+
 } // namespace arcticdb::storage::azure

cpp/arcticdb/storage/python_bindings.cpp

@@ -124,7 +124,8 @@ void register_bindings(py::module& storage) {
         .def(py::init<>())
         .def_property("container_name", &AzureOverride::container_name, &AzureOverride::set_container_name)
         .def_property("endpoint", &AzureOverride::endpoint, &AzureOverride::set_endpoint)
-        .def_property("ca_cert_path", &AzureOverride::ca_cert_path, &AzureOverride::set_ca_cert_path);
+        .def_property("ca_cert_path", &AzureOverride::ca_cert_path, &AzureOverride::set_ca_cert_path)
+        .def_property("ca_cert_dir", &AzureOverride::ca_cert_dir, &AzureOverride::set_ca_cert_dir);
 
     py::class_<LmdbOverride>(storage, "LmdbOverride")
             .def(py::init<>())

cpp/arcticdb/storage/storage_override.hpp

@@ -127,6 +127,7 @@ class AzureOverride {
     std::string container_name_;
     std::string endpoint_;
     std::string ca_cert_path_;
+    std::string ca_cert_dir_;
 
 public:
     std::string container_name() const {
@@ -153,6 +154,14 @@ class AzureOverride {
         ca_cert_path_ = ca_cert_path;
     }
 
+    std::string ca_cert_dir() const {
+        return ca_cert_dir_;
+    }
+
+    void set_ca_cert_dir(std::string_view ca_cert_dir){
+        ca_cert_dir_ = ca_cert_dir;
+    }
+
     void modify_storage_config(arcticdb::proto::storage::VariantStorage& storage) const {
         if(storage.config().Is<arcticdb::proto::azure_storage::Config>()) {
             arcticdb::proto::azure_storage::Config azure_storage;
@@ -161,6 +170,7 @@ class AzureOverride {
             azure_storage.set_container_name(container_name_);
             azure_storage.set_endpoint(endpoint_);
             azure_storage.set_ca_cert_path(ca_cert_path_);
+            azure_storage.set_ca_cert_dir(ca_cert_dir_);
 
             util::pack_to_any(azure_storage, *storage.mutable_config());
         }

cpp/proto/arcticc/pb2/azure_storage.proto

@@ -18,4 +18,5 @@ message Config {
     string prefix = 5;
     string ca_cert_path = 6;
     bool use_mock_storage_for_testing = 7;
+    string ca_cert_dir = 8;
 }

cpp/third_party/vcpkg_overlays/azure-core-cpp/capath.patch

@@ -0,0 +1,54 @@
+diff --git a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp
+index a8e28364..4beebde6 100644
+--- a/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp
++++ b/sdk/core/azure-core/inc/azure/core/http/curl_transport.hpp
+@@ -120,6 +120,18 @@ namespace Azure { namespace Core { namespace Http {
+      */
+     std::string CAInfo;
+
++    /**
++     * @brief Path to a directory which holds PEM encoded file, containing the certificate authorities
++     * sent to libcurl handle directly.
++     *
++     * @remark The Azure SDK will not check if the path is valid or not.
++     *
++     * @remark The default is the built-in system specific path. More about this option:
++     * https://curl.se/libcurl/c/CURLOPT_CAPATH.html
++     *
++     */
++    std::string CAPath;
++
+     /**
+      * @brief All HTTP requests will keep the connection channel open to the service.
+      *
+diff --git a/sdk/core/azure-core/src/http/curl/curl.cpp b/sdk/core/azure-core/src/http/curl/curl.cpp
+index 89c0ade9..c307aa68 100644
+--- a/sdk/core/azure-core/src/http/curl/curl.cpp
++++ b/sdk/core/azure-core/src/http/curl/curl.cpp
+@@ -1281,6 +1281,8 @@ inline std::string GetConnectionKey(std::string const& host, CurlTransportOption
+   key.append(",");
+   key.append(!options.CAInfo.empty() ? options.CAInfo : "0");
+   key.append(",");
++  key.append(!options.CAPath.empty() ? options.CAPath : "0");
++  key.append(",");
+   key.append(
+       options.Proxy.HasValue() ? (options.Proxy.Value().empty() ? "NoProxy" : options.Proxy.Value())
+                                : "0");
+@@ -2308,6 +2310,17 @@ CurlConnection::CurlConnection(
+     }
+   }
+
++  if (!options.CAPath.empty())
++  {
++    if (!SetLibcurlOption(m_handle, CURLOPT_CAPATH, options.CAPath.c_str(), &result))
++    {
++      throw Azure::Core::Http::TransportException(
++          _detail::DefaultFailedToGetNewConnectionTemplate + hostDisplayName
++          + ". Failed to set CA path to:" + options.CAPath + ". "
++          + std::string(curl_easy_strerror(result)));
++    }
++  }
++
+   if (!options.SslOptions.PemEncodedExpectedRootCertificates.empty())
+   {
+     curl_blob rootCertBlob

cpp/third_party/vcpkg_overlays/azure-core-cpp/portfile.cmake

@@ -0,0 +1,28 @@
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO Azure/azure-sdk-for-cpp
+    REF azure-core_1.10.0
+    SHA512 8917d5a3934a743bd8b44dacc12c3cd410cd59f1fa596c36a50f78562a1c7fe7a189c762e4099f3b24d23fcf5dcb0243ac3f139f1d345e6283ff4b5db418ecda
+    PATCHES
+        capath.patch
+)
+
+vcpkg_check_features(
+    OUT_FEATURE_OPTIONS FEATURE_OPTIONS
+    FEATURES
+        curl BUILD_TRANSPORT_CURL
+        winhttp BUILD_TRANSPORT_WINHTTP
+)
+
+vcpkg_cmake_configure(
+    SOURCE_PATH "${SOURCE_PATH}/sdk/core/azure-core/"
+    OPTIONS
+        ${FEATURE_OPTIONS}
+        -DWARNINGS_AS_ERRORS=OFF
+)
+
+vcpkg_cmake_install()
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include")
+vcpkg_cmake_config_fixup()
+file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share")
+vcpkg_copy_pdbs()