diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake
index 29517567ce6e5..6e6a74c9c78f8 100644
--- a/cpp/cmake_modules/DefineOptions.cmake
+++ b/cpp/cmake_modules/DefineOptions.cmake
@@ -470,6 +470,15 @@ takes precedence over ccache if a storage backend is configured" ON)
                 "Rely on jemalloc shared libraries where relevant"
                 ${ARROW_DEPENDENCY_USE_SHARED})
 
+  if(MSVC)
+    # LLVM doesn't support shared library with MSVC.
+    set(ARROW_LLVM_USE_SHARED_DEFAULT OFF)
+  else()
+    set(ARROW_LLVM_USE_SHARED_DEFAULT ${ARROW_DEPENDENCY_USE_SHARED})
+  endif()
+  define_option(ARROW_LLVM_USE_SHARED "Rely on LLVM shared libraries where relevant"
+                ${ARROW_LLVM_USE_SHARED_DEFAULT})
+
   define_option(ARROW_LZ4_USE_SHARED "Rely on lz4 shared libraries where relevant"
                 ${ARROW_DEPENDENCY_USE_SHARED})
 
diff --git a/cpp/cmake_modules/FindLLVMAlt.cmake b/cpp/cmake_modules/FindLLVMAlt.cmake
index b3d77978f153c..e980f53fd3407 100644
--- a/cpp/cmake_modules/FindLLVMAlt.cmake
+++ b/cpp/cmake_modules/FindLLVMAlt.cmake
@@ -68,18 +68,6 @@ if(NOT LLVM_FOUND)
 endif()
 
 if(LLVM_FOUND)
-  # Find the libraries that correspond to the LLVM components
-  llvm_map_components_to_libnames(LLVM_LIBS
-                                  core
-                                  mcjit
-                                  native
-                                  ipo
-                                  bitreader
-                                  target
-                                  linker
-                                  analysis
-                                  debuginfodwarf)
-
   find_program(LLVM_LINK_EXECUTABLE llvm-link HINTS ${LLVM_TOOLS_BINARY_DIR})
 
   find_program(CLANG_EXECUTABLE
@@ -94,22 +82,37 @@ if(LLVM_FOUND)
                                    INTERFACE_COMPILE_FLAGS "${LLVM_DEFINITIONS}")
 
   add_library(LLVM::LLVM_LIBS INTERFACE IMPORTED)
-  set_target_properties(LLVM::LLVM_LIBS PROPERTIES INTERFACE_LINK_LIBRARIES
-                                                   "${LLVM_LIBS}")
+  if(ARROW_LLVM_USE_SHARED)
+    target_link_libraries(LLVM::LLVM_LIBS INTERFACE LLVM)
+  else()
+    # Find the libraries that correspond to the LLVM components
+    llvm_map_components_to_libnames(LLVM_LIBS
+                                    core
+                                    mcjit
+                                    native
+                                    ipo
+                                    bitreader
+                                    target
+                                    linker
+                                    analysis
+                                    debuginfodwarf)
+    target_link_libraries(LLVM::LLVM_LIBS INTERFACE ${LLVM_LIBS})
 
-  if(TARGET LLVMSupport AND NOT ARROW_ZSTD_USE_SHARED)
-    get_target_property(LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES LLVMSupport
-                        INTERFACE_LINK_LIBRARIES)
-    list(FIND LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES zstd::libzstd_shared
-         LLVM_SUPPORT_LIBZSTD_INDEX)
-    if(NOT LLVM_SUPPORT_LIBZSTD_INDEX EQUAL -1)
-      list(REMOVE_AT LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES ${LLVM_SUPPORT_LIBZSTD_INDEX})
-      list(INSERT LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES ${LLVM_SUPPORT_LIBZSTD_INDEX}
-           zstd::libzstd_static)
+    if(TARGET LLVMSupport AND NOT ARROW_ZSTD_USE_SHARED)
+      get_target_property(LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES LLVMSupport
+                          INTERFACE_LINK_LIBRARIES)
+      list(FIND LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES zstd::libzstd_shared
+           LLVM_SUPPORT_LIBZSTD_INDEX)
+      if(NOT LLVM_SUPPORT_LIBZSTD_INDEX EQUAL -1)
+        list(REMOVE_AT LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES
+             ${LLVM_SUPPORT_LIBZSTD_INDEX})
+        list(INSERT LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES ${LLVM_SUPPORT_LIBZSTD_INDEX}
+             zstd::libzstd_static)
+      endif()
+      set_target_properties(LLVMSupport
+                            PROPERTIES INTERFACE_LINK_LIBRARIES
+                                       "${LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES}")
     endif()
-    set_target_properties(LLVMSupport
-                          PROPERTIES INTERFACE_LINK_LIBRARIES
-                                     "${LLVM_SUPPORT_INTERFACE_LINK_LIBRARIES}")
   endif()
 endif()
 
diff --git a/cpp/src/gandiva/CMakeLists.txt b/cpp/src/gandiva/CMakeLists.txt
index 6a92224e9113d..db260b5acc933 100644
--- a/cpp/src/gandiva/CMakeLists.txt
+++ b/cpp/src/gandiva/CMakeLists.txt
@@ -175,14 +175,14 @@ add_dependencies(gandiva ${GANDIVA_LIBRARIES})
 arrow_install_all_headers("gandiva")
 
 set(GANDIVA_STATIC_TEST_LINK_LIBS gandiva_static ${ARROW_TEST_LINK_LIBS})
-set(GANDIVA_SHARED_TEST_LINK_LIBS gandiva_shared ${ARROW_TEST_LINK_LIBS})
+set(GANDIVA_SHARED_TEST_LINK_LIBS gandiva_shared ${ARROW_TEST_LINK_LIBS} LLVM::LLVM_LIBS)
 if(ARROW_WITH_UTF8PROC)
   list(APPEND GANDIVA_SHARED_TEST_LINK_LIBS utf8proc::utf8proc)
   list(APPEND GANDIVA_STATIC_TEST_LINK_LIBS utf8proc::utf8proc)
 endif()
 if(WIN32)
   list(APPEND GANDIVA_STATIC_TEST_LINK_LIBS ${GANDIVA_OPENSSL_TARGETS})
-  list(APPEND GANDIVA_SHARED_TEST_LINK_LIBS LLVM::LLVM_LIBS ${GANDIVA_OPENSSL_TARGETS})
+  list(APPEND GANDIVA_SHARED_TEST_LINK_LIBS ${GANDIVA_OPENSSL_TARGETS})
 endif()
 
 function(ADD_GANDIVA_TEST REL_TEST_NAME)