diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 67e77ad6ec91..8f12528069c1 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -29,7 +29,7 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake .. -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON -DPLUGIN_DENSE_PARSER=ON -GNinja -DBUILD_DEPRECATED_CLI=ON
+        cmake .. -DGOOGLE_TEST=ON -DUSE_OPENMP=ON -DUSE_DMLC_GTEST=ON -GNinja -DBUILD_DEPRECATED_CLI=ON
         ninja -v
     - name: Run gtest binary
       run: |
diff --git a/CMakeLists.txt b/CMakeLists.txt
index bf8f0cf62c79..a9c6f7410684 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -99,7 +99,6 @@ set(ENABLED_SANITIZERS "address" "leak" CACHE STRING
   "Semicolon separated list of sanitizer names. E.g 'address;leak'. Supported sanitizers are
 address, leak, undefined and thread.")
 ## Plugins
-option(PLUGIN_DENSE_PARSER "Build dense parser plugin" OFF)
 option(PLUGIN_RMM "Build with RAPIDS Memory Manager (RMM)" OFF)
 option(PLUGIN_FEDERATED "Build with Federated Learning" OFF)
 ## TODO: 1. Add check if DPC++ compiler is used for building
@@ -185,6 +184,9 @@ endif()
 if(USE_HDFS)
   message(SEND_ERROR "The option `USE_HDFS` has been removed from XGBoost")
 endif()
+if(PLUGIN_DENSE_PARSER)
+  message(SEND_ERROR "The option `PLUGIN_DENSE_PARSER` has been removed from XGBoost.")
+endif()
 
 #-- Sanitizer
 if(USE_SANITIZER)
diff --git a/plugin/CMakeLists.txt b/plugin/CMakeLists.txt
index 6089ae486e40..58b31053ffdc 100644
--- a/plugin/CMakeLists.txt
+++ b/plugin/CMakeLists.txt
@@ -1,7 +1,3 @@
-if(PLUGIN_DENSE_PARSER)
-  target_sources(objxgboost PRIVATE ${xgboost_SOURCE_DIR}/plugin/dense_parser/dense_libsvm.cc)
-endif()
-
 if(PLUGIN_UPDATER_ONEAPI)
   add_library(oneapi_plugin OBJECT
     ${xgboost_SOURCE_DIR}/plugin/updater_oneapi/regression_obj_oneapi.cc
diff --git a/plugin/README.md b/plugin/README.md
index 6e115c465e26..008f4ad492b6 100644
--- a/plugin/README.md
+++ b/plugin/README.md
@@ -36,5 +36,3 @@ The register macros available to plugin writers are:
 And from dmlc-core:
 
  - DMLC_REGISTER_PARAMETER - Register a set of parameter for a specific usecase
- - DMLC_REGISTER_DATA_PARSER - Register a data parser where the data can be
-   represented by a URL. This is used by DMatrix.
diff --git a/plugin/dense_parser/dense_libsvm.cc b/plugin/dense_parser/dense_libsvm.cc
deleted file mode 100644
index 0dd2d0419c17..000000000000
--- a/plugin/dense_parser/dense_libsvm.cc
+++ /dev/null
@@ -1,87 +0,0 @@
-/*!
- * Copyright 2015 by Contributors
- * \file dense_libsvm.cc
- * \brief Plugin to load in libsvm, but fill all the missing entries with zeros.
- *  This plugin is mainly used for benchmark purposes and do not need to be included.
- */
-#include <xgboost/base.h>
-#include <dmlc/data.h>
-#include <memory>
-
-namespace dmlc {
-namespace data {
-
-template<typename IndexType>
-class DensifyParser : public dmlc::Parser<IndexType> {
- public:
-  DensifyParser(dmlc::Parser<IndexType>* parser, uint32_t num_col)
-      : parser_(parser), num_col_(num_col) {
-  }
-
-  void BeforeFirst() override {
-    parser_->BeforeFirst();
-  }
-
-  bool Next() override {
-    if (!parser_->Next()) return false;
-    const RowBlock<IndexType>& batch = parser_->Value();
-    LOG(INFO) << batch.size;
-    dense_index_.resize(num_col_ * batch.size);
-    dense_value_.resize(num_col_ * batch.size);
-    std::fill(dense_value_.begin(), dense_value_.end(), 0.0);
-    offset_.resize(batch.size + 1);
-    offset_[0] = 0;
-
-    for (size_t i = 0; i < batch.size; ++i) {
-      offset_[i + 1] = (i + 1) * num_col_;
-      Row<IndexType> row = batch[i];
-      for (uint32_t j = 0; j < num_col_; ++j) {
-        dense_index_[i * num_col_ + j] = j;
-      }
-      for (unsigned k = 0; k < row.length; ++k) {
-        uint32_t index = row.get_index(k);
-        CHECK_LT(index, num_col_)
-            << "Featuere index larger than num_col";
-        dense_value_[i * num_col_ + index]  = row.get_value(k);
-      }
-    }
-    out_ = batch;
-    out_.index = dmlc::BeginPtr(dense_index_);
-    out_.value = dmlc::BeginPtr(dense_value_);
-    out_.offset = dmlc::BeginPtr(offset_);
-    return true;
-  }
-
-  const dmlc::RowBlock<IndexType>& Value() const override {
-    return out_;
-  }
-
-  size_t BytesRead() const override {
-    return parser_->BytesRead();
-  }
-
- private:
-  RowBlock<IndexType> out_;
-  std::unique_ptr<Parser<IndexType> > parser_;
-  uint32_t num_col_;
-  std::vector<size_t> offset_;
-  std::vector<IndexType> dense_index_;
-  std::vector<xgboost::bst_float> dense_value_;
-};
-
-template<typename IndexType, typename DType = real_t>
-Parser<IndexType> *
-CreateDenseLibSVMParser(const std::string& path,
-                        const std::map<std::string, std::string>& args,
-                        unsigned part_index,
-                        unsigned num_parts) {
-  CHECK_NE(args.count("num_col"), 0) << "expect num_col in dense_libsvm";
-  return new DensifyParser<IndexType>(
-            Parser<IndexType>::Create(path.c_str(), part_index, num_parts, "libsvm"),
-           uint32_t(atoi(args.at("num_col").c_str())));
-}
-}  // namespace data
-
-DMLC_REGISTER_DATA_PARSER(uint32_t, real_t, dense_libsvm,
-  data::CreateDenseLibSVMParser<uint32_t __DMLC_COMMA real_t>);
-}  // namespace dmlc
diff --git a/python-package/packager/build_config.py b/python-package/packager/build_config.py
index d3733d628afe..933bfdce2b0e 100644
--- a/python-package/packager/build_config.py
+++ b/python-package/packager/build_config.py
@@ -17,14 +17,10 @@ class BuildConfiguration:  # pylint: disable=R0902
     use_nccl: bool = False
     # Whether to load nccl dynamically
     use_dlopen_nccl: bool = False
-    # Whether to enable HDFS
-    use_hdfs: bool = False
-    # Whether to enable Azure Storage
-    use_azure: bool = False
-    # Whether to enable AWS S3
-    use_s3: bool = False
-    # Whether to enable the dense parser plugin
-    plugin_dense_parser: bool = False
+    # Whether to enable federated learning
+    plugin_federated: bool = False
+    # Whether to enable rmm support
+    plugin_rmm: bool = False
     # Special option: See explanation below
     use_system_libxgboost: bool = False
 
diff --git a/tests/buildkite/build-cpu.sh b/tests/buildkite/build-cpu.sh
index 88da7d39504a..73e88d8aac02 100755
--- a/tests/buildkite/build-cpu.sh
+++ b/tests/buildkite/build-cpu.sh
@@ -15,7 +15,7 @@ $command_wrapper rm -fv dmlc-core/include/dmlc/build_config_default.h
   # include/dmlc/build_config_default.h.
 echo "--- Build libxgboost from the source"
 $command_wrapper tests/ci_build/build_via_cmake.sh -DCMAKE_PREFIX_PATH=/opt/grpc \
-  -DPLUGIN_DENSE_PARSER=ON -DPLUGIN_FEDERATED=ON
+		 -DPLUGIN_FEDERATED=ON
 echo "--- Run Google Test"
 $command_wrapper bash -c "cd build && ctest --extra-verbose"
 echo "--- Stash XGBoost CLI executable"