AgglomerativeClustering support single cluster and ignore only zero d…

…istances from self-loops (#3824)

Closes #3801 
Closes #3802 

Corresponding RAFT PR: rapidsai/raft#217

Authors:
  - Corey J. Nolet (https://github.com/cjnolet)

Approvers:
  - Dante Gama Dessavre (https://github.com/dantegd)

URL: #3824

cpp/src/hierarchy/pw_dist_graph.cuh

@@ -24,6 +24,7 @@
 #include <raft/distance/distance.cuh>
 
 #include <rmm/device_uvector.hpp>
+#include <rmm/exec_policy.hpp>
 
 #include <raft/linalg/distance_type.h>
 #include <raft/mr/device/buffer.hpp>
@@ -71,6 +72,7 @@ void pairwise_distances(const raft::handle_t &handle, const value_t *X,
                         value_idx *indptr, value_idx *indices, value_t *data) {
   auto d_alloc = handle.get_device_allocator();
   auto stream = handle.get_stream();
+  auto exec_policy = rmm::exec_policy(stream);
 
   value_idx nnz = m * m;
 
@@ -90,6 +92,19 @@ void pairwise_distances(const raft::handle_t &handle, const value_t *X,
   // usage to hand it a sparse array here.
   raft::distance::pairwise_distance<value_t, value_idx>(
     X, X, data, m, m, n, workspace, metric, stream);
+
+  // self-loops get max distance
+  auto transform_in = thrust::make_zip_iterator(
+    thrust::make_tuple(thrust::make_counting_iterator(0), data));
+
+  thrust::transform(
+    exec_policy, transform_in, transform_in + nnz, data,
+    [=] __device__(const thrust::tuple<value_idx, value_t> &tup) {
+      value_idx idx = thrust::get<0>(tup);
+      bool self_loop = idx % m == idx / m;
+      return (self_loop * std::numeric_limits<value_t>::max()) +
+             (!self_loop * thrust::get<1>(tup));
+    });
 }
 
 /**

python/cuml/test/test_agglomerative.py

@@ -25,9 +25,27 @@
 import cupy as cp
 
 
+@pytest.mark.parametrize('connectivity', ['knn', 'pairwise'])
+def test_duplicate_distances(connectivity):
+    X = cp.array([[0.0, 0.0, 0.0], [0.0, 0.0, 0.0], [2.0, 2.0, 2.0]])
+
+    cuml_agg = AgglomerativeClustering(n_clusters=2, affinity="euclidean",
+                                       linkage="single", n_neighbors=3,
+                                       connectivity=connectivity)
+
+    sk_agg = cluster.AgglomerativeClustering(n_clusters=2,
+                                             affinity="euclidean",
+                                             linkage="single")
+
+    cuml_agg.fit(X)
+    sk_agg.fit(X.get())
+
+    assert(adjusted_rand_score(cuml_agg.labels_, sk_agg.labels_) == 1.0)
+
+
 @pytest.mark.parametrize('nrows', [100, 1000])
 @pytest.mark.parametrize('ncols', [25, 50])
-@pytest.mark.parametrize('nclusters', [2, 10, 50])
+@pytest.mark.parametrize('nclusters', [1, 2, 10, 50])
 @pytest.mark.parametrize('k', [3, 5, 15])
 @pytest.mark.parametrize('connectivity', ['knn', 'pairwise'])
 def test_single_linkage_sklearn_compare(nrows, ncols, nclusters,
@@ -37,17 +55,13 @@ def test_single_linkage_sklearn_compare(nrows, ncols, nclusters,
                       ncols,
                       nclusters,
                       cluster_std=1.0,
-                      shuffle=False,
-                      random_state=42)
+                      shuffle=False)
 
     cuml_agg = AgglomerativeClustering(
         n_clusters=nclusters, affinity='euclidean', linkage='single',
         n_neighbors=k, connectivity=connectivity)
 
-    try:
-        cuml_agg.fit(X)
-    except Exception:
-        cuml_agg.fit(X)
+    cuml_agg.fit(X)
 
     sk_agg = cluster.AgglomerativeClustering(
         n_clusters=nclusters, affinity='euclidean', linkage='single')