From bc127a24be5ba2ae9f2f2720a7f1a299cb0fac5d Mon Sep 17 00:00:00 2001 From: agandhi Date: Tue, 8 May 2018 09:16:52 -0700 Subject: [PATCH 1/7] Test commit: Adding method comments --- .../cloudmanager/state/SolrState.scala | 39 +++++++++++++++++-- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index e88f0c5..17084a5 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -87,22 +87,54 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config lazy val activeReplicas = allReplicas.filter(_.active) lazy val inactiveReplicas = allReplicas.filterNot(activeReplicas.contains) + /** + * Returns all replicas for a given collection + * @param collection + * @return + */ def replicasFor(collection: String): Seq[SolrReplica] = allReplicas.filter(_.collection == collection) + + /** + * Returns all replicas for a given collection, slice combination + * @param collection + * @param sliceName + * @return + */ def replicasFor(collection: String, sliceName: String): Seq[SolrReplica] = replicasFor(collection).filter(_.slice.getName == sliceName) def liveReplicasFor(collection: String): Seq[SolrReplica] = replicasFor(collection).filter(_.active) - def nodesWithCollection(collection: String) = replicasFor(collection).map(_.node).distinct - def nodesWithoutCollection(collection: String) = liveNodes -- nodesWithCollection(collection) + def nodesWithCollection(collection: String): Seq[String] = replicasFor(collection).map(_.node).distinct + def nodesWithoutCollection(collection: String): Set[String] = liveNodes -- nodesWithCollection(collection) + /** + * + * @param nodeList + * @return Map (canonical host name -> node) + */ def dnsNameMap(nodeList: Set[String] = liveNodes): Map[String,String] = { nodeList.map( node => InetAddress.getByName(node.take(node.indexOf(':'))).getCanonicalHostName -> node ).toMap } + /** + * + * @param indicators + * @param allowOfflineReferences + * @param ignoreUnrecognized + * @return + */ def mapToNodes(indicators: Option[Seq[String]], allowOfflineReferences: Boolean = false, ignoreUnrecognized: Boolean = false): Option[Seq[String]] = { indicators.map(mapToNodes(_, allowOfflineReferences, ignoreUnrecognized)) } + + /** + * + * @param indicators + * @param allowOfflineReferences + * @param ignoreUnrecognized + * @return + */ def mapToNodes(indicators: Seq[String], allowOfflineReferences: Boolean, ignoreUnrecognized: Boolean): Seq[String] = { - val nodeList = indicators.foldLeft(Seq[String]())( (acc, indicator) => { + val nodeList: Seq[String] = indicators.foldLeft(Seq[String]())((acc, indicator) => { indicator.toLowerCase match { case "all" => val nodeList = if (allowOfflineReferences) allNodes else liveNodes @@ -141,6 +173,7 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config findUnambigousNode(dnsMap, (s: String) => s == fragment) .orElse(findUnambigousNode(dnsMap, (s: String) => s.contains(fragment))) } + def findUnambigousNode(dnsMap: Map[String,String], comparison: (String) => Boolean): Option[String] = { val matchingMaps = dnsMap.filter{ case (dnsName,canonName) => comparison(dnsName) || comparison(canonName) From c5bf43468685d102522b832efb28da5b13efe77f Mon Sep 17 00:00:00 2001 From: agandhi Date: Tue, 8 May 2018 13:55:29 -0700 Subject: [PATCH 2/7] Some more comments --- .../cloudmanager/state/SolrState.scala | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index 17084a5..5a56d1a 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -107,15 +107,18 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config def nodesWithoutCollection(collection: String): Set[String] = liveNodes -- nodesWithCollection(collection) /** - * + * Create a map of the fully qualified domain name to the representation of the node in ZK + * InetAddress.getByName: Determines the IP address of a host, given the host's name + * InetAddress.getCanonicalHostName: Gets the fully qualified domain name for this IP address * @param nodeList - * @return Map (canonical host name -> node) + * @return Map (dns name -> node zk representation) */ def dnsNameMap(nodeList: Set[String] = liveNodes): Map[String,String] = { nodeList.map( node => InetAddress.getByName(node.take(node.indexOf(':'))).getCanonicalHostName -> node ).toMap } /** + * Takes the value specified for nodes by the user via the CLI and returns their equivalent representations in Zookeeper * * @param indicators * @param allowOfflineReferences @@ -128,8 +131,9 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config /** * - * @param indicators - * @param allowOfflineReferences + * @param indicators user passed in argument for nodes via the command line e.g. "all","empty", a comma separated + * list of IPs or a regular expression + * @param allowOfflineReferences allow down nodes to be considered * @param ignoreUnrecognized * @return */ @@ -143,10 +147,12 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config val nodeList = if (allowOfflineReferences) unusedNodes else unusedNodes -- downNodes acc ++ nodeList.toSeq case r if r.startsWith("regex=") => + //If the user specified a regular expression val pattern = r.stripPrefix("regex=").r - val nodeList = dnsNameMap(if (allowOfflineReferences) allNodes else liveNodes) - nodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq + val nodeList = dnsNameMap(if (allowOfflineReferences) allNodes else liveNodes) //resolve the list of available nodes + nodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq //filter these nodes based on the provided pattern case i => + //If a comma separated list of nodes is specified, then for each node val nodeName = Try(Seq(canonicalNodeName(i, allowOfflineReferences))).recover({ case e: ManagerException if ignoreUnrecognized => comment.warn(e.getMessage) @@ -185,15 +191,18 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config } val nodeList = if (allowOfflineReferences) allNodes else liveNodes + + //If the value specified by the user exactly matches a ZK node representation, return this value as is if (nodeList.contains(hostIndicator)) { hostIndicator } else { + unambiguousFragment(hostIndicator, dnsNameMap(nodeList)).getOrElse { val chunks = hostIndicator.split(':') val host = chunks.head val port = if (chunks.length > 1) ":" + chunks.last else "" - val ipAndPort = InetAddress.getByName(host).getHostAddress + port + val ipAndPort = InetAddress.getByName(host).getHostAddress + port// val matches = nodeList.filter((node) => node.contains(ipAndPort)) matches.size match { case 0 => throw new ManagerException(s"Could not determine a live node from '$hostIndicator'") From 4c11ad234396ba439955d2dda9a4b15849c2a232 Mon Sep 17 00:00:00 2001 From: agandhi Date: Tue, 8 May 2018 16:57:55 -0700 Subject: [PATCH 3/7] Saving progress --- .../cloudmanager/state/SolrState.scala | 136 +++++++++++++----- 1 file changed, 97 insertions(+), 39 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index 5a56d1a..cf4613f 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -7,6 +7,7 @@ import org.apache.solr.common.cloud.{ClusterState, Replica, Slice, ZkStateReader import scala.collection.JavaConverters._ import scala.util.Try +import scala.util.matching.Regex object SolrReplica { def hostName(nodeName: String) = { @@ -107,16 +108,28 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config def nodesWithoutCollection(collection: String): Set[String] = liveNodes -- nodesWithCollection(collection) /** - * Create a map of the fully qualified domain name to the representation of the node in ZK - * InetAddress.getByName: Determines the IP address of a host, given the host's name - * InetAddress.getCanonicalHostName: Gets the fully qualified domain name for this IP address - * @param nodeList + * + * This method take the representation of each node, resolves it and creates a map of + * fully qualified domain name -> node ZK representation + * + * @param nodeList list of ZK represenation of each node in the cluster * @return Map (dns name -> node zk representation) */ def dnsNameMap(nodeList: Set[String] = liveNodes): Map[String,String] = { nodeList.map( node => InetAddress.getByName(node.take(node.indexOf(':'))).getCanonicalHostName -> node ).toMap } + /** + * This method take the representation of each node, resolves it and creates a map of + * * IP address -> node ZK representation + * + * @param nodeList list of ZK represenation of each node in the cluster + * @return Map (ip address -> node zk representation) + */ + def ipNameMap(nodeList: Set[String] = liveNodes): Map[String,String] = { + nodeList.map( node => InetAddress.getByName(node.take(node.indexOf(':'))).getHostAddress -> node ).toMap + } + /** * Takes the value specified for nodes by the user via the CLI and returns their equivalent representations in Zookeeper * @@ -141,16 +154,15 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config val nodeList: Seq[String] = indicators.foldLeft(Seq[String]())((acc, indicator) => { indicator.toLowerCase match { case "all" => - val nodeList = if (allowOfflineReferences) allNodes else liveNodes + val nodeList: Set[String] = if (allowOfflineReferences) allNodes else liveNodes acc ++ nodeList case "empty" => val nodeList = if (allowOfflineReferences) unusedNodes else unusedNodes -- downNodes acc ++ nodeList.toSeq case r if r.startsWith("regex=") => //If the user specified a regular expression - val pattern = r.stripPrefix("regex=").r - val nodeList = dnsNameMap(if (allowOfflineReferences) allNodes else liveNodes) //resolve the list of available nodes - nodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq //filter these nodes based on the provided pattern + val pattern: Regex = r.stripPrefix("regex=").r + getNodeListUsingRegEx(pattern, allowOfflineReferences) case i => //If a comma separated list of nodes is specified, then for each node val nodeName = Try(Seq(canonicalNodeName(i, allowOfflineReferences))).recover({ @@ -166,52 +178,98 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config nodeList } + /** * Gets a known host name for a given string - * @param hostIndicator + * @param hostIndicator node indicator passed in by the user, this could be a host name or an IP, or + * an exact representation of the node in ZK * @param allowOfflineReferences * @throw ManagerException if a host could not be safely determined * @return A known canonical host */ def canonicalNodeName(hostIndicator: String, allowOfflineReferences: Boolean = false): String = { - - def unambiguousFragment(fragment: String, dnsMap: Map[String,String]): Option[String] = { - findUnambigousNode(dnsMap, (s: String) => s == fragment) - .orElse(findUnambigousNode(dnsMap, (s: String) => s.contains(fragment))) - } - - def findUnambigousNode(dnsMap: Map[String,String], comparison: (String) => Boolean): Option[String] = { - val matchingMaps = dnsMap.filter{ - case (dnsName,canonName) => comparison(dnsName) || comparison(canonName) - } - matchingMaps.toList match { - case (dnsName, canonName) :: Nil => Some(canonName) - case _ => None - } - } - - val nodeList = if (allowOfflineReferences) allNodes else liveNodes + val rawNodeList = if (allowOfflineReferences) allNodes else liveNodes //If the value specified by the user exactly matches a ZK node representation, return this value as is - if (nodeList.contains(hostIndicator)) { + if (rawNodeList.contains(hostIndicator)) { hostIndicator } else { + unambiguousFragment(hostIndicator, dnsNameMap(rawNodeList)).getOrElse( + unambiguousFragment(hostIndicator, ipNameMap(rawNodeList)).getOrElse( + { + //Here we attempt to resolve the indicator(user passed in value) instead of the values from ZK + val chunks = hostIndicator.split(':') + val host = chunks.head + val port = if (chunks.length > 1) ":" + chunks.last else "" + val ipAndPort = InetAddress.getByName(host).getHostAddress + port + val matches = rawNodeList.filter((node) => node.contains(ipAndPort)) + matches.size match { + case 0 => throw new ManagerException(s"Could not determine a live node from '$hostIndicator'") + case 1 => matches.head + case _ => throw new ManagerException(s"Ambiguous node name '$hostIndicator', possible matches: $matches") + } + }) + ) + } - unambiguousFragment(hostIndicator, dnsNameMap(nodeList)).getOrElse { - val chunks = hostIndicator.split(':') - val host = chunks.head - val port = if (chunks.length > 1) ":" + chunks.last else "" - val ipAndPort = InetAddress.getByName(host).getHostAddress + port// - val matches = nodeList.filter((node) => node.contains(ipAndPort)) - matches.size match { - case 0 => throw new ManagerException(s"Could not determine a live node from '$hostIndicator'") - case 1 => matches.head - case _ => throw new ManagerException(s"Ambiguous node name '$hostIndicator', possible matches: $matches") - } - } + } + + + /** + * + * @param fragment + * @param resolvedNodeMap + * @return + */ + def unambiguousFragment(fragment: String, resolvedNodeMap: Map[String,String]): Option[String] = { + findUnambigousNode(resolvedNodeMap, (s: String) => s == fragment) + .orElse(findUnambigousNode(resolvedNodeMap, (s: String) => s.contains(fragment))) + } + + /** + * + * @param resolvedNodeMap + * @param comparison + * @return + */ + def findUnambigousNode(resolvedNodeMap: Map[String,String], comparison: (String) => Boolean): Option[String] = { + val matchingMaps = resolvedNodeMap.filter{ + case (resolvedNodeName, zkNodeName) => comparison(resolvedNodeName) || comparison(zkNodeName) } + matchingMaps.toList match { + case (resolvedNodeName, zkNodeName) :: Nil => Some(zkNodeName) + case _ => None + } + } + + /** + * This method takes the passed in regular expression and searches for nodes that match this pattern + * In the first pass, the pattern attempts to match against fully qualified domain names + * If no nodes matched in the first pass, it then attempts to match against the IP addresses + * If no nodes matched in the second pass, it then attempts to match against the raw node representation returned by ZK + * Return an empty sequence if none of these work + * + * @param pattern the pattern to use for matching nodes + * @param allowOfflineReferences + * @return + */ + def getNodeListUsingRegEx(pattern: Regex, allowOfflineReferences: Boolean): Seq[String] = { + val rawNodeList = if (allowOfflineReferences) allNodes else liveNodes + + //First try by matching to the fully qualified domain name + val dnsNodeList: Map[String, String] = dnsNameMap(rawNodeList) + dnsNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq + + //If matching via domain name didn't work, try matching to the IP addresses + val ipNodeList: Map[String, String] = ipNameMap(rawNodeList) + ipNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq + + //If either of these approaches do not work, trying matching with the unresolved list of nodes i.e. use the + //node list from the cluster state without transforming it + rawNodeList.filter{pattern.findFirstIn(_).nonEmpty}.toSeq } + } From 92f881208f19e08237c275923cfd00dae75bcda4 Mon Sep 17 00:00:00 2001 From: agandhi Date: Wed, 9 May 2018 00:05:35 -0700 Subject: [PATCH 4/7] Some more changes, method comments --- .../cloudmanager/state/SolrState.scala | 81 ++++++++++--------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index cf4613f..bd16093 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -131,9 +131,11 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config } /** - * Takes the value specified for nodes by the user via the CLI and returns their equivalent representations in Zookeeper + * Takes the value specified for nodes by the user via the CLI and returns their + * corresponding string representation from the ZK cluster state * - * @param indicators + * @param indicators user passed in argument for nodes via the command line e.g. "all","empty", a comma separated + * list of IPs or a regular expression * @param allowOfflineReferences * @param ignoreUnrecognized * @return @@ -190,55 +192,57 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config def canonicalNodeName(hostIndicator: String, allowOfflineReferences: Boolean = false): String = { val rawNodeList = if (allowOfflineReferences) allNodes else liveNodes - //If the value specified by the user exactly matches a ZK node representation, return this value as is + //If the value specified by the user exactly matches a node from the cluster state, return this value as is if (rawNodeList.contains(hostIndicator)) { hostIndicator } else { unambiguousFragment(hostIndicator, dnsNameMap(rawNodeList)).getOrElse( unambiguousFragment(hostIndicator, ipNameMap(rawNodeList)).getOrElse( - { - //Here we attempt to resolve the indicator(user passed in value) instead of the values from ZK - val chunks = hostIndicator.split(':') - val host = chunks.head - val port = if (chunks.length > 1) ":" + chunks.last else "" - val ipAndPort = InetAddress.getByName(host).getHostAddress + port - val matches = rawNodeList.filter((node) => node.contains(ipAndPort)) - matches.size match { - case 0 => throw new ManagerException(s"Could not determine a live node from '$hostIndicator'") - case 1 => matches.head - case _ => throw new ManagerException(s"Ambiguous node name '$hostIndicator', possible matches: $matches") + { + //Here we attempt to transform the indicator(user passed in value) instead of the values from the cluster state + val chunks = hostIndicator.split(':') + val host = chunks.head + val port = if (chunks.length > 1) ":" + chunks.last else "" + val ipAndPort = InetAddress.getByName(host).getHostAddress + port + val matches = rawNodeList.filter((node) => node.contains(ipAndPort)) + matches.size match { + case 0 => throw new ManagerException(s"Could not determine a live node from '$hostIndicator'") + case 1 => matches.head + case _ => throw new ManagerException(s"Ambiguous node name '$hostIndicator', possible matches: $matches") + } } - }) + ) ) } - } /** * - * @param fragment - * @param resolvedNodeMap + * @param fragment user passed in string for node identification + * @param nodeComparisonMap map where key is the resolved name of a node(IP or Host) and value is its + * string representation in the cluster state * @return */ - def unambiguousFragment(fragment: String, resolvedNodeMap: Map[String,String]): Option[String] = { - findUnambigousNode(resolvedNodeMap, (s: String) => s == fragment) - .orElse(findUnambigousNode(resolvedNodeMap, (s: String) => s.contains(fragment))) + def unambiguousFragment(fragment: String, nodeComparisonMap: Map[String,String]): Option[String] = { + findUnambigousNode(nodeComparisonMap, (s: String) => s == fragment) + .orElse(findUnambigousNode(nodeComparisonMap, (s: String) => s.contains(fragment))) } /** * - * @param resolvedNodeMap - * @param comparison + * @param nodeComparisonMap map where key is the resolved name of a node(IP or Host) and value is its + * string representation in the cluster state + * @param comparison function to use for comparison * @return */ - def findUnambigousNode(resolvedNodeMap: Map[String,String], comparison: (String) => Boolean): Option[String] = { - val matchingMaps = resolvedNodeMap.filter{ - case (resolvedNodeName, zkNodeName) => comparison(resolvedNodeName) || comparison(zkNodeName) + def findUnambigousNode(nodeComparisonMap: Map[String,String], comparison: (String) => Boolean): Option[String] = { + val matchingMaps = nodeComparisonMap.filter{ + case (resolvedNodeString, clusterNodeString) => comparison(resolvedNodeString) || comparison(clusterNodeString) } matchingMaps.toList match { - case (resolvedNodeName, zkNodeName) :: Nil => Some(zkNodeName) + case (resolvedNodeName, clusterNodeString) :: Nil => Some(clusterNodeString) case _ => None } } @@ -256,19 +260,22 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config * @return */ def getNodeListUsingRegEx(pattern: Regex, allowOfflineReferences: Boolean): Seq[String] = { - val rawNodeList = if (allowOfflineReferences) allNodes else liveNodes + val clusterStateNodeList = if (allowOfflineReferences) allNodes else liveNodes + var nodes = Seq() //First try by matching to the fully qualified domain name - val dnsNodeList: Map[String, String] = dnsNameMap(rawNodeList) + val dnsNodeList: Map[String, String] = dnsNameMap(clusterStateNodeList) dnsNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq - - //If matching via domain name didn't work, try matching to the IP addresses - val ipNodeList: Map[String, String] = ipNameMap(rawNodeList) - ipNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq - - //If either of these approaches do not work, trying matching with the unresolved list of nodes i.e. use the - //node list from the cluster state without transforming it - rawNodeList.filter{pattern.findFirstIn(_).nonEmpty}.toSeq + if(dnsNodeList.isEmpty){ + //If matching via domain name didn't work, try matching to the IP addresses + val ipNodeList: Map[String, String] = ipNameMap(clusterStateNodeList) + ipNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq + if(ipNodeList.isEmpty){ + //If either of these approaches do not work, trying matching with the unresolved list of nodes i.e. use the + //node list from the cluster state without transforming it + clusterStateNodeList.filter{pattern.findFirstIn(_).nonEmpty}.toSeq + } + } } From 774e9f409217ed557d6e42953f6354a622874e72 Mon Sep 17 00:00:00 2001 From: agandhi Date: Wed, 9 May 2018 00:27:40 -0700 Subject: [PATCH 5/7] Refactored getNodeListUsingRegEx, couldn't figure out a idiomatic way to do this --- .../cloudmanager/state/SolrState.scala | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index bd16093..5ffd3af 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -262,21 +262,22 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config def getNodeListUsingRegEx(pattern: Regex, allowOfflineReferences: Boolean): Seq[String] = { val clusterStateNodeList = if (allowOfflineReferences) allNodes else liveNodes - var nodes = Seq() //First try by matching to the fully qualified domain name val dnsNodeList: Map[String, String] = dnsNameMap(clusterStateNodeList) - dnsNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq - if(dnsNodeList.isEmpty){ + val dnsMatch = dnsNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq + if(!dnsMatch.isEmpty){ dnsMatch } else{ //If matching via domain name didn't work, try matching to the IP addresses val ipNodeList: Map[String, String] = ipNameMap(clusterStateNodeList) - ipNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq - if(ipNodeList.isEmpty){ + val ipMatch = ipNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq + if(!ipMatch.isEmpty){ ipMatch } else { //If either of these approaches do not work, trying matching with the unresolved list of nodes i.e. use the //node list from the cluster state without transforming it - clusterStateNodeList.filter{pattern.findFirstIn(_).nonEmpty}.toSeq + val clusterStateMatch = clusterStateNodeList.filter{pattern.findFirstIn(_).nonEmpty}.toSeq + if(!clusterStateMatch.isEmpty){ clusterStateMatch } else { + //return an empty sequence if nothing matches + Seq() + } } } } - - } From 5a4ddfa9b5e31b384f90632e455f9f271e5854e4 Mon Sep 17 00:00:00 2001 From: agandhi Date: Wed, 9 May 2018 10:25:33 -0700 Subject: [PATCH 6/7] Removing unwanted comments --- .../whitepages/cloudmanager/state/SolrState.scala | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index 5ffd3af..33cd2bd 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -88,21 +88,12 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config lazy val activeReplicas = allReplicas.filter(_.active) lazy val inactiveReplicas = allReplicas.filterNot(activeReplicas.contains) - /** - * Returns all replicas for a given collection - * @param collection - * @return - */ + def replicasFor(collection: String): Seq[SolrReplica] = allReplicas.filter(_.collection == collection) - /** - * Returns all replicas for a given collection, slice combination - * @param collection - * @param sliceName - * @return - */ def replicasFor(collection: String, sliceName: String): Seq[SolrReplica] = replicasFor(collection).filter(_.slice.getName == sliceName) + def liveReplicasFor(collection: String): Seq[SolrReplica] = replicasFor(collection).filter(_.active) def nodesWithCollection(collection: String): Seq[String] = replicasFor(collection).map(_.node).distinct def nodesWithoutCollection(collection: String): Set[String] = liveNodes -- nodesWithCollection(collection) From 8fad29cb4ad126c7542fa7b5c64fe594bc6fa76d Mon Sep 17 00:00:00 2001 From: agandhi Date: Thu, 10 May 2018 08:00:18 -0700 Subject: [PATCH 7/7] Refactored to use common methods that take functions for comparison and to determine success --- .../cloudmanager/state/SolrState.scala | 179 +++++++++++------- 1 file changed, 108 insertions(+), 71 deletions(-) diff --git a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala index 33cd2bd..c478c9d 100644 --- a/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala +++ b/src/main/scala/com/whitepages/cloudmanager/state/SolrState.scala @@ -157,13 +157,12 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config val pattern: Regex = r.stripPrefix("regex=").r getNodeListUsingRegEx(pattern, allowOfflineReferences) case i => - //If a comma separated list of nodes is specified, then for each node - val nodeName = Try(Seq(canonicalNodeName(i, allowOfflineReferences))).recover({ - case e: ManagerException if ignoreUnrecognized => - comment.warn(e.getMessage) - Seq[String]() - }).get - acc ++ nodeName + val matches: Seq[String] = getNodeListUsingIndicators(i, allowOfflineReferences) + matches.size match { + case 0 => throw new ManagerException(s"Could not determine a live node from '$indicator'") + case 1 => acc ++ matches + case _ => throw new ManagerException(s"Ambiguous node name '$indicator', possible matches: $matches") + } } }) if (nodeList.isEmpty) @@ -171,104 +170,142 @@ case class SolrState(state: ClusterState, collectionInfo: CollectionInfo, config nodeList } - /** * Gets a known host name for a given string - * @param hostIndicator node indicator passed in by the user, this could be a host name or an IP, or + * @param indicator node indicator passed in by the user, this could be a host name or an IP, or * an exact representation of the node in ZK * @param allowOfflineReferences * @throw ManagerException if a host could not be safely determined * @return A known canonical host */ - def canonicalNodeName(hostIndicator: String, allowOfflineReferences: Boolean = false): String = { - val rawNodeList = if (allowOfflineReferences) allNodes else liveNodes + def canonicalNodeName(indicator: String, allowOfflineReferences: Boolean = false): String = { + val matches: Seq[String] = getNodeListUsingIndicators(indicator, allowOfflineReferences) + matches.size match { + case 0 => throw new ManagerException(s"Could not determine a live node from '$indicator'") + case 1 => matches.head + case _ => throw new ManagerException(s"Ambiguous node name '$indicator', possible matches: $matches") + } + } + + def getNodeListUsingRegEx(pattern: Regex, allowOfflineReferences: Boolean): Seq[String] = { + val clusterStateNodeList: Set[String] = if (allowOfflineReferences) allNodes else liveNodes + getNodeList(clusterStateNodeList, + comparison = (s: String) => pattern.findFirstIn(s).nonEmpty, + mapSuccessCriteria = (matches: Map[String,String]) => matches.size>0, + setSuccessCriteria = (matches: Seq[String]) => matches.size>0 + ) + } + + + /** + * + * @param indicator + * @param allowOfflineReferences + * @return returns Seq[String] instead of String to facilitate error handling + */ + def getNodeListUsingIndicators(indicator:String, allowOfflineReferences: Boolean): Seq[String] = { + val clusterStateNodeList: Set[String] = if (allowOfflineReferences) allNodes else liveNodes //If the value specified by the user exactly matches a node from the cluster state, return this value as is - if (rawNodeList.contains(hostIndicator)) { - hostIndicator - } - else { - unambiguousFragment(hostIndicator, dnsNameMap(rawNodeList)).getOrElse( - unambiguousFragment(hostIndicator, ipNameMap(rawNodeList)).getOrElse( - { - //Here we attempt to transform the indicator(user passed in value) instead of the values from the cluster state - val chunks = hostIndicator.split(':') - val host = chunks.head - val port = if (chunks.length > 1) ":" + chunks.last else "" - val ipAndPort = InetAddress.getByName(host).getHostAddress + port - val matches = rawNodeList.filter((node) => node.contains(ipAndPort)) - matches.size match { - case 0 => throw new ManagerException(s"Could not determine a live node from '$hostIndicator'") - case 1 => matches.head - case _ => throw new ManagerException(s"Ambiguous node name '$hostIndicator', possible matches: $matches") - } + if (clusterStateNodeList.contains(indicator)) { + Seq(indicator) + } else { + def exactMatchComparison = (s: String) => s == indicator + def subStringMatchComparison = (s: String) => s.contains(indicator) + def singleNodeMapSuccessCriteria: Map[String, String] => Boolean = (matches:Map[String,String]) => matches.size == 1 + def singleNodeSetSuccessCriteria: Seq[String] => Boolean = (matches: Seq[String]) => matches.size == 1 + + //Attempt to resolve using exact match + val exactMatches = getNodeList(clusterStateNodeList, exactMatchComparison, singleNodeMapSuccessCriteria, singleNodeSetSuccessCriteria) + if(!exactMatches.isEmpty) { exactMatches } else { + //Attempt to resolve using fuzzy match + val subStringMatches = getNodeList(clusterStateNodeList, subStringMatchComparison, singleNodeMapSuccessCriteria, singleNodeSetSuccessCriteria) + if(!subStringMatches.isEmpty) { subStringMatches } else { + val ipPortMatches = matchWithIPAndPort(indicator,clusterStateNodeList) + if(!ipPortMatches.isEmpty) { ipPortMatches } else { + Seq() } - ) - ) + } + } } } + /** + * Useful for matching when there are multiple solr instances on the same node registered with + * cluster (registration will be with same node but different port) + * @param indicator + * @param clusterStateNodeList + * @return + */ + def matchWithIPAndPort(indicator: String, clusterStateNodeList: Set[String]): Seq[String] = { + val chunks = indicator.split(':') + val host = chunks.head + val port = if (chunks.length > 1) ":" + chunks.last else "" + val ipAndPort = InetAddress.getByName(host).getHostAddress + port + val matches: Set[String] = clusterStateNodeList.filter((node) => node.contains(ipAndPort)) + matches.toSeq + } /** * - * @param fragment user passed in string for node identification - * @param nodeComparisonMap map where key is the resolved name of a node(IP or Host) and value is its - * string representation in the cluster state + * @param clusterStateNodeList list of nodes from the cluster state + * @param comparison + * @param mapSuccessCriteria function to determine matching success when comparing against resolved node representations (IP or host) + * @param setSuccessCriteria function to determine matching success when comparing against node representations from cluster state * @return */ - def unambiguousFragment(fragment: String, nodeComparisonMap: Map[String,String]): Option[String] = { - findUnambigousNode(nodeComparisonMap, (s: String) => s == fragment) - .orElse(findUnambigousNode(nodeComparisonMap, (s: String) => s.contains(fragment))) + def getNodeList(clusterStateNodeList: Set[String], comparison: (String) => Boolean, + mapSuccessCriteria: Map[String,String] => Boolean, + setSuccessCriteria: Seq[String] => Boolean ): Seq[String] = { + //First try by matching to the fully qualified domain name + val dnsNodeMap: Map[String, String] = dnsNameMap(clusterStateNodeList) + val dnsMatches = findNodes(dnsNodeMap, comparison, mapSuccessCriteria) + if(!dnsMatches.isEmpty){ dnsMatches } else { + //If matching via domain name didn't work, try matching to the IP address + val ipNodeMap: Map[String, String] = ipNameMap(clusterStateNodeList) + val ipMatches = findNodes(ipNodeMap, comparison, mapSuccessCriteria) + if(!ipMatches.isEmpty){ ipMatches } else { + //If either of these approaches do not work, trying matching with the list of node strings from the cluster state + val clusterStateMatches = findNodes(clusterStateNodeList, comparison, setSuccessCriteria) + if(!clusterStateMatches.isEmpty){ ipMatches } else { + Seq() + } + } + } } /** - * + * Performs a comparison against a map which contains a resolved version of a node in the cluster (IP or host) as the key + * and the string representation of this node in the cluster state * @param nodeComparisonMap map where key is the resolved name of a node(IP or Host) and value is its * string representation in the cluster state * @param comparison function to use for comparison + * @param successCriteria function that takes the "matched" elements and determines success or failure e.g. match only 1, match atleast 1 * @return */ - def findUnambigousNode(nodeComparisonMap: Map[String,String], comparison: (String) => Boolean): Option[String] = { - val matchingMaps = nodeComparisonMap.filter{ + def findNodes(nodeComparisonMap: Map[String,String], comparison: (String) => Boolean, successCriteria: Map[String,String] => Boolean):Seq[String] = { + val matchingMaps: Map[String, String] = nodeComparisonMap.filter{ case (resolvedNodeString, clusterNodeString) => comparison(resolvedNodeString) || comparison(clusterNodeString) } - matchingMaps.toList match { - case (resolvedNodeName, clusterNodeString) :: Nil => Some(clusterNodeString) - case _ => None + if(successCriteria(matchingMaps)){ + matchingMaps.map({case (k,v) => v}).toSeq + } else{ + Seq() } } - /** - * This method takes the passed in regular expression and searches for nodes that match this pattern - * In the first pass, the pattern attempts to match against fully qualified domain names - * If no nodes matched in the first pass, it then attempts to match against the IP addresses - * If no nodes matched in the second pass, it then attempts to match against the raw node representation returned by ZK - * Return an empty sequence if none of these work - * - * @param pattern the pattern to use for matching nodes - * @param allowOfflineReferences + * Performs the comparison directly against list of nodes from the cluster state + * @param clusterStateNodeList + * @param comparison + * @param successCriteria * @return */ - def getNodeListUsingRegEx(pattern: Regex, allowOfflineReferences: Boolean): Seq[String] = { - val clusterStateNodeList = if (allowOfflineReferences) allNodes else liveNodes - - //First try by matching to the fully qualified domain name - val dnsNodeList: Map[String, String] = dnsNameMap(clusterStateNodeList) - val dnsMatch = dnsNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq - if(!dnsMatch.isEmpty){ dnsMatch } else{ - //If matching via domain name didn't work, try matching to the IP addresses - val ipNodeList: Map[String, String] = ipNameMap(clusterStateNodeList) - val ipMatch = ipNodeList.filter{ case (k, v) => pattern.findFirstIn(k).nonEmpty}.values.toSeq - if(!ipMatch.isEmpty){ ipMatch } else { - //If either of these approaches do not work, trying matching with the unresolved list of nodes i.e. use the - //node list from the cluster state without transforming it - val clusterStateMatch = clusterStateNodeList.filter{pattern.findFirstIn(_).nonEmpty}.toSeq - if(!clusterStateMatch.isEmpty){ clusterStateMatch } else { - //return an empty sequence if nothing matches - Seq() - } - } + def findNodes(clusterStateNodeList: Set[String], comparison: (String) => Boolean, successCriteria: Seq[String] => Boolean ): Seq[String] = { + val clusterStateMatch: Seq[String] = clusterStateNodeList.filter{comparison(_)}.toSeq + if(successCriteria(clusterStateMatch)){ clusterStateMatch } else { + Seq() } } + }