Addition of HasMax/HasMin/HasStandardDeviation/HasMean constraint suggestions

.gitignore

@@ -6,4 +6,5 @@ target/
 .vscode/
 .bloop/
 .DS_Store
-
+.scalafmt.conf
+*.log

Makefile

@@ -1,2 +1,4 @@
 build:
 	mvn clean install
+compile:
+	mvn clean compile

src/main/scala/com/amazon/deequ/examples/ConstraintSuggestionExample.scala

@@ -50,7 +50,7 @@ private[examples] object ConstraintSuggestionExample extends App {
     // to suggest constraints
     val suggestionResult = ConstraintSuggestionRunner()
       .onData(data)
-      .addConstraintRules(Rules.DEFAULT)
+      .addConstraintRules(Rules.EXTENDED)
       .run()
 
     // We can now investigate the constraints that deequ suggested. We get a textual description

src/main/scala/com/amazon/deequ/suggestions/ConstraintSuggestionRunner.scala

@@ -34,6 +34,11 @@ object Rules {
     Seq(CompleteIfCompleteRule(), RetainCompletenessRule(), RetainTypeRule(),
       CategoricalRangeRule(), FractionalCategoricalRangeRule(),
       NonNegativeNumbersRule())
+
+  val NUMERICAL: Seq[ConstraintRule[ColumnProfile]] =
+    Seq(HasMax(), HasMin(), HasMean(), HasStandardDeviation())
+
+  val EXTENDED: Seq[ConstraintRule[ColumnProfile]] = DEFAULT ++ NUMERICAL
 }
 
 private[suggestions] case class ConstraintSuggestionMetricsRepositoryOptions(

src/main/scala/com/amazon/deequ/suggestions/rules/HasMax.scala

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not
+ * use this file except in compliance with the License. A copy of the License
+ * is located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.deequ.suggestions.rules
+
+import com.amazon.deequ.checks.Check
+import com.amazon.deequ.constraints.Constraint.maxConstraint
+import com.amazon.deequ.profiles.ColumnProfile
+import com.amazon.deequ.profiles.NumericColumnProfile
+import com.amazon.deequ.suggestions.ConstraintSuggestion
+import com.amazon.deequ.checks
+
+/** If we see only non-negative numbers in a column, we suggest a corresponding
+  * constraint
+  */
+case class HasMax() extends ConstraintRule[ColumnProfile] {
+
+  override def shouldBeApplied(profile: ColumnProfile, numRecords: Long): Boolean = {
+    profile match {
+      case np: NumericColumnProfile => np.maximum.isDefined
+      case _ => false
+    }
+  }
+
+  override def candidate(profile: ColumnProfile, numRecords: Long): ConstraintSuggestion = {
+    val maximum: Double = profile match { case np: NumericColumnProfile => np.maximum.get }
+
+    val description = s"'${profile.column}' <= $maximum"
+    val constraint = maxConstraint(profile.column, _ == maximum)
+
+    ConstraintSuggestion(
+      constraint,
+      profile.column,
+      s"Maximum: $maximum",
+      description,
+      this,
+      s""".hasMax("${profile.column}", _ == $maximum)"""
+    )
+  }
+
+  override val ruleDescription: String = "If we see a numeric column, " +
+    "we suggest a corresponding Maximum value constraint"
+}

src/main/scala/com/amazon/deequ/suggestions/rules/HasMean.scala

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not
+ * use this file except in compliance with the License. A copy of the License
+ * is located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.deequ.suggestions.rules
+
+import com.amazon.deequ.checks.Check
+import com.amazon.deequ.constraints.Constraint.meanConstraint
+import com.amazon.deequ.profiles.ColumnProfile
+import com.amazon.deequ.profiles.NumericColumnProfile
+import com.amazon.deequ.suggestions.ConstraintSuggestion
+import com.amazon.deequ.checks
+
+/** If we see only non-negative numbers in a column, we suggest a corresponding
+  * constraint
+  */
+case class HasMean() extends ConstraintRule[ColumnProfile] {
+
+  override def shouldBeApplied(profile: ColumnProfile, numRecords: Long): Boolean = {
+    profile match {
+      case np: NumericColumnProfile => np.mean.isDefined
+      case _ => false
+    }
+  }
+
+  override def candidate(profile: ColumnProfile, numRecords: Long): ConstraintSuggestion = {
+    val mean: Double = profile match { case np: NumericColumnProfile => np.maximum.get }
+
+    val description = s"'${profile.column}' <= $mean"
+    val constraint = meanConstraint(profile.column, _ == mean)
+
+    ConstraintSuggestion(
+      constraint,
+      profile.column,
+      s"Mean: $mean",
+      description,
+      this,
+      s""".hasMean("${profile.column}", _ == $mean)"""
+    )
+  }
+
+  override val ruleDescription: String = "If we see a numeric column, " +
+    "we suggest a corresponding Mean value constraint"
+}

src/main/scala/com/amazon/deequ/suggestions/rules/HasMin.scala

@@ -0,0 +1,55 @@
+/**
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not
+ * use this file except in compliance with the License. A copy of the License
+ * is located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.deequ.suggestions.rules
+
+import com.amazon.deequ.checks.Check
+import com.amazon.deequ.constraints.Constraint.minConstraint
+import com.amazon.deequ.profiles.ColumnProfile
+import com.amazon.deequ.profiles.NumericColumnProfile
+import com.amazon.deequ.suggestions.ConstraintSuggestion
+
+/** If we see only non-negative numbers in a column, we suggest a corresponding
+  * constraint
+  */
+case class HasMin() extends ConstraintRule[ColumnProfile] {
+
+  override def shouldBeApplied(profile: ColumnProfile, numRecords: Long): Boolean = {
+    profile match {
+      case np: NumericColumnProfile => np.minimum.isDefined
+      case _ => false
+    }
+  }
+
+  override def candidate(profile: ColumnProfile, numRecords: Long): ConstraintSuggestion = {
+    val minimum: Double = profile match { case np: NumericColumnProfile => np.minimum.get }
+
+    val description = s"'${profile.column}' >= $minimum"
+    val constraint = minConstraint(profile.column, _ == minimum)
+
+    ConstraintSuggestion(
+      constraint,
+      profile.column,
+      s"Minimum: $minimum",
+      description,
+      this,
+      s""".hasMin("${profile.column}", _ == $minimum)"""
+    )
+  }
+
+  override val ruleDescription: String = "If we see a numeric column, " +
+    "we suggest a corresponding Minimum value constraint"
+}

src/main/scala/com/amazon/deequ/suggestions/rules/HasStandardDeviation.scala

@@ -0,0 +1,56 @@
+/**
+ * Copyright 2023 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"). You may not
+ * use this file except in compliance with the License. A copy of the License
+ * is located at
+ *
+ *     http://aws.amazon.com/apache2.0/
+ *
+ * or in the "license" file accompanying this file. This file is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ *
+ */
+
+package com.amazon.deequ.suggestions.rules
+
+import com.amazon.deequ.checks.Check
+import com.amazon.deequ.constraints.Constraint.standardDeviationConstraint
+import com.amazon.deequ.profiles.ColumnProfile
+import com.amazon.deequ.profiles.NumericColumnProfile
+import com.amazon.deequ.suggestions.ConstraintSuggestion
+import com.amazon.deequ.checks
+
+/** If we see only non-negative numbers in a column, we suggest a corresponding
+  * constraint
+  */
+case class HasStandardDeviation() extends ConstraintRule[ColumnProfile] {
+
+  override def shouldBeApplied(profile: ColumnProfile, numRecords: Long): Boolean = {
+    profile match {
+      case np: NumericColumnProfile => np.mean.isDefined
+      case _ => false
+    }
+  }
+
+  override def candidate(profile: ColumnProfile, numRecords: Long): ConstraintSuggestion = {
+    val stdDev: Double = profile match { case np: NumericColumnProfile => np.stdDev.get }
+
+    val description = s"'${profile.column}' <= $stdDev"
+    val constraint = standardDeviationConstraint(profile.column, _ == stdDev)
+
+    ConstraintSuggestion(
+      constraint,
+      profile.column,
+      s"stdDev: $stdDev",
+      description,
+      this,
+      s""".hasStandardDeviation("${profile.column}", _ == $stdDev)"""
+    )
+  }
+
+  override val ruleDescription: String = "If we see a numeric column, " +
+    "we suggest a corresponding standard deviation value constraint"
+}

src/test/scala/com/amazon/deequ/suggestions/ConstraintSuggestionsIntegrationTest.scala

@@ -76,7 +76,7 @@ class ConstraintSuggestionsIntegrationTest extends WordSpec with SparkContextSpe
 
       val constraintSuggestionResult = ConstraintSuggestionRunner()
         .onData(data)
-        .addConstraintRules(Rules.DEFAULT)
+        .addConstraintRules(Rules.EXTENDED)
         .addConstraintRule(UniqueIfApproximatelyUniqueRule())
         .run()
 

src/test/scala/com/amazon/deequ/suggestions/rules/ConstraintRulesTest.scala

@@ -37,8 +37,46 @@ class ConstraintRulesTest extends WordSpec with FixtureSupport with SparkContext
       val complete = StandardColumnProfile("col1", 1.0, 100, String, false, Map.empty, None)
       val incomplete = StandardColumnProfile("col1", .25, 100, String, false, Map.empty, None)
 
+      val completeInteger =
+        getFakeNumericColumnProfileWithMinMaxMeanAndStdDev(
+          columnName = "integer1",
+          completeness = 1.0,
+          dataType = DataTypeInstances.Integral,
+          mean = 12,
+          maximum = 123,
+          minimum = -123,
+          stdDev = 1
+        )
+
+      val incompleteFractional =
+        getFakeNumericColumnProfileWithMinMaxMeanAndStdDev(
+          columnName = "fractional1",
+          completeness = 0.9,
+          dataType = DataTypeInstances.Fractional,
+          mean = 12.5,
+          maximum = 123.7,
+          minimum = -123.89,
+          stdDev = 1.023
+        )
+
       assert(CompleteIfCompleteRule().shouldBeApplied(complete, 1000))
       assert(!CompleteIfCompleteRule().shouldBeApplied(incomplete, 1000))
+      assert(!HasMax().shouldBeApplied(complete, 1000))
+      assert(!HasMean().shouldBeApplied(complete, 1000))
+      assert(!HasMin().shouldBeApplied(complete, 1000))
+      assert(!HasStandardDeviation().shouldBeApplied(complete, 1000))
+      assert(CompleteIfCompleteRule().shouldBeApplied(complete, 1000))
+      assert(CompleteIfCompleteRule().shouldBeApplied(complete, 1000))
+      assert(HasMax().shouldBeApplied(completeInteger, 1000))
+      assert(HasMax().shouldBeApplied(incompleteFractional, 1000))
+      assert(HasMean().shouldBeApplied(completeInteger, 1000))
+      assert(HasMean().shouldBeApplied(completeInteger, 1000))
+      assert(HasMin().shouldBeApplied(completeInteger, 1000))
+      assert(HasMin().shouldBeApplied(incompleteFractional, 1000))
+      assert(HasStandardDeviation().shouldBeApplied(completeInteger, 1000))
+      assert(HasStandardDeviation().shouldBeApplied(completeInteger, 1000))
+      assert(HasStandardDeviation().shouldBeApplied(incompleteFractional, 1000))
+      assert(HasStandardDeviation().shouldBeApplied(incompleteFractional, 1000))
     }
 
     "return evaluable constraint candidates" in

src/test/scala/com/amazon/deequ/utils/FixtureSupport.scala

@@ -16,6 +16,8 @@
 
 package com.amazon.deequ.utils
 
+import com.amazon.deequ.analyzers.DataTypeInstances
+import com.amazon.deequ.profiles.NumericColumnProfile
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.DataFrame
 import org.apache.spark.sql.Row
@@ -414,4 +416,32 @@ trait FixtureSupport {
       ("bar", 20)
     ).toDF("name", "age")
   }
+
+  def getFakeNumericColumnProfileWithMinMaxMeanAndStdDev(
+    columnName: String,
+    completeness: Double,
+    dataType: DataTypeInstances.Value,
+    minimum: Double,
+    maximum: Double,
+    mean: Double,
+    stdDev: Double
+  ): NumericColumnProfile = {
+
+    NumericColumnProfile(
+      column = columnName,
+      completeness = completeness,
+      approximateNumDistinctValues = 1000,
+      dataType = dataType,
+      isDataTypeInferred = false,
+      typeCounts = Map[String, Long](),
+      histogram = None,
+      kll = None,
+      mean = Some(mean),
+      maximum = Some(maximum),
+      minimum = Some(minimum),
+      sum = Some(1000.879),
+      stdDev = Some(1.023),
+      approxPercentiles = None
+    )
+  }
 }