Add support of upsert (insert or update, on conflict) for Postgres an…

…d MySQL
zio · Apr 10, 2018 · 3aec36e · 3aec36e
1 parent a42d0fb
commit 3aec36e
Show file tree

Hide file tree

Showing 36 changed files with 925 additions and 119 deletions.
diff --git a/README.md b/README.md
@@ -703,7 +703,74 @@ val a = quote {
 ctx.run(a)
 // DELETE FROM Person WHERE name = ''
 ```
-
+
+### insert or update (upsert, conflict)
+
+Upsert is only supported by Postgres and MySQL
+
+#### Postgres
+Ignore conflict
+```scala
+val a = quote {
+  query[Product].insert(_.id -> 1, _.sku -> 10).onConflictIgnore
+}
+
+// INSERT INTO Product AS t (id,sku) VALUES (1, 10) ON CONFLICT DO NOTHING
+```
+
+Ignore conflict by explicitly setting conflict target
+```scala
+val a = quote {
+  query[Product].insert(_.id -> 1, _.sku -> 10).onConflictIgnore(_.id)
+}
+
+// INSERT INTO Product AS t (id,sku) VALUES (1, 10) ON CONFLICT (id) DO NOTHING
+```
+
+Resolve conflict by updating existing row if needed. In `onConflictUpdate(target)((t, e) => assignment)`: `target` refers to
+conflict target, `t` - to existing row and `e` - to excluded, e.g. row proposed for insert.
+```scala
+val a = quote {
+  query[Product]
+    .insert(_.id -> 1, _.sku -> 10)
+    .onConflictUpdate(_.id)((t, e) => t.sku -> (t.sku + e.sku))
+}
+
+// INSERT INTO Product AS t (id,sku) VALUES (1, 10) ON CONFLICT (id) DO UPDATE SET sku = (t.sku + EXCLUDED.sku)
+```
+
+#### MySQL
+
+Ignore any conflict, e.g. `insert ignore`
+```scala
+val a = quote {
+  query[Product].insert(_.id -> 1, _.sku -> 10).onConflictIgnore
+}
+
+// INSERT IGNORE INTO Product (id,sku) VALUES (1, 10)
+```
+
+Ignore duplicate key conflict by explicitly setting it
+```scala
+val a = quote {
+  query[Product].insert(_.id -> 1, _.sku -> 10).onConflictIgnore(_.id)
+}
+
+// INSERT INTO Product (id,sku) VALUES (1, 10) ON DUPLICATE KEY UPDATE id=id
+```
+
+Resolve duplicate key by updating existing row if needed. In `onConflictUpdate((t, e) => assignment)`: `t` refers to
+existing row and `e` - to values, e.g. values proposed for insert.
+```scala
+val a = quote {
+  query[Product]
+    .insert(_.id -> 1, _.sku -> 10)
+    .onConflictUpdate((t, e) => t.sku -> (t.sku + e.sku))
+}
+
+// INSERT INTO Product (id,sku) VALUES (1, 10) ON DUPLICATE KEY UPDATE sku = (sku + VALUES(sku))
+```
+
 ## IO Monad
 
 Quill provides an IO monad that allows the user to express multiple computations and execute them separately. This mechanism is also known as a free monad, which provides a way of expressing computations as referentially-transparent values and isolates the unsafe IO operations into a single operation. For instance:

diff --git a/quill-async-mysql/src/test/scala/io/getquill/context/async/mysql/ConflictAsyncSpec.scala b/quill-async-mysql/src/test/scala/io/getquill/context/async/mysql/ConflictAsyncSpec.scala
@@ -0,0 +1,38 @@
+package io.getquill.context.async.mysql
+
+import io.getquill.context.sql.ConflictSpec
+
+import scala.concurrent.ExecutionContext.Implicits.global
+
+class ConflictAsyncSpec extends ConflictSpec {
+  val ctx = testContext
+  import ctx._
+
+  override protected def beforeAll(): Unit = {
+    await(ctx.run(qr1.delete))
+    ()
+  }
+
+  "INSERT IGNORE" in {
+    import `onConflictIgnore`._
+    await(ctx.run(testQuery1)) mustEqual res1
+    await(ctx.run(testQuery2)) mustEqual res2
+    await(ctx.run(testQuery3)) mustEqual res3
+  }
+
+  "ON DUPLICATE KEY UPDATE i=i " in {
+    import `onConflictIgnore(_.i)`._
+    await(ctx.run(testQuery1)) mustEqual res1
+    await(ctx.run(testQuery2)) mustEqual res2
+    await(ctx.run(testQuery3)) mustEqual res3
+  }
+
+  "ON DUPLICATE KEY UPDATE ..." in {
+    import `onConflictUpdate((t, e) => ...)`._
+    await(ctx.run(testQuery(e1))) mustEqual res1
+    await(ctx.run(testQuery(e2))) mustEqual res2 + 1
+    await(ctx.run(testQuery(e3))) mustEqual res3 + 1
+    await(ctx.run(testQuery4)) mustEqual res4
+  }
+}
+
diff --git a/quill-async-mysql/src/test/scala/io/getquill/context/async/mysql/MysqlAsyncContextSpec.scala b/quill-async-mysql/src/test/scala/io/getquill/context/async/mysql/MysqlAsyncContextSpec.scala
@@ -45,4 +45,9 @@ class MysqlAsyncContextSpec extends Spec {
     }
     ctx.close
   }
+
+  override protected def beforeAll(): Unit = {
+    await(testContext.run(qr1.delete))
+    ()
+  }
 }
diff --git a/...-async-postgres/src/test/scala/io/getquill/context/async/postgres/ConflictAsyncSpec.scala b/...-async-postgres/src/test/scala/io/getquill/context/async/postgres/ConflictAsyncSpec.scala
@@ -0,0 +1,37 @@
+package io.getquill.context.async.postgres
+
+import io.getquill.context.sql.ConflictSpec
+
+import scala.concurrent.ExecutionContext.Implicits.global
+
+class ConflictAsyncSpec extends ConflictSpec {
+  val ctx = testContext
+  import ctx._
+
+  override protected def beforeAll(): Unit = {
+    await(ctx.run(qr1.delete))
+    ()
+  }
+
+  "ON CONFLICT DO NOTHING" in {
+    import `onConflictIgnore`._
+    await(ctx.run(testQuery1)) mustEqual res1
+    await(ctx.run(testQuery2)) mustEqual res2
+    await(ctx.run(testQuery3)) mustEqual res3
+  }
+
+  "ON CONFLICT (i) DO NOTHING" in {
+    import `onConflictIgnore(_.i)`._
+    await(ctx.run(testQuery1)) mustEqual res1
+    await(ctx.run(testQuery2)) mustEqual res2
+    await(ctx.run(testQuery3)) mustEqual res3
+  }
+
+  "ON CONFLICT (i) DO UPDATE ..." in {
+    import `onConflictUpdate(_.i)((t, e) => ...)`._
+    await(ctx.run(testQuery(e1))) mustEqual res1
+    await(ctx.run(testQuery(e2))) mustEqual res2
+    await(ctx.run(testQuery(e3))) mustEqual res3
+    await(ctx.run(testQuery4)) mustEqual res4
+  }
+}
diff --git a/...postgres/src/test/scala/io/getquill/context/async/postgres/PostgresAsyncContextSpec.scala b/...postgres/src/test/scala/io/getquill/context/async/postgres/PostgresAsyncContextSpec.scala
@@ -45,4 +45,9 @@ class PostgresAsyncContextSpec extends Spec {
     }
     ctx.close
   }
+
+  override protected def beforeAll(): Unit = {
+    await(testContext.run(qr1.delete))
+    ()
+  }
 }
diff --git a/quill-cassandra/src/main/scala/io/getquill/context/cassandra/CqlIdiom.scala b/quill-cassandra/src/main/scala/io/getquill/context/cassandra/CqlIdiom.scala
@@ -42,7 +42,7 @@ trait CqlIdiom extends Idiom {
       case a: TraversableOperation => a.token
       case a @ (
         _: Function | _: FunctionApply | _: Dynamic | _: OptionOperation | _: Block |
-        _: Val | _: Ordering | _: QuotedReference | _: If
+        _: Val | _: Ordering | _: QuotedReference | _: If | _: Excluded | _: Existing
         ) =>
         fail(s"Invalid cql: '$a'")
     }

diff --git a/quill-core/src/main/scala/io/getquill/MirrorIdiom.scala b/quill-core/src/main/scala/io/getquill/MirrorIdiom.scala
@@ -40,6 +40,8 @@ class MirrorIdiom extends Idiom {
     case ast: QuotedReference      => ast.ast.token
     case ast: Lift                 => ast.token
     case ast: Assignment           => ast.token
+    case ast: Excluded             => ast.token
+    case ast: Existing             => ast.token
   }
 
   implicit def ifTokenizer(implicit liftTokenizer: Tokenizer[Lift]): Tokenizer[If] = Tokenizer[If] {
@@ -181,12 +183,45 @@ class MirrorIdiom extends Idiom {
     case e => stmt"${e.name.token}"
   }
 
+  implicit val excludedTokenizer: Tokenizer[Excluded] = Tokenizer[Excluded] {
+    case Excluded(ident) => stmt"${ident.token}"
+  }
+
+  implicit val existingTokenizer: Tokenizer[Existing] = Tokenizer[Existing] {
+    case Existing(ident) => stmt"${ident.token}"
+  }
+
   implicit def actionTokenizer(implicit liftTokenizer: Tokenizer[Lift]): Tokenizer[Action] = Tokenizer[Action] {
     case Update(query, assignments)    => stmt"${query.token}.update(${assignments.token})"
     case Insert(query, assignments)    => stmt"${query.token}.insert(${assignments.token})"
     case Delete(query)                 => stmt"${query.token}.delete"
     case Returning(query, alias, body) => stmt"${query.token}.returning((${alias.token}) => ${body.token})"
     case Foreach(query, alias, body)   => stmt"${query.token}.foreach((${alias.token}) => ${body.token})"
+    case c: Conflict                   => stmt"${c.token}"
+  }
+
+  implicit def conflictTokenizer(implicit liftTokenizer: Tokenizer[Lift]): Tokenizer[Conflict] = {
+
+    def targetProps(l: List[Property]) = l.map(p => Transform(p) {
+      case Ident(_) => Ident("_")
+    })
+
+    implicit val conflictTargetTokenizer = Tokenizer[Conflict.Target] {
+      case Conflict.NoTarget          => stmt""
+      case Conflict.Properties(props) => stmt"(${targetProps(props).token})"
+    }
+
+    val updateAssignsTokenizer = Tokenizer[Assignment] {
+      case Assignment(i, p, v) =>
+        stmt"(${i.token}, e) => ${p.token} -> ${scopedTokenizer(v)}"
+    }
+
+    Tokenizer[Conflict] {
+      case Conflict(i, t, Conflict.Update(assign)) =>
+        stmt"${i.token}.onConflictUpdate${t.token}(${assign.map(updateAssignsTokenizer.token).mkStmt()})"
+      case Conflict(i, t, Conflict.Ignore) =>
+        stmt"${i.token}.onConflictIgnore${t.token}"
+    }
   }
 
   implicit def assignmentTokenizer(implicit liftTokenizer: Tokenizer[Lift]): Tokenizer[Assignment] = Tokenizer[Assignment] {

diff --git a/quill-core/src/main/scala/io/getquill/ast/Ast.scala b/quill-core/src/main/scala/io/getquill/ast/Ast.scala
@@ -92,6 +92,8 @@ case class If(condition: Ast, `then`: Ast, `else`: Ast) extends Ast
 
 case class Assignment(alias: Ident, property: Ast, value: Ast) extends Ast
 
+case class Excluded(alias: Ident) extends Ast
+case class Existing(alias: Ident) extends Ast
 //************************************************************
 
 sealed trait Operation extends Ast
@@ -129,6 +131,16 @@ case class Returning(action: Ast, alias: Ident, property: Ast) extends Action
 
 case class Foreach(query: Ast, alias: Ident, body: Ast) extends Action
 
+case class Conflict(insert: Ast, target: Conflict.Target, action: Conflict.Action) extends Action
+object Conflict {
+  sealed trait Target
+  case object NoTarget extends Target
+  case class Properties(props: List[Property]) extends Target
+
+  sealed trait Action
+  case object Ignore extends Action
+  case class Update(assignments: List[Assignment]) extends Action
+}
 //************************************************************
 
 case class Dynamic(tree: Any) extends Ast

diff --git a/quill-core/src/main/scala/io/getquill/ast/StatefulTransformer.scala b/quill-core/src/main/scala/io/getquill/ast/StatefulTransformer.scala
@@ -14,15 +14,14 @@ trait StatefulTransformer[T] {
       case e: Ident                => (e, this)
       case e: OptionOperation      => apply(e)
       case e: TraversableOperation => apply(e)
+      case e: Property             => apply(e)
+      case e: Existing             => (e, this)
+      case e: Excluded             => (e, this)
 
       case Function(a, b) =>
         val (bt, btt) = apply(b)
         (Function(a, bt), btt)
 
-      case Property(a, b) =>
-        val (at, att) = apply(a)
-        (Property(at, b), att)
-
       case Infix(a, b) =>
         val (bt, btt) = apply(b)(_.apply)
         (Infix(a, bt), btt)
@@ -179,6 +178,13 @@ trait StatefulTransformer[T] {
         (Assignment(a, bt, ct), ctt)
     }
 
+  def apply(e: Property): (Property, StatefulTransformer[T]) =
+    e match {
+      case Property(a, b) =>
+        val (at, att) = apply(a)
+        (Property(at, b), att)
+    }
+
   def apply(e: Operation): (Operation, StatefulTransformer[T]) =
     e match {
       case UnaryOperation(o, a) =>
@@ -228,6 +234,27 @@ trait StatefulTransformer[T] {
         val (at, att) = apply(a)
         val (ct, ctt) = att.apply(c)
         (Foreach(at, b, ct), ctt)
+      case Conflict(a, b, c) =>
+        val (at, att) = apply(a)
+        val (bt, btt) = att.apply(b)
+        val (ct, ctt) = btt.apply(c)
+        (Conflict(at, bt, ct), ctt)
+    }
+
+  def apply(e: Conflict.Target): (Conflict.Target, StatefulTransformer[T]) =
+    e match {
+      case Conflict.NoTarget => (e, this)
+      case Conflict.Properties(a) =>
+        val (at, att) = apply(a)(_.apply)
+        (Conflict.Properties(at), att)
+    }
+
+  def apply(e: Conflict.Action): (Conflict.Action, StatefulTransformer[T]) =
+    e match {
+      case Conflict.Ignore => (e, this)
+      case Conflict.Update(a) =>
+        val (at, att) = apply(a)(_.apply)
+        (Conflict.Update(at), att)
     }
 
   def apply[U, R](list: List[U])(f: StatefulTransformer[T] => U => (R, StatefulTransformer[T])) =

diff --git a/quill-core/src/main/scala/io/getquill/ast/StatelessTransformer.scala b/quill-core/src/main/scala/io/getquill/ast/StatelessTransformer.scala
@@ -11,7 +11,7 @@ trait StatelessTransformer {
       case e: Assignment           => apply(e)
       case Function(params, body)  => Function(params, apply(body))
       case e: Ident                => e
-      case Property(a, name)       => Property(apply(a), name)
+      case e: Property             => apply(e)
       case Infix(a, b)             => Infix(a, b.map(apply))
       case e: OptionOperation      => apply(e)
       case e: TraversableOperation => apply(e)
@@ -22,6 +22,8 @@ trait StatelessTransformer {
       case Block(statements)       => Block(statements.map(apply))
       case Val(name, body)         => Val(name, apply(body))
       case o: Ordering             => o
+      case e: Excluded             => e
+      case e: Existing             => e
     }
 
   def apply(o: OptionOperation): OptionOperation =
@@ -72,6 +74,11 @@ trait StatelessTransformer {
       case Assignment(a, b, c) => Assignment(a, apply(b), apply(c))
     }
 
+  def apply(e: Property): Property =
+    e match {
+      case Property(a, name) => Property(apply(a), name)
+    }
+
   def apply(e: Operation): Operation =
     e match {
       case UnaryOperation(o, a)            => UnaryOperation(o, apply(a))
@@ -97,6 +104,19 @@ trait StatelessTransformer {
       case Delete(query)                     => Delete(apply(query))
       case Returning(query, alias, property) => Returning(apply(query), alias, apply(property))
       case Foreach(query, alias, body)       => Foreach(apply(query), alias, apply(body))
+      case Conflict(query, target, action)   => Conflict(apply(query), apply(target), apply(action))
+    }
+
+  def apply(e: Conflict.Target): Conflict.Target =
+    e match {
+      case Conflict.NoTarget          => e
+      case Conflict.Properties(props) => Conflict.Properties(props.map(apply))
+    }
+
+  def apply(e: Conflict.Action): Conflict.Action =
+    e match {
+      case Conflict.Ignore          => e
+      case Conflict.Update(assigns) => Conflict.Update(assigns.map(apply))
     }
 
 }