Add possibility to parse JSON to set

* Use `jsonutils.nim` hookable API to add possibility to deserialize
  JSON arrays directly to `HashSet` and `OrderedSet` types and
  respectively to serialize those types to JSON arrays.

* Move serialization/deserialization functionality for `Table` and
  `OrderedTable` types from `jsonutils.nim` to `tables.nim` via the
  hookable API.

* Add object `jsonutils.Joptions` and parameter from its type to
  `jsonutils.fromJson` procedure to control whether to allow
  deserializeing JSON objects to Nim objects when the JSON has some
  extra or missing keys.

* Add unit tests for the added functionalities to `tjsonutils.nim`.

lib/pure/collections/sets.nim

@@ -80,6 +80,8 @@ type
     ## <#initOrderedSet,int>`_ before calling other procs on it.
     data: OrderedKeyValuePairSeq[A]
     counter, first, last: int
+  SomeSet*[A] = HashSet[A] | OrderedSet[A]
+    ## Type union representing `HashSet` or `OrderedSet`.
 
 const
   defaultInitialSize* = 64
@@ -907,7 +909,52 @@ iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] =
   forAllOrderedPairs:
     yield (idx, s.data[h].key)
 
-
+proc fromJsonHook*[A, B](s: var SomeSet[A], jsonNode: B) =
+  ## Enables `fromJson` for `HashSet` and `OrderedSet` types.
+  ## 
+  ## See also:
+  ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_
+  runnableExamples:
+    import std/[json, jsonutils]
+    type
+      Foo = object
+        hs: HashSet[string]
+        os: OrderedSet[string]
+    var foo: Foo
+    fromJson(foo, parseJson("""
+      {"hs": ["hash", "set"], "os": ["ordered", "set"]}"""))
+    assert foo.hs == ["hash", "set"].toHashSet
+    assert foo.os == ["ordered", "set"].toOrderedSet
+
+  mixin jsonTo
+  assert jsonNode.kind == JArray,
+          "The kind of the `jsonNode` must be JArray, but its actual " &
+          "type is " & $jsonNode.kind & "."
+  clear(s)
+  for v in jsonNode:
+    incl(s, jsonTo(v, A))
+
+proc toJsonHook*[A](s: SomeSet[A]): auto =
+  ## Enables `toJson` for `HashSet` and `OrderedSet` types.
+  ##
+  ## See also:
+  ## * `fromJsonHook proc<#fromJsonHook,SomeSet[A],B>`_
+  runnableExamples:
+    import std/[json, jsonutils]
+    type
+      Foo = object
+        hs: HashSet[string]
+        os: OrderedSet[string]
+    let foo = Foo(
+      hs: ["hash"].toHashSet,
+      os: ["ordered", "set"].toOrderedSet)
+    assert $toJson(foo) == """{"hs":["hash"],"os":["ordered","set"]}"""
+
+  mixin newJArray
+  mixin toJson
+  result = newJArray()
+  for k in s:
+    add(result, toJson(k))
 
 # -----------------------------------------------------------------------
 

lib/pure/collections/tables.nim

@@ -1750,9 +1750,53 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B =
     yield t.data[h].val
     assert(len(t) == L, "the length of the table changed while iterating over it")
 
-
-
-
+proc fromJsonHook*[K, V, JN](t: var (Table[K, V] | OrderedTable[K, V]),
+                             jsonNode: JN) =
+  ## Enables `fromJson` for `Table` and `OrderedTable` types.
+  ## 
+  ## See also:
+  ## * `toJsonHook proc<#toJsonHook,(Table[K,V]|OrderedTable[K,V])>`_
+  runnableExamples:
+    import std/[json, jsonutils]
+    type
+      Foo = object
+        t: Table[string, int]
+        ot: OrderedTable[string, int]
+    var foo: Foo
+    fromJson(foo, parseJson("""
+      {"t":{"two":2,"one":1},"ot":{"one":1,"three":3}}"""))
+    assert foo.t == [("one", 1), ("two", 2)].toTable
+    assert foo.ot == [("one", 1), ("three", 3)].toOrderedTable
+
+  mixin jsonTo
+  assert jsonNode.kind == JObject,
+          "The kind of the `jsonNode` must be JObject, but its actual " &
+          "type is " & $jsonNode.kind & "."
+  clear(t)
+  for k, v in jsonNode:
+    t[k] = jsonTo(v, V)
+
+proc toJsonHook*[K, V](t: (Table[K, V] | OrderedTable[K, V])): auto =
+  ## Enables `toJson` for `Table` and `OrderedTable` types.
+  ##
+  ## See also:
+  ## * `fromJsonHook proc<#fromJsonHook,(Table[K,V]|OrderedTable[K,V]),JN>`_
+  runnableExamples:
+    import std/[json, jsonutils]
+    type
+      Foo = object
+        t: Table[string, int]
+        ot: OrderedTable[string, int]
+    let foo = Foo(
+      t: [("two", 2)].toTable,
+      ot: [("one", 1), ("three", 3)].toOrderedTable)
+    assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}"""
+
+  mixin newJObject
+  mixin toJson
+  result = newJObject()
+  for k, v in pairs(t):
+    result[k] = toJson(v)
 
 # ---------------------------------------------------------------------------
 # --------------------------- OrderedTableRef -------------------------------

lib/std/jsonutils.nim

@@ -13,16 +13,14 @@ runnableExamples:
   let j = a.toJson
   doAssert j.jsonTo(type(a)).toJson == j
 
-import std/[json,tables,strutils]
+import std/[json,strutils]
 
 #[
 xxx
-use toJsonHook,fromJsonHook for Table|OrderedTable
 add Options support also using toJsonHook,fromJsonHook and remove `json=>options` dependency
 
 Future directions:
 add a way to customize serialization, for eg:
-* allowing missing or extra fields in JsonNode
 * field renaming
 * allow serializing `enum` and `char` as `string` instead of `int`
   (enum is more compact/efficient, and robust to enum renamings, but string
@@ -32,6 +30,18 @@ add a way to customize serialization, for eg:
 
 import std/macros
 
+type
+  Joptions* = object
+    ## Options controlling the behavior of `fromJson`.
+    allowExtraKeys*: bool
+      ## If `true` Nim's object to which the JSON is parsed is not required to
+      ## have a field for every JSON key.
+    allowMissingKeys*: bool
+      ## If `true` Nim's object to which JSON is parsed is allowed to have
+      ## fields without corresponding JSON keys. This is allowed only for
+      ## non-discriminant fields.
+    # in future work: a key rename could be added
+
 proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
 proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".}
 template distinctBase[T](a: T): untyped = distinctBase(type(a))(a)
@@ -92,31 +102,43 @@ proc checkJsonImpl(cond: bool, condStr: string, msg = "") =
 template checkJson(cond: untyped, msg = "") =
   checkJsonImpl(cond, astToStr(cond), msg)
 
-template fromJsonFields(a, b, T, keys) =
-  checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull
+template fromJsonFields(obj, json, T, discKeys, opt) =
+  checkJson json.kind == JObject, $json.kind # we could customize whether to allow JNull
   var num = 0
-  for key, val in fieldPairs(a):
+  var numMatched = discKeys.len
+  for key, val in fieldPairs(obj):
     num.inc
-    when key notin keys:
-      if b.hasKey key:
-        fromJson(val, b[key])
-      else:
-        # we could customize to allow this
-        checkJson false, $($T, key, b)
-  checkJson b.len == num, $(b.len, num, $T, b) # could customize
-
-proc fromJson*[T](a: var T, b: JsonNode) =
+    when key notin discKeys:
+      if json.hasKey key:
+        numMatched.inc
+        fromJson(val, json[key])
+      elif not opt.allowMissingKeys:
+        checkJson false, $($T, key, json)
+
+  let ok =
+    if opt.allowExtraKeys and opt.allowMissingKeys:
+      true
+    elif opt.allowExtraKeys:
+      # This check is redundant because if here missing keys are not allowed,
+      # and if `num != numMatched` it will fail in the loop above but it is left
+      # for clarity.
+      assert num == numMatched
+      num == numMatched
+    elif opt.allowMissingKeys:
+      json.len == numMatched
+    else:
+      json.len == num and num == numMatched
+
+  checkJson ok, $(json.len, num, numMatched, $T, json)
+
+proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) =
   ## inplace version of `jsonTo`
   #[
   adding "json path" leading to `b` can be added in future work.
   ]#
   checkJson b != nil, $($T, b)
   when compiles(fromJsonHook(a, b)): fromJsonHook(a, b)
   elif T is bool: a = to(b,T)
-  elif T is Table | OrderedTable:
-    a.clear
-    for k,v in b:
-      a[k] = jsonTo(v, typeof(a[k]))
   elif T is enum:
     case b.kind
     of JInt: a = T(b.getBiggestInt())
@@ -152,10 +174,10 @@ proc fromJson*[T](a: var T, b: JsonNode) =
       jsonTo(b[key], typ)
     a = initCaseObject(T, fun)
     const keys = getDiscriminants(T)
-    fromJsonFields(a, b, T, keys)
+    fromJsonFields(a, b, T, keys, opt)
   elif T is tuple:
     when isNamedTuple(T):
-      fromJsonFields(a, b, T, seq[string].default)
+      fromJsonFields(a, b, T, seq[string].default, opt)
     else:
       checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull
       var i = 0
@@ -175,9 +197,6 @@ proc toJson*[T](a: T): JsonNode =
   ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to
   ## customize serialization, see strtabs.toJsonHook for an example.
   when compiles(toJsonHook(a)): result = toJsonHook(a)
-  elif T is Table | OrderedTable:
-    result = newJObject()
-    for k, v in pairs(a): result[k] = toJson(v)
   elif T is object | tuple:
     when T is object or isNamedTuple(T):
       result = newJObject()

tests/stdlib/tjsonutils.nim

@@ -13,7 +13,7 @@ proc testRoundtrip[T](t: T, expected: string) =
   t2.fromJson(j)
   doAssert t2.toJson == j
 
-import tables
+import tables, sets
 import strtabs
 
 type Foo = ref object
@@ -119,5 +119,95 @@ template fn() =
     testRoundtrip(Foo[int](t1: false, z2: 7)): """{"t1":false,"z2":7}"""
     # pending https://github.com/nim-lang/Nim/issues/14698, test with `type Foo[T] = ref object`
 
+  block hashSet:
+    testRoundtrip(HashSet[string]()): "[]"
+    testRoundtrip([""].toHashSet): """[""]"""
+    testRoundtrip(["one"].toHashSet): """["one"]"""
+
+    var s: HashSet[string]
+    fromJson(s, parseJson("""["one","two"]"""))
+    doAssert s == ["one", "two"].toHashSet
+    let jsonNode = toJson(s)
+    doAssert jsonNode.kind == JArray
+    doAssert jsonNode.len == 2
+    let elem0 = jsonNode.elems[0]
+    let elem1 = jsonNode.elems[1]
+    doAssert elem0.kind == JString
+    doAssert elem1.kind == JString
+    doAssert elem0.str == "one" or elem0.str == "two"
+    doAssert elem1.str == "one" or elem1.str == "two"
+    doAssert elem0.str != elem1.str
+
+  block orderedSet:
+    testRoundtrip(["one", "two", "three"].toOrderedSet):
+      """["one","two","three"]"""
+
+  block testJoptions:
+    type
+      AboutLifeUniverseAndEverythingElse = object
+        question: string
+        answer: int
+
+    block testExceptionOnExtraKeys:
+      var guide: AboutLifeUniverseAndEverythingElse
+      let json = parseJson(
+        """{"question":"6*9=?","answer":42,"author":"Douglas Adams"}""")
+      doAssertRaises ValueError, fromJson(guide, json)
+      doAssertRaises ValueError,
+                     fromJson(guide, json, Joptions(allowMissingKeys: true))
+
+      type
+        A = object
+          a1,a2,a3: int
+      var a: A
+      let j = parseJson("""{"a3": 1, "a4": 2}""")
+      doAssertRaises ValueError,
+                     fromJson(a, j, Joptions(allowMissingKeys: true))
+
+    block testExceptionOnMissingKeys:
+      var guide: AboutLifeUniverseAndEverythingElse
+      let json = parseJson("""{"answer":42}""")
+      doAssertRaises ValueError, fromJson(guide, json)
+      doAssertRaises ValueError,
+                     fromJson(guide, json, Joptions(allowExtraKeys: true))
+
+    block testAllowExtraKeys:
+      var guide: AboutLifeUniverseAndEverythingElse
+      let json = parseJson(
+        """{"question":"6*9=?","answer":42,"author":"Douglas Adams"}""")
+      fromJson(guide, json, Joptions(allowExtraKeys: true))
+      doAssert guide.question == "6*9=?"
+      doAssert guide.answer == 42
+
+    block testAllowMissingKeys:
+      var guide: AboutLifeUniverseAndEverythingElse
+      let json = parseJson("""{"answer":42}""")
+      fromJson(guide, json, Joptions(allowMissingKeys: true))
+      doAssert guide.question == ""
+      doAssert guide.answer == 42
+
+    block testAllowExtraAndMissingKeys:
+      var guide: AboutLifeUniverseAndEverythingElse
+      let json = parseJson(
+        """{"answer":42,"author":"Douglas Adams"}""")
+      fromJson(guide, json, Joptions(
+        allowExtraKeys: true, allowMissingKeys: true))
+      doAssert guide.question == ""
+      doAssert guide.answer == 42
+
+    block testExceptionOnMissingDiscriminantKey:
+      type
+        Foo = object
+          a: array[2, string]
+          case b: bool
+          of false: f: float
+          of true: t: tuple[i: int, s: string]
+
+      var foo: Foo
+      let json = parseJson("""{"a":["one","two"]}""")
+      doAssertRaises ValueError, fromJson(foo, json)
+      doAssertRaises ValueError,
+                     fromJson(foo, json, Joptions(allowMissingKeys: true))
+
 static: fn()
 fn()