scope based symbol resolution; fixes generic sandwich problem; fix #13747, #17965, #13970: declarations at non-module scope now work with generics

compiler/ast.nim

@@ -603,6 +603,7 @@ type
                           # mean: never)
     skPackage,            # symbol is a package (used for canonicalization)
     skAlias               # an alias (needs to be resolved immediately)
+    skMixin               # in `mixin foo`, declares `foo` as skMixin if no other symbols exist
   TSymKinds* = set[TSymKind]
 
 const

compiler/astalgo.nim

@@ -777,10 +777,18 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
       replaceSlot = h
     h = nextTry(h, high(t.data))
   if replaceSlot >= 0:
-    result = t.data[replaceSlot] # found it
-    if not onConflictKeepOld:
-      t.data[replaceSlot] = n # overwrite it with newer definition!
-    return result # but return the old one
+    # another option would be to keep both but handle conflict resolution by prefering the non-skMixin
+    # see D20210521T082944
+    if t.data[replaceSlot].kind == skMixin:
+      t.data[replaceSlot] = n
+      return nil # report no conflict
+    elif n.kind == skMixin:
+      return nil # report no conflict
+    else:
+      result = t.data[replaceSlot] # found it
+      if not onConflictKeepOld:
+        t.data[replaceSlot] = n # overwrite it with newer definition!
+      return result # but return the old one
   elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)

compiler/debugutils.nim

@@ -13,6 +13,15 @@ useful debugging flags:
 ## future work
 * expose and improve astalgo.debug, replacing it by std/prettyprints,
   refs https://github.com/nim-lang/RFCs/issues/385
+
+## examples:
+{.define(nimCompilerDebug).}
+code
+{.undef(nimCompilerDebug).}
+
+when defined(nimDebugUtils):
+  if isCompilerDebug():
+    dbg t, ?.t.sym, ?.t.sym.owner, cl.owner
 ]#
 
 import options
@@ -54,3 +63,25 @@ proc isCompilerDebug*(): bool =
       {.undef(nimCompilerDebug).}
       echo 'x'
   conf0.isDefined("nimCompilerDebug")
+
+template debugScopesIf* =
+  if isCompilerDebug():
+    # TODO: callSite?
+    const infom1 = instantiationInfo(-1, true)
+    dbg(info = infom1)
+    debugScopes(c, limit = 100)
+
+template debugScopes2* =
+  # TODO: callSite?
+  const infom1 = instantiationInfo(-1, true)
+  dbg(info = infom1)
+  debugScopes(c, limit = 100)
+
+template dbgIf*(args: varargs[untyped]) =
+  if isCompilerDebug():
+    const infom1 = instantiationInfo(-1, true)
+    when varargsLen(args) > 0:
+      dbg(info = infom1, args)
+    else:
+      # otherwise info is wrong (points to here)
+      dbg(info = infom1)

compiler/evaltempl.nim

@@ -46,7 +46,7 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
            s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
         handleParam actual[s.owner.typ.len + s.position - 1]
       else:
-        internalAssert c.config, sfGenSym in s.flags or s.kind == skType
+        internalAssert c.config, sfGenSym in s.flags or s.kind in {skType, skMixin}
         var x = PSym(idTableGet(c.mapping, s))
         if x == nil:
           x = copySym(s, nextSymId(c.idgen))

compiler/lookups.nim

@@ -187,9 +187,45 @@ proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PS
           ambiguous = true
 
 proc searchInScopes*(c: PContext, s: PIdent; ambiguous: var bool): PSym =
+  #[
+  This is the key algorithm to solve the generic sandwich problem:
+  Inside generics, we stop the search at c.genericInstStack[^1] unless we found
+  a mixin; this means:
+
+  # module m1:
+  proc bar1()=discard
+  proc fn*[T] =
+    mixin bar3
+    proc bar2() = discard
+    bar1() # ok, this is resolved as a symbol during generic prepass
+    bar2() # ok, ditto
+    bar3() # ok, this is resolved as a skMixin symbol during generic prepass
+    bar4() # error, this is not visible
+    bar5() # error, ditto, even if bar5 is at module scope in m2
+
+  # module m2:
+  import m1
+  proc bar5() = discard
+  proc main =
+    proc bar3() = discard
+    proc bar4() = discard
+    fn[int]()
+  ]#
+  var foundMixin = false
   for scope in allScopes(c.currentScope):
     result = strTableGet(scope.symbols, s)
-    if result != nil: return result
+    if result != nil:
+      if result.kind == skMixin:
+        foundMixin = true
+        continue
+      if c.inGenericInst > 0 and not foundMixin:
+        var parent = result.owner
+        while true:
+          if parent == c.genericInstStack[^1]: break
+          if parent != nil: parent = parent.owner
+          else: return nil
+      return result
+  if c.inGenericInst > 0 and not foundMixin: return nil
   result = someSymFromImportTable(c, s, ambiguous)
 
 proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
@@ -200,7 +236,8 @@ proc debugScopes*(c: PContext; limit=0, max = int.high) {.deprecated.} =
     for h in 0..high(scope.symbols.data):
       if scope.symbols.data[h] != nil:
         if count >= max: return
-        echo count, ": ", scope.symbols.data[h].name.s
+        let sym = scope.symbols.data[h]
+        echo count, ": ", sym, " owner:", sym.owner
         count.inc
     if i == limit: return
     inc i
@@ -213,6 +250,8 @@ proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSy
     while candidate != nil:
       if candidate.kind in filter:
         if result.len == 0:
+          # xxx check whether we need similar logic as in `searchInScopes`
+          # (`inGenericInst` etc)
           result.add candidate
       candidate = nextIdentIter(ti, scope.symbols)
 

compiler/sem.nim

@@ -19,6 +19,9 @@ import
   lowerings, plugins/active, lineinfos, strtabs, int128,
   isolation_check, typeallowed, modulegraphs, enumtostr, concepts, astmsgs
 
+when defined(nimCompilerStackraceHints):
+  import std/stackframes
+
 when defined(nimfix):
   import nimfix/prettybase
 
@@ -496,6 +499,7 @@ proc semConstBoolExpr(c: PContext, n: PNode): PNode =
 
 proc semGenericStmt(c: PContext, n: PNode): PNode
 proc semConceptBody(c: PContext, n: PNode): PNode
+proc semGenericStmtInTypeSection(c: PContext, n: PNode): PNode
 
 include semtypes
 
@@ -637,11 +641,13 @@ proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
   else:
     let oldContextLen = msgs.getInfoContextLen(c.config)
     let oldInGenericInst = c.inGenericInst
+    let oldgenericInstStackLen = c.genericInstStack.len
     try:
       result = semStmtAndGenerateGenerics(c, n)
     except ERecoverableError, ESuggestDone:
       recoverContext(c)
       c.inGenericInst = oldInGenericInst
+      c.genericInstStack.setLen oldgenericInstStackLen
       msgs.setInfoContextLen(c.config, oldContextLen)
       if getCurrentException() of ESuggestDone:
         c.suggestionsMade = true

compiler/semdata.nim

@@ -112,6 +112,7 @@ type
     compilesContextId*: int    # > 0 if we are in a ``compiles`` magic
     compilesContextIdGenerator*: int
     inGenericInst*: int        # > 0 if we are instantiating a generic
+    genericInstStack*: seq[PSym] # 
     converters*: seq[PSym]
     patterns*: seq[PSym]       # sequence of pattern matchers
     optionStack*: seq[POptionEntry]

compiler/semexprs.nim

@@ -2149,6 +2149,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   let oldInGenericContext = c.inGenericContext
   let oldInUnrolledContext = c.inUnrolledContext
   let oldInGenericInst = c.inGenericInst
+  let oldgenericInstStackLen = c.genericInstStack.len
   let oldInStaticContext = c.inStaticContext
   let oldProcCon = c.p
   c.generics = @[]
@@ -2167,6 +2168,10 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   c.inGenericContext = oldInGenericContext
   c.inUnrolledContext = oldInUnrolledContext
   c.inGenericInst = oldInGenericInst
+
+  assert c.genericInstStack.len >= oldgenericInstStackLen
+  c.genericInstStack.setLen oldgenericInstStackLen
+
   c.inStaticContext = oldInStaticContext
   c.p = oldProcCon
   msgs.setInfoContextLen(c.config, oldContextLen)
@@ -2723,7 +2728,6 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     defer:
       if isCompilerDebug():
         echo ("<", c.config$n.info, n, ?.result.typ)
-
   result = n
   if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
   if nfSem in n.flags: return
@@ -3005,7 +3009,10 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     for i in 0..<n.len:
       n[i] = semExpr(c, n[i])
   of nkComesFrom: discard "ignore the comes from information for now"
-  of nkMixinStmt: discard
+  of nkMixinStmt:
+    for ni in n:
+      if ni.kind == nkOpenSymChoice and ni.len == 1 and ni[0].sym.kind == skMixin:
+        addDecl(c, ni[0].sym)
   of nkBindStmt:
     if c.p != nil:
       if n.len > 0 and n[0].kind == nkSym:

compiler/semgnrc.nim

@@ -178,14 +178,16 @@ proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
 
 proc semGenericStmt(c: PContext, n: PNode,
                     flags: TSemGenericFlags, ctx: var GenericCtx): PNode =
+  when defined(nimCompilerStackraceHints):
+    setFrameMsg c.config$n.info & " " & $n.kind
+
   result = n
 
   when defined(nimsuggest):
     if withinTypeDesc in flags: inc c.inTypeContext
 
   #if conf.cmd == cmdIdeTools: suggestStmt(c, n)
   semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
-
   case n.kind
   of nkIdent, nkAccQuoted:
     result = lookup(c, n, flags, ctx)
@@ -488,7 +490,15 @@ proc semGenericStmt(c: PContext, n: PNode,
     else: body = n[bodyPos]
     n[bodyPos] = semGenericStmtScope(c, body, flags, ctx)
     closeScope(c)
-  of nkPragma, nkPragmaExpr: discard
+  of nkPragma, nkPragmaExpr:
+    # similar to treatment in `semTemplBody`
+    for i in 0..<n.len:
+      if n[i].kind == nkExprColonExpr:
+        if n[i][0].kind == nkIdent and getIdent(c.cache, $wPragma) == n[i][0].ident:
+          # handles: `{.pragma: myprag, inline.}`, where `myprag` shouldn't be passed through `semGenericStmt`
+          discard
+        else:
+          result[i][1] = semGenericStmt(c, n[i][1], flags, ctx)
   of nkExprColonExpr, nkExprEqExpr:
     checkMinSonsLen(n, 2, c.config)
     result[1] = semGenericStmt(c, n[1], flags, ctx)
@@ -499,17 +509,17 @@ proc semGenericStmt(c: PContext, n: PNode,
   when defined(nimsuggest):
     if withinTypeDesc in flags: dec c.inTypeContext
 
-proc semGenericStmt(c: PContext, n: PNode): PNode =
+proc semGenericStmtImpl(c: PContext, n: PNode, flags: TSemGenericFlags): PNode {.inline.} =
   var ctx: GenericCtx
   ctx.toMixin = initIntSet()
   ctx.toBind = initIntSet()
-  result = semGenericStmt(c, n, {}, ctx)
+  result = semGenericStmt(c, n, flags, ctx)
   semIdeForTemplateOrGeneric(c, result, ctx.cursorInBody)
 
-proc semConceptBody(c: PContext, n: PNode): PNode =
-  var ctx: GenericCtx
-  ctx.toMixin = initIntSet()
-  ctx.toBind = initIntSet()
-  result = semGenericStmt(c, n, {withinConcept}, ctx)
-  semIdeForTemplateOrGeneric(c, result, ctx.cursorInBody)
+proc semGenericStmt(c: PContext, n: PNode): PNode = semGenericStmtImpl(c, n, {})
+
+proc semGenericStmtInTypeSection(c: PContext, n: PNode): PNode =
+  # xxx consider merging with `semGenericStmt`
+  result = semGenericStmt(c, n)
 
+proc semConceptBody(c: PContext, n: PNode): PNode = semGenericStmtImpl(c, n, {withinConcept})