@@ -63,6 +83,10 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos */ def breakAsLineFeed(): Boolean = lineBreak(LineFeed) + private def errorUntil(n:Int, untilFN: () => Boolean): Boolean = { + !emptyScalar(n) && oneOrMore( !untilFN() && indentedErrorLine(n)) && emitError() + } + /** * Outside scalar content, YAML allows any line break to be used to terminate lines.
* [30] b-non-content ::= b-break @@ -81,7 +105,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos consume() true } - else if (currentChar == '%' && isNsHexDigit(lookAhead(1)) && isNsHexDigit(lookAhead(2))) { + else if ( isDirectiveChar && isNsHexDigit(lookAhead(1)) && isNsHexDigit(lookAhead(2))) { consume(3) true } @@ -278,7 +302,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * s-l-comments */ private def directive() = - currentChar == '%' && emit(BeginDirective) && emitIndicator() && ( + isDirectiveChar && emit(BeginDirective) && emitIndicator() && ( matches(yamlDirective()) || matches(tagDirective()) || matches(reservedDirective()) ) && emit(EndDirective) && multilineComment() @@ -436,7 +460,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos emit(MetaText) } } || - emit(Error) + emitError() ) } } && @@ -541,7 +565,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos } else { inText = false - emit(Error) + emitError() done = true } case '\\' if quoteChar == '"' => @@ -575,7 +599,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos } case EofChar => - emit(Error) + emitError() done = true case _ => @@ -792,10 +816,10 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos @tailrec private def flowMapEntries(n: Int, ctx: YamlContext): Boolean = currentChar != '}' && { flowMapEntry(n, ctx) && { separateFlow(n, ctx) - indicator(',') && { + if(indicator(',')) { separateFlow(n, ctx) flowMapEntries(n, ctx) - } + }else true } } @@ -1101,7 +1125,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos def chompedEmpty(n: Int, t: Char): Boolean = { if (t != '+') { if (t != '-') emit(EndScalar) - zeroOrMore(indentLessOrEqual(n) && breakNonContent()) //strip empty + zeroOrMore(indentLessOrEqual(n) && breakComment()) //strip empty } else { zeroOrMore(emptyLine(n, BlockIn)) // keep empty @@ -1272,12 +1296,29 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos m > 0 && emit(BeginSequence) && oneOrMore { - val b1 = indent(n + m) - b1 && blockSeqEntry(n + m) + matches(seqMember(n + m)) || matches(errorUntilSeq(n,m)) || matches(errorsUntilBack(n,m)) } && emit(EndSequence) } + // advance with error until sequence char: errorsUntilSeq + // if not - is detected, advance until the indentation is the same that the father (n+1?) and go out. Father map recovery + private def errorUntilSeq(n:Int, m:Int) = oneOrMoreErrorSeqEntries(n,m,() => !looksLikeSeq(n,m)) && looksLikeSeq(n,m) && emitError() + + private def errorsUntilBack(n:Int, m:Int) = oneOrMoreErrorSeqEntries(n,m, () => seqIndentedFromFather(n)) && emitError() || !nonEof + + private def seqIndentedFromFather(n:Int): Boolean = input.countSpaces() > n + 1 + + private def oneOrMoreErrorSeqEntries(n:Int,m:Int, whileFN:() => Boolean): Boolean = oneOrMore(whileFN() && checkSeqIndent(n,m) && seqEntryError(n)) + + private def seqEntryError(n:Int) = nonEof && consumeErrorLine() + + private def looksLikeSeq(n:Int, m:Int) = detectSequenceStart(n) == m + + private def checkSeqIndent(n:Int, m:Int) = !(n == -1 && input.countSpaces()==0) && indent(n+m) + + private def seqMember(idn:Int): Boolean = indent(idn) && blockSeqEntry(idn) + private def detectSequenceStart(n: Int) = { val spaces = input.countSpaces(0, MAX_VALUE) if (lookAhead(spaces) == '-' && !isNsChar(lookAhead(spaces + 1))) spaces - n else 0 @@ -1342,26 +1383,52 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * )+ *
For some fixed auto-detected m greater than 0 */ - def blockMapping(n: Int): Boolean = beginOfLine && { + def blockMapping(n: Int, ctx:YamlContext): Boolean = beginOfLine && { val m = detectMapStart(n) - (m > 0) && emit(BeginMapping) && oneOrMore { - indent(n + m) && blockMapEntry(n + m) - } && emit(EndMapping) + (m > 0) && emit(BeginMapping) && + entryList(n+m) && + emit(EndMapping) } + private def isFlowOrPlainIndicator(chr:Int) = isFlowIndicator(chr) || chr == '\'' || chr == '"' + + private def entryList(n:Int): Boolean = oneOrMore { + entry(n) || + (entryErrors(n) && emitError()) + } + + private def entryErrors(n:Int) = oneOrMore( !validEntryContent(n) && entryError(n)) + + private def validEntryContent(n:Int) = looksLikeEntry(n) || singleDocRootException(n) + + private def singleDocRootException(n:Int):Boolean = !isSingleDocument && isRootException(n) + + private def looksLikeEntry(n:Int) = matches(exactEntryIndent(n) && someEntryIndicator()) + + private def exactEntryIndent(n:Int):Boolean = (n<=0 || input.countSpaces() == n) && indent(n) + + private def isRootException(n:Int) = n <= 0 && (isDirectiveChar || isDirectivesEnd || isDocumentEnd ) + + private def isDirectiveChar: Boolean = currentChar == '%' + + private def entryError(n:Int) = nonEof && indent(n) && consumeErrorLine() + + private def entry(ind:Int): Boolean = matches(exactEntryIndent(ind) && blockMapEntry(ind)) + + private def someEntryIndicator(): Boolean = explicitEntry || lineContainsMapIndicator() || isFlowOrPlainIndicator(currentChar) + + private def explicitEntry:Boolean = currentChar == '?' /** * [188] ns-l-block-map-entry(n) ::= * [[mapExplicitEntry c-l-block-map-explicit-entry(n)]] * | [[mapImplicitEntry ns-l-block-map-implicit-entry(n)]] */ @failfast def blockMapEntry(n: Int): Boolean = { - val explicit = currentChar == '?' - val entry = explicit || lineContainsMapIndicator() - if (!entry) false + if (!someEntryIndicator()) false else matches { emit(BeginPair) - (if (explicit) mapExplicitEntry(n) else mapImplicitEntry(n)) && + (if (explicitEntry) mapExplicitEntry(n) else mapImplicitEntry(n)) && emit(EndPair) } } @@ -1418,8 +1485,39 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * ) */ def mapImplicitValue(n: Int): Boolean = indicator(':') && { - blockNode(n, BlockOut) || - emptyNode() && (matches(multilineComment()) || matches(error() && multilineComment())) + blockNode(n, BlockOut) || blockNodeRecovery(n) || commentOrErrorLine(n) + } + + private def commentOrErrorLine(n:Int): Boolean = matches(emptyNode() && (multilineComment() || sameErrorLine(n))) + + private def blockNodeRecovery(n:Int): Boolean = matches { + multilineComment() && + errorUntil(n, () => blockNode(n,BlockOut)) && + blockNode(n,BlockOut) + } + + private def emptyScalar(n:Int) = input.countSpaces() == 0 && lineBreakSequence() != 0 && input.column > n + + private def sameErrorLine(n:Int) = { + n < input.column && { + consumeWhile(!isBreakComment(_)) + emitError() + breakComment() + } + } + + private def indentedErrorLine(n:Int): Boolean = nonEof && input.countSpaces() > n && consumeErrorLine() + + private def consumeErrorLine(): Boolean = { + consumeWhile(c => !isBreakComment(c)) + consumeLineBreak() || !nonEof + } + + private def consumeLineBreak(): Boolean = { + val n = lineBreakSequence() + n != 0 && { + consume(n); true + } } /** @@ -1434,11 +1532,14 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * [197] s-l+flow-in-block(n) ::= s-separate(n+1,flow-out) ns-flow-node(n+1,flow-out) s-l-comments */ def blockNode(n: Int, ctx: YamlContext): Boolean = - matches(blockInBlock(n, ctx)) || matches { - val b1 = separate(n + 1, FlowOut) && emit(BeginNode) - b1 && flowNode(n + 1, FlowOut) && emit(EndNode) && multilineComment() - } + blockIndentedNode(n,ctx) || flowNode(n) + + private def blockIndentedNode(n:Int, ctx:YamlContext) = matches(blockInBlock(n, ctx)) + private def flowNode(n:Int): Boolean = matches { + val b1 = separate(n + 1, FlowOut) && emit(BeginNode) + b1 && flowNode(n + 1, FlowOut) && emit(EndNode) && multilineComment() + } /* * [198] s-l+block-in-block(n,c) ::= s-l+block-scalar(n,c) | s-l+block-collection(n,c) */ @@ -1475,7 +1576,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * */ def blockCollection(n: Int, ctx: YamlContext): Boolean = { - def bc() = multilineComment() && (blockSequence(if (ctx == BlockOut) n - 1 else n) || blockMapping(n)) + def bc() = multilineComment() && (blockSequence(if (ctx == BlockOut) n - 1 else n) || blockMapping(n, ctx)) matches(separate(n + 1, ctx) && nodeProperties(n + 1, ctx) && bc()) || matches(bc()) } @@ -1534,7 +1635,7 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * * [209] l-directive-document ::= l-directive+ l-explicit-document */ - private def directiveDocument() = currentChar == '%' && oneOrMore(directive()) && explicitDocument() + private def directiveDocument() = isDirectiveChar && oneOrMore(directive()) && explicitDocument() /** [210] l-any-document ::= l-directive-document | l-explicit-document | l-bare-document */ private def anyDocument() = nonEof && matches { @@ -1543,6 +1644,8 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos emit(EndDocument) } + private def errorLine():Boolean = error() && breakNonContent() + /** * A YAML stream consists of zero or more documents. * Subsequent documents require some sort of separation marker line. @@ -1561,22 +1664,14 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos * )* * */ - def yamlStream(): Unit = { + def yamlStream(): Unit = advanceUntilEnd(anyDocument, Some(() => rootContent())) + + private def advanceUntilEnd(docFunction:() => Boolean, otherContentFN: Option[() => Boolean]): Boolean = { while (documentPrefix()) {} - anyDocument() + docFunction() var currentOffset = input.offset while (nonEof) { - matches { - oneOrMore(documentSuffix()) && - zeroOrMore(documentPrefix()) && - optional(anyDocument()) - } || - matches { - zeroOrMore(documentPrefix()) - emit(BeginDocument) && - explicitDocument() && - emit(EndDocument) - } + otherContentFN.foreach(f => f()) if (input.offset == currentOffset) { consumeWhile(_ != EofChar) emit(Error) @@ -1586,6 +1681,20 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos emit(EndStream) } + private def rootContent(): Boolean = { + matches { + oneOrMore(documentSuffix()) && + zeroOrMore(documentPrefix()) && + optional(anyDocument()) + } || + matches { + zeroOrMore(documentPrefix()) + emit(BeginDocument) && + explicitDocument() && + emit(EndDocument) + } + } + /** Check if the line contains a map Indicator (":" plus space or end of text) */ def lineContainsMapIndicator(): Boolean = { var i = 0 @@ -1615,12 +1724,14 @@ final class YamlLexer private (input: LexerInput, positionOffset: Position = Pos case _ => None } + def emitError(): Boolean = emit(Error) + /** Emit an error and Consume until end of line or file */ def error(): Boolean = if (isBreakComment(currentChar)) true else { do consume() while (!isBreakComment(currentChar)) - emit(Error) + emitError() } private def nonEndOfDocument = { // Todo optimize diff --git a/shared/src/main/scala/org/yaml/parser/JsonParser.scala b/shared/src/main/scala/org/yaml/parser/JsonParser.scala index 91a8dd0c..8639f38a 100644 --- a/shared/src/main/scala/org/yaml/parser/JsonParser.scala +++ b/shared/src/main/scala/org/yaml/parser/JsonParser.scala @@ -14,6 +14,9 @@ import scala.collection.mutable.ArrayBuffer class JsonParser private[parser] (val lexer: JsonLexer)(implicit val eh: ParseErrorHandler) extends YParser { type TD = TokenData[YamlToken] + + lexer.initialize() + private var keepTokens = false private var current = new JsonBuilder private var stack = List(current) diff --git a/shared/src/main/scala/org/yaml/parser/YamlParser.scala b/shared/src/main/scala/org/yaml/parser/YamlParser.scala index d6771778..8f586375 100644 --- a/shared/src/main/scala/org/yaml/parser/YamlParser.scala +++ b/shared/src/main/scala/org/yaml/parser/YamlParser.scala @@ -1,6 +1,6 @@ package org.yaml.parser -import org.mulesoft.lexer.{Position, SourceLocation} +import org.mulesoft.lexer._ import org.yaml.lexer.YamlLexer import org.yaml.model._ @@ -23,7 +23,14 @@ class YamlParser private[parser] (val lexer: YamlLexer)(implicit val eh: ParseEr } /** Parse the Yaml and return an Indexed Seq of the Parts */ - def parse(keepTokens: Boolean = true): IndexedSeq[YPart] = new YamlLoader(lexer, keepTokens, includeTag, eh).parse() + def parse(keepTokens: Boolean = true): IndexedSeq[YPart] = parse(keepTokens, StreamLexerContext) + + private def parse(keepTokens: Boolean, ctx:LexerContext):IndexedSeq[YPart] = + new YamlLoader(lexer.initialize(ctx), keepTokens, includeTag, eh).parse() + + override def document(): YDocument = { + new YDocument(SourceLocation(lexer.sourceName), parse(keepTokens = false, SingleDocumentLexerContext)) + } /** Define an Include Tag if not empty it will generate Mutable Node References for tagged nodes */ def withIncludeTag(s: String): this.type = { diff --git a/shared/src/test/data/location/example-8.3.yloc b/shared/src/test/data/location/example-8.3.yloc index 32a7aeb0..1d384e8f 100644 --- a/shared/src/test/data/location/example-8.3.yloc +++ b/shared/src/test/data/location/example-8.3.yloc @@ -1,5 +1,9 @@ -YDocument [1,0..3,0] example-8.3.yaml -YNodePlain [1,0..3,0] example-8.3.yaml -YSequence [1,0..3,0] example-8.3.yaml +YDocument [1,0..9,0] example-8.3.yaml +YNodePlain [1,0..9,0] example-8.3.yaml +YSequence [1,0..9,0] example-8.3.yaml YNodePlain [1,1..3,0] example-8.3.yaml YScalar [1,2..2,0] example-8.3.yaml +YNodePlain [4,1..6,0] example-8.3.yaml +YScalar [4,2..6,0] example-8.3.yaml +YNodePlain [7,1..8,0] example-8.3.yaml +YScalar [7,2..8,0] example-8.3.yaml diff --git a/shared/src/test/data/location/new-line-flow-map.yloc b/shared/src/test/data/location/new-line-flow-map.yloc new file mode 100644 index 00000000..772ddfdf --- /dev/null +++ b/shared/src/test/data/location/new-line-flow-map.yloc @@ -0,0 +1,13 @@ +YDocument [1,0..3,0] new-line-flow-map.yaml +YNodePlain [1,0..3,0] new-line-flow-map.yaml +YMap [1,0..3,0] new-line-flow-map.yaml +YMapEntry [1,0..3,0] new-line-flow-map.yaml +YNodePlain [1,0..1,4] new-line-flow-map.yaml +YScalar [1,0..1,4] new-line-flow-map.yaml +YNodePlain [2,2..2,12] new-line-flow-map.yaml +YMap [2,2..2,12] new-line-flow-map.yaml +YMapEntry [2,3..2,11] new-line-flow-map.yaml +YNodePlain [2,3..2,6] new-line-flow-map.yaml +YScalar [2,3..2,6] new-line-flow-map.yaml +YNodePlain [2,8..2,11] new-line-flow-map.yaml +YScalar [2,8..2,11] new-line-flow-map.yaml diff --git a/shared/src/test/data/location/same-level-array.yloc b/shared/src/test/data/location/same-level-array.yloc new file mode 100644 index 00000000..ad4096ee --- /dev/null +++ b/shared/src/test/data/location/same-level-array.yloc @@ -0,0 +1,30 @@ +YDocument [1,0..6,11] same-level-array.yaml +YNodePlain [1,0..6,11] same-level-array.yaml +YMap [1,0..6,11] same-level-array.yaml +YMapEntry [1,0..6,11] same-level-array.yaml +YNodePlain [1,0..1,3] same-level-array.yaml +YScalar [1,0..1,3] same-level-array.yaml +YNodePlain [1,4..6,11] same-level-array.yaml +YMap [2,0..6,11] same-level-array.yaml +YMapEntry [2,2..3,0] same-level-array.yaml +YNodePlain [2,2..2,13] same-level-array.yaml +YScalar [2,2..2,13] same-level-array.yaml +YNodePlain [2,15..2,16] same-level-array.yaml +YScalar [2,15..2,16] same-level-array.yaml +YMapEntry [3,2..5,0] same-level-array.yaml +YNodePlain [3,2..3,6] same-level-array.yaml +YScalar [3,2..3,6] same-level-array.yaml +YNodePlain [3,7..5,0] same-level-array.yaml +YSequence [4,0..5,0] same-level-array.yaml +YNodePlain [4,4..4,9] same-level-array.yaml +YScalar [4,4..4,9] same-level-array.yaml +YMapEntry [5,2..6,11] same-level-array.yaml +YNodePlain [5,2..5,7] same-level-array.yaml +YScalar [5,2..5,7] same-level-array.yaml +YNodePlain [5,8..6,11] same-level-array.yaml +YMap [6,0..6,11] same-level-array.yaml +YMapEntry [6,4..6,11] same-level-array.yaml +YNodePlain [6,4..6,8] same-level-array.yaml +YScalar [6,4..6,8] same-level-array.yaml +YNodePlain [6,10..6,11] same-level-array.yaml +YScalar [6,10..6,11] same-level-array.yaml diff --git a/shared/src/test/data/render/example-8.3.json b/shared/src/test/data/render/example-8.3.json index b344d1e2..6e434811 100644 --- a/shared/src/test/data/render/example-8.3.json +++ b/shared/src/test/data/render/example-8.3.json @@ -1,3 +1,5 @@ [ + "", + "text\n", "" ] diff --git a/shared/src/test/data/render/example-8.3.yaml b/shared/src/test/data/render/example-8.3.yaml index 8087903c..310d37bb 100644 --- a/shared/src/test/data/render/example-8.3.yaml +++ b/shared/src/test/data/render/example-8.3.yaml @@ -1 +1,4 @@ - "" +- | + text +- "" diff --git a/shared/src/test/data/render/new-line-flow-map.yaml b/shared/src/test/data/render/new-line-flow-map.yaml new file mode 100644 index 00000000..8b8edfba --- /dev/null +++ b/shared/src/test/data/render/new-line-flow-map.yaml @@ -0,0 +1,2 @@ +zulu: + "a": "b" diff --git a/shared/src/test/data/render/same-level-array.yaml b/shared/src/test/data/render/same-level-array.yaml new file mode 100644 index 00000000..4715aec0 --- /dev/null +++ b/shared/src/test/data/render/same-level-array.yaml @@ -0,0 +1,6 @@ +key: + displayName: b + type: + - array + items: + type: A diff --git a/shared/src/test/data/yaml/new-line-flow-map.yaml b/shared/src/test/data/yaml/new-line-flow-map.yaml new file mode 100644 index 00000000..96167c06 --- /dev/null +++ b/shared/src/test/data/yaml/new-line-flow-map.yaml @@ -0,0 +1,2 @@ +zulu: + {"a": "b"} diff --git a/shared/src/test/data/yaml/same-level-array.yaml b/shared/src/test/data/yaml/same-level-array.yaml new file mode 100644 index 00000000..8359e34f --- /dev/null +++ b/shared/src/test/data/yaml/same-level-array.yaml @@ -0,0 +1,6 @@ +key: + displayName: b + type: + - array + items: + type: A \ No newline at end of file diff --git a/shared/src/test/data/yaml12/example-8.3.jyaml b/shared/src/test/data/yaml12/example-8.3.jyaml index ad220b17..fb5f6cca 100644 --- a/shared/src/test/data/yaml12/example-8.3.jyaml +++ b/shared/src/test/data/yaml12/example-8.3.jyaml @@ -1,5 +1,7 @@ %YAML 1.2 --- !!seq [ + !!str "", + !!str "text\n", !!str "" ] diff --git a/shared/src/test/data/yaml12/example-8.3.yaml b/shared/src/test/data/yaml12/example-8.3.yaml index ad220b17..fb5f6cca 100644 --- a/shared/src/test/data/yaml12/example-8.3.yaml +++ b/shared/src/test/data/yaml12/example-8.3.yaml @@ -1,5 +1,7 @@ %YAML 1.2 --- !!seq [ + !!str "", + !!str "text\n", !!str "" ] diff --git a/shared/src/test/data/yaml12/new-line-flow-map.yaml b/shared/src/test/data/yaml12/new-line-flow-map.yaml new file mode 100644 index 00000000..39821b1a --- /dev/null +++ b/shared/src/test/data/yaml12/new-line-flow-map.yaml @@ -0,0 +1,9 @@ +%YAML 1.2 +--- +!!map { + ? !!str "zulu" + : !!map { + ? !!str "a" + : !!str "b" + } +} diff --git a/shared/src/test/data/yaml12/same-level-array.yaml b/shared/src/test/data/yaml12/same-level-array.yaml new file mode 100644 index 00000000..4f2267a5 --- /dev/null +++ b/shared/src/test/data/yaml12/same-level-array.yaml @@ -0,0 +1,18 @@ +%YAML 1.2 +--- +!!map { + ? !!str "key" + : !!map { + ? !!str "displayName" + : !!str "b", + ? !!str "type" + : !!seq [ + !!str "array" + ], + ? !!str "items" + : !!map { + ? !!str "type" + : !!str "A" + } + } +} diff --git a/shared/src/test/data/yeast/example-5.10.yt b/shared/src/test/data/yeast/example-5.10.yt index e8b730cf..f57a367f 100644 --- a/shared/src/test/data/yeast/example-5.10.yt +++ b/shared/src/test/data/yeast/example-5.10.yt @@ -12,7 +12,8 @@ BeginNode 14, 1, 14 '' BeginScalar 14, 1, 14 '' EndScalar 14, 1, 14 '' EndNode 14, 1, 14 '' -Error 14, 1, 14 ' @text' +WhiteSpace 14, 1, 14 ' ' +Error 15, 1, 15 '@text' LineBreak 20, 1, 20 '\n' EndPair 21, 2, 0 '' BeginPair 21, 2, 0 '' @@ -26,7 +27,8 @@ BeginNode 34, 2, 13 '' BeginScalar 34, 2, 13 '' EndScalar 34, 2, 13 '' EndNode 34, 2, 13 '' -Error 34, 2, 13 ' `text' +WhiteSpace 34, 2, 13 ' ' +Error 35, 2, 14 '`text' EndPair 40, 2, 19 '' EndMapping 40, 2, 19 '' EndNode 40, 2, 19 '' diff --git a/shared/src/test/data/yeast/example-8.3.yt b/shared/src/test/data/yeast/example-8.3.yt index da033bda..28e58b37 100644 --- a/shared/src/test/data/yeast/example-8.3.yt +++ b/shared/src/test/data/yeast/example-8.3.yt @@ -11,7 +11,29 @@ EndScalar 4, 2, 0 '' Indent 4, 2, 0 ' ' LineBreak 6, 2, 2 '\n' EndNode 7, 3, 0 '' -EndSequence 7, 3, 0 '' -EndNode 7, 3, 0 '' -EndDocument 7, 3, 0 '' -Error 7, 3, 0 ' text\n- >\n text\n text\n- |2\n text\n' +Error 7, 3, 0 ' text\n' +Indicator 13, 4, 0 '-' +BeginNode 14, 4, 1 '' +WhiteSpace 14, 4, 1 ' ' +BeginScalar 15, 4, 2 '' +Indicator 15, 4, 2 '>' +LineBreak 16, 4, 3 '\n' +Indent 17, 5, 0 ' ' +Text 19, 5, 2 'text' +LineFeed 23, 5, 6 '\n' +EndScalar 24, 6, 0 '' +EndNode 24, 6, 0 '' +Error 24, 6, 0 ' text\n' +Indicator 30, 7, 0 '-' +BeginNode 31, 7, 1 '' +WhiteSpace 31, 7, 1 ' ' +BeginScalar 32, 7, 2 '' +Indicator 32, 7, 2 '|' +Indicator 33, 7, 3 '2' +LineBreak 34, 7, 4 '\n' +EndScalar 35, 8, 0 '' +EndNode 35, 8, 0 '' +Error 35, 8, 0 ' text\n' +EndSequence 41, 9, 0 '' +EndNode 41, 9, 0 '' +EndDocument 41, 9, 0 '' diff --git a/shared/src/test/data/yeast/new-line-flow-map.yt b/shared/src/test/data/yeast/new-line-flow-map.yt new file mode 100644 index 00000000..ac710a84 --- /dev/null +++ b/shared/src/test/data/yeast/new-line-flow-map.yt @@ -0,0 +1,42 @@ +BeginDocument 0, 1, 0 '' +BeginNode 0, 1, 0 '' +BeginMapping 0, 1, 0 '' +BeginPair 0, 1, 0 '' +BeginNode 0, 1, 0 '' +BeginScalar 0, 1, 0 '' +Text 0, 1, 0 'zulu' +EndScalar 4, 1, 4 '' +EndNode 4, 1, 4 '' +Indicator 4, 1, 4 ':' +LineBreak 5, 1, 5 '\n' +Indent 6, 2, 0 ' ' +WhiteSpace 7, 2, 1 ' ' +BeginNode 8, 2, 2 '' +BeginMapping 8, 2, 2 '' +Indicator 8, 2, 2 '{' +BeginPair 9, 2, 3 '' +BeginNode 9, 2, 3 '' +BeginScalar 9, 2, 3 '' +Indicator 9, 2, 3 '\"' +Text 10, 2, 4 'a' +Indicator 11, 2, 5 '\"' +EndScalar 12, 2, 6 '' +EndNode 12, 2, 6 '' +Indicator 12, 2, 6 ':' +WhiteSpace 13, 2, 7 ' ' +BeginNode 14, 2, 8 '' +BeginScalar 14, 2, 8 '' +Indicator 14, 2, 8 '\"' +Text 15, 2, 9 'b' +Indicator 16, 2, 10 '\"' +EndScalar 17, 2, 11 '' +EndNode 17, 2, 11 '' +EndPair 17, 2, 11 '' +Indicator 17, 2, 11 '}' +EndMapping 18, 2, 12 '' +EndNode 18, 2, 12 '' +LineBreak 18, 2, 12 '\n' +EndPair 19, 3, 0 '' +EndMapping 19, 3, 0 '' +EndNode 19, 3, 0 '' +EndDocument 19, 3, 0 '' diff --git a/shared/src/test/data/yeast/same-level-array.yt b/shared/src/test/data/yeast/same-level-array.yt new file mode 100644 index 00000000..74575b17 --- /dev/null +++ b/shared/src/test/data/yeast/same-level-array.yt @@ -0,0 +1,87 @@ +BeginDocument 0, 1, 0 '' +BeginNode 0, 1, 0 '' +BeginMapping 0, 1, 0 '' +BeginPair 0, 1, 0 '' +BeginNode 0, 1, 0 '' +BeginScalar 0, 1, 0 '' +Text 0, 1, 0 'key' +EndScalar 3, 1, 3 '' +EndNode 3, 1, 3 '' +Indicator 3, 1, 3 ':' +BeginNode 4, 1, 4 '' +LineBreak 4, 1, 4 '\n' +BeginMapping 5, 2, 0 '' +Indent 5, 2, 0 ' ' +BeginPair 7, 2, 2 '' +BeginNode 7, 2, 2 '' +BeginScalar 7, 2, 2 '' +Text 7, 2, 2 'displayName' +EndScalar 18, 2, 13 '' +EndNode 18, 2, 13 '' +Indicator 18, 2, 13 ':' +WhiteSpace 19, 2, 14 ' ' +BeginNode 20, 2, 15 '' +BeginScalar 20, 2, 15 '' +Text 20, 2, 15 'b' +EndScalar 21, 2, 16 '' +EndNode 21, 2, 16 '' +LineBreak 21, 2, 16 '\n' +EndPair 22, 3, 0 '' +Indent 22, 3, 0 ' ' +BeginPair 24, 3, 2 '' +BeginNode 24, 3, 2 '' +BeginScalar 24, 3, 2 '' +Text 24, 3, 2 'type' +EndScalar 28, 3, 6 '' +EndNode 28, 3, 6 '' +Indicator 28, 3, 6 ':' +BeginNode 29, 3, 7 '' +LineBreak 29, 3, 7 '\n' +BeginSequence 30, 4, 0 '' +Indent 30, 4, 0 ' ' +Indicator 32, 4, 2 '-' +WhiteSpace 33, 4, 3 ' ' +BeginNode 34, 4, 4 '' +BeginScalar 34, 4, 4 '' +Text 34, 4, 4 'array' +EndScalar 39, 4, 9 '' +EndNode 39, 4, 9 '' +LineBreak 39, 4, 9 '\n' +EndSequence 40, 5, 0 '' +EndNode 40, 5, 0 '' +EndPair 40, 5, 0 '' +Indent 40, 5, 0 ' ' +BeginPair 42, 5, 2 '' +BeginNode 42, 5, 2 '' +BeginScalar 42, 5, 2 '' +Text 42, 5, 2 'items' +EndScalar 47, 5, 7 '' +EndNode 47, 5, 7 '' +Indicator 47, 5, 7 ':' +BeginNode 48, 5, 8 '' +LineBreak 48, 5, 8 '\n' +BeginMapping 49, 6, 0 '' +Indent 49, 6, 0 ' ' +BeginPair 53, 6, 4 '' +BeginNode 53, 6, 4 '' +BeginScalar 53, 6, 4 '' +Text 53, 6, 4 'type' +EndScalar 57, 6, 8 '' +EndNode 57, 6, 8 '' +Indicator 57, 6, 8 ':' +WhiteSpace 58, 6, 9 ' ' +BeginNode 59, 6, 10 '' +BeginScalar 59, 6, 10 '' +Text 59, 6, 10 'A' +EndScalar 60, 6, 11 '' +EndNode 60, 6, 11 '' +EndPair 60, 6, 11 '' +EndMapping 60, 6, 11 '' +EndNode 60, 6, 11 '' +EndPair 60, 6, 11 '' +EndMapping 60, 6, 11 '' +EndNode 60, 6, 11 '' +EndPair 60, 6, 11 '' +EndMapping 60, 6, 11 '' +EndNode 60, 6, 11 '' +EndDocument 60, 6, 11 '' diff --git a/shared/src/test/data/yts/example-5.10.yts b/shared/src/test/data/yts/example-5.10.yts index f87fcaaf..61bba8a9 100644 --- a/shared/src/test/data/yts/example-5.10.yts +++ b/shared/src/test/data/yts/example-5.10.yts @@ -16,7 +16,7 @@ YD [1,0..2,19] YI N [1,14..1,14] YS S s [1,14..1,14] YI n [1,14..1,14] - YI ! b x [1,14..2,0] + YI w ! b x [1,14..2,0] YM [2,0..2,19] YI X [2,0..2,0] YN [2,0..2,12] @@ -28,8 +28,8 @@ YD [1,0..2,19] YI N [2,13..2,13] YS S s [2,13..2,13] YI n [2,13..2,13] - YI ! x [2,13..2,19] + YI w ! x [2,13..2,19] YI m [2,19..2,19] YI n [2,19..2,19] YI o [2,19..2,19] -33 tokens dumped. +35 tokens dumped. diff --git a/shared/src/test/data/yts/example-8.3.yts b/shared/src/test/data/yts/example-8.3.yts index 34ebf520..561fdba3 100644 --- a/shared/src/test/data/yts/example-8.3.yts +++ b/shared/src/test/data/yts/example-8.3.yts @@ -1,16 +1,25 @@ File: example-8.3.yaml -YD [1,0..3,0] +YD [1,0..9,0] YI O [1,0..1,0] - YN [1,0..3,0] + YN [1,0..9,0] YI N [1,0..1,0] - YS [1,0..3,0] + YS [1,0..9,0] YI Q I [1,0..1,1] YN [1,1..3,0] YI N w [1,1..1,2] YS S I b s [1,2..2,0] YI i b n [2,0..3,0] - YI q [3,0..3,0] - YI n [3,0..3,0] - YI o [3,0..3,0] -YI ! [3,0..9,0] -17 tokens dumped. + YI ! I [3,0..4,1] + YN [4,1..6,0] + YI N w [4,1..4,2] + YS S I b i T L s [4,2..6,0] + YI n [6,0..6,0] + YI ! I [6,0..7,1] + YN [7,1..8,0] + YI N w [7,1..7,2] + YS S I I b s [7,2..8,0] + YI n [8,0..8,0] + YI ! q [8,0..9,0] + YI n [9,0..9,0] + YI o [9,0..9,0] +39 tokens dumped. diff --git a/shared/src/test/data/yts/new-line-flow-map.yts b/shared/src/test/data/yts/new-line-flow-map.yts new file mode 100644 index 00000000..6466740c --- /dev/null +++ b/shared/src/test/data/yts/new-line-flow-map.yts @@ -0,0 +1,37 @@ +File: new-line-flow-map.yaml +YD [1,0..3,0] + YI O [1,0..1,0] + YN [1,0..3,0] + YI N [1,0..1,0] + YM [1,0..3,0] + YI M [1,0..1,0] + YM [1,0..3,0] + YI X [1,0..1,0] + YN [1,0..1,4] + YI N [1,0..1,0] + YS S T s [1,0..1,4] + YI n [1,4..1,4] + YI I b i w [1,4..2,2] + YN [2,2..2,12] + YI N [2,2..2,2] + YM [2,2..2,12] + YI M I [2,2..2,3] + YM [2,3..2,11] + YI X [2,3..2,3] + YN [2,3..2,6] + YI N [2,3..2,3] + YS S I T I s [2,3..2,6] + YI n [2,6..2,6] + YI I w [2,6..2,8] + YN [2,8..2,11] + YI N [2,8..2,8] + YS S I T I s [2,8..2,11] + YI n [2,11..2,11] + YI x [2,11..2,11] + YI I m [2,11..2,12] + YI n [2,12..2,12] + YI b x [2,12..3,0] + YI m [3,0..3,0] + YI n [3,0..3,0] + YI o [3,0..3,0] +42 tokens dumped. diff --git a/shared/src/test/data/yts/same-level-array.yts b/shared/src/test/data/yts/same-level-array.yts new file mode 100644 index 00000000..ab0c0658 --- /dev/null +++ b/shared/src/test/data/yts/same-level-array.yts @@ -0,0 +1,83 @@ +File: same-level-array.yaml +YD [1,0..6,11] + YI O [1,0..1,0] + YN [1,0..6,11] + YI N [1,0..1,0] + YM [1,0..6,11] + YI M [1,0..1,0] + YM [1,0..6,11] + YI X [1,0..1,0] + YN [1,0..1,3] + YI N [1,0..1,0] + YS S T s [1,0..1,3] + YI n [1,3..1,3] + YI I [1,3..1,4] + YN [1,4..6,11] + YI N b [1,4..2,0] + YM [2,0..6,11] + YI M i [2,0..2,2] + YM [2,2..3,0] + YI X [2,2..2,2] + YN [2,2..2,13] + YI N [2,2..2,2] + YS S T s [2,2..2,13] + YI n [2,13..2,13] + YI I w [2,13..2,15] + YN [2,15..2,16] + YI N [2,15..2,15] + YS S T s [2,15..2,16] + YI n [2,16..2,16] + YI b x [2,16..3,0] + YI i [3,0..3,2] + YM [3,2..5,0] + YI X [3,2..3,2] + YN [3,2..3,6] + YI N [3,2..3,2] + YS S T s [3,2..3,6] + YI n [3,6..3,6] + YI I [3,6..3,7] + YN [3,7..5,0] + YI N b [3,7..4,0] + YS [4,0..5,0] + YI Q i I w [4,0..4,4] + YN [4,4..4,9] + YI N [4,4..4,4] + YS S T s [4,4..4,9] + YI n [4,9..4,9] + YI b q [4,9..5,0] + YI n [5,0..5,0] + YI x [5,0..5,0] + YI i [5,0..5,2] + YM [5,2..6,11] + YI X [5,2..5,2] + YN [5,2..5,7] + YI N [5,2..5,2] + YS S T s [5,2..5,7] + YI n [5,7..5,7] + YI I [5,7..5,8] + YN [5,8..6,11] + YI N b [5,8..6,0] + YM [6,0..6,11] + YI M i [6,0..6,4] + YM [6,4..6,11] + YI X [6,4..6,4] + YN [6,4..6,8] + YI N [6,4..6,4] + YS S T s [6,4..6,8] + YI n [6,8..6,8] + YI I w [6,8..6,10] + YN [6,10..6,11] + YI N [6,10..6,10] + YS S T s [6,10..6,11] + YI n [6,11..6,11] + YI x [6,11..6,11] + YI m [6,11..6,11] + YI n [6,11..6,11] + YI x [6,11..6,11] + YI m [6,11..6,11] + YI n [6,11..6,11] + YI x [6,11..6,11] + YI m [6,11..6,11] + YI n [6,11..6,11] + YI o [6,11..6,11] +87 tokens dumped. diff --git a/shared/src/test/scala/org/mulesoft/yaml/BasicTest.scala b/shared/src/test/scala/org/mulesoft/yaml/BasicTest.scala index 0f6556f7..c1dd876b 100644 --- a/shared/src/test/scala/org/mulesoft/yaml/BasicTest.scala +++ b/shared/src/test/scala/org/mulesoft/yaml/BasicTest.scala @@ -10,7 +10,7 @@ object BasicTest { val yaml = " #aaaaa\n # hhh\n" def main(args: Array[String]): Unit = { - val lexer = YamlLexer(yaml) + val lexer = YamlLexer(yaml).initialize() while (lexer.token != YamlToken.EndStream) { println(YeastData(lexer.tokenData, lexer.tokenString)) lexer.advance() diff --git a/shared/src/test/scala/org/mulesoft/yaml/InvalidJsonTest.scala b/shared/src/test/scala/org/mulesoft/yaml/InvalidJsonTest.scala index 68e50d93..a74731ed 100644 --- a/shared/src/test/scala/org/mulesoft/yaml/InvalidJsonTest.scala +++ b/shared/src/test/scala/org/mulesoft/yaml/InvalidJsonTest.scala @@ -39,7 +39,7 @@ trait InvalidJsonTest extends GoldenSuite { private def generate(yamlFile: SyncFile, yeastFile: SyncFile) = { val out = new StringBuilder - val lexer = JsonLexer(yamlFile.read()) + val lexer = JsonLexer(yamlFile.read()).initialize() while (lexer.token != YamlToken.EndDocument) { // json files have only 1 document and ends with that token val data = YeastData(lexer.tokenData, lexer.tokenString) out.append(data).append('\n') diff --git a/shared/src/test/scala/org/mulesoft/yaml/InvalidYamlTest.scala b/shared/src/test/scala/org/mulesoft/yaml/InvalidYamlTest.scala index 03f67e3f..b63704b3 100644 --- a/shared/src/test/scala/org/mulesoft/yaml/InvalidYamlTest.scala +++ b/shared/src/test/scala/org/mulesoft/yaml/InvalidYamlTest.scala @@ -32,7 +32,7 @@ trait InvalidYamlTest extends GoldenSuite { private def generate(yamlFile: SyncFile, yeastFile: SyncFile) = { val out = new StringBuilder - val lexer = YamlLexer(yamlFile.read()) + val lexer = YamlLexer(yamlFile.read()).initialize() while (lexer.token != YamlToken.EndStream) { val data = YeastData(lexer.tokenData, lexer.tokenString) out.append(data).append('\n') diff --git a/shared/src/test/scala/org/mulesoft/yaml/JsonToYeastTest.scala b/shared/src/test/scala/org/mulesoft/yaml/JsonToYeastTest.scala index 7172d69e..ae54d97a 100644 --- a/shared/src/test/scala/org/mulesoft/yaml/JsonToYeastTest.scala +++ b/shared/src/test/scala/org/mulesoft/yaml/JsonToYeastTest.scala @@ -33,7 +33,7 @@ trait JsonToYeastTest extends GoldenSuite { private def generate(yamlFile: SyncFile, yeastFile: SyncFile) = { val out = new StringBuilder - val lexer = JsonLexer(yamlFile.read()) + val lexer = JsonLexer(yamlFile.read()).initialize() while (lexer.token != YamlToken.EndDocument) { val data = YeastData(lexer.tokenData, lexer.tokenString) out.append(data).append('\n') diff --git a/shared/src/test/scala/org/mulesoft/yaml/ParseInvalidYamlsTest.scala b/shared/src/test/scala/org/mulesoft/yaml/ParseInvalidYamlsTest.scala index 42f4540e..45dc1e47 100644 --- a/shared/src/test/scala/org/mulesoft/yaml/ParseInvalidYamlsTest.scala +++ b/shared/src/test/scala/org/mulesoft/yaml/ParseInvalidYamlsTest.scala @@ -3,7 +3,7 @@ package org.mulesoft.yaml import org.mulesoft.common.io.Fs import org.mulesoft.lexer.{InputRange, SourceLocation} import org.scalatest.FunSuite -import org.yaml.model.{ParseErrorHandler, SyamlException, YPart} +import org.yaml.model.{ParseErrorHandler, SyamlException, YMap, YPart} import org.yaml.parser.YamlParser import scala.collection.mutable @@ -37,7 +37,7 @@ trait ParseInvalidYamlsTest extends FunSuite { YamlParser(yamlFile.read())(handler).parse() assert(handler.errors.lengthCompare(2) == 0) - assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: ' name'")) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: 'name'")) assert(handler.errors.last.error.getMessage.equals("Syntax error in the following text: ' get:\n post:'")) } @@ -51,6 +51,167 @@ trait ParseInvalidYamlsTest extends FunSuite { assert(handler.errors.head.error.getMessage.equals("Undefined anchor : 'other'")) } + test("Single Unclosed key at root") { + val handler = TestErrorHandler() + + val text = + """key1 + |otherkey: value + |""".stripMargin + val doc = YamlParser(text)(handler).document() + val map = doc.node.as[YMap] + assert(map.entries.length == 1) + assert(handler.errors.lengthCompare(1) == 0) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: 'key1\n'")) + } + + test("Unclosed key at root in middle") { + val handler = TestErrorHandler() + + val text = + """otherkey: value + |invalid + |recover: value + |""".stripMargin + val docs = YamlParser(text)(handler).documents() + assert(docs.head.node.as[YMap].entries.length == 2) + assert(handler.errors.lengthCompare(1) == 0) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: 'invalid\n'")) + } + + test("Two Unclosed key at root") { + val handler = TestErrorHandler() + + val text = + """key1 + |key2 + |otherkey: value + |""".stripMargin + val doc = YamlParser(text)(handler).document() + val map = doc.node.as[YMap] + assert(map.entries.length == 1) + assert(handler.errors.lengthCompare(1) == 0) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: 'key1\nkey2\n'")) + } + + test("Unclosed key and recover") { + val handler = TestErrorHandler() + + val text = + """key1: valie + |key2: + | otherLevel: + | invalid + | valid: value + | brother + |key3: + |""".stripMargin + val doc = YamlParser(text)(handler).document() + val map = doc.node.as[YMap] + assert(map.entries.length == 3) + val secondMap = map.entries(1).value.as[YMap] + assert(secondMap.entries.length == 1) + assert(secondMap.entries.head.value.as[YMap].entries.length == 1) + + assert(handler.errors.lengthCompare(2) == 0) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: ' invalid\n'")) + assert(handler.errors.last.error.getMessage.equals("Syntax error in the following text: 'brother\n'")) + } + + test("Valid map with unclosed key in middle and recover") { + val handler = TestErrorHandler() + + val text = + """key: + | valid1: value + | invalid + | valid2: value""".stripMargin + val doc = YamlParser(text)(handler).document() + val map = doc.node.as[YMap] + val secondMap = map.entries.head.value.as[YMap] + assert(secondMap.entries.length == 2) + } + + test("Invalid first key in non root map recover") { + val handler = TestErrorHandler() + + val text = + """key: + | Unclosed key at root in middle + | valid1: value""".stripMargin + val doc = YamlParser(text)(handler).document() + val map = doc.node.as[YMap] + val secondMap = map.entries.head.value.as[YMap] + assert(secondMap.entries.length == 1) + } + + test("Sequence recovery in middle") { + val handler = TestErrorHandler() + + val text = + """key: + | - seq1 + | error + | - seq2""".stripMargin + val docs = YamlParser(text)(handler).document() + val map = docs.node.as[YMap] + assert(map.entries.length == 1) + val seq: Seq[String] = map.entries.head.value.as[Seq[String]] + assert(seq.length == 2) + assert(handler.errors.lengthCompare(1) == 0) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: 'error\n'")) + } + + test("Parse invalid entry value single document as scalar and map and recovery") { + val handler = TestErrorHandler() + + val text = + """key: + | key2: value + | map: value + | recovery: value""".stripMargin + + val doc = YamlParser(text)(handler).document() + assert(doc.node.as[YMap].entries.head.value.as[YMap].entries.length == 2) + + assert(handler.errors.lengthCompare(2) == 0) + assert(handler.errors.head.error.getMessage.equals("Syntax error in the following text: 'value'")) + assert(handler.errors.last.error.getMessage.equals("Syntax error in the following text: ' map: value\n'")) + } + + test("Empty line indented"){ + val handler = TestErrorHandler() + + val text = "example: |\n a multiline string\n " + + YamlParser(text)(handler).document() + assert(handler.errors.lengthCompare(0) == 0) + } + + test("Quoted string with flow tokens"){ + val handler = TestErrorHandler() + val text = "example: \n '{\"message\": \"Flight added (but not really)\"'" + + YamlParser(text)(handler).document() + assert(handler.errors.lengthCompare(0) == 0) + } + + test("Array as key"){ + val handler = TestErrorHandler() + val text = "toplevel:\n []:\n son: value" + + YamlParser(text)(handler).document() + assert(handler.errors.lengthCompare(0) == 0) + } + + test("Flow map as key"){ + val handler = TestErrorHandler() + val text = "toplevel:\n {}:\n son: value" + + YamlParser(text)(handler).document() + assert(handler.errors.lengthCompare(0) == 0) + } + case class TestErrorHandler() extends ParseErrorHandler { val errors = new mutable.ListBuffer[ErrorContainer]() diff --git a/shared/src/test/scala/org/mulesoft/yaml/YamlToYeastTest.scala b/shared/src/test/scala/org/mulesoft/yaml/YamlToYeastTest.scala index a64829c2..b349d04f 100644 --- a/shared/src/test/scala/org/mulesoft/yaml/YamlToYeastTest.scala +++ b/shared/src/test/scala/org/mulesoft/yaml/YamlToYeastTest.scala @@ -30,7 +30,7 @@ trait YamlToYeastTest extends GoldenSuite { } private def generate(yamlFile: SyncFile, yeastFile: SyncFile) = { - val lexer = YamlLexer(yamlFile.read()) + val lexer = YamlLexer(yamlFile.read()).initialize() val out = new StringBuilder while (lexer.token != YamlToken.EndStream) { val data = YeastData(lexer.tokenData, lexer.tokenString)