From a5dacbc539b88411ac08e23f5f313e6e4a190c17 Mon Sep 17 00:00:00 2001 From: John Kerl Date: Tue, 9 Aug 2022 08:22:21 -0400 Subject: [PATCH] allow multi-line string literals in the DSL --- go.mod | 1 + go.sum | 27 ++ internal/pkg/parsing/lexer/lexer.go | 415 +++++++++--------- internal/pkg/parsing/lexer/transitiontable.go | 8 + internal/pkg/parsing/mlr.bnf | 1 + .../pkg/parsing/parser/productionstable.go | 24 +- 6 files changed, 249 insertions(+), 227 deletions(-) diff --git a/go.mod b/go.mod index c468ea0cbf..32a0f22208 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ go 1.15 require ( github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb + github.com/goccmack/gocc v0.0.0-20211213154817-7ea699349eca // indirect github.com/johnkerl/lumin v1.0.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/lestrrat-go/strftime v1.0.6 diff --git a/go.sum b/go.sum index 952915574b..536920354d 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb h1:IT4JYU7k4ikYg1SCxNI1/Tieq/NFvh6dzLdgi7eu0tM= github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc= +github.com/goccmack/gocc v0.0.0-20211213154817-7ea699349eca h1:NuA6w6b01Ojdig+4K1l9p4Pp3unlv4owphbOiENm8m4= +github.com/goccmack/gocc v0.0.0-20211213154817-7ea699349eca/go.mod h1:c4Mb67Mg9+pl6OlxvnFBUiiQOSlXfh0QukINLl54OD0= github.com/johnkerl/lumin v1.0.0 h1:CV34cHZOJ92Y02RbQ0rd4gA0C06Qck9q8blOyaPoWpU= github.com/johnkerl/lumin v1.0.0/go.mod h1:eLf5AdQOaLvzZ2zVy4REr/DSeEwG+CZreHwNLICqv9E= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= @@ -27,12 +29,37 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.5.1 h1:OJxoQ/rynoF0dcCdI7cLPktw/hR2cueqYfjm43oqK38= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654 h1:id054HUawV2/6IGm2IV8KZQjqtwAOo2CYlOToYqa0d0= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf h1:MZ2shdL+ZM/XzY3ZGOnh4Nlpnxz5GSOhOmtHo3iPU6M= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/pkg/parsing/lexer/lexer.go b/internal/pkg/parsing/lexer/lexer.go index dea79dd4f9..fe57dda580 100644 --- a/internal/pkg/parsing/lexer/lexer.go +++ b/internal/pkg/parsing/lexer/lexer.go @@ -12,7 +12,7 @@ import ( const ( NoState = -1 NumStates = 336 - NumSymbols = 639 + NumSymbols = 640 ) type Lexer struct { @@ -559,213 +559,214 @@ Lexer symbols: 427: '[' 428: '_' 429: '_' -430: ' ' -431: '!' -432: '#' -433: '$' -434: '%' -435: '&' -436: ''' -437: '\' -438: '(' -439: ')' -440: '*' -441: '+' -442: ',' -443: '-' -444: '.' -445: '/' -446: ':' -447: ';' -448: '<' -449: '=' -450: '>' -451: '?' -452: '@' -453: '[' -454: ']' -455: '^' -456: '_' -457: '`' -458: '{' -459: '|' -460: '}' -461: '~' -462: '\' +430: '\n' +431: ' ' +432: '!' +433: '#' +434: '$' +435: '%' +436: '&' +437: ''' +438: '\' +439: '(' +440: ')' +441: '*' +442: '+' +443: ',' +444: '-' +445: '.' +446: '/' +447: ':' +448: ';' +449: '<' +450: '=' +451: '>' +452: '?' +453: '@' +454: '[' +455: ']' +456: '^' +457: '_' +458: '`' +459: '{' +460: '|' +461: '}' +462: '~' 463: '\' 464: '\' -465: '"' -466: '\' -467: '[' -468: '\' -469: ']' -470: '\' -471: '.' -472: '\' -473: '*' -474: '\' -475: '%' -476: '\' -477: 'A' -478: '\' -479: 'B' -480: '\' -481: 'C' -482: '\' -483: 'D' -484: '\' -485: 'G' -486: '\' -487: 'H' -488: '\' -489: 'K' -490: '\' -491: 'L' -492: '\' -493: 'N' -494: '\' -495: 'P' -496: '\' -497: 'R' -498: '\' -499: 'S' -500: '\' -501: 'U' -502: '\' -503: 'V' -504: '\' -505: 'W' -506: '\' -507: 'X' -508: '\' -509: 'Z' -510: '\' -511: 'a' -512: '\' -513: 'b' -514: '\' -515: 'c' -516: '\' -517: 'd' -518: '\' -519: 'f' -520: '\' -521: 'g' -522: '\' -523: 'h' -524: '\' -525: 'k' -526: '\' -527: 'l' -528: '\' -529: 'n' -530: '\' -531: 'p' -532: '\' -533: 'r' -534: '\' -535: 's' -536: '\' -537: 't' -538: '\' -539: 'u' -540: '\' -541: 'v' -542: '\' -543: 'w' -544: '\' -545: 'x' -546: '\' -547: 'z' -548: '\' -549: '0' -550: '\' -551: '1' -552: '\' -553: '2' -554: '\' -555: '3' -556: '\' -557: '4' -558: '\' -559: '5' -560: '\' -561: '6' -562: '\' -563: '7' -564: '\' -565: '8' -566: '\' -567: '9' -568: 'e' -569: 'E' -570: 't' -571: 'r' -572: 'u' -573: 'e' -574: 'f' -575: 'a' -576: 'l' -577: 's' -578: 'e' -579: ' ' -580: '!' -581: '#' -582: '$' -583: '%' -584: '&' -585: ''' -586: '\' -587: '(' -588: ')' -589: '*' -590: '+' -591: ',' -592: '-' -593: '.' -594: '/' -595: ':' -596: ';' -597: '<' -598: '=' -599: '>' -600: '?' -601: '@' -602: '[' -603: ']' -604: '^' -605: '_' -606: '`' -607: '|' -608: '~' -609: '\' -610: '{' -611: '\' -612: '}' -613: ' ' -614: '\t' -615: '\n' -616: '\r' -617: '#' -618: '\n' -619: 'a'-'z' -620: 'A'-'Z' -621: \u00a0-\u00ff -622: \u0100-\U0010ffff -623: '0'-'9' +465: '\' +466: '"' +467: '\' +468: '[' +469: '\' +470: ']' +471: '\' +472: '.' +473: '\' +474: '*' +475: '\' +476: '%' +477: '\' +478: 'A' +479: '\' +480: 'B' +481: '\' +482: 'C' +483: '\' +484: 'D' +485: '\' +486: 'G' +487: '\' +488: 'H' +489: '\' +490: 'K' +491: '\' +492: 'L' +493: '\' +494: 'N' +495: '\' +496: 'P' +497: '\' +498: 'R' +499: '\' +500: 'S' +501: '\' +502: 'U' +503: '\' +504: 'V' +505: '\' +506: 'W' +507: '\' +508: 'X' +509: '\' +510: 'Z' +511: '\' +512: 'a' +513: '\' +514: 'b' +515: '\' +516: 'c' +517: '\' +518: 'd' +519: '\' +520: 'f' +521: '\' +522: 'g' +523: '\' +524: 'h' +525: '\' +526: 'k' +527: '\' +528: 'l' +529: '\' +530: 'n' +531: '\' +532: 'p' +533: '\' +534: 'r' +535: '\' +536: 's' +537: '\' +538: 't' +539: '\' +540: 'u' +541: '\' +542: 'v' +543: '\' +544: 'w' +545: '\' +546: 'x' +547: '\' +548: 'z' +549: '\' +550: '0' +551: '\' +552: '1' +553: '\' +554: '2' +555: '\' +556: '3' +557: '\' +558: '4' +559: '\' +560: '5' +561: '\' +562: '6' +563: '\' +564: '7' +565: '\' +566: '8' +567: '\' +568: '9' +569: 'e' +570: 'E' +571: 't' +572: 'r' +573: 'u' +574: 'e' +575: 'f' +576: 'a' +577: 'l' +578: 's' +579: 'e' +580: ' ' +581: '!' +582: '#' +583: '$' +584: '%' +585: '&' +586: ''' +587: '\' +588: '(' +589: ')' +590: '*' +591: '+' +592: ',' +593: '-' +594: '.' +595: '/' +596: ':' +597: ';' +598: '<' +599: '=' +600: '>' +601: '?' +602: '@' +603: '[' +604: ']' +605: '^' +606: '_' +607: '`' +608: '|' +609: '~' +610: '\' +611: '{' +612: '\' +613: '}' +614: ' ' +615: '\t' +616: '\n' +617: '\r' +618: '#' +619: '\n' +620: 'a'-'z' +621: 'A'-'Z' +622: \u00a0-\u00ff +623: \u0100-\U0010ffff 624: '0'-'9' -625: 'a'-'f' -626: 'A'-'F' -627: '0'-'7' -628: '0'-'1' -629: 'A'-'Z' -630: 'a'-'z' -631: '0'-'9' -632: \u00a0-\u00ff -633: \u0100-\U0010ffff -634: 'A'-'Z' -635: 'a'-'z' -636: '0'-'9' -637: \u0100-\U0010ffff -638: . +625: '0'-'9' +626: 'a'-'f' +627: 'A'-'F' +628: '0'-'7' +629: '0'-'1' +630: 'A'-'Z' +631: 'a'-'z' +632: '0'-'9' +633: \u00a0-\u00ff +634: \u0100-\U0010ffff +635: 'A'-'Z' +636: 'a'-'z' +637: '0'-'9' +638: \u0100-\U0010ffff +639: . */ diff --git a/internal/pkg/parsing/lexer/transitiontable.go b/internal/pkg/parsing/lexer/transitiontable.go index f7566d621c..87c9c5d879 100644 --- a/internal/pkg/parsing/lexer/transitiontable.go +++ b/internal/pkg/parsing/lexer/transitiontable.go @@ -169,6 +169,8 @@ var TransTab = TransitionTable{ // S3 func(r rune) int { switch { + case r == 10: // ['\n','\n'] + return 57 case r == 32: // [' ',' '] return 57 case r == 33: // ['!','!'] @@ -1142,6 +1144,8 @@ var TransTab = TransitionTable{ // S57 func(r rune) int { switch { + case r == 10: // ['\n','\n'] + return 57 case r == 32: // [' ',' '] return 57 case r == 33: // ['!','!'] @@ -2964,6 +2968,8 @@ var TransTab = TransitionTable{ // S157 func(r rune) int { switch { + case r == 10: // ['\n','\n'] + return 57 case r == 32: // [' ',' '] return 57 case r == 33: // ['!','!'] @@ -4412,6 +4418,8 @@ var TransTab = TransitionTable{ // S236 func(r rune) int { switch { + case r == 10: // ['\n','\n'] + return 57 case r == 32: // [' ',' '] return 57 case r == 33: // ['!','!'] diff --git a/internal/pkg/parsing/mlr.bnf b/internal/pkg/parsing/mlr.bnf index 95b7d82c9b..6503e61ef0 100644 --- a/internal/pkg/parsing/mlr.bnf +++ b/internal/pkg/parsing/mlr.bnf @@ -83,6 +83,7 @@ _idchar : _letter | _decdig | '_' ; _string_literal_element : 'A'-'Z' | 'a'-'z' | '0'-'9' + | '\n' | ' ' | '!' | '#' | '$' | '%' | '&' | '\'' | '\\' | '(' | ')' | '*' | '+' | ',' | '-' | '.' | '/' diff --git a/internal/pkg/parsing/parser/productionstable.go b/internal/pkg/parsing/parser/productionstable.go index 04ab8120d7..6447de3749 100644 --- a/internal/pkg/parsing/parser/productionstable.go +++ b/internal/pkg/parsing/parser/productionstable.go @@ -2708,31 +2708,15 @@ var productionsTable = ProdTab{ }, }, ProdTabEntry{ - String: `PowTerm : PrecedenceChainEnd "**" "-" PowTerm << dsl.NewASTNodeBinary( - X[1], - X[0], - dsl.NewASTNodeUnaryNestable( - X[2], - X[3], - dsl.NodeTypeOperator, - ), - dsl.NodeTypeOperator, - ) >>`, + String: `PowTerm : PrecedenceChainEnd "**" "-" PowTerm << dsl.NewASTNodeBinary( X[1], X[0], + dsl.NewASTNodeUnaryNestable( X[2], X[3], dsl.NodeTypeOperator,), dsl.NodeTypeOperator,) >>`, Id: "PowTerm", NTType: 60, Index: 205, NumSymbols: 4, ReduceFunc: func(X []Attrib, C interface{}) (Attrib, error) { - return dsl.NewASTNodeBinary( - X[1], - X[0], - dsl.NewASTNodeUnaryNestable( - X[2], - X[3], - dsl.NodeTypeOperator, - ), - dsl.NodeTypeOperator, - ) + return dsl.NewASTNodeBinary(X[1], X[0], + dsl.NewASTNodeUnaryNestable(X[2], X[3], dsl.NodeTypeOperator), dsl.NodeTypeOperator) }, }, ProdTabEntry{