[parser] digester: avoid infinite loop when an invalid token is seen (p…

…ingcap#313)
xhebox · May 5, 2019 · 671650d · 671650d
1 parent bf3b1cf
commit 671650d
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 0 deletions.
diff --git a/parser/digester.go b/parser/digester.go
@@ -111,6 +111,9 @@ func (d *sqlDigester) normalize(sql string) {
 	d.lexer.reset(sql)
 	for {
 		tok, pos, lit := d.lexer.scan()
+		if tok == invalid {
+			break
+		}
 		if tok == unicode.ReplacementChar && d.lexer.r.eof() {
 			break
 		}

diff --git a/parser/digester_test.go b/parser/digester_test.go
@@ -50,6 +50,7 @@ func (s *testSQLDigestSuite) TestNormalize(c *C) {
 		// test syntax error, it will be checked by parser, but it should not make normalize dead loop.
 		{"select * from t ignore index(", "select * from t ignore index"},
 		{"select /*+ ", "select "},
+		{"select * from 🥳", "select * from"},
 	}
 	for _, test := range tests {
 		actual := parser.Normalize(test.input)