- added the ability that one can add comments on an own line or inser…

…t empty lines in a tokenVocab
robstoll · Jul 20, 2014 · 20d859a · 20d859a
1 parent 823b8fd
commit 20d859a
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 3 deletions.
diff --git a/tool/src/main/java/org/antlr/tool/Grammar.java b/tool/src/main/java/org/antlr/tool/Grammar.java
@@ -2425,6 +2425,9 @@ else if ( token == '\'' ) {
 					continue;
 				}
 				token = tokenizer.nextToken(); // skip newline
+				while(token == StreamTokenizer.TT_EOL ){
+					token = tokenizer.nextToken(); 
+				}
 			}
 			br.close();
 		}

diff --git a/tool/src/test/java/org/antlr/test/TestCompositeGrammars.java b/tool/src/test/java/org/antlr/test/TestCompositeGrammars.java
@@ -535,13 +535,36 @@ public class TestCompositeGrammars extends BaseTest {
 		assertEquals("unexpected warnings: "+equeue, 0, equeue.warnings.size());
 	}
 
-	@Test public void testTokenVocabCommentsOnOwnLineNoWarningsLikeNoLexerRuleCorrespondingToToken() 
+	@Test public void testTokenVocabCommentsOnOwnLineNoErrorNoWarnings() 
 			throws Exception {
 		ErrorQueue equeue = new ErrorQueue();
 		ErrorManager.setErrorListener(equeue);		
 
 		mkdir(tmpdir);
-		writeFile(tmpdir, "Foo.tokens", "TokenFromTokenVocab=4\n//some comments on a new line\n'token'=4\n");
+		writeFile(tmpdir, "Foo.tokens", "TokenFromTokenVocab=4\n"
+				  + "//some comments on a new line\n'token'=4\n");
+
+		String grammar = "grammar Foo;\n" 
+				+ "options {tokenVocab=Foo;}\n"
+				+ "tokens{TokenFromTokenVocab='token';}\n" 
+				+ "a : TokenFromTokenVocab;\n"
+				+ "WS : (' '|'\\n') {$channel=HIDDEN;} ;\n";
+
+		writeFile(tmpdir, "Foo.g", grammar);
+		Tool antlr = newTool(new String[] {"-lib", tmpdir, tmpdir + "/Foo.g"});
+		antlr.process();
+
+		assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
+		assertEquals("unexpected warnings: "+equeue, 0, equeue.warnings.size());
+	}
+
+	@Test public void testTokenVocabWithEmptyLineNoErrorNoWarnings() 
+			throws Exception {
+		ErrorQueue equeue = new ErrorQueue();
+		ErrorManager.setErrorListener(equeue);		
+
+		mkdir(tmpdir);
+		writeFile(tmpdir, "Foo.tokens", "TokenFromTokenVocab=4\n\n'token'=4\n");
 
 		String grammar = "grammar Foo;\n" 
 				+ "options {tokenVocab=Foo;}\n"
@@ -578,7 +601,6 @@ public class TestCompositeGrammars extends BaseTest {
 		assertEquals("unexpected errors: "+equeue, 0, equeue.errors.size());
 		assertEquals("unexpected warnings: "+equeue, 0, equeue.warnings.size());
 	}
-
 
 	@Test public void testSyntaxErrorsInImportsNotThrownOut() throws Exception {
 		ErrorQueue equeue = new ErrorQueue();