From f0b12a2769734ddf716d7fa4f54f2da90a4dd6ef Mon Sep 17 00:00:00 2001
From: "Michael[tm] Smith" <mike@w3.org>
Date: Fri, 10 Jun 2016 21:39:28 +0900
Subject: [PATCH] Make consecutive hyphens in comments a non-error

This is experiment and may be reverted later.
---
 .../impl/ErrorReportingTokenizer.java         |   4 +-
 .../validator/htmlparser/impl/Tokenizer.java  | 104 +++++++++++++++++-
 2 files changed, 104 insertions(+), 4 deletions(-)

diff --git a/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java b/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
index f1749e0b..b2bfe3ae 100644
--- a/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
+++ b/src/nu/validator/htmlparser/impl/ErrorReportingTokenizer.java
@@ -413,8 +413,8 @@ private boolean isAstralPrivateUse(int c) {
         err("Nameless doctype.");
     }
 
-    @Override protected void errConsecutiveHyphens() throws SAXException {
-        err("Consecutive hyphens did not terminate a comment. \u201C--\u201D is not permitted inside a comment, but e.g. \u201C- -\u201D is.");
+    @Override protected void errNestedComment() throws SAXException {
+        err("Saw \u201C<!--\u201D within a comment. Probable cause: Nested comment (not allowed).");
     }
 
     @Override protected void errPrematureEndOfComment() throws SAXException {
diff --git a/src/nu/validator/htmlparser/impl/Tokenizer.java b/src/nu/validator/htmlparser/impl/Tokenizer.java
index cb3281fe..d3dc34b3 100755
--- a/src/nu/validator/htmlparser/impl/Tokenizer.java
+++ b/src/nu/validator/htmlparser/impl/Tokenizer.java
@@ -222,6 +222,12 @@ public class Tokenizer implements Locator {
 
     public static final int AMBIGUOUS_AMPERSAND = 75;
 
+    public static final int COMMENT_LESSTHAN = 76;
+
+    public static final int COMMENT_LESSTHAN_BANG = 77;
+
+    public static final int COMMENT_LESSTHAN_BANG_DASH = 78;
+
     /**
      * Magic value for UTF-16 operations.
      */
@@ -923,7 +929,6 @@ private void maybeAppendSpaceToBogusComment() throws SAXException {
 
     @Inline private void adjustDoubleHyphenAndAppendToStrBufAndErr(char c)
             throws SAXException {
-        errConsecutiveHyphens();
         // [NOCPP[
         switch (commentPolicy) {
             case ALTER_INFOSET:
@@ -2446,6 +2451,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  */
                                 state = transition(state, Tokenizer.DATA, reconsume, pos);
                                 continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2491,6 +2500,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 state = transition(state, Tokenizer.COMMENT_END_DASH, reconsume, pos);
                                 break commentloop;
                             // continue stateloop;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 appendStrBufCarriageReturn();
                                 break stateloop;
@@ -2592,6 +2605,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                  * Stay in the comment end state.
                                  */
                                 continue;
+                            case '<':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                                continue stateloop;
                             case '\r':
                                 adjustDoubleHyphenAndAppendToStrBufCarriageReturn();
                                 state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -2683,6 +2700,85 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                                 continue stateloop;
                         }
                     }
+                case COMMENT_LESSTHAN:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '!':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '-':
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT_LESSTHAN_BANG_DASH, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
+                case COMMENT_LESSTHAN_BANG_DASH:
+                    for (;;) {
+                        if (++pos == endPos) {
+                            break stateloop;
+                        }
+                        c = checkChar(buf, pos);
+                        switch (c) {
+                            case '-':
+                                errNestedComment();
+                                adjustDoubleHyphenAndAppendToStrBufAndErr(c);
+                                state = transition(state, Tokenizer.COMMENT_END, reconsume, pos);
+                                continue stateloop;
+                            case '\r':
+                                appendStrBufCarriageReturn();
+                                break stateloop;
+                            case '\n':
+                                appendStrBufLineFeed();
+                                continue;
+                            case '\u0000':
+                                c = '\uFFFD';
+                                // fall thru
+                            default:
+                                appendStrBuf(c);
+                                state = transition(state, Tokenizer.COMMENT, reconsume, pos);
+                                continue stateloop;
+                        }
+                    }
                     // XXX reorder point
                 case COMMENT_START_DASH:
                     if (++pos == endPos) {
@@ -2712,6 +2808,10 @@ private void ensureBufferSpace(int inputLength) throws SAXException {
                              */
                             state = transition(state, Tokenizer.DATA, reconsume, pos);
                             continue stateloop;
+                        case '<':
+                            appendStrBuf(c);
+                            state = transition(state, Tokenizer.COMMENT_LESSTHAN, reconsume, pos);
+                            continue stateloop;
                         case '\r':
                             appendStrBufCarriageReturn();
                             state = transition(state, Tokenizer.COMMENT, reconsume, pos);
@@ -6777,7 +6877,7 @@ protected void errGtInPublicId() throws SAXException {
     protected void errNamelessDoctype() throws SAXException {
     }
 
-    protected void errConsecutiveHyphens() throws SAXException {
+    protected void errNestedComment() throws SAXException {
     }
 
     protected void errPrematureEndOfComment() throws SAXException {