Merge pull request #1 from danielbrzn/markdown-parsing-fix

Update Tokenizer to treat Markdown code as text instead of HTML

lib/Tokenizer.js

@@ -7,6 +7,7 @@ var decodeCodePoint = require("entities/lib/decode_codepoint.js"),
 
     i = 0,
 
+    MARKDOWN                  = i++,
     TEXT                      = i++,
     BEFORE_TAG_NAME           = i++, //after <
     IN_TAG_NAME               = i++,
@@ -146,13 +147,24 @@ function Tokenizer(options, cbs){
 	this._decodeEntities = !!(options && options.decodeEntities);
 }
 
+Tokenizer.prototype._stateMarkdown = function(c){
+	if(c === '`') {
+		this._state = TEXT;
+	}
+}
+
 Tokenizer.prototype._stateText = function(c){
-	if(c === "<"){
-		if(this._index > this._sectionStart){
-			this._cbs.ontext(this._getSection());
+	if(c === '`'){
+		this._state = MARKDOWN;
+	} else if(c === "<"){
+		var isInequality = (this._index + 1 < this._buffer.length) && (this._buffer.charAt(this._index + 1) === '=');
+		if(!isInequality){
+			if(this._index > this._sectionStart){
+				this._cbs.ontext(this._getSection());
+			}
+			this._state = BEFORE_TAG_NAME;
+			this._sectionStart = this._index;
 		}
-		this._state = BEFORE_TAG_NAME;
-		this._sectionStart = this._index;
 	} else if(this._decodeEntities && this._special === SPECIAL_NONE && c === "&"){
 		if(this._index > this._sectionStart){
 			this._cbs.ontext(this._getSection());
@@ -635,11 +647,14 @@ Tokenizer.prototype.write = function(chunk){
 Tokenizer.prototype._parse = function(){
 	while(this._index < this._buffer.length && this._running){
 		var c = this._buffer.charAt(this._index);
-		if(this._state === TEXT) {
+
+		if(this._state === MARKDOWN){
+			this._stateMarkdown(c);
+		} else if(this._state === TEXT){
 			this._stateText(c);
 		} else if(this._state === BEFORE_TAG_NAME){
 			this._stateBeforeTagName(c);
-		} else if(this._state === IN_TAG_NAME) {
+		} else if(this._state === IN_TAG_NAME){
 			this._stateInTagName(c);
 		} else if(this._state === BEFORE_CLOSING_TAG_NAME){
 			this._stateBeforeCloseingTagName(c);