Added HTML parser support

Gruntfile.js

@@ -43,5 +43,10 @@ module.exports = function (grunt) {
   grunt.loadTasks('tasks');
 
   grunt.registerTask('default', ['jshint', 'jscs', 'mochacli']);
-  grunt.registerTask('test', 'default');
+  grunt.registerTask('test', function (file) {
+    if (file) {
+      grunt.config('mochacli.all', 'test/test-' + file + '.js');
+    }
+    grunt.task.run('mochacli');
+  });
 };

lib/file.js

@@ -1,14 +1,14 @@
 'use strict';
 var path = require('path');
 var fs = require('fs');
+var parse5 = require('parse5');
 
 // File is responsible to gather all information related to a given parsed file, as:
 //  - its dir and name
 //  - its content
 //  - the search paths where referenced resource will be looked at
 //  - the list of parsed blocks
 //
-
 //
 // Returns an array object of all the directives for the given html.
 // Each item of the array has the following form:
@@ -33,6 +33,14 @@ var fs = require('fs');
 //    <!-- build:css /foo/css/site.css -->
 // then dest will equal foo/css/site.css (note missing trailing /)
 //
+var getAttributes = function (attributes) {
+  var tmp = {};
+  return attributes.reduce(function (previousValue, currentValue) {
+    tmp[currentValue.name] = currentValue.value;
+    return tmp;
+  }, tmp);
+};
+
 var getBlocks = function (content) {
   // start build pattern: will match
   //  * <!-- build:[target] output -->
@@ -43,107 +51,119 @@ var getBlocks = function (content) {
   //   * 2 : the alternate search path
   //   * 3 : the output
   //
-  var regbuild = /<!--\s*build:(\w+)(?:\(([^\)]+)\))?\s*([^\s]+)\s*-->/;
+  var regbuild = /\s*build:(\w+)(?:\(([^\)]+)\))?\s*([^\s]+)\s*/;
   // end build pattern -- <!-- endbuild -->
-  var regend = /<!--\s*endbuild\s*-->/;
-
-  var lines = content.replace(/\r\n/g, '\n').split(/\n/);
+  var regend = /\s*endbuild\s*/;
   var block = false;
   var sections = [];
   var last;
-
-  lines.forEach(function (l) {
-    var indent = (l.match(/^\s*/) || [])[0];
-    var build = l.match(regbuild);
-    var endbuild = regend.test(l);
-    var startFromRoot = false;
-
-    // discard empty lines
-    if (build) {
-      block = true;
-      // Handle absolute path (i.e. with respect to the server root)
-      // if (build[3][0] === '/') {
-      //   startFromRoot = true;
-      //   build[3] = build[3].substr(1);
-      // }
-      last = {
-        type: build[1],
-        dest: build[3],
-        startFromRoot: startFromRoot,
-        indent: indent,
-        searchPath: [],
-        src: [],
-        raw: []
-      };
-
-      if (build[2]) {
-        // Alternate search path
-        last.searchPath.push(build[2]);
-      }
-    }
-    // Check IE conditionals
-    var isConditionalStart = l.match(/(<!--\[if.*\]>)(<!-->)?( -->)?/g);
-    var isConditionalEnd = l.match(/(<!--\s?)?(<!\[endif\]-->)/g);
-    if (block && isConditionalStart) {
-      last.conditionalStart = isConditionalStart;
-    }
-    if (block && isConditionalEnd) {
-      last.conditionalEnd = isConditionalEnd;
-    }
-
-    // switch back block flag when endbuild
-    if (block && endbuild) {
-      last.raw.push(l);
-      sections.push(last);
-      block = false;
-    }
-
-    if (block && last) {
-      var asset = l.match(/(href|src)=["']([^'"]+)["']/);
-      if (asset && asset[2]) {
-        last.src.push(asset[2]);
-
-        var media = l.match(/media=['"]([^'"]+)['"]/);
-        // FIXME: media attribute should be present for all members of the block *and* having the same value
-        if (media) {
-          last.media = media[1];
+  var startFromRoot = false;
+  var asset = null;
+
+  var parser = new parse5.SimpleApiParser({
+    startTag: function (tagName, attrs, selfClosing , location) {
+      if ((tagName === 'script' || tagName === 'link') && block && last) {
+        var attributes = getAttributes(attrs);
+        if (attributes) {
+          if (attributes.src && !!attributes.src) {
+            asset = attributes.src;
+          } else if (attributes.href && !!attributes.href) {
+            asset = attributes.href;
+          }
         }
 
-        // preserve defer attribute
-        var defer = / defer/.test(l);
-        if (defer && last.defer === false || last.defer && !defer) {
-          throw new Error('Error: You are not supposed to mix deferred and non-deferred scripts in one block.');
-        } else if (defer) {
-          last.defer = true;
-        } else {
-          last.defer = false;
+        if (asset) {
+          last.src.push(asset);
+
+          // FIXME: media attribute should be present for all members of the block *and* having the same value
+          if (attributes && attributes.media && !!attributes.media) {
+            last.media = attributes.media;
+          }
+
+          // preserve defer attribute
+          var defer = attributes && attributes.hasOwnProperty('defer');
+          if (defer && last.defer === false || last.defer && !defer) {
+            throw new Error('Error: You are not supposed to mix deferred and non-deferred scripts in one block.');
+          } else if (defer) {
+            last.defer = true;
+          } else {
+            last.defer = false;
+          }
+
+          // preserve async attribute
+          var async = attributes && attributes.hasOwnProperty('async');
+          if (async && last.async === false || last.async && !async) {
+            throw new Error('Error: You are not supposed to mix asynced and non-asynced scripts in one block.');
+          } else if (async) {
+            last.async = true;
+          } else {
+            last.async = false;
+          }
+
+          // RequireJS uses a data-main attribute on the script tag to tell it
+          // to load up the main entry point of the amp app
+          //
+          // If we find one, we must record the name of the main entry point,
+          // as well the name of the destination file, and treat
+          // the furnished requirejs as an asset (src)
+          var main = attributes && attributes['data-main'];
+          if (main) {
+            throw new Error('require.js blocks are no more supported.');
+          }
         }
 
-        // preserve async attribute
-        var async = / async/.test(l);
-        if (async && last.async === false || last.async && !async) {
-          throw new Error('Error: You are not supposed to mix asynced and non-asynced scripts in one block.');
-        } else if (async) {
-          last.async = true;
-        } else {
-          last.async = false;
+        last.raw.push(content.substring(location.start, location.end));
+        asset = null;
+      }
+    },
+    comment: function (text, location) {
+      var indent = (text.match(/^\s*/) || [])[0];
+      var endbuild = regend.test(text);
+      var build = text.match(regbuild);
+
+      // discard empty lines
+      if (build) {
+        block = true;
+        last = {
+          type: build[1],
+          dest: build[3],
+          startFromRoot: startFromRoot,
+          indent: indent,
+          searchPath: [],
+          src: [],
+          raw: [content.substring(location.start, location.end)]
+        };
+
+        if (build[2]) {
+          // Alternate search path
+          last.searchPath.push(build[2]);
         }
+      }
 
-        // RequireJS uses a data-main attribute on the script tag to tell it
-        // to load up the main entry point of the amp app
-        //
-        // If we find one, we must record the name of the main entry point,
-        // as well the name of the destination file, and treat
-        // the furnished requirejs as an asset (src)
-        var main = l.match(/data-main=['"]([^'"]+)['"]/);
-        if (main) {
-          throw new Error('require.js blocks are no more supported.');
-        }
+      // Check IE conditionals
+      var isConditionalStart = text.match(/(\[if.*\]>)(<!-->)?( -->)?/g);
+      var isConditionalEnd = text.match(/(<!--\s?)?(<!\[endif\])/g);
+      if (block && isConditionalStart) {
+        last.conditionalStart = isConditionalStart;
+      }
+      if (block && isConditionalEnd) {
+        last.conditionalEnd = isConditionalEnd;
+      }
+
+      // switch back block flag when endbuild
+      if (block && endbuild) {
+        last.raw.push(content.substring(location.start, location.end));
+        sections.push(last);
+        block = false;
       }
-      last.raw.push(l);
     }
+  }, {
+    decodeEntities: true,
+    locationInfo: true
   });
 
+  parser.parse(content);
+
   return sections;
 };
 

lib/fileprocessor.js

@@ -1,5 +1,6 @@
 'use strict';
 var debug = require('debug')('fileprocessor');
+var escapeStringRegexp = require('escape-string-regexp');
 var File = require('./file');
 var _ = require('lodash');
 var chalk = require('chalk');
@@ -151,10 +152,16 @@ var FileProcessor = module.exports = function (type, patterns, finder, logcb, bl
 //
 FileProcessor.prototype.replaceBlocks = function replaceBlocks(file) {
   var result = file.content;
-  var linefeed = /\r\n/g.test(result) ? '\r\n' : '\n';
+
   file.blocks.forEach(function (block) {
-    var blockLine = block.raw.join(linefeed);
-    result = result.replace(blockLine, this.replaceWith(block));
+    var firstBlock = escapeStringRegexp(block.raw[0] + '');
+    var lastBlock = escapeStringRegexp(block.raw[block.raw.length - 1] + '');
+
+    var expression = (firstBlock + '[\\s\\S]*?' + lastBlock);
+
+    var regex = new RegExp(expression, 'm');
+
+    result = result.replace(regex, this.replaceWith(block));
   }, this);
   return result;
 };

lib/revvedfinder.js

@@ -1,5 +1,6 @@
 'use strict';
 var debug = require('debug')('revvedfinder');
+var escapeStringRegexp = require('escape-string-regexp');
 var path = require('path');
 var _ = require('lodash');
 
@@ -19,10 +20,6 @@ var RevvedFinder = module.exports = function (locator) {
   }
 };
 
-var regexpQuote = function (str) {
-  return (str + '').replace(/([.?*+\^$\[\]\\(){}|\-])/g, '\\$1');
-};
-
 RevvedFinder.prototype.getCandidatesFromMapping = function (file, searchPaths) {
   var dirname = path.dirname(file);
   var filepath = dirname === '.' ? '' : dirname + '/';
@@ -53,8 +50,8 @@ RevvedFinder.prototype.getCandidatesFromFS = function (file, searchPaths) {
   var basename = path.basename(file, extname);
   var dirname = path.dirname(file);
   var hex = '[0-9a-fA-F]+';
-  var regPrefix = '(' + hex + '\\.' + regexpQuote(basename) + ')';
-  var regSuffix = '(' + regexpQuote(basename) + '\\.' + hex + regexpQuote(extname) + ')';
+  var regPrefix = '(' + hex + '\\.' + escapeStringRegexp(basename) + ')';
+  var regSuffix = '(' + escapeStringRegexp(basename) + '\\.' + hex + escapeStringRegexp(extname) + ')';
   var revvedRx = new RegExp(regPrefix + '|' + regSuffix);
   var candidates = [];
   var self = this;

package.json

@@ -26,7 +26,9 @@
   "dependencies": {
     "chalk": "^1.0.0",
     "debug": "^2.1.1",
-    "lodash": "^3.3.1"
+    "escape-string-regexp": "^1.0.3",
+    "lodash": "^3.3.1",
+    "parse5": "^1.4.1"
   },
   "devDependencies": {
     "grunt": "^0.4.5",

test/fixtures/block_compressed.html

@@ -0,0 +1 @@
+<!-- build:js foo.js --><script src="bar01.js"></script><script src="bar02.js"></script><script src="bar03.js"></script><script src="bar04.js"></script><!-- endbuild -->

test/fixtures/block_with_custom_attributes.html

@@ -0,0 +1,4 @@
+<!-- build:css styles/main.css -->
+<link rel="stylesheet" href="styles/foo.css" data-attr="1" data-test="2">
+<link rel="stylesheet" href="styles/bar.css" my-custom-attribute>
+<!-- endbuild -->

test/fixtures/block_with_mixed_async.html

@@ -1,4 +1,4 @@
 <!-- build:js foo.js -->
-  <script src="bar.js" async ></script>
+  <script src="bar.js" async></script>
   <script src="baz.js"></script>
 <!-- endbuild -->

test/fixtures/block_with_mixed_defer.html

@@ -1,4 +1,4 @@
 <!-- build:js foo.js -->
-  <script src="bar.js" defer ></script>
+  <script src="bar.js" defer></script>
   <script src="baz.js"></script>
 <!-- endbuild -->

test/fixtures/block_with_mixed_defer2.html

@@ -1,4 +1,4 @@
 <!-- build:js foo.js -->
   <script src="baz.js"></script>
-  <script src="bar.js" defer ></script>
+  <script src="bar.js" defer></script>
 <!-- endbuild -->

test/test-file.js

@@ -16,15 +16,16 @@ describe('File', function () {
     assert.ok(file.blocks.length, 2);
   });
 
-  it('should *not* skip blank lines', function () {
+  it('should skip blank lines', function () {
     var filename = __dirname + '/fixtures/block_with_empty_line.html';
     var file = new File(filename);
 
     assert.equal(1, file.blocks.length);
     assert.equal('foo.css', file.blocks[0].dest);
-    assert.equal(5, file.blocks[0].raw.length);
     assert.equal(2, file.blocks[0].src.length);
-    assert.equal('  ', file.blocks[0].indent);
+    assert.equal(4, file.blocks[0].raw.length);
+    assert.equal(' ', file.blocks[0].indent);
+    assert.equal(32, file.blocks[0].indent.charCodeAt(0));
   });
 
   it('should return the right number of blocks with the right number of lines', function () {
@@ -36,7 +37,7 @@ describe('File', function () {
     assert.equal(3, b1.raw.length);
     assert.equal('css', b1.type);
     assert.equal(1, b1.src.length);
-    assert.equal(16, b2.raw.length);
+    assert.equal(15, b2.raw.length);
     assert.equal('js', b2.type);
     assert.equal(13, b2.src.length);
   });
@@ -63,8 +64,8 @@ describe('File', function () {
     assert.equal(1, file.blocks.length);
     assert.ok(file.blocks[0].conditionalStart);
     assert.ok(file.blocks[0].conditionalEnd);
-    assert.equal('<!--[if (lt IE 9) & (!IEmobile)]>', file.blocks[0].conditionalStart);
-    assert.equal('<![endif]-->', file.blocks[0].conditionalEnd);
+    assert.equal('[if (lt IE 9) & (!IEmobile)]>', file.blocks[0].conditionalStart);
+    assert.equal('<![endif]', file.blocks[0].conditionalEnd);
   });
 
   it('should also detect block that has IE conditionals within block', function () {
@@ -73,8 +74,8 @@ describe('File', function () {
     assert.equal(1, file.blocks.length);
     assert.ok(file.blocks[0].conditionalStart);
     assert.ok(file.blocks[0].conditionalEnd);
-    assert.equal('<!--[if (lt IE 9) & (!IEmobile)]>', file.blocks[0].conditionalStart);
-    assert.equal('<![endif]-->', file.blocks[0].conditionalEnd);
+    assert.equal('[if (lt IE 9) & (!IEmobile)]>', file.blocks[0].conditionalStart);
+    assert.equal('<![endif]', file.blocks[0].conditionalEnd);
   });
 
   it('should throw an exception if it finds RequireJS blocks', function () {
@@ -166,6 +167,4 @@ describe('File', function () {
     assert.ok(file.blocks[0].media);
     assert.equal('(min-width:980px)', file.blocks[0].media);
   });
-
-
 });