Enable support for multiple fields

twitter · Jul 9, 2014 · bca7441 · bca7441
1 parent 0973c23
commit bca7441
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 4 deletions.
diff --git a/src/bloodhound/tokenizers.js b/src/bloodhound/tokenizers.js
@@ -15,13 +15,31 @@ var tokenizers = (function(root) {
     }
   };
 
-  function whitespace(s) { return s.split(/\s+/); }
+  function toStr(s) { return (_.isUndefined(s) || s === null) ? '' : s + ''; }
 
-  function nonword(s) { return s.split(/\W+/); }
+  function whitespace(str) {
+    str = toStr(str);
+    return str ? str.split(/\s+/) : [];
+  }
+
+  function nonword(str) {
+    str = toStr(str);
+    return str ? str.split(/\W+/) : [];
+  }
 
   function getObjTokenizer(tokenizer) {
-    return function setKey(key) {
-      return function tokenize(o) { return tokenizer(o[key]); };
+    return function setKey(/* key, ... */) {
+      var args = [].slice.call(arguments, 0);
+
+      return function tokenize(o) {
+        var val, tokens = [];
+
+        _.each(args, function(k) {
+          tokens = tokens.concat(tokenizer(toStr(o[k])));
+        });
+
+        return tokens;
+      };
     };
   }
 })();
diff --git a/test/tokenizers_spec.js b/test/tokenizers_spec.js
@@ -5,22 +5,56 @@ describe('tokenizers', function() {
     expect(tokens).toEqual(['big-deal', 'ok']);
   });
 
+  it('.whitespace should treat null as empty string', function() {
+    var tokens = tokenizers.whitespace(null);
+    expect(tokens).toEqual([]);
+  });
+
+  it('.whitespace should treat undefined as empty string', function() {
+    var tokens = tokenizers.whitespace(undefined);
+    expect(tokens).toEqual([]);
+  });
+
   it('.nonword should tokenize on non-word characters', function() {
     var tokens = tokenizers.nonword('big-deal ok');
     expect(tokens).toEqual(['big', 'deal', 'ok']);
   });
 
+  it('.nonword should treat null as empty string', function() {
+    var tokens = tokenizers.nonword(null);
+    expect(tokens).toEqual([]);
+  });
+
+  it('.nonword should treat undefined as empty string', function() {
+    var tokens = tokenizers.nonword(undefined);
+    expect(tokens).toEqual([]);
+  });
+
   it('.obj.whitespace should tokenize on whitespace', function() {
     var t = tokenizers.obj.whitespace('val');
     var tokens = t({ val: 'big-deal ok' });
 
     expect(tokens).toEqual(['big-deal', 'ok']);
   });
 
+  it('.obj.whitespace should accept multiple properties', function() {
+    var t = tokenizers.obj.whitespace('one', 'two');
+    var tokens = t({ one: 'big-deal ok', two: 'buzz' });
+
+    expect(tokens).toEqual(['big-deal', 'ok', 'buzz']);
+  });
+
   it('.obj.nonword should tokenize on non-word characters', function() {
     var t = tokenizers.obj.nonword('val');
     var tokens = t({ val: 'big-deal ok' });
 
     expect(tokens).toEqual(['big', 'deal', 'ok']);
   });
+
+  it('.obj.nonword should accept multiple properties', function() {
+    var t = tokenizers.obj.nonword('one', 'two');
+    var tokens = t({ one: 'big-deal ok', two: 'buzz' });
+
+    expect(tokens).toEqual(['big', 'deal', 'ok', 'buzz']);
+  });
 });