Merge pull request #1 from RunDevelopment/tremor-improvements

Improved Tremor tokenization
PrismJS · Sep 21, 2021 · 5bb9ac9 · 5bb9ac9
2 parents 1b3316a + dba2ff6
commit 5bb9ac9
Show file tree

Hide file tree

Showing 10 changed files with 281 additions and 207 deletions.
diff --git a/components/prism-tremor.js b/components/prism-tremor.js
@@ -1,20 +1,55 @@
-// NOTE: Tremor supports recursive nested interpolation in strings
-// and heredocs but we use a partial interpolation method here. The
-// string language rule pattern supports both single and triple double-quote
-// delimited strings.
-//
-
-Prism.languages.tremor = {
-	'comment': {
-		pattern: /(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:--|\/\/|#).*)/,
-		lookbehind: true
-	},
-	'string': {
-		pattern: /"""(?:[^"\\\r\n#]|\\(?:\r\n|[\s\S])|#(?!\{)|#\{(?:[^"{}]|\{[^{}]*\}|"(?:[^"\\\r\n]|\\(?:\r\n|[\s\S]))")*\})*"""|"(?:[^"\\\r\n#]|\\(?:\r\n|[\s\S])|#(?!\{)|#\{(?:[^"{}]|\{[^{}]*\}|"(?:[^"\\\r\n]|\\(?:\r\n|[\s\S]))")*\})*"/,
+(function (Prism) {
+
+	Prism.languages.tremor = {
+		'comment': {
+			pattern: /(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:--|\/\/|#).*)/,
+			lookbehind: true
+		},
+		'interpolated-string': null, // see below
+		'extractor': {
+			pattern: /\b[a-z_]\w*\|(?:[^\r\n\\|]|\\(?:\r\n|[\s\S]))*\|/i,
+			greedy: true,
+			inside: {
+				'function': /^\w+/,
+				'regex': /\|[\s\S]+/,
+			}
+		},
+		'identifier': {
+			pattern: /`[^`]*`/,
+			greedy: true
+		},
+
+		'function': /\b[a-z_]\w*(?=\s*(?:::\s*<|\())\b/,
+
+		'keyword': /\b(?:event|state|select|create|define|deploy|operator|script|connector|pipeline|flow|config|links|connect|to|from|into|with|group|by|args|window|stream|tumbling|sliding|where|having|set|each|emit|drop|const|let|for|match|of|case|when|default|end|patch|insert|update|erase|move|copy|merge|fn|intrinsic|recur|use|as|mod)\b/,
+		'boolean': /\b(?:true|false|null)\b/i,
+
+		'number': /\b(?:0b[0-1_]*|0x[0-9a-fA-F_]*|\d[0-9_]*(?:\.\d[0-9_]*)?(?:[Ee][+-]?[0-9_]+)?)\b/,
+
+		'pattern-punctuation': {
+			pattern: /%(?=[({[])/,
+			alias: 'punctuation'
+		},
+		'operator': /[-+*\/%~!^]=?|=[=>]?|&[&=]?|\|[|=]?|<<?=?|>>?>?=?|(?:not|and|or|xor|present|absent)\b/,
+		'punctuation': /::|[;\[\]()\{\},.:]/,
+	};
+
+	var interpolationPattern = /#\{(?:[^"{}]|\{[^{}]*\}|"(?:[^"\\\r\n]|\\(?:\r\n|[\s\S]))*")*\}/.source;
+
+	Prism.languages.tremor['interpolated-string'] = {
+		pattern: RegExp(
+			/(^|[^\\])/.source +
+			'(?:' +
+			'"""(?:' + /[^"\\#]|\\[\s\S]|"(?!"")|#(?!\{)/.source + '|' + interpolationPattern + ')*"""' +
+			'|' +
+			'"(?:' + /[^"\\\r\n#]|\\(?:\r\n|[\s\S])|#(?!\{)/.source + '|' + interpolationPattern + ')*"' +
+			')'
+		),
+		lookbehind: true,
 		greedy: true,
 		inside: {
 			'interpolation': {
-				pattern: /#\{(?:[^"{}]|\{[^{}]*\}|"(?:[^"\\\r\n]|\\(?:\r\n|[\s\S]))")*\}/,
+				pattern: RegExp(interpolationPattern),
 				inside: {
 					'punctuation': /^#\{|\}$/,
 					'expression': {
@@ -25,21 +60,9 @@ Prism.languages.tremor = {
 			},
 			'string': /[\s\S]+/
 		}
-	},
-	'function': /\b[a-z_]\w*(?=\s*(?:::\s*<|\())\b/,
-	'keyword': /\b(?:event|state|select|create|define|deploy|operator|script|connector|pipeline|flow|config|links|connect|to|from|into|with|group|by|args|window|stream|tumbling|sliding|where|having|set|each|emit|drop|const|let|for|match|of|case|when|default|end|patch|insert|update|erase|move|copy|merge|fn|intrinsic|recur|use|as|mod)\b/,
-	'boolean': /\b(?:true|false|null)\b/i,
-	'number': /\b(?:0b[0-1_]*|0x[0-9a-fA-F_]*|\d[0-9_]*(?:\.\d[0-9_]*)?(?:[Ee][+-]?[0-9_]+)?)\b/,
-	'operator': /[-+*\/%!^]=?|=[=>]?|&[&=]?|\|[|=]?|<<?=?|>>?>?=?|(?:not|and|or|xor|present|absent)\b/,
-	'punctuation': /[;\[\]()\{\},.%#]/,
-	'variable': {
-		pattern: /(?:[a-z](?:[a-z_\d]+\s*)?::|`[^`]+`::)*(?:[a-z][a-z_\d]*|`[^`]+`)/,
-		greedy: true,
-		inside: {
-			'punctuation': /::/
-		}
-	},
-};
+	};
+
+	Prism.languages.troy = Prism.languages['tremor'];
+	Prism.languages.trickle = Prism.languages['tremor'];
 
-Prism.languages.troy = Prism.languages['tremor'];
-Prism.languages.trickle = Prism.languages['tremor'];
+}(Prism));
diff --git a/components/prism-tremor.min.js b/components/prism-tremor.min.js
diff --git a/tests/languages/tremor/extractor_feature.test b/tests/languages/tremor/extractor_feature.test
@@ -0,0 +1,15 @@
+re|^(?P<kind>bat.*)$|
+datetime|%Y-%m-%d %H:%M|
+
+----------------------------------------------------
+
+[
+	["extractor", [
+		["function", "re"],
+		["regex", "|^(?P<kind>bat.*)$|"]
+	]],
+	["extractor", [
+		["function", "datetime"],
+		["regex", "|%Y-%m-%d %H:%M|"]
+	]]
+]
diff --git a/tests/languages/tremor/identifier_feature.test b/tests/languages/tremor/identifier_feature.test
@@ -0,0 +1,12 @@
+foo
+foo_bar_42
+`foo`
+`bar`
+
+----------------------------------------------------
+
+[
+	"foo\r\nfoo_bar_42\r\n",
+	["identifier", "`foo`"],
+	["identifier", "`bar`"]
+]
diff --git a/tests/languages/tremor/keyword_escape_feature.test b/tests/languages/tremor/keyword_escape_feature.test
@@ -55,59 +55,59 @@
 ----------------------------------------------------
 
 [
-	["variable", ["`args`"]],
-	["variable", ["`as`"]],
-	["variable", ["`by`"]],
-	["variable", ["`case`"]],
-	["variable", ["`config`"]],
-	["variable", ["`connect`"]],
-	["variable", ["`connector`"]],
-	["variable", ["`const`"]],
-	["variable", ["`copy`"]],
-	["variable", ["`create`"]],
-	["variable", ["`default`"]],
-	["variable", ["`define`"]],
-	["variable", ["`deploy`"]],
-	["variable", ["`drop`"]],
-	["variable", ["`each`"]],
-	["variable", ["`emit`"]],
-	["variable", ["`end`"]],
-	["variable", ["`erase`"]],
-	["variable", ["`event`"]],
-	["variable", ["`flow`"]],
-	["variable", ["`fn`"]],
-	["variable", ["`for`"]],
-	["variable", ["`from`"]],
-	["variable", ["`group`"]],
-	["variable", ["`having`"]],
-	["variable", ["`insert`"]],
-	["variable", ["`into`"]],
-	["variable", ["`intrinsic`"]],
-	["variable", ["`let`"]],
-	["variable", ["`links`"]],
-	["variable", ["`match`"]],
-	["variable", ["`merge`"]],
-	["variable", ["`mod`"]],
-	["variable", ["`move`"]],
-	["variable", ["`of`"]],
-	["variable", ["`operator`"]],
-	["variable", ["`patch`"]],
-	["variable", ["`pipeline`"]],
-	["variable", ["`recur`"]],
-	["variable", ["`script`"]],
-	["variable", ["`select`"]],
-	["variable", ["`set`"]],
-	["variable", ["`sliding`"]],
-	["variable", ["`state`"]],
-	["variable", ["`stream`"]],
-	["variable", ["`to`"]],
-	["variable", ["`tumbling`"]],
-	["variable", ["`update`"]],
-	["variable", ["`use`"]],
-	["variable", ["`when`"]],
-	["variable", ["`where`"]],
-	["variable", ["`window`"]],
-	["variable", ["`with`"]]
+	["identifier", "`args`"],
+	["identifier", "`as`"],
+	["identifier", "`by`"],
+	["identifier", "`case`"],
+	["identifier", "`config`"],
+	["identifier", "`connect`"],
+	["identifier", "`connector`"],
+	["identifier", "`const`"],
+	["identifier", "`copy`"],
+	["identifier", "`create`"],
+	["identifier", "`default`"],
+	["identifier", "`define`"],
+	["identifier", "`deploy`"],
+	["identifier", "`drop`"],
+	["identifier", "`each`"],
+	["identifier", "`emit`"],
+	["identifier", "`end`"],
+	["identifier", "`erase`"],
+	["identifier", "`event`"],
+	["identifier", "`flow`"],
+	["identifier", "`fn`"],
+	["identifier", "`for`"],
+	["identifier", "`from`"],
+	["identifier", "`group`"],
+	["identifier", "`having`"],
+	["identifier", "`insert`"],
+	["identifier", "`into`"],
+	["identifier", "`intrinsic`"],
+	["identifier", "`let`"],
+	["identifier", "`links`"],
+	["identifier", "`match`"],
+	["identifier", "`merge`"],
+	["identifier", "`mod`"],
+	["identifier", "`move`"],
+	["identifier", "`of`"],
+	["identifier", "`operator`"],
+	["identifier", "`patch`"],
+	["identifier", "`pipeline`"],
+	["identifier", "`recur`"],
+	["identifier", "`script`"],
+	["identifier", "`select`"],
+	["identifier", "`set`"],
+	["identifier", "`sliding`"],
+	["identifier", "`state`"],
+	["identifier", "`stream`"],
+	["identifier", "`to`"],
+	["identifier", "`tumbling`"],
+	["identifier", "`update`"],
+	["identifier", "`use`"],
+	["identifier", "`when`"],
+	["identifier", "`where`"],
+	["identifier", "`window`"],
+	["identifier", "`with`"]
 ]
 
 ----------------------------------------------------

diff --git a/tests/languages/tremor/modularity_feature.test b/tests/languages/tremor/modularity_feature.test
@@ -8,34 +8,27 @@ foo::bar::baz
 ----------------------------------------------------
 
 [
-	["variable", ["foo"]],
-	["variable", [
-		"foo",
-		["punctuation", "::"],
-		"bar"
-	]],
-	["variable", [
-		"foo",
-		["punctuation", "::"],
-		"bar",
-		["punctuation", "::"],
-		"baz"
-	]],
-	["variable", [
-		"`foo`",
-		["punctuation", "::"],
-		"bar",
-		["punctuation", "::"],
-		"`baz`"
-	]],
-	["variable", ["`foo`"]],
-	["variable", [
-		"`foo`",
-		["punctuation", "::"],
-		"`bar`",
-		["punctuation", "::"],
-		"`baz`"
-	]]
+	"foo\r\nfoo",
+	["punctuation", "::"],
+	"bar\r\nfoo",
+	["punctuation", "::"],
+	"bar",
+	["punctuation", "::"],
+	"baz\r\n",
+
+	["identifier", "`foo`"],
+	["punctuation", "::"],
+	"bar",
+	["punctuation", "::"],
+	["identifier", "`baz`"],
+
+	["identifier", "`foo`"],
+
+	["identifier", "`foo`"],
+	["punctuation", "::"],
+	["identifier", "`bar`"],
+	["punctuation", "::"],
+	["identifier", "`baz`"]
 ]
 
 ----------------------------------------------------