From 829d91f29ba45713e89e15a143455e4755099e83 Mon Sep 17 00:00:00 2001
From: Gusted <postmaster@gusted.xyz>
Date: Tue, 2 Apr 2024 00:35:19 +0200
Subject: [PATCH] Typescript: Several fixes

- Allow comments to be inside of tags.
- Fixes the lexing of `<!--` comments.
- Allow tag names to include `-`.
- Modify module rule to be more strict which is to match `declare module
"Module name"`.
- Added tests.
- Resolves https://github.com/alecthomas/chroma/issues/940
- Resolves https://github.com/alecthomas/chroma/issues/939
- Resolves https://github.com/alecthomas/chroma/issues/801
---
 lexers/embedded/typescript.xml      | 20 ++++++++++++--
 lexers/testdata/tsx.actual          | 15 +++++++++++
 lexers/testdata/tsx.expected        | 41 +++++++++++++++++++++++++++++
 lexers/testdata/typescript.actual   |  5 ++++
 lexers/testdata/typescript.expected | 37 ++++++++++++++++++++++++++
 5 files changed, 116 insertions(+), 2 deletions(-)
 create mode 100644 lexers/testdata/typescript.actual
 create mode 100644 lexers/testdata/typescript.expected

diff --git a/lexers/embedded/typescript.xml b/lexers/embedded/typescript.xml
index d49241e6f..9828e92f5 100644
--- a/lexers/embedded/typescript.xml
+++ b/lexers/embedded/typescript.xml
@@ -51,10 +51,13 @@
       </rule>
     </state>
     <state name="tag">
+      <rule>
+        <include state="commentsandwhitespace"/>
+      </rule>
       <rule pattern="\s+">
         <token type="Text"/>
       </rule>
-      <rule pattern="([\w]+\s*)(=)(\s*)">
+      <rule pattern="([\w-]+\s*)(=)(\s*)">
         <bygroups>
           <token type="NameAttribute"/>
           <token type="Operator"/>
@@ -77,12 +80,25 @@
         <pop depth="1"/>
       </rule>
     </state>
+   <state name="comment">
+      <rule pattern="[^-]+">
+        <token type="Comment"/>
+      </rule>
+      <rule pattern="--&gt;">
+        <token type="Comment"/>
+        <pop depth="1"/>
+      </rule>
+      <rule pattern="-">
+        <token type="Comment"/>
+      </rule>
+    </state>
     <state name="commentsandwhitespace">
       <rule pattern="\s+">
         <token type="Text"/>
       </rule>
       <rule pattern="&lt;!--">
         <token type="Comment"/>
+        <push state="comment"/>
       </rule>
       <rule pattern="//.*?\n">
         <token type="CommentSingle"/>
@@ -200,7 +216,7 @@
       <rule pattern="(Array|Boolean|Date|Error|Function|Math|Number|Object|Packages|RegExp|String|decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|isNaN|parseFloat|parseInt|document|this|window)\b">
         <token type="NameBuiltin"/>
       </rule>
-      <rule pattern="\b(module)(\s*)(\s*[\w?.$][\w?.$]*)(\s*)">
+      <rule pattern="\b(module)(\s+)(&quot;[\w\./@]+&quot;)(\s+)">
         <bygroups>
           <token type="KeywordReserved"/>
           <token type="Text"/>
diff --git a/lexers/testdata/tsx.actual b/lexers/testdata/tsx.actual
index 2762b50da..78cce86e2 100644
--- a/lexers/testdata/tsx.actual
+++ b/lexers/testdata/tsx.actual
@@ -10,4 +10,19 @@ ReactDOM.render(
     </React.StrictMode>
   </>,
   document.getElementById('root'),
+)
+
+ReactDOM.render(
+  <p
+  // This is a inline comment.
+  /*
+    This is a
+    Multi line comment
+  */
+  <!-- Another multiline
+  comment !-->
+  data-test-id="outro" disabled>
+	Some text here.
+  </p>,
+  document.getElementById('root'),
 )
\ No newline at end of file
diff --git a/lexers/testdata/tsx.expected b/lexers/testdata/tsx.expected
index c7ddc3213..fa1088092 100644
--- a/lexers/testdata/tsx.expected
+++ b/lexers/testdata/tsx.expected
@@ -60,5 +60,46 @@
   {"type":"LiteralStringSingle","value":"'root'"},
   {"type":"Punctuation","value":"),"},
   {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameOther","value":"ReactDOM"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"render"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"\u003c"},
+  {"type":"NameTag","value":"p"},
+  {"type":"Text","value":"\n  "},
+  {"type":"CommentSingle","value":"// This is a inline comment.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentMultiline","value":"/*\n    This is a\n    Multi line comment\n  */"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Comment","value":"\u003c!-- Another multiline\n  comment !--\u003e"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"data-test-id"},
+  {"type":"Operator","value":"="},
+  {"type":"LiteralString","value":"\"outro\""},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"disabled"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"\n\t"},
+  {"type":"NameOther","value":"Some"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"text"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"here"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"\u003c/"},
+  {"type":"NameTag","value":"p"},
+  {"type":"Punctuation","value":"\u003e,"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameBuiltin","value":"document"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"getElementById"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralStringSingle","value":"'root'"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":"\n"},
   {"type":"Punctuation","value":")"}
 ]
diff --git a/lexers/testdata/typescript.actual b/lexers/testdata/typescript.actual
new file mode 100644
index 000000000..5c76fcdc7
--- /dev/null
+++ b/lexers/testdata/typescript.actual
@@ -0,0 +1,5 @@
+const moduleName = modules.map();
+
+declare module "fs" {}
+declare module "@custom/plugin" {}
+declare module "../../compiler/types" {}
diff --git a/lexers/testdata/typescript.expected b/lexers/testdata/typescript.expected
new file mode 100644
index 000000000..1fbff172f
--- /dev/null
+++ b/lexers/testdata/typescript.expected
@@ -0,0 +1,37 @@
+[
+  {"type":"KeywordReserved","value":"const"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"moduleName"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"modules"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"map"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordReserved","value":"declare"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordReserved","value":"module"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"\"fs\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{}"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordReserved","value":"declare"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordReserved","value":"module"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"\"@custom/plugin\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{}"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordReserved","value":"declare"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordReserved","value":"module"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"\"../../compiler/types\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{}"},
+  {"type":"Text","value":"\n"}
+]