Merge pull request #137 from phereford/master

Remove HTML style tags in preprocessing
JuliaText · Apr 1, 2019 · c8ae7a2 · c8ae7a2
2 parents 53799b2 + bbf872e
commit c8ae7a2
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 0 deletions.
diff --git a/src/preprocessing.jl b/src/preprocessing.jl
@@ -124,10 +124,12 @@ end
 #
 ##############################################################################
 const script_tags = Regex("<script\\b[^>]*>([\\s\\S]*?)</script>")
+const style_tags = Regex("<style\\b[^>]*>([\\s\\S]*?)</style>")
 const html_tags = Regex("<[^>]*>")
 
 function remove_html_tags(s::AbstractString)
     s = remove_patterns(s, script_tags)
+    s = remove_patterns(s, style_tags)
     remove_patterns(s, html_tags)
 end
 

diff --git a/test/preprocessing.jl b/test/preprocessing.jl
@@ -82,6 +82,26 @@
     remove_html_tags!(d)
     @test "Hello world" == strip(d.text)
 
+    style_html_doc = StringDocument(
+      """
+        <html>
+            <head>
+                <script language=\"javascript\"> x = 20; </script>
+            </head>
+            <body>
+                <style>
+                  .fake-style {
+                    color: #00ff00;
+                  }
+                </style>
+                <h1>Hello</h1><a href=\"world\">world</a>
+            </body>
+        </html>
+      """
+     )
+    remove_html_tags!(style_html_doc)
+    @test "Hello world" == strip(style_html_doc.text)
+
     #Test #62
     remove_corrupt_utf8("abc") == "abc"
     remove_corrupt_utf8(String([0x43, 0xf0])) == "C "