remove unused functions (#194)

JuliaLang · Feb 4, 2022 · 860bdc6 · 860bdc6
1 parent bed7f32
commit 860bdc6
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 141 deletions.
diff --git a/src/_precompile.jl b/src/_precompile.jl
@@ -15,7 +15,6 @@ function _precompile_()
     precompile(Tokenize.Tokens.untokenize, (Array{Tokenize.Tokens.Token, 1},))
     precompile(Tokenize.Tokens.untokenize, (Tokenize.Lexers.Lexer{GenericIOBuffer{Array{UInt8, 1}},Tokenize.Tokens.Token},))
 
-    precompile(Tokenize.Lexers.is_cat_id_start, (Char, Int32,))
     precompile(Tokenize.Lexers.is_identifier_char, (Char,))
     precompile(Tokenize.Lexers.is_identifier_start_char, (Char,))
     precompile(Tokenize.Lexers.peekchar, (GenericIOBuffer{Array{UInt8, 1}},))
@@ -92,5 +91,5 @@ function _precompile_()
     precompile(Tokenize.Lexers.accept, (Tokenize.Lexers.Lexer{GenericIOBuffer{Array{UInt8, 1}},Tokenize.Tokens.RawToken}, Char,))
 
 
-    precompile(Tokenize.Lexers.readchar, (GenericIOBuffer{Array{UInt8, 1}},))    
+    precompile(Tokenize.Lexers.readchar, (GenericIOBuffer{Array{UInt8, 1}},))
 end
diff --git a/src/lexer.jl b/src/lexer.jl
@@ -323,6 +323,10 @@ function emit_error(l::Lexer, err::TokenError = Tokens.UNKNOWN)
     return emit(l, Tokens.ERROR, err)
 end
 
+function is_identifier_start_char(c::Char)
+    c == EOF_CHAR && return false
+    return Base.is_id_start_char(c)
+end
 
 """
     next_token(l::Lexer)
@@ -1021,6 +1025,10 @@ function lex_cmd(l::Lexer, doemit=true)
     end
 end
 
+function is_identifier_char(c::Char)
+    c == EOF_CHAR && return false
+    return Base.is_id_char(c)
+end
 function lex_identifier(l::Lexer{IO_t,T}, c) where {IO_t,T}
     if T == Token
         readon(l)

diff --git a/src/utilities.jl b/src/utilities.jl
@@ -28,151 +28,12 @@ The JuliaParser.jl package is licensed under the MIT "Expat" License:
 
 import Base.Unicode
 
-
-@inline function utf8_trailing(i)
-    if i < 193
-        return 0
-    elseif i < 225
-        return 1
-    elseif i < 241
-        return 2
-    elseif i < 249
-        return 3
-    elseif i < 253
-        return 4
-    else
-        return 5
-    end
-end
-
-const utf8_offset = [0x00000000
-                    0x00003080
-                    0x000e2080
-                    0x03c82080
-                    0xfa082080
-                    0x82082080]
-# const EOF_CHAR = convert(Char,typemax(UInt32))
 const EOF_CHAR = typemax(Char)
 
-
-function is_cat_id_start(ch::Char, cat::Integer)
-    c = UInt32(ch)
-    return (cat == Unicode.UTF8PROC_CATEGORY_LU || cat == Unicode.UTF8PROC_CATEGORY_LL ||
-            cat == Unicode.UTF8PROC_CATEGORY_LT || cat == Unicode.UTF8PROC_CATEGORY_LM ||
-            cat == Unicode.UTF8PROC_CATEGORY_LO || cat == Unicode.UTF8PROC_CATEGORY_NL ||
-            cat == Unicode.UTF8PROC_CATEGORY_SC ||  # allow currency symbols
-            cat == Unicode.UTF8PROC_CATEGORY_SO ||  # other symbols
-
-            # math symbol (category Sm) whitelist
-            (c >= 0x2140 && c <= 0x2a1c &&
-             ((c >= 0x2140 && c <= 0x2144) || # ⅀, ⅁, ⅂, ⅃, ⅄
-              c == 0x223f || c == 0x22be || c == 0x22bf || # ∿, ⊾, ⊿
-              c == 0x22a4 || c == 0x22a5 || # ⊤ ⊥
-
-              (c >= 0x2202 && c <= 0x2233 &&
-               (c == 0x2202 || c == 0x2205 || c == 0x2206 || # ∂, ∅, ∆
-                c == 0x2207 || c == 0x220e || c == 0x220f || # ∇, ∎, ∏
-                c == 0x2210 || c == 0x2211 || # ∐, ∑
-                c == 0x221e || c == 0x221f || # ∞, ∟
-                c >= 0x222b)) || # ∫, ∬, ∭, ∮, ∯, ∰, ∱, ∲, ∳
-
-              (c >= 0x22c0 && c <= 0x22c3) ||  # N-ary big ops: ⋀, ⋁, ⋂, ⋃
-              (c >= 0x25F8 && c <= 0x25ff) ||  # ◸, ◹, ◺, ◻, ◼, ◽, ◾, ◿
-
-              (c >= 0x266f &&
-               (c == 0x266f || c == 0x27d8 || c == 0x27d9 || # ♯, ⟘, ⟙
-                (c >= 0x27c0 && c <= 0x27c1) ||  # ⟀, ⟁
-                (c >= 0x29b0 && c <= 0x29b4) ||  # ⦰, ⦱, ⦲, ⦳, ⦴
-                (c >= 0x2a00 && c <= 0x2a06) ||  # ⨀, ⨁, ⨂, ⨃, ⨄, ⨅, ⨆
-                (c >= 0x2a09 && c <= 0x2a16) ||  # ⨉, ⨊, ⨋, ⨌, ⨍, ⨎, ⨏, ⨐, ⨑, ⨒,
-                                                 # ⨓, ⨔, ⨕, ⨖
-                c == 0x2a1b || c == 0x2a1c)))) || # ⨛, ⨜
-
-            (c >= 0x1d6c1 && # variants of \nabla and \partial
-             (c == 0x1d6c1 || c == 0x1d6db ||
-              c == 0x1d6fb || c == 0x1d715 ||
-              c == 0x1d735 || c == 0x1d74f ||
-              c == 0x1d76f || c == 0x1d789 ||
-              c == 0x1d7a9 || c == 0x1d7c3)) ||
-
-            # super- and subscript +-=()
-            (c >= 0x207a && c <= 0x207e) ||
-            (c >= 0x208a && c <= 0x208e) ||
-
-            # angle symbols
-            (c >= 0x2220 && c <= 0x2222) || # ∠, ∡, ∢
-            (c >= 0x299b && c <= 0x29af) || # ⦛, ⦜, ⦝, ⦞, ⦟, ⦠, ⦡, ⦢, ⦣, ⦤, ⦥,
-                                            # ⦦, ⦧, ⦨, ⦩, ⦪, ⦫, ⦬, ⦭, ⦮, ⦯
-            # Other_ID_Start
-            c == 0x2118 || c == 0x212E || # ℘, ℮
-            (c >= 0x309B && c <= 0x309C)) # katakana-hiragana sound marks
-end
-
-function is_identifier_char(c::Char)
-    c == EOF_CHAR && return false
-    return Base.is_id_char(c)
-end
-
-function is_identifier_start_char(c::Char)
-    c == EOF_CHAR && return false
-    return Base.is_id_start_char(c)
-end
-
-
-function peekchar(io::Base.GenericIOBuffer)
-    if !io.readable || io.ptr > io.size
-        return EOF_CHAR
-    end
-    ch, _ = readutf(io)
-    return ch
-end
-
-function readutf(io, offset = 0)
-    ch = convert(UInt8, io.data[io.ptr + offset])
-    if ch < 0x80
-        return convert(Char, ch), 0
-    end
-    trailing = utf8_trailing(ch + 1)
-    c::UInt32 = 0
-    for j = 1:trailing
-        c += ch
-        c <<= 6
-        ch = convert(UInt8, io.data[io.ptr + j + offset])
-    end
-    c += ch
-    c -= utf8_offset[trailing + 1]
-    return convert(Char, c), trailing
-end
-
-function dpeekchar(io::IOBuffer)
-    if !io.readable || io.ptr > io.size
-        return EOF_CHAR, EOF_CHAR
-    end
-    ch1, trailing = readutf(io)
-    offset = trailing + 1
-
-    if io.ptr + offset > io.size
-        return ch1, EOF_CHAR
-    end
-    ch2, _ = readutf(io, offset)
-
-    return ch1, ch2
-end
-
-# this implementation is copied from Base
-peekchar(s::IOStream) = begin
-    _CHTMP = Ref{Char}()
-    if ccall(:ios_peekutf8, Int32, (Ptr{Nothing}, Ptr{Char}), s, _CHTMP) < 0
-        return EOF_CHAR
-    end
-    return _CHTMP[]
-end
-
 eof(io::IO) = Base.eof(io)
 eof(c::Char) = c === EOF_CHAR
 
 readchar(io::IO) = eof(io) ? EOF_CHAR : read(io, Char)
-takechar(io::IO) = (readchar(io); io)
 
 # Checks whether a Char is an operator, which can not be juxtaposed with another
 # Char to be an operator (i.e <=), and can be prefixed by a dot (.)