From c0011c46e19e65e565db9ef3a731bb140a8a3a43 Mon Sep 17 00:00:00 2001 From: KristofferC Date: Wed, 17 Aug 2022 13:19:08 +0200 Subject: [PATCH] move to JuliaSyntax.jl over Tokenize.jl --- Project.toml | 4 +- docs/Manifest.toml | 78 +++++++++++++++++++++----------- docs/src/internals/passes.md | 73 +++++++++++++++--------------- src/MarkdownHighlighter.jl | 6 +-- src/OhMyREPL.jl | 2 +- src/passes/BracketHighlighter.jl | 19 ++++---- src/passes/RainbowBrackets.jl | 27 +++++------ src/passes/SyntaxHighlighter.jl | 46 ++++++++++--------- src/precompile.jl | 4 ++ src/repl.jl | 10 ++-- src/repl_pass.jl | 30 ++++++------ test/test_bracketmatch.jl | 1 - test/test_custompass.jl | 9 ++-- test/test_highlighter.jl | 3 +- 14 files changed, 175 insertions(+), 137 deletions(-) create mode 100644 src/precompile.jl diff --git a/Project.toml b/Project.toml index b5d25ff..cabde2f 100644 --- a/Project.toml +++ b/Project.toml @@ -5,15 +5,15 @@ version = "0.5.12" [deps] Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" JLFzf = "1019f520-868f-41f5-a6de-eb00f4b6a39c" +JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" -Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624" [compat] Crayons = "1, 2, 3, 4" -Tokenize = "0.5" +JuliaSyntax = "0.3.0" JLFzf = "^0.1.1" julia = "1.0" diff --git a/docs/Manifest.toml b/docs/Manifest.toml index c358061..2c8e258 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -2,6 +2,7 @@ [[ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[Artifacts]] uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" @@ -10,9 +11,9 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" [[Crayons]] -git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d" +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.0.4" +version = "4.1.1" [[Dates]] deps = ["Printf"] @@ -20,9 +21,9 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" [[DocStringExtensions]] deps = ["LibGit2"] -git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f" +git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b" uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" -version = "0.8.5" +version = "0.8.6" [[Documenter]] deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"] @@ -31,8 +32,12 @@ uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.27.3" [[Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" [[IOCapture]] deps = ["Logging", "Random"] @@ -46,29 +51,36 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" [[JLFzf]] deps = ["Pipe", "REPL", "Random", "fzf_jll"] -git-tree-sha1 = "fa300b5b580549186c2b1558a199ac5615cc8928" +git-tree-sha1 = "f377670cda23b6b7c1c0b3893e37451c5c1a2185" uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c" -version = "0.1.3" +version = "0.1.5" [[JLLWrappers]] deps = ["Preferences"] -git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e" +git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.3.0" +version = "1.4.1" [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" +git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.1" +version = "0.21.3" + +[[JuliaSyntax]] +git-tree-sha1 = "db2bdeda30e452485863799be4515f6305431a46" +uuid = "70703baa-626e-46a2-a12c-08ffd08c73b4" +version = "0.3.2" [[LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" [[LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -77,6 +89,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -91,27 +104,30 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" [[MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" [[NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[OhMyREPL]] -deps = ["Crayons", "JLFzf", "Markdown", "Pkg", "Printf", "REPL", "Tokenize"] +deps = ["Crayons", "JLFzf", "JuliaSyntax", "Markdown", "Pkg", "Printf", "REPL"] path = ".." uuid = "5fb14364-9ced-5910-84b2-373655c76a03" -version = "0.5.9" +version = "0.5.12" [[Parsers]] -deps = ["Dates"] -git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc" +deps = ["Dates", "SnoopPrecompile"] +git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "1.1.0" +version = "2.5.8" [[Pipe]] git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d" @@ -119,14 +135,15 @@ uuid = "b98c9c47-44ae-5843-9183-064241ee97a0" version = "1.3.0" [[Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.0" [[Preferences]] deps = ["TOML"] -git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a" +git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.2.2" +version = "1.3.0" [[Printf]] deps = ["Unicode"] @@ -137,35 +154,39 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" [[Random]] -deps = ["Serialization"] +deps = ["SHA", "Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +[[SnoopPrecompile]] +deps = ["Preferences"] +git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" +uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" +version = "1.0.3" + [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" [[TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" [[Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" [[Test]] deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -[[Tokenize]] -git-tree-sha1 = "37018506dc445ad7db288442fbb846105f26c43f" -uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624" -version = "0.5.17" - [[UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" @@ -176,17 +197,20 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" [[fzf_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "b2d7b205b9518f8a5ebcd05764c5536dbdf460bb" +git-tree-sha1 = "868e669ccb12ba16eaf50cb2957ee2ff61261c56" uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09" -version = "0.24.4+0" +version = "0.29.0+0" [[nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" [[p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/docs/src/internals/passes.md b/docs/src/internals/passes.md index 2271889..7694603 100644 --- a/docs/src/internals/passes.md +++ b/docs/src/internals/passes.md @@ -7,10 +7,10 @@ All the passes are registered in a global pass handler. To show all the passes u ```jl julia> OhMyREPL.showpasses() ---------------------------------- - # Pass name Enabled + # Pass name Enabled ---------------------------------- - 1 BracketHighlighter true - 2 SyntaxHighlighter true + 1 BracketHighlighter true + 2 SyntaxHighlighter true ---------------------------------- ``` @@ -34,24 +34,24 @@ julia> using Tokenize julia> tokens = collect(Tokenize.tokenize(str)) 20-element Array{Tokenize.Tokens.Token,1}: 1,1-1,8: KEYWORD "function" - 1,9-1,9: WHITESPACE " " - 1,10-1,10: IDENTIFIER "f" - 1,11-1,11: LPAREN "(" - 1,12-1,12: IDENTIFIER "x" - 1,13-1,14: OP "::" + 1,9-1,9: WHITESPACE " " + 1,10-1,10: IDENTIFIER "f" + 1,11-1,11: LPAREN "(" + 1,12-1,12: IDENTIFIER "x" + 1,13-1,14: OP "::" 1,15-1,21: IDENTIFIER "Float64" - 1,22-1,22: RPAREN ")" - 1,23-1,23: WHITESPACE " " - 1,24-1,29: KEYWORD "return" - 1,30-1,30: WHITESPACE " " - 1,31-1,31: OP ":" - 1,32-1,32: IDENTIFIER "x" - 1,33-1,33: WHITESPACE " " - 1,34-1,34: OP "+" - 1,35-1,35: WHITESPACE " " - 1,36-1,38: CHAR "'a'" - 1,39-1,39: WHITESPACE " " - 1,40-1,42: KEYWORD "end" + 1,22-1,22: RPAREN ")" + 1,23-1,23: WHITESPACE " " + 1,24-1,29: KEYWORD "return" + 1,30-1,30: WHITESPACE " " + 1,31-1,31: OP ":" + 1,32-1,32: IDENTIFIER "x" + 1,33-1,33: WHITESPACE " " + 1,34-1,34: OP "+" + 1,35-1,35: WHITESPACE " " + 1,36-1,38: CHAR "'a'" + 1,39-1,39: WHITESPACE " " + 1,40-1,42: KEYWORD "end" 1,43-1,42: ENDMARKER "" ``` @@ -62,17 +62,17 @@ crayons = Vector{Crayon}(length(tokens)); fill!(crayons, Crayon()) # Crayon is a bits type so this is OK ``` -These two vectors are then sent to the syntax highlighter pass together with an integer that represent what character offset the cursor currently is located. The syntax highlighter does not use this information but the bracket highlighter does. +These two vectors and the source code are then sent to the syntax highlighter pass together with an integer that represent what character offset the cursor currently is located. The syntax highlighter does not use this information but the bracket highlighter does. ``` -OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0) +OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0, str) ``` Running this function has the effect of updating the `crayons` vector. If we print this vector we see that they have been updated: ![](ansitokens_after.png) -To print the original string with the updated vector of `Crayon`s we use the `OhMyREPL.untokenize_with_ANSI([io::IO], crayons, tokens)` function as: +To print the original string with the updated vector of `Crayon`s we use the `OhMyREPL.untokenize_with_ANSI([io::IO], crayons, tokens, str)` function as: ![](print_ansi.png) @@ -80,7 +80,7 @@ Each registered and enabled pass does this updating and the contributions from e ## Creating a pass -This section shows how to create a very pass that let the user define a `Crayon` for each typeassertion / declaration that happens to be a `Float64`. +This section shows how to create a pass that let the user define a `Crayon` for each typeassertion / declaration that happens to be a `Float64`. !!! info Please refer to the [Tokenize.jl API](https://github.com/KristofferC/Tokenize.jl#api) section and the [`Crayons.jl` documentation](https://github.com/KristofferC/Crayons.jl) while reading this section. @@ -89,7 +89,8 @@ We start off with a few imports and creating a new struct which will hold the se ```jl using Crayons -import Tokenize.Tokens: Token, untokenize, exactkind +import JuliaSyntax +import JuliaSyntax.Tokenize: Token, untokenize, kind using OhMyREPL mutable struct Float64Modifier @@ -104,11 +105,11 @@ We then use call overloading to define a function for the type. The function wil ```jl # The pass function, the cursor position is not used but it needs to be given an argument -function (float64modifier::Float64Modifier)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int) +function (float64modifier::Float64Modifier)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString) # Loop over all tokens and crayons for i in 1:length(crayons) - if untokenize(tokens[i]) == "Float64" - if i > 1 && exactkind(tokens[i-1]) == Tokenize.Tokens.DECLARATION + if untokenize(tokens[i], str) == "Float64" + if i > 1 && kind(tokens[i-1]) == JuliaSyntax.K"::" # Update the crayon crayons[i] = float64modifier.crayon end @@ -130,11 +131,11 @@ To register and start using the pass simply use `OhMyREPL.add_pass!(passname::St ```jl julia> OhMyREPL.add_pass!("Redify Float64", FLOAT64_MODIFIER) ---------------------------------- - # Pass name Enabled + # Pass name Enabled ---------------------------------- - 1 Redify Float64 true - 2 BracketHighlighter true - 3 SyntaxHighlighter true + 1 Redify Float64 true + 2 BracketHighlighter true + 3 SyntaxHighlighter true ---------------------------------- ``` @@ -151,11 +152,11 @@ The prescedence of a pass can be modified with the `OhMyREPL.prescedence!(pass:: ```jl julia> OhMyREPL.prescedence!("Redify Float64", 3) ---------------------------------- - # Pass name Enabled + # Pass name Enabled ---------------------------------- - 1 BracketHighlighter true - 2 SyntaxHighlighter true - 3 Redify Float64 true + 1 BracketHighlighter true + 2 SyntaxHighlighter true + 3 Redify Float64 true ---------------------------------- ``` diff --git a/src/MarkdownHighlighter.jl b/src/MarkdownHighlighter.jl index 1438ebc..44a9ea3 100644 --- a/src/MarkdownHighlighter.jl +++ b/src/MarkdownHighlighter.jl @@ -1,6 +1,6 @@ using Crayons -using Tokenize +using JuliaSyntax.Tokenize import Markdown import .OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS @@ -40,14 +40,14 @@ function Markdown.term(io::IO, md::Markdown.Code, columns) for (sourcecode, output) in zip(sourcecodes, outputs) tokens = collect(tokenize(sourcecode)) crayons = fill(Crayon(), length(tokens)) - SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0) + SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0, sourcecode) buff = IOBuffer() if lang == "jldoctest" || lang == "julia-repl" print(buff, Crayon(foreground = :red, bold = true), "julia> ", Crayon(reset = true)) end for (token, crayon) in zip(tokens, crayons) print(buff, crayon) - print(buff, untokenize(token)) + print(buff, untokenize(token, sourcecode)) print(buff, Crayon(reset = true)) end print(buff, output) diff --git a/src/OhMyREPL.jl b/src/OhMyREPL.jl index 894ab6a..f2306fe 100644 --- a/src/OhMyREPL.jl +++ b/src/OhMyREPL.jl @@ -5,7 +5,7 @@ bracket matching and other nifty features. """ module OhMyREPL -using Tokenize +using JuliaSyntax.Tokenize using Crayons if VERSION > v"1.3" import JLFzf diff --git a/src/passes/BracketHighlighter.jl b/src/passes/BracketHighlighter.jl index 10a7d4d..e8a1fe4 100644 --- a/src/passes/BracketHighlighter.jl +++ b/src/passes/BracketHighlighter.jl @@ -1,11 +1,12 @@ module BracketHighlighter -using Tokenize -using Tokenize.Tokens -import Tokenize.Tokens: Token, kind, startpos, endpos, untokenize +using JuliaSyntax.Tokenize +import .Tokenize: Token, kind, startpos, untokenize using Crayons +import JuliaSyntax.@K_str + import OhMyREPL: add_pass!, PASS_HANDLER mutable struct BracketHighlighterSettings @@ -15,8 +16,8 @@ end const BRACKETMATCHER_SETTINGS = BracketHighlighterSettings(Crayon(bold = :true, underline = :true)) -function (matcher::BracketHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int) - left_bracket_match, right_bracket_match, matched = bracket_match(tokens, cursorpos) +function (matcher::BracketHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString) + left_bracket_match, right_bracket_match, matched = bracket_match(tokens, cursorpos, str) !matched && return crayons[left_bracket_match] = matcher.crayon crayons[right_bracket_match] = matcher.crayon @@ -31,14 +32,14 @@ add_pass!(PASS_HANDLER, "BracketHighlighter", BRACKETMATCHER_SETTINGS, true) # Takes a string and a cursor index. # Returns index of left matching bracket, right matching bracket # and if there was a match at all as a 3 tuple. -const LEFT_DELIMS = [Tokens.LPAREN, Tokens.LSQUARE, Tokens.LBRACE] -const RIGHT_DELIMS = [Tokens.RPAREN, Tokens.RSQUARE, Tokens.RBRACE] -function bracket_match(tokens::Vector{Token}, cursoridx::Int) +const LEFT_DELIMS = [K"(", K"[", K"{"] +const RIGHT_DELIMS = [K")", K"]", K"}"] +function bracket_match(tokens::Vector{Token}, cursoridx::Int, str) enclosing_token_idx = -1 char_counter = 0 # find which token we are currently inside for (i, token) in enumerate(tokens) - char_counter += length(untokenize(token)) + char_counter += length(untokenize(token, str)) if char_counter >= cursoridx enclosing_token_idx = i break diff --git a/src/passes/RainbowBrackets.jl b/src/passes/RainbowBrackets.jl index 8d04f88..f6d3901 100644 --- a/src/passes/RainbowBrackets.jl +++ b/src/passes/RainbowBrackets.jl @@ -1,9 +1,10 @@ module RainbowBrackets using Crayons -using Tokenize -using Tokenize.Tokens -import Tokenize.Tokens: Token, kind, startpos, endpos, untokenize +import JuliaSyntax.@K_str +import JuliaSyntax.Tokenize +using .Tokenize +import .Tokenize: Token, kind, startpos, untokenize, Kind import OhMyREPL: add_pass!, PASS_HANDLER @@ -20,13 +21,13 @@ mutable struct RainbowBracketsSettings active::RainBowTokens end -function get_color(rainbow::RainbowBracketsSettings, k::Tokens.Kind, i) +function get_color(rainbow::RainbowBracketsSettings, k::Kind, i) @assert i >= 0 toks = rainbow.active i == 0 && return toks.error_token - if k == Tokens.LPAREN || k == Tokens.RPAREN + if k == K"(" || k == K")" return toks.parenthesis_tokens[mod1(i, length(toks.parenthesis_tokens))] - elseif k == Tokens.LSQUARE || k == Tokens.RSQUARE + elseif k == K"[" || k == K"]" return toks.brackets_tokens[mod1(i, length(toks.brackets_tokens))] else return toks.curly_tokens[mod1(i, length(toks.curly_tokens))] @@ -52,26 +53,26 @@ const RAINBOW_TOKENS_256 = const RAINBOWBRACKETS_SETTINGS = RainbowBracketsSettings(RAINBOW_TOKENS_256, RAINBOW_TOKENS_16, Sys.iswindows() ? RAINBOW_TOKENS_16 : RAINBOW_TOKENS_256) -function (rainbow::RainbowBracketsSettings)(ansitokens::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int) +function (rainbow::RainbowBracketsSettings)(ansitokens::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString) p, s, b = 0, 0, 0 for (i, t) in enumerate(tokens) k = kind(t) - if k == Tokens.LPAREN + if k == K"(" p += 1 ansitokens[i] = get_color(rainbow, k, p) - elseif k == Tokens.RPAREN + elseif k == K")" ansitokens[i] = get_color(rainbow, k, p) p = max(0, p - 1) - elseif k == Tokens.LSQUARE + elseif k == K"[" s += 1 ansitokens[i] = get_color(rainbow, k, s) - elseif k == Tokens.RSQUARE + elseif k == K"]" ansitokens[i] = get_color(rainbow, k, s) s = max(0, s - 1) - elseif k == Tokens.LBRACE + elseif k == K"{" b += 1 ansitokens[i] = get_color(rainbow, k, b) - elseif k == Tokens.RBRACE + elseif k == K"}" ansitokens[i] = get_color(rainbow, k, b) b = max(0, b - 1) end diff --git a/src/passes/SyntaxHighlighter.jl b/src/passes/SyntaxHighlighter.jl index 15750b4..860f7bf 100644 --- a/src/passes/SyntaxHighlighter.jl +++ b/src/passes/SyntaxHighlighter.jl @@ -1,8 +1,10 @@ module SyntaxHighlighter -using Tokenize -using Tokenize.Tokens -import Tokenize.Tokens: Token, kind, exactkind, iskeyword, untokenize +import JuliaSyntax +import .JuliaSyntax.Tokenize +import JuliaSyntax.@K_str +using .Tokenize +import .Tokenize: Token, kind, kind, untokenize using Crayons @@ -102,56 +104,58 @@ end add_pass!(PASS_HANDLER, "SyntaxHighlighter", SYNTAX_HIGHLIGHTER_SETTINGS, false) -function (highlighter::SyntaxHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, ::Int) +function (highlighter::SyntaxHighlighterSettings)(crayons::Vector{Crayon}, tokens::Vector{Token}, ::Int, str::AbstractString) cscheme = highlighter.active - prev_t = Tokens.Token() - pprev_t = Tokens.Token() + prev_t = Token() + pprev_t = Token() for (i, t) in enumerate(tokens) # a::x - if exactkind(prev_t) == Tokens.DECLARATION + #= + if kind(prev_t) == Tokens.DECLARATION crayons[i-1] = cscheme.argdef crayons[i] = cscheme.argdef + =# # :foo - elseif kind(t) == Tokens.IDENTIFIER && exactkind(prev_t) == Tokens.COLON && - kind(pprev_t) ∉ (Tokens.INTEGER, Tokens.FLOAT, Tokens.IDENTIFIER, Tokens.RPAREN) + if kind(t) == K"Identifier" && kind(prev_t) == K":" && + kind(pprev_t) ∉ (K"Integer", K"Float", K"Identifier", K")") crayons[i-1] = cscheme.symbol crayons[i] = cscheme.symbol # function - elseif iskeyword(kind(t)) - if kind(t) == Tokens.TRUE || kind(t) == Tokens.FALSE + elseif JuliaSyntax.is_keyword(kind(t)) + if kind(t) == K"true" || kind(t) == K"false" crayons[i] = cscheme.symbol else crayons[i] = cscheme.keyword end # "foo" - elseif kind(t) == Tokens.STRING || kind(t) == Tokens.TRIPLE_STRING || kind(t) == Tokens.CHAR || kind(t) == Tokens.CMD || kind(t) == Tokens.TRIPLE_CMD + elseif kind(t) == K"String" || kind(t) == K"Char" || kind(t) == K"CmdString" crayons[i] = cscheme.string # * - - elseif Tokens.isoperator(kind(t)) || exactkind(t) == Tokens.TRUE || exactkind(t) == Tokens.FALSE + elseif JuliaSyntax.is_operator(kind(t)) || kind(t) == K"true" || kind(t) == K"false" crayons[i] = cscheme.op # #= foo =# - elseif kind(t) == Tokens.COMMENT + elseif kind(t) == K"Comment" crayons[i] = cscheme.comment # f(...) - elseif kind(t) == Tokens.LPAREN - if kind(prev_t) == Tokens.IDENTIFIER && !(i > 2 && exactkind(tokens[i-2]) == Tokens.AT_SIGN) + elseif kind(t) == K"(" + if kind(prev_t) == K"Identifier" && !(i > 2 && kind(tokens[i-2]) == K"@") crayons[i-1] = cscheme.call - elseif exactkind(prev_t) == Tokens.DOT && kind(pprev_t) == Tokens.IDENTIFIER + elseif kind(prev_t) == K"." && kind(pprev_t) == K"Identifier" crayons[i-1] = cscheme.call crayons[i-2] = cscheme.call end # function f(...) - if i > 3 && kind(tokens[i-2]) == Tokens.WHITESPACE && exactkind(tokens[i-3]) == Tokens.FUNCTION + if i > 3 && JuliaSyntax.is_whitespace(kind(tokens[i-2])) && kind(tokens[i-3]) == K"function" crayons[i-1] = cscheme.function_def end # @fdsafds - elseif kind(t) == Tokens.IDENTIFIER && exactkind(prev_t) == Tokens.AT_SIGN + elseif kind(t) == K"Identifier" && kind(prev_t) == K"@" crayons[i-1] = cscheme._macro crayons[i] = cscheme._macro # 2] = 32.32 - elseif kind(t) ∈ (Tokens.INTEGER, Tokens.BIN_INT, Tokens.OCT_INT, Tokens.HEX_INT, Tokens.FLOAT) || (kind(t) == Tokens.IDENTIFIER && untokenize(t) == "NaN") + elseif kind(t) ∈ (K"Integer", K"BinInt", K"OctInt", K"HexInt", K"Float") || (kind(t) == K"Identifier" && untokenize(t, str) == "NaN") crayons[i] = cscheme.number - elseif kind(t) == Tokens.WHITESPACE + elseif JuliaSyntax.is_whitespace(kind(t)) crayons[i] = Crayon() else crayons[i] = cscheme.text diff --git a/src/precompile.jl b/src/precompile.jl new file mode 100644 index 0000000..e210832 --- /dev/null +++ b/src/precompile.jl @@ -0,0 +1,4 @@ +# collect(Tokenize.tokenize("foo")) +precompile(Tuple{OhMyREPL.Passes.SyntaxHighlighter.SyntaxHighlighterSettings, Array{Crayons.Crayon, 1}, Array{Tokenize.Tokens.Token, 1}, Int64}) +precompile(Tuple{OhMyREPL.Passes.BracketHighlighter.BracketHighlighterSettings, Array{Crayons.Crayon, 1}, Array{Tokenize.Tokens.Token, 1}, Int64}) +precompile(Tuple{OhMyREPL.Passes.RainbowBrackets.RainbowBracketsSettings, Array{Crayons.Crayon, 1}, Array{Tokenize.Tokens.Token, 1}, Int64}) diff --git a/src/repl.jl b/src/repl.jl index ff2fb41..9cf6ac4 100644 --- a/src/repl.jl +++ b/src/repl.jl @@ -11,7 +11,8 @@ import REPL: respond, return_callback import REPL.LineEdit: buffer, cmove_col, cmove_up, InputAreaState, transition, terminal, buffer, on_enter, move_input_end, add_history, state, mode, edit_insert -import Tokenize.Lexers +import JuliaSyntax.Tokenize +import .Tokenize.Lexer import REPL.Terminals: raw!, width, height, cmove, getX, TerminalBuffer, getY, clear_line, beep, disable_bracketed_paste, enable_bracketed_paste @@ -58,9 +59,10 @@ function rewrite_with_ANSI(s, cursormove::Bool = false) # Insert colorized text from running the passes seekstart(buffer(s)) - tokens = collect(Lexers.Lexer(buffer(s))) - apply_passes!(PASS_HANDLER, tokens, cursoridx, cursormove) - untokenize_with_ANSI(outbuf, PASS_HANDLER , tokens, l) + str = read(buffer(s), String) + tokens = collect(Lexer(str)) + apply_passes!(PASS_HANDLER, tokens, str, cursoridx, cursormove) + untokenize_with_ANSI(outbuf, PASS_HANDLER, tokens, str) # Reset the buffer since the Lexer messed with it (maybe the Lexer should reset it on done) seek(buffer(s), p) diff --git a/src/repl_pass.jl b/src/repl_pass.jl index aa82bfa..535613e 100644 --- a/src/repl_pass.jl +++ b/src/repl_pass.jl @@ -1,5 +1,5 @@ -import Tokenize:Tokens, Lexers -import Tokenize.Tokens: Token, kind, untokenize +import JuliaSyntax.Tokenize +import .Tokenize: Token, kind, untokenize, Lexer using Printf const RESET = Crayon(reset = true) @@ -22,9 +22,9 @@ const PASS_HANDLER = PassHandler() function test_pass(io::IO, f, str::Union{String, IO}, cursorpos::Int = 1, cursormovement::Bool = false) rpc = PassHandler() add_pass!(rpc, "TestPass", f) - tokens = collect(Lexers.Lexer(str)) - apply_passes!(rpc, tokens, cursorpos, cursormovement) - untokenize_with_ANSI(io, rpc.accum_crayons, tokens) + tokens = collect(Lexer(str)) + apply_passes!(rpc, tokens, str, cursorpos, cursormovement) + untokenize_with_ANSI(io, rpc.accum_crayons, tokens, str) end test_pass(f, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::Bool = false) = @@ -32,30 +32,30 @@ test_pass(f, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::B function test_passes(io::IO, rpc::PassHandler, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::Bool = false) b = IOBuffer() - tokens = collect(Lexers.Lexer(str)) - apply_passes!(rpc, tokens, cursorpos, cursormovement) - untokenize_with_ANSI(io, rpc.accum_crayons, tokens) + tokens = collect(Lexer(str)) + apply_passes!(rpc, tokens, str, cursorpos, cursormovement) + untokenize_with_ANSI(io, rpc.accum_crayons, tokens, str) end test_passes(rpc::PassHandler, str::Union{String, IOBuffer}, cursorpos::Int = 1, cursormovement::Bool = false) = test_passes(stdout, rpc, str, cursorpos, cursormovement) -function untokenize_with_ANSI(io::IO, crayons::Vector{Crayon}, tokens::Vector{Token}, indent = 7) +function untokenize_with_ANSI(io::IO, crayons::Vector{Crayon}, tokens::Vector{Token}, str, indent = 7) @assert length(tokens) == length(crayons) print(io, RESET) z = 1 for (token, crayon) in zip(tokens, crayons) print(io, crayon) - for c in untokenize(token) + for c in untokenize(token, str) print(io, c) c == '\n' && print(io, " "^indent) end print(io, RESET) end end -untokenize_with_ANSI(io::IO, rpc::PassHandler, tokens::Vector{Token}, indent = 7) = untokenize_with_ANSI(io, rpc.accum_crayons, tokens, indent) -untokenize_with_ANSI(crayons::Vector{Crayon}, tokens::Vector{Token}, indent = 7) = - untokenize_with_ANSI(stdout, crayons, tokens, indent) +untokenize_with_ANSI(io::IO, rpc::PassHandler, tokens::Vector{Token}, str::AbstractString, indent = 7) = untokenize_with_ANSI(io, rpc.accum_crayons, tokens, str, indent) +untokenize_with_ANSI(crayons::Vector{Crayon}, tokens::Vector{Token}, str::AbstractString, indent = 7) = + untokenize_with_ANSI(stdout, crayons, tokens, str, indent) function merge!(t1::Vector{Crayon}, t2::Vector{Crayon}) @assert length(t1) == length(t2) @@ -65,7 +65,7 @@ function merge!(t1::Vector{Crayon}, t2::Vector{Crayon}) return t1 end -function apply_passes!(rpc::PassHandler, tokens::Vector{Token}, cursorpos::Int = 1, cursormovement::Bool = false) +function apply_passes!(rpc::PassHandler, tokens::Vector{Token}, str::AbstractString, cursorpos::Int = 1, cursormovement::Bool = false) resize!(rpc.crayons, length(tokens)) resize!(rpc.accum_crayons, length(tokens)) fill!(rpc.accum_crayons, Crayon()) @@ -77,7 +77,7 @@ function apply_passes!(rpc::PassHandler, tokens::Vector{Token}, cursorpos::Int = continue end # This is an enabled pass that should be run, so run it! - pass.f!(rpc.crayons, tokens, cursorpos) + pass.f!(rpc.crayons, tokens, cursorpos, str) merge!(rpc.accum_crayons, rpc.crayons) end end diff --git a/test/test_bracketmatch.jl b/test/test_bracketmatch.jl index d7c5ec2..3377a69 100644 --- a/test/test_bracketmatch.jl +++ b/test/test_bracketmatch.jl @@ -2,7 +2,6 @@ module TestBracketHighlighter using Test using OhMyREPL -using Tokenize b = IOBuffer() str = "(function :foobar, foobar )# foobar" diff --git a/test/test_custompass.jl b/test/test_custompass.jl index 923f264..d7a58bd 100644 --- a/test/test_custompass.jl +++ b/test/test_custompass.jl @@ -2,14 +2,15 @@ module CustomPassTest using Test using OhMyREPL -using Tokenize +import JuliaSyntax.Tokenize +using .Tokenize using Crayons -function foobar_bluify(crayons, tokens, ::Int) +function foobar_bluify(crayons, tokens, ::Int, str::AbstractString) for (i, (crayon, tok)) in enumerate(zip(crayons, tokens)) println(tok) - if (Tokenize.Tokens.kind(tok) == Tokenize.Tokens.IDENTIFIER - && Tokenize.Tokens.untokenize(tok) == "foobar") + if (Tokenize.kind(tok) == Tokenize.K"Identifier" + && Tokenize.untokenize(tok, str) == "foobar") crayons[i] = Crayon(foreground = :blue) end end diff --git a/test/test_highlighter.jl b/test/test_highlighter.jl index 2c38423..955adf4 100644 --- a/test/test_highlighter.jl +++ b/test/test_highlighter.jl @@ -5,7 +5,8 @@ using Test using OhMyREPL import OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS -using Tokenize +import JuliaSyntax.Tokenize +using .Tokenize OhMyREPL.colorscheme!("Monokai16")