Skip to content

Commit

Permalink
move to JuliaSyntax.jl over Tokenize.jl
Browse files Browse the repository at this point in the history
  • Loading branch information
KristofferC committed Mar 8, 2023
1 parent aa1d7e2 commit c0011c4
Show file tree
Hide file tree
Showing 14 changed files with 175 additions and 137 deletions.
4 changes: 2 additions & 2 deletions Project.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@ version = "0.5.12"
[deps]
Crayons = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
JLFzf = "1019f520-868f-41f5-a6de-eb00f4b6a39c"
JuliaSyntax = "70703baa-626e-46a2-a12c-08ffd08c73b4"
Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
Tokenize = "0796e94c-ce3b-5d07-9a54-7f471281c624"

[compat]
Crayons = "1, 2, 3, 4"
Tokenize = "0.5"
JuliaSyntax = "0.3.0"
JLFzf = "^0.1.1"
julia = "1.0"

Expand Down
78 changes: 51 additions & 27 deletions docs/Manifest.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

[[ArgTools]]
uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
version = "1.1.1"

[[Artifacts]]
uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
Expand All @@ -10,19 +11,19 @@ uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

[[Crayons]]
git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d"
git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
version = "4.0.4"
version = "4.1.1"

[[Dates]]
deps = ["Printf"]
uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"

[[DocStringExtensions]]
deps = ["LibGit2"]
git-tree-sha1 = "a32185f5428d3986f47c2ab78b1f216d5e6cc96f"
git-tree-sha1 = "b19534d1895d702889b219c382a6e18010797f0b"
uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
version = "0.8.5"
version = "0.8.6"

[[Documenter]]
deps = ["Base64", "Dates", "DocStringExtensions", "IOCapture", "InteractiveUtils", "JSON", "LibGit2", "Logging", "Markdown", "REPL", "Test", "Unicode"]
Expand All @@ -31,8 +32,12 @@ uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
version = "0.27.3"

[[Downloads]]
deps = ["ArgTools", "LibCURL", "NetworkOptions"]
deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
version = "1.6.0"

[[FileWatching]]
uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"

[[IOCapture]]
deps = ["Logging", "Random"]
Expand All @@ -46,29 +51,36 @@ uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"

[[JLFzf]]
deps = ["Pipe", "REPL", "Random", "fzf_jll"]
git-tree-sha1 = "fa300b5b580549186c2b1558a199ac5615cc8928"
git-tree-sha1 = "f377670cda23b6b7c1c0b3893e37451c5c1a2185"
uuid = "1019f520-868f-41f5-a6de-eb00f4b6a39c"
version = "0.1.3"
version = "0.1.5"

[[JLLWrappers]]
deps = ["Preferences"]
git-tree-sha1 = "642a199af8b68253517b80bd3bfd17eb4e84df6e"
git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1"
uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
version = "1.3.0"
version = "1.4.1"

[[JSON]]
deps = ["Dates", "Mmap", "Parsers", "Unicode"]
git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4"
git-tree-sha1 = "3c837543ddb02250ef42f4738347454f95079d4e"
uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
version = "0.21.1"
version = "0.21.3"

[[JuliaSyntax]]
git-tree-sha1 = "db2bdeda30e452485863799be4515f6305431a46"
uuid = "70703baa-626e-46a2-a12c-08ffd08c73b4"
version = "0.3.2"

[[LibCURL]]
deps = ["LibCURL_jll", "MozillaCACerts_jll"]
uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
version = "0.6.3"

[[LibCURL_jll]]
deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
version = "7.84.0+0"

[[LibGit2]]
deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
Expand All @@ -77,6 +89,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
[[LibSSH2_jll]]
deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
version = "1.10.2+0"

[[Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
Expand All @@ -91,42 +104,46 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
[[MbedTLS_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
version = "2.28.2+0"

[[Mmap]]
uuid = "a63ad114-7e13-5084-954f-fe012c677804"

[[MozillaCACerts_jll]]
uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
version = "2022.10.11"

[[NetworkOptions]]
uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
version = "1.2.0"

[[OhMyREPL]]
deps = ["Crayons", "JLFzf", "Markdown", "Pkg", "Printf", "REPL", "Tokenize"]
deps = ["Crayons", "JLFzf", "JuliaSyntax", "Markdown", "Pkg", "Printf", "REPL"]
path = ".."
uuid = "5fb14364-9ced-5910-84b2-373655c76a03"
version = "0.5.9"
version = "0.5.12"

[[Parsers]]
deps = ["Dates"]
git-tree-sha1 = "c8abc88faa3f7a3950832ac5d6e690881590d6dc"
deps = ["Dates", "SnoopPrecompile"]
git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf"
uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
version = "1.1.0"
version = "2.5.8"

[[Pipe]]
git-tree-sha1 = "6842804e7867b115ca9de748a0cf6b364523c16d"
uuid = "b98c9c47-44ae-5843-9183-064241ee97a0"
version = "1.3.0"

[[Pkg]]
deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
version = "1.9.0"

[[Preferences]]
deps = ["TOML"]
git-tree-sha1 = "00cfd92944ca9c760982747e9a1d0d5d86ab1e5a"
git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d"
uuid = "21216c6a-2e73-6563-6e65-726566657250"
version = "1.2.2"
version = "1.3.0"

[[Printf]]
deps = ["Unicode"]
Expand All @@ -137,35 +154,39 @@ deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"

[[Random]]
deps = ["Serialization"]
deps = ["SHA", "Serialization"]
uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"

[[SHA]]
uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
version = "0.7.0"

[[Serialization]]
uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"

[[SnoopPrecompile]]
deps = ["Preferences"]
git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c"
uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c"
version = "1.0.3"

[[Sockets]]
uuid = "6462fe0b-24de-5631-8697-dd941f90decc"

[[TOML]]
deps = ["Dates"]
uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
version = "1.0.3"

[[Tar]]
deps = ["ArgTools", "SHA"]
uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
version = "1.10.0"

[[Test]]
deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[[Tokenize]]
git-tree-sha1 = "37018506dc445ad7db288442fbb846105f26c43f"
uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
version = "0.5.17"

[[UUIDs]]
deps = ["Random", "SHA"]
uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
Expand All @@ -176,17 +197,20 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
[[Zlib_jll]]
deps = ["Libdl"]
uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
version = "1.2.13+0"

[[fzf_jll]]
deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
git-tree-sha1 = "b2d7b205b9518f8a5ebcd05764c5536dbdf460bb"
git-tree-sha1 = "868e669ccb12ba16eaf50cb2957ee2ff61261c56"
uuid = "214eeab7-80f7-51ab-84ad-2988db7cef09"
version = "0.24.4+0"
version = "0.29.0+0"

[[nghttp2_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
version = "1.48.0+0"

[[p7zip_jll]]
deps = ["Artifacts", "Libdl"]
uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
version = "17.4.0+0"
73 changes: 37 additions & 36 deletions docs/src/internals/passes.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ All the passes are registered in a global pass handler. To show all the passes u
```jl
julia> OhMyREPL.showpasses()
----------------------------------
# Pass name Enabled
# Pass name Enabled
----------------------------------
1 BracketHighlighter true
2 SyntaxHighlighter true
1 BracketHighlighter true
2 SyntaxHighlighter true
----------------------------------
```

Expand All @@ -34,24 +34,24 @@ julia> using Tokenize
julia> tokens = collect(Tokenize.tokenize(str))
20-element Array{Tokenize.Tokens.Token,1}:
1,1-1,8: KEYWORD "function"
1,9-1,9: WHITESPACE " "
1,10-1,10: IDENTIFIER "f"
1,11-1,11: LPAREN "("
1,12-1,12: IDENTIFIER "x"
1,13-1,14: OP "::"
1,9-1,9: WHITESPACE " "
1,10-1,10: IDENTIFIER "f"
1,11-1,11: LPAREN "("
1,12-1,12: IDENTIFIER "x"
1,13-1,14: OP "::"
1,15-1,21: IDENTIFIER "Float64"
1,22-1,22: RPAREN ")"
1,23-1,23: WHITESPACE " "
1,24-1,29: KEYWORD "return"
1,30-1,30: WHITESPACE " "
1,31-1,31: OP ":"
1,32-1,32: IDENTIFIER "x"
1,33-1,33: WHITESPACE " "
1,34-1,34: OP "+"
1,35-1,35: WHITESPACE " "
1,36-1,38: CHAR "'a'"
1,39-1,39: WHITESPACE " "
1,40-1,42: KEYWORD "end"
1,22-1,22: RPAREN ")"
1,23-1,23: WHITESPACE " "
1,24-1,29: KEYWORD "return"
1,30-1,30: WHITESPACE " "
1,31-1,31: OP ":"
1,32-1,32: IDENTIFIER "x"
1,33-1,33: WHITESPACE " "
1,34-1,34: OP "+"
1,35-1,35: WHITESPACE " "
1,36-1,38: CHAR "'a'"
1,39-1,39: WHITESPACE " "
1,40-1,42: KEYWORD "end"
1,43-1,42: ENDMARKER ""
```

Expand All @@ -62,25 +62,25 @@ crayons = Vector{Crayon}(length(tokens));
fill!(crayons, Crayon()) # Crayon is a bits type so this is OK
```

These two vectors are then sent to the syntax highlighter pass together with an integer that represent what character offset the cursor currently is located. The syntax highlighter does not use this information but the bracket highlighter does.
These two vectors and the source code are then sent to the syntax highlighter pass together with an integer that represent what character offset the cursor currently is located. The syntax highlighter does not use this information but the bracket highlighter does.

```
OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0)
OhMyREPL.Passes.SyntaxHighlighter.SYNTAX_HIGHLIGHTER_SETTINGS(crayons, tokens, 0, str)
```

Running this function has the effect of updating the `crayons` vector. If we print this vector we see that they have been updated:

![](ansitokens_after.png)

To print the original string with the updated vector of `Crayon`s we use the `OhMyREPL.untokenize_with_ANSI([io::IO], crayons, tokens)` function as:
To print the original string with the updated vector of `Crayon`s we use the `OhMyREPL.untokenize_with_ANSI([io::IO], crayons, tokens, str)` function as:

![](print_ansi.png)

Each registered and enabled pass does this updating and the contributions from each pass to the `Crayon` vector is merged in to a separate vector. After each pass is done, the result is printed to the REPL.

## Creating a pass

This section shows how to create a very pass that let the user define a `Crayon` for each typeassertion / declaration that happens to be a `Float64`.
This section shows how to create a pass that let the user define a `Crayon` for each typeassertion / declaration that happens to be a `Float64`.

!!! info
Please refer to the [Tokenize.jl API](https://github.com/KristofferC/Tokenize.jl#api) section and the [`Crayons.jl` documentation](https://github.com/KristofferC/Crayons.jl) while reading this section.
Expand All @@ -89,7 +89,8 @@ We start off with a few imports and creating a new struct which will hold the se

```jl
using Crayons
import Tokenize.Tokens: Token, untokenize, exactkind
import JuliaSyntax
import JuliaSyntax.Tokenize: Token, untokenize, kind
using OhMyREPL

mutable struct Float64Modifier
Expand All @@ -104,11 +105,11 @@ We then use call overloading to define a function for the type. The function wil

```jl
# The pass function, the cursor position is not used but it needs to be given an argument
function (float64modifier::Float64Modifier)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int)
function (float64modifier::Float64Modifier)(crayons::Vector{Crayon}, tokens::Vector{Token}, cursorpos::Int, str::AbstractString)
# Loop over all tokens and crayons
for i in 1:length(crayons)
if untokenize(tokens[i]) == "Float64"
if i > 1 && exactkind(tokens[i-1]) == Tokenize.Tokens.DECLARATION
if untokenize(tokens[i], str) == "Float64"
if i > 1 && kind(tokens[i-1]) == JuliaSyntax.K"::"
# Update the crayon
crayons[i] = float64modifier.crayon
end
Expand All @@ -130,11 +131,11 @@ To register and start using the pass simply use `OhMyREPL.add_pass!(passname::St
```jl
julia> OhMyREPL.add_pass!("Redify Float64", FLOAT64_MODIFIER)
----------------------------------
# Pass name Enabled
# Pass name Enabled
----------------------------------
1 Redify Float64 true
2 BracketHighlighter true
3 SyntaxHighlighter true
1 Redify Float64 true
2 BracketHighlighter true
3 SyntaxHighlighter true
----------------------------------
```

Expand All @@ -151,11 +152,11 @@ The prescedence of a pass can be modified with the `OhMyREPL.prescedence!(pass::
```jl
julia> OhMyREPL.prescedence!("Redify Float64", 3)
----------------------------------
# Pass name Enabled
# Pass name Enabled
----------------------------------
1 BracketHighlighter true
2 SyntaxHighlighter true
3 Redify Float64 true
1 BracketHighlighter true
2 SyntaxHighlighter true
3 Redify Float64 true
----------------------------------
```

Expand Down
Loading

0 comments on commit c0011c4

Please sign in to comment.