Cell metadata and persistent cell deactivation

Project.toml

@@ -21,6 +21,7 @@ REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
 RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00"
 Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [compat]

sample/notebook_with_metadata.jl

@@ -0,0 +1,30 @@
+### A Pluto.jl notebook ###
+# v0.16.1
+
+using Markdown
+using InteractiveUtils
+
+# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error).
+macro bind(def, element)
+    quote
+        local el = $(esc(element))
+        global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing
+        el
+    end
+end
+
+# ╔═╡ 7d550d46-7dc4-11eb-3df0-db7ad0594043
+# ╠═╡ disabled = "directly"
+using PlutoUI
+
+# ╔═╡ 8399ed34-7dc4-11eb-16ec-e572834e149d
+# ╠═╡ disabled = "indirectly"
+@bind x Slider(1:10)
+
+# ╔═╡ 88c83d42-7dc4-11eb-1b5e-a1ecad4b6ff1
+x
+
+# ╔═╡ Cell order:
+# ╠═7d550d46-7dc4-11eb-3df0-db7ad0594043
+# ╠═8399ed34-7dc4-11eb-16ec-e572834e149d
+# ╠═88c83d42-7dc4-11eb-1b5e-a1ecad4b6ff1

src/analysis/DependencyCache.jl

@@ -44,3 +44,22 @@ function update_dependency_cache!(notebook::Notebook)
         update_dependency_cache!(cell, notebook)
     end
 end
+
+"""
+find (indirectly) deactivated cells and update their status
+"""
+function disable_dependent_cells!(notebook:: Notebook, topology:: NotebookTopology):: Vector{Cell}
+    for cell in notebook.cells
+		cell.depends_on_disabled_cells = false
+	end
+	deactivated = filter(c -> c.running_disabled, notebook.cells)
+    indirectly_deactivated = collect(topological_order(notebook, topology, deactivated))
+    for cell in indirectly_deactivated
+        cell.running = false
+        cell.queued = false
+        cell.depends_on_disabled_cells = true
+    end
+	return indirectly_deactivated
+end
+
+disable_dependent_cells!(notebook:: Notebook) = disable_dependent_cells!(notebook, notebook.topology)

src/evaluation/Run.jl

@@ -6,7 +6,7 @@ import .WorkspaceManager: macroexpand_in_workspace
 Base.push!(x::Set{Cell}) = x
 
 "Run given cells and all the cells that depend on them, based on the topology information before and after the changes."
-function run_reactive!(session::ServerSession, notebook::Notebook, old_topology::NotebookTopology, new_topology::NotebookTopology, roots::Vector{Cell}; deletion_hook::Function = WorkspaceManager.move_vars, user_requested_run::Bool = true, already_in_run::Bool = false, already_run::Vector{Cell} = Cell[])::TopologicalOrder
+function run_reactive!(session::ServerSession, notebook::Notebook, old_topology::NotebookTopology, new_topology::NotebookTopology, roots::Vector{Cell}; deletion_hook::Function = WorkspaceManager.move_vars, user_requested_run::Bool = true, already_in_run::Bool = false, already_run::Vector{Cell} = Cell[], indirectly_deactivated::Union{Nothing, Vector{Cell}}=nothing)::TopologicalOrder
     if !already_in_run
         # make sure that we're the only `run_reactive!` being executed - like a semaphor
         take!(notebook.executetoken)
@@ -22,6 +22,7 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology:
 
         # update cache and save notebook because the dependencies might have changed after expanding macros
         update_dependency_cache!(notebook)
+		disable_dependent_cells!(notebook)
         save_notebook(session, notebook)
     end
 
@@ -54,14 +55,9 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology:
     to_run_raw = setdiff(union(new_runnable, old_runnable), keys(new_order.errable))::Vector{Cell} # TODO: think if old error cell order matters
 
     # find (indirectly) deactivated cells and update their status
-    deactivated = filter(c -> c.running_disabled, notebook.cells)
-    indirectly_deactivated = collect(topological_order(notebook, new_topology, deactivated))
-    for cell in indirectly_deactivated
-        cell.running = false
-        cell.queued = false
-        cell.depends_on_disabled_cells = true
-    end
-
+	if indirectly_deactivated === nothing
+		indirectly_deactivated = disable_dependent_cells!(notebook, new_topology)
+	end
     to_run = setdiff(to_run_raw, indirectly_deactivated)
 
     # change the bar on the sides of cells to "queued"
@@ -137,6 +133,7 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology:
 
             # update cache and save notebook because the dependencies might have changed after expanding macros
             update_dependency_cache!(notebook)
+			disable_dependent_cells!(notebook)
             save_notebook(session, notebook)
 
             return run_reactive!(session, notebook, new_topology, new_new_topology, to_run; deletion_hook, user_requested_run, already_in_run = true, already_run = to_run[1:i])
@@ -146,6 +143,7 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology:
 
             # update cache and save notebook because the dependencies might have changed after expanding macros
             update_dependency_cache!(notebook)
+			disable_dependent_cells!(notebook)
             save_notebook(session, notebook)
 
             return run_reactive!(session, notebook, new_topology, new_new_topology, to_run; deletion_hook, user_requested_run, already_in_run = true, already_run = to_run[1:i])
@@ -407,6 +405,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr
 	new = notebook.topology = updated_topology(old, notebook, cells) # macros are not yet resolved
 
 	update_dependency_cache!(notebook)
+	indirectly_deactivated = disable_dependent_cells!(notebook)
 	save && save_notebook(session, notebook)
 
 	# _assume `prerender_text == false` if you want to skip some details_
@@ -458,7 +457,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr
 		sync_nbpkg(session, notebook; save=(save && !session.options.server.disable_writing_notebook_files))
 		if !(isempty(to_run_online) && session.options.evaluation.lazy_workspace_creation) && will_run_code(notebook)
 			# not async because that would be double async
-			run_reactive_async!(session, notebook, old, new, to_run_online; run_async=false, kwargs...)
+			run_reactive_async!(session, notebook, old, new, to_run_online; run_async=false, indirectly_deactivated, kwargs...)
 			# run_reactive_async!(session, notebook, old, new, to_run_online; deletion_hook=deletion_hook, run_async=false, kwargs...)
 		end
 	end

src/notebook/Cell.jl

@@ -46,6 +46,8 @@ Base.@kwdef mutable struct Cell
 
     running_disabled::Bool=false
     depends_on_disabled_cells::Bool=false
+
+    metadata::Dict{String,Any}=Dict{String,Dict{String,Any}}()
 end
 
 Cell(cell_id, code) = Cell(cell_id=cell_id, code=code)
@@ -64,3 +66,16 @@ end
 function Base.convert(::Type{UUID}, string::String)
     UUID(string)
 end
+
+function get_cell_metadata(cell::Cell)::Dict{String,Any}
+    Dict(
+        if cell.running_disabled
+            Dict("disabled" => "directly")
+        elseif cell.depends_on_disabled_cells
+            Dict("disabled" => "indirectly")
+        else
+            Dict()
+        end...,
+        cell.metadata...,
+    )
+end

src/notebook/Notebook.jl

@@ -3,6 +3,7 @@ import .ExpressionExplorer: SymbolsState, FunctionNameSignaturePair, FunctionNam
 import .Configuration
 import .PkgCompat: PkgCompat, PkgContext
 import Pkg
+import TOML
 
 mutable struct BondValue
     value::Any
@@ -85,10 +86,14 @@ const _notebook_header = "### A Pluto.jl notebook ###"
 # We use a creative delimiter to avoid accidental use in code
 # so don't get inspired to suddenly use these in your code!
 const _cell_id_delimiter = "# ╔═╡ "
+const _cell_metadata_prefix = "# ╠═╡ "
 const _order_delimiter = "# ╠═"
 const _order_delimiter_folded = "# ╟─"
 const _cell_suffix = "\n\n"
 
+const _disabled_prefix = "#=╠═╡\n"
+const _disabled_suffix = "\n  ╠═╡ =#"
+
 const _ptoml_cell_id = UUID(1)
 const _mtoml_cell_id = UUID(2)
 
@@ -120,9 +125,23 @@ function save_notebook(io, notebook::Notebook)
 
     for c in cells_ordered
         println(io, _cell_id_delimiter, string(c.cell_id))
-        # write the cell code and prevent collisions with the cell delimiter
-        print(io, replace(c.code, _cell_id_delimiter => "# "))
-        print(io, _cell_suffix)
+        metadata_toml = strip(sprint(TOML.print, get_cell_metadata(c)))
+        if metadata_toml != ""
+            for line in split(metadata_toml, "\n")
+                println(io, _cell_metadata_prefix, line)
+            end
+        end
+
+        if c.running_disabled || c.depends_on_disabled_cells
+            print(io, _disabled_prefix)
+            print(io, replace(c.code, _cell_id_delimiter => "# "))
+            print(io, _disabled_suffix)
+            print(io, _cell_suffix)
+        else
+            # write the cell code and prevent collisions with the cell delimiter
+            print(io, replace(c.code, _cell_id_delimiter => "# "))
+            print(io, _cell_suffix)
+        end
     end
 
 
@@ -204,13 +223,37 @@ function load_notebook_nobackup(io, path)::Notebook
             break
         else
             cell_id = UUID(cell_id_str)
-            code_raw = String(readuntil(io, _cell_id_delimiter))
+
+            metadata_toml_lines = String[]
+            initial_code_line = ""
+            while !eof(io)
+                line = String(readline(io))
+                if startswith(line, _cell_metadata_prefix)
+                    prefix_length = ncodeunits(_cell_metadata_prefix)
+                    push!(metadata_toml_lines, line[begin+prefix_length:end])
+                else
+                    initial_code_line = line
+                    break
+                end
+            end
+
+            code_raw = initial_code_line * "\n" * String(readuntil(io, _cell_id_delimiter))
             # change Windows line endings to Linux
             code_normalised = replace(code_raw, "\r\n" => "\n")
+
+            # remove the disabled on startup comments for further processing in Julia
+            code_normalised = replace(replace(code_normalised, _disabled_prefix => ""), _disabled_suffix => "")
+
             # remove the cell suffix
             code = code_normalised[1:prevind(code_normalised, end, length(_cell_suffix))]
 
-            read_cell = Cell(cell_id, code)
+            # parse metadata
+            metadata = TOML.parse(join(metadata_toml_lines, "\n"))
+            cell_disabling_status = pop!(metadata, "disabled", "") # the disabling metadata is not part of the "normal" metadata but stored in separate fields due to its importance in notebook evaluation
+            running_disabled = cell_disabling_status == "directly"
+            depends_on_disabled_cells = cell_disabling_status == "indirectly"
+
+            read_cell = Cell(; cell_id, code, running_disabled, depends_on_disabled_cells, metadata)
             collected_cells[cell_id] = read_cell
         end
     end
@@ -305,6 +348,7 @@ function load_notebook(path::String; disable_writing_notebook_files::Bool=false)
     # Analyze cells so that the initial save is in topological order
     loaded.topology = updated_topology(loaded.topology, loaded, loaded.cells) |> static_resolve_topology
     update_dependency_cache!(loaded)
+    disable_dependent_cells!(loaded)
 
     disable_writing_notebook_files || save_notebook(loaded)
     loaded.topology = NotebookTopology()

test/Notebook.jl

@@ -29,6 +29,22 @@ function basic_notebook()
     ]) |> init_packages!
 end
 
+function metadata_notebook()
+    Notebook([
+        Cell(
+            code="100*a + b",
+            running_disabled=true,
+            metadata=Dict(
+                "a metadata tag" => Dict(
+                    "boolean" => true,
+                    "string" => "String",
+                    "number" => 10000,
+                ),
+            ),
+        ),
+    ]) |> init_packages!
+end
+
 function shuffled_notebook()
     Notebook([
         Cell("z = y"),
@@ -141,6 +157,14 @@ end
         end
     end
 
+    @testset "Metadata" begin
+        nb = metadata_notebook()
+        save_notebook(nb)
+        @info "File" Text(read(nb.path,String))
+        result = load_notebook_nobackup(nb.path)
+        @test notebook_inputs_equal(nb, result)
+    end
+
     @testset "I/O overloaded" begin
         @testset "$(name)" for (name, nb) in nbs
             @test let

test/helpers.jl

@@ -132,7 +132,7 @@ end
 function notebook_inputs_equal(nbA, nbB; check_paths_equality=true)
     x = !check_paths_equality || (normpath(nbA.path) == normpath(nbB.path))
 
-    to_compare(cell) = (cell.cell_id, cell.code_folded, cell.code)
+    to_compare(cell) = (cell.cell_id, cell.code_folded, cell.code, cell.metadata)
     y = to_compare.(nbA.cells) == to_compare.(nbB.cells)
 
     x && y