Loading saved Jobs from a JLD2 file after closing the Julia session fail to reconstruct

[singularitti]

After saving the Jobs which were run into a JLD2 file, close the Julia session, and reload this file, the following error occurs:

julia> JLD2.load("saved.jld2")["jobs"]
┌ Warning: type Main.#9#10 does not exist in workspace; reconstructing
└ @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/data/reconstructing_datatypes.jl:369
Error encountered while load FileIO.File{FileIO.DataFormat{:JLD2}, String}("saved.jld2").

Fatal error:
ERROR: MethodError: Cannot `convert` an object of type JLD2.ReconstructedTypes.var"##Main.#9#10#312" to an object of type Function
Closest candidates are:
  convert(::Type{T}, ::T) where T at Base.jl:61
Stacktrace:
  [1] rconvert(T::Type, x::JLD2.ReconstructedTypes.var"##Main.#9#10#312")
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/data/custom_serialization.jl:10
  [2] jlconvert
    @ ~/.julia/packages/JLD2/k9Gt0/src/data/writing_datatypes.jl:305 [inlined]
  [3] macro expansion
    @ ~/.julia/packages/JLD2/k9Gt0/src/data/reconstructing_datatypes.jl:528 [inlined]
  [4] jlconvert(#unused#::JLD2.ReadRepresentation{Job, JLD2.OnDiskRepresentation{(0, 8, 16, 32, 48, 56, 64, 72, 76, 84, 92, 100), Tuple{Int64, Function, String, String, Dates.DateTime, Dates.DateTime, Dates.DateTime, SimpleWorkflows.JobStatus, Union{Nothing, Task}, UInt64, Vector{Job}, Vector{Job}}, Tuple{Int64, JLD2.RelOffset, JLD2.Vlen{String}, JLD2.Vlen{String}, Dates.DateTime, Dates.DateTime, Dates.DateTime, SimpleWorkflows.JobStatus, JLD2.RelOffset, UInt64, JLD2.RelOffset, JLD2.RelOffset}}()}, f::JLD2.JLDFile{JLD2.MmapIO}, ptr::Ptr{Nothing}, header_offset::JLD2.RelOffset)
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/data/reconstructing_datatypes.jl:513
  [5] read_scalar(f::JLD2.JLDFile{JLD2.MmapIO}, rr::JLD2.ReadRepresentation{Job, JLD2.OnDiskRepresentation{(0, 8, 16, 32, 48, 56, 64, 72, 76, 84, 92, 100), Tuple{Int64, Function, String, String, Dates.DateTime, Dates.DateTime, Dates.DateTime, SimpleWorkflows.JobStatus, Union{Nothing, Task}, UInt64, Vector{Job}, Vector{Job}}, Tuple{Int64, JLD2.RelOffset, JLD2.Vlen{String}, JLD2.Vlen{String}, Dates.DateTime, Dates.DateTime, Dates.DateTime, SimpleWorkflows.JobStatus, JLD2.RelOffset, UInt64, JLD2.RelOffset, JLD2.RelOffset}}()}, header_offset::JLD2.RelOffset)
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/dataio.jl:37
  [6] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, rr::Any, read_dataspace::Tuple{JLD2.ReadDataspace, JLD2.RelOffset, Int64, UInt16}, attributes::Vector{JLD2.ReadAttribute})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:170
  [7] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, data_offset::Int64, data_length::Int64, filter_id::UInt16, header_offset::JLD2.RelOffset, attributes::Vector{JLD2.ReadAttribute})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:149
  [8] load_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, offset::JLD2.RelOffset)
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:92
  [9] jlconvert
    @ ~/.julia/packages/JLD2/k9Gt0/src/data/writing_datatypes.jl:304 [inlined]
 [10] macro expansion
    @ ~/.julia/packages/JLD2/k9Gt0/src/dataio.jl:70 [inlined]
 [11] macro expansion
    @ ./simdloop.jl:77 [inlined]
 [12] read_array!(v::Vector{Job}, f::JLD2.JLDFile{JLD2.MmapIO}, rr::JLD2.ReadRepresentation{Job, JLD2.RelOffset})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/dataio.jl:68
 [13] read_array(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, rr::JLD2.ReadRepresentation{Job, JLD2.RelOffset}, data_length::Int64, filter_id::UInt16, header_offset::JLD2.RelOffset, attributes::Vector{JLD2.ReadAttribute})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:329
 [14] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, rr::JLD2.ReadRepresentation{Any, JLD2.RelOffset}, read_dataspace::Tuple{JLD2.ReadDataspace, JLD2.RelOffset, Int64, UInt16}, attributes::Vector{JLD2.ReadAttribute})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:200
 [15] macro expansion
    @ ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:156 [inlined]
 [16] macro expansion
    @ ~/.julia/packages/JLD2/k9Gt0/src/datatypes.jl:76 [inlined]
 [17] read_data(f::JLD2.JLDFile{JLD2.MmapIO}, dataspace::JLD2.ReadDataspace, datatype_class::UInt8, datatype_offset::Int64, data_offset::Int64, data_length::Int64, filter_id::UInt16, header_offset::JLD2.RelOffset, attributes::Vector{JLD2.ReadAttribute})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:152
 [18] load_dataset(f::JLD2.JLDFile{JLD2.MmapIO}, offset::JLD2.RelOffset)
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/datasets.jl:92
 [19] getindex(g::JLD2.Group{JLD2.JLDFile{JLD2.MmapIO}}, name::String)
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/groups.jl:109
 [20] getindex
    @ ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:393 [inlined]
 [21] loadtodict!(d::Dict{String, Any}, g::JLD2.JLDFile{JLD2.MmapIO}, prefix::String)
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:154
 [22] loadtodict!
    @ ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:153 [inlined]
 [23] (::JLD2.var"#72#73")(file::JLD2.JLDFile{JLD2.MmapIO})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/fileio.jl:39
 [24] jldopen(::Function, ::String, ::Vararg{String}; kws::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:4
 [25] jldopen
    @ ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:1 [inlined]
 [26] #fileio_load#71
    @ ~/.julia/packages/JLD2/k9Gt0/src/fileio.jl:38 [inlined]
 [27] fileio_load(f::FileIO.File{FileIO.DataFormat{:JLD2}, String})
    @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/fileio.jl:37
 [28] #invokelatest#2
    @ ./essentials.jl:729 [inlined]
 [29] invokelatest
    @ ./essentials.jl:726 [inlined]
 [30] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::FileIO.Formatted; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:219
 [31] action
    @ ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:196 [inlined]
 [32] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::Symbol, ::String; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:185
 [33] action
    @ ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:185 [inlined]
 [34] load(::String; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
    @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:113
 [35] load(::String)
    @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:109
 [36] top-level scope
    @ REPL[3]:1
Stacktrace:
 [1] handle_error(e::MethodError, q::Base.PkgId, bt::Vector{Union{Ptr{Nothing}, Base.InterpreterIP}})
   @ FileIO ~/.julia/packages/FileIO/DtNtF/src/error_handling.jl:61
 [2] handle_exceptions(exceptions::Vector{Tuple{Any, Union{Base.PkgId, Module}, Vector}}, action::String)
   @ FileIO ~/.julia/packages/FileIO/DtNtF/src/error_handling.jl:56
 [3] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::FileIO.Formatted; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
   @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:228
 [4] action
   @ ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:196 [inlined]
 [5] action(::Symbol, ::Vector{Union{Base.PkgId, Module}}, ::Symbol, ::String; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
   @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:185
 [6] action
   @ ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:185 [inlined]
 [7] load(::String; options::Base.Pairs{Symbol, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
   @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:113
 [8] load(::String)
   @ FileIO ~/.julia/packages/FileIO/DtNtF/src/loadsave.jl:109
 [9] top-level scope
   @ REPL[3]:1

[singularitti]

After PR #140, I still have a similar issue. I found that it is because JLD2.jl does not support saving anonymous functions and closures, which Thunk functions are.

Error encountered while load FileIO.File{FileIO.DataFormat{:JLD2}, String}("saved.jld2").

Fatal error:
ERROR: ConcurrencyViolationError("setfield!: atomic field cannot be written non-atomically")

I have tried JLD.jl and BSON.jl, but either of them has problems like this or that.

[singularitti]

Closed by #149 since I use Serialization.serialize and it seems to work.