Skip to content

Commit

Permalink
Update JLD for JuliaLang/julia#10380
Browse files Browse the repository at this point in the history
  • Loading branch information
@simonster
simonster committed Apr 25, 2015
1 parent 0e816df commit 62e9b24
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 36 deletions.
28 changes: 23 additions & 5 deletions src/JLD.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,11 @@ immutable AssociativeWrapper{K,V,T<:Associative}
values::Vector{V}
end

# Wrapper for SimpleVector
immutable SimpleVectorWrapper
elements::Vector
end

include("jld_types.jl")

file(x::JldFile) = x
Expand Down Expand Up @@ -376,6 +381,11 @@ function after_read{K,V,T}(x::AssociativeWrapper{K,V,T})
ret
end

# Special case for SimpleVector
if VERSION >= v"0.4.0-dev+4319"
after_read(x::SimpleVectorWrapper) = Base.svec(x.elements...)
end

## Arrays

# Read an array
Expand Down Expand Up @@ -488,7 +498,7 @@ write(parent::Union(JldFile, JldGroup), name::ByteString,
close(_write(parent, name, data, wsession; kargs...))

# Pick whether to use compact or default storage based on data size
function dset_create_properties(parent, sz::Int, obj, chunk=Int[]; mmap = false)
function dset_create_properties(parent, sz::Int, obj, chunk=Int[]; mmap::Bool=false)
if sz <= 8192 && !ismmapped(parent) && !mmap
return compact_properties(), false
end
Expand Down Expand Up @@ -634,18 +644,24 @@ write_ref(parent::JldGroup, data, wsession::JldWriteSession) =
write_ref(file(parent), data, wsession)

# Special case for associative, to rehash keys
function _write(parent::Union(JldFile, JldGroup), name::ByteString,
d::Associative, wsession::JldWriteSession; kargs...)
function _write{K,V}(parent::Union(JldFile, JldGroup), name::ByteString,
d::Associative{K,V}, wsession::JldWriteSession; kargs...)
n = length(d)
K, V = eltype(d)
ks = Array(K, n)
vs = Array(V, n)
i = 0
for (k,v) in d
ks[i+=1] = k
vs[i] = v
end
write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession)
write_compound(parent, name, AssociativeWrapper{K,V,typeof(d)}(ks, vs), wsession; kargs...)
end

# Special case for SimpleVector
if VERSION >= v"0.4.0-dev+4319"
_write(parent::Union(JldFile, JldGroup), name::ByteString,
d::SimpleVector, wsession::JldWriteSession; kargs...) =
write_compound(parent, name, SimpleVectorWrapper([d...]), wsession; kargs...)
end

# Expressions, drop line numbers
Expand Down Expand Up @@ -859,6 +875,8 @@ function full_typename(io::IO, file::JldFile, jltype::DataType)
full_typename(io, file, jltype.parameters[i])
end
print(io, '}')
elseif jltype <: Tuple
print(io, "{}")
end
end
function full_typename(file::JldFile, x)
Expand Down
86 changes: 57 additions & 29 deletions src/jld_types.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,20 @@ const BUILTIN_TYPES = Set([Symbol, Type, UTF16String, BigFloat, BigInt])
const H5CONVERT_DEFINED = ObjectIdDict()
const JLCONVERT_DEFINED = ObjectIdDict()

if VERSION >= v"0.4.0-dev+4319"
const EMPTY_TUPLE_TYPE = Tuple{}
typealias TypesType SimpleVector
typealias TupleType{T<:Tuple} Type{T}
tupletypes(T::TupleType) = T.parameters
typetuple(types) = Tuple{types...}
else
const EMPTY_TUPLE_TYPE = ()
typealias TypesType (Type...)
typealias TupleType (Type...)
tupletypes(T::TupleType) = T
typetuple(types) = tuple(types...)
end

## Helper functions

# Holds information about the mapping between a Julia and HDF5 type
Expand All @@ -20,7 +34,7 @@ immutable JldTypeInfo
end

# Get information about the HDF5 types corresponding to Julia types
function JldTypeInfo(parent::JldFile, types::(@compat Tuple{Vararg{Type}}), commit::Bool)
function JldTypeInfo(parent::JldFile, types::TypesType, commit::Bool)
dtypes = Array(JldDatatype, length(types))
offsets = Array(Int, length(types))
offset = 0
Expand Down Expand Up @@ -257,7 +271,7 @@ function h5type{T<:Type}(parent::JldFile, ::Type{T}, commit::Bool)
id = HDF5.h5t_create(HDF5.H5T_COMPOUND, 8)
HDF5.h5t_insert(id, "typename_", 0, h5fieldtype(parent, UTF8String, commit))
dtype = HDF5Datatype(id, parent.plain)
commit ? commit_datatype(parent, dtype, Type) : JldDatatype(dtype, -1)
out = commit ? commit_datatype(parent, dtype, Type) : JldDatatype(dtype, -1)
end

gen_h5convert{T<:Type}(::JldFile, ::Type{T}) = nothing
Expand Down Expand Up @@ -292,16 +306,13 @@ h5fieldtype{T,N}(parent::JldFile, ::Type{Array{T,N}}, ::Bool) = JLD_REF_TYPE
## Tuples

if INLINE_TUPLE
h5fieldtype(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool) =
h5fieldtype(parent::JldFile, T::TupleType, commit::Bool) =
isleaftype(T) ? h5type(parent, T, commit) : JLD_REF_TYPE
else
h5fieldtype(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), ::Bool) = JLD_REF_TYPE
h5fieldtype(parent::JldFile, T::TupleType, ::Bool) = JLD_REF_TYPE
end

function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool)
!isa(T, (@compat Tuple{Vararg{Union((@compat Tuple), DataType)}})) && unknown_type_err(T)
T = T::(@compat Tuple{Vararg{Union((@compat Tuple), DataType)}})

function h5type(parent::JldFile, T::TupleType, commit::Bool)
haskey(parent.jlh5type, T) && return parent.jlh5type[T]
# Tuples should always be concretely typed, unless we're
# reconstructing a tuple, in which case commit will be false
Expand All @@ -321,7 +332,7 @@ function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool)
dtype = HDF5Datatype(id, parent.plain)
if commit
jlddtype = commit_datatype(parent, dtype, T)
if isempty(T)
if T == EMPTY_TUPLE_TYPE
# to allow recovery of empty tuples, which HDF5 does not allow
a_write(dtype, "empty", @compat UInt8(1))
end
Expand All @@ -331,21 +342,22 @@ function h5type(parent::JldFile, T::(@compat Tuple{Vararg{Type}}), commit::Bool)
end
end

function gen_jlconvert(typeinfo::JldTypeInfo, T::(@compat Tuple{Vararg{Type}}))
function gen_jlconvert(typeinfo::JldTypeInfo, T::TupleType)
haskey(JLCONVERT_DEFINED, T) && return

ex = Expr(:block)
args = ex.args
tup = Expr(:tuple)
tupargs = tup.args
types = tupletypes(T)
for i = 1:length(typeinfo.dtypes)
h5offset = typeinfo.offsets[i]
field = symbol(string("field", i))

if HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE
push!(args, :($field = read_ref(file, unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset))))
else
push!(args, :($field = jlconvert($(T[i]), file, ptr+$h5offset)))
push!(args, :($field = jlconvert($(types[i]), file, ptr+$h5offset)))
end
push!(tupargs, field)
end
Expand Down Expand Up @@ -434,7 +446,20 @@ function _gen_jlconvert_immutable(typeinfo::JldTypeInfo, T::ANY)
h5offset = typeinfo.offsets[i]
jloffset = jloffsets[i]

if HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE
if isa(T.types[i], TupleType) && VERSION >= v"0.4.0-dev+4319" && T.types[i].pointerfree
# We continue to store tuples as references for the sake of
# backwards compatibility, but on 0.4 they are now stored
# inline
push!(args, quote
ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset)
if ref == HDF5.HDF5ReferenceObj_NULL
warn("""A pointerfree tuple field was undefined.
This is not supported in Julia 0.4 and the corresponding tuple will be uninitialized.""")
else
unsafe_store!(convert(Ptr{$(T.types[i])}, out)+$jloffset, read_ref(file, ref))
end
end)
elseif HDF5.h5t_get_class(typeinfo.dtypes[i]) == HDF5.H5T_REFERENCE
obj = gensym("obj")
push!(args, quote
ref = unsafe_load(convert(Ptr{HDF5ReferenceObj}, ptr)+$h5offset)
Expand Down Expand Up @@ -523,19 +548,20 @@ end
## Common functions for all non-special types (including gen_h5convert)

# Whether this datatype should be stored as opaque
isopaque(t::@compat Tuple{Vararg{Type}}) = isa(t, ())
isopaque(t::DataType) = isempty(fieldnames(t))
isopaque(t::TupleType) = t == EMPTY_TUPLE_TYPE
# isopaque(t::DataType) = isempty(fieldnames(t))
isopaque(t::DataType) = isa(t, TupleType) ? t == EMPTY_TUPLE_TYPE : isempty(fieldnames(t))

# The size of this datatype in the HDF5 file (if opaque)
opaquesize(t::@compat Tuple{Vararg{DataType}}) = 1
opaquesize(t::TupleType) = 1
opaquesize(t::DataType) = max(1, t.size)

# Whether a type that is stored inline in HDF5 should be stored as a
# reference in Julia. This will only be called such that it returns
# true for some unions of special types defined above, unless either
# INLINE_TUPLE or INLINE_POINTER_IMMUTABLE is true.
uses_reference(T::DataType) = !T.pointerfree
uses_reference(::@compat Tuple) = true
uses_reference(::TupleType) = true
uses_reference(::UnionType) = true

unknown_type_err(T) =
Expand All @@ -548,19 +574,21 @@ gen_h5convert(parent::JldFile, T) =
# There is no point in specializing this
function _gen_h5convert(parent::JldFile, T::ANY)
dtype = parent.jlh5type[T].dtype
istuple = isa(T, @compat Tuple)
istuple = isa(T, TupleType)

if isopaque(T)
if T.size == 0
@eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = nothing
else
@eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) =
unsafe_store!(convert(Ptr{$T}, out), x)
end
return
end

if istuple
types = T
types = tupletypes(T)
else
if isopaque(T::DataType)
if (T::DataType).size == 0
@eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) = nothing
else
@eval h5convert!(out::Ptr, ::JldFile, x::$T, ::JldWriteSession) =
unsafe_store!(convert(Ptr{$T}, out), x)
end
return
end
types = (T::DataType).types
end

Expand Down Expand Up @@ -704,9 +732,9 @@ function reconstruct_type(parent::JldFile, dtype::HDF5Datatype, savedname::Abstr
end
end

if startswith(savedname, "(")
if startswith(savedname, "(") || startswith(savedname, "Core.Tuple{")
# We're reconstructing a tuple
tuple(fieldtypes...)
typetuple(fieldtypes)
else
# We're reconstructing some other type
@eval begin
Expand Down
12 changes: 10 additions & 2 deletions test/jld.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,13 +205,19 @@ bitsparamint16 = BitsParams{@compat Int16(1)}()
# Tuple of tuples
tuple_of_tuples = (1, 2, (3, 4, [5, 6]), [7, 8])

# SimpleVector
if VERSION >= v"0.4.0-dev+4319"
simplevec = Base.svec(1, 2, Int64, "foo")
iseq(x::SimpleVector, y::SimpleVector) = collect(x) == collect(y)
end

iseq(x,y) = isequal(x,y)
iseq(x::MyStruct, y::MyStruct) = (x.len == y.len && x.data == y.data)
iseq(x::MyImmutable, y::MyImmutable) = (isequal(x.x, y.x) && isequal(x.y, y.y) && isequal(x.z, y.z))
iseq(x::Union(EmptyTI, EmptyTT), y::Union(EmptyTI, EmptyTT)) = isequal(x.x, y.x)
iseq(c1::Array{Base.Sys.CPUinfo}, c2::Array{Base.Sys.CPUinfo}) = length(c1) == length(c2) && all([iseq(c1[i], c2[i]) for i = 1:length(c1)])
function iseq(c1::Base.Sys.CPUinfo, c2::Base.Sys.CPUinfo)
for n in Base.Sys.CPUinfo.names
for n in fieldnames(Base.Sys.CPUinfo)
if getfield(c1, n) != getfield(c2, n)
return false
end
Expand Down Expand Up @@ -387,6 +393,7 @@ for compress in (true,false)
@write fid bitsparamint
@write fid bitsparamuint
@write fid tuple_of_tuples
VERSION >= v"0.4.0-dev+4319" && @write fid simplevec

# Make sure we can create groups (i.e., use HDF5 features)
g = g_create(fid, "mygroup")
Expand Down Expand Up @@ -515,6 +522,7 @@ for compress in (true,false)
@check fidr bitsparamint
@check fidr bitsparamuint
@check fidr tuple_of_tuples
VERSION >= v"0.4.0-dev+4319" && @check fidr simplevec

x1 = read(fidr, "group1/x")
@assert x1 == Any[1]
Expand Down Expand Up @@ -739,7 +747,7 @@ jldopen(fn, "r") do file
@test reinterpret(UInt8, x.d) == 0x12

x = read(file, "x9")
@test isa(x, (@compat Tuple))
@test isa(x, Tuple)
@test length(x) == 3
@test x[1].x == 1
@test isa(x[2], Tuple)
Expand Down

0 comments on commit 62e9b24

Please sign in to comment.