Skip to content

Commit

Permalink
Allow re-initialization and caching of foreign types (#47407)
Browse files Browse the repository at this point in the history
Co-authored-by: Tim Holy <[email protected]>
Co-authored-by: Max Horn <[email protected]>
  • Loading branch information
3 people committed Nov 29, 2022
1 parent 93b9429 commit a397a1d
Show file tree
Hide file tree
Showing 17 changed files with 254 additions and 6 deletions.
16 changes: 16 additions & 0 deletions src/datatype.c
Original file line number Diff line number Diff line change
Expand Up @@ -826,6 +826,22 @@ JL_DLLEXPORT jl_datatype_t * jl_new_foreign_type(jl_sym_t *name,
return bt;
}

JL_DLLEXPORT int jl_reinit_foreign_type(jl_datatype_t *dt,
jl_markfunc_t markfunc,
jl_sweepfunc_t sweepfunc)
{
if (!jl_is_foreign_type(dt))
return 0;
const jl_datatype_layout_t *layout = dt->layout;
jl_fielddescdyn_t * desc =
(jl_fielddescdyn_t *) ((char *)layout + sizeof(*layout));
assert(!desc->markfunc);
assert(!desc->sweepfunc);
desc->markfunc = markfunc;
desc->sweepfunc = sweepfunc;
return 1;
}

JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt)
{
return jl_is_datatype(dt) && dt->layout && dt->layout->fielddesc_type == 3;
Expand Down
1 change: 1 addition & 0 deletions src/jl_exported_funcs.inc
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@
XX(jl_new_code_info_uninit) \
XX(jl_new_datatype) \
XX(jl_new_foreign_type) \
XX(jl_reinit_foreign_type) \
XX(jl_new_method_instance_uninit) \
XX(jl_new_method_table) \
XX(jl_new_method_uninit) \
Expand Down
7 changes: 7 additions & 0 deletions src/julia_gcext.h
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,13 @@ JL_DLLEXPORT jl_datatype_t *jl_new_foreign_type(
int haspointers,
int large);


#define HAVE_JL_REINIT_FOREIGN_TYPE 1
JL_DLLEXPORT int jl_reinit_foreign_type(
jl_datatype_t *dt,
jl_markfunc_t markfunc,
jl_sweepfunc_t sweepfunc);

JL_DLLEXPORT int jl_is_foreign_type(jl_datatype_t *dt);

JL_DLLEXPORT size_t jl_gc_max_internal_obj_size(void);
Expand Down
19 changes: 17 additions & 2 deletions src/staticdata.c
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ External links:

#include "julia.h"
#include "julia_internal.h"
#include "julia_gcext.h"
#include "builtin_proto.h"
#include "processor.h"
#include "serialize.h"
Expand Down Expand Up @@ -1249,6 +1250,9 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
ios_write(s->s, (char*)v, sizeof(void*) + jl_string_len(v));
write_uint8(s->s, '\0'); // null-terminated strings for easier C-compatibility
}
else if (jl_is_foreign_type(t) == 1) {
jl_error("Cannot serialize instances of foreign datatypes");
}
else if (jl_datatype_nfields(t) == 0) {
// The object has no fields, so we just snapshot its byte representation
assert(!t->layout->npointers);
Expand Down Expand Up @@ -1438,10 +1442,14 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
if (dt->layout != NULL) {
size_t nf = dt->layout->nfields;
size_t np = dt->layout->npointers;
size_t fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type);
size_t fieldsize = 0;
uint8_t is_foreign_type = dt->layout->fielddesc_type == 3;
if (!is_foreign_type) {
fieldsize = jl_fielddesc_size(dt->layout->fielddesc_type);
}
char *flddesc = (char*)dt->layout;
size_t fldsize = sizeof(jl_datatype_layout_t) + nf * fieldsize;
if (dt->layout->first_ptr != -1)
if (!is_foreign_type && dt->layout->first_ptr != -1)
fldsize += np << dt->layout->fielddesc_type;
uintptr_t layout = LLT_ALIGN(ios_pos(s->const_data), sizeof(void*));
write_padding(s->const_data, layout - ios_pos(s->const_data)); // realign stream
Expand All @@ -1450,6 +1458,13 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED
arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_datatype_t, layout))); // relocation location
arraylist_push(&s->relocs_list, (void*)(((uintptr_t)ConstDataRef << RELOC_TAG_OFFSET) + layout)); // relocation target
ios_write(s->const_data, flddesc, fldsize);
if (is_foreign_type) {
// make sure we have space for the extra hidden pointers
// zero them since they will need to be re-initialized externally
assert(fldsize == sizeof(jl_datatype_layout_t));
jl_fielddescdyn_t dyn = {0, 0};
ios_write(s->const_data, (char*)&dyn, sizeof(jl_fielddescdyn_t));
}
}
}
else if (jl_is_typename(v)) {
Expand Down
1 change: 1 addition & 0 deletions test/gcext/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,3 @@
/gcext
/gcext-debug
/Foreign/deps
14 changes: 14 additions & 0 deletions test/gcext/DependsOnForeign/Manifest.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.8.3"
manifest_format = "2.0"
project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb"

[[deps.Foreign]]
deps = ["Libdl"]
path = "../Foreign"
uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
version = "0.1.0"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
6 changes: 6 additions & 0 deletions test/gcext/DependsOnForeign/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
name = "DependsOnForeign"
uuid = "4b0716e0-dfb5-4e00-8b44-e2685a41517f"
version = "0.1.0"

[deps]
Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
14 changes: 14 additions & 0 deletions test/gcext/DependsOnForeign/src/DependsOnForeign.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
module DependsOnForeign

using Foreign

f(obj::FObj) = Base.pointer_from_objref(obj)
precompile(f, (FObj,))

const FObjRef = Ref{FObj}()

function __init__()
FObjRef[] = FObj()
end

end # module DependsOnForeign
8 changes: 8 additions & 0 deletions test/gcext/Foreign/Manifest.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.9.0-DEV"
manifest_format = "2.0"
project_hash = "7b70172a2edbdc772ed789e79d4411d7528eae86"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
6 changes: 6 additions & 0 deletions test/gcext/Foreign/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
name = "Foreign"
uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
version = "0.1.0"

[deps]
Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
56 changes: 56 additions & 0 deletions test/gcext/Foreign/deps/foreignlib.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// This file is a part of Julia. License is MIT: https://julialang.org/license

#include "julia.h"
#include "julia_gcext.h"

// TODO make these atomics
int nmarks = 0;
int nsweeps = 0;

uintptr_t mark(jl_ptls_t ptls, jl_value_t *p)
{
nmarks += 1;
return 0;
}

void sweep(jl_value_t *p)
{
nsweeps++;
}

JL_DLLEXPORT jl_datatype_t *declare_foreign(jl_sym_t* name, jl_module_t *module, jl_datatype_t *parent)
{
return jl_new_foreign_type(name, module, parent, mark, sweep, 1, 0);
}

// #define GC_MAX_SZCLASS (2032 - sizeof(void *))

JL_DLLEXPORT int reinit_foreign(jl_datatype_t *dt)
{
int ret = jl_reinit_foreign_type(dt, mark, sweep);
nmarks = nsweeps = 0;
if (ret == 0)
return 0;
if (dt->layout->npointers != 1)
return -1;
if (dt->layout->size != 0)
return -2;
return ret;
}

JL_DLLEXPORT jl_value_t *allocate_foreign(jl_ptls_t ptls, size_t sz, jl_datatype_t *dt)
{
jl_value_t* obj = jl_gc_alloc_typed(ptls, sz, dt);
jl_gc_schedule_foreign_sweepfunc(ptls, obj);
return obj;
}

JL_DLLEXPORT int nmark_counter()
{
return nmarks;
}

JL_DLLEXPORT int nsweep_counter()
{
return nsweeps;
}
29 changes: 29 additions & 0 deletions test/gcext/Foreign/src/Foreign.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# This file is a part of Julia. License is MIT: https://julialang.org/license

module Foreign

using Libdl

const foreignlib = joinpath(ENV["BINDIR"], "foreignlib.$(dlext)")

const FObj = ccall((:declare_foreign, foreignlib), Any, (Any, Any, Any), :FObj, @__MODULE__, Any)
FObj() = ccall((:allocate_foreign, foreignlib), Any, (Ptr{Cvoid}, Csize_t, Any,), Core.getptls(), sizeof(Ptr{Cvoid}), FObj)::FObj

export FObj

get_nmark() = ccall((:nmark_counter, foreignlib), Cint, ())
get_nsweep() = ccall((:nsweep_counter, foreignlib), Cint, ())

function __init__()
@assert ccall((:reinit_foreign, foreignlib), Cint, (Any,), FObj) == 1
end

allocs(N) = [Foreign.FObj() for _ in 1:N]

function test(N)
x = allocs(N)
Core.donotdelete(x)
x = nothing
end

end # module Foreign
14 changes: 14 additions & 0 deletions test/gcext/ForeignObjSerialization/Manifest.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.8.3"
manifest_format = "2.0"
project_hash = "e7199d961a5f4ebad68a3deaf5beaa7406a0afcb"

[[deps.Foreign]]
deps = ["Libdl"]
path = "../Foreign"
uuid = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
version = "0.1.0"

[[deps.Libdl]]
uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
6 changes: 6 additions & 0 deletions test/gcext/ForeignObjSerialization/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
name = "ForeignObjSerialization"
uuid = "2c015d96-a6ca-42f0-bc68-f9090de6bc2c"
version = "0.1.0"

[deps]
Foreign = "de1f6f7a-d7b3-400f-91c2-33f248ee89c4"
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
module ForeignObjSerialization

using Foreign
const FObjRef = Ref{FObj}(FObj())

end # module ForeignObjSerialization
24 changes: 20 additions & 4 deletions test/gcext/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -19,38 +19,54 @@ SRCDIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST))))
# get the executable suffix, if any
EXE := $(suffix $(abspath $(JULIA)))

OS := $(shell uname)
ifeq ($(OS), Darwin)
DYLIB := .dylib
else
DYLIB := .so
endif

# get compiler and linker flags. (see: `contrib/julia-config.jl`)
JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "julia-config.jl"))' --
CPPFLAGS_ADD :=
CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags)
LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs)
DYLIBFLAGS := --shared -fPIC

DEBUGFLAGS += -g

#=============================================================================

release: $(BIN)/gcext$(EXE)
debug: $(BIN)/gcext-debug$(EXE)
release: $(BIN)/gcext$(EXE) $(BIN)/Foreign/deps/foreignlib$(DYLIB)
debug: $(BIN)/gcext-debug$(EXE) $(BIN)/Foreign/deps/foreignlib-debug$(DYLIB)

$(BIN)/gcext$(EXE): $(SRCDIR)/gcext.c
$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)

$(BIN)/gcext-debug$(EXE): $(SRCDIR)/gcext.c
$(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS)

$(BIN)/foreignlib$(DYLIB): $(SRCDIR)/Foreign/deps/foreignlib.c
$(CC) $^ -o $@ $(DYLIBFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS)

$(BIN)/foreignlib-debug$(DYLIB): $(SRCDIR)/Foreign/deps/foreignlib.c
$(CC) $^ -o $@ $(DYLIBFLAGS) $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS)

ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR)))
# for demonstration purposes, our demo code is also installed
# in $BIN, although this would likely not be typical
$(BIN)/LocalTest.jl: $(SRCDIR)/LocalTest.jl
cp $< $@
endif

check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl
$(JULIA) --depwarn=error $(SRCDIR)/gcext-test.jl $<
check: $(BIN)/gcext$(EXE) $(BIN)/LocalTest.jl $(BIN)/foreignlib$(DYLIB)
BINDIR=$(BIN) $(JULIA) --depwarn=error $(SRCDIR)/gcext-test.jl $<
@echo SUCCESS

clean:
-rm -f $(BIN)/gcext-debug$(EXE) $(BIN)/gcext$(EXE)
-rm -f $(BIN)/foreignlib$(DYLIB)
-rm -f $(BIN)/foreignlib-debug$(DYLIB)

.PHONY: release debug clean check

Expand Down
33 changes: 33 additions & 0 deletions test/gcext/gcext-test.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

# tests the output of the embedding example is correct
using Test
using Pkg

if Sys.iswindows()
# libjulia needs to be in the same directory as the embedding executable or in path
Expand Down Expand Up @@ -43,3 +44,35 @@ end
@test checknum(lines[5], r"([0-9]+) corrupted auxiliary roots",
n -> n == 0)
end

@testset "Package with foreign type" begin
load_path = copy(LOAD_PATH)
push!(LOAD_PATH, joinpath(@__DIR__, "Foreign"))
push!(LOAD_PATH, joinpath(@__DIR__, "DependsOnForeign"))
try
# Force recaching
Base.compilecache(Base.identify_package("Foreign"))
Base.compilecache(Base.identify_package("DependsOnForeign"))

push!(LOAD_PATH, joinpath(@__DIR__, "ForeignObjSerialization"))
@test_throws ErrorException Base.compilecache(Base.identify_package("ForeignObjSerialization"), Base.DevNull())
pop!(LOAD_PATH)

(@eval (using Foreign))
@test Base.invokelatest(Foreign.get_nmark) == 0
@test Base.invokelatest(Foreign.get_nsweep) == 0

obj = Base.invokelatest(Foreign.FObj)
GC.@preserve obj begin
GC.gc(true)
end
@test Base.invokelatest(Foreign.get_nmark) > 0
@time Base.invokelatest(Foreign.test, 10)
GC.gc(true)
@test Base.invokelatest(Foreign.get_nsweep) > 0
(@eval (using DependsOnForeign))
Base.invokelatest(DependsOnForeign.f, obj)
finally
copy!(LOAD_PATH, load_path)
end
end

0 comments on commit a397a1d

Please sign in to comment.