Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Partial balloon copy failure #398

Open
avikivity opened this issue Jul 17, 2014 · 1 comment
Open

Partial balloon copy failure #398

avikivity opened this issue Jul 17, 2014 · 1 comment
Assignees

Comments

@avikivity
Copy link
Member

#0 0x00000000003d2a72 in cli_hlt () at /home/avi/osv/arch/x64/processor.hh:242
#1 halt_no_interrupts () at /home/avi/osv/arch/x64/arch.hh:48
#2 osv::halt () at /home/avi/osv/core/power.cc:34
#3 0x0000000000223df0 in abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at /home/avi/osv/runtime.cc:139
#4 0x0000000000223e99 in __assert_fail (expr=, file=, line=, func=) at /home/avi/osv/runtime.cc:145
#5 0x0000000000372709 in page_fault (ef=0xffff80003fd03038) at /home/avi/osv/arch/x64/mmu.cc:34
#6
#7 0x0000000000337047 in elf::object::symtab_len (this=this@entry=0xffffa00038740c00) at /home/avi/osv/core/elf.cc:737
#8 0x0000000000337159 in elf::object::lookup_addr (this=0xffffa00038740c00, addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292) at /home/avi/osv/core/elf.cc:767
#9 0x00000000003394f7 in operator() (ml=..., __closure=) at /home/avi/osv/core/elf.cc:1147
#10 with_modules<elf::program::lookup_addr(void const*)::__lambda6> (f=..., this=0xffffa0003fc05400) at /home/avi/osv/include/osv/elf.hh:597
#11 elf::program::lookup_addr (this=, addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292) at /home/avi/osv/core/elf.cc:1153
#12 0x00000000003da98c in osv::lookup_name_demangled (addr=addr@entry=0x100001438964 MoveAndUpdateClosure::copy_partial_obj()+292, buf=buf@entry=0xffff800035bf87a0 "???+3610006",

len=len@entry=1024) at /home/avi/osv/core/demangle.cc:44

#13 0x0000000000223ca1 in print_backtrace () at /home/avi/osv/runtime.cc:97
#14 0x0000000000223e57 in abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at /home/avi/osv/runtime.cc:131
#15 0x0000000000223e99 in __assert_fail (expr=, file=, line=, func=) at /home/avi/osv/runtime.cc:145
#16 0x0000000000326273 in mmu::jvm_balloon_vma::add_partial (this=this@entry=0xffffa00094d2ef00, partial=partial@entry=524288,

eff=eff@entry=0x78b8aadd0 <error: Cannot access memory at address 0x78b8aadd0>) at /home/avi/osv/core/mmu.cc:1409

#17 0x0000000000341924 in jvm_balloon_fault (b=std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00, ef=ef@entry=0xffff800035bf9068, vma=vma@entry=0xffffa00094d2ef00)

at /home/avi/osv/java/jvm_balloon.cc:400

#18 0x0000000000329e04 in mmu::jvm_balloon_vma::fault (this=0xffffa00094d2ef00, fault_addr=32431996928, ef=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1430
#19 0x0000000000325da5 in mmu::vm_fault (addr=, addr@entry=32431999832, ef=ef@entry=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1217
#20 0x00000000003726aa in page_fault (ef=0xffff800035bf9068) at /home/avi/osv/arch/x64/mmu.cc:38
#21
#22 0x000000000036e973 in repmovsb (n=, src=, dest=) at /home/avi/osv/arch/x64/string.cc:100
#23 memcpy_repmov (dest=0x78c480000, src=0x78d190b58, n=524288) at /home/avi/osv/arch/x64/string.cc:225
#24 0x0000100001438965 in MoveAndUpdateClosure::copy_partial_obj() ()
#25 0x0000100001439093 in PSParallelCompact::fill_region(ParCompactionManager*, unsigned long) ()
#26 0x000010000142b62c in ParCompactionManager::drain_region_stacks() ()
#27 0x0000100001410c9a in DrainStacksCompactionTask::do_it(GCTaskManager*, unsigned int) ()
#28 0x0000100001148c62 in GCTaskThread::run() ()
#29 0x00001000013d0f22 in java_start(Thread*) ()
#30 0x0000000000423136 in operator() (__closure=0xffffa00037057700) at /home/avi/osv/libc/pthread.cc:79
#31 std::_Function_handler<void(), pthread_private::pthread::pthread(void* ()(void), void*, sigset_t, const pthread_private::thread_attr*)::__lambda5>::_M_invoke(const std::_Any_data &) (

__functor=...) at /home/avi/osv/external/x64/gcc.bin/usr/include/c++/4.8.2/functional:2071

#32 0x00000000003be11b in main (this=0xffff800035bf4040) at /home/avi/osv/core/sched.cc:915
#33 sched::thread_main_c (t=0xffff800035bf4040) at /home/avi/osv/arch/x64/arch-switch.hh:137
#34 0x0000000000372516 in thread_main () at /home/avi/osv/arch/x64/entry.S:113

1406 bool jvm_balloon_vma::add_partial(size_t partial, unsigned char *eff)
1407 {
1408 if (_effective_jvm_addr) {
1409 assert(eff == _effective_jvm_addr);
1410 } else {
1411 _effective_jvm_addr= eff;
1412 }
1413
1414 _partial_copy += partial;

(gdb) p partial
$1 = 524288
(gdb) p eff
$2 = (unsigned char *) 0x78b8aadd0 <error: Cannot access memory at address 0x78b8aadd0>
(gdb) p _effective_jvm_addr
$3 = (unsigned char *) 0x77e6c4470 ""
(gdb)

p _this
$4 = {
mmu::vma = {
_vptr.vma = 0x638690 <vtable for mmu::jvm_balloon_vma+16>,
_range = {
_start = 32419872768,
_end = 32551993344
},
perm = 3,
flags = 81,
map_dirty = true,
page_ops = 0x9b0598 mmu::page_allocator_noinit,
vma_list_hook = {
<boost::intrusive::detail::generic_hook<boost::intrusive::get_set_node_algo<void
, false>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1, 0>> = {
boost::intrusive::detail::no_default_definer = {},
boost::intrusive::rbtree_node<void*> = {
parent
= 0xffffa00061893e30,
left
= 0x0,
right
= 0x0,
color
= boost::intrusive::rbtree_node<void*>::black_t
}, }, }
},
members of mmu::jvm_balloon_vma:
_balloon = std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00,
_jvm_addr = 0x78c5bb928 "",
_effective_jvm_addr = 0x77e6c4470 "",
_partial_addr = 0,
_partial_vma = 0x0,
_partial_copy = 488632,
_real_perm = 3,
_real_flags = 17,
_real_size = 132120576
}

p *_balloon ._M_ptr
$9 = {
_jvm_addr = 0x7c249a3a8 "T",
_jref = 0xffffa00033fc8860,
_alignment = 2097152,
_balloon_size = 134217728
}

to reproduce: tomcat + json-servlet + wrk, for a long time

@glommer
Copy link
Contributor

glommer commented Jul 17, 2014

Thanks Avi. I will take a look soon

On Thu, Jul 17, 2014 at 3:06 PM, Avi Kivity [email protected]
wrote:

#0 0x00000000003d2a72 in cli_hlt () at
/home/avi/osv/arch/x64/processor.hh:242
#1 #1 halt_no_interrupts ()
at /home/avi/osv/arch/x64/arch.hh:48
#2 #2 osv::halt () at
/home/avi/osv/core/power.cc:34
#3 #3 0x0000000000223df0 in
abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s (%s: %s: %d)\n") at
/home/avi/osv/runtime.cc:139
#4 #4 0x0000000000223e99
in __assert_fail (expr=, file=, line=, func=) at
/home/avi/osv/runtime.cc:145
#5 #5 0x0000000000372709
in page_fault (ef=0xffff80003fd03038) at /home/avi/osv/arch/x64/mmu.cc:34
#6 #6
#7 #7 0x0000000000337047
in elf::object::symtab_len (this=this@entry=0xffffa00038740c00) at
/home/avi/osv/core/elf.cc:737
#8 #8 0x0000000000337159
in elf::object::lookup_addr (this=0xffffa00038740c00, addr=addr@entry=0x100001438964
MoveAndUpdateClosure::copy_partial_obj()+292) at
/home/avi/osv/core/elf.cc:767
#9 #9 0x00000000003394f7
in operator() (ml=..., __closure=) at /home/avi/osv/core/elf.cc:1147
#10 #10 with_modules
(f=..., this=0xffffa0003fc05400) at /home/avi/osv/include/osv/elf.hh:597
#11 #11
elf::program::lookup_addr (this=, addr=addr@entry=0x100001438964
MoveAndUpdateClosure::copy_partial_obj()+292) at
/home/avi/osv/core/elf.cc:1153
#12 #12
0x00000000003da98c in osv::lookup_name_demangled (addr=addr@entry=0x100001438964
MoveAndUpdateClosure::copy_partial_obj()+292, buf=buf@entry=0xffff800035bf87a0
"???+3610006",
len=len@entry=1024) at /home/avi/osv/core/demangle.cc:44
#13 #13
0x0000000000223ca1 in print_backtrace () at /home/avi/osv/runtime.cc:97
#14 #14
0x0000000000223e57 in abort (fmt=fmt@entry=0x5c99c0 "Assertion failed: %s
(%s: %s: %d)\n") at /home/avi/osv/runtime.cc:131
#15 #15
0x0000000000223e99 in assert_fail (expr=, file=, line=, func=) at
/home/avi/osv/runtime.cc:145
#16 #16
0x0000000000326273 in mmu::jvm_balloon_vma::add_partial (this=this@entry=0xffffa00094d2ef00,
partial=partial@entry=524288,
eff=eff@entry=0x78b8aadd0 ) at /home/avi/osv/core/mmu.cc:1409
#17 #17
0x0000000000341924 in jvm_balloon_fault (b=std::shared_ptr (count 3, weak
0) 0xffffa0003e0d1c00, ef=ef@entry=0xffff800035bf9068, vma=vma@entry
=0xffffa00094d2ef00)
at /home/avi/osv/java/jvm_balloon.cc:400
#18 #18
0x0000000000329e04 in mmu::jvm_balloon_vma::fault (this=0xffffa00094d2ef00,
fault_addr=32431996928, ef=0xffff800035bf9068) at
/home/avi/osv/core/mmu.cc:1430
#19 #19
0x0000000000325da5 in mmu::vm_fault (addr=, addr@entry=32431999832,
ef=ef@entry=0xffff800035bf9068) at /home/avi/osv/core/mmu.cc:1217
#20 #20
0x00000000003726aa in page_fault (ef=0xffff800035bf9068) at
/home/avi/osv/arch/x64/mmu.cc:38
#21 #21
#22 #22
0x000000000036e973 in repmovsb (n=, src=, dest=) at
/home/avi/osv/arch/x64/string.cc:100
#23 #23 memcpy_repmov
(dest=0x78c480000, src=0x78d190b58, n=524288) at
/home/avi/osv/arch/x64/string.cc:225
#24 #24
0x0000100001438965 in MoveAndUpdateClosure::copy_partial_obj() ()
#25 #25
0x0000100001439093 in PSParallelCompact::fill_region(ParCompactionManager
,
unsigned long) ()
#26 #26
0x000010000142b62c in ParCompactionManager::drain_region_stacks() ()
#27 #27
0x0000100001410c9a in DrainStacksCompactionTask::do_it(GCTaskManager
,
unsigned int) ()
#28 #28
0x0000100001148c62 in GCTaskThread::run() ()
#29 #29
0x00001000013d0f22 in java_start(Thread*) ()
#30 #30
0x0000000000423136 in operator() (__closure=0xffffa00037057700) at
/home/avi/osv/libc/pthread.cc:79
#31 #31
std::_Function_handler::_M_invoke(const std::_Any_data &) (
__functor=...) at
/home/avi/osv/external/x64/gcc.bin/usr/include/c++/4.8.2/functional:2071
#32 #32 0x00000000003be11b
in main (this=0xffff800035bf4040) at /home/avi/osv/core/sched.cc:915
#33 #33
sched::thread_main_c (t=0xffff800035bf4040) at
/home/avi/osv/arch/x64/arch-switch.hh:137
#34 #34
0x0000000000372516 in thread_main () at /home/avi/osv/arch/x64/entry.S:113

1406 bool jvm_balloon_vma::add_partial(size_t partial, unsigned char *eff)
1407 {
1408 if (_effective_jvm_addr) {
1409 assert(eff == _effective_jvm_addr);
1410 } else {
1411 _effective_jvm_addr= eff;
1412 }
1413
1414 _partial_copy += partial;

(gdb) p partial
$1 = 524288
(gdb) p eff
$2 = (unsigned char *) 0x78b8aadd0
(gdb) p _effective_jvm_addr
$3 = (unsigned char *) 0x77e6c4470 ""
(gdb)

p

_this $4 = {mmu::vma = { _vptr.vma = 0x638690 , _range = { _start =
32419872768, _end = 32551993344 }, perm = 3, flags = 81, map_dirty =
true, page_ops = 0x9b0598 mmu::page_allocator_noinit, vma_list_hook = {
<boost::intrusive::detail::generic_hook<boost::intrusive::get_set_node_algo<void
,
false>, boost::intrusive::member_tag, (boost::intrusive::link_mode_type)1,
0>> = {
boost::intrusive::detail::no_default_definer = {},
boost::intrusive::rbtree_node<void*> = {
parent
= 0xffffa00061893e30,
left
= 0x0,
right
= 0x0,
color
= boost::intrusive::rbtree_node::black_t
}, }, }
},
members of mmu::jvm_balloon_vma:
_balloon = std::shared_ptr (count 3, weak 0) 0xffffa0003e0d1c00,
_jvm_addr = 0x78c5bb928 "",
_effective_jvm_addr = 0x77e6c4470 "",
_partial_addr = 0,
_partial_vma = 0x0,
_partial_copy = 488632,
_real_perm = 3,
_real_flags = 17,
_real_size = 132120576
}

p *_balloon ._M_ptr
$9 = {
_jvm_addr = 0x7c249a3a8 "T",
_jref = 0xffffa00033fc8860,
_alignment = 2097152,
_balloon_size = 134217728
}

to reproduce: tomcat + json-servlet + wrk, for a long time


Reply to this email directly or view it on GitHub
#398.

@slivne slivne added this to the release 0.12 milestone Jul 31, 2014
@slivne slivne added the bug label Jul 31, 2014
@slivne slivne modified the milestones: release 0.14, release 0.12 Sep 29, 2014
@slivne slivne removed this from the release 0.14 milestone Oct 28, 2014
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Development

No branches or pull requests

4 participants