Still crashity crashing
Charles Oliver Nutter
headius at headius.com
Tue Aug 4 20:34:55 PDT 2009
Ok, found it...there are 9 threads active:
(gdb) info threads
9 process 43094 thread 0x2903 0x96ebf3ae in __semwait_signal ()
8 process 43094 thread 0x2803 0x96ebf3ae in __semwait_signal ()
7 process 43094 thread 0x2703 0x96ebf3ae in __semwait_signal ()
6 process 43094 thread 0x2603 0x96eb8202 in semaphore_wait_trap ()
5 process 43094 thread 0x2303 0x96ebf3ae in __semwait_signal ()
4 process 43094 thread 0x2203 0x96ebf3ae in __semwait_signal ()
3 process 43094 thread 0x2103 0x96efc796 in __wait4 ()
* 2 process 43094 thread 0x213 0x96ebf3ae in __semwait_signal ()
1 process 43094 thread 0x717 0x96ebf3ae in __semwait_signal ()
Their backtraces are as follows:
(gdb) thread 1
[Switching to thread 1 (process 43094 thread 0x717)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96f04398 in pthread_join$UNIX2003 ()
#2 0x0000668b in ContinueInNewThread0 ()
#3 0x0000440e in JLI_Launch ()
#4 0x0000d388 in main ()
(gdb) thread 2
[Switching to thread 2 (process 43094 thread 0x213)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96eea326 in _pthread_cond_wait ()
#2 0x96ee9d0d in pthread_cond_wait$UNIX2003 ()
#3 0x0136aea9 in os::PlatformEvent::park ()
#4 0x0134f32f in Monitor::IWait ()
#5 0x0134f5c4 in Monitor::wait ()
#6 0x01463070 in VMThread::execute ()
#7 0x0113e3a0 in GenCollectorPolicy::mem_allocate_work ()
#8 0x011cec88 in GenCollectedHeap::mem_allocate ()
#9 0x0142c27e in typeArrayKlass::allocate ()
#10 0x01362222 in oopFactory::new_typeArray ()
#11 0x010ef68f in Runtime1::new_type_array ()
#12 0x01ab508a in ?? ()
#13 0x01ae41f3 in ?? ()
#14 0x01ae3aa4 in ?? ()
#15 0x01aede5c in ?? ()
#16 0x01aef690 in ?? ()
#17 0x01af1684 in ?? ()
#18 0x01af1244 in ?? ()
#19 0x01af0cb0 in ?? ()
#20 0x01a4282f in ?? ()
#21 0x01a422cf in ?? ()
#22 0x01a421c7 in ?? ()
#23 0x01a421c7 in ?? ()
#24 0x01a421c7 in ?? ()
#25 0x01a421c7 in ?? ()
#26 0x01a421c7 in ?? ()
#27 0x01a421c7 in ?? ()
#28 0x01a421c7 in ?? ()
#29 0x01a421c7 in ?? ()
#30 0x01a421c7 in ?? ()
#31 0x01a41ff3 in ?? ()
#32 0x01a41ff3 in ?? ()
#33 0x01a3f2cc in ?? ()
#34 0x01222eb2 in JavaCalls::call_helper ()
#35 0x0136a241 in os::os_exception_wrapper ()
#36 0x01221a83 in JavaCalls::call ()
#37 0x011f67ac in instanceKlass::call_class_initializer_impl ()
#38 0x011f6a6e in instanceKlass::initialize_impl ()
#39 0x011f70c8 in instanceKlass::initialize ()
#40 0x013181f5 in LinkResolver::resolve_static_call ()
#41 0x01319e2b in LinkResolver::resolve_invokestatic ()
#42 0x01319f15 in LinkResolver::resolve_invoke ()
#43 0x0121b66b in InterpreterRuntime::resolve_invoke ()
#44 0x01a4f8cf in ?? ()
#45 0x01a421c7 in ?? ()
#46 0x01a421c7 in ?? ()
#47 0x01a421c7 in ?? ()
#48 0x01a41ff3 in ?? ()
#49 0x01a421c7 in ?? ()
#50 0x01a4216f in ?? ()
#51 0x01a4216f in ?? ()
#52 0x01a3f2cc in ?? ()
#53 0x01222eb2 in JavaCalls::call_helper ()
#54 0x0136a241 in os::os_exception_wrapper ()
#55 0x01221a83 in JavaCalls::call ()
#56 0x0122e6d8 in jni_invoke_static ()
#57 0x0124253d in jni_CallStaticVoidMethod ()
#58 0x00001f31 in JavaMain ()
#59 0x96ee9095 in _pthread_start ()
#60 0x96ee8f52 in thread_start ()
(gdb) thread 3
[Switching to thread 3 (process 43094 thread 0x2103)]
0x96efc796 in __wait4 ()
(gdb) bt
#0 0x96efc796 in __wait4 ()
#1 0x96efc787 in waitpid$UNIX2003 ()
#2 0x0136a856 in os::fork_and_exec ()
#3 0x0145eae4 in VMError::show_message_box ()
#4 0x0145e87d in VMError::report_and_die ()
#5 0x01370b5e in JVM_handle_bsd_signal ()
#6 0x0136a031 in signalHandler ()
#7 <signal handler called>
#8 0x011a43aa in frame::oops_interpreted_do ()
#9 0x011a4aa6 in frame::oops_do_internal ()
#10 0x0141f175 in JavaThread::oops_do ()
#11 0x0141f5ff in Threads::verify ()
#12 0x0142db1c in Universe::verify ()
#13 0x011d1506 in GenCollectedHeap::do_collection ()
#14 0x0113e5e3 in GenCollectorPolicy::satisfy_failed_allocation ()
#15 0x011cecc5 in GenCollectedHeap::satisfy_failed_allocation ()
#16 0x0145f23b in VM_GenCollectForAllocation::doit ()
#17 0x01464125 in VM_Operation::evaluate ()
#18 0x014627fd in VMThread::evaluate_operation ()
#19 0x0146346c in VMThread::loop ()
#20 0x0146375c in VMThread::run ()
#21 0x0136dab5 in java_start ()
#22 0x96ee9095 in _pthread_start ()
#23 0x96ee8f52 in thread_start ()
(gdb) thread 4
[Switching to thread 4 (process 43094 thread 0x2203)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96eea326 in _pthread_cond_wait ()
#2 0x96ee9d0d in pthread_cond_wait$UNIX2003 ()
#3 0x0136aea9 in os::PlatformEvent::park ()
#4 0x013ed57a in ObjectMonitor::wait ()
#5 0x013ed711 in ObjectSynchronizer::wait ()
#6 0x01293a5d in JVM_MonitorWait ()
#7 0x01a4971d in ?? ()
#8 0x01a41ff3 in ?? ()
#9 0x01a41ff3 in ?? ()
#10 0x01a3f2cc in ?? ()
#11 0x01222eb2 in JavaCalls::call_helper ()
#12 0x0136a241 in os::os_exception_wrapper ()
#13 0x012222d2 in JavaCalls::call_virtual ()
#14 0x0122250d in JavaCalls::call_virtual ()
#15 0x0128fefe in thread_entry ()
#16 0x01422671 in JavaThread::thread_main_inner ()
#17 0x0142275e in JavaThread::run ()
#18 0x0136dab5 in java_start ()
#19 0x96ee9095 in _pthread_start ()
#20 0x96ee8f52 in thread_start ()
(gdb) thread 5
[Switching to thread 5 (process 43094 thread 0x2303)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96eea326 in _pthread_cond_wait ()
#2 0x96ee9d0d in pthread_cond_wait$UNIX2003 ()
#3 0x0136aea9 in os::PlatformEvent::park ()
#4 0x013ed57a in ObjectMonitor::wait ()
#5 0x013ed711 in ObjectSynchronizer::wait ()
#6 0x01293a5d in JVM_MonitorWait ()
#7 0x01a4971d in ?? ()
#8 0x01a41ff3 in ?? ()
#9 0x01a421c7 in ?? ()
#10 0x01a421c7 in ?? ()
#11 0x01a3f2cc in ?? ()
#12 0x01222eb2 in JavaCalls::call_helper ()
#13 0x0136a241 in os::os_exception_wrapper ()
#14 0x012222d2 in JavaCalls::call_virtual ()
#15 0x0122250d in JavaCalls::call_virtual ()
#16 0x0128fefe in thread_entry ()
#17 0x01422671 in JavaThread::thread_main_inner ()
#18 0x0142275e in JavaThread::run ()
#19 0x0136dab5 in java_start ()
#20 0x96ee9095 in _pthread_start ()
#21 0x96ee8f52 in thread_start ()
(gdb) thread 6
[Switching to thread 6 (process 43094 thread 0x2603)]
0x96eb8202 in semaphore_wait_trap ()
(gdb) bt
#0 0x96eb8202 in semaphore_wait_trap ()
#1 0x0136d8a0 in check_pending_signals ()
#2 0x0136d9c9 in os::signal_wait ()
#3 0x01367c85 in signal_thread_entry ()
#4 0x01422671 in JavaThread::thread_main_inner ()
#5 0x0142275e in JavaThread::run ()
#6 0x0136dab5 in java_start ()
#7 0x96ee9095 in _pthread_start ()
#8 0x96ee8f52 in thread_start ()
(gdb) thread 7
[Switching to thread 7 (process 43094 thread 0x2703)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96eea326 in _pthread_cond_wait ()
#2 0x96ee9d0d in pthread_cond_wait$UNIX2003 ()
#3 0x0136aea9 in os::PlatformEvent::park ()
#4 0x0134f32f in Monitor::IWait ()
#5 0x0134f5c4 in Monitor::wait ()
#6 0x01148fff in CompileQueue::get ()
#7 0x0114c91c in CompileBroker::compiler_thread_loop ()
#8 0x0141b154 in compiler_thread_entry ()
#9 0x01422671 in JavaThread::thread_main_inner ()
#10 0x0142275e in JavaThread::run ()
#11 0x0136dab5 in java_start ()
#12 0x96ee9095 in _pthread_start ()
#13 0x96ee8f52 in thread_start ()
(gdb) thread 8
[Switching to thread 8 (process 43094 thread 0x2803)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96eea326 in _pthread_cond_wait ()
#2 0x96ee9d0d in pthread_cond_wait$UNIX2003 ()
#3 0x0136aea9 in os::PlatformEvent::park ()
#4 0x0134f32f in Monitor::IWait ()
#5 0x0134f701 in Monitor::wait ()
#6 0x0131cfa4 in LowMemoryDetector::low_memory_detector_thread_entry ()
#7 0x01422671 in JavaThread::thread_main_inner ()
#8 0x0142275e in JavaThread::run ()
#9 0x0136dab5 in java_start ()
#10 0x96ee9095 in _pthread_start ()
#11 0x96ee8f52 in thread_start ()
(gdb) thread 9
[Switching to thread 9 (process 43094 thread 0x2903)]
0x96ebf3ae in __semwait_signal ()
(gdb) bt
#0 0x96ebf3ae in __semwait_signal ()
#1 0x96eea326 in _pthread_cond_wait ()
#2 0x96f0f9f0 in pthread_cond_timedwait$UNIX2003 ()
#3 0x0136c019 in os::PlatformEvent::park ()
#4 0x0136f164 in os::sleep ()
#5 0x0142086f in WatcherThread::run ()
#6 0x0136dab5 in java_start ()
#7 0x96ee9095 in _pthread_start ()
#8 0x96ee8f52 in thread_start ()
On Tue, Aug 4, 2009 at 10:28 PM, Charles Oliver
Nutter<headius at headius.com> wrote:
> The ShowMessageBox option also gave me the option of launching gdb.
> I'm not a gdb expert, but "bt" produced this:
>
> (gdb) bt
> #0 0x96ebf3ae in __semwait_signal ()
> #1 0x96f04398 in pthread_join$UNIX2003 ()
> #2 0x0000668b in ContinueInNewThread0 ()
> #3 0x0000440e in JLI_Launch ()
> #4 0x0000d388 in main ()
>
> Trying now to see if I can view stacks for other threads or something.
>
> On Tue, Aug 4, 2009 at 10:24 PM, Charles Oliver
> Nutter<headius at headius.com> wrote:
>> Here's the output with the verify flags on:
>>
>> ~/projects/jruby ➔ jruby -J-XX:+UnlockDiagnosticVMOptions
>> -J-XX:+VerifyBeforeGC -J-XX:+VerifyAfterGC
>> -J-Djruby.compile.invokedynamic -J-XX:+EnableInvokeDynamic
>> bench/bench_fib_recursive.rb 100
>> [Verifying threads permgen tenured generation def new generation
>> remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyBeforeGC:[Verifying threads permgen tenured generation def new
>> generation remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyAfterGC:[Verifying threads permgen tenured generation def new
>> generation remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyBeforeGC:[Verifying threads permgen tenured generation def new
>> generation remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyAfterGC:[Verifying threads permgen tenured generation def new
>> generation remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyBeforeGC:[Verifying threads permgen tenured generation def new
>> generation remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyAfterGC:[Verifying threads permgen tenured generation def new
>> generation remset ref_proc syms strs zone dict hand C-heap ]
>> VerifyBeforeGC:[Verifying threads #
>>
>> The hs_err file is attached. I do not have pstack on OS X...do you
>> know what the equivalent would be?
>>
>> On Tue, Aug 4, 2009 at 8:56 PM, <Y.S.Ramakrishna at sun.com> wrote:
>>> The crash is in GC. Together with yr conjecture that the crash happens
>>> after a while when you expect the method to have been jitted, this might
>>> imply that the jit is not correctly conveying to GC where the
>>> references in jitted frames are or is not marking cards for the
>>> generational collector correctly, or GC isn't looking at all the
>>> slots it should be looking at or .... (But that is a long assumption.)
>>> The funny part about "running long enough" is that the hs_err log
>>> says the crash occurred after 0 seconds, so the VM is probably
>>> not correctly printing the time it's been running.
>>>
>>> I'd suggest running the JVM with the options -XX:+VerifyBeforeGC and -XX:+VerifyAfterGC
>>> to perhaps catch the problem sooner or get closer to the root cause.
>>> If you run with the additional JVM option -XX:+ShowMessageBoxOnError
>>> you might be able to pstack the process and get some info. (Or
>>> use the hs_err script to decode the stack trace from the hs_err log.)
>>>
>>> -- ramki
>>>
>>> On 08/04/09 18:07, Charles Oliver Nutter wrote:
>>>> I can't seem to catch a break. My local build and Attila's build still
>>>> both crash for me. There's got to be something about my environment
>>>> that's different, eh?
>>>>
>>>> As before, interpreter runs fine. Anything that runs long enough to
>>>> jit seems to crash. I've attached the latest log.
>>>>
>>>> On Tue, Aug 4, 2009 at 7:58 PM, Charles Oliver
>>>> Nutter<headius at headius.com> wrote:
>>>>> On Tue, Aug 4, 2009 at 4:17 PM, Attila Szegedi<szegedia at gmail.com> wrote:
>>>>>> I can now confirm that it indeed does work with jruby-1.4.0dev that
>>>>>> indeed emits invokedynamic.
>>>>>>
>>>>>> The only catch is, it's very slow with invokedynamic, something that
>>>>>> Christian also mentioned.
>>>>> The fact that these numbers are so bad and even degrade shows me that
>>>>> although it's compiling indy, it's not eliminating the handles (which
>>>>> is true, the inlining and handle-walking stuff is not ready yet). So
>>>>> it's got many layers of polymorphic calls, probably some argument
>>>>> boxing and unboxing, and basically a ton more overhead than our
>>>>> heavily tuned non-indy call path. I'm confident in the Hotspot guys :)
>>>>>
>>>>> Just wanted to make sure anyone watching didn't think this was the
>>>>> best it would get...
>>>>>
>>>>>> MacBook-Pro-Ati:jruby-1.4.0dev aszegedi$ bin/jruby --server -J-
>>>>>> Djruby.compile.invokedynamic=true -J-XX:+EnableInvokeDynamic bench/
>>>>>> bench_fib_recursive.rb 15
>>>>>> 2.993000 0.000000 2.993000 ( 2.915000)
>>>>>> 2.720000 0.000000 2.720000 ( 2.720000)
>>>>>> 2.773000 0.000000 2.773000 ( 2.773000)
>>>>>> 2.712000 0.000000 2.712000 ( 2.712000)
>>>>>> 3.057000 0.000000 3.057000 ( 3.057000)
>>>>>> 3.183000 0.000000 3.183000 ( 3.184000)
>>>>>> 3.167000 0.000000 3.167000 ( 3.167000)
>>>>>> 3.145000 0.000000 3.145000 ( 3.145000)
>>>>>> 3.249000 0.000000 3.249000 ( 3.249000)
>>>>>> 3.160000 0.000000 3.160000 ( 3.160000)
>>>>>> 3.104000 0.000000 3.104000 ( 3.104000)
>>>>>> 3.152000 0.000000 3.152000 ( 3.153000)
>>>>>> 3.172000 0.000000 3.172000 ( 3.171000)
>>>>>> 3.111000 0.000000 3.111000 ( 3.111000)
>>>>>> 3.201000 0.000000 3.201000 ( 3.201000)
>>>>>> MacBook-Pro-Ati:jruby-1.4.0dev aszegedi$ bin/jruby --server bench/
>>>>>> bench_fib_recursive.rb 15
>>>>>> 0.523000 0.000000 0.523000 ( 0.481000)
>>>>>> 0.292000 0.000000 0.292000 ( 0.291000)
>>>>>> 0.279000 0.000000 0.279000 ( 0.278000)
>>>>>> 0.283000 0.000000 0.283000 ( 0.283000)
>>>>>> 0.284000 0.000000 0.284000 ( 0.284000)
>>>>>> 0.292000 0.000000 0.292000 ( 0.292000)
>>>>>> 0.289000 0.000000 0.289000 ( 0.289000)
>>>>>> 0.286000 0.000000 0.286000 ( 0.287000)
>>>>>> 0.285000 0.000000 0.285000 ( 0.285000)
>>>>>> 0.285000 0.000000 0.285000 ( 0.285000)
>>>>>> 0.289000 0.000000 0.289000 ( 0.290000)
>>>>>> 0.282000 0.000000 0.282000 ( 0.282000)
>>>>>> 0.290000 0.000000 0.290000 ( 0.290000)
>>>>>> 0.293000 0.000000 0.293000 ( 0.293000)
>>>>>> 0.285000 0.000000 0.285000 ( 0.284000)
>>>>>
>>>>> ------------------------------------------------------------------------
>>>>>
>>>>> _______________________________________________
>>>>> mlvm-dev mailing list
>>>>> mlvm-dev at openjdk.java.net
>>>>> http://mail.openjdk.java.net/mailman/listinfo/mlvm-dev
>>>
>>> _______________________________________________
>>> mlvm-dev mailing list
>>> mlvm-dev at openjdk.java.net
>>> http://mail.openjdk.java.net/mailman/listinfo/mlvm-dev
>>>
>>
>
More information about the mlvm-dev
mailing list