JRuby thread stuck on IntHashMap.rehash #3394
Description
I have a multi-threaded application running on JRuby 9.0.1.0 (Java 1.8.0_45-b14). The application runs with 8 Ruby threads. One of those threads has been stuck on the IntHashMap.rehash method for the last 13 hours. The remaining threads are all functioning normally. Here is (most of) the stack trace for the thread in question as reported by jstack:
"Ruby-0-Thread-7: /data/apps/my_app/current/vendor/bundle/jruby/2.2.0/gems/my-gem-java/lib/java_executor_simple.rb:22" #21 daemon prio=5 os_prio=0 tid=0x00007f1f585bf800 nid=0x505e runnable [0x00007f1f0db99000]
java.lang.Thread.State: RUNNABLE
at org.jruby.util.collections.IntHashMap.rehash(IntHashMap.java:139)
at org.jruby.util.collections.IntHashMap.put(IntHashMap.java:170)
at org.jruby.java.dispatch.CallableSelector.matchingCallableArityOne(CallableSelector.java:118)
at org.jruby.java.invokers.RubyToJavaInvoker.findCallableArityOne(RubyToJavaInvoker.java:282)
at org.jruby.java.invokers.InstanceMethodInvoker.call(InstanceMethodInvoker.java:98)
at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:139)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:145)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.application.invokeOther31:time(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/application.rb)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.application.RUBY$method$md?$0(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/application.rb:149)
at java.lang.invoke.LambdaForm$DMH/885284298.invokeStatic_L7_L(LambdaForm$DMH)
at java.lang.invoke.LambdaForm$MH/2130192211.invokeExact_MT(LambdaForm$MH)
at org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:131)
at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:181)
at org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:197)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:161)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.application.invokeOther3:md?(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/application.rb)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.application.RUBY$block$process$2(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/application.rb:182)
at java.lang.invoke.LambdaForm$DMH/885284298.invokeStatic_L7_L(LambdaForm$DMH)
at java.lang.invoke.LambdaForm$MH/2130192211.invokeExact_MT(LambdaForm$MH)
at org.jruby.runtime.CompiledIRBlockBody.commonYieldPath(CompiledIRBlockBody.java:70)
at org.jruby.runtime.IRBlockBody.yieldSpecific(IRBlockBody.java:67)
at org.jruby.runtime.Block.yieldSpecific(Block.java:116)
at org.jruby.ir.runtime.IRRuntimeHelpers.yieldSpecific(IRRuntimeHelpers.java:457)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.my_method.RUBY$method$cache$0(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/bdcc.rb:19)
at java.lang.invoke.LambdaForm$DMH/664223387.invokeStatic_L8_L(LambdaForm$DMH)
at java.lang.invoke.LambdaForm$MH/684822005.invokeExact_MT(LambdaForm$MH)
at org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:146)
at org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:216)
at org.jruby.runtime.callsite.CachingCallSite.callBlock(CachingCallSite.java:203)
at org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:209)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.application.invokeOther8:cache(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/application.rb)
at data.apps.my_app.releases.my_app_minus_03751e0c_minus_1caa0727.lib.my_app.application.RUBY$block$process$1(/data/apps/my_app/releases/my_app-03751e0c-1caa0727/lib/my_app/application.rb:176)
at java.lang.invoke.LambdaForm$DMH/885284298.invokeStatic_L7_L(LambdaForm$DMH)
at java.lang.invoke.LambdaForm$MH/2130192211.invokeExact_MT(LambdaForm$MH)
at org.jruby.runtime.CompiledIRBlockBody.commonYieldPath(CompiledIRBlockBody.java:70)
at org.jruby.runtime.IRBlockBody.doYield(IRBlockBody.java:140)
at org.jruby.runtime.BlockBody.yield(BlockBody.java:77)
I cannot yet reliably reproduce this, but it does reliably occur once the app has been running for a few hours.