Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 56 additions & 23 deletions core/src/main/java/org/jruby/RubyModule.java
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArraySet;
import java.util.concurrent.atomic.AtomicReferenceFieldUpdater;
import java.util.function.Consumer;

import jnr.constants.Constant;
Expand All @@ -76,9 +77,8 @@
import org.jruby.api.JRubyAPI;
import org.jruby.api.Warn;
import org.jruby.embed.Extension;
import org.jruby.exceptions.LoadError;
import org.jruby.exceptions.RaiseException;
import org.jruby.exceptions.RuntimeError;
import org.jruby.ext.thread.Mutex;
import org.jruby.internal.runtime.AbstractIRMethod;
import org.jruby.internal.runtime.methods.AliasMethod;
import org.jruby.internal.runtime.methods.AttrReaderMethod;
Expand Down Expand Up @@ -321,7 +321,7 @@ public IRubyObject hasAutoload(ThreadContext context, String idString, boolean i
if (autoload == null) return context.nil;

// autoload has been completed
if (autoload.getValue() != null) return context.nil;
if (autoload.getValue(context) != null) return context.nil;

file = autoload.getPath();

Expand Down Expand Up @@ -5487,7 +5487,8 @@ public IRubyObject getConstantWithAutoload(ThreadContext context, String name, I
final RubyModule.Autoload autoload = getAutoloadMap().get(name);

if (autoload == null) return null;
if (autoload.getValue() != null) return autoload.getValue();
IRubyObject value = autoload.getValue(context);
if (value != null) return value;

autoload.load(context);
continue;
Expand Down Expand Up @@ -5849,7 +5850,7 @@ public boolean autoloadingValue(ThreadContext context, String name) {
final Autoload autoload = getAutoloadMap().get(name);

// Has autoload not been evacuated and is in progress on this thread and has updated the value?
return autoload != null && autoload.isSelf(context) && autoload.getValue() != null;
return autoload != null && autoload.isSelf(context) && autoload.getValue(context) != null;
}

@CRuby(value = "check_autoload_required", file = "variable.c")
Expand Down Expand Up @@ -6355,7 +6356,7 @@ protected final void defineAutoload(String symbol, RubyString path) {
*/
protected final void defineAutoload(ThreadContext context, String symbol, RubyString path) {
final Autoload existingAutoload = getAutoloadMap().get(symbol);
if (existingAutoload == null || existingAutoload.getValue() == null) {
if (existingAutoload == null || existingAutoload.getValue(context) == null) {
storeConstant(context, symbol, RubyObject.UNDEF, false, context.getFile(), context.getLine());
RubyStackTraceElement caller = context.getSingleBacktrace();
getAutoloadMapForWrite().put(symbol, new Autoload(symbol, path, caller.getFileName(), caller.getLineNumber()));
Expand All @@ -6373,7 +6374,7 @@ protected final IRubyObject finishAutoload(String name) {
protected final IRubyObject finishAutoload(ThreadContext context, String name) {
final Autoload autoload = getAutoloadMap().get(name);
if ( autoload == null ) return null;
final IRubyObject value = autoload.getValue();
final IRubyObject value = autoload.getValue(context);

if (value != null && value != UNDEF) {
storeConstant(context, name, value, autoload.hidden, autoload.getFile(), autoload.getLine());
Expand Down Expand Up @@ -6594,6 +6595,8 @@ public final class Autoload implements SourceLocation {
private final RubyString path;
public String file;
public int line;
// Held while loading, created by the first load; a Thread::Mutex so waiting fibers go through the scheduler
private volatile Mutex loading;

Autoload(String symbol, RubyString path, String file, int line) {
this.symbol = symbol;
Expand All @@ -6614,18 +6617,27 @@ public boolean isHidden() {
return hidden;
}

// Returns an object for the constant if the caller is the autoloading thread.
// Otherwise, try to start autoloading and returns the defined object by autoload.
synchronized IRubyObject load(ThreadContext context) {
if (this.ctx == null) {
this.ctx = context;
} else if (isSelf(context)) {
return getValue();
// Returns the constant's value if the caller is the one autoloading it. Otherwise waits for a load in
// progress in another thread or fiber, loads if still needed, and returns the value.
@CRuby(value = "rb_autoload_load", file = "variable.c")
IRubyObject load(ThreadContext context) {
// a constant referenced while its own autoload runs; we already hold the lock
if (isSelf(context)) return getValue(context);

Mutex loading = loading(context);
loading.lock(context);
try {
return loadLocked(context);
} finally {
loading.unlock(context);
}
}

private IRubyObject loadLocked(ThreadContext context) {
// we hold the lock, so we are the loader now
this.ctx = context;

try {
// This method needs to be synchronized for removing Autoload
// from autoloadMap when it's loaded.
LoadService loadService = loadService(context);
String[] loading = new String[1];
// If another thread is still requiring the file, autoloadRequire waits for it.
Expand All @@ -6639,17 +6651,27 @@ synchronized IRubyObject load(ThreadContext context) {
// Do not finish autoloading by cyclic autoload
finishAutoload(context, symbol);
}
} catch (LoadError | RuntimeError lre) {
// reset ctx to null for a future attempt to load
} catch (Throwable t) {
// whatever failed the load, the next attempt starts over, in this thread or another
this.ctx = null;
throw lre;
throw t;
}

return getValue();
return getValue(context);
}

@CRuby(value = "autoload_load_needed", file = "variable.c", note = "lazy mutex creation")
private Mutex loading(ThreadContext context) {
Mutex loading = this.loading;
if (loading != null) return loading;

// a plain, visible Mutex like MRI's; only require locks are hidden, as thread shields are
LOADING_UPDATER.compareAndSet(this, null, new Mutex(context.runtime, context.runtime.getMutex()));
return this.loading;
}

// Update an object for the constant if the caller is the autoloading thread.
synchronized boolean setConstant(ThreadContext ctx, IRubyObject newValue, boolean hidden, String file, int line) {
boolean setConstant(ThreadContext ctx, IRubyObject newValue, boolean hidden, String file, int line) {
boolean isSelf = isSelf(ctx);

if (isSelf) {
Expand All @@ -6666,8 +6688,16 @@ synchronized boolean setConstant(ThreadContext ctx, IRubyObject newValue, boolea
return isSelf;
}

// Returns an object for the constant defined by autoload.
synchronized IRubyObject getValue() {
// Returns an object for the constant defined by autoload, once any load by another thread or fiber is
// done, since the value is set as soon as the constant is defined, before the file finishes loading.
IRubyObject getValue(ThreadContext context) {
Mutex loading = this.loading;

if (loading != null && loading.isLocked() && !isSelf(context) && !loading.isOwned()) {
loading.lock(context);
loading.unlock(context);
}

return value;
}

Expand All @@ -6680,6 +6710,9 @@ private boolean isSelf(ThreadContext rhs) {
ThreadContext ctx = this.ctx;
return ctx != null && ctx.getThread() == rhs.getThread();
}

private static final AtomicReferenceFieldUpdater<Autoload, Mutex> LOADING_UPDATER =
AtomicReferenceFieldUpdater.newUpdater(Autoload.class, Mutex.class, "loading");
}

/**
Expand Down
15 changes: 14 additions & 1 deletion core/src/main/java/org/jruby/ext/thread/Mutex.java
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,15 @@ public Mutex(Ruby runtime, RubyClass type) {
super(runtime, type);
}

private Mutex(Ruby runtime, RubyClass type, boolean useObjectSpace) {
super(runtime, type, useObjectSpace);
}

/** A Mutex for runtime internals, kept out of ObjectSpace like MRI's hidden mutexes. */
public static Mutex newInternal(Ruby runtime) {
return new Mutex(runtime, runtime.getMutex(), false);
}

public static RubyClass setup(ThreadContext context, RubyClass Thread, RubyClass Object) {
return (RubyClass) Object.setConstant(context, "Mutex",
Thread.defineClassUnder(context, "Mutex", Object, Mutex::new).reifiedClass(Mutex.class).defineMethods(context, Mutex.class));
Expand Down Expand Up @@ -339,7 +348,11 @@ public IRubyObject synchronize(ThreadContext context, Block block) {

@JRubyMethod(name = "owned?")
public IRubyObject owned_p(ThreadContext context) {
return asBoolean(context, lock.isHeldByCurrentThread());
return asBoolean(context, isOwned());
}

public boolean isOwned() {
return lock.isHeldByCurrentThread();
}

private void checkRelocking(ThreadContext context) {
Expand Down
33 changes: 14 additions & 19 deletions core/src/main/java/org/jruby/runtime/load/LoadService.java
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Function;
import java.util.jar.JarFile;
import java.util.regex.Matcher;
Expand All @@ -57,13 +56,13 @@
import org.jruby.RubyModule;
import org.jruby.RubyString;
import org.jruby.RubySymbol;
import org.jruby.RubyThread;
import org.jruby.anno.CRuby;
import org.jruby.anno.JRubyMethod;
import org.jruby.exceptions.JumpException;
import org.jruby.exceptions.RaiseException;
import org.jruby.exceptions.Unrescuable;
import org.jruby.ext.rbconfig.RbConfigLibrary;
import org.jruby.ext.thread.Mutex;
import org.jruby.platform.Platform;
import org.jruby.runtime.ThreadContext;
import org.jruby.runtime.builtin.IRubyObject;
Expand Down Expand Up @@ -420,7 +419,10 @@ private enum RequireState {
final RequireLocks requireLocks = new RequireLocks();

final class RequireLocks {
final class RequireLock extends ReentrantLock {
// A Mutex like MRI's thread shield, so a fiber waiting for another's load goes through the fiber scheduler
@CRuby(value = "rb_thread_shield_new", file = "thread.c")
final class RequireLock {
final Mutex mutex = Mutex.newInternal(runtime);
volatile boolean destroyed;
}
final ConcurrentHashMap<String, RequireLock> pool;
Expand Down Expand Up @@ -448,7 +450,6 @@ private RequireState lock(
RequireState defaultResult,
Function<String, RequireState> ifLocked) {
ThreadContext currentContext = runtime.getCurrentContext();
RubyThread thread = currentContext.getThread();

// loop until require fails for us or succeeds for another thread
while (true) {
Expand All @@ -460,23 +461,24 @@ private RequireState lock(

// If lock is new, lock and return LOCKED
lock = pool.computeIfAbsent(path, (name) -> {
thread.lock(newLock);
// a fresh mutex, so this always succeeds, and never blocks inside computeIfAbsent
newLock.mutex.tryLock(currentContext);
return newLock;
});

RequireState state = null;
if (lock == newLock) {
// Lock is ours, run ifLocked and then clean up
try {
return state = executeAndClearLock(path, ifLocked, thread, lock);
return state = executeAndClearLock(currentContext, path, ifLocked, lock);
} finally {
// failed load, remove our lock and let other threads fight it out
if (state == null) pool.remove(path);
}
}
}

if (lock.isHeldByCurrentThread()) {
if (lock.mutex.isOwned()) {
// we hold the lock, which means we're re-locking for the same file; warn about this
if (circularRequireWarning && runtime.isVerbose()) {
warnCircularRequire(path);
Expand All @@ -486,17 +488,10 @@ private RequireState lock(
}

// Other thread holds the lock, wait to acquire
while (true) {
try {
thread.lockInterruptibly(lock);
break;
} catch (InterruptedException ie) {
currentContext.pollThreadEvents();
}
}
lock.mutex.lock(currentContext);

// Lock has been acquired, confirm other thread has completed and return default
thread.unlock(lock);
lock.mutex.unlock(currentContext);
if (lock.destroyed) {
return defaultResult;
}
Expand All @@ -505,7 +500,7 @@ private RequireState lock(
}
}

private RequireState executeAndClearLock(String path, Function<String, RequireState> ifLocked, RubyThread thread, RequireLock lock) {
private RequireState executeAndClearLock(ThreadContext context, String path, Function<String, RequireState> ifLocked, RequireLock lock) {
boolean destroyLock = false;
try {
RequireState state = ifLocked.apply(path);
Expand All @@ -521,7 +516,7 @@ private RequireState executeAndClearLock(String path, Function<String, RequireSt
lock.destroyed = true;
pool.remove(path);
}
thread.unlock(lock);
lock.mutex.unlock(context);
}
}
}
Expand Down Expand Up @@ -737,7 +732,7 @@ protected LibrarySearcher.FoundLibrary searchForLoad(String file) {
*/
public boolean isRequiring(String key) {
RequireLocks.RequireLock lock = requireLocks.pool.get(key);
return lock != null && lock.isHeldByCurrentThread();
return lock != null && lock.mutex.isOwned();
}

public boolean featureAlreadyLoaded(String feature) {
Expand Down
3 changes: 3 additions & 0 deletions spec/ruby/core/kernel/fixtures/require_fiber_yield.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
ScratchPad << :loading
Fiber.yield
ScratchPad << :loaded
50 changes: 50 additions & 0 deletions spec/ruby/core/kernel/require_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -59,3 +59,53 @@
Kernel.public_methods(false).should.include?(:require)
end
end

describe "Kernel#require of a file another Fiber is still requiring" do
before :each do
@path = fixture(__FILE__, "require_fiber_yield.rb")
@fibers = []
ScratchPad.record []
end

after :each do
@fibers.each { |fiber| fiber.resume while fiber.alive? }
Fiber.set_scheduler(nil)
$LOADED_FEATURES.delete(@path)
end

it "raises a ThreadError without a Fiber scheduler" do
@fibers << Fiber.new { require @path }
@fibers.last.resume

-> { require @path }.should.raise(ThreadError, "deadlock; lock already owned by another fiber belonging to the same thread")
end

describe "with a Fiber scheduler" do
require_relative '../fiber/fixtures/scheduler'

before :each do
Fiber.set_scheduler(FiberSpecs::LoggingScheduler.new)
end

it "blocks through the scheduler until the other Fiber finishes" do
@fibers << loader = Fiber.schedule { require @path }
ScratchPad.recorded.should == [:loading]

result = nil
@fibers << waiter = Fiber.schedule { result = require @path }
event = Fiber.scheduler.events.last
event[:event].should == :block
event[:fiber].should.equal?(waiter)

loader.resume
ScratchPad.recorded.should == [:loading, :loaded]
event = Fiber.scheduler.events.last
event[:event].should == :unblock
event[:fiber].should.equal?(loader)
event[:args].last.should.equal?(waiter)

waiter.resume
result.should == false
end
end
end
Loading
Loading