Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 72 additions & 15 deletions crates/stdlib/src/faulthandler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ mod decl {
use crate::vm::{
AsObject, PyObjectRef, PyResult, VirtualMachine,
builtins::PyInt,
function::{ArgIntoBool, ArgIntoFloat, OptionalArg},
function::{ArgIntoBool, OptionalArg},
stdlib::time::pytime,
};
use alloc::sync::Arc;
use core::sync::atomic::{AtomicBool, AtomicI32, Ordering};
Expand Down Expand Up @@ -673,7 +674,8 @@ mod decl {
return;
}
let timeout = Duration::from_micros(guard.timeout_us);
cvar.wait_for(&mut guard, timeout);
// A notification or spurious wake does not mean the deadline passed.
cvar.wait_while_for(&mut guard, |state| !state.cancel, timeout);

// Check if cancelled after wait
if guard.cancel {
Expand Down Expand Up @@ -797,8 +799,9 @@ mod decl {
#[derive(FromArgs)]
#[allow(unused)]
struct DumpTracebackLaterArgs {
#[pyarg(positional, error_msg = "timeout must be a number (int or float)")]
timeout: ArgIntoFloat,
// Clinic converts repeat/exit before interpreting the timeout object.
#[pyarg(any)]
timeout: PyObjectRef,
#[pyarg(any, default = ArgIntoBool::FALSE)]
repeat: ArgIntoBool,
#[pyarg(any, default, py_default = "sys.stderr")]
Expand All @@ -811,20 +814,21 @@ mod decl {

#[pyfunction]
fn dump_traceback_later(args: DumpTracebackLaterArgs, vm: &VirtualMachine) -> PyResult<()> {
let timeout: f64 = args.timeout.into_float();

if timeout <= 0.0 {
let timeout = pytime::from_seconds_object(&args.timeout, pytime::Round::Up, vm)?;
let timeout_us = pytime::Round::Up.divide(timeout, 1_000);
if timeout_us <= 0 {
return Err(vm.new_value_error("timeout must be greater than 0"));
}

let (fd, _file) = get_fd_from_file_opt(args.file, vm)?;

// Convert timeout to microseconds
let timeout_us = (timeout * 1_000_000.0) as u64;
if timeout_us == 0 {
return Err(vm.new_value_error("timeout must be greater than 0"));
pytime::check_thread_timeout(timeout, vm)?;
// CPython also bounds the seconds passed to format_timeout by LONG_MAX.
#[allow(clippy::unnecessary_cast)] // c_long is i32 or i64.
let max_seconds = core::ffi::c_long::MAX as i64;
if timeout_us / 1_000_000 > max_seconds {
return Err(vm.new_overflow_error("timeout value is too large"));
}
let timeout_us = timeout_us as u64;

let (fd, _file) = get_fd_from_file_opt(args.file, vm)?;
let header = format_timeout(timeout_us);

// Snapshot thread frame slots so watchdog can dump tracebacks
Expand Down Expand Up @@ -929,7 +933,7 @@ mod decl {
#[derive(FromArgs)]
#[allow(unused)]
struct RegisterArgs {
#[pyarg(positional)]
#[pyarg(any)]
signum: i32,
#[pyarg(any, default, py_default = "sys.stderr")]
file: OptionalArg<PyObjectRef>,
Expand Down Expand Up @@ -1095,4 +1099,57 @@ mod decl {
suppress_crash_report();
host_faulthandler::raise_exception(args.code, args.flags);
}

#[cfg(test)]
#[cfg(any(unix, windows))]
mod tests {
use super::*;
use std::sync::mpsc;
use std::time::Instant;

#[test]
fn watchdog_notification_does_not_expire_timeout() {
let state = Arc::new((
Mutex::new(WatchdogState {
cancel: false,
fd: 2,
timeout_us: 60_000_000,
max_threads: 100,
repeat: false,
exit: false,
header: String::new(),
#[cfg(feature = "threading")]
thread_frame_slots: Vec::new(),
}),
Condvar::new(),
));
let worker_state = Arc::clone(&state);
let (done_tx, done_rx) = mpsc::channel();
let worker = thread::spawn(move || {
watchdog_thread(worker_state);
done_tx.send(()).unwrap();
});

// Wait for a real waiter so the notification cannot be lost before
// the watchdog starts. No Python frames are exposed to this worker.
let deadline = Instant::now() + Duration::from_secs(5);
let notified = loop {
if state.1.notify_one() {
break true;
}
if Instant::now() >= deadline {
break false;
}
thread::sleep(Duration::from_millis(1));
};
let completed = done_rx.recv_timeout(Duration::from_millis(100));

// Always stop and join before asserting, including failure paths.
state.0.lock().cancel = true;
state.1.notify_all();
worker.join().unwrap();
assert!(notified, "watchdog did not start waiting");
assert!(matches!(completed, Err(mpsc::RecvTimeoutError::Timeout)));
}
}
}
2 changes: 1 addition & 1 deletion crates/vm/src/stdlib/time/pytime.rs
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ fn from_object(object: &PyObject, scale: i64, round: Round, vm: &VirtualMachine)

/// Check the platform wait limit after the caller has handled negative values
/// and any API-specific indefinite-wait sentinel.
pub(crate) fn check_thread_timeout(nanoseconds: i64, vm: &VirtualMachine) -> PyResult<()> {
pub fn check_thread_timeout(nanoseconds: i64, vm: &VirtualMachine) -> PyResult<()> {
if Round::Up.divide(nanoseconds, 1_000) > THREAD_TIMEOUT_MAX_MICROSECONDS {
Err(vm.new_overflow_error("timeout value is too large"))
} else {
Expand Down
201 changes: 199 additions & 2 deletions extra_tests/snippets/stdlib_faulthandler.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,210 @@
import faulthandler
import signal
import struct
import sys
import time

from testutils import assert_raises

# Faulthandler watchdogs use the same signed-nanosecond conversion as sleep.


def rejected_timeout(expected, message, timeout, **kwargs):
try:
with assert_raises(expected) as caught:
faulthandler.dump_traceback_later(timeout, **kwargs)
assert str(caught.exception) == message
finally:
faulthandler.cancel_dump_traceback_later()


# Keyword binding must reach timeout validation before inspecting the file.
try:
with assert_raises(ValueError) as caught:
faulthandler.dump_traceback_later(timeout=0, file=-1)
assert str(caught.exception) == "timeout must be greater than 0"
assert_raises(TypeError, faulthandler.dump_traceback_later)
assert_raises(TypeError, faulthandler.dump_traceback_later, 60, timeout=60)
assert_raises(TypeError, faulthandler.dump_traceback_later, 60, False, -1, False, 1)
finally:
faulthandler.cancel_dump_traceback_later()


# All probes below reject the file before arming a watchdog, including tiny
# positive durations that could expire before a subsequent cancellation.
file_error = "file is not a valid file descriptor"
for timeout in (1e-20, 5e-324, 1e-9):
rejected_timeout(ValueError, file_error, timeout, file=-1)
for timeout in (0, -0.0, -1, -1e-20, -5e-324):
rejected_timeout(ValueError, "timeout must be greater than 0", timeout, file=-1)
rejected_timeout(ValueError, "Invalid value NaN (not a number)", float("nan"), file=-1)
for timeout in (float("inf"), -float("inf"), 10**100, -(10**100), 9223372037):
rejected_timeout(
OverflowError, "timestamp out of range for C PyTime_t", timeout, file=-1
)
for timeout in (None, "60", object()):
rejected_timeout(
TypeError,
"must be real number, not " + type(timeout).__name__,
timeout,
file=-1,
)


calls = []


class TruthValue:
def __init__(self, name, error=None):
self.name = name
self.error = error

def __bool__(self):
calls.append(self.name)
if self.error is not None:
raise self.error
return False


class IndexedTimeout:
def __index__(self):
calls.append("index")
return 60

def __float__(self):
raise AssertionError("timeout must prefer __index__")


class FloatTimeout:
def __float__(self):
calls.append("float")
return 60.0


class InvalidFile:
def fileno(self):
calls.append("fileno")
return -1

def flush(self):
raise AssertionError("invalid descriptor must not be flushed")


for timeout, conversion in ((IndexedTimeout(), "index"), (FloatTimeout(), "float")):
calls.clear()
rejected_timeout(
RuntimeError,
"file.fileno() is not a valid file descriptor",
timeout,
repeat=TruthValue("repeat"),
exit=TruthValue("exit"),
file=InvalidFile(),
)
assert calls == ["repeat", "exit", conversion, "fileno"]

for failing_argument in ("repeat", "exit"):
calls.clear()
failure = RuntimeError(failing_argument + " failed")
rejected_timeout(
RuntimeError,
str(failure),
None,
repeat=TruthValue("repeat", failure if failing_argument == "repeat" else None),
exit=TruthValue("exit", failure if failing_argument == "exit" else None),
file=InvalidFile(),
)
assert calls == (["repeat"] if failing_argument == "repeat" else ["repeat", "exit"])

for timeout, expected, message in (
(0, ValueError, "timeout must be greater than 0"),
(float("nan"), ValueError, "Invalid value NaN (not a number)"),
(float("inf"), OverflowError, "timestamp out of range for C PyTime_t"),
):
calls.clear()
rejected_timeout(
expected,
message,
timeout,
repeat=TruthValue("repeat"),
exit=TruthValue("exit"),
file=InvalidFile(),
)
assert calls == ["repeat", "exit"]


class FailingIndex:
def __index__(self):
raise RuntimeError("index failed")

def __float__(self):
raise AssertionError("failed index must not fall back to float")


class FailingFloat:
def __float__(self):
raise RuntimeError("float failed")


class InvalidIndex:
def __index__(self):
return 60.0

def __float__(self):
raise AssertionError("invalid index must not fall back to float")


rejected_timeout(RuntimeError, "index failed", FailingIndex(), file=-1)
rejected_timeout(RuntimeError, "float failed", FailingFloat(), file=-1)
try:
assert_raises(TypeError, faulthandler.dump_traceback_later, InvalidIndex(), file=-1)
finally:
faulthandler.cancel_dump_traceback_later()


class FloatSubclass(float):
def __float__(self):
raise AssertionError("float subclass override must be bypassed")


class IndexedFloatSubclass(FloatSubclass):
def __index__(self):
return -1


rejected_timeout(ValueError, file_error, FloatSubclass(60), file=-1)
rejected_timeout(
ValueError, "timeout must be greater than 0", IndexedFloatSubclass(60), file=-1
)

# The platform limit and the signed C long seconds limit precede file lookup.
# This is below the signed-nanosecond limit on every tested platform.
if sys.platform == "win32":
rejected_timeout(OverflowError, "timeout value is too large", 4_294_968, file=-1)
elif struct.calcsize("l") == 4:
rejected_timeout(OverflowError, "timeout value is too large", 2**31, file=-1)
else:
rejected_timeout(ValueError, file_error, 9223372036, file=-1)

# Exercise the successful keyword path only with a long timeout, immediately
# cancelled. Wasm targets do not provide the native watchdog thread.
if sys.platform not in ("wasi", "emscripten"):
try:
assert faulthandler.dump_traceback_later(timeout=60) is None
finally:
faulthandler.cancel_dump_traceback_later()


# An invalid file proves register's signum keyword binds without installing
# any signal handler. This API is absent on Windows.
if hasattr(faulthandler, "register") and hasattr(signal, "SIGUSR1"):
with assert_raises(ValueError) as caught:
faulthandler.register(signum=signal.SIGUSR1, file=-1)
assert str(caught.exception) == "file is not a valid file descriptor"


# The watchdog's thread cap is keyword-only and uses Py_ssize_t conversion.
# Its output must distinguish an exact cap from a truncated thread dump.
if sys.platform not in ("wasi", "emscripten"):
import faulthandler

for cap in (None, 1.5):
assert_raises(TypeError, faulthandler.dump_traceback_later, 60, max_threads=cap)
for cap in (sys.maxsize + 1, -sys.maxsize - 2):
Expand Down
Loading