Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .cspell.dict/python-more.txt
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ PYTHONHASHSEED
PYTHONHOME
PYTHONINSPECT
PYTHONINTMAXSTRDIGITS
PYTHONIOENCODING
PYTHONNODEBUGRANGES
PYTHONNOUSERSITE
PYTHONOPTIMIZE
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_tarfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -2494,8 +2494,6 @@ def test_test_command_invalid_file(self):
finally:
os_helper.unlink(tmpname)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_list_command(self):
for tar_name in testtarnames:
with support.captured_stdout() as t:
Expand All @@ -2507,8 +2505,6 @@ def test_list_command(self):
PYTHONIOENCODING='ascii')
self.assertEqual(out, expected)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_list_command_verbose(self):
for tar_name in testtarnames:
with support.captured_stdout() as t:
Expand Down
1 change: 1 addition & 0 deletions crates/vm/Lib/core_modules/encodings_ascii.py
51 changes: 36 additions & 15 deletions crates/vm/src/vm/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -261,17 +261,28 @@ impl VirtualMachine {
Ok(())
}

fn import_utf8_encodings(&mut self) -> PyResult<()> {
fn import_ascii_utf8_encodings(&mut self) -> PyResult<()> {
import::import_frozen(self, "codecs")?;
// FIXME: See corresponding part of `core_frozen_inits`
// let encoding_module_name = if cfg!(feature = "freeze-stdlib") {
// "encodings.utf_8"
// } else {
// "encodings_utf_8"
// };
let encoding_module_name = "encodings_utf_8";
let encoding_module = import::import_frozen(self, encoding_module_name)?;
let getregentry = encoding_module.get_attr("getregentry", self)?;

// Use dotted names when freeze-stdlib is enabled (modules come from Lib/encodings/),
// otherwise use underscored names (modules come from core_modules/).
let (ascii_module_name, utf8_module_name) = if cfg!(feature = "freeze-stdlib") {
("encodings.ascii", "encodings.utf_8")
} else {
("encodings_ascii", "encodings_utf_8")
};

// Register ascii encoding
let ascii_module = import::import_frozen(self, ascii_module_name)?;
let getregentry = ascii_module.get_attr("getregentry", self)?;
let codec_info = getregentry.call((), self)?;
self.state
.codec_registry
.register_manual("ascii", codec_info.try_into_value(self)?)?;

// Register utf-8 encoding
let utf8_module = import::import_frozen(self, utf8_module_name)?;
let getregentry = utf8_module.get_attr("getregentry", self)?;
let codec_info = getregentry.call((), self)?;
self.state
.codec_registry
Expand All @@ -298,7 +309,7 @@ impl VirtualMachine {
#[cfg(not(feature = "threading"))]
import::import_frozen(self, "_thread")?;
let importlib = import::init_importlib_base(self)?;
self.import_utf8_encodings()?;
self.import_ascii_utf8_encodings()?;

#[cfg(any(not(target_arch = "wasm32"), target_os = "wasi"))]
{
Expand Down Expand Up @@ -327,17 +338,25 @@ impl VirtualMachine {
let line_buffering = buffered_stdio && (isatty || fd == 2);

let newline = if cfg!(windows) { None } else { Some("\n") };
// stderr uses backslashreplace error handler
let errors: Option<&str> = if fd == 2 {
let encoding = self.state.config.settings.stdio_encoding.as_deref();
// stderr always uses backslashreplace (ignores stdio_errors)
let errors = if fd == 2 {
Some("backslashreplace")
} else {
None
self.state.config.settings.stdio_errors.as_deref()
};

let stdio = self.call_method(
&io,
"TextIOWrapper",
(buf, (), errors, newline, line_buffering, write_through),
(
buf,
encoding,
errors,
newline,
line_buffering,
write_through,
),
)?;
let mode = if write { "w" } else { "r" };
stdio.set_attr("mode", self.ctx.new_str(mode), self)?;
Expand Down Expand Up @@ -1007,6 +1026,8 @@ pub fn resolve_frozen_alias(name: &str) -> &str {
match name {
"_frozen_importlib" => "importlib._bootstrap",
"_frozen_importlib_external" => "importlib._bootstrap_external",
"encodings_ascii" => "encodings.ascii",
"encodings_utf_8" => "encodings.utf_8",
_ => name,
}
}
Expand Down
8 changes: 6 additions & 2 deletions crates/vm/src/vm/setting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,11 @@ pub struct Settings {
/// -u, PYTHONUNBUFFERED=x
pub buffered_stdio: bool,

// wchar_t *stdio_encoding;
/// PYTHONIOENCODING - stdio encoding
pub stdio_encoding: Option<String>,
/// PYTHONIOENCODING - stdio error handler
pub stdio_errors: Option<String>,
pub utf8_mode: u8,
// wchar_t *stdio_errors;
/// --check-hash-based-pycs
pub check_hash_pycs_mode: CheckHashPycsMode,

Expand Down Expand Up @@ -197,6 +199,8 @@ impl Default for Settings {
buffered_stdio: true,
check_hash_pycs_mode: CheckHashPycsMode::Default,
allow_external_library: cfg!(feature = "importlib"),
stdio_encoding: None,
stdio_errors: None,
utf8_mode: 1,
int_max_str_digits: 4300,
#[cfg(feature = "flame-it")]
Expand Down
17 changes: 17 additions & 0 deletions src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,23 @@ pub fn parse_opts() -> Result<(Settings, RunMode), lexopt::Error> {
settings.code_debug_ranges = false;
}

// Parse PYTHONIOENCODING=encoding[:errors]
if let Some(val) = get_env("PYTHONIOENCODING")
&& let Some(val_str) = val.to_str()
&& !val_str.is_empty()
{
if let Some((enc, err)) = val_str.split_once(':') {
if !enc.is_empty() {
settings.stdio_encoding = Some(enc.to_owned());
}
if !err.is_empty() {
settings.stdio_errors = Some(err.to_owned());
}
} else {
settings.stdio_encoding = Some(val_str.to_owned());
}
}

if settings.dev_mode {
settings.warnoptions.push("default".to_owned());
settings.faulthandler = true;
Expand Down
Loading