Skip to content

Commit c2bb450

Browse files
youknowdotcodex
youknowdot
and
codex
authored
Preserve native trailing command-line argument bytes (#9494)
Carry arguments and program names as OsString until Python filesystem decoding, and restore the canonical invalid UTF-8 argument test through update_lib. Keep remaining markers and startup adapters unchanged. Assisted-by: OpenAI Codex: model version unavailable Co-authored-by: Codex <codex@openai.com>
1 parent 8051c05 commit c2bb450

7 files changed

Lines changed: 32 additions & 21 deletions

File tree

‎Lib/test/test_cmd_line.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,6 @@ def test_undecodable_code(self):
330330
if not stdout.startswith(pattern):
331331
raise AssertionError("%a doesn't start with %a" % (stdout, pattern))
332332

333-
@unittest.expectedFailure # TODO: RUSTPYTHON
334333
@unittest.skipIf(sys.platform == 'win32',
335334
'Windows has a native unicode API')
336335
def test_invalid_utf8_arg(self):

‎crates/stdlib/src/syslog.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,10 @@ mod syslog {
5353
if let Some(argv) = vm.state.config.settings.argv.first()
5454
&& !argv.is_empty()
5555
{
56+
let argv = vm.fsdecode(argv.clone());
57+
let argv = argv.as_wtf8();
5658
return Some(
57-
PyStr::from(match argv.find('\\') {
59+
PyStr::from(match argv.as_bytes().iter().position(|&b| b == b'\\') {
5860
Some(value) => &argv[value..],
5961
None => argv,
6062
})

‎crates/vm/src/getpath.rs‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
88
use crate::vm::{Paths, Settings};
99
use std::env;
10+
use std::ffi::OsString;
1011
use std::io::{self, Read};
1112
use std::path::{Path, PathBuf};
1213

@@ -622,13 +623,19 @@ fn env_executable_override() -> Option<String> {
622623

623624
/// CALCULATE program_name: `argv[0]`, with the executable suffix added when
624625
/// only the suffixed file exists.
625-
fn program_name() -> String {
626-
let program_name = env::args()
626+
fn program_name() -> OsString {
627+
let program_name = env::args_os()
627628
.next()
628629
.filter(|arg| !arg.is_empty())
629-
.unwrap_or_else(|| platform::DEFAULT_PROGRAM_NAME.to_owned());
630-
if cfg!(windows) && !program_name.to_lowercase().ends_with(".exe") {
631-
let suffixed = format!("{program_name}.exe");
630+
.unwrap_or_else(|| platform::DEFAULT_PROGRAM_NAME.into());
631+
if cfg!(windows)
632+
&& !program_name
633+
.to_string_lossy()
634+
.to_lowercase()
635+
.ends_with(".exe")
636+
{
637+
let mut suffixed = program_name.clone();
638+
suffixed.push(".exe");
632639
if Path::new(&suffixed).is_file() {
633640
return suffixed;
634641
}
@@ -651,7 +658,7 @@ fn get_executable_path() -> Option<PathBuf> {
651658
}
652659
#[cfg(target_arch = "wasm32")]
653660
{
654-
let exec_arg = env::args().next()?;
661+
let exec_arg = env::args_os().next()?;
655662
Some(PathBuf::from(exec_arg))
656663
}
657664
}

‎crates/vm/src/stdlib/sys.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@ pub mod sys {
313313
.settings
314314
.argv
315315
.iter()
316-
.map(|arg| vm.ctx.new_str(arg.clone()).into())
316+
.map(|arg| vm.fsdecode(arg.clone()).into())
317317
.collect()
318318
}
319319

@@ -708,7 +708,7 @@ pub mod sys {
708708

709709
#[pyattr]
710710
fn orig_argv(vm: &VirtualMachine) -> Vec<PyObjectRef> {
711-
env::args().map(|arg| vm.ctx.new_str(arg).into()).collect()
711+
env::args_os().map(|arg| vm.fsdecode(arg).into()).collect()
712712
}
713713

714714
#[pyattr]

‎crates/vm/src/vm/setting.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use std::ffi::OsString;
55
#[derive(Debug, Clone, Default)]
66
pub struct Paths {
77
/// program_name: the name the interpreter was invoked as
8-
pub program_name: String,
8+
pub program_name: OsString,
99
/// sys.executable
1010
pub executable: String,
1111
/// sys._base_executable (original interpreter in venv)
@@ -77,8 +77,8 @@ pub struct Settings {
7777
pub pycache_prefix: Option<OsString>,
7878
// int parse_argv;
7979
// PyWideStringList orig_argv;
80-
/// sys.argv
81-
pub argv: Vec<String>,
80+
/// sys.argv, decoded with the filesystem encoding during sys initialization.
81+
pub argv: Vec<OsString>,
8282

8383
// spell-checker:ignore Xfoo
8484
/// -Xfoo[=bar]

‎src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -252,7 +252,7 @@ fn run_file(vm: &VirtualMachine, scope: Scope, argv0: &str) -> PyResult<u32> {
252252
}
253253

254254
fn cant_open_file(vm: &VirtualMachine, path: &str, err: &std::io::Error) -> PyResult<u32> {
255-
let program_name = &vm.state.config.paths.program_name;
255+
let program_name = vm.fsdecode(vm.state.config.paths.program_name.clone());
256256
let filename_repr = vm.ctx.new_str(path).as_object().repr(vm)?;
257257
let errno = err.raw_os_error().unwrap_or(2);
258258
let strerror = vm::host_env::errno::strerror_string(errno).unwrap_or_default();

‎src/settings.rs‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ use lexopt::Arg::{Long, Short, Value};
55
use lexopt::ValueExt;
66
use rustpython_vm::{Settings, vm::CheckHashPycsMode};
77
use std::env;
8+
use std::ffi::OsString;
89

910
pub enum RunMode {
1011
Script(String),
@@ -112,13 +113,15 @@ file : program read from script file
112113
arg ...: arguments passed to program in sys.argv[1:]
113114
";
114115

115-
fn parse_args() -> Result<(CliArgs, RunMode, Vec<String>), lexopt::Error> {
116+
fn parse_args() -> Result<(CliArgs, RunMode, Vec<OsString>), lexopt::Error> {
116117
let mut args = CliArgs::default();
117118
let mut parser = lexopt::Parser::from_env();
118-
fn argv(argv0: String, mut parser: lexopt::Parser) -> Result<Vec<String>, lexopt::Error> {
119-
core::iter::once(Ok(argv0))
120-
.chain(parser.raw_args()?.map(|arg| arg.string()))
121-
.collect()
119+
fn argv(argv0: String, mut parser: lexopt::Parser) -> Result<Vec<OsString>, lexopt::Error> {
120+
// Arguments after the entry point are filesystem-encoded data, not
121+
// necessarily Unicode. Decode only when constructing Python strings.
122+
Ok(core::iter::once(argv0.into())
123+
.chain(parser.raw_args()?)
124+
.collect())
122125
}
123126
while let Some(arg) = parser.next()? {
124127
match arg {
@@ -171,15 +174,15 @@ fn parse_args() -> Result<(CliArgs, RunMode, Vec<String>), lexopt::Error> {
171174

172175
Long("install-pip") => {
173176
let (mode, argv) = if let Some(val) = parser.optional_value() {
174-
(val.parse()?, vec![val.string()?])
177+
(val.parse()?, vec![val])
175178
} else if let Ok(argv0) = parser.value() {
176179
let mode = argv0.parse()?;
177180
(mode, argv(argv0.string()?, parser)?)
178181
} else {
179182
(
180183
InstallPipMode::Ensurepip,
181184
["ensurepip", "--upgrade", "--default-pip"]
182-
.map(str::to_owned)
185+
.map(OsString::from)
183186
.into(),
184187
)
185188
};

0 commit comments

Comments
 (0)