Skip to content

[3.7] bpo-37369: Fix venv and test symlinking #14456

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zooba merged 2 commits into from
Jun 28, 2019
Merged

[3.7] bpo-37369: Fix venv and test symlinking #14456

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77594f6
Fix venv and test symlinking
zooba Jun 28, 2019
228a3a8
Improve check for unlinkable venv
zooba Jun 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 18 additions & 6 deletions Lib/test/test_platform.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,24 @@ def test_architecture(self):

@support.skip_unless_symlink
def test_architecture_via_symlink(self): # issue3762
if sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
real = _winapi.GetModuleFileName(0)
else:
real = os.path.realpath(sys.executable)
link = os.path.abspath(support.TESTFN)
os.symlink(real, link)

# On Windows, the EXE needs to know where pythonXY.dll and *.pyd is at
# so we add the directory to the path, PYTHONHOME and PYTHONPATH.
env = None
if sys.platform == "win32":
env = {k.upper(): os.environ[k] for k in os.environ}
env["PATH"] = "{};{}".format(
os.path.dirname(sys.executable), env.get("PATH", ""))
env["PYTHONHOME"] = os.path.dirname(sys.executable)
os.path.dirname(real), env.get("PATH", ""))
env["PYTHONHOME"] = os.path.dirname(real)
if sysconfig.is_python_build(True):
env["PYTHONPATH"] = os.path.dirname(os.__file__)

Expand All @@ -40,11 +50,8 @@ def get(python, env=None):
.format(p.returncode))
return r

real = os.path.realpath(sys.executable)
link = os.path.abspath(support.TESTFN)
os.symlink(real, link)
try:
self.assertEqual(get(real), get(link, env=env))
self.assertEqual(get(sys.executable), get(link, env=env))
finally:
os.remove(link)

Expand Down Expand Up @@ -280,6 +287,11 @@ def test_libc_ver(self):
os.path.exists(sys.executable+'.exe'):
# Cygwin horror
executable = sys.executable + '.exe'
elif sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
executable = _winapi.GetModuleFileName(0)
else:
executable = sys.executable
res = platform.libc_ver(executable)
Expand Down
17 changes: 12 additions & 5 deletions Lib/test/test_sysconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,16 +233,26 @@ def test_get_scheme_names(self):

@skip_unless_symlink
def test_symlink(self):
if sys.platform == "win32" and not os.path.exists(sys.executable):
# App symlink appears to not exist, but we want the
# real executable here anyway
import _winapi
real = _winapi.GetModuleFileName(0)
else:
real = os.path.realpath(sys.executable)
link = os.path.abspath(TESTFN)
os.symlink(real, link)

# On Windows, the EXE needs to know where pythonXY.dll is at so we have
# to add the directory to the path.
env = None
if sys.platform == "win32":
env = {k.upper(): os.environ[k] for k in os.environ}
env["PATH"] = "{};{}".format(
os.path.dirname(sys.executable), env.get("PATH", ""))
os.path.dirname(real), env.get("PATH", ""))
# Requires PYTHONHOME as well since we locate stdlib from the
# EXE path and not the DLL path (which should be fixed)
env["PYTHONHOME"] = os.path.dirname(sys.executable)
env["PYTHONHOME"] = os.path.dirname(real)
if sysconfig.is_python_build(True):
env["PYTHONPATH"] = os.path.dirname(os.__file__)

Expand All @@ -258,9 +268,6 @@ def get(python, env=None):
self.fail('Non-zero return code {0} (0x{0:08X})'
.format(p.returncode))
return out, err
real = os.path.realpath(sys.executable)
link = os.path.abspath(TESTFN)
os.symlink(real, link)
try:
self.assertEqual(get(real), get(link, env))
finally:
Expand Down
13 changes: 12 additions & 1 deletion Lib/test/test_venv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,12 @@ def setUp(self):
self.include = 'include'
executable = getattr(sys, '_base_executable', sys.executable)
self.exe = os.path.split(executable)[-1]
if (sys.platform == 'win32'
and os.path.lexists(executable)
and not os.path.exists(executable)):
self.cannot_link_exe = True
else:
self.cannot_link_exe = False

def tearDown(self):
rmtree(self.env_dir)
Expand Down Expand Up @@ -248,7 +254,12 @@ def test_symlinking(self):
# symlinked to 'python3.3' in the env, even when symlinking in
# general isn't wanted.
if usl:
self.assertTrue(os.path.islink(fn))
if self.cannot_link_exe:
# Symlinking is skipped when our executable is already a
# special app symlink
self.assertFalse(os.path.islink(fn))
else:
self.assertTrue(os.path.islink(fn))

# If a venv is created from a source build and that venv is used to
# run the test, the pyvenv.cfg in the venv created in the test will
Expand Down
89 changes: 54 additions & 35 deletions Lib/venv/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,47 +155,66 @@ def create_configuration(self, context):
f.write('include-system-site-packages = %s\n' % incl)
f.write('version = %d.%d.%d\n' % sys.version_info[:3])

def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
"""
force_copy = not self.symlinks
if not force_copy:
try:
if not os.path.islink(dst): # can't link to itself!
if os.name != 'nt':
def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
"""
force_copy = not self.symlinks
if not force_copy:
try:
if not os.path.islink(dst): # can't link to itself!
if relative_symlinks_ok:
assert os.path.dirname(src) == os.path.dirname(dst)
os.symlink(os.path.basename(src), dst)
else:
os.symlink(src, dst)
except Exception: # may need to use a more specific exception
logger.warning('Unable to symlink %r to %r', src, dst)
force_copy = True
if force_copy:
shutil.copyfile(src, dst)
else:
def symlink_or_copy(self, src, dst, relative_symlinks_ok=False):
"""
Try symlinking a file, and if that fails, fall back to copying.
"""
bad_src = os.path.lexists(src) and not os.path.exists(src)
if self.symlinks and not bad_src and not os.path.islink(dst):
try:
if relative_symlinks_ok:
assert os.path.dirname(src) == os.path.dirname(dst)
os.symlink(os.path.basename(src), dst)
else:
os.symlink(src, dst)
except Exception: # may need to use a more specific exception
logger.warning('Unable to symlink %r to %r', src, dst)
force_copy = True
if force_copy:
if os.name == 'nt':
# On Windows, we rewrite symlinks to our base python.exe into
# copies of venvlauncher.exe
basename, ext = os.path.splitext(os.path.basename(src))
srcfn = os.path.join(os.path.dirname(__file__),
"scripts",
"nt",
basename + ext)
# Builds or venv's from builds need to remap source file
# locations, as we do not put them into Lib/venv/scripts
if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
if basename.endswith('_d'):
ext = '_d' + ext
basename = basename[:-2]
if basename == 'python':
basename = 'venvlauncher'
elif basename == 'pythonw':
basename = 'venvwlauncher'
src = os.path.join(os.path.dirname(src), basename + ext)
else:
src = srcfn
if not os.path.exists(src):
logger.warning('Unable to copy %r', src)
return
except Exception: # may need to use a more specific exception
logger.warning('Unable to symlink %r to %r', src, dst)

# On Windows, we rewrite symlinks to our base python.exe into
# copies of venvlauncher.exe
basename, ext = os.path.splitext(os.path.basename(src))
srcfn = os.path.join(os.path.dirname(__file__),
"scripts",
"nt",
basename + ext)
# Builds or venv's from builds need to remap source file
# locations, as we do not put them into Lib/venv/scripts
if sysconfig.is_python_build(True) or not os.path.isfile(srcfn):
if basename.endswith('_d'):
ext = '_d' + ext
basename = basename[:-2]
if basename == 'python':
basename = 'venvlauncher'
elif basename == 'pythonw':
basename = 'venvwlauncher'
src = os.path.join(os.path.dirname(src), basename + ext)
else:
src = srcfn
if not os.path.exists(src):
if not bad_src:
logger.warning('Unable to copy %r', src)
return

shutil.copyfile(src, dst)

Expand Down