Fix unicode/str inconsistency in Python 2 (theskumar#177)

bbc2 · Saurabh Kumar · commit 62df65a63e5d · 2019-05-12T16:10:27.000+05:30
This encodes environment keys and values for `os.environ` in Python 2
and fixes the following issues:

* Values loaded from `.env` being `unicode` whereas regular values being
  `str` (i.e. `bytes`) in Python 2.
* Error when loading `.env` with UTF-8 characters in Python 2.
* `# type: ignore` hiding the issue from the type checker.

`sys.getfilesystemencoding()` is used because it seems to match the
encoding used for environment variables with Python 2.  Since it can
apparently return `None`, we fall back to a default encoding (UTF-8
here) if that's the case.

I tested this on Windows where `"mbcs"` is the file system encoding.
Hopefully, this will also work on other systems.
diff --git a/README.md b/README.md
@@ -300,7 +300,10 @@ Unreleased
 -----
 
 - Add type hints and expose them to users ([@qnighy])([#172])
-- `load_dotenv` and `dotenv_values` now accepts `encoding` paramater, defaults to `None` ([@theskumar])([@earlbread]) (#161)
+- `load_dotenv` and `dotenv_values` now accept an `encoding` parameter, defaults to `None`
+  ([@theskumar])([@earlbread])([#161])
+- Fix `str`/`unicode` inconsistency in Python 2: values are always `str` now. ([@bbc2])([#121])
+- Fix Unicode error in Python 2, introduced in 0.10.0. ([@bbc2])([#176])
 
 0.10.1
 -----
@@ -417,6 +420,8 @@ Unreleased
 [#148]: https://github.com/theskumar/python-dotenv/issues/148
 [#158]: https://github.com/theskumar/python-dotenv/issues/158
 [#172]: https://github.com/theskumar/python-dotenv/issues/172
+[#121]: https://github.com/theskumar/python-dotenv/issues/121
+[#176]: https://github.com/theskumar/python-dotenv/issues/176
 
 [@asyncee]: https://github.com/asyncee
 [@greyli]: https://github.com/greyli
diff --git a/src/dotenv/compat.py b/src/dotenv/compat.py
@@ -1,10 +1,8 @@
-from typing import Text, Type
 import sys
+
 if sys.version_info >= (3, 0):
     from io import StringIO  # noqa
 else:
     from StringIO import StringIO  # noqa
 
 PY2 = sys.version_info[0] == 2  # type: bool
-WIN = sys.platform.startswith('win')  # type: bool
-text_type = Text  # type: Type[Text]
diff --git a/src/dotenv/main.py b/src/dotenv/main.py
@@ -15,7 +15,7 @@
 from collections import OrderedDict
 from contextlib import contextmanager
 
-from .compat import StringIO, PY2, WIN, text_type
+from .compat import StringIO, PY2
 
 if TYPE_CHECKING:  # pragma: no cover
     if sys.version_info >= (3, 6):
@@ -109,6 +109,17 @@ def parse_stream(stream):
         yield binding
 
 
+def to_env(text):
+    # type: (Text) -> str
+    """
+    Encode a string the same way whether it comes from the environment or a `.env` file.
+    """
+    if PY2:
+        return text.encode(sys.getfilesystemencoding() or "utf-8")
+    else:
+        return text
+
+
 class DotEnv():
 
     def __init__(self, dotenv_path, verbose=False, encoding=None):
@@ -156,13 +167,7 @@ def set_as_environment_variables(self, override=False):
         for k, v in self.dict().items():
             if k in os.environ and not override:
                 continue
-            # With Python2 on Windows, force environment variables to str to avoid
-            # "TypeError: environment can only contain strings" in Python's subprocess.py.
-            if PY2 and WIN:
-                if isinstance(k, text_type) or isinstance(v, text_type):
-                    k = k.encode('ascii')
-                    v = v.encode('ascii')
-            os.environ[k] = v  # type: ignore
+            os.environ[to_env(k)] = to_env(v)
 
         return True
 
diff --git a/tests/test_core.py b/tests/test_core.py
@@ -12,7 +12,7 @@
 from IPython.terminal.embed import InteractiveShellEmbed
 
 from dotenv import dotenv_values, find_dotenv, load_dotenv, set_key
-from dotenv.compat import StringIO
+from dotenv.compat import PY2, StringIO
 from dotenv.main import Binding, parse_stream
 
 
@@ -229,6 +229,15 @@ def test_dotenv_values_export():
     assert os.environ['foo'] == 'bar'
 
 
+def test_dotenv_values_utf_8():
+    stream = StringIO(u"a=à\n")
+    load_dotenv(stream=stream)
+    if PY2:
+        assert os.environ["a"] == u"à".encode(sys.getfilesystemencoding())
+    else:
+        assert os.environ["a"] == "à"
+
+
 def test_dotenv_empty_selfreferential_interpolation():
     stream = StringIO(u'some_path="${some_path}:a/b/c"\n')
     stream.seek(0)
