Skip to content

Allow pickling objects defined interactively. #384

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
takluyver wants to merge 6 commits into from
Closed

Allow pickling objects defined interactively. #384

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
89f3d69
Allow pickling objects defined interactively.
takluyver Apr 14, 2011
5411be8
Objects can still be pickled after a %reset.
takluyver Apr 14, 2011
66c14cf
Add test that we can pickle objects defined interactively.
takluyver Apr 15, 2011
3590d11
Add test for accessing global namespace inside functions.
takluyver Apr 16, 2011
7977a47
Tweak IPython doctesting so it doesn't interfere with our other tests.
takluyver Apr 16, 2011
afe8078
Change namespace initialisation code so that user_ns and user_global_…
takluyver Apr 16, 2011
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 53 additions & 72 deletions IPython/core/interactiveshell.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -813,17 +813,44 @@ def init_create_namespaces(self, user_ns=None, user_global_ns=None):
# that if you need to access the built-in namespace directly, you
# should start with "import __builtin__" (note, no 's') which will
# definitely give you a module. Yeah, it's somewhat confusing:-(.

# These routines return properly built dicts as needed by the rest of
# the code, and can also be used by extension writers to generate
# properly initialized namespaces.
user_ns, user_global_ns = self.make_user_namespaces(user_ns,
user_global_ns)

# We must ensure that __builtin__ (without the final 's') is always
# available and pointing to the __builtin__ *module*. For more details:
# http://mail.python.org/pipermail/python-dev/2001-April/014068.html

separate_user_ns = True
if user_ns is None:
user_ns = {}
separate_user_ns = False

# Set __name__ to __main__ to better match the behavior of the
# normal interpreter.
user_ns.setdefault('__name__','__main__')
user_ns.setdefault('__builtin__',__builtin__)
user_ns.setdefault('__builtins__',__builtin__)

if (user_global_ns is not None) and \
not isinstance(user_global_ns, dict):
raise TypeError("user_global_ns must be a true dict; got %r"
% type(user_global_ns))

# Assign namespaces
# This is the namespace where all normal user variables live
self.user_ns = user_ns
self.user_global_ns = user_global_ns
# user_ns the namespace where all normal user variables live
self.user_ns_mod = FakeModule(user_ns)
if not separate_user_ns:
# In the normal case, we replace user_ns with a module dict, so that
# user-defined objects are in the __main__ module, and can be pickled.
self.user_ns = self.user_ns_mod.__dict__
else:
# Otherwise, we will need to manually update it after execution.
self.user_ns = user_ns

# The global namespace will normally be identical to the user namespace.
# The exceptions are certain embedding cases.
if user_global_ns is None:
self.user_global_ns = self.user_ns
else:
self.user_global_ns = user_global_ns

# An auxiliary namespace that checks what parts of the user_ns were
# loaded at startup, so we can list later only variables defined in
Expand Down Expand Up @@ -866,8 +893,8 @@ def init_create_namespaces(self, user_ns=None, user_global_ns=None):

# A table holding all the namespaces IPython deals with, so that
# introspection facilities can search easily.
self.ns_table = {'user':user_ns,
'user_global':user_global_ns,
self.ns_table = {'user':self.user_ns,
'user_global':self.user_global_ns,
'internal':self.internal_ns,
'builtin':__builtin__.__dict__
}
Expand All @@ -880,65 +907,7 @@ def init_create_namespaces(self, user_ns=None, user_global_ns=None):
# clears them manually and carefully.
self.ns_refs_table = [ self.user_ns_hidden,
self.internal_ns, self._main_ns_cache ]

def make_user_namespaces(self, user_ns=None, user_global_ns=None):
"""Return a valid local and global user interactive namespaces.

This builds a dict with the minimal information needed to operate as a
valid IPython user namespace, which you can pass to the various
embedding classes in ipython. The default implementation returns the
same dict for both the locals and the globals to allow functions to
refer to variables in the namespace. Customized implementations can
return different dicts. The locals dictionary can actually be anything
following the basic mapping protocol of a dict, but the globals dict
must be a true dict, not even a subclass. It is recommended that any
custom object for the locals namespace synchronize with the globals
dict somehow.

Raises TypeError if the provided globals namespace is not a true dict.

Parameters
----------
user_ns : dict-like, optional
The current user namespace. The items in this namespace should
be included in the output. If None, an appropriate blank
namespace should be created.
user_global_ns : dict, optional
The current user global namespace. The items in this namespace
should be included in the output. If None, an appropriate
blank namespace should be created.

Returns
-------
A pair of dictionary-like object to be used as the local namespace
of the interpreter and a dict to be used as the global namespace.
"""


# We must ensure that __builtin__ (without the final 's') is always
# available and pointing to the __builtin__ *module*. For more details:
# http://mail.python.org/pipermail/python-dev/2001-April/014068.html

if user_ns is None:
# Set __name__ to __main__ to better match the behavior of the
# normal interpreter.
user_ns = {'__name__' :'__main__',
'__builtin__' : __builtin__,
'__builtins__' : __builtin__,
}
else:
user_ns.setdefault('__name__','__main__')
user_ns.setdefault('__builtin__',__builtin__)
user_ns.setdefault('__builtins__',__builtin__)

if user_global_ns is None:
user_global_ns = user_ns
if type(user_global_ns) is not dict:
raise TypeError("user_global_ns must be a true dict; got %r"
% type(user_global_ns))

return user_ns, user_global_ns


def init_sys_modules(self):
# We need to insert into sys.modules something that looks like a
# module but which accesses the IPython namespace, for shelve and
Expand All @@ -962,7 +931,7 @@ def init_sys_modules(self):
except KeyError:
raise KeyError('user_ns dictionary MUST have a "__name__" key')
else:
sys.modules[main_name] = FakeModule(self.user_ns)
sys.modules[main_name] = self.user_ns_mod

def init_user_ns(self):
"""Initialize all user-visible namespaces to their minimum defaults.
Expand Down Expand Up @@ -1057,13 +1026,19 @@ def reset(self, new_session=True):
# would cause errors in many object's __del__ methods.
for ns in [self.user_ns, self.user_global_ns]:
drop_keys = set(ns.keys())
drop_keys.discard('__name__')
drop_keys.discard('__builtin__')
drop_keys.discard('__builtins__')
for k in drop_keys:
del ns[k]

# Restore the user namespaces to minimal usability
self.init_user_ns()

# In some situations, e.g. testing, our fake main module has a __dict__
# separate from the user_ns: if so, we need to clear it manually.
if self.user_ns is not self.user_ns_mod.__dict__:
init_fakemod_dict(self.user_ns_mod, self.user_ns)

# Restore the default and user aliases
self.alias_manager.clear_aliases()
Expand Down Expand Up @@ -2145,7 +2120,13 @@ def run_cell(self, raw_cell, store_history=True):

self.run_ast_nodes(code_ast.body, cell_name,
interactivity="last_expr")


# In the normal case, user_ns is the __dict__ of the
# corresponding module. But e.g. in tests, it isn't, so we need
# to copy the contents across.
if self.user_ns is not self.user_ns_mod.__dict__:
init_fakemod_dict(self.user_ns_mod, self.user_ns)

# Execute any registered post-execution functions.
for func, status in self._post_execute.iteritems():
if not status:
Expand Down
22 changes: 22 additions & 0 deletions IPython/core/tests/test_interactiveshell.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,3 +91,25 @@ def test_magic_names_in_string(self):
ip = get_ipython()
ip.run_cell('a = """\n%exit\n"""')
self.assertEquals(ip.user_ns['a'], '\n%exit\n')

def test_can_pickle(self):
ip = get_ipython()
ip.run_cell(("class Mylist(list):\n"
" def __init__(self,x=[]):\n"
" list.__init__(self,x)"))
ip.run_cell("w=Mylist([1,2,3])")

from cPickle import dumps
res = dumps(ip.user_ns["w"])
self.assertTrue(isinstance(res, bytes))

def test_global_ns(self):
ip = get_ipython()
ip.run_cell("a = 10")
ip.run_cell(("def f(x):"
" return x + a"))
ip.run_cell("b = f(12)")
self.assertEqual(ip.user_ns["b"], 22)



5 changes: 4 additions & 1 deletion IPython/testing/globalipapp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,9 @@ def __init__(self,*a):
# namespace in doctests that call '_'.
self.protect_underscore = False

# We set this so that the tests don't clash with __main__
self["__name__"] = "__test_main__"

def clear(self):
dict.clear(self)
self.update(self._savedict)
Expand Down Expand Up @@ -158,7 +161,7 @@ def start_ipython():
# Create and initialize our test-friendly IPython instance.
shell = TerminalInteractiveShell.instance(config=config,
user_ns=ipnsdict(),
user_global_ns={}
user_global_ns=None
)

# A few more tweaks needed for playing nicely with doctests...
Expand Down
4 changes: 4 additions & 0 deletions IPython/testing/plugin/ipdoctest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,8 @@ def setUp(self):
# for IPython examples *only*, we swap the globals with the ipython
# namespace, after updating it with the globals (which doctest
# fills with the necessary info from the module being tested).
self.user_ns_orig = {}
self.user_ns_orig.update(_ip.user_ns)
_ip.user_ns.update(self._dt_test.globs)
self._dt_test.globs = _ip.user_ns
# IPython must protect the _ key in the namespace (it can't exist)
Expand All @@ -286,6 +288,8 @@ def tearDown(self):
# teardown doesn't destroy the ipython namespace
if isinstance(self._dt_test.examples[0],IPExample):
self._dt_test.globs = self._dt_test_globs_ori
_ip.user_ns.clear()
_ip.user_ns.update(self.user_ns_orig)
# Restore the behavior of the '_' key in the user namespace to
# normal after each doctest, so that unittests behave normally
_ip.user_ns.protect_underscore = False
Expand Down