Skip to content

Commit f9a3974

Browse files
BadSingletonBadSingleton
committed
Create a clr.loader module
* Return ModuleObject.pyHandle, do not convert. * Write domain tests generated code to file.
1 parent 45b0a0c commit f9a3974

File tree

6 files changed

+46
-62
lines changed

6 files changed

+46
-62
lines changed

‎src/runtime/converter.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,14 @@ internal static IntPtr ToPython(object value, Type type)
199199
return ClassDerivedObject.ToPython(pyderived);
200200
}
201201

202+
// ModuleObjects are created in a way that their wrapping them as
203+
// a CLRObject fails, the ClassObject has no tpHandle. Return the
204+
// pyHandle as is, do not convert.
205+
if (value is ModuleObject modobj)
206+
{
207+
return modobj.pyHandle;
208+
}
209+
202210
// hmm - from Python, we almost never care what the declared
203211
// type is. we'd rather have the object bound to the actual
204212
// implementing class.

‎src/runtime/importhook.cs‎

Lines changed: 20 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections.Generic;
23
using System.Runtime.InteropServices;
34

45
namespace Python.Runtime
@@ -18,9 +19,6 @@ import sys
1819
1920
class DotNetLoader(importlib.abc.Loader):
2021
21-
def __init__(self):
22-
super().__init__()
23-
2422
@classmethod
2523
def exec_module(klass, mod):
2624
# This method needs to exist.
@@ -32,13 +30,13 @@ import clr
3230
return clr._load_clr_module(spec)
3331
3432
class DotNetFinder(importlib.abc.MetaPathFinder):
35-
36-
def __init__(self):
37-
super().__init__()
38-
33+
3934
@classmethod
4035
def find_spec(klass, fullname, paths=None, target=None):
41-
import clr
36+
# Don't import, we might call ourselves recursively!
37+
if 'clr' not in sys.modules:
38+
return None
39+
clr = sys.modules['clr']
4240
if clr._available_namespaces and fullname in clr._available_namespaces:
4341
return importlib.machinery.ModuleSpec(fullname, DotNetLoader(), is_package=True)
4442
return None
@@ -64,13 +62,10 @@ internal static unsafe void Initialize()
6462
BorrowedReference dict = Runtime.PyImport_GetModuleDict();
6563
Runtime.PyDict_SetItemString(dict, "CLR", ClrModuleReference);
6664
Runtime.PyDict_SetItemString(dict, "clr", ClrModuleReference);
67-
68-
// Add/create the MetaPathLoader
6965
SetupNamespaceTracking();
7066
SetupImportHook();
7167
}
7268

73-
7469
/// <summary>
7570
/// Cleanup resources upon shutdown of the Python runtime.
7671
/// </summary>
@@ -81,11 +76,10 @@ internal static void Shutdown()
8176
return;
8277
}
8378

84-
bool shouldFreeDef = Runtime.Refcount(py_clr_module) == 1;
79+
TeardownNameSpaceTracking();
8580
Runtime.XDecref(py_clr_module);
8681
py_clr_module = IntPtr.Zero;
8782

88-
TeardownNameSpaceTracking();
8983
Runtime.XDecref(root.pyHandle);
9084
root = null;
9185
CLRModule.Reset();
@@ -107,62 +101,42 @@ internal static void RestoreRuntimeData(RuntimeDataStorage storage)
107101
var rootHandle = storage.GetValue<IntPtr>("root");
108102
root = (CLRModule)ManagedType.GetManagedObject(rootHandle);
109103
BorrowedReference dict = Runtime.PyImport_GetModuleDict();
110-
Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "CLR", py_clr_module);
111104
Runtime.PyDict_SetItemString(dict.DangerousGetAddress(), "clr", py_clr_module);
112105
SetupNamespaceTracking();
113106
}
114107

115108
static void SetupImportHook()
116109
{
117110
// Create the import hook module
118-
var import_hook_module_def = ModuleDefOffset.AllocModuleDef("clr.loader");
119-
var import_hook_module = Runtime.PyModule_Create2(import_hook_module_def, 3);
111+
var import_hook_module = Runtime.PyModule_New("clr.loader");
120112

121113
// Run the python code to create the module's classes.
122-
var mod_dict = Runtime.PyModule_GetDict(new BorrowedReference(import_hook_module));
123114
var builtins = Runtime.PyEval_GetBuiltins();
124115
var exec = Runtime.PyDict_GetItemString(builtins, "exec");
125116
using var args = NewReference.DangerousFromPointer(Runtime.PyTuple_New(2));
126117

127-
var codeStr = Runtime.PyString_FromString(LoaderCode);
118+
IntPtr codeStr = Runtime.PyString_FromString(LoaderCode);
128119
Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 0, codeStr);
129120
// PyTuple_SetItem steals a reference, mod_dict is borrowed.
121+
var mod_dict = Runtime.PyModule_GetDict(import_hook_module);
130122
Runtime.XIncref(mod_dict.DangerousGetAddress());
131123
Runtime.PyTuple_SetItem(args.DangerousGetAddress(), 1, mod_dict.DangerousGetAddress());
132124
Runtime.PyObject_Call(exec.DangerousGetAddress(), args.DangerousGetAddress(), IntPtr.Zero);
133125

134-
var loader = Runtime.PyDict_GetItemString(mod_dict, "DotNetLoader").DangerousGetAddressOrNull();
135-
Runtime.XIncref(loader);
136-
137-
// Add the classes to the module
138-
// PyModule_AddObject steals a reference only on success
139-
if (Runtime.PyModule_AddObject(import_hook_module, "DotNetLoader", loader) != 0)
140-
{
141-
Runtime.XDecref(loader);
142-
throw new PythonException();
143-
}
144-
145-
var finder = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder").DangerousGetAddressOrNull();
146-
Runtime.XIncref(finder);
147-
if (Runtime.PyModule_AddObject(import_hook_module, "DotNetFinder", finder) != 0)
148-
{
149-
Runtime.XDecref(finder);
150-
throw new PythonException();
151-
}
152-
153126
// Set as a sub-module of clr.
154-
Runtime.XIncref(import_hook_module);
155-
if(Runtime.PyModule_AddObject(py_clr_module, "loader", import_hook_module) != 0)
127+
if(Runtime.PyModule_AddObject(ClrModuleReference, "loader", import_hook_module) != 0)
156128
{
157-
Runtime.XDecref(import_hook_module);
129+
Runtime.XDecref(import_hook_module.DangerousGetAddress());
158130
throw new PythonException();
159131
}
160132

161133
// Finally, add the hook to the meta path
162-
var finder_inst = Runtime.PyDict_GetItemString(mod_dict, "finder_inst").DangerousGetAddressOrNull();
163-
Runtime.XIncref(finder);
134+
var findercls = Runtime.PyDict_GetItemString(mod_dict, "DotNetFinder");
135+
var finderCtorArgs = Runtime.PyTuple_New(0);
136+
var finder_inst = Runtime.PyObject_CallObject(findercls.DangerousGetAddress(), finderCtorArgs);
137+
Runtime.XDecref(finderCtorArgs);
164138
var metapath = Runtime.PySys_GetObject("meta_path");
165-
Runtime.PyList_Append(metapath, finder);
139+
Runtime.PyList_Append(metapath, finder_inst);
166140
}
167141

168142
/// <summary>
@@ -268,7 +242,8 @@ public static unsafe NewReference GetCLRModule()
268242
}
269243

270244
/// <summary>
271-
/// The hook to import a CLR module into Python
245+
/// The hook to import a CLR module into Python. Returns a new reference
246+
/// to the module.
272247
/// </summary>
273248
public static ModuleObject Import(string modname)
274249
{
@@ -305,7 +280,7 @@ public static ModuleObject Import(string modname)
305280
tail.LoadNames();
306281
}
307282
}
308-
283+
tail.IncrRefCount();
309284
return tail;
310285
}
311286

‎src/runtime/moduleobject.cs‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ public ModuleObject(string name)
5353
var dictRef = Runtime.PyObject_GenericGetDict(ObjectReference);
5454
PythonException.ThrowIfIsNull(dictRef);
5555
dict = dictRef.DangerousMoveToPointer();
56-
56+
__all__ = Runtime.PyList_New(0);
5757
using var pyname = NewReference.DangerousFromPointer(Runtime.PyString_FromString(moduleName));
5858
using var pyfilename = NewReference.DangerousFromPointer(Runtime.PyString_FromString(filename));
5959
using var pydocstring = NewReference.DangerousFromPointer(Runtime.PyString_FromString(docstring));
@@ -576,13 +576,6 @@ public static string[] ListAssemblies(bool verbose)
576576
return names;
577577
}
578578

579-
[ModuleFunction]
580-
public static int _AtExit()
581-
{
582-
return Runtime.AtExit();
583-
}
584-
585-
586579
/// <summary>
587580
/// Note: This should *not* be called directly.
588581
/// The function that get/import a CLR assembly as a python module.
@@ -593,16 +586,14 @@ public static int _AtExit()
593586
/// <returns>A new reference to the imported module, as a PyObject.</returns>
594587
[ModuleFunction]
595588
[ForbidPythonThreads]
596-
public static PyObject _load_clr_module(PyObject spec)
589+
public static ModuleObject _load_clr_module(PyObject spec)
597590
{
598591
ModuleObject mod = null;
599592
using (var modname = spec.GetAttr("name"))
600593
{
601594
mod = ImportHook.Import(modname.ToString());
602595
}
603-
// We can't return directly a ModuleObject, because the tpHandle is
604-
// not set, but we can return a PyObject.
605-
return new PyObject(mod.pyHandle);
596+
return mod;
606597
}
607598
}
608599
}

‎src/runtime/runtime.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1931,7 +1931,7 @@ internal static string PyModule_GetFilename(IntPtr module)
19311931

19321932

19331933
internal static IntPtr PyImport_Import(IntPtr name) => Delegates.PyImport_Import(name);
1934-
internal static int PyModule_AddObject(IntPtr module, string name, IntPtr stolenObject)
1934+
internal static int PyModule_AddObject(BorrowedReference module, string name, BorrowedReference stolenObject)
19351935
{
19361936
using var namePtr = new StrPtr(name, Encoding.UTF8);
19371937
return Delegates.PyModule_AddObject(module, namePtr, stolenObject);
@@ -2486,7 +2486,7 @@ static Delegates()
24862486
{
24872487
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll));
24882488
}
2489-
PyModule_AddObject = (delegate* unmanaged[Cdecl]<IntPtr, StrPtr, IntPtr, int>)GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll));
2489+
PyModule_AddObject = (delegate* unmanaged[Cdecl]<BorrowedReference, StrPtr, BorrowedReference, int>)GetFunctionByName(nameof(PyModule_AddObject), GetUnmanagedDll(_PythonDll));
24902490
PyImport_Import = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll));
24912491
PyImport_ImportModule = (delegate* unmanaged[Cdecl]<StrPtr, NewReference>)GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll));
24922492
PyImport_ReloadModule = (delegate* unmanaged[Cdecl]<BorrowedReference, NewReference>)GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll));
@@ -2773,7 +2773,7 @@ static Delegates()
27732773
internal static delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference> PyModule_GetDict { get; }
27742774
internal static delegate* unmanaged[Cdecl]<IntPtr, StrPtr> PyModule_GetFilename { get; }
27752775
internal static delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr> PyModule_Create2 { get; }
2776-
internal static delegate* unmanaged[Cdecl]<IntPtr, StrPtr, IntPtr, int> PyModule_AddObject { get; }
2776+
internal static delegate* unmanaged[Cdecl]<BorrowedReference, StrPtr, BorrowedReference, int> PyModule_AddObject { get; }
27772777
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyImport_Import { get; }
27782778
internal static delegate* unmanaged[Cdecl]<StrPtr, NewReference> PyImport_ImportModule { get; }
27792779
internal static delegate* unmanaged[Cdecl]<BorrowedReference, NewReference> PyImport_ReloadModule { get; }

‎tests/domain_tests/TestRunner.cs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,11 @@ namespace Python.DomainReloadTests
3030
/// which test case to run. That's because pytest assumes we'll run
3131
/// everything in one process, but we really want a clean process on each
3232
/// test case to test the init/reload/teardown parts of the domain reload.
33+
///
34+
/// ### Debugging tips: ###
35+
/// * Running pytest with the `-s` argument prevents stdout capture by pytest
36+
/// * Add a sleep into the python test case before the crash/failure, then while
37+
/// sleeping, attach the debugger to the Python.TestDomainReload.exe process.
3338
/// </summary>
3439
///
3540
class TestRunner
@@ -1287,7 +1292,13 @@ static string CreateAssembly(string name, string code, bool exe = false)
12871292
}
12881293
parameters.ReferencedAssemblies.Add(netstandard);
12891294
parameters.ReferencedAssemblies.Add(PythonDllLocation);
1290-
CompilerResults results = provider.CompileAssemblyFromSource(parameters, code);
1295+
// Write code to file so it can debugged.
1296+
var sourcePath = Path.Combine(TestPath, name+"_source.cs");
1297+
using(var file = new StreamWriter(sourcePath))
1298+
{
1299+
file.Write(code);
1300+
}
1301+
CompilerResults results = provider.CompileAssemblyFromFile(parameters, sourcePath);
12911302
if (results.NativeCompilerReturnValue != 0)
12921303
{
12931304
var stderr = System.Console.Error;

‎tests/domain_tests/test_domain_reload.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,5 @@ def test_in_to_ref_param():
8989
def test_nested_type():
9090
_run_test("nested_type")
9191

92-
@pytest.mark.skipif(platform.system() == 'Darwin', reason='FIXME: macos can\'t find the python library')
9392
def test_import_after_reload():
9493
_run_test("import_after_reload")

0 commit comments

Comments
 (0)