Skip to content

Commit 14f2fd2

Browse files
lostmsulostmsu
committed
detect Py_TRACE_REFS at runtime and calculate object offsets accordingly
1 parent daccc43 commit 14f2fd2

File tree

5 files changed

+91
-51
lines changed

5 files changed

+91
-51
lines changed

‎src/runtime/classderived.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,7 @@ internal static IntPtr ToPython(IPythonDerivedType obj)
102102
// collected while Python still has a reference to it.
103103
if (Runtime.Refcount(self.pyHandle) == 1)
104104
{
105-
106-
#if PYTHON_WITH_PYDEBUG
107-
Runtime._Py_NewReference(self.pyHandle);
108-
#endif
105+
Runtime._Py_NewReference(self.ObjectReference);
109106
GCHandle gc = GCHandle.Alloc(self, GCHandleType.Normal);
110107
SetGCHandle(self.ObjectReference, self.TypeReference, gc);
111108
self.gcHandle.Free();

‎src/runtime/converter.cs‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,16 @@ internal static bool ToManagedValue(IntPtr value, Type obType,
525525

526526
internal delegate bool TryConvertFromPythonDelegate(IntPtr pyObj, out object result);
527527

528+
internal static int ToInt32(BorrowedReference value)
529+
{
530+
nint num = Runtime.PyLong_AsSignedSize_t(value);
531+
if (num == -1 && Exceptions.ErrorOccurred())
532+
{
533+
throw new PythonException();
534+
}
535+
return checked((int)num);
536+
}
537+
528538
/// <summary>
529539
/// Convert a Python value to an instance of a primitive managed type.
530540
/// </summary>

‎src/runtime/native/ABI.cs‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ namespace Python.Runtime.Native
77

88
static class ABI
99
{
10+
public static int RefCountOffset { get; } = GetRefCountOffset();
11+
public static int ObjectHeadOffset => RefCountOffset;
12+
1013
internal static void Initialize(Version version, BorrowedReference pyType)
1114
{
1215
string offsetsClassSuffix = string.Format(CultureInfo.InvariantCulture,
@@ -16,18 +19,34 @@ internal static void Initialize(Version version, BorrowedReference pyType)
1619

1720
const string nativeTypeOffsetClassName = "Python.Runtime.NativeTypeOffset";
1821
string className = "Python.Runtime.TypeOffset" + offsetsClassSuffix;
22+
Type nativeOffsetsClass = thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false);
1923
Type typeOffsetsClass =
2024
// Try platform native offsets first. It is only present when generated by setup.py
21-
thisAssembly.GetType(nativeTypeOffsetClassName, throwOnError: false)
22-
?? thisAssembly.GetType(className, throwOnError: false);
25+
nativeOffsetsClass ?? thisAssembly.GetType(className, throwOnError: false);
2326
if (typeOffsetsClass is null)
2427
{
2528
var types = thisAssembly.GetTypes().Select(type => type.Name).Where(name => name.StartsWith("TypeOffset"));
2629
string message = $"Searching for {className}, found {string.Join(",", types)}.";
2730
throw new NotSupportedException($"Python ABI v{version} is not supported: {message}");
2831
}
32+
2933
var typeOffsets = (ITypeOffsets)Activator.CreateInstance(typeOffsetsClass);
30-
TypeOffset.Use(typeOffsets);
34+
TypeOffset.Use(typeOffsets, nativeOffsetsClass == null ? ObjectHeadOffset : 0);
35+
}
36+
37+
static unsafe int GetRefCountOffset()
38+
{
39+
IntPtr tempObject = Runtime.PyList_New(0);
40+
IntPtr* tempPtr = (IntPtr*)tempObject;
41+
int offset = 0;
42+
while(tempPtr[offset] != (IntPtr)1)
43+
{
44+
offset++;
45+
if (offset > 100)
46+
throw new InvalidProgramException("PyObject_HEAD could not be found withing reasonable distance from the start of PyObject");
47+
}
48+
Runtime.XDecref(tempObject);
49+
return offset * IntPtr.Size;
3150
}
3251
}
3352
}

‎src/runtime/native/TypeOffset.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ static partial class TypeOffset
7575
internal static int tp_str { get; private set; }
7676
internal static int tp_traverse { get; private set; }
7777

78-
internal static void Use(ITypeOffsets offsets)
78+
internal static void Use(ITypeOffsets offsets, int extraHeadOffset)
7979
{
8080
if (offsets is null) throw new ArgumentNullException(nameof(offsets));
8181

@@ -87,6 +87,7 @@ internal static void Use(ITypeOffsets offsets)
8787

8888
var sourceProperty = typeof(ITypeOffsets).GetProperty(offsetProperty.Name);
8989
int value = (int)sourceProperty.GetValue(offsets, null);
90+
value += extraHeadOffset;
9091
offsetProperty.SetValue(obj: null, value: value, index: null);
9192
}
9293

‎src/runtime/runtime.cs‎

Lines changed: 56 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -85,9 +85,9 @@ internal static Version PyVersion
8585
{
8686
using (var versionTuple = new PyTuple(PySys_GetObject("version_info")))
8787
{
88-
var major = versionTuple[0].As<int>();
89-
var minor = versionTuple[1].As<int>();
90-
var micro = versionTuple[2].As<int>();
88+
var major = Converter.ToInt32(versionTuple[0].Reference);
89+
var minor = Converter.ToInt32(versionTuple[1].Reference);
90+
var micro = Converter.ToInt32(versionTuple[2].Reference);
9191
return new Version(major, minor, micro);
9292
}
9393
}
@@ -198,15 +198,15 @@ private static void InitPyMembers()
198198
SetPyMember(ref PyFalse, PyObject_GetAttrString(builtins, "False"),
199199
() => PyFalse = IntPtr.Zero);
200200

201-
SetPyMember(ref PyBoolType, PyObject_Type(PyTrue),
201+
SetPyMemberTypeOf(ref PyBoolType, PyTrue,
202202
() => PyBoolType = IntPtr.Zero);
203-
SetPyMember(ref PyNoneType, PyObject_Type(PyNone),
203+
SetPyMemberTypeOf(ref PyNoneType, PyNone,
204204
() => PyNoneType = IntPtr.Zero);
205-
SetPyMember(ref PyTypeType, PyObject_Type(PyNoneType),
205+
SetPyMemberTypeOf(ref PyTypeType, PyNoneType,
206206
() => PyTypeType = IntPtr.Zero);
207207

208208
op = PyObject_GetAttrString(builtins, "len");
209-
SetPyMember(ref PyMethodType, PyObject_Type(op),
209+
SetPyMemberTypeOf(ref PyMethodType, op,
210210
() => PyMethodType = IntPtr.Zero);
211211
XDecref(op);
212212

@@ -215,7 +215,7 @@ private static void InitPyMembers()
215215
//
216216
// object.__init__ seems safe, though.
217217
op = PyObject_GetAttr(PyBaseObjectType, PyIdentifier.__init__);
218-
SetPyMember(ref PyWrapperDescriptorType, PyObject_Type(op),
218+
SetPyMemberTypeOf(ref PyWrapperDescriptorType, op,
219219
() => PyWrapperDescriptorType = IntPtr.Zero);
220220
XDecref(op);
221221

@@ -226,47 +226,47 @@ private static void InitPyMembers()
226226
}
227227

228228
op = PyString_FromString("string");
229-
SetPyMember(ref PyStringType, PyObject_Type(op),
229+
SetPyMemberTypeOf(ref PyStringType, op,
230230
() => PyStringType = IntPtr.Zero);
231231
XDecref(op);
232232

233233
op = PyUnicode_FromString("unicode");
234-
SetPyMember(ref PyUnicodeType, PyObject_Type(op),
234+
SetPyMemberTypeOf(ref PyUnicodeType, op,
235235
() => PyUnicodeType = IntPtr.Zero);
236236
XDecref(op);
237237

238238
op = EmptyPyBytes();
239-
SetPyMember(ref PyBytesType, PyObject_Type(op),
239+
SetPyMemberTypeOf(ref PyBytesType, op,
240240
() => PyBytesType = IntPtr.Zero);
241241
XDecref(op);
242242

243243
op = PyTuple_New(0);
244-
SetPyMember(ref PyTupleType, PyObject_Type(op),
244+
SetPyMemberTypeOf(ref PyTupleType, op,
245245
() => PyTupleType = IntPtr.Zero);
246246
XDecref(op);
247247

248248
op = PyList_New(0);
249-
SetPyMember(ref PyListType, PyObject_Type(op),
249+
SetPyMemberTypeOf(ref PyListType, op,
250250
() => PyListType = IntPtr.Zero);
251251
XDecref(op);
252252

253253
op = PyDict_New();
254-
SetPyMember(ref PyDictType, PyObject_Type(op),
254+
SetPyMemberTypeOf(ref PyDictType, op,
255255
() => PyDictType = IntPtr.Zero);
256256
XDecref(op);
257257

258258
op = PyInt_FromInt32(0);
259-
SetPyMember(ref PyIntType, PyObject_Type(op),
259+
SetPyMemberTypeOf(ref PyIntType, op,
260260
() => PyIntType = IntPtr.Zero);
261261
XDecref(op);
262262

263263
op = PyLong_FromLong(0);
264-
SetPyMember(ref PyLongType, PyObject_Type(op),
264+
SetPyMemberTypeOf(ref PyLongType, op,
265265
() => PyLongType = IntPtr.Zero);
266266
XDecref(op);
267267

268268
op = PyFloat_FromDouble(0);
269-
SetPyMember(ref PyFloatType, PyObject_Type(op),
269+
SetPyMemberTypeOf(ref PyFloatType, op,
270270
() => PyFloatType = IntPtr.Zero);
271271
XDecref(op);
272272

@@ -278,7 +278,8 @@ private static void InitPyMembers()
278278
_PyObject_NextNotImplemented = Get_PyObject_NextNotImplemented();
279279
{
280280
using var sys = PyImport_ImportModule("sys");
281-
PyModuleType = PyObject_Type(sys.DangerousMoveToPointer());
281+
SetPyMemberTypeOf(ref PyModuleType, sys.DangerousGetAddress(),
282+
() => PyModuleType = IntPtr.Zero);
282283
}
283284
}
284285

@@ -455,6 +456,12 @@ private static void SetPyMember(ref IntPtr obj, IntPtr value, Action onRelease)
455456
_pyRefs.Add(value, onRelease);
456457
}
457458

459+
private static void SetPyMemberTypeOf(ref IntPtr obj, IntPtr value, Action onRelease)
460+
{
461+
var type = PyObject_Type(new BorrowedReference(value)).DangerousMoveToPointer();
462+
SetPyMember(ref obj, type, onRelease);
463+
}
464+
458465
private static void ResetPyMembers()
459466
{
460467
_pyRefs.Release();
@@ -761,16 +768,12 @@ internal static unsafe void XDecref(IntPtr op)
761768
[Pure]
762769
internal static unsafe long Refcount(IntPtr op)
763770
{
764-
#if PYTHON_WITH_PYDEBUG
765-
var p = (void*)(op + TypeOffset.ob_refcnt);
766-
#else
767-
var p = (void*)op;
768-
#endif
769-
if ((void*)0 == p)
771+
if (op == IntPtr.Zero)
770772
{
771773
return 0;
772774
}
773-
return Is32Bit ? (*(int*)p) : (*(long*)p);
775+
var p = (nint*)(op + ABI.RefCountOffset);
776+
return *p;
774777
}
775778

776779
/// <summary>
@@ -977,14 +980,9 @@ internal static unsafe IntPtr PyObject_TYPE(IntPtr op)
977980
{
978981
return IntPtr.Zero;
979982
}
980-
#if PYTHON_WITH_PYDEBUG
981-
var n = 3;
982-
#else
983-
var n = 1;
984-
#endif
985-
return Is32Bit
986-
? new IntPtr((void*)(*((uint*)p + n)))
987-
: new IntPtr((void*)(*((ulong*)p + n)));
983+
Debug.Assert(TypeOffset.ob_type > 0);
984+
IntPtr* typePtr = (IntPtr*)(op + TypeOffset.ob_type);
985+
return *typePtr;
988986
}
989987
internal static unsafe BorrowedReference PyObject_TYPE(BorrowedReference op)
990988
=> new BorrowedReference(PyObject_TYPE(op.DangerousGetAddress()));
@@ -1001,6 +999,9 @@ internal static IntPtr PyObject_Type(IntPtr op)
1001999
return tp;
10021000
}
10031001

1002+
internal static NewReference PyObject_Type(BorrowedReference o)
1003+
=> Delegates.PyObject_Type(o);
1004+
10041005
internal static string PyObject_GetTypeName(IntPtr op)
10051006
{
10061007
IntPtr pyType = PyObject_TYPE(op);
@@ -1145,10 +1146,11 @@ internal static IntPtr PyObject_Str(IntPtr pointer)
11451146

11461147
internal static IntPtr PyObject_Dir(IntPtr pointer) => Delegates.PyObject_Dir(pointer);
11471148

1148-
#if PYTHON_WITH_PYDEBUG
1149-
[DllImport(_PythonDll, CallingConvention = CallingConvention.Cdecl)]
1150-
internal static extern void _Py_NewReference(IntPtr ob);
1151-
#endif
1149+
internal static void _Py_NewReference(BorrowedReference ob)
1150+
{
1151+
if (Delegates._Py_NewReference != null)
1152+
Delegates._Py_NewReference(ob);
1153+
}
11521154

11531155
//====================================================================
11541156
// Python buffer API
@@ -1912,11 +1914,6 @@ internal static string PyModule_GetName(IntPtr module)
19121914
internal static string PyModule_GetFilename(IntPtr module)
19131915
=> Delegates.PyModule_GetFilename(module).ToString(Encoding.UTF8);
19141916

1915-
#if PYTHON_WITH_PYDEBUG
1916-
[DllImport(_PythonDll, EntryPoint = "PyModule_Create2TraceRefs", CallingConvention = CallingConvention.Cdecl)]
1917-
#else
1918-
1919-
#endif
19201917
internal static IntPtr PyModule_Create2(IntPtr module, int apiver) => Delegates.PyModule_Create2(module, apiver);
19211918

19221919

@@ -2331,6 +2328,7 @@ static Delegates()
23312328
PyObject_Hash = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Hash), GetUnmanagedDll(_PythonDll));
23322329
PyObject_Repr = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Repr), GetUnmanagedDll(_PythonDll));
23332330
PyObject_Str = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Str), GetUnmanagedDll(_PythonDll));
2331+
PyObject_Type = (delegate* unmanaged[Cdecl]<BorrowedReference, NewReference>)GetFunctionByName(nameof(PyObject_Type), GetUnmanagedDll(_PythonDll));
23342332
PyObject_Dir = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyObject_Dir), GetUnmanagedDll(_PythonDll));
23352333
PyObject_GetBuffer = (delegate* unmanaged[Cdecl]<IntPtr, ref Py_buffer, int, int>)GetFunctionByName(nameof(PyObject_GetBuffer), GetUnmanagedDll(_PythonDll));
23362334
PyBuffer_Release = (delegate* unmanaged[Cdecl]<ref Py_buffer, void>)GetFunctionByName(nameof(PyBuffer_Release), GetUnmanagedDll(_PythonDll));
@@ -2466,7 +2464,14 @@ static Delegates()
24662464
PyModule_GetName = (delegate* unmanaged[Cdecl]<IntPtr, StrPtr>)GetFunctionByName(nameof(PyModule_GetName), GetUnmanagedDll(_PythonDll));
24672465
PyModule_GetDict = (delegate* unmanaged[Cdecl]<BorrowedReference, BorrowedReference>)GetFunctionByName(nameof(PyModule_GetDict), GetUnmanagedDll(_PythonDll));
24682466
PyModule_GetFilename = (delegate* unmanaged[Cdecl]<IntPtr, StrPtr>)GetFunctionByName(nameof(PyModule_GetFilename), GetUnmanagedDll(_PythonDll));
2469-
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll));
2467+
try
2468+
{
2469+
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName(nameof(PyModule_Create2), GetUnmanagedDll(_PythonDll));
2470+
}
2471+
catch (MissingMethodException)
2472+
{
2473+
PyModule_Create2 = (delegate* unmanaged[Cdecl]<IntPtr, int, IntPtr>)GetFunctionByName("PyModule_Create2TraceRefs", GetUnmanagedDll(_PythonDll));
2474+
}
24702475
PyImport_Import = (delegate* unmanaged[Cdecl]<IntPtr, IntPtr>)GetFunctionByName(nameof(PyImport_Import), GetUnmanagedDll(_PythonDll));
24712476
PyImport_ImportModule = (delegate* unmanaged[Cdecl]<StrPtr, NewReference>)GetFunctionByName(nameof(PyImport_ImportModule), GetUnmanagedDll(_PythonDll));
24722477
PyImport_ReloadModule = (delegate* unmanaged[Cdecl]<BorrowedReference, NewReference>)GetFunctionByName(nameof(PyImport_ReloadModule), GetUnmanagedDll(_PythonDll));
@@ -2521,6 +2526,12 @@ static Delegates()
25212526
PyThreadState_SetAsyncExcLLP64 = (delegate* unmanaged[Cdecl]<uint, IntPtr, int>)GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll));
25222527
PyThreadState_SetAsyncExcLP64 = (delegate* unmanaged[Cdecl]<ulong, IntPtr, int>)GetFunctionByName("PyThreadState_SetAsyncExc", GetUnmanagedDll(_PythonDll));
25232528
PyType_FromSpecWithBases = (delegate* unmanaged[Cdecl]<in NativeTypeSpec, BorrowedReference, NewReference>)GetFunctionByName(nameof(PyType_FromSpecWithBases), GetUnmanagedDll(PythonDLL));
2529+
2530+
try
2531+
{
2532+
_Py_NewReference = (delegate* unmanaged[Cdecl]<BorrowedReference, void>)GetFunctionByName(nameof(_Py_NewReference), GetUnmanagedDll(_PythonDll));
2533+
}
2534+
catch (MissingMethodException) { }
25242535
}
25252536

25262537
static global::System.IntPtr GetUnmanagedDll(string libraryName)
@@ -2616,6 +2627,7 @@ static Delegates()
26162627
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Hash { get; }
26172628
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Repr { get; }
26182629
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Str { get; }
2630+
internal static delegate* unmanaged[Cdecl]<BorrowedReference, NewReference> PyObject_Type { get; }
26192631
internal static delegate* unmanaged[Cdecl]<IntPtr, IntPtr> PyObject_Dir { get; }
26202632
internal static delegate* unmanaged[Cdecl]<IntPtr, ref Py_buffer, int, int> PyObject_GetBuffer { get; }
26212633
internal static delegate* unmanaged[Cdecl]<ref Py_buffer, void> PyBuffer_Release { get; }
@@ -2799,6 +2811,7 @@ static Delegates()
27992811
internal static delegate* unmanaged[Cdecl]<ulong, IntPtr, int> PyThreadState_SetAsyncExcLP64 { get; }
28002812
internal static delegate* unmanaged[Cdecl]<BorrowedReference, IntPtr, NewReference> PyObject_GenericGetDict { get; }
28012813
internal static delegate* unmanaged[Cdecl]<in NativeTypeSpec, BorrowedReference, NewReference> PyType_FromSpecWithBases { get; }
2814+
internal static delegate* unmanaged[Cdecl]<BorrowedReference, void> _Py_NewReference { get; }
28022815
}
28032816
}
28042817

0 commit comments

Comments
 (0)