#include "Python.h"

#include "pycore_ceval.h" // _Py_set_eval_breaker_bit()

#include "pycore_context.h"

#include "pycore_dict.h" // _PyDict_MaybeUntrack()

#include "pycore_initconfig.h"

#include "pycore_interp.h" // PyInterpreterState.gc

#include "pycore_object.h"

#include "pycore_pyerrors.h"

#include "pycore_pystate.h" // _PyThreadState_GET()

#include "pycore_weakref.h" // _PyWeakref_ClearRef()

#include "pydtrace.h"

typedef struct _gc_runtime_state GCState;

#ifdef Py_DEBUG

# define GC_DEBUG

#define GC_NEXT _PyGCHead_NEXT

#define GC_PREV _PyGCHead_PREV

#define PREV_MASK_COLLECTING _PyGC_PREV_MASK_COLLECTING

#define NEXT_MASK_UNREACHABLE (1)

#define AS_GC(op) _Py_AS_GC(op)

#define FROM_GC(gc) _Py_FROM_GC(gc)

static inline int

gc_is_collecting(PyGC_Head *g)

{

return (g->_gc_prev & PREV_MASK_COLLECTING) != 0;

}

static inline void

gc_clear_collecting(PyGC_Head *g)

{

g->_gc_prev &= ~PREV_MASK_COLLECTING;

}

static inline Py_ssize_t

gc_get_refs(PyGC_Head *g)

{

return (Py_ssize_t)(g->_gc_prev >> _PyGC_PREV_SHIFT);

}

static inline void

gc_set_refs(PyGC_Head *g, Py_ssize_t refs)

{

g->_gc_prev = (g->_gc_prev & ~_PyGC_PREV_MASK)

| ((uintptr_t)(refs) << _PyGC_PREV_SHIFT);

}

static inline void

gc_reset_refs(PyGC_Head *g, Py_ssize_t refs)

{

g->_gc_prev = (g->_gc_prev & _PyGC_PREV_MASK_FINALIZED)

| PREV_MASK_COLLECTING

| ((uintptr_t)(refs) << _PyGC_PREV_SHIFT);

}

static inline void

gc_decref(PyGC_Head *g)

{

_PyObject_ASSERT_WITH_MSG(FROM_GC(g),

gc_get_refs(g) > 0,

"refcount is too small");

g->_gc_prev -= 1 << _PyGC_PREV_SHIFT;

}

#define DEBUG_STATS (1<<0) /* print collection statistics */

#define DEBUG_COLLECTABLE (1<<1) /* print collectable objects */

#define DEBUG_UNCOLLECTABLE (1<<2) /* print uncollectable objects */

#define DEBUG_SAVEALL (1<<5) /* save all garbage in gc.garbage */

#define DEBUG_LEAK DEBUG_COLLECTABLE | \

DEBUG_UNCOLLECTABLE | \

DEBUG_SAVEALL

#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head)

static GCState *

get_gc_state(void)

{

PyInterpreterState *interp = _PyInterpreterState_GET();

return &interp->gc;

}

_PyGC_InitState(GCState *gcstate)

{

#define INIT_HEAD(GEN) \

do { \

GEN.head._gc_next = (uintptr_t)&GEN.head; \

GEN.head._gc_prev = (uintptr_t)&GEN.head; \

} while (0)

for (int i = 0; i < NUM_GENERATIONS; i++) {

assert(gcstate->generations[i].count == 0);

INIT_HEAD(gcstate->generations[i]);

};

gcstate->generation0 = GEN_HEAD(gcstate, 0);

INIT_HEAD(gcstate->permanent_generation);

#undef INIT_HEAD

}

_PyGC_Init(PyInterpreterState *interp)

{

GCState *gcstate = &interp->gc;

gcstate->garbage = PyList_New(0);

if (gcstate->garbage == NULL) {

return _PyStatus_NO_MEMORY();

}

gcstate->callbacks = PyList_New(0);

if (gcstate->callbacks == NULL) {

return _PyStatus_NO_MEMORY();

}

return _PyStatus_OK();

}

static inline void

gc_list_init(PyGC_Head *list)

{

// List header must not have flags.

// We can assign pointer by simple cast.

list->_gc_prev = (uintptr_t)list;

list->_gc_next = (uintptr_t)list;

}

static inline int

gc_list_is_empty(PyGC_Head *list)

{

return (list->_gc_next == (uintptr_t)list);

}

static inline void

gc_list_append(PyGC_Head *node, PyGC_Head *list)

{

PyGC_Head *last = (PyGC_Head *)list->_gc_prev;

// last <-> node

_PyGCHead_SET_PREV(node, last);

_PyGCHead_SET_NEXT(last, node);

// node <-> list

_PyGCHead_SET_NEXT(node, list);

list->_gc_prev = (uintptr_t)node;

}

static inline void

gc_list_remove(PyGC_Head *node)

{

PyGC_Head *prev = GC_PREV(node);

PyGC_Head *next = GC_NEXT(node);

_PyGCHead_SET_NEXT(prev, next);

_PyGCHead_SET_PREV(next, prev);

node->_gc_next = 0; /* object is not currently tracked */

}

static void

gc_list_move(PyGC_Head *node, PyGC_Head *list)

{

/* Unlink from current list. */

PyGC_Head *from_prev = GC_PREV(node);

PyGC_Head *from_next = GC_NEXT(node);

_PyGCHead_SET_NEXT(from_prev, from_next);

_PyGCHead_SET_PREV(from_next, from_prev);

/* Relink at end of new list. */

// list must not have flags. So we can skip macros.

PyGC_Head *to_prev = (PyGC_Head*)list->_gc_prev;

_PyGCHead_SET_PREV(node, to_prev);

_PyGCHead_SET_NEXT(to_prev, node);

list->_gc_prev = (uintptr_t)node;

_PyGCHead_SET_NEXT(node, list);

}

static void

gc_list_merge(PyGC_Head *from, PyGC_Head *to)

{

assert(from != to);

if (!gc_list_is_empty(from)) {

PyGC_Head *to_tail = GC_PREV(to);

PyGC_Head *from_head = GC_NEXT(from);

PyGC_Head *from_tail = GC_PREV(from);

assert(from_head != from);

assert(from_tail != from);

_PyGCHead_SET_NEXT(to_tail, from_head);

_PyGCHead_SET_PREV(from_head, to_tail);

_PyGCHead_SET_NEXT(from_tail, to);

_PyGCHead_SET_PREV(to, from_tail);

}

gc_list_init(from);

}

static Py_ssize_t

gc_list_size(PyGC_Head *list)

{

PyGC_Head *gc;

Py_ssize_t n = 0;

for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) {

n++;

}

return n;

}

static inline void

gc_list_clear_collecting(PyGC_Head *collectable)

{

PyGC_Head *gc;

for (gc = GC_NEXT(collectable); gc != collectable; gc = GC_NEXT(gc)) {

gc_clear_collecting(gc);

}

}

static int

append_objects(PyObject *py_list, PyGC_Head *gc_list)

{

PyGC_Head *gc;

for (gc = GC_NEXT(gc_list); gc != gc_list; gc = GC_NEXT(gc)) {

PyObject *op = FROM_GC(gc);

if (op != py_list) {

if (PyList_Append(py_list, op)) {

return -1; /* exception */

}

}

}

return 0;

}

enum flagstates {collecting_clear_unreachable_clear,

collecting_clear_unreachable_set,

collecting_set_unreachable_clear,

collecting_set_unreachable_set};

#ifdef GC_DEBUG

static void

validate_list(PyGC_Head *head, enum flagstates flags)

{

assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0);

assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0);

uintptr_t prev_value = 0, next_value = 0;

switch (flags) {

case collecting_clear_unreachable_clear:

break;

case collecting_set_unreachable_clear:

prev_value = PREV_MASK_COLLECTING;

break;

case collecting_clear_unreachable_set:

next_value = NEXT_MASK_UNREACHABLE;

break;

case collecting_set_unreachable_set:

prev_value = PREV_MASK_COLLECTING;

next_value = NEXT_MASK_UNREACHABLE;

break;

default:

assert(! "bad internal flags argument");

}

PyGC_Head *prev = head;

PyGC_Head *gc = GC_NEXT(head);

while (gc != head) {

PyGC_Head *trueprev = GC_PREV(gc);

PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE);

assert(truenext != NULL);

assert(trueprev == prev);

assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value);

assert((gc->_gc_next & NEXT_MASK_UNREACHABLE) == next_value);

prev = gc;

gc = truenext;

}

assert(prev == GC_PREV(head));

}

#define validate_list(x, y) do{}while(0)

static void

update_refs(PyGC_Head *containers)

{

PyGC_Head *next;

PyGC_Head *gc = GC_NEXT(containers);

while (gc != containers) {

next = GC_NEXT(gc);

/* Move any object that might have become immortal to the

if (_Py_IsImmortal(FROM_GC(gc))) {

gc_list_move(gc, &get_gc_state()->permanent_generation.head);

gc = next;

continue;

}

gc_reset_refs(gc, Py_REFCNT(FROM_GC(gc)));

/* Python's cyclic gc should never see an incoming refcount

_PyObject_ASSERT(FROM_GC(gc), gc_get_refs(gc) != 0);

gc = next;

}

}

static int

visit_decref(PyObject *op, void *parent)

{

OBJECT_STAT_INC(object_visits);

_PyObject_ASSERT(_PyObject_CAST(parent), !_PyObject_IsFreed(op));

if (_PyObject_IS_GC(op)) {

PyGC_Head *gc = AS_GC(op);

/* We're only interested in gc_refs for objects in the

if (gc_is_collecting(gc)) {

gc_decref(gc);

}

}

return 0;

}

static void

subtract_refs(PyGC_Head *containers)

{

traverseproc traverse;

PyGC_Head *gc = GC_NEXT(containers);

for (; gc != containers; gc = GC_NEXT(gc)) {

PyObject *op = FROM_GC(gc);

traverse = Py_TYPE(op)->tp_traverse;

(void) traverse(op,

(visitproc)visit_decref,

op);

}

}

static int

visit_reachable(PyObject *op, PyGC_Head *reachable)

{

OBJECT_STAT_INC(object_visits);

if (!_PyObject_IS_GC(op)) {

return 0;

}

PyGC_Head *gc = AS_GC(op);

const Py_ssize_t gc_refs = gc_get_refs(gc);

// Ignore objects in other generation.

// This also skips objects "to the left" of the current position in

// move_unreachable's scan of the 'young' list - they've already been

// traversed, and no longer have the PREV_MASK_COLLECTING flag.

if (! gc_is_collecting(gc)) {

return 0;

}

// It would be a logic error elsewhere if the collecting flag were set on

// an untracked object.

assert(gc->_gc_next != 0);

if (gc->_gc_next & NEXT_MASK_UNREACHABLE) {

/* This had gc_refs = 0 when move_unreachable got

// Manually unlink gc from unreachable list because the list functions

// don't work right in the presence of NEXT_MASK_UNREACHABLE flags.

PyGC_Head *prev = GC_PREV(gc);

PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE);

_PyObject_ASSERT(FROM_GC(prev),

prev->_gc_next & NEXT_MASK_UNREACHABLE);

_PyObject_ASSERT(FROM_GC(next),

next->_gc_next & NEXT_MASK_UNREACHABLE);

prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE

_PyGCHead_SET_PREV(next, prev);

gc_list_append(gc, reachable);

gc_set_refs(gc, 1);

}

else if (gc_refs == 0) {

/* This is in move_unreachable's 'young' list, but

gc_set_refs(gc, 1);

}

/* Else there's nothing to do.

else {

_PyObject_ASSERT_WITH_MSG(op, gc_refs > 0, "refcount is too small");

}

return 0;

}

static void

move_unreachable(PyGC_Head *young, PyGC_Head *unreachable)

{

// previous elem in the young list, used for restore gc_prev.

PyGC_Head *prev = young;

PyGC_Head *gc = GC_NEXT(young);

/* Invariants: all objects "to the left" of us in young are reachable

while (gc != young) {

if (gc_get_refs(gc)) {

/* gc is definitely reachable from outside the

PyObject *op = FROM_GC(gc);

traverseproc traverse = Py_TYPE(op)->tp_traverse;

_PyObject_ASSERT_WITH_MSG(op, gc_get_refs(gc) > 0,

"refcount is too small");

// NOTE: visit_reachable may change gc->_gc_next when

// young->_gc_prev == gc. Don't do gc = GC_NEXT(gc) before!

(void) traverse(op,

(visitproc)visit_reachable,

(void *)young);

// relink gc_prev to prev element.

_PyGCHead_SET_PREV(gc, prev);

// gc is not COLLECTING state after here.

gc_clear_collecting(gc);

prev = gc;

}

else {

/* This *may* be unreachable. To make progress,

// Move gc to unreachable.

// No need to gc->next->prev = prev because it is single linked.

prev->_gc_next = gc->_gc_next;

// We can't use gc_list_append() here because we use

// NEXT_MASK_UNREACHABLE here.

PyGC_Head *last = GC_PREV(unreachable);

// NOTE: Since all objects in unreachable set has

// NEXT_MASK_UNREACHABLE flag, we set it unconditionally.

// But this may pollute the unreachable list head's 'next' pointer

// too. That's semantically senseless but expedient here - the

// damage is repaired when this function ends.

last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc);

_PyGCHead_SET_PREV(gc, last);

gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable);

unreachable->_gc_prev = (uintptr_t)gc;

}

gc = (PyGC_Head*)prev->_gc_next;

}

// young->_gc_prev must be last element remained in the list.

young->_gc_prev = (uintptr_t)prev;

// don't let the pollution of the list head's next pointer leak

unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE;

}

static void

untrack_tuples(PyGC_Head *head)

{

PyGC_Head *next, *gc = GC_NEXT(head);

while (gc != head) {

PyObject *op = FROM_GC(gc);

next = GC_NEXT(gc);

if (PyTuple_CheckExact(op)) {

_PyTuple_MaybeUntrack(op);

}

gc = next;

}

}

static void

untrack_dicts(PyGC_Head *head)

{

PyGC_Head *next, *gc = GC_NEXT(head);

while (gc != head) {

PyObject *op = FROM_GC(gc);

next = GC_NEXT(gc);

if (PyDict_CheckExact(op)) {

_PyDict_MaybeUntrack(op);

}

gc = next;

}

}

static int

has_legacy_finalizer(PyObject *op)

{

return Py_TYPE(op)->tp_del != NULL;

}

static void

move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers)

{

PyGC_Head *gc, *next;

assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0);

/* March over unreachable. Move objects with finalizers into

for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {

PyObject *op = FROM_GC(gc);

_PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE);

gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;

next = (PyGC_Head*)gc->_gc_next;

if (has_legacy_finalizer(op)) {

gc_clear_collecting(gc);

gc_list_move(gc, finalizers);

}

}

}

static inline void

clear_unreachable_mask(PyGC_Head *unreachable)

{

/* Check that the list head does not have the unreachable bit set */

assert(((uintptr_t)unreachable & NEXT_MASK_UNREACHABLE) == 0);

PyGC_Head *gc, *next;

assert((unreachable->_gc_next & NEXT_MASK_UNREACHABLE) == 0);

for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {

_PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE);

gc->_gc_next &= ~NEXT_MASK_UNREACHABLE;

next = (PyGC_Head*)gc->_gc_next;

}

validate_list(unreachable, collecting_set_unreachable_clear);

}

static int

visit_move(PyObject *op, PyGC_Head *tolist)

{

OBJECT_STAT_INC(object_visits);

if (_PyObject_IS_GC(op)) {

PyGC_Head *gc = AS_GC(op);

if (gc_is_collecting(gc)) {

gc_list_move(gc, tolist);

gc_clear_collecting(gc);

}

}

return 0;

}

static void

move_legacy_finalizer_reachable(PyGC_Head *finalizers)

{

traverseproc traverse;

PyGC_Head *gc = GC_NEXT(finalizers);

for (; gc != finalizers; gc = GC_NEXT(gc)) {

/* Note that the finalizers list may grow during this. */

traverse = Py_TYPE(FROM_GC(gc))->tp_traverse;

(void) traverse(FROM_GC(gc),

(visitproc)visit_move,

(void *)finalizers);

}

}

static int

handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old)

{

PyGC_Head *gc;

PyObject *op; /* generally FROM_GC(gc) */

PyWeakReference *wr; /* generally a cast of op */

PyGC_Head wrcb_to_call; /* weakrefs with callbacks to call */

PyGC_Head *next;

int num_freed = 0;

gc_list_init(&wrcb_to_call);

/* Clear all weakrefs to the objects in unreachable. If such a weakref

for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) {

PyWeakReference **wrlist;

op = FROM_GC(gc);

next = GC_NEXT(gc);

if (PyWeakref_Check(op)) {

/* A weakref inside the unreachable set must be cleared. If we

_PyWeakref_ClearRef((PyWeakReference *)op);

}

if (! _PyType_SUPPORTS_WEAKREFS(Py_TYPE(op)))

continue;

/* It supports weakrefs. Does it have any?

wrlist = _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(op);

/* `op` may have some weakrefs. March over the list, clear

for (wr = *wrlist; wr != NULL; wr = *wrlist) {

PyGC_Head *wrasgc; /* AS_GC(wr) */

/* _PyWeakref_ClearRef clears the weakref but leaves

_PyObject_ASSERT((PyObject *)wr, wr->wr_object == op);

_PyWeakref_ClearRef(wr);

_PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None);

if (wr->wr_callback == NULL) {

/* no callback */

continue;

}

/* Headache time. `op` is going away, and is weakly referenced by

if (gc_is_collecting(AS_GC((PyObject *)wr))) {

/* it should already have been cleared above */

assert(wr->wr_object == Py_None);

continue;

}

/* Create a new reference so that wr can't go away

Py_INCREF(wr);

/* Move wr to wrcb_to_call, for the next pass. */

wrasgc = AS_GC((PyObject *)wr);

assert(wrasgc != next); /* wrasgc is reachable, but

gc_list_move(wrasgc, &wrcb_to_call);

}

}

/* Invoke the callbacks we decided to honor. It's safe to invoke them

while (! gc_list_is_empty(&wrcb_to_call)) {

PyObject *temp;

PyObject *callback;

gc = (PyGC_Head*)wrcb_to_call._gc_next;

op = FROM_GC(gc);

_PyObject_ASSERT(op, PyWeakref_Check(op));

wr = (PyWeakReference *)op;

callback = wr->wr_callback;

_PyObject_ASSERT(op, callback != NULL);

/* copy-paste of weakrefobject.c's handle_callback() */

temp = PyObject_CallOneArg(callback, (PyObject *)wr);

if (temp == NULL)

PyErr_WriteUnraisable(callback);

else

Py_DECREF(temp);

/* Give up the reference we created in the first pass. When

Py_DECREF(op);

if (wrcb_to_call._gc_next == (uintptr_t)gc) {

/* object is still alive -- move it */

gc_list_move(gc, old);

}

else {

++num_freed;

}

}

return num_freed;

}

static void

debug_cycle(const char *msg, PyObject *op)

{

PySys_FormatStderr("gc: %s <%s %p>\n",

msg, Py_TYPE(op)->tp_name, op);

}

static void

handle_legacy_finalizers(PyThreadState *tstate,

GCState *gcstate,

PyGC_Head *finalizers, PyGC_Head *old)

{

assert(!_PyErr_Occurred(tstate));

assert(gcstate->garbage != NULL);

PyGC_Head *gc = GC_NEXT(finalizers);

for (; gc != finalizers; gc = GC_NEXT(gc)) {

PyObject *op = FROM_GC(gc);

if ((gcstate->debug & DEBUG_SAVEALL) || has_legacy_finalizer(op)) {

if (PyList_Append(gcstate->garbage, op) < 0) {

_PyErr_Clear(tstate);

break;

}

}

}

gc_list_merge(finalizers, old);

}

static void

finalize_garbage(PyThreadState *tstate, PyGC_Head *collectable)

{

destructor finalize;

PyGC_Head seen;

/* While we're going through the loop, `finalize(op)` may cause op, or

gc_list_init(&seen);

while (!gc_list_is_empty(collectable)) {

PyGC_Head *gc = GC_NEXT(collectable);

PyObject *op = FROM_GC(gc);

gc_list_move(gc, &seen);

if (!_PyGCHead_FINALIZED(gc) &&

(finalize = Py_TYPE(op)->tp_finalize) != NULL) {

_PyGCHead_SET_FINALIZED(gc);

Py_INCREF(op);

finalize(op);

assert(!_PyErr_Occurred(tstate));

Py_DECREF(op);