Skip to content

Commit adb69fc

Browse files
nnorwitznnorwitz
committed
Merge from ast-arena. This reduces the code in Python/ast.c by ~300 lines,
simplifies a lot of error handling code, and fixes many memory leaks.
1 parent 23a6958 commit adb69fc

File tree

16 files changed

+704
-1242
lines changed

16 files changed

+704
-1242
lines changed

‎Include/Python-ast.h‎

Lines changed: 66 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -328,81 +328,79 @@ struct _alias {
328328
};
329329

330330

331-
mod_ty Module(asdl_seq * body);
332-
mod_ty Interactive(asdl_seq * body);
333-
mod_ty Expression(expr_ty body);
334-
mod_ty Suite(asdl_seq * body);
331+
mod_ty Module(asdl_seq * body, PyArena *arena);
332+
mod_ty Interactive(asdl_seq * body, PyArena *arena);
333+
mod_ty Expression(expr_ty body, PyArena *arena);
334+
mod_ty Suite(asdl_seq * body, PyArena *arena);
335335
stmt_ty FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
336-
asdl_seq * decorators, int lineno);
336+
asdl_seq * decorators, int lineno, PyArena *arena);
337337
stmt_ty ClassDef(identifier name, asdl_seq * bases, asdl_seq * body, int
338-
lineno);
339-
stmt_ty Return(expr_ty value, int lineno);
340-
stmt_ty Delete(asdl_seq * targets, int lineno);
341-
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno);
342-
stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno);
343-
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno);
338+
lineno, PyArena *arena);
339+
stmt_ty Return(expr_ty value, int lineno, PyArena *arena);
340+
stmt_ty Delete(asdl_seq * targets, int lineno, PyArena *arena);
341+
stmt_ty Assign(asdl_seq * targets, expr_ty value, int lineno, PyArena *arena);
342+
stmt_ty AugAssign(expr_ty target, operator_ty op, expr_ty value, int lineno,
343+
PyArena *arena);
344+
stmt_ty Print(expr_ty dest, asdl_seq * values, bool nl, int lineno, PyArena
345+
*arena);
344346
stmt_ty For(expr_ty target, expr_ty iter, asdl_seq * body, asdl_seq * orelse,
345-
int lineno);
346-
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
347-
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno);
348-
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno);
347+
int lineno, PyArena *arena);
348+
stmt_ty While(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
349+
PyArena *arena);
350+
stmt_ty If(expr_ty test, asdl_seq * body, asdl_seq * orelse, int lineno,
351+
PyArena *arena);
352+
stmt_ty Raise(expr_ty type, expr_ty inst, expr_ty tback, int lineno, PyArena
353+
*arena);
349354
stmt_ty TryExcept(asdl_seq * body, asdl_seq * handlers, asdl_seq * orelse, int
350-
lineno);
351-
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno);
352-
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno);
353-
stmt_ty Import(asdl_seq * names, int lineno);
354-
stmt_ty ImportFrom(identifier module, asdl_seq * names, int lineno);
355-
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno);
356-
stmt_ty Global(asdl_seq * names, int lineno);
357-
stmt_ty Expr(expr_ty value, int lineno);
358-
stmt_ty Pass(int lineno);
359-
stmt_ty Break(int lineno);
360-
stmt_ty Continue(int lineno);
361-
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno);
362-
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno);
363-
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno);
364-
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno);
365-
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno);
366-
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno);
367-
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno);
368-
expr_ty Yield(expr_ty value, int lineno);
355+
lineno, PyArena *arena);
356+
stmt_ty TryFinally(asdl_seq * body, asdl_seq * finalbody, int lineno, PyArena
357+
*arena);
358+
stmt_ty Assert(expr_ty test, expr_ty msg, int lineno, PyArena *arena);
359+
stmt_ty Import(asdl_seq * names, int lineno, PyArena *arena);
360+
stmt_ty ImportFrom(identifier module, asdl_seq * names, int lineno, PyArena
361+
*arena);
362+
stmt_ty Exec(expr_ty body, expr_ty globals, expr_ty locals, int lineno, PyArena
363+
*arena);
364+
stmt_ty Global(asdl_seq * names, int lineno, PyArena *arena);
365+
stmt_ty Expr(expr_ty value, int lineno, PyArena *arena);
366+
stmt_ty Pass(int lineno, PyArena *arena);
367+
stmt_ty Break(int lineno, PyArena *arena);
368+
stmt_ty Continue(int lineno, PyArena *arena);
369+
expr_ty BoolOp(boolop_ty op, asdl_seq * values, int lineno, PyArena *arena);
370+
expr_ty BinOp(expr_ty left, operator_ty op, expr_ty right, int lineno, PyArena
371+
*arena);
372+
expr_ty UnaryOp(unaryop_ty op, expr_ty operand, int lineno, PyArena *arena);
373+
expr_ty Lambda(arguments_ty args, expr_ty body, int lineno, PyArena *arena);
374+
expr_ty Dict(asdl_seq * keys, asdl_seq * values, int lineno, PyArena *arena);
375+
expr_ty ListComp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
376+
*arena);
377+
expr_ty GeneratorExp(expr_ty elt, asdl_seq * generators, int lineno, PyArena
378+
*arena);
379+
expr_ty Yield(expr_ty value, int lineno, PyArena *arena);
369380
expr_ty Compare(expr_ty left, asdl_seq * ops, asdl_seq * comparators, int
370-
lineno);
381+
lineno, PyArena *arena);
371382
expr_ty Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
372-
starargs, expr_ty kwargs, int lineno);
373-
expr_ty Repr(expr_ty value, int lineno);
374-
expr_ty Num(object n, int lineno);
375-
expr_ty Str(string s, int lineno);
383+
starargs, expr_ty kwargs, int lineno, PyArena *arena);
384+
expr_ty Repr(expr_ty value, int lineno, PyArena *arena);
385+
expr_ty Num(object n, int lineno, PyArena *arena);
386+
expr_ty Str(string s, int lineno, PyArena *arena);
376387
expr_ty Attribute(expr_ty value, identifier attr, expr_context_ty ctx, int
377-
lineno);
388+
lineno, PyArena *arena);
378389
expr_ty Subscript(expr_ty value, slice_ty slice, expr_context_ty ctx, int
379-
lineno);
380-
expr_ty Name(identifier id, expr_context_ty ctx, int lineno);
381-
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno);
382-
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno);
383-
slice_ty Ellipsis(void);
384-
slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step);
385-
slice_ty ExtSlice(asdl_seq * dims);
386-
slice_ty Index(expr_ty value);
387-
comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs);
388-
excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body);
390+
lineno, PyArena *arena);
391+
expr_ty Name(identifier id, expr_context_ty ctx, int lineno, PyArena *arena);
392+
expr_ty List(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
393+
expr_ty Tuple(asdl_seq * elts, expr_context_ty ctx, int lineno, PyArena *arena);
394+
slice_ty Ellipsis(PyArena *arena);
395+
slice_ty Slice(expr_ty lower, expr_ty upper, expr_ty step, PyArena *arena);
396+
slice_ty ExtSlice(asdl_seq * dims, PyArena *arena);
397+
slice_ty Index(expr_ty value, PyArena *arena);
398+
comprehension_ty comprehension(expr_ty target, expr_ty iter, asdl_seq * ifs,
399+
PyArena *arena);
400+
excepthandler_ty excepthandler(expr_ty type, expr_ty name, asdl_seq * body,
401+
PyArena *arena);
389402
arguments_ty arguments(asdl_seq * args, identifier vararg, identifier kwarg,
390-
asdl_seq * defaults);
391-
keyword_ty keyword(identifier arg, expr_ty value);
392-
alias_ty alias(identifier name, identifier asname);
393-
394-
void free_mod(mod_ty);
395-
void free_stmt(stmt_ty);
396-
void free_expr(expr_ty);
397-
void free_expr_context(expr_context_ty);
398-
void free_slice(slice_ty);
399-
void free_boolop(boolop_ty);
400-
void free_operator(operator_ty);
401-
void free_unaryop(unaryop_ty);
402-
void free_cmpop(cmpop_ty);
403-
void free_comprehension(comprehension_ty);
404-
void free_excepthandler(excepthandler_ty);
405-
void free_arguments(arguments_ty);
406-
void free_keyword(keyword_ty);
407-
void free_alias(alias_ty);
403+
asdl_seq * defaults, PyArena *arena);
404+
keyword_ty keyword(identifier arg, expr_ty value, PyArena *arena);
405+
alias_ty alias(identifier name, identifier asname, PyArena *arena);
408406

‎Include/Python.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@
113113

114114
#include "pystate.h"
115115

116+
#include "pyarena.h"
116117
#include "modsupport.h"
117118
#include "pythonrun.h"
118119
#include "ceval.h"

‎Include/asdl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ typedef struct {
2323
void *elements[1];
2424
} asdl_seq;
2525

26-
asdl_seq *asdl_seq_new(int size);
26+
asdl_seq *asdl_seq_new(int size, PyArena *arena);
2727
void asdl_seq_free(asdl_seq *);
2828

2929
#ifdef Py_DEBUG

‎Include/ast.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ extern "C" {
55
#endif
66

77
PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
8-
const char *);
8+
const char *, PyArena *);
99

1010
#ifdef __cplusplus
1111
}

‎Include/compile.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ typedef struct {
2525

2626
struct _mod; /* Declare the existence of this type */
2727
PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
28-
PyCompilerFlags *);
28+
PyCompilerFlags *, PyArena *);
2929
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
3030

3131
#define ERR_LATE_FUTURE \

‎Include/pyarena.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/* An arena-like memory interface for the compiler.
2+
*/
3+
4+
#ifndef Py_PYARENA_H
5+
#define Py_PYARENA_H
6+
7+
#ifdef __cplusplus
8+
extern "C" {
9+
#endif
10+
11+
typedef struct _arena PyArena;
12+
13+
/* PyArena_New() and PyArena_Free() create a new arena and free it,
14+
respectively. Once an arena has been created, it can be used
15+
to allocate memory. Once it is freed, all the memory it allocated
16+
is freed and none of its pointers are valid.
17+
18+
PyArena_New() returns an arena pointer. On error, it
19+
returns a negative number and sets an exception.
20+
*/
21+
PyAPI_FUNC(PyArena *) PyArena_New(void);
22+
PyAPI_FUNC(void) PyArena_Free(PyArena *);
23+
24+
PyAPI_FUNC(void *) PyArena_Malloc(PyArena *, size_t);
25+
26+
/* The next two routines aren't proper arena allocation routines.
27+
They exist to experiment with the arena API without making wholesale
28+
changes to the implementation.
29+
30+
The two functions register pointers with the arena id. These
31+
are externally allocated pointers that will be freed when the
32+
arena is freed. One takes a pointer allocated with malloc. The
33+
other takes a PyObject that is DECREFed when the arena is freed.
34+
*/
35+
PyAPI_FUNC(int) PyArena_AddMallocPointer(PyArena *, void *);
36+
PyAPI_FUNC(int) PyArena_AddPyObject(PyArena *, PyObject *);
37+
38+
#ifdef __cplusplus
39+
}
40+
#endif
41+
42+
#endif /* !Py_PYARENA_H */

‎Include/pythonrun.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,10 +37,12 @@ PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags
3737
PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
3838

3939
PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
40-
int, PyCompilerFlags *flags);
40+
int, PyCompilerFlags *flags,
41+
PyArena *);
4142
PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int,
4243
char *, char *,
43-
PyCompilerFlags *, int *);
44+
PyCompilerFlags *, int *,
45+
PyArena *);
4446
#define PyParser_SimpleParseString(S, B) \
4547
PyParser_SimpleParseStringFlags(S, B, 0)
4648
#define PyParser_SimpleParseFile(FP, S, B) \

‎Makefile.pre.in‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -254,6 +254,7 @@ PYTHON_OBJS= \
254254
Python/modsupport.o \
255255
Python/mystrtoul.o \
256256
Python/mysnprintf.o \
257+
Python/pyarena.o \
257258
Python/pyfpe.o \
258259
Python/pystate.o \
259260
Python/pythonrun.o \
@@ -520,6 +521,7 @@ PYTHON_HEADERS= \
520521
Include/object.h \
521522
Include/objimpl.h \
522523
Include/patchlevel.h \
524+
Include/pyarena.h \
523525
Include/pydebug.h \
524526
Include/pyerrors.h \
525527
Include/pyfpe.h \

‎Parser/asdl_c.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -249,8 +249,9 @@ def emit_function(self, name, ctype, args, attrs, union=1):
249249
if args:
250250
argstr = ", ".join(["%s %s" % (atype, aname)
251251
for atype, aname, opt in args])
252+
argstr += ", PyArena *arena"
252253
else:
253-
argstr = "void"
254+
argstr = "PyArena *arena"
254255
self.emit("%s %s(%s);" % (ctype, name, argstr), 0)
255256

256257
def visitProduct(self, prod, name):
@@ -265,6 +266,10 @@ def emit(s, depth=0, reflow=1):
265266
self.emit(s, depth, reflow)
266267
argstr = ", ".join(["%s %s" % (atype, aname)
267268
for atype, aname, opt in args + attrs])
269+
if argstr:
270+
argstr += ", PyArena *arena"
271+
else:
272+
argstr = "PyArena *arena"
268273
self.emit("%s" % ctype, 0)
269274
emit("%s(%s)" % (name, argstr))
270275
emit("{")
@@ -280,7 +285,7 @@ def emit(s, depth=0, reflow=1):
280285
emit('return NULL;', 2)
281286
emit('}', 1)
282287

283-
emit("p = (%s)malloc(sizeof(*p));" % ctype, 1)
288+
emit("p = (%s)PyArena_Malloc(arena, sizeof(*p));" % ctype, 1);
284289
emit("if (!p) {", 1)
285290
emit("PyErr_NoMemory();", 2)
286291
emit("return NULL;", 2)
@@ -655,7 +660,7 @@ def main(srcfile):
655660
c = ChainOfVisitors(TypeDefVisitor(f),
656661
StructVisitor(f),
657662
PrototypeVisitor(f),
658-
FreePrototypeVisitor(f),
663+
## FreePrototypeVisitor(f),
659664
)
660665
c.visit(mod)
661666
f.close()
@@ -671,8 +676,8 @@ def main(srcfile):
671676
print >> f
672677
v = ChainOfVisitors(MarshalPrototypeVisitor(f),
673678
FunctionVisitor(f),
674-
FreeUtilVisitor(f),
675-
FreeVisitor(f),
679+
## FreeUtilVisitor(f),
680+
## FreeVisitor(f),
676681
MarshalUtilVisitor(f),
677682
MarshalFunctionVisitor(f),
678683
)

0 commit comments

Comments
 (0)