Skip to content

Commit fe2127d

Browse files
tim-onetim-one
committed
Ugly. A pile of new xxxFlags() functions, to communicate to the parser
that 'yield' is a keyword. This doesn't help test_generators at all! I don't know why not. These things do work now (and didn't before this patch): 1. "from __future__ import generators" now works in a native shell. 2. Similarly "python -i xxx.py" now has generators enabled in the shell if xxx.py had them enabled. 3. This program (which was my doctest proxy) works fine: from __future__ import generators source = """\ def f(): yield 1 """ exec compile(source, "", "single") in globals() print type(f())
1 parent e75785a commit fe2127d

5 files changed

Lines changed: 65 additions & 14 deletions

File tree

‎Include/parsetok.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,19 @@ typedef struct {
1717
int expected;
1818
} perrdetail;
1919

20+
#define PyPARSE_YIELD_IS_KEYWORD 0x0001
21+
2022
extern DL_IMPORT(node *) PyParser_ParseString(char *, grammar *, int,
2123
perrdetail *);
2224
extern DL_IMPORT(node *) PyParser_ParseFile (FILE *, char *, grammar *, int,
2325
char *, char *, perrdetail *);
2426

27+
extern DL_IMPORT(node *) PyParser_ParseStringFlags(char *, grammar *, int,
28+
perrdetail *, int);
29+
extern DL_IMPORT(node *) PyParser_ParseFileFlags(FILE *, char *, grammar *,
30+
int, char *, char *,
31+
perrdetail *, int);
32+
2533
#ifdef __cplusplus
2634
}
2735
#endif

‎Include/pythonrun.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,9 @@ DL_IMPORT(int) PyRun_InteractiveLoopFlags(FILE *, char *, PyCompilerFlags *);
4545

4646
DL_IMPORT(struct _node *) PyParser_SimpleParseString(char *, int);
4747
DL_IMPORT(struct _node *) PyParser_SimpleParseFile(FILE *, char *, int);
48+
DL_IMPORT(struct _node *) PyParser_SimpleParseStringFlags(char *, int, int);
49+
DL_IMPORT(struct _node *) PyParser_SimpleParseFileFlags(FILE *, char *,
50+
int, int);
4851

4952
DL_IMPORT(PyObject *) PyRun_String(char *, int, PyObject *, PyObject *);
5053
DL_IMPORT(PyObject *) PyRun_File(FILE *, char *, int, PyObject *, PyObject *);

‎Lib/doctest.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -348,6 +348,8 @@ def _test():
348348
# 0,9,7 9-Feb-2001
349349
# string method conversion
350350

351+
from __future__ import generators
352+
351353
__version__ = 0, 9, 7
352354

353355
import types

‎Parser/parsetok.c‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,18 @@ int Py_TabcheckFlag;
1313

1414

1515
/* Forward */
16-
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *);
16+
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
1717

1818
/* Parse input coming from a string. Return error code, print some errors. */
19-
2019
node *
2120
PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
21+
{
22+
return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
23+
}
24+
25+
node *
26+
PyParser_ParseStringFlags(char *s, grammar *g, int start,
27+
perrdetail *err_ret, int flags)
2228
{
2329
struct tok_state *tok;
2430

@@ -42,7 +48,7 @@ PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
4248
tok->alterror++;
4349
}
4450

45-
return parsetok(tok, g, start, err_ret);
51+
return parsetok(tok, g, start, err_ret, flags);
4652
}
4753

4854

@@ -51,6 +57,14 @@ PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
5157
node *
5258
PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
5359
char *ps1, char *ps2, perrdetail *err_ret)
60+
{
61+
return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
62+
err_ret, 0);
63+
}
64+
65+
node *
66+
PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start,
67+
char *ps1, char *ps2, perrdetail *err_ret, int flags)
5468
{
5569
struct tok_state *tok;
5670

@@ -72,14 +86,15 @@ PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
7286
}
7387

7488

75-
return parsetok(tok, g, start, err_ret);
89+
return parsetok(tok, g, start, err_ret, flags);
7690
}
7791

7892
/* Parse input coming from the given tokenizer structure.
7993
Return error code. */
8094

8195
static node *
82-
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
96+
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
97+
int flags)
8398
{
8499
parser_state *ps;
85100
node *n;
@@ -90,6 +105,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
90105
err_ret->error = E_NOMEM;
91106
return NULL;
92107
}
108+
if (flags & PyPARSE_YIELD_IS_KEYWORD)
109+
ps->p_generators = 1;
93110

94111
for (;;) {
95112
char *a, *b;

‎Python/pythonrun.c‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -533,6 +533,7 @@ PyRun_InteractiveOneFlags(FILE *fp, char *filename, PyCompilerFlags *flags)
533533
node *n;
534534
perrdetail err;
535535
char *ps1 = "", *ps2 = "";
536+
536537
v = PySys_GetObject("ps1");
537538
if (v != NULL) {
538539
v = PyObject_Str(v);
@@ -549,8 +550,11 @@ PyRun_InteractiveOneFlags(FILE *fp, char *filename, PyCompilerFlags *flags)
549550
else if (PyString_Check(w))
550551
ps2 = PyString_AsString(w);
551552
}
552-
n = PyParser_ParseFile(fp, filename, &_PyParser_Grammar,
553-
Py_single_input, ps1, ps2, &err);
553+
n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
554+
Py_single_input, ps1, ps2, &err,
555+
(flags &&
556+
flags->cf_flags & PyCF_GENERATORS) ?
557+
PyPARSE_YIELD_IS_KEYWORD : 0);
554558
Py_XDECREF(v);
555559
Py_XDECREF(w);
556560
if (n == NULL) {
@@ -1017,7 +1021,9 @@ PyObject *
10171021
PyRun_FileExFlags(FILE *fp, char *filename, int start, PyObject *globals,
10181022
PyObject *locals, int closeit, PyCompilerFlags *flags)
10191023
{
1020-
node *n = PyParser_SimpleParseFile(fp, filename, start);
1024+
node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
1025+
(flags && flags->cf_flags & PyCF_GENERATORS) ?
1026+
PyPARSE_YIELD_IS_KEYWORD : 0);
10211027
if (closeit)
10221028
fclose(fp);
10231029
return run_err_node(n, filename, globals, locals, flags);
@@ -1101,7 +1107,9 @@ Py_CompileStringFlags(char *str, char *filename, int start,
11011107
{
11021108
node *n;
11031109
PyCodeObject *co;
1104-
n = PyParser_SimpleParseString(str, start);
1110+
n = PyParser_SimpleParseStringFlags(str, start,
1111+
(flags && flags->cf_flags & PyCF_GENERATORS) ?
1112+
PyPARSE_YIELD_IS_KEYWORD : 0);
11051113
if (n == NULL)
11061114
return NULL;
11071115
co = PyNode_CompileFlags(n, filename, flags);
@@ -1125,30 +1133,43 @@ Py_SymtableString(char *str, char *filename, int start)
11251133
/* Simplified interface to parsefile -- return node or set exception */
11261134

11271135
node *
1128-
PyParser_SimpleParseFile(FILE *fp, char *filename, int start)
1136+
PyParser_SimpleParseFileFlags(FILE *fp, char *filename, int start, int flags)
11291137
{
11301138
node *n;
11311139
perrdetail err;
1132-
n = PyParser_ParseFile(fp, filename, &_PyParser_Grammar, start,
1133-
(char *)0, (char *)0, &err);
1140+
n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start,
1141+
(char *)0, (char *)0, &err, flags);
11341142
if (n == NULL)
11351143
err_input(&err);
11361144
return n;
11371145
}
11381146

1147+
node *
1148+
PyParser_SimpleParseFile(FILE *fp, char *filename, int start)
1149+
{
1150+
return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
1151+
}
1152+
11391153
/* Simplified interface to parsestring -- return node or set exception */
11401154

11411155
node *
1142-
PyParser_SimpleParseString(char *str, int start)
1156+
PyParser_SimpleParseStringFlags(char *str, int start, int flags)
11431157
{
11441158
node *n;
11451159
perrdetail err;
1146-
n = PyParser_ParseString(str, &_PyParser_Grammar, start, &err);
1160+
n = PyParser_ParseStringFlags(str, &_PyParser_Grammar, start, &err,
1161+
flags);
11471162
if (n == NULL)
11481163
err_input(&err);
11491164
return n;
11501165
}
11511166

1167+
node *
1168+
PyParser_SimpleParseString(char *str, int start)
1169+
{
1170+
return PyParser_SimpleParseStringFlags(str, start, 0);
1171+
}
1172+
11521173
/* Set the error appropriate to the given input error code (see errcode.h) */
11531174

11541175
static void

0 commit comments

Comments
 (0)