From 3e0a6f37dfdd595be737baae00ec0e036a912615 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Fri, 1 May 2020 06:27:52 +0300 Subject: bpo-40334: Add support for feature_version in new PEG parser (GH-19827) `ast.parse` and `compile` support a `feature_version` parameter that tells the parser to parse the input string, as if it were written in an older Python version. The `feature_version` is propagated to the tokenizer, which uses it to handle the three different stages of support for `async` and `await`. Additionally, it disallows the following at parser level: - The '@' operator in < 3.5 - Async functions in < 3.5 - Async comprehensions in < 3.6 - Underscores in numeric literals in < 3.6 - Await expression in < 3.5 - Variable annotations in < 3.6 - Async for-loops in < 3.5 - Async with-statements in < 3.5 - F-strings in < 3.6 Closes we-like-parsers/cpython#124. --- Grammar/python.gram | 80 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 51 insertions(+), 29 deletions(-) (limited to 'Grammar') diff --git a/Grammar/python.gram b/Grammar/python.gram index e1164d990a..38107fcf73 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -80,10 +80,14 @@ compound_stmt[stmt_ty]: # NOTE: annotated_rhs may start with 'yield'; yield_expr must start with 'yield' assignment: | a=NAME ':' b=expression c=['=' d=annotated_rhs { d }] { - _Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) } + CHECK_VERSION( + 6, + "Variable annotation syntax is", + _Py_AnnAssign(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, c, 1, EXTRA) + ) } | a=('(' b=inside_paren_ann_assign_target ')' { b } | ann_assign_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { - _Py_AnnAssign(a, b, c, 0, EXTRA)} + CHECK_VERSION(6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) } | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) tc=[TYPE_COMMENT] { _Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } | a=target b=augassign c=(yield_expr | star_expressions) { @@ -91,19 +95,19 @@ assignment: | invalid_assignment augassign[AugOperator*]: - | '+=' {_PyPegen_augoperator(p, Add)} - | '-=' {_PyPegen_augoperator(p, Sub)} - | '*=' {_PyPegen_augoperator(p, Mult)} - | '@=' {_PyPegen_augoperator(p, MatMult)} - | '/=' {_PyPegen_augoperator(p, Div)} - | '%=' {_PyPegen_augoperator(p, Mod)} - | '&=' {_PyPegen_augoperator(p, BitAnd)} - | '|=' {_PyPegen_augoperator(p, BitOr)} - | '^=' {_PyPegen_augoperator(p, BitXor)} - | '<<=' {_PyPegen_augoperator(p, LShift)} - | '>>=' {_PyPegen_augoperator(p, RShift)} - | '**=' {_PyPegen_augoperator(p, Pow)} - | '//=' {_PyPegen_augoperator(p, FloorDiv)} + | '+=' { _PyPegen_augoperator(p, Add) } + | '-=' { _PyPegen_augoperator(p, Sub) } + | '*=' { _PyPegen_augoperator(p, Mult) } + | '@=' { CHECK_VERSION(5, "The '@' operator is", _PyPegen_augoperator(p, MatMult)) } + | '/=' { _PyPegen_augoperator(p, Div) } + | '%=' { _PyPegen_augoperator(p, Mod) } + | '&=' { _PyPegen_augoperator(p, BitAnd) } + | '|=' { _PyPegen_augoperator(p, BitOr) } + | '^=' { _PyPegen_augoperator(p, BitXor) } + | '<<=' { _PyPegen_augoperator(p, LShift) } + | '>>=' { _PyPegen_augoperator(p, RShift) } + | '**=' { _PyPegen_augoperator(p, Pow) } + | '//=' { _PyPegen_augoperator(p, FloorDiv) } global_stmt[stmt_ty]: 'global' a=','.NAME+ { _Py_Global(CHECK(_PyPegen_map_names_to_ids(p, a)), EXTRA) } @@ -156,14 +160,20 @@ while_stmt[stmt_ty]: | 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) } for_stmt[stmt_ty]: - | is_async=[ASYNC] 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { - (is_async ? _Py_AsyncFor : _Py_For)(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { + _Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | ASYNC 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { + CHECK_VERSION(5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) } with_stmt[stmt_ty]: - | is_async=[ASYNC] 'with' '(' a=','.with_item+ ')' ':' b=block { - (is_async ? _Py_AsyncWith : _Py_With)(a, b, NULL, EXTRA) } - | is_async=[ASYNC] 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { - (is_async ? _Py_AsyncWith : _Py_With)(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | 'with' '(' a=','.with_item+ ')' ':' b=block { + _Py_With(a, b, NULL, EXTRA) } + | 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { + _Py_With(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | ASYNC 'with' '(' a=','.with_item+ ')' ':' b=block { + CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NULL, EXTRA)) } + | ASYNC 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { + CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } with_item[withitem_ty]: | e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) } @@ -188,10 +198,18 @@ function_def[stmt_ty]: | function_def_raw function_def_raw[stmt_ty]: - | is_async=[ASYNC] 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { - (is_async ? _Py_AsyncFunctionDef : _Py_FunctionDef)(n->v.Name.id, - (params) ? params : CHECK(_PyPegen_empty_arguments(p)), - b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { + _Py_FunctionDef(n->v.Name.id, + (params) ? params : CHECK(_PyPegen_empty_arguments(p)), + b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) } + | ASYNC 'def' n=NAME '(' params=[params] ')' a=['->' z=expression { z }] ':' tc=[func_type_comment] b=block { + CHECK_VERSION( + 5, + "Async functions are", + _Py_AsyncFunctionDef(n->v.Name.id, + (params) ? params : CHECK(_PyPegen_empty_arguments(p)), + b, NULL, a, NEW_TYPE_COMMENT(p, tc), EXTRA) + ) } func_type_comment[PyObject*]: | NEWLINE t=TYPE_COMMENT &(NEWLINE INDENT) { t } # Must be followed by indented block | invalid_double_type_comments @@ -399,7 +417,7 @@ term[expr_ty]: | a=term '/' b=factor { _Py_BinOp(a, Div, b, EXTRA) } | a=term '//' b=factor { _Py_BinOp(a, FloorDiv, b, EXTRA) } | a=term '%' b=factor { _Py_BinOp(a, Mod, b, EXTRA) } - | a=term '@' b=factor { _Py_BinOp(a, MatMult, b, EXTRA) } + | a=term '@' b=factor { CHECK_VERSION(5, "The '@' operator is", _Py_BinOp(a, MatMult, b, EXTRA)) } | factor factor[expr_ty] (memo): | '+' a=factor { _Py_UnaryOp(UAdd, a, EXTRA) } @@ -410,7 +428,7 @@ power[expr_ty]: | a=await_primary '**' b=factor { _Py_BinOp(a, Pow, b, EXTRA) } | await_primary await_primary[expr_ty] (memo): - | AWAIT a=primary { _Py_Await(a, EXTRA) } + | AWAIT a=primary { CHECK_VERSION(5, "Await expressions are", _Py_Await(a, EXTRA)) } | primary primary[expr_ty]: | a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } @@ -469,8 +487,12 @@ kvpair[KeyValuePair*]: | '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) } | a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) } for_if_clauses[asdl_seq*]: - | a=(y=[ASYNC] 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* - { _Py_comprehension(a, b, c, y != NULL, p->arena) })+ { a } + | for_if_clause+ +for_if_clause[comprehension_ty]: + | ASYNC 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* { + CHECK_VERSION(6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) } + | 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* { + _Py_comprehension(a, b, c, 0, p->arena) } yield_expr[expr_ty]: | 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) } -- cgit v1.2.3-65-gdbad