From c49e67f64364ff287218c0997415c3bf6abe4815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 18 Mar 2026 18:29:34 -0400 Subject: [PATCH 1/2] Changes from 3.6 --- .../Compiler/Ast/AsyncForStatement.cs | 52 +++---- .../Compiler/Ast/AsyncWithStatement.cs | 55 +++---- .../Compiler/Ast/PythonNameBinder.cs | 5 - .../Compiler/Ast/PythonWalker.Generated.cs | 8 - .../Compiler/Ast/UnaryExpression.cs | 26 ++-- src/core/IronPython/Compiler/Parser.cs | 146 +++++++++++------- .../Runtime/Binding/PythonOperationKind.cs | 3 + .../Binding/PythonProtocol.Operations.cs | 6 + .../IronPython/Runtime/FunctionAttributes.cs | 2 +- tests/suite/test_async.py | 14 ++ 10 files changed, 162 insertions(+), 155 deletions(-) diff --git a/src/core/IronPython/Compiler/Ast/AsyncForStatement.cs b/src/core/IronPython/Compiler/Ast/AsyncForStatement.cs index 16def8c82..b32b891b1 100644 --- a/src/core/IronPython/Compiler/Ast/AsyncForStatement.cs +++ b/src/core/IronPython/Compiler/Ast/AsyncForStatement.cs @@ -6,7 +6,9 @@ using System.Threading; -using Microsoft.Scripting; +using IronPython.Runtime.Binding; +using IronPython.Runtime.Exceptions; + using MSAst = System.Linq.Expressions; namespace IronPython.Compiler.Ast { @@ -70,60 +72,42 @@ private Statement BuildDesugared() { var iterName = $"__asyncfor_iter{id}"; var runningName = $"__asyncfor_running{id}"; - // Helper to create nodes with proper parent and span - NameExpression MakeName(string name) { - var n = new NameExpression(name) { Parent = parent }; - n.IndexSpan = span; - return n; - } - - T WithSpan(T node) where T : Node { + // Helper to assign proper parent and span to nodes + T SetScope(T node) where T : Node { + node.Parent = parent; node.IndexSpan = span; return node; } // _iter = ITER.__aiter__() - var aiterAttr = WithSpan(new MemberExpression(List, "__aiter__") { Parent = parent }); - var aiterCall = WithSpan(new CallExpression(aiterAttr, null, null) { Parent = parent }); - var assignIter = WithSpan(new AssignmentStatement(new Expression[] { MakeName(iterName) }, aiterCall) { Parent = parent }); + var aiterCall = SetScope(new UnaryExpression(PythonOperationKind.AIter, List)); + var assignIter = SetScope(new AssignmentStatement([SetScope(new NameExpression(iterName))], aiterCall)); // running = True - var trueConst = new ConstantExpression(true) { Parent = parent }; trueConst.IndexSpan = span; - var assignRunning = WithSpan(new AssignmentStatement(new Expression[] { MakeName(runningName) }, trueConst) { Parent = parent }); + var trueConst = SetScope(new ConstantExpression(true)); + var assignRunning = SetScope(new AssignmentStatement([SetScope(new NameExpression(runningName))], trueConst)); // TARGET = await __aiter.__anext__() - var anextAttr = WithSpan(new MemberExpression(MakeName(iterName), "__anext__") { Parent = parent }); - var anextCall = WithSpan(new CallExpression(anextAttr, null, null) { Parent = parent }); + var anextCall = SetScope(new UnaryExpression(PythonOperationKind.ANext, SetScope(new NameExpression(iterName)))); var awaitNext = new AwaitExpression(anextCall); - var assignTarget = WithSpan(new AssignmentStatement(new Expression[] { Left }, awaitNext) { Parent = parent }); + var assignTarget = SetScope(new AssignmentStatement([Left], awaitNext)); // except StopAsyncIteration: __running = False - var falseConst = new ConstantExpression(false) { Parent = parent }; falseConst.IndexSpan = span; - var stopRunning = WithSpan(new AssignmentStatement( - new Expression[] { MakeName(runningName) }, falseConst) { Parent = parent }); - var handler = WithSpan(new TryStatementHandler( - MakeName("StopAsyncIteration"), - null!, - WithSpan(new SuiteStatement(new Statement[] { stopRunning }) { Parent = parent }) - ) { Parent = parent }); + var falseConst = SetScope(new ConstantExpression(false)); + var stopRunning = SetScope(new AssignmentStatement([SetScope(new NameExpression(runningName))], falseConst)); + var handler = SetScope(new TryStatementHandler(SetScope(new NameExpression(nameof(PythonExceptions.StopAsyncIteration))), null!, SetScope(new SuiteStatement([stopRunning])))); handler.HeaderIndex = span.End; // try/except/else block - var tryExcept = WithSpan(new TryStatement( - assignTarget, - new[] { handler }, - WithSpan(new SuiteStatement(new Statement[] { Body }) { Parent = parent }), - null! - ) { Parent = parent }); + var tryExcept = SetScope(new TryStatement(assignTarget, [handler], SetScope(new SuiteStatement([Body])), null)); tryExcept.HeaderIndex = span.End; // while __running: try/except/else - var whileStmt = new WhileStatement(MakeName(runningName), tryExcept, Else); + var whileStmt = new WhileStatement(SetScope(new NameExpression(runningName)), tryExcept, Else); whileStmt.SetLoc(GlobalParent, span.Start, span.End, span.End); whileStmt.Parent = parent; - var suite = WithSpan(new SuiteStatement(new Statement[] { assignIter, assignRunning, whileStmt }) { Parent = parent }); - return suite; + return SetScope(new SuiteStatement([assignIter, assignRunning, whileStmt])); } public override MSAst.Expression Reduce() { diff --git a/src/core/IronPython/Compiler/Ast/AsyncWithStatement.cs b/src/core/IronPython/Compiler/Ast/AsyncWithStatement.cs index 804926de8..3a33076eb 100644 --- a/src/core/IronPython/Compiler/Ast/AsyncWithStatement.cs +++ b/src/core/IronPython/Compiler/Ast/AsyncWithStatement.cs @@ -52,57 +52,42 @@ private Statement BuildDesugared() { // finally: // await mgr.__aexit__(None, None, None) - // Helper to create nodes with proper parent and span - NameExpression MakeName(string name) { - var n = new NameExpression(name) { Parent = parent }; - n.IndexSpan = span; - return n; + // Helper to assign proper parent and span to nodes + T SetScope(T node) where T : Node { + node.Parent = parent; + node.IndexSpan = span; + return node; } // mgr = EXPR - var assignMgr = new AssignmentStatement(new Expression[] { MakeName("__asyncwith_mgr") }, ContextManager) { Parent = parent }; - assignMgr.IndexSpan = span; + var assignMgr = SetScope(new AssignmentStatement([SetScope(new NameExpression("__asyncwith_mgr"))], ContextManager)); // await mgr.__aenter__() - var aenterAttr = new MemberExpression(MakeName("__asyncwith_mgr"), "__aenter__") { Parent = parent }; - aenterAttr.IndexSpan = span; - var aenterCall = new CallExpression(aenterAttr, null, null) { Parent = parent }; - aenterCall.IndexSpan = span; + var aenterAttr = SetScope(new MemberExpression(SetScope(new NameExpression("__asyncwith_mgr")), "__aenter__")); + var aenterCall = SetScope(new CallExpression(aenterAttr, null, null)); var awaitEnter = new AwaitExpression(aenterCall); Statement bodyStmt; - if (Variable != null) { + if (Variable is not null) { // VAR = await value; BLOCK - var assignVar = new AssignmentStatement(new Expression[] { Variable }, awaitEnter) { Parent = parent }; - assignVar.IndexSpan = span; - bodyStmt = new SuiteStatement(new Statement[] { assignVar, Body }) { Parent = parent }; + var assignVar = SetScope(new AssignmentStatement([Variable], awaitEnter)); + bodyStmt = new SuiteStatement([assignVar, Body]) { Parent = parent }; } else { - var exprStmt = new ExpressionStatement(awaitEnter) { Parent = parent }; - exprStmt.IndexSpan = span; - bodyStmt = new SuiteStatement(new Statement[] { exprStmt, Body }) { Parent = parent }; + var exprStmt = SetScope(new ExpressionStatement(awaitEnter)); + bodyStmt = new SuiteStatement([exprStmt, Body]) { Parent = parent }; } // await mgr.__aexit__(None, None, None) - var aexitAttr = new MemberExpression(MakeName("__asyncwith_mgr"), "__aexit__") { Parent = parent }; - aexitAttr.IndexSpan = span; - var none1 = new ConstantExpression(null) { Parent = parent }; none1.IndexSpan = span; - var none2 = new ConstantExpression(null) { Parent = parent }; none2.IndexSpan = span; - var none3 = new ConstantExpression(null) { Parent = parent }; none3.IndexSpan = span; - var aexitCallNormal = new CallExpression(aexitAttr, - new Expression[] { none1, none2, none3 }, null) { Parent = parent }; - aexitCallNormal.IndexSpan = span; + var aexitAttr = SetScope(new MemberExpression(SetScope(new NameExpression("__asyncwith_mgr")), "__aexit__")); + var none = SetScope(new ConstantExpression(null)); + var aexitCallNormal = SetScope(new CallExpression(aexitAttr, [none, none, none], null)); var awaitExitNormal = new AwaitExpression(aexitCallNormal); // try/finally: await __aexit__ on normal exit - var finallyExprStmt = new ExpressionStatement(awaitExitNormal) { Parent = parent }; - finallyExprStmt.IndexSpan = span; - var tryFinally = new TryStatement(bodyStmt, null, null, finallyExprStmt) { Parent = parent }; - tryFinally.IndexSpan = span; - tryFinally.HeaderIndex = span.End; - - var suite = new SuiteStatement(new Statement[] { assignMgr, tryFinally }) { Parent = parent }; - suite.IndexSpan = span; - return suite; + var finallyExprStmt = SetScope(new ExpressionStatement(awaitExitNormal)); + var tryFinally = SetScope(new TryStatement(bodyStmt, null, null, finallyExprStmt) { HeaderIndex = span.End }); + + return SetScope(new SuiteStatement([assignMgr, tryFinally])); } public override MSAst.Expression Reduce() { diff --git a/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs b/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs index 2110c69db..c2b118472 100644 --- a/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs +++ b/src/core/IronPython/Compiler/Ast/PythonNameBinder.cs @@ -351,11 +351,6 @@ public override bool Walk(AsyncForStatement node) { node.Parent = _currentScope; return base.Walk(node); } - // AsyncStatement - public override bool Walk(AsyncStatement node) { - node.Parent = _currentScope; - return base.Walk(node); - } // AsyncWithStatement public override bool Walk(AsyncWithStatement node) { node.Parent = _currentScope; diff --git a/src/core/IronPython/Compiler/Ast/PythonWalker.Generated.cs b/src/core/IronPython/Compiler/Ast/PythonWalker.Generated.cs index c97de6a5c..6175f6402 100644 --- a/src/core/IronPython/Compiler/Ast/PythonWalker.Generated.cs +++ b/src/core/IronPython/Compiler/Ast/PythonWalker.Generated.cs @@ -144,10 +144,6 @@ public virtual void PostWalk(AssignmentStatement node) { } public virtual bool Walk(AsyncForStatement node) { return true; } public virtual void PostWalk(AsyncForStatement node) { } - // AsyncStatement - public virtual bool Walk(AsyncStatement node) { return true; } - public virtual void PostWalk(AsyncStatement node) { } - // AsyncWithStatement public virtual bool Walk(AsyncWithStatement node) { return true; } public virtual void PostWalk(AsyncWithStatement node) { } @@ -415,10 +411,6 @@ public override void PostWalk(AssignmentStatement node) { } public override bool Walk(AsyncForStatement node) { return false; } public override void PostWalk(AsyncForStatement node) { } - // AsyncStatement - public override bool Walk(AsyncStatement node) { return false; } - public override void PostWalk(AsyncStatement node) { } - // AsyncWithStatement public override bool Walk(AsyncWithStatement node) { return false; } public override void PostWalk(AsyncWithStatement node) { } diff --git a/src/core/IronPython/Compiler/Ast/UnaryExpression.cs b/src/core/IronPython/Compiler/Ast/UnaryExpression.cs index dbdc75af7..8d02c67a7 100644 --- a/src/core/IronPython/Compiler/Ast/UnaryExpression.cs +++ b/src/core/IronPython/Compiler/Ast/UnaryExpression.cs @@ -2,20 +2,25 @@ // The .NET Foundation licenses this file to you under the Apache 2.0 License. // See the LICENSE file in the project root for more information. -using MSAst = System.Linq.Expressions; +#nullable enable -using System; using System.Diagnostics; using IronPython.Runtime.Binding; -namespace IronPython.Compiler.Ast { - using Ast = MSAst.Expression; - using AstUtils = Microsoft.Scripting.Ast.Utils; +using MSAst = System.Linq.Expressions; +namespace IronPython.Compiler.Ast { public class UnaryExpression : Expression { public UnaryExpression(PythonOperator op, Expression expression) { Operator = op; + OperationKind = PythonOperatorToOperatorString(op); + Expression = expression; + EndIndex = expression.EndIndex; + } + + internal UnaryExpression(PythonOperationKind op, Expression expression) { + OperationKind = op; Expression = expression; EndIndex = expression.EndIndex; } @@ -24,13 +29,10 @@ public UnaryExpression(PythonOperator op, Expression expression) { public PythonOperator Operator { get; } - public override MSAst.Expression Reduce() { - return GlobalParent.Operation( - typeof(object), - PythonOperatorToOperatorString(Operator), - Expression - ); - } + internal PythonOperationKind OperationKind { get; } + + public override MSAst.Expression Reduce() + => GlobalParent.Operation(typeof(object), OperationKind, Expression); public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { diff --git a/src/core/IronPython/Compiler/Parser.cs b/src/core/IronPython/Compiler/Parser.cs index 8853fe6f9..502658084 100644 --- a/src/core/IronPython/Compiler/Parser.cs +++ b/src/core/IronPython/Compiler/Parser.cs @@ -45,7 +45,7 @@ public class Parser : IDisposable { // TODO: remove IDisposable private SourceUnit _sourceUnit; /// - /// Language features initialized on parser construction and possibly updated during parsing. + /// Language features initialized on parser construction and possibly updated during parsing. /// The code can set the language features (e.g. "from __future__ import division"). /// private ModuleOptions _languageFeatures; @@ -376,7 +376,7 @@ internal void ReportSyntaxError(int start, int end, string message, int errorCod Severity.FatalError); } - #endregion + #endregion #region LL(1) Parsing @@ -403,7 +403,7 @@ private string ReadName() { } //stmt: simple_stmt | compound_stmt - //compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt + //compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt private Statement ParseStmt() { switch (PeekToken().Kind) { case TokenKind.KeywordIf: @@ -465,7 +465,7 @@ private Statement ParseSimpleStmt() { /* small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt - + del_stmt: 'del' exprlist pass_stmt: 'pass' flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt @@ -557,7 +557,7 @@ private Statement FinishSmallStmt(Statement stmt) { // yield_stmt: yield_expr private Statement ParseYieldStmt() { - // For yield statements, continue to enforce that it's currently in a function. + // For yield statements, continue to enforce that it's currently in a function. // This gives us better syntax error reporting for yield-statements than for yield-expressions. FunctionDefinition current = CurrentFunction; if (current == null) { @@ -577,7 +577,7 @@ private Statement ParseYieldStmt() { /// /// Peek if the next token is a 'yield' and parse a yield expression. Else return null. - /// + /// /// Called w/ yield already eaten. /// /// A yield expression if present, else null. @@ -589,7 +589,7 @@ private Expression ParseYieldExpression() { // Mark that this function is actually a generator. // If we're in a generator expression, then we don't have a function yet. // g=((yield i) for i in range(5)) - // In that acse, the genexp will mark IsGenerator. + // In that acse, the genexp will mark IsGenerator. FunctionDefinition current = CurrentFunction; if (current != null) { current.IsGenerator = true; @@ -761,7 +761,7 @@ private PythonOperator GetBinaryOperator(OperatorToken token) { } } - // import_stmt: 'import' module ['as' name"] (',' module ['as' name])* + // import_stmt: 'import' module ['as' name"] (',' module ['as' name])* // name: identifier private ImportStatement ParseImportStmt() { Eat(TokenKind.KeywordImport); @@ -834,8 +834,8 @@ private string[] ReadNames() { // 'from' relative_module 'import' identifier ['as' name] (',' identifier ['as' name]) * - // 'from' relative_module 'import' '(' identifier ['as' name] (',' identifier ['as' name])* [','] ')' - // 'from' module 'import' "*" + // 'from' relative_module 'import' '(' identifier ['as' name] (',' identifier ['as' name])* [','] ')' + // 'from' module 'import' "*" private FromImportStatement ParseFromImportStmt() { Eat(TokenKind.KeywordFrom); var start = GetStart(); @@ -1336,7 +1336,7 @@ private Expression FinishLambdef() { return ParseLambdaHelperEnd(func, expr); } - // Helpers for parsing lambda expressions. + // Helpers for parsing lambda expressions. // Usage // FunctionDefinition f = ParseLambdaHelperStart(string); // Expression expr = ParseXYZ(); @@ -1357,7 +1357,7 @@ private FunctionDefinition ParseLambdaHelperStart(string name) { } private Expression ParseLambdaHelperEnd(FunctionDefinition func, Expression expr) { - // Pep 342 in Python 2.5 allows Yield Expressions, which can occur inside a Lambda body. + // Pep 342 in Python 2.5 allows Yield Expressions, which can occur inside a Lambda body. // In this case, the lambda is a generator and will yield it's final result instead of just return it. Statement body; if (func.IsGenerator) { @@ -1417,17 +1417,16 @@ private WithStatement ParseWithStmt() { var withItem = ParseWithItem(); List items = null; while (MaybeEat(TokenKind.Comma)) { - if (items == null) { + if (items is null) { items = new List(); } items.Add(ParseWithItem()); } - var header = GetEnd(); Statement body = ParseSuite(); - if (items != null) { + if (items is not null) { for (int i = items.Count - 1; i >= 0; i--) { var curItem = items[i]; var innerWith = new WithStatement(curItem.ContextManager, curItem.Variable, body); @@ -1456,16 +1455,18 @@ private WithItem ParseWithItem() { } // async_stmt: 'async' (funcdef | with_stmt | for_stmt) - private Statement ParseAsyncStmt() { - var start = GetStart(); + private Statement ParseAsyncStmt(bool onlyAllowDef = false) { Eat(TokenKind.KeywordAsync); + var start = GetStart(); switch (PeekToken().Kind) { case TokenKind.KeywordDef: return ParseFuncDef(true); case TokenKind.KeywordWith: + if (onlyAllowDef) goto default; return ParseAsyncWithStmt(start); case TokenKind.KeywordFor: + if (onlyAllowDef) goto default; return ParseAsyncForStmt(start); default: ReportSyntaxError("invalid syntax"); @@ -1480,12 +1481,33 @@ private AsyncWithStatement ParseAsyncWithStmt(int asyncStart) { } Eat(TokenKind.KeywordWith); + var withItem = ParseWithItem(); + List items = null; + while (MaybeEat(TokenKind.Comma)) { + if (items is null) { + items = new List(); + } + + items.Add(ParseWithItem()); + } + var header = GetEnd(); Statement body = ParseSuite(); + if (items is not null) { + for (int i = items.Count - 1; i >= 0; i--) { + var curItem = items[i]; + var innerWith = new AsyncWithStatement(curItem.ContextManager, curItem.Variable, body); + innerWith.HeaderIndex = header; + innerWith.SetLoc(_globalParent, withItem.Start, GetEnd()); + body = innerWith; + header = GetEnd(); + } + } + AsyncWithStatement ret = new AsyncWithStatement(withItem.ContextManager, withItem.Variable, body); ret.HeaderIndex = header; - ret.SetLoc(_globalParent, asyncStart, GetEnd()); + ret.SetLoc(_globalParent, withItem.Start, GetEnd()); return ret; } @@ -1958,13 +1980,9 @@ private Expression FinishUnaryNegate() { return new UnaryExpression(PythonOperator.Negate, ParseFactor()); } - // power: ['await'] atom trailer* ['**' factor] + // power: atom_expr ['**' factor] private Expression ParsePower() { - if (MaybeEat(TokenKind.KeywordAwait)) { - return ParseAwaitExpression(); - } - Expression ret = ParseAtom(); - ret = AddTrailers(ret); + Expression ret = ParseAtomExpr(); if (MaybeEat(TokenKind.Power)) { var start = ret.StartIndex; ret = new BinaryExpression(PythonOperator.Power, ret, ParseFactor()); @@ -1973,25 +1991,27 @@ private Expression ParsePower() { return ret; } - // await_expr: 'await' unary_expr (essentially power level) - private Expression ParseAwaitExpression() { - FunctionDefinition current = CurrentFunction; - if (current == null || !current.IsAsync) { - ReportSyntaxError("'await' outside async function"); + // atom_expr: ['await'] atom trailer* + private Expression ParseAtomExpr() { + var isAsync = MaybeEat(TokenKind.KeywordAwait); + if (isAsync) { + FunctionDefinition current = CurrentFunction; + if (current is null || !current.IsAsync) { + ReportSyntaxError("'await' outside async function"); + } + if (current is not null) { + current.IsGenerator = true; + current.GeneratorStop = GeneratorStop; + } } - if (current != null) { - current.IsGenerator = true; - current.GeneratorStop = GeneratorStop; + Expression ret = ParseAtom(); + ret = AddTrailers(ret); + if (isAsync) { + var start = ret.StartIndex; + ret = new AwaitExpression(ret); + ret.SetLoc(_globalParent, start, GetEnd()); } - - var start = GetStart(); - - // Parse the awaitable expression at the unary level - Expression expr = ParsePower(); - - var ret = new AwaitExpression(expr); - ret.SetLoc(_globalParent, start, GetEnd()); return ret; } @@ -2620,7 +2640,7 @@ private Expression ParseGeneratorExpression(Expression expr) { // Generator Expressions have an implicit function definition and yield around their expression. // (x for i in R) // becomes: - // def f(): + // def f(): // for i in R: yield (x) ExpressionStatement ys = new ExpressionStatement(new YieldExpression(expr)); ys.Expression.SetLoc(_globalParent, expr.IndexSpan); @@ -3129,13 +3149,13 @@ private PythonAst ParseFileWorker(bool makeModule, bool returnValue) { List l = new List(); // - // A future statement must appear near the top of the module. - // The only lines that can appear before a future statement are: - // - the module docstring (if any), - // - comments, - // - blank lines, and - // - other future statements. - // + // A future statement must appear near the top of the module. + // The only lines that can appear before a future statement are: + // - the module docstring (if any), + // - comments, + // - blank lines, and + // - other future statements. + // MaybeEatNewLine(); @@ -3209,6 +3229,12 @@ private Statement InternalParseInteractiveInput(out bool parsingMultiLineCmpdStm } return null; + case TokenKind.KeywordAsync: + parsingMultiLineCmpdStmt = true; + s = ParseAsyncStmt(onlyAllowDef: true); + EatEndOfInput(); + break; + case TokenKind.KeywordIf: case TokenKind.KeywordWhile: case TokenKind.KeywordFor: @@ -3246,11 +3272,11 @@ private Expression ParseTestListAsExpression() { /// /// Maybe eats a new line token returning true if the token was /// eaten. - /// - /// Python always tokenizes to have only 1 new line character in a - /// row. But we also craete NLToken's and ignore them except for - /// error reporting purposes. This gives us the same errors as - /// CPython and also matches the behavior of the standard library + /// + /// Python always tokenizes to have only 1 new line character in a + /// row. But we also craete NLToken's and ignore them except for + /// error reporting purposes. This gives us the same errors as + /// CPython and also matches the behavior of the standard library /// tokenize module. This function eats any present NL tokens and throws /// them away. /// @@ -3263,12 +3289,12 @@ private bool MaybeEatNewLine() { } /// - /// Eats a new line token throwing if the next token isn't a new line. - /// - /// Python always tokenizes to have only 1 new line character in a - /// row. But we also craete NLToken's and ignore them except for - /// error reporting purposes. This gives us the same errors as - /// CPython and also matches the behavior of the standard library + /// Eats a new line token throwing if the next token isn't a new line. + /// + /// Python always tokenizes to have only 1 new line character in a + /// row. But we also craete NLToken's and ignore them except for + /// error reporting purposes. This gives us the same errors as + /// CPython and also matches the behavior of the standard library /// tokenize module. This function eats any present NL tokens and throws /// them away. /// @@ -3294,7 +3320,7 @@ private Token EatEndOfInput() { if (_sourceReader.BaseReader is StreamReader sr && sr.BaseStream.CanSeek) { // TODO: Convert exception index to proper SourceLocation } - // BUG: We have some weird stream and we can't accurately track the + // BUG: We have some weird stream and we can't accurately track the // position where the exception came from. There are too many levels // of buffering below us to re-wind and calculate the actual line number, so // we'll give the last line number the tokenizer was at. diff --git a/src/core/IronPython/Runtime/Binding/PythonOperationKind.cs b/src/core/IronPython/Runtime/Binding/PythonOperationKind.cs index 3f540daed..c654361eb 100644 --- a/src/core/IronPython/Runtime/Binding/PythonOperationKind.cs +++ b/src/core/IronPython/Runtime/Binding/PythonOperationKind.cs @@ -106,6 +106,9 @@ internal enum PythonOperationKind { /// GetEnumeratorForIteration, + AIter, + ANext, + ///Operator for performing add Add, ///Operator for performing sub diff --git a/src/core/IronPython/Runtime/Binding/PythonProtocol.Operations.cs b/src/core/IronPython/Runtime/Binding/PythonProtocol.Operations.cs index 61f916e14..5bdad3a51 100644 --- a/src/core/IronPython/Runtime/Binding/PythonProtocol.Operations.cs +++ b/src/core/IronPython/Runtime/Binding/PythonProtocol.Operations.cs @@ -190,6 +190,12 @@ internal static partial class PythonProtocol { case PythonOperationKind.GetEnumeratorForIteration: res = MakeEnumeratorOperation(operation, args[0]); break; + case PythonOperationKind.AIter: + res = MakeUnaryOperation(operation, args[0], "__aiter__", TypeError(operation, "'async for' requires an object with __aiter__ method, got {0}", args)); + break; + case PythonOperationKind.ANext: + res = MakeUnaryOperation(operation, args[0], "__anext__", TypeError(operation, "'async for' received an invalid object from __aiter__: {0}", args)); + break; default: res = BindingHelpers.AddPythonBoxing(MakeBinaryOperation(operation, args, operation.Operation, null)); break; diff --git a/src/core/IronPython/Runtime/FunctionAttributes.cs b/src/core/IronPython/Runtime/FunctionAttributes.cs index 8e7f667e5..5f2a7196c 100644 --- a/src/core/IronPython/Runtime/FunctionAttributes.cs +++ b/src/core/IronPython/Runtime/FunctionAttributes.cs @@ -25,7 +25,7 @@ public enum FunctionAttributes { /// /// Set if the function is a coroutine (async def). /// - Coroutine = 0x100, + Coroutine = 0x80, /// /// IronPython specific: Set if the function includes nested exception handling and therefore can alter /// sys.exc_info(). diff --git a/tests/suite/test_async.py b/tests/suite/test_async.py index 59ae3500a..b6c24bd09 100644 --- a/tests/suite/test_async.py +++ b/tests/suite/test_async.py @@ -296,6 +296,20 @@ async def test(): self.assertEqual(run_coro(test()), [110, 120, 210, 220]) + def test_special_method_lookup(self): + """Ensure async for looks up __aiter__/__anext__ on the type, not the instance.""" + + a = AsyncIter([1, 2, 3]) + a.__aiter__ = lambda: AsyncIter([98]) # should be ignored + a.__anext__ = lambda: 99 # should be ignored + + async def test(): + result = [] + async for x in a: + result.append(x) + return result + + self.assertEqual(run_coro(test()), [1, 2, 3]) class AsyncCombinedTest(unittest.TestCase): """Tests combining async with and async for.""" From 13242cc1b5b4acc416fa7c3671dfc727c4d03425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Lozier?= Date: Wed, 18 Mar 2026 22:36:20 -0400 Subject: [PATCH 2/2] Remove AsyncStatement.cs --- .../IronPython/Compiler/Ast/AsyncStatement.cs | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 src/core/IronPython/Compiler/Ast/AsyncStatement.cs diff --git a/src/core/IronPython/Compiler/Ast/AsyncStatement.cs b/src/core/IronPython/Compiler/Ast/AsyncStatement.cs deleted file mode 100644 index cfae57eef..000000000 --- a/src/core/IronPython/Compiler/Ast/AsyncStatement.cs +++ /dev/null @@ -1,15 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the Apache 2.0 License. -// See the LICENSE file in the project root for more information. - -#nullable enable - -using System; - -namespace IronPython.Compiler.Ast { - public class AsyncStatement : Statement { - public override void Walk(PythonWalker walker) { - throw new NotImplementedException(); - } - } -}