From 417206a05c4545bde96c2bbbea92b53e6cac0d48 Mon Sep 17 00:00:00 2001
From: Pablo Galindo Salgado <Pablogsal@gmail.com>
Date: Wed, 30 Nov 2022 11:36:06 +0000
Subject: [PATCH] gh-99891: Fix infinite recursion in the tokenizer when
showing warnings (GH-99893)
Automerge-Triggered-By: GH:pablogsal
---
Lib/test/test_source_encoding.py | 12 ++++++++++++
.../2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst | 3 +++
Parser/tokenizer.c | 8 ++++++++
Parser/tokenizer.h | 1 +
4 files changed, 24 insertions(+)
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst
diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py
index cfc4b13f18f..b05173ad00d 100644
--- a/Lib/test/test_source_encoding.py
+++ b/Lib/test/test_source_encoding.py
@@ -160,6 +160,18 @@ def test_file_parse_error_multiline(self):
finally:
os.unlink(TESTFN)
+ def test_tokenizer_fstring_warning_in_first_line(self):
+ source = "0b1and 2"
+ with open(TESTFN, "w") as fd:
+ fd.write("{}".format(source))
+ try:
+ retcode, stdout, stderr = script_helper.assert_python_ok(TESTFN)
+ self.assertIn(b"SyntaxWarning: invalid binary litera", stderr)
+ self.assertEqual(stderr.count(source.encode()), 1)
+ finally:
+ os.unlink(TESTFN)
+
+
class AbstractSourceEncodingTest:
def test_default_coding(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst
new file mode 100644
index 00000000000..20cd361affe
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2022-11-30-11-09-40.gh-issue-99891.9VomwB.rst
@@ -0,0 +1,3 @@
+Fix a bug in the tokenizer that could cause infinite recursion when showing
+syntax warnings that happen in the first line of the source. Patch by Pablo
+Galindo
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index ce72e152902..463c0e00ca1 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -97,6 +97,7 @@ tok_new(void)
tok->async_def_nl = 0;
tok->interactive_underflow = IUNDERFLOW_NORMAL;
tok->str = NULL;
+ tok->report_warnings = 1;
#ifdef Py_DEBUG
tok->debug = _Py_GetConfig()->parser_debug;
#endif
@@ -1201,6 +1202,10 @@ indenterror(struct tok_state *tok)
static int
parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...)
{
+ if (!tok->report_warnings) {
+ return 0;
+ }
+
PyObject *errmsg;
va_list vargs;
va_start(vargs, format);
@@ -2239,6 +2244,9 @@ _PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
}
}
struct token token;
+ // We don't want to report warnings here because it could cause infinite recursion
+ // if fetching the encoding shows a warning.
+ tok->report_warnings = 0;
while (tok->lineno < 2 && tok->done == E_OK) {
_PyTokenizer_Get(tok, &token);
}
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 2542d30e1da..16a94d5f51d 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -92,6 +92,7 @@ struct tok_state {
NEWLINE token after it. */
/* How to proceed when asked for a new token in interactive mode */
enum interactive_underflow_t interactive_underflow;
+ int report_warnings;
#ifdef Py_DEBUG
int debug;
#endif
--
GitLab