diff options
| author | Lysandros Nikolaou <lisandrosnik@gmail.com> | 2020-06-11 02:56:08 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-06-11 00:56:08 +0100 |
| commit | 896f4cf63f9ab93e30572d879a5719d5aa2499fb (patch) | |
| tree | b24213dfd64cf197fcab434adce912f37c3b87a7 | |
| parent | 7f888c7ef905842bf7739cc03bd20398329951b5 (diff) | |
| download | cpython-896f4cf63f9ab93e30572d879a5719d5aa2499fb.tar.gz cpython-896f4cf63f9ab93e30572d879a5719d5aa2499fb.zip | |
bpo-40847: Consider a line with only a LINECONT a blank line (GH-20769)
A line with only a line continuation character should be considered
a blank line at tokenizer level so that only a single NEWLINE token
gets emitted. The old parser was working around the issue, but the
new parser threw a `SyntaxError` for valid input. For example,
an empty line following a line continuation character was interpreted
as a `SyntaxError`.
Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
| -rw-r--r-- | Lib/test/test_peg_parser.py | 7 | ||||
| -rw-r--r-- | Lib/test/test_syntax.py | 14 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst | 4 | ||||
| -rw-r--r-- | Parser/tokenizer.c | 3 |
4 files changed, 27 insertions, 1 deletions
diff --git a/Lib/test/test_peg_parser.py b/Lib/test/test_peg_parser.py index 6ccb2573176..fae85e323da 100644 --- a/Lib/test/test_peg_parser.py +++ b/Lib/test/test_peg_parser.py @@ -153,6 +153,13 @@ TEST_CASES = [ ('dict_comp', '{x:1 for x in a}'), ('dict_comp_if', '{x:1+2 for x in a if b}'), ('dict_empty', '{}'), + ('empty_line_after_linecont', + r''' + pass + \ + + pass + '''), ('for', ''' for i in a: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index f41426a4e9d..0c207ec8fc0 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -858,6 +858,20 @@ class SyntaxTestCase(unittest.TestCase): "iterable argument unpacking follows " "keyword argument unpacking") + def test_empty_line_after_linecont(self): + # See issue-40847 + s = r"""\ +pass + \ + +pass +""" + try: + compile(s, '<string>', 'exec') + except SyntaxError: + self.fail("Empty line after a line continuation character is valid.") + + def test_main(): support.run_unittest(SyntaxTestCase) from test import test_syntax diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst b/Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst new file mode 100644 index 00000000000..0b489f24832 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-06-09-23-52-32.bpo-40847.4XAACw.rst @@ -0,0 +1,4 @@ +Fix a bug where a line with only a line continuation character is not considered a blank line at tokenizer level.
+In such cases, more than a single `NEWLINE` token was emitted. The old parser was working around the issue,
+but the new parser threw a :exc:`SyntaxError` for valid input due to this. For example, an empty line following
+a line continuation character was interpreted as a :exc:`SyntaxError`.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index cebfadc8e89..d461e4e24e7 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -1203,8 +1203,9 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end) } } tok_backup(tok, c); - if (c == '#' || c == '\n') { + if (c == '#' || c == '\n' || c == '\\') { /* Lines with only whitespace and/or comments + and/or a line continuation character shouldn't affect the indentation and are not passed to the parser as NEWLINE tokens, except *totally* empty lines in interactive |