Description: Explicitly raise TokenError when the source cannot be parsed by ast
Author: Miro Hrončok <miro@hroncok.cz>
Origin: upstream
Bug: https://github.com/ZanderBrown/nudatus/pull/11
Bug-Debian: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1058114
Applied-Upstream: https://github.com/ZanderBrown/nudatus/commit/7503c8fdef5cb11f8118a3342085055f0accbca7
Reviewed-by: Nick Morrott <nickm@debian.org>
Last-Update: 2024-02-23
---
--- a/nudatus.py
+++ b/nudatus.py
@@ -10,11 +10,13 @@
 """
 
 import argparse
+import ast
 import sys
 import token
 import tokenize
 from io import BytesIO
 from tokenize import tokenize as tokenizer
+from tokenize import TokenError
 from typing import List, Optional
 
 _VERSION = (
@@ -48,6 +50,15 @@
     last_line_text = ""
     open_list_dicts = 0
 
+    # Parsing invalid code via the tokenizer is documented as "undefined".
+    # Python 3.12 sometimes successfully parses invalid code.
+    # To restore behavior before Python 3.12, we raise explicitly
+    # if ast cannot parse this:
+    try:
+        ast.parse(text_bytes)
+    except SyntaxError as e:
+        raise TokenError(e)
+
     # Build tokens from the script
     tokens = tokenizer(buff.readline)
     for t, text, (line_s, col_s), (line_e, col_e), line in tokens:
