aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Tools
diff options
context:
space:
mode:
Diffstat (limited to 'Tools')
-rw-r--r--Tools/build/generate_sbom.py2
-rw-r--r--Tools/cases_generator/analyzer.py16
-rw-r--r--Tools/cases_generator/generators_common.py4
-rw-r--r--Tools/cases_generator/optimizer_generator.py4
-rw-r--r--Tools/cases_generator/parsing.py12
-rw-r--r--Tools/cases_generator/stack.py22
-rwxr-xr-xTools/i18n/makelocalealias.py3
-rw-r--r--Tools/jit/_targets.py5
-rw-r--r--Tools/jit/build.py4
-rw-r--r--Tools/msi/freethreaded/freethreaded_files.wxs2
-rw-r--r--Tools/msi/lib/lib_files.wxs2
-rw-r--r--Tools/peg_generator/pegen/c_generator.py60
12 files changed, 74 insertions, 62 deletions
diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py
index ecb7b54f6d8..df52f8de762 100644
--- a/Tools/build/generate_sbom.py
+++ b/Tools/build/generate_sbom.py
@@ -169,7 +169,7 @@ def download_with_retries(download_location: str,
base_delay: float = 2.25,
max_jitter: float = 1.0) -> typing.Any:
"""Download a file with exponential backoff retry."""
- for attempt in range(max_retries):
+ for attempt in range(max_retries + 1):
try:
resp = urllib.request.urlopen(download_location)
except urllib.error.URLError as ex:
diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py
index 3070559db8a..ca6d0301f35 100644
--- a/Tools/cases_generator/analyzer.py
+++ b/Tools/cases_generator/analyzer.py
@@ -135,15 +135,13 @@ class Flush:
@dataclass
class StackItem:
name: str
- type: str | None
size: str
peek: bool = False
used: bool = False
def __str__(self) -> str:
size = f"[{self.size}]" if self.size else ""
- type = "" if self.type is None else f"{self.type} "
- return f"{type}{self.name}{size} {self.peek}"
+ return f"{self.name}{size} {self.peek}"
def is_array(self) -> bool:
return self.size != ""
@@ -345,7 +343,7 @@ def override_error(
def convert_stack_item(
item: parser.StackEffect, replace_op_arg_1: str | None
) -> StackItem:
- return StackItem(item.name, item.type, item.size)
+ return StackItem(item.name, item.size)
def check_unused(stack: list[StackItem], input_names: dict[str, lexer.Token]) -> None:
"Unused items cannot be on the stack above used, non-peek items"
@@ -637,6 +635,10 @@ NON_ESCAPING_FUNCTIONS = (
"_PyLong_IsNegative",
"_PyLong_IsNonNegativeCompact",
"_PyLong_IsZero",
+ "_PyLong_BothAreCompact",
+ "_PyCompactLong_Add",
+ "_PyCompactLong_Multiply",
+ "_PyCompactLong_Subtract",
"_PyManagedDictPointer_IsValues",
"_PyObject_GC_IS_SHARED",
"_PyObject_GC_IS_TRACKED",
@@ -681,6 +683,10 @@ NON_ESCAPING_FUNCTIONS = (
"PyStackRef_UntagInt",
"PyStackRef_IncrementTaggedIntNoOverflow",
"PyStackRef_IsNullOrInt",
+ "PyStackRef_IsError",
+ "PyStackRef_IsValid",
+ "PyStackRef_Wrap",
+ "PyStackRef_Unwrap",
)
@@ -809,7 +815,7 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool:
if len(stack_inputs) == 0:
return False
return all(
- (s.name == other.name and s.type == other.type and s.size == other.size)
+ (s.name == other.name and s.size == other.size)
for s, other in zip(stack_inputs, instr.outputs)
)
diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py
index 02f9a952754..47de205c0e9 100644
--- a/Tools/cases_generator/generators_common.py
+++ b/Tools/cases_generator/generators_common.py
@@ -56,9 +56,7 @@ def root_relative_path(filename: str) -> str:
def type_and_null(var: StackItem) -> tuple[str, str]:
- if var.type:
- return var.type, "NULL"
- elif var.is_array():
+ if var.is_array():
return "_PyStackRef *", "NULL"
else:
return "_PyStackRef", "PyStackRef_NULL"
diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py
index fda022a44e5..75805dbd7f3 100644
--- a/Tools/cases_generator/optimizer_generator.py
+++ b/Tools/cases_generator/optimizer_generator.py
@@ -73,8 +73,6 @@ def validate_uop(override: Uop, uop: Uop) -> None:
def type_name(var: StackItem) -> str:
if var.is_array():
return "JitOptSymbol **"
- if var.type:
- return var.type
return "JitOptSymbol *"
@@ -230,7 +228,7 @@ def generate_abstract_interpreter(
declare_variables(override, out, skip_inputs=False)
else:
declare_variables(uop, out, skip_inputs=True)
- stack = Stack(extract_bits=False, cast_type="JitOptSymbol *")
+ stack = Stack()
write_uop(override, uop, out, stack, debug, skip_inputs=(override is None))
out.start_line()
out.emit("break;\n")
diff --git a/Tools/cases_generator/parsing.py b/Tools/cases_generator/parsing.py
index 9c9b0053a59..a6dac481875 100644
--- a/Tools/cases_generator/parsing.py
+++ b/Tools/cases_generator/parsing.py
@@ -247,12 +247,11 @@ class SimpleStmt(Stmt):
@dataclass
class StackEffect(Node):
name: str = field(compare=False) # __eq__ only uses type, cond, size
- type: str = "" # Optional `:type`
size: str = "" # Optional `[size]`
# Note: size cannot be combined with type or cond
def __repr__(self) -> str:
- items = [self.name, self.type, self.size]
+ items = [self.name, self.size]
while items and items[-1] == "":
del items[-1]
return f"StackEffect({', '.join(repr(item) for item in items)})"
@@ -463,20 +462,13 @@ class Parser(PLexer):
# IDENTIFIER [':' IDENTIFIER [TIMES]] ['if' '(' expression ')']
# | IDENTIFIER '[' expression ']'
if tkn := self.expect(lx.IDENTIFIER):
- type_text = ""
- if self.expect(lx.COLON):
- type_text = self.require(lx.IDENTIFIER).text.strip()
- if self.expect(lx.TIMES):
- type_text += " *"
size_text = ""
if self.expect(lx.LBRACKET):
- if type_text:
- raise self.make_syntax_error("Unexpected [")
if not (size := self.expression()):
raise self.make_syntax_error("Expected expression")
self.require(lx.RBRACKET)
size_text = size.text.strip()
- return StackEffect(tkn.text, type_text, size_text)
+ return StackEffect(tkn.text, size_text)
return None
@contextual
diff --git a/Tools/cases_generator/stack.py b/Tools/cases_generator/stack.py
index 6b681775f48..3a0e7e5d0d5 100644
--- a/Tools/cases_generator/stack.py
+++ b/Tools/cases_generator/stack.py
@@ -168,7 +168,7 @@ class Local:
@staticmethod
def register(name: str) -> "Local":
- item = StackItem(name, None, "", False, True)
+ item = StackItem(name, "", False, True)
return Local(item, None, True)
def kill(self) -> None:
@@ -216,13 +216,11 @@ def array_or_scalar(var: StackItem | Local) -> str:
return "array" if var.is_array() else "scalar"
class Stack:
- def __init__(self, extract_bits: bool=True, cast_type: str = "uintptr_t") -> None:
+ def __init__(self) -> None:
self.base_offset = PointerOffset.zero()
self.physical_sp = PointerOffset.zero()
self.logical_sp = PointerOffset.zero()
self.variables: list[Local] = []
- self.extract_bits = extract_bits
- self.cast_type = cast_type
def drop(self, var: StackItem, check_liveness: bool) -> None:
self.logical_sp = self.logical_sp.pop(var)
@@ -268,10 +266,8 @@ class Stack:
self.base_offset = self.logical_sp
if var.name in UNUSED or not var.used:
return Local.unused(var, self.base_offset)
- cast = f"({var.type})" if (not indirect and var.type) else ""
- bits = ".bits" if cast and self.extract_bits else ""
c_offset = (self.base_offset - self.physical_sp).to_c()
- assign = f"{var.name} = {cast}{indirect}stack_pointer[{c_offset}]{bits};\n"
+ assign = f"{var.name} = {indirect}stack_pointer[{c_offset}];\n"
out.emit(assign)
self._print(out)
return Local.from_memory(var, self.base_offset)
@@ -292,12 +288,8 @@ class Stack:
out: CWriter,
var: StackItem,
stack_offset: PointerOffset,
- cast_type: str,
- extract_bits: bool,
) -> None:
- cast = f"({cast_type})" if var.type else ""
- bits = ".bits" if cast and extract_bits else ""
- out.emit(f"stack_pointer[{stack_offset.to_c()}]{bits} = {cast}{var.name};\n")
+ out.emit(f"stack_pointer[{stack_offset.to_c()}] = {var.name};\n")
def _save_physical_sp(self, out: CWriter) -> None:
if self.physical_sp != self.logical_sp:
@@ -320,7 +312,7 @@ class Stack:
self._print(out)
var.memory_offset = var_offset
stack_offset = var_offset - self.physical_sp
- Stack._do_emit(out, var.item, stack_offset, self.cast_type, self.extract_bits)
+ Stack._do_emit(out, var.item, stack_offset)
self._print(out)
var_offset = var_offset.push(var.item)
@@ -350,7 +342,7 @@ class Stack:
out.emit(self.as_comment() + "\n")
def copy(self) -> "Stack":
- other = Stack(self.extract_bits, self.cast_type)
+ other = Stack()
other.base_offset = self.base_offset
other.physical_sp = self.physical_sp
other.logical_sp = self.logical_sp
@@ -496,7 +488,7 @@ class Storage:
f"Expected '{undefined}' to be defined before '{out.name}'"
else:
undefined = out.name
- while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[0]):
+ while len(self.outputs) > self.peeks and not self.needs_defining(self.outputs[self.peeks]):
out = self.outputs.pop(self.peeks)
self.stack.push(out)
diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py
index b407a8a643b..02af1caff7d 100755
--- a/Tools/i18n/makelocalealias.py
+++ b/Tools/i18n/makelocalealias.py
@@ -140,6 +140,9 @@ if __name__ == '__main__':
data = locale.locale_alias.copy()
data.update(parse_glibc_supported(args.glibc_supported))
data.update(parse(args.locale_alias))
+ # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist
+ # on all platforms.
+ data['c.utf8'] = 'C.UTF-8'
while True:
# Repeat optimization while the size is decreased.
n = len(data)
diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py
index d0a1c081ffe..b383e39da19 100644
--- a/Tools/jit/_targets.py
+++ b/Tools/jit/_targets.py
@@ -10,6 +10,7 @@ import re
import sys
import tempfile
import typing
+import shlex
import _llvm
import _schema
@@ -46,6 +47,7 @@ class _Target(typing.Generic[_S, _R]):
stable: bool = False
debug: bool = False
verbose: bool = False
+ cflags: str = ""
known_symbols: dict[str, int] = dataclasses.field(default_factory=dict)
pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve()
@@ -62,6 +64,7 @@ class _Target(typing.Generic[_S, _R]):
hasher = hashlib.sha256()
hasher.update(self.triple.encode())
hasher.update(self.debug.to_bytes())
+ hasher.update(self.cflags.encode())
# These dependencies are also reflected in _JITSources in regen.targets:
hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes())
hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes())
@@ -155,6 +158,8 @@ class _Target(typing.Generic[_S, _R]):
f"{o}",
f"{c}",
*self.args,
+ # Allow user-provided CFLAGS to override any defaults
+ *shlex.split(self.cflags),
]
await _llvm.run("clang", args, echo=self.verbose)
return await self._parse(o)
diff --git a/Tools/jit/build.py b/Tools/jit/build.py
index 1afd0c76bad..a0733005929 100644
--- a/Tools/jit/build.py
+++ b/Tools/jit/build.py
@@ -39,11 +39,15 @@ if __name__ == "__main__":
parser.add_argument(
"-v", "--verbose", action="store_true", help="echo commands as they are run"
)
+ parser.add_argument(
+ "--cflags", help="additional flags to pass to the compiler", default=""
+ )
args = parser.parse_args()
for target in args.target:
target.debug = args.debug
target.force = args.force
target.verbose = args.verbose
+ target.cflags = args.cflags
target.pyconfig_dir = args.pyconfig_dir
target.build(
comment=comment,
diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs
index 86d9a8b83f6..0707e77b5e9 100644
--- a/Tools/msi/freethreaded/freethreaded_files.wxs
+++ b/Tools/msi/freethreaded/freethreaded_files.wxs
@@ -103,7 +103,7 @@
</ComponentGroup>
</Fragment>
- <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_uuid;_wmi;_zoneinfo;_zstd;_testcapi;_ctypes_test;_testbuffer;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testinternalcapi;_testclinic;_testclinic_limited;_tkinter ?>
+ <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_remote_debugging;_uuid;_wmi;_zoneinfo;_zstd;_testcapi;_ctypes_test;_testbuffer;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testinternalcapi;_testclinic;_testclinic_limited;_tkinter ?>
<Fragment>
<DirectoryRef Id="Lib_venv_scripts_nt__freethreaded" />
diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs
index 8439518bcbd..4d44299f783 100644
--- a/Tools/msi/lib/lib_files.wxs
+++ b/Tools/msi/lib/lib_files.wxs
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
- <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_uuid;_wmi;_zoneinfo;_zstd ?>
+ <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_remote_debugging;_uuid;_wmi;_zoneinfo;_zstd ?>
<Fragment>
<DirectoryRef Id="Lib_venv_scripts_nt" />
diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index 2be85a163b4..04f66eec1a0 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -44,7 +44,7 @@ EXTENSION_PREFIX = """\
# define MAXSTACK 4000
# endif
#else
-# define MAXSTACK 4000
+# define MAXSTACK 6000
#endif
"""
@@ -214,33 +214,47 @@ class CCallMakerVisitor(GrammarVisitor):
call.assigned_variable_type = node.type
return call
+ def assert_no_undefined_behavior(
+ self, call: FunctionCall, wrapper: str, expected_rtype: str | None,
+ ) -> None:
+ if call.return_type != expected_rtype:
+ raise RuntimeError(
+ f"{call.function} return type is incompatible with {wrapper}: "
+ f"expect: {expected_rtype}, actual: {call.return_type}"
+ )
+
def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
call = self.generate_call(node.node)
- if call.nodetype == NodeTypes.NAME_TOKEN:
- return FunctionCall(
- function=f"_PyPegen_lookahead_with_name",
- arguments=[positive, call.function, *call.arguments],
- return_type="int",
- )
+ comment = None
+ if call.nodetype is NodeTypes.NAME_TOKEN:
+ function = "_PyPegen_lookahead_for_expr"
+ self.assert_no_undefined_behavior(call, function, "expr_ty")
+ elif call.nodetype is NodeTypes.STRING_TOKEN:
+ # _PyPegen_string_token() returns 'void *' instead of 'Token *';
+ # in addition, the overall function call would return 'expr_ty'.
+ assert call.function == "_PyPegen_string_token"
+ function = "_PyPegen_lookahead"
+ self.assert_no_undefined_behavior(call, function, "expr_ty")
elif call.nodetype == NodeTypes.SOFT_KEYWORD:
- return FunctionCall(
- function=f"_PyPegen_lookahead_with_string",
- arguments=[positive, call.function, *call.arguments],
- return_type="int",
- )
+ function = "_PyPegen_lookahead_with_string"
+ self.assert_no_undefined_behavior(call, function, "expr_ty")
elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}:
- return FunctionCall(
- function=f"_PyPegen_lookahead_with_int",
- arguments=[positive, call.function, *call.arguments],
- return_type="int",
- comment=f"token={node.node}",
- )
+ function = "_PyPegen_lookahead_with_int"
+ self.assert_no_undefined_behavior(call, function, "Token *")
+ comment = f"token={node.node}"
+ elif call.return_type == "expr_ty":
+ function = "_PyPegen_lookahead_for_expr"
+ elif call.return_type == "stmt_ty":
+ function = "_PyPegen_lookahead_for_stmt"
else:
- return FunctionCall(
- function=f"_PyPegen_lookahead",
- arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments],
- return_type="int",
- )
+ function = "_PyPegen_lookahead"
+ self.assert_no_undefined_behavior(call, function, None)
+ return FunctionCall(
+ function=function,
+ arguments=[positive, call.function, *call.arguments],
+ return_type="int",
+ comment=comment,
+ )
def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall:
return self.lookahead_call_helper(node, 1)
generated by cgit v1.2.3 (git 2.39.1) at 2025-06-16 02:30:06 +0000