aboutsummaryrefslogtreecommitdiffstatshomepage
path: root/Tools
diff options
context:
space:
mode:
Diffstat (limited to 'Tools')
-rw-r--r--Tools/build/mypy.ini10
-rw-r--r--Tools/build/update_file.py24
-rwxr-xr-xTools/build/verify_ensurepip_wheels.py6
-rw-r--r--Tools/cases_generator/optimizer_generator.py44
-rw-r--r--Tools/clinic/libclinic/converters.py124
-rw-r--r--Tools/msi/lib/lib.wixproj3
-rw-r--r--Tools/peg_generator/pegen/parser_generator.py5
-rw-r--r--Tools/tsan/suppressions_free_threading.txt10
-rw-r--r--Tools/wasm/emscripten/.editorconfig (renamed from Tools/wasm/.editorconfig)0
-rw-r--r--Tools/wasm/mypy.ini11
-rwxr-xr-xTools/wasm/wasi-env3
-rw-r--r--Tools/wasm/wasi.py373
-rw-r--r--Tools/wasm/wasi/__main__.py368
-rw-r--r--Tools/wasm/wasi/config.site-wasm32-wasi (renamed from Tools/wasm/config.site-wasm32-wasi)0
-rwxr-xr-xTools/wasm/wasm_build.py932
15 files changed, 525 insertions, 1388 deletions
diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini
index 06224163884..fab35bf6890 100644
--- a/Tools/build/mypy.ini
+++ b/Tools/build/mypy.ini
@@ -1,7 +1,13 @@
[mypy]
+
+# Please, when adding new files here, also add them to:
+# .github/workflows/mypy.yml
files =
Tools/build/compute-changes.py,
- Tools/build/generate_sbom.py
+ Tools/build/generate_sbom.py,
+ Tools/build/verify_ensurepip_wheels.py,
+ Tools/build/update_file.py
+
pretty = True
# Make sure Python can still be built
@@ -10,6 +16,8 @@ python_version = 3.10
# ...And be strict:
strict = True
+strict_bytes = True
+local_partial_types = True
extra_checks = True
enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined
warn_unreachable = True
diff --git a/Tools/build/update_file.py b/Tools/build/update_file.py
index b4182c1d0cb..b4a5fb6e778 100644
--- a/Tools/build/update_file.py
+++ b/Tools/build/update_file.py
@@ -6,14 +6,27 @@ This avoids wholesale rebuilds when a code (re)generation phase does not
actually change the in-tree generated code.
"""
+from __future__ import annotations
+
import contextlib
import os
import os.path
import sys
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+ import typing
+ from collections.abc import Iterator
+ from io import TextIOWrapper
+
+ _Outcome: typing.TypeAlias = typing.Literal['created', 'updated', 'same']
+
@contextlib.contextmanager
-def updating_file_with_tmpfile(filename, tmpfile=None):
+def updating_file_with_tmpfile(
+ filename: str,
+ tmpfile: str | None = None,
+) -> Iterator[tuple[TextIOWrapper, TextIOWrapper]]:
"""A context manager for updating a file via a temp file.
The context manager provides two open files: the source file open
@@ -46,13 +59,18 @@ def updating_file_with_tmpfile(filename, tmpfile=None):
update_file_with_tmpfile(filename, tmpfile)
-def update_file_with_tmpfile(filename, tmpfile, *, create=False):
+def update_file_with_tmpfile(
+ filename: str,
+ tmpfile: str,
+ *,
+ create: bool = False,
+) -> _Outcome:
try:
targetfile = open(filename, 'rb')
except FileNotFoundError:
if not create:
raise # re-raise
- outcome = 'created'
+ outcome: _Outcome = 'created'
os.replace(tmpfile, filename)
else:
with targetfile:
diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py
index a37da2f7075..46c42916d93 100755
--- a/Tools/build/verify_ensurepip_wheels.py
+++ b/Tools/build/verify_ensurepip_wheels.py
@@ -20,13 +20,13 @@ ENSURE_PIP_INIT_PY_TEXT = (ENSURE_PIP_ROOT / "__init__.py").read_text(encoding="
GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true"
-def print_notice(file_path: str, message: str) -> None:
+def print_notice(file_path: str | Path, message: str) -> None:
if GITHUB_ACTIONS:
message = f"::notice file={file_path}::{message}"
print(message, end="\n\n")
-def print_error(file_path: str, message: str) -> None:
+def print_error(file_path: str | Path, message: str) -> None:
if GITHUB_ACTIONS:
message = f"::error file={file_path}::{message}"
print(message, end="\n\n")
@@ -67,6 +67,7 @@ def verify_wheel(package_name: str) -> bool:
return False
release_files = json.loads(raw_text)["releases"][package_version]
+ expected_digest = ""
for release_info in release_files:
if package_path.name != release_info["filename"]:
continue
@@ -95,6 +96,7 @@ def verify_wheel(package_name: str) -> bool:
return True
+
if __name__ == "__main__":
exit_status = int(not verify_wheel("pip"))
raise SystemExit(exit_status)
diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py
index 7a32275347e..fda022a44e5 100644
--- a/Tools/cases_generator/optimizer_generator.py
+++ b/Tools/cases_generator/optimizer_generator.py
@@ -30,16 +30,52 @@ DEFAULT_ABSTRACT_INPUT = (ROOT / "Python/optimizer_bytecodes.c").absolute().as_p
def validate_uop(override: Uop, uop: Uop) -> None:
- # To do
- pass
+ """
+ Check that the overridden uop (defined in 'optimizer_bytecodes.c')
+ has the same stack effects as the original uop (defined in 'bytecodes.c').
+
+ Ensure that:
+ - The number of inputs and outputs is the same.
+ - The names of the inputs and outputs are the same
+ (except for 'unused' which is ignored).
+ - The sizes of the inputs and outputs are the same.
+ """
+ for stack_effect in ('inputs', 'outputs'):
+ orig_effects = getattr(uop.stack, stack_effect)
+ new_effects = getattr(override.stack, stack_effect)
+
+ if len(orig_effects) != len(new_effects):
+ msg = (
+ f"{uop.name}: Must have the same number of {stack_effect} "
+ "in bytecodes.c and optimizer_bytecodes.c "
+ f"({len(orig_effects)} != {len(new_effects)})"
+ )
+ raise analysis_error(msg, override.body.open)
+
+ for orig, new in zip(orig_effects, new_effects, strict=True):
+ if orig.name != new.name and orig.name != "unused" and new.name != "unused":
+ msg = (
+ f"{uop.name}: {stack_effect.capitalize()} must have "
+ "equal names in bytecodes.c and optimizer_bytecodes.c "
+ f"({orig.name} != {new.name})"
+ )
+ raise analysis_error(msg, override.body.open)
+
+ if orig.size != new.size:
+ msg = (
+ f"{uop.name}: {stack_effect.capitalize()} must have "
+ "equal sizes in bytecodes.c and optimizer_bytecodes.c "
+ f"({orig.size!r} != {new.size!r})"
+ )
+ raise analysis_error(msg, override.body.open)
def type_name(var: StackItem) -> str:
if var.is_array():
- return f"JitOptSymbol **"
+ return "JitOptSymbol **"
if var.type:
return var.type
- return f"JitOptSymbol *"
+ return "JitOptSymbol *"
def declare_variables(uop: Uop, out: CWriter, skip_inputs: bool) -> None:
diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py
index 633fb5f56a6..39d0ac557a6 100644
--- a/Tools/clinic/libclinic/converters.py
+++ b/Tools/clinic/libclinic/converters.py
@@ -17,6 +17,54 @@ from libclinic.converter import (
TypeSet = set[bltns.type[object]]
+class BaseUnsignedIntConverter(CConverter):
+
+ def use_converter(self) -> None:
+ if self.converter:
+ self.add_include('pycore_long.h',
+ f'{self.converter}()')
+
+ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
+ if not limited_capi:
+ return super().parse_arg(argname, displayname, limited_capi=limited_capi)
+ return self.format_code("""
+ {{{{
+ Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}),
+ Py_ASNATIVEBYTES_NATIVE_ENDIAN |
+ Py_ASNATIVEBYTES_ALLOW_INDEX |
+ Py_ASNATIVEBYTES_REJECT_NEGATIVE |
+ Py_ASNATIVEBYTES_UNSIGNED_BUFFER);
+ if (_bytes < 0) {{{{
+ goto exit;
+ }}}}
+ if ((size_t)_bytes > sizeof({type})) {{{{
+ PyErr_SetString(PyExc_OverflowError,
+ "Python int too large for C {type}");
+ goto exit;
+ }}}}
+ }}}}
+ """,
+ argname=argname,
+ type=self.type)
+
+
+class uint8_converter(BaseUnsignedIntConverter):
+ type = "uint8_t"
+ converter = '_PyLong_UInt8_Converter'
+
+class uint16_converter(BaseUnsignedIntConverter):
+ type = "uint16_t"
+ converter = '_PyLong_UInt16_Converter'
+
+class uint32_converter(BaseUnsignedIntConverter):
+ type = "uint32_t"
+ converter = '_PyLong_UInt32_Converter'
+
+class uint64_converter(BaseUnsignedIntConverter):
+ type = "uint64_t"
+ converter = '_PyLong_UInt64_Converter'
+
+
class bool_converter(CConverter):
type = 'int'
default_type = bool
@@ -211,29 +259,7 @@ class short_converter(CConverter):
return super().parse_arg(argname, displayname, limited_capi=limited_capi)
-def format_inline_unsigned_int_converter(self: CConverter, argname: str) -> str:
- return self.format_code("""
- {{{{
- Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}),
- Py_ASNATIVEBYTES_NATIVE_ENDIAN |
- Py_ASNATIVEBYTES_ALLOW_INDEX |
- Py_ASNATIVEBYTES_REJECT_NEGATIVE |
- Py_ASNATIVEBYTES_UNSIGNED_BUFFER);
- if (_bytes < 0) {{{{
- goto exit;
- }}}}
- if ((size_t)_bytes > sizeof({type})) {{{{
- PyErr_SetString(PyExc_OverflowError,
- "Python int too large for C {type}");
- goto exit;
- }}}}
- }}}}
- """,
- argname=argname,
- type=self.type)
-
-
-class unsigned_short_converter(CConverter):
+class unsigned_short_converter(BaseUnsignedIntConverter):
type = 'unsigned short'
default_type = int
c_ignored_default = "0"
@@ -244,11 +270,6 @@ class unsigned_short_converter(CConverter):
else:
self.converter = '_PyLong_UnsignedShort_Converter'
- def use_converter(self) -> None:
- if self.converter == '_PyLong_UnsignedShort_Converter':
- self.add_include('pycore_long.h',
- '_PyLong_UnsignedShort_Converter()')
-
def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
if self.format_unit == 'H':
return self.format_code("""
@@ -258,9 +279,7 @@ class unsigned_short_converter(CConverter):
}}}}
""",
argname=argname)
- if not limited_capi:
- return super().parse_arg(argname, displayname, limited_capi=limited_capi)
- return format_inline_unsigned_int_converter(self, argname)
+ return super().parse_arg(argname, displayname, limited_capi=limited_capi)
@add_legacy_c_converter('C', accept={str})
@@ -311,7 +330,7 @@ class int_converter(CConverter):
return super().parse_arg(argname, displayname, limited_capi=limited_capi)
-class unsigned_int_converter(CConverter):
+class unsigned_int_converter(BaseUnsignedIntConverter):
type = 'unsigned int'
default_type = int
c_ignored_default = "0"
@@ -322,11 +341,6 @@ class unsigned_int_converter(CConverter):
else:
self.converter = '_PyLong_UnsignedInt_Converter'
- def use_converter(self) -> None:
- if self.converter == '_PyLong_UnsignedInt_Converter':
- self.add_include('pycore_long.h',
- '_PyLong_UnsignedInt_Converter()')
-
def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
if self.format_unit == 'I':
return self.format_code("""
@@ -336,9 +350,7 @@ class unsigned_int_converter(CConverter):
}}}}
""",
argname=argname)
- if not limited_capi:
- return super().parse_arg(argname, displayname, limited_capi=limited_capi)
- return format_inline_unsigned_int_converter(self, argname)
+ return super().parse_arg(argname, displayname, limited_capi=limited_capi)
class long_converter(CConverter):
@@ -359,7 +371,7 @@ class long_converter(CConverter):
return super().parse_arg(argname, displayname, limited_capi=limited_capi)
-class unsigned_long_converter(CConverter):
+class unsigned_long_converter(BaseUnsignedIntConverter):
type = 'unsigned long'
default_type = int
c_ignored_default = "0"
@@ -370,11 +382,6 @@ class unsigned_long_converter(CConverter):
else:
self.converter = '_PyLong_UnsignedLong_Converter'
- def use_converter(self) -> None:
- if self.converter == '_PyLong_UnsignedLong_Converter':
- self.add_include('pycore_long.h',
- '_PyLong_UnsignedLong_Converter()')
-
def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
if self.format_unit == 'k':
return self.format_code("""
@@ -387,9 +394,7 @@ class unsigned_long_converter(CConverter):
argname=argname,
bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi),
)
- if not limited_capi:
- return super().parse_arg(argname, displayname, limited_capi=limited_capi)
- return format_inline_unsigned_int_converter(self, argname)
+ return super().parse_arg(argname, displayname, limited_capi=limited_capi)
class long_long_converter(CConverter):
@@ -410,7 +415,7 @@ class long_long_converter(CConverter):
return super().parse_arg(argname, displayname, limited_capi=limited_capi)
-class unsigned_long_long_converter(CConverter):
+class unsigned_long_long_converter(BaseUnsignedIntConverter):
type = 'unsigned long long'
default_type = int
c_ignored_default = "0"
@@ -421,11 +426,6 @@ class unsigned_long_long_converter(CConverter):
else:
self.converter = '_PyLong_UnsignedLongLong_Converter'
- def use_converter(self) -> None:
- if self.converter == '_PyLong_UnsignedLongLong_Converter':
- self.add_include('pycore_long.h',
- '_PyLong_UnsignedLongLong_Converter()')
-
def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
if self.format_unit == 'K':
return self.format_code("""
@@ -438,9 +438,7 @@ class unsigned_long_long_converter(CConverter):
argname=argname,
bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi),
)
- if not limited_capi:
- return super().parse_arg(argname, displayname, limited_capi=limited_capi)
- return format_inline_unsigned_int_converter(self, argname)
+ return super().parse_arg(argname, displayname, limited_capi=limited_capi)
class Py_ssize_t_converter(CConverter):
@@ -557,15 +555,11 @@ class slice_index_converter(CConverter):
argname=argname)
-class size_t_converter(CConverter):
+class size_t_converter(BaseUnsignedIntConverter):
type = 'size_t'
converter = '_PyLong_Size_t_Converter'
c_ignored_default = "0"
- def use_converter(self) -> None:
- self.add_include('pycore_long.h',
- '_PyLong_Size_t_Converter()')
-
def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None:
if self.format_unit == 'n':
return self.format_code("""
@@ -575,9 +569,7 @@ class size_t_converter(CConverter):
}}}}
""",
argname=argname)
- if not limited_capi:
- return super().parse_arg(argname, displayname, limited_capi=limited_capi)
- return format_inline_unsigned_int_converter(self, argname)
+ return super().parse_arg(argname, displayname, limited_capi=limited_capi)
class fildes_converter(CConverter):
diff --git a/Tools/msi/lib/lib.wixproj b/Tools/msi/lib/lib.wixproj
index 02078e503d7..3ea46dd40ea 100644
--- a/Tools/msi/lib/lib.wixproj
+++ b/Tools/msi/lib/lib.wixproj
@@ -15,12 +15,11 @@
<EmbeddedResource Include="*.wxl" />
</ItemGroup>
<ItemGroup>
- <ExcludeFolders Include="Lib\test;Lib\tests;Lib\tkinter;Lib\idlelib;Lib\turtledemo" />
+ <ExcludeFolders Include="Lib\site-packages;Lib\test;Lib\tests;Lib\tkinter;Lib\idlelib;Lib\turtledemo" />
<InstallFiles Include="$(PySourcePath)Lib\**\*"
Exclude="$(PySourcePath)Lib\**\*.pyc;
$(PySourcePath)Lib\**\*.pyo;
$(PySourcePath)Lib\turtle.py;
- $(PySourcePath)Lib\site-packages\README;
@(ExcludeFolders->'$(PySourcePath)%(Identity)\*');
@(ExcludeFolders->'$(PySourcePath)%(Identity)\**\*')">
<SourceBase>$(PySourcePath)Lib</SourceBase>
diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py
index 6ce0649aefe..52ae743c26b 100644
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -81,6 +81,11 @@ class RuleCheckingVisitor(GrammarVisitor):
self.tokens.add("FSTRING_START")
self.tokens.add("FSTRING_END")
self.tokens.add("FSTRING_MIDDLE")
+ # If python < 3.14 add the virtual tstring tokens
+ if sys.version_info < (3, 14, 0, 'beta', 1):
+ self.tokens.add("TSTRING_START")
+ self.tokens.add("TSTRING_END")
+ self.tokens.add("TSTRING_MIDDLE")
def visit_NameLeaf(self, node: NameLeaf) -> None:
if node.value not in self.rules and node.value not in self.tokens:
diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt
index 21224e490b8..3230f969436 100644
--- a/Tools/tsan/suppressions_free_threading.txt
+++ b/Tools/tsan/suppressions_free_threading.txt
@@ -46,4 +46,12 @@ race:list_inplace_repeat_lock_held
# PyObject_Realloc internally does memcpy which isn't atomic so can race
# with non-locking reads. See #132070
-race:PyObject_Realloc \ No newline at end of file
+race:PyObject_Realloc
+
+# gh-133467. Some of these could be hard to trigger.
+race_top:update_one_slot
+race_top:_Py_slot_tp_getattr_hook
+race_top:slot_tp_descr_get
+race_top:type_set_name
+race_top:set_tp_bases
+race_top:type_set_bases_unlocked
diff --git a/Tools/wasm/.editorconfig b/Tools/wasm/emscripten/.editorconfig
index 4de5fe5954d..4de5fe5954d 100644
--- a/Tools/wasm/.editorconfig
+++ b/Tools/wasm/emscripten/.editorconfig
diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini
deleted file mode 100644
index 4de0a30c260..00000000000
--- a/Tools/wasm/mypy.ini
+++ /dev/null
@@ -1,11 +0,0 @@
-[mypy]
-files = Tools/wasm/wasm_*.py
-pretty = True
-show_traceback = True
-
-# Make sure the wasm can be run using Python 3.8:
-python_version = 3.8
-
-# Be strict...
-strict = True
-enable_error_code = truthy-bool,ignore-without-code
diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env
index 4c5078a1f67..08d4f499baa 100755
--- a/Tools/wasm/wasi-env
+++ b/Tools/wasm/wasi-env
@@ -1,7 +1,8 @@
#!/bin/sh
set -e
-# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py .
+# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py ;
+# expected in Python 3.18 as 3.13 is when `wasi.py` was introduced.
# function
usage() {
diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py
index a742043e4be..b49b27cbbbe 100644
--- a/Tools/wasm/wasi.py
+++ b/Tools/wasm/wasi.py
@@ -1,367 +1,10 @@
-#!/usr/bin/env python3
-
-import argparse
-import contextlib
-import functools
-import os
-try:
- from os import process_cpu_count as cpu_count
-except ImportError:
- from os import cpu_count
-import pathlib
-import shutil
-import subprocess
-import sys
-import sysconfig
-import tempfile
-
-
-CHECKOUT = pathlib.Path(__file__).parent.parent.parent
-
-CROSS_BUILD_DIR = CHECKOUT / "cross-build"
-BUILD_DIR = CROSS_BUILD_DIR / "build"
-
-LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
-LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8")
-
-WASMTIME_VAR_NAME = "WASMTIME"
-WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
-
-
-def updated_env(updates={}):
- """Create a new dict representing the environment to use.
-
- The changes made to the execution environment are printed out.
- """
- env_defaults = {}
- # https://reproducible-builds.org/docs/source-date-epoch/
- git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
- try:
- epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip()
- env_defaults["SOURCE_DATE_EPOCH"] = epoch
- except subprocess.CalledProcessError:
- pass # Might be building from a tarball.
- # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
- environment = env_defaults | os.environ | updates
-
- env_diff = {}
- for key, value in environment.items():
- if os.environ.get(key) != value:
- env_diff[key] = value
-
- print("๐ŸŒŽ Environment changes:")
- for key in sorted(env_diff.keys()):
- print(f" {key}={env_diff[key]}")
-
- return environment
-
-
-def subdir(working_dir, *, clean_ok=False):
- """Decorator to change to a working directory."""
- def decorator(func):
- @functools.wraps(func)
- def wrapper(context):
- nonlocal working_dir
-
- if callable(working_dir):
- working_dir = working_dir(context)
- try:
- tput_output = subprocess.check_output(["tput", "cols"],
- encoding="utf-8")
- except subprocess.CalledProcessError:
- terminal_width = 80
- else:
- terminal_width = int(tput_output.strip())
- print("โŽฏ" * terminal_width)
- print("๐Ÿ“", working_dir)
- if (clean_ok and getattr(context, "clean", False) and
- working_dir.exists()):
- print(f"๐Ÿšฎ Deleting directory (--clean)...")
- shutil.rmtree(working_dir)
-
- working_dir.mkdir(parents=True, exist_ok=True)
-
- with contextlib.chdir(working_dir):
- return func(context, working_dir)
-
- return wrapper
-
- return decorator
-
-
-def call(command, *, quiet, **kwargs):
- """Execute a command.
-
- If 'quiet' is true, then redirect stdout and stderr to a temporary file.
- """
- print("โฏ", " ".join(map(str, command)))
- if not quiet:
- stdout = None
- stderr = None
- else:
- stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8",
- delete=False,
- prefix="cpython-wasi-",
- suffix=".log")
- stderr = subprocess.STDOUT
- print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...")
-
- subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
-
-
-def build_platform():
- """The name of the build/host platform."""
- # Can also be found via `config.guess`.`
- return sysconfig.get_config_var("BUILD_GNU_TYPE")
-
-
-def build_python_path():
- """The path to the build Python binary."""
- binary = BUILD_DIR / "python"
- if not binary.is_file():
- binary = binary.with_suffix(".exe")
- if not binary.is_file():
- raise FileNotFoundError("Unable to find `python(.exe)` in "
- f"{BUILD_DIR}")
-
- return binary
-
-
-@subdir(BUILD_DIR, clean_ok=True)
-def configure_build_python(context, working_dir):
- """Configure the build/host Python."""
- if LOCAL_SETUP.exists():
- print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...")
- else:
- print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...")
- LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
-
- configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)]
- if context.args:
- configure.extend(context.args)
-
- call(configure, quiet=context.quiet)
-
-
-@subdir(BUILD_DIR)
-def make_build_python(context, working_dir):
- """Make/build the build Python."""
- call(["make", "--jobs", str(cpu_count()), "all"],
- quiet=context.quiet)
-
- binary = build_python_path()
- cmd = [binary, "-c",
- "import sys; "
- "print(f'{sys.version_info.major}.{sys.version_info.minor}')"]
- version = subprocess.check_output(cmd, encoding="utf-8").strip()
-
- print(f"๐ŸŽ‰ {binary} {version}")
-
-
-def find_wasi_sdk():
- """Find the path to wasi-sdk."""
- if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"):
- return pathlib.Path(wasi_sdk_path)
- elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists():
- return default_path
-
-
-def wasi_sdk_env(context):
- """Calculate environment variables for building with wasi-sdk."""
- wasi_sdk_path = context.wasi_sdk_path
- sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
- env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++",
- "AR": "llvm-ar", "RANLIB": "ranlib"}
-
- for env_var, binary_name in list(env.items()):
- env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
-
- if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"):
- for compiler in ["CC", "CPP", "CXX"]:
- env[compiler] += f" --sysroot={sysroot}"
-
- env["PKG_CONFIG_PATH"] = ""
- env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
- map(os.fsdecode,
- [sysroot / "lib" / "pkgconfig",
- sysroot / "share" / "pkgconfig"]))
- env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
-
- env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
- env["WASI_SYSROOT"] = os.fsdecode(sysroot)
-
- env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"),
- os.environ["PATH"]])
-
- return env
-
-
-@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
-def configure_wasi_python(context, working_dir):
- """Configure the WASI/host build."""
- if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
- raise ValueError("WASI-SDK not found; "
- "download from "
- "https://github.com/WebAssembly/wasi-sdk and/or "
- "specify via $WASI_SDK_PATH or --wasi-sdk")
-
- config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi")
-
- wasi_build_dir = working_dir.relative_to(CHECKOUT)
-
- python_build_dir = BUILD_DIR / "build"
- lib_dirs = list(python_build_dir.glob("lib.*"))
- assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}"
- lib_dir = os.fsdecode(lib_dirs[0])
- pydebug = lib_dir.endswith("-pydebug")
- python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
- sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
- if pydebug:
- sysconfig_data += "-pydebug"
-
- # Use PYTHONPATH to include sysconfig data which must be anchored to the
- # WASI guest's `/` directory.
- args = {"GUEST_DIR": "/",
- "HOST_DIR": CHECKOUT,
- "ENV_VAR_NAME": "PYTHONPATH",
- "ENV_VAR_VALUE": f"/{sysconfig_data}",
- "PYTHON_WASM": working_dir / "python.wasm"}
- # Check dynamically for wasmtime in case it was specified manually via
- # `--host-runner`.
- if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
- if wasmtime := shutil.which("wasmtime"):
- args[WASMTIME_VAR_NAME] = wasmtime
- else:
- raise FileNotFoundError("wasmtime not found; download from "
- "https://github.com/bytecodealliance/wasmtime")
- host_runner = context.host_runner.format_map(args)
- env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
- build_python = os.fsdecode(build_python_path())
- # The path to `configure` MUST be relative, else `python.wasm` is unable
- # to find the stdlib due to Python not recognizing that it's being
- # executed from within a checkout.
- configure = [os.path.relpath(CHECKOUT / 'configure', working_dir),
- f"--host={context.host_triple}",
- f"--build={build_platform()}",
- f"--with-build-python={build_python}"]
- if pydebug:
- configure.append("--with-pydebug")
- if context.args:
- configure.extend(context.args)
- call(configure,
- env=updated_env(env_additions | wasi_sdk_env(context)),
- quiet=context.quiet)
-
- python_wasm = working_dir / "python.wasm"
- exec_script = working_dir / "python.sh"
- with exec_script.open("w", encoding="utf-8") as file:
- file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
- exec_script.chmod(0o755)
- print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} ... ")
- sys.stdout.flush()
-
-
-@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
-def make_wasi_python(context, working_dir):
- """Run `make` for the WASI/host build."""
- call(["make", "--jobs", str(cpu_count()), "all"],
- env=updated_env(),
- quiet=context.quiet)
-
- exec_script = working_dir / "python.sh"
- subprocess.check_call([exec_script, "--version"])
- print(
- f"๐ŸŽ‰ Use '{exec_script.relative_to(context.init_dir)}' "
- "to run CPython in wasm runtime"
- )
-
-
-def build_all(context):
- """Build everything."""
- steps = [configure_build_python, make_build_python, configure_wasi_python,
- make_wasi_python]
- for step in steps:
- step(context)
-
-def clean_contents(context):
- """Delete all files created by this script."""
- if CROSS_BUILD_DIR.exists():
- print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...")
- shutil.rmtree(CROSS_BUILD_DIR)
-
- if LOCAL_SETUP.exists():
- with LOCAL_SETUP.open("rb") as file:
- if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER:
- print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...")
-
-
-def main():
- default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run "
- # Make sure the stack size will work for a pydebug
- # build.
- # Use 16 MiB stack.
- "--wasm max-wasm-stack=16777216 "
- # Enable thread support; causes use of preview1.
- #"--wasm threads=y --wasi threads=y "
- # Map the checkout to / to load the stdlib from /Lib.
- "--dir {HOST_DIR}::{GUEST_DIR} "
- # Set PYTHONPATH to the sysconfig data.
- "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}")
-
- parser = argparse.ArgumentParser()
- subcommands = parser.add_subparsers(dest="subcommand")
- build = subcommands.add_parser("build", help="Build everything")
- configure_build = subcommands.add_parser("configure-build-python",
- help="Run `configure` for the "
- "build Python")
- make_build = subcommands.add_parser("make-build-python",
- help="Run `make` for the build Python")
- configure_host = subcommands.add_parser("configure-host",
- help="Run `configure` for the "
- "host/WASI (pydebug builds "
- "are inferred from the build "
- "Python)")
- make_host = subcommands.add_parser("make-host",
- help="Run `make` for the host/WASI")
- clean = subcommands.add_parser("clean", help="Delete files and directories "
- "created by this script")
- for subcommand in build, configure_build, make_build, configure_host, make_host:
- subcommand.add_argument("--quiet", action="store_true", default=False,
- dest="quiet",
- help="Redirect output from subprocesses to a log file")
- for subcommand in configure_build, configure_host:
- subcommand.add_argument("--clean", action="store_true", default=False,
- dest="clean",
- help="Delete any relevant directories before building")
- for subcommand in build, configure_build, configure_host:
- subcommand.add_argument("args", nargs="*",
- help="Extra arguments to pass to `configure`")
- for subcommand in build, configure_host:
- subcommand.add_argument("--wasi-sdk", type=pathlib.Path,
- dest="wasi_sdk_path",
- default=find_wasi_sdk(),
- help="Path to wasi-sdk; defaults to "
- "$WASI_SDK_PATH or /opt/wasi-sdk")
- subcommand.add_argument("--host-runner", action="store",
- default=default_host_runner, dest="host_runner",
- help="Command template for running the WASI host "
- "(default designed for wasmtime 14 or newer: "
- f"`{default_host_runner}`)")
- for subcommand in build, configure_host, make_host:
- subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1",
- help="The target triple for the WASI host build")
-
- context = parser.parse_args()
- context.init_dir = pathlib.Path().absolute()
-
- dispatch = {"configure-build-python": configure_build_python,
- "make-build-python": make_build_python,
- "configure-host": configure_wasi_python,
- "make-host": make_wasi_python,
- "build": build_all,
- "clean": clean_contents}
- dispatch[context.subcommand](context)
+if __name__ == "__main__":
+ import pathlib
+ import runpy
+ import sys
+ print("โš ๏ธ WARNING: This script is deprecated and slated for removal in Python 3.20; "
+ "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n",
+ file=sys.stderr)
-if __name__ == "__main__":
- main()
+ runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__")
diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py
new file mode 100644
index 00000000000..ba5faeb9e20
--- /dev/null
+++ b/Tools/wasm/wasi/__main__.py
@@ -0,0 +1,368 @@
+#!/usr/bin/env python3
+
+import argparse
+import contextlib
+import functools
+import os
+try:
+ from os import process_cpu_count as cpu_count
+except ImportError:
+ from os import cpu_count
+import pathlib
+import shutil
+import subprocess
+import sys
+import sysconfig
+import tempfile
+
+
+CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent
+assert (CHECKOUT / "configure").is_file(), "Please update the location of the file"
+
+CROSS_BUILD_DIR = CHECKOUT / "cross-build"
+BUILD_DIR = CROSS_BUILD_DIR / "build"
+
+LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
+LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8")
+
+WASMTIME_VAR_NAME = "WASMTIME"
+WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}"
+
+
+def updated_env(updates={}):
+ """Create a new dict representing the environment to use.
+
+ The changes made to the execution environment are printed out.
+ """
+ env_defaults = {}
+ # https://reproducible-builds.org/docs/source-date-epoch/
+ git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"]
+ try:
+ epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip()
+ env_defaults["SOURCE_DATE_EPOCH"] = epoch
+ except subprocess.CalledProcessError:
+ pass # Might be building from a tarball.
+ # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence.
+ environment = env_defaults | os.environ | updates
+
+ env_diff = {}
+ for key, value in environment.items():
+ if os.environ.get(key) != value:
+ env_diff[key] = value
+
+ print("๐ŸŒŽ Environment changes:")
+ for key in sorted(env_diff.keys()):
+ print(f" {key}={env_diff[key]}")
+
+ return environment
+
+
+def subdir(working_dir, *, clean_ok=False):
+ """Decorator to change to a working directory."""
+ def decorator(func):
+ @functools.wraps(func)
+ def wrapper(context):
+ nonlocal working_dir
+
+ if callable(working_dir):
+ working_dir = working_dir(context)
+ try:
+ tput_output = subprocess.check_output(["tput", "cols"],
+ encoding="utf-8")
+ except subprocess.CalledProcessError:
+ terminal_width = 80
+ else:
+ terminal_width = int(tput_output.strip())
+ print("โŽฏ" * terminal_width)
+ print("๐Ÿ“", working_dir)
+ if (clean_ok and getattr(context, "clean", False) and
+ working_dir.exists()):
+ print(f"๐Ÿšฎ Deleting directory (--clean)...")
+ shutil.rmtree(working_dir)
+
+ working_dir.mkdir(parents=True, exist_ok=True)
+
+ with contextlib.chdir(working_dir):
+ return func(context, working_dir)
+
+ return wrapper
+
+ return decorator
+
+
+def call(command, *, quiet, **kwargs):
+ """Execute a command.
+
+ If 'quiet' is true, then redirect stdout and stderr to a temporary file.
+ """
+ print("โฏ", " ".join(map(str, command)))
+ if not quiet:
+ stdout = None
+ stderr = None
+ else:
+ stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8",
+ delete=False,
+ prefix="cpython-wasi-",
+ suffix=".log")
+ stderr = subprocess.STDOUT
+ print(f"๐Ÿ“ Logging output to {stdout.name} (--quiet)...")
+
+ subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr)
+
+
+def build_platform():
+ """The name of the build/host platform."""
+ # Can also be found via `config.guess`.`
+ return sysconfig.get_config_var("BUILD_GNU_TYPE")
+
+
+def build_python_path():
+ """The path to the build Python binary."""
+ binary = BUILD_DIR / "python"
+ if not binary.is_file():
+ binary = binary.with_suffix(".exe")
+ if not binary.is_file():
+ raise FileNotFoundError("Unable to find `python(.exe)` in "
+ f"{BUILD_DIR}")
+
+ return binary
+
+
+@subdir(BUILD_DIR, clean_ok=True)
+def configure_build_python(context, working_dir):
+ """Configure the build/host Python."""
+ if LOCAL_SETUP.exists():
+ print(f"๐Ÿ‘ {LOCAL_SETUP} exists ...")
+ else:
+ print(f"๐Ÿ“ Touching {LOCAL_SETUP} ...")
+ LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER)
+
+ configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)]
+ if context.args:
+ configure.extend(context.args)
+
+ call(configure, quiet=context.quiet)
+
+
+@subdir(BUILD_DIR)
+def make_build_python(context, working_dir):
+ """Make/build the build Python."""
+ call(["make", "--jobs", str(cpu_count()), "all"],
+ quiet=context.quiet)
+
+ binary = build_python_path()
+ cmd = [binary, "-c",
+ "import sys; "
+ "print(f'{sys.version_info.major}.{sys.version_info.minor}')"]
+ version = subprocess.check_output(cmd, encoding="utf-8").strip()
+
+ print(f"๐ŸŽ‰ {binary} {version}")
+
+
+def find_wasi_sdk():
+ """Find the path to wasi-sdk."""
+ if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"):
+ return pathlib.Path(wasi_sdk_path)
+ elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists():
+ return default_path
+
+
+def wasi_sdk_env(context):
+ """Calculate environment variables for building with wasi-sdk."""
+ wasi_sdk_path = context.wasi_sdk_path
+ sysroot = wasi_sdk_path / "share" / "wasi-sysroot"
+ env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++",
+ "AR": "llvm-ar", "RANLIB": "ranlib"}
+
+ for env_var, binary_name in list(env.items()):
+ env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name)
+
+ if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"):
+ for compiler in ["CC", "CPP", "CXX"]:
+ env[compiler] += f" --sysroot={sysroot}"
+
+ env["PKG_CONFIG_PATH"] = ""
+ env["PKG_CONFIG_LIBDIR"] = os.pathsep.join(
+ map(os.fsdecode,
+ [sysroot / "lib" / "pkgconfig",
+ sysroot / "share" / "pkgconfig"]))
+ env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot)
+
+ env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path)
+ env["WASI_SYSROOT"] = os.fsdecode(sysroot)
+
+ env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"),
+ os.environ["PATH"]])
+
+ return env
+
+
+@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True)
+def configure_wasi_python(context, working_dir):
+ """Configure the WASI/host build."""
+ if not context.wasi_sdk_path or not context.wasi_sdk_path.exists():
+ raise ValueError("WASI-SDK not found; "
+ "download from "
+ "https://github.com/WebAssembly/wasi-sdk and/or "
+ "specify via $WASI_SDK_PATH or --wasi-sdk")
+
+ config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi")
+
+ wasi_build_dir = working_dir.relative_to(CHECKOUT)
+
+ python_build_dir = BUILD_DIR / "build"
+ lib_dirs = list(python_build_dir.glob("lib.*"))
+ assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}"
+ lib_dir = os.fsdecode(lib_dirs[0])
+ pydebug = lib_dir.endswith("-pydebug")
+ python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1]
+ sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}"
+ if pydebug:
+ sysconfig_data += "-pydebug"
+
+ # Use PYTHONPATH to include sysconfig data which must be anchored to the
+ # WASI guest's `/` directory.
+ args = {"GUEST_DIR": "/",
+ "HOST_DIR": CHECKOUT,
+ "ENV_VAR_NAME": "PYTHONPATH",
+ "ENV_VAR_VALUE": f"/{sysconfig_data}",
+ "PYTHON_WASM": working_dir / "python.wasm"}
+ # Check dynamically for wasmtime in case it was specified manually via
+ # `--host-runner`.
+ if WASMTIME_HOST_RUNNER_VAR in context.host_runner:
+ if wasmtime := shutil.which("wasmtime"):
+ args[WASMTIME_VAR_NAME] = wasmtime
+ else:
+ raise FileNotFoundError("wasmtime not found; download from "
+ "https://github.com/bytecodealliance/wasmtime")
+ host_runner = context.host_runner.format_map(args)
+ env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner}
+ build_python = os.fsdecode(build_python_path())
+ # The path to `configure` MUST be relative, else `python.wasm` is unable
+ # to find the stdlib due to Python not recognizing that it's being
+ # executed from within a checkout.
+ configure = [os.path.relpath(CHECKOUT / 'configure', working_dir),
+ f"--host={context.host_triple}",
+ f"--build={build_platform()}",
+ f"--with-build-python={build_python}"]
+ if pydebug:
+ configure.append("--with-pydebug")
+ if context.args:
+ configure.extend(context.args)
+ call(configure,
+ env=updated_env(env_additions | wasi_sdk_env(context)),
+ quiet=context.quiet)
+
+ python_wasm = working_dir / "python.wasm"
+ exec_script = working_dir / "python.sh"
+ with exec_script.open("w", encoding="utf-8") as file:
+ file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n')
+ exec_script.chmod(0o755)
+ print(f"๐Ÿƒโ€โ™€๏ธ Created {exec_script} (--host-runner)... ")
+ sys.stdout.flush()
+
+
+@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple)
+def make_wasi_python(context, working_dir):
+ """Run `make` for the WASI/host build."""
+ call(["make", "--jobs", str(cpu_count()), "all"],
+ env=updated_env(),
+ quiet=context.quiet)
+
+ exec_script = working_dir / "python.sh"
+ call([exec_script, "--version"], quiet=False)
+ print(
+ f"๐ŸŽ‰ Use `{exec_script.relative_to(context.init_dir)}` "
+ "to run CPython w/ the WASI host specified by --host-runner"
+ )
+
+
+def build_all(context):
+ """Build everything."""
+ steps = [configure_build_python, make_build_python, configure_wasi_python,
+ make_wasi_python]
+ for step in steps:
+ step(context)
+
+def clean_contents(context):
+ """Delete all files created by this script."""
+ if CROSS_BUILD_DIR.exists():
+ print(f"๐Ÿงน Deleting {CROSS_BUILD_DIR} ...")
+ shutil.rmtree(CROSS_BUILD_DIR)
+
+ if LOCAL_SETUP.exists():
+ with LOCAL_SETUP.open("rb") as file:
+ if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER:
+ print(f"๐Ÿงน Deleting generated {LOCAL_SETUP} ...")
+
+
+def main():
+ default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run "
+ # Make sure the stack size will work for a pydebug
+ # build.
+ # Use 16 MiB stack.
+ "--wasm max-wasm-stack=16777216 "
+ # Enable thread support; causes use of preview1.
+ #"--wasm threads=y --wasi threads=y "
+ # Map the checkout to / to load the stdlib from /Lib.
+ "--dir {HOST_DIR}::{GUEST_DIR} "
+ # Set PYTHONPATH to the sysconfig data.
+ "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}")
+
+ parser = argparse.ArgumentParser()
+ subcommands = parser.add_subparsers(dest="subcommand")
+ build = subcommands.add_parser("build", help="Build everything")
+ configure_build = subcommands.add_parser("configure-build-python",
+ help="Run `configure` for the "
+ "build Python")
+ make_build = subcommands.add_parser("make-build-python",
+ help="Run `make` for the build Python")
+ configure_host = subcommands.add_parser("configure-host",
+ help="Run `configure` for the "
+ "host/WASI (pydebug builds "
+ "are inferred from the build "
+ "Python)")
+ make_host = subcommands.add_parser("make-host",
+ help="Run `make` for the host/WASI")
+ clean = subcommands.add_parser("clean", help="Delete files and directories "
+ "created by this script")
+ for subcommand in build, configure_build, make_build, configure_host, make_host:
+ subcommand.add_argument("--quiet", action="store_true", default=False,
+ dest="quiet",
+ help="Redirect output from subprocesses to a log file")
+ for subcommand in configure_build, configure_host:
+ subcommand.add_argument("--clean", action="store_true", default=False,
+ dest="clean",
+ help="Delete any relevant directories before building")
+ for subcommand in build, configure_build, configure_host:
+ subcommand.add_argument("args", nargs="*",
+ help="Extra arguments to pass to `configure`")
+ for subcommand in build, configure_host:
+ subcommand.add_argument("--wasi-sdk", type=pathlib.Path,
+ dest="wasi_sdk_path",
+ default=find_wasi_sdk(),
+ help="Path to wasi-sdk; defaults to "
+ "$WASI_SDK_PATH or /opt/wasi-sdk")
+ subcommand.add_argument("--host-runner", action="store",
+ default=default_host_runner, dest="host_runner",
+ help="Command template for running the WASI host "
+ "(default designed for wasmtime 14 or newer: "
+ f"`{default_host_runner}`)")
+ for subcommand in build, configure_host, make_host:
+ subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1",
+ help="The target triple for the WASI host build")
+
+ context = parser.parse_args()
+ context.init_dir = pathlib.Path().absolute()
+
+ dispatch = {"configure-build-python": configure_build_python,
+ "make-build-python": make_build_python,
+ "configure-host": configure_wasi_python,
+ "make-host": make_wasi_python,
+ "build": build_all,
+ "clean": clean_contents}
+ dispatch[context.subcommand](context)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/wasi/config.site-wasm32-wasi
index c5d8b3e205d..c5d8b3e205d 100644
--- a/Tools/wasm/config.site-wasm32-wasi
+++ b/Tools/wasm/wasi/config.site-wasm32-wasi
diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py
deleted file mode 100755
index bcb80212362..00000000000
--- a/Tools/wasm/wasm_build.py
+++ /dev/null
@@ -1,932 +0,0 @@
-#!/usr/bin/env python3
-"""Build script for Python on WebAssembly platforms.
-
- $ ./Tools/wasm/wasm_builder.py emscripten-browser build repl
- $ ./Tools/wasm/wasm_builder.py emscripten-node-dl build test
- $ ./Tools/wasm/wasm_builder.py wasi build test
-
-Primary build targets are "emscripten-node-dl" (NodeJS, dynamic linking),
-"emscripten-browser", and "wasi".
-
-Emscripten builds require a recent Emscripten SDK. The tools looks for an
-activated EMSDK environment (". /path/to/emsdk_env.sh"). System packages
-(Debian, Homebrew) are not supported.
-
-WASI builds require WASI SDK and wasmtime. The tool looks for 'WASI_SDK_PATH'
-and falls back to /opt/wasi-sdk.
-
-The 'build' Python interpreter must be rebuilt every time Python's byte code
-changes.
-
- ./Tools/wasm/wasm_builder.py --clean build build
-
-"""
-import argparse
-import enum
-import dataclasses
-import logging
-import os
-import pathlib
-import re
-import shlex
-import shutil
-import socket
-import subprocess
-import sys
-import sysconfig
-import tempfile
-import time
-import warnings
-import webbrowser
-
-# for Python 3.8
-from typing import (
- cast,
- Any,
- Callable,
- Dict,
- Iterable,
- List,
- Optional,
- Tuple,
- Union,
-)
-
-logger = logging.getLogger("wasm_build")
-
-SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute()
-WASMTOOLS = SRCDIR / "Tools" / "wasm"
-BUILDDIR = SRCDIR / "builddir"
-CONFIGURE = SRCDIR / "configure"
-SETUP_LOCAL = SRCDIR / "Modules" / "Setup.local"
-
-HAS_CCACHE = shutil.which("ccache") is not None
-
-# path to WASI-SDK root
-WASI_SDK_PATH = pathlib.Path(os.environ.get("WASI_SDK_PATH", "/opt/wasi-sdk"))
-
-# path to Emscripten SDK config file.
-# auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh".
-EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten"))
-EMSDK_MIN_VERSION = (3, 1, 19)
-EMSDK_BROKEN_VERSION = {
- (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338",
- (3, 1, 16): "https://github.com/emscripten-core/emscripten/issues/17393",
- (3, 1, 20): "https://github.com/emscripten-core/emscripten/issues/17720",
-}
-_MISSING = pathlib.Path("MISSING")
-
-WASM_WEBSERVER = WASMTOOLS / "wasm_webserver.py"
-
-CLEAN_SRCDIR = f"""
-Builds require a clean source directory. Please use a clean checkout or
-run "make clean -C '{SRCDIR}'".
-"""
-
-INSTALL_NATIVE = """
-Builds require a C compiler (gcc, clang), make, pkg-config, and development
-headers for dependencies like zlib.
-
-Debian/Ubuntu: sudo apt install build-essential git curl pkg-config zlib1g-dev
-Fedora/CentOS: sudo dnf install gcc make git-core curl pkgconfig zlib-devel
-"""
-
-INSTALL_EMSDK = """
-wasm32-emscripten builds need Emscripten SDK. Please follow instructions at
-https://emscripten.org/docs/getting_started/downloads.html how to install
-Emscripten and how to activate the SDK with "emsdk_env.sh".
-
- git clone https://github.com/emscripten-core/emsdk.git /path/to/emsdk
- cd /path/to/emsdk
- ./emsdk install latest
- ./emsdk activate latest
- source /path/to/emsdk_env.sh
-"""
-
-INSTALL_WASI_SDK = """
-wasm32-wasi builds need WASI SDK. Please fetch the latest SDK from
-https://github.com/WebAssembly/wasi-sdk/releases and install it to
-"/opt/wasi-sdk". Alternatively you can install the SDK in a different location
-and point the environment variable WASI_SDK_PATH to the root directory
-of the SDK. The SDK is available for Linux x86_64, macOS x86_64, and MinGW.
-"""
-
-INSTALL_WASMTIME = """
-wasm32-wasi tests require wasmtime on PATH. Please follow instructions at
-https://wasmtime.dev/ to install wasmtime.
-"""
-
-
-def parse_emconfig(
- emconfig: pathlib.Path = EM_CONFIG,
-) -> Tuple[pathlib.Path, pathlib.Path]:
- """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS.
-
- The ".emscripten" config file is a Python snippet that uses "EM_CONFIG"
- environment variable. EMSCRIPTEN_ROOT is the "upstream/emscripten"
- subdirectory with tools like "emconfigure".
- """
- if not emconfig.exists():
- return _MISSING, _MISSING
- with open(emconfig, encoding="utf-8") as f:
- code = f.read()
- # EM_CONFIG file is a Python snippet
- local: Dict[str, Any] = {}
- exec(code, globals(), local)
- emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"])
- node_js = pathlib.Path(local["NODE_JS"])
- return emscripten_root, node_js
-
-
-EMSCRIPTEN_ROOT, NODE_JS = parse_emconfig()
-
-
-def read_python_version(configure: pathlib.Path = CONFIGURE) -> str:
- """Read PACKAGE_VERSION from configure script
-
- configure and configure.ac are the canonical source for major and
- minor version number.
- """
- version_re = re.compile(r"^PACKAGE_VERSION='(\d\.\d+)'")
- with configure.open(encoding="utf-8") as f:
- for line in f:
- mo = version_re.match(line)
- if mo:
- return mo.group(1)
- raise ValueError(f"PACKAGE_VERSION not found in {configure}")
-
-
-PYTHON_VERSION = read_python_version()
-
-
-class ConditionError(ValueError):
- def __init__(self, info: str, text: str) -> None:
- self.info = info
- self.text = text
-
- def __str__(self) -> str:
- return f"{type(self).__name__}: '{self.info}'\n{self.text}"
-
-
-class MissingDependency(ConditionError):
- pass
-
-
-class DirtySourceDirectory(ConditionError):
- pass
-
-
-@dataclasses.dataclass
-class Platform:
- """Platform-specific settings
-
- - CONFIG_SITE override
- - configure wrapper (e.g. emconfigure)
- - make wrapper (e.g. emmake)
- - additional environment variables
- - check function to verify SDK
- """
-
- name: str
- pythonexe: str
- config_site: Optional[pathlib.PurePath]
- configure_wrapper: Optional[pathlib.Path]
- make_wrapper: Optional[pathlib.PurePath]
- environ: Dict[str, Any]
- check: Callable[[], None]
- # Used for build_emports().
- ports: Optional[pathlib.PurePath]
- cc: Optional[pathlib.PurePath]
-
- def getenv(self, profile: "BuildProfile") -> Dict[str, Any]:
- return self.environ.copy()
-
-
-def _check_clean_src() -> None:
- candidates = [
- SRCDIR / "Programs" / "python.o",
- SRCDIR / "Python" / "frozen_modules" / "importlib._bootstrap.h",
- ]
- for candidate in candidates:
- if candidate.exists():
- raise DirtySourceDirectory(os.fspath(candidate), CLEAN_SRCDIR)
-
-
-def _check_native() -> None:
- if not any(shutil.which(cc) for cc in ["cc", "gcc", "clang"]):
- raise MissingDependency("cc", INSTALL_NATIVE)
- if not shutil.which("make"):
- raise MissingDependency("make", INSTALL_NATIVE)
- if sys.platform == "linux":
- # skip pkg-config check on macOS
- if not shutil.which("pkg-config"):
- raise MissingDependency("pkg-config", INSTALL_NATIVE)
- # zlib is needed to create zip files
- for devel in ["zlib"]:
- try:
- subprocess.check_call(["pkg-config", "--exists", devel])
- except subprocess.CalledProcessError:
- raise MissingDependency(devel, INSTALL_NATIVE) from None
- _check_clean_src()
-
-
-NATIVE = Platform(
- "native",
- # macOS has python.exe
- pythonexe=sysconfig.get_config_var("BUILDPYTHON") or "python",
- config_site=None,
- configure_wrapper=None,
- ports=None,
- cc=None,
- make_wrapper=None,
- environ={},
- check=_check_native,
-)
-
-
-def _check_emscripten() -> None:
- if EMSCRIPTEN_ROOT is _MISSING:
- raise MissingDependency("Emscripten SDK EM_CONFIG", INSTALL_EMSDK)
- # sanity check
- emconfigure = EMSCRIPTEN.configure_wrapper
- if emconfigure is not None and not emconfigure.exists():
- raise MissingDependency(os.fspath(emconfigure), INSTALL_EMSDK)
- # version check
- version_txt = EMSCRIPTEN_ROOT / "emscripten-version.txt"
- if not version_txt.exists():
- raise MissingDependency(os.fspath(version_txt), INSTALL_EMSDK)
- with open(version_txt) as f:
- version = f.read().strip().strip('"')
- if version.endswith("-git"):
- # git / upstream / tot-upstream installation
- version = version[:-4]
- version_tuple = cast(
- Tuple[int, int, int],
- tuple(int(v) for v in version.split("."))
- )
- if version_tuple < EMSDK_MIN_VERSION:
- raise ConditionError(
- os.fspath(version_txt),
- f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' is older than "
- "minimum required version "
- f"{'.'.join(str(v) for v in EMSDK_MIN_VERSION)}.",
- )
- broken = EMSDK_BROKEN_VERSION.get(version_tuple)
- if broken is not None:
- raise ConditionError(
- os.fspath(version_txt),
- (
- f"Emscripten SDK {version} in '{EMSCRIPTEN_ROOT}' has known "
- f"bugs, see {broken}."
- ),
- )
- if os.environ.get("PKG_CONFIG_PATH"):
- warnings.warn(
- "PKG_CONFIG_PATH is set and not empty. emconfigure overrides "
- "this environment variable. Use EM_PKG_CONFIG_PATH instead."
- )
- _check_clean_src()
-
-
-EMSCRIPTEN = Platform(
- "emscripten",
- pythonexe="python.js",
- config_site=WASMTOOLS / "config.site-wasm32-emscripten",
- configure_wrapper=EMSCRIPTEN_ROOT / "emconfigure",
- ports=EMSCRIPTEN_ROOT / "embuilder",
- cc=EMSCRIPTEN_ROOT / "emcc",
- make_wrapper=EMSCRIPTEN_ROOT / "emmake",
- environ={
- # workaround for https://github.com/emscripten-core/emscripten/issues/17635
- "TZ": "UTC",
- "EM_COMPILER_WRAPPER": "ccache" if HAS_CCACHE else None,
- "PATH": [EMSCRIPTEN_ROOT, os.environ["PATH"]],
- },
- check=_check_emscripten,
-)
-
-
-def _check_wasi() -> None:
- wasm_ld = WASI_SDK_PATH / "bin" / "wasm-ld"
- if not wasm_ld.exists():
- raise MissingDependency(os.fspath(wasm_ld), INSTALL_WASI_SDK)
- wasmtime = shutil.which("wasmtime")
- if wasmtime is None:
- raise MissingDependency("wasmtime", INSTALL_WASMTIME)
- _check_clean_src()
-
-
-WASI = Platform(
- "wasi",
- pythonexe="python.wasm",
- config_site=WASMTOOLS / "config.site-wasm32-wasi",
- configure_wrapper=WASMTOOLS / "wasi-env",
- ports=None,
- cc=WASI_SDK_PATH / "bin" / "clang",
- make_wrapper=None,
- environ={
- "WASI_SDK_PATH": WASI_SDK_PATH,
- # workaround for https://github.com/python/cpython/issues/95952
- "HOSTRUNNER": (
- "wasmtime run "
- "--wasm max-wasm-stack=16777216 "
- "--wasi preview2 "
- "--dir {srcdir}::/ "
- "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib"
- ),
- "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]],
- },
- check=_check_wasi,
-)
-
-
-class Host(enum.Enum):
- """Target host triplet"""
-
- wasm32_emscripten = "wasm32-unknown-emscripten"
- wasm64_emscripten = "wasm64-unknown-emscripten"
- wasm32_wasi = "wasm32-unknown-wasi"
- wasm64_wasi = "wasm64-unknown-wasi"
- # current platform
- build = sysconfig.get_config_var("BUILD_GNU_TYPE")
-
- @property
- def platform(self) -> Platform:
- if self.is_emscripten:
- return EMSCRIPTEN
- elif self.is_wasi:
- return WASI
- else:
- return NATIVE
-
- @property
- def is_emscripten(self) -> bool:
- cls = type(self)
- return self in {cls.wasm32_emscripten, cls.wasm64_emscripten}
-
- @property
- def is_wasi(self) -> bool:
- cls = type(self)
- return self in {cls.wasm32_wasi, cls.wasm64_wasi}
-
- def get_extra_paths(self) -> Iterable[pathlib.PurePath]:
- """Host-specific os.environ["PATH"] entries.
-
- Emscripten's Node version 14.x works well for wasm32-emscripten.
- wasm64-emscripten requires more recent v8 version, e.g. node 16.x.
- Attempt to use system's node command.
- """
- cls = type(self)
- if self == cls.wasm32_emscripten:
- return [NODE_JS.parent]
- elif self == cls.wasm64_emscripten:
- # TODO: look for recent node
- return []
- else:
- return []
-
- @property
- def emport_args(self) -> List[str]:
- """Host-specific port args (Emscripten)."""
- cls = type(self)
- if self is cls.wasm64_emscripten:
- return ["-sMEMORY64=1"]
- elif self is cls.wasm32_emscripten:
- return ["-sMEMORY64=0"]
- else:
- return []
-
- @property
- def embuilder_args(self) -> List[str]:
- """Host-specific embuilder args (Emscripten)."""
- cls = type(self)
- if self is cls.wasm64_emscripten:
- return ["--wasm64"]
- else:
- return []
-
-
-class EmscriptenTarget(enum.Enum):
- """Emscripten-specific targets (--with-emscripten-target)"""
-
- browser = "browser"
- browser_debug = "browser-debug"
- node = "node"
- node_debug = "node-debug"
-
- @property
- def is_browser(self) -> bool:
- cls = type(self)
- return self in {cls.browser, cls.browser_debug}
-
- @property
- def emport_args(self) -> List[str]:
- """Target-specific port args."""
- cls = type(self)
- if self in {cls.browser_debug, cls.node_debug}:
- # some libs come in debug and non-debug builds
- return ["-O0"]
- else:
- return ["-O2"]
-
-
-class SupportLevel(enum.Enum):
- supported = "tier 3, supported"
- working = "working, unsupported"
- experimental = "experimental, may be broken"
- broken = "broken / unavailable"
-
- def __bool__(self) -> bool:
- cls = type(self)
- return self in {cls.supported, cls.working}
-
-
-@dataclasses.dataclass
-class BuildProfile:
- name: str
- support_level: SupportLevel
- host: Host
- target: Union[EmscriptenTarget, None] = None
- dynamic_linking: Union[bool, None] = None
- pthreads: Union[bool, None] = None
- default_testopts: str = "-j2"
-
- @property
- def is_browser(self) -> bool:
- """Is this a browser build?"""
- return self.target is not None and self.target.is_browser
-
- @property
- def builddir(self) -> pathlib.Path:
- """Path to build directory"""
- return BUILDDIR / self.name
-
- @property
- def python_cmd(self) -> pathlib.Path:
- """Path to python executable"""
- return self.builddir / self.host.platform.pythonexe
-
- @property
- def makefile(self) -> pathlib.Path:
- """Path to Makefile"""
- return self.builddir / "Makefile"
-
- @property
- def configure_cmd(self) -> List[str]:
- """Generate configure command"""
- # use relative path, so WASI tests can find lib prefix.
- # pathlib.Path.relative_to() does not work here.
- configure = os.path.relpath(CONFIGURE, self.builddir)
- cmd = [configure, "-C"]
- platform = self.host.platform
- if platform.configure_wrapper:
- cmd.insert(0, os.fspath(platform.configure_wrapper))
-
- cmd.append(f"--host={self.host.value}")
- cmd.append(f"--build={Host.build.value}")
-
- if self.target is not None:
- assert self.host.is_emscripten
- cmd.append(f"--with-emscripten-target={self.target.value}")
-
- if self.dynamic_linking is not None:
- assert self.host.is_emscripten
- opt = "enable" if self.dynamic_linking else "disable"
- cmd.append(f"--{opt}-wasm-dynamic-linking")
-
- if self.pthreads is not None:
- opt = "enable" if self.pthreads else "disable"
- cmd.append(f"--{opt}-wasm-pthreads")
-
- if self.host != Host.build:
- cmd.append(f"--with-build-python={BUILD.python_cmd}")
-
- if platform.config_site is not None:
- cmd.append(f"CONFIG_SITE={platform.config_site}")
-
- return cmd
-
- @property
- def make_cmd(self) -> List[str]:
- """Generate make command"""
- cmd = ["make"]
- platform = self.host.platform
- if platform.make_wrapper:
- cmd.insert(0, os.fspath(platform.make_wrapper))
- return cmd
-
- def getenv(self) -> Dict[str, Any]:
- """Generate environ dict for platform"""
- env = os.environ.copy()
- if hasattr(os, 'process_cpu_count'):
- cpu_count = os.process_cpu_count()
- else:
- cpu_count = os.cpu_count()
- env.setdefault("MAKEFLAGS", f"-j{cpu_count}")
- platenv = self.host.platform.getenv(self)
- for key, value in platenv.items():
- if value is None:
- env.pop(key, None)
- elif key == "PATH":
- # list of path items, prefix with extra paths
- new_path: List[pathlib.PurePath] = []
- new_path.extend(self.host.get_extra_paths())
- new_path.extend(value)
- env[key] = os.pathsep.join(os.fspath(p) for p in new_path)
- elif isinstance(value, str):
- env[key] = value.format(
- relbuilddir=self.builddir.relative_to(SRCDIR),
- srcdir=SRCDIR,
- version=PYTHON_VERSION,
- )
- else:
- env[key] = value
- return env
-
- def _run_cmd(
- self,
- cmd: Iterable[str],
- args: Iterable[str] = (),
- cwd: Optional[pathlib.Path] = None,
- ) -> int:
- cmd = list(cmd)
- cmd.extend(args)
- if cwd is None:
- cwd = self.builddir
- logger.info('Running "%s" in "%s"', shlex.join(cmd), cwd)
- return subprocess.check_call(
- cmd,
- cwd=os.fspath(cwd),
- env=self.getenv(),
- )
-
- def _check_execute(self) -> None:
- if self.is_browser:
- raise ValueError(f"Cannot execute on {self.target}")
-
- def run_build(self, *args: str) -> None:
- """Run configure (if necessary) and make"""
- if not self.makefile.exists():
- logger.info("Makefile not found, running configure")
- self.run_configure(*args)
- self.run_make("all", *args)
-
- def run_configure(self, *args: str) -> int:
- """Run configure script to generate Makefile"""
- os.makedirs(self.builddir, exist_ok=True)
- return self._run_cmd(self.configure_cmd, args)
-
- def run_make(self, *args: str) -> int:
- """Run make (defaults to build all)"""
- return self._run_cmd(self.make_cmd, args)
-
- def run_pythoninfo(self, *args: str) -> int:
- """Run 'make pythoninfo'"""
- self._check_execute()
- return self.run_make("pythoninfo", *args)
-
- def run_test(self, target: str, testopts: Optional[str] = None) -> int:
- """Run buildbottests"""
- self._check_execute()
- if testopts is None:
- testopts = self.default_testopts
- return self.run_make(target, f"TESTOPTS={testopts}")
-
- def run_py(self, *args: str) -> int:
- """Run Python with hostrunner"""
- self._check_execute()
- return self.run_make(
- "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run"
- )
-
- def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None:
- """Run WASM webserver and open build in browser"""
- relbuilddir = self.builddir.relative_to(SRCDIR)
- url = f"http://{bind}:{port}/{relbuilddir}/python.html"
- args = [
- sys.executable,
- os.fspath(WASM_WEBSERVER),
- "--bind",
- bind,
- "--port",
- str(port),
- ]
- srv = subprocess.Popen(args, cwd=SRCDIR)
- # wait for server
- end = time.monotonic() + 3.0
- while time.monotonic() < end and srv.returncode is None:
- try:
- with socket.create_connection((bind, port), timeout=0.1) as _:
- pass
- except OSError:
- time.sleep(0.01)
- else:
- break
-
- webbrowser.open(url)
-
- try:
- srv.wait()
- except KeyboardInterrupt:
- pass
-
- def clean(self, all: bool = False) -> None:
- """Clean build directory"""
- if all:
- if self.builddir.exists():
- shutil.rmtree(self.builddir)
- elif self.makefile.exists():
- self.run_make("clean")
-
- def build_emports(self, force: bool = False) -> None:
- """Pre-build emscripten ports."""
- platform = self.host.platform
- if platform.ports is None or platform.cc is None:
- raise ValueError("Need ports and CC command")
-
- embuilder_cmd = [os.fspath(platform.ports)]
- embuilder_cmd.extend(self.host.embuilder_args)
- if force:
- embuilder_cmd.append("--force")
-
- ports_cmd = [os.fspath(platform.cc)]
- ports_cmd.extend(self.host.emport_args)
- if self.target:
- ports_cmd.extend(self.target.emport_args)
-
- if self.dynamic_linking:
- # Trigger PIC build.
- ports_cmd.append("-sMAIN_MODULE")
- embuilder_cmd.append("--pic")
-
- if self.pthreads:
- # Trigger multi-threaded build.
- ports_cmd.append("-sUSE_PTHREADS")
-
- # Pre-build libbz2, libsqlite3, libz, and some system libs.
- ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"])
- # Multi-threaded sqlite3 has different suffix
- embuilder_cmd.extend(
- ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"]
- )
-
- self._run_cmd(embuilder_cmd, cwd=SRCDIR)
-
- with tempfile.TemporaryDirectory(suffix="-py-emport") as tmpdir:
- tmppath = pathlib.Path(tmpdir)
- main_c = tmppath / "main.c"
- main_js = tmppath / "main.js"
- with main_c.open("w") as f:
- f.write("int main(void) { return 0; }\n")
- args = [
- os.fspath(main_c),
- "-o",
- os.fspath(main_js),
- ]
- self._run_cmd(ports_cmd, args, cwd=tmppath)
-
-
-# native build (build Python)
-BUILD = BuildProfile(
- "build",
- support_level=SupportLevel.working,
- host=Host.build,
-)
-
-_profiles = [
- BUILD,
- # wasm32-emscripten
- BuildProfile(
- "emscripten-browser",
- support_level=SupportLevel.supported,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.browser,
- dynamic_linking=True,
- ),
- BuildProfile(
- "emscripten-browser-debug",
- support_level=SupportLevel.working,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.browser_debug,
- dynamic_linking=True,
- ),
- BuildProfile(
- "emscripten-node-dl",
- support_level=SupportLevel.supported,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.node,
- dynamic_linking=True,
- ),
- BuildProfile(
- "emscripten-node-dl-debug",
- support_level=SupportLevel.working,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.node_debug,
- dynamic_linking=True,
- ),
- BuildProfile(
- "emscripten-node-pthreads",
- support_level=SupportLevel.supported,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.node,
- pthreads=True,
- ),
- BuildProfile(
- "emscripten-node-pthreads-debug",
- support_level=SupportLevel.working,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.node_debug,
- pthreads=True,
- ),
- # Emscripten build with both pthreads and dynamic linking is crashing.
- BuildProfile(
- "emscripten-node-dl-pthreads-debug",
- support_level=SupportLevel.broken,
- host=Host.wasm32_emscripten,
- target=EmscriptenTarget.node_debug,
- dynamic_linking=True,
- pthreads=True,
- ),
- # wasm64-emscripten (requires Emscripten >= 3.1.21)
- BuildProfile(
- "wasm64-emscripten-node-debug",
- support_level=SupportLevel.experimental,
- host=Host.wasm64_emscripten,
- target=EmscriptenTarget.node_debug,
- # MEMORY64 is not compatible with dynamic linking
- dynamic_linking=False,
- pthreads=False,
- ),
- # wasm32-wasi
- BuildProfile(
- "wasi",
- support_level=SupportLevel.supported,
- host=Host.wasm32_wasi,
- ),
- # wasm32-wasi-threads
- BuildProfile(
- "wasi-threads",
- support_level=SupportLevel.experimental,
- host=Host.wasm32_wasi,
- pthreads=True,
- ),
- # no SDK available yet
- # BuildProfile(
- # "wasm64-wasi",
- # support_level=SupportLevel.broken,
- # host=Host.wasm64_wasi,
- # ),
-]
-
-PROFILES = {p.name: p for p in _profiles}
-
-parser = argparse.ArgumentParser(
- "wasm_build.py",
- description=__doc__,
- formatter_class=argparse.RawTextHelpFormatter,
-)
-
-parser.add_argument(
- "--clean",
- "-c",
- help="Clean build directories first",
- action="store_true",
-)
-
-parser.add_argument(
- "--verbose",
- "-v",
- help="Verbose logging",
- action="store_true",
-)
-
-parser.add_argument(
- "--silent",
- help="Run configure and make in silent mode",
- action="store_true",
-)
-
-parser.add_argument(
- "--testopts",
- help=(
- "Additional test options for 'test' and 'hostrunnertest', e.g. "
- "--testopts='-v test_os'."
- ),
- default=None,
-)
-
-# Don't list broken and experimental variants in help
-platforms_choices = list(p.name for p in _profiles) + ["cleanall"]
-platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"]
-parser.add_argument(
- "platform",
- metavar="PLATFORM",
- help=f"Build platform: {', '.join(platforms_help)}",
- choices=platforms_choices,
-)
-
-ops = dict(
- build="auto build (build 'build' Python, emports, configure, compile)",
- configure="run ./configure",
- compile="run 'make all'",
- pythoninfo="run 'make pythoninfo'",
- test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)",
- hostrunnertest="run 'make hostrunnertest TESTOPTS=...'",
- repl="start interactive REPL / webserver + browser session",
- clean="run 'make clean'",
- cleanall="remove all build directories",
- emports="build Emscripten port with embuilder (only Emscripten)",
-)
-ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items())
-parser.add_argument(
- "ops",
- metavar="OP",
- help=f"operation (default: build)\n\n{ops_help}",
- choices=tuple(ops),
- default="build",
- nargs="*",
-)
-
-
-def main() -> None:
- args = parser.parse_args()
- logging.basicConfig(
- level=logging.INFO if args.verbose else logging.ERROR,
- format="%(message)s",
- )
-
- if args.platform == "cleanall":
- for builder in PROFILES.values():
- builder.clean(all=True)
- parser.exit(0)
-
- # additional configure and make args
- cm_args = ("--silent",) if args.silent else ()
-
- # nargs=* with default quirk
- if args.ops == "build":
- args.ops = ["build"]
-
- builder = PROFILES[args.platform]
- try:
- builder.host.platform.check()
- except ConditionError as e:
- parser.error(str(e))
-
- if args.clean:
- builder.clean(all=False)
-
- # hack for WASI
- if builder.host.is_wasi and not SETUP_LOCAL.exists():
- SETUP_LOCAL.touch()
-
- # auto-build
- if "build" in args.ops:
- # check and create build Python
- if builder is not BUILD:
- logger.info("Auto-building 'build' Python.")
- try:
- BUILD.host.platform.check()
- except ConditionError as e:
- parser.error(str(e))
- if args.clean:
- BUILD.clean(all=False)
- BUILD.run_build(*cm_args)
- # build Emscripten ports with embuilder
- if builder.host.is_emscripten and "emports" not in args.ops:
- builder.build_emports()
-
- for op in args.ops:
- logger.info("\n*** %s %s", args.platform, op)
- if op == "build":
- builder.run_build(*cm_args)
- elif op == "configure":
- builder.run_configure(*cm_args)
- elif op == "compile":
- builder.run_make("all", *cm_args)
- elif op == "pythoninfo":
- builder.run_pythoninfo(*cm_args)
- elif op == "repl":
- if builder.is_browser:
- builder.run_browser()
- else:
- builder.run_py()
- elif op == "test":
- builder.run_test("buildbottest", testopts=args.testopts)
- elif op == "hostrunnertest":
- builder.run_test("hostrunnertest", testopts=args.testopts)
- elif op == "clean":
- builder.clean(all=False)
- elif op == "cleanall":
- builder.clean(all=True)
- elif op == "emports":
- builder.build_emports(force=args.clean)
- else:
- raise ValueError(op)
-
- print(builder.builddir)
- parser.exit(0)
-
-
-if __name__ == "__main__":
- main()
generated by cgit v1.2.3 (git 2.39.1) at 2025-05-09 09:26:48 +0000