bpo-45188: Windows now regenerates frozen modules at the start of build instead of late (GH-28322)

This will enable us to drop the frozen module header files from the repository.

It does currently cause many source files to be built twice, which just takes more time. For whoever comes to fix this in the future, the files shared between freeze_module and pythoncore should be put into a static library that is consumed by both.

1 files changed, 47 insertions, 13 deletions

diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py
index a9111eca634..044ea095fdc 100644
--- a/Tools/scripts/freeze_modules.py
+++ b/Tools/scripts/freeze_modules.py
@@ -6,7 +6,8 @@ See the notes at the top of Python/frozen.c for more info.
 from collections import namedtuple
 import hashlib
 import os
-import os.path
+import ntpath
+import posixpath
 import subprocess
 import sys
 import textwrap
@@ -21,7 +22,19 @@ ROOT_DIR = os.path.dirname(TOOLS_DIR)
 STDLIB_DIR = os.path.join(ROOT_DIR, 'Lib')
 # If MODULES_DIR is changed then the .gitattributes file needs to be updated.
 MODULES_DIR = os.path.join(ROOT_DIR, 'Python/frozen_modules')
-TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
+
+if sys.platform != "win32":
+    TOOL = os.path.join(ROOT_DIR, 'Programs', '_freeze_module')
+else:
+    def find_tool():
+        for arch in ['amd64', 'win32']:
+            for exe in ['_freeze_module.exe', '_freeze_module_d.exe']:
+                tool = os.path.join(ROOT_DIR, 'PCbuild', arch, exe)
+                if os.path.isfile(tool):
+                    return tool
+        sys.exit("ERROR: missing _freeze_module.exe; you need to run PCbuild/build.bat")
+    TOOL = find_tool()
+    del find_tool
 
 MANIFEST = os.path.join(MODULES_DIR, 'MANIFEST')
 FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
@@ -82,6 +95,28 @@ ESSENTIAL = {
 
 
 #######################################
+# platform-specific helpers
+
+if os.path is posixpath:
+    relpath_for_posix_display = os.path.relpath
+
+    def relpath_for_windows_display(path, base):
+        return ntpath.relpath(
+            ntpath.join(*path.split(os.path.sep)),
+            ntpath.join(*base.split(os.path.sep)),
+        )
+
+else:
+    relpath_for_windows_display = ntpath.relpath
+
+    def relpath_for_posix_display(path, base):
+        return posixpath.relpath(
+            posixpath.join(*path.split(os.path.sep)),
+            posixpath.join(*base.split(os.path.sep)),
+        )
+
+
+#######################################
 # specs
 
 def parse_frozen_specs(sectionalspecs=FROZEN, destdir=None):
@@ -253,7 +288,7 @@ class FrozenModule(namedtuple('FrozenModule', 'name ispkg section source')):
         if source:
             source = f'<{source}>'
         else:
-            source = os.path.relpath(self.pyfile, ROOT_DIR)
+            source = relpath_for_posix_display(self.pyfile, ROOT_DIR)
         return {
             'module': self.name,
             'ispkg': self.ispkg,
@@ -397,7 +432,7 @@ def replace_block(lines, start_marker, end_marker, replacements, file):
         raise Exception(f"End marker {end_marker!r} "
                         f"occurs before start marker {start_marker!r} "
                         f"in file {file}")
-    replacements = [line.rstrip() + os.linesep for line in replacements]
+    replacements = [line.rstrip() + '\n' for line in replacements]
     return lines[:start_pos + 1] + replacements + lines[end_pos:]
 
 
@@ -446,7 +481,7 @@ def regen_frozen(modules):
     for src in _iter_sources(modules):
         # Adding a comment to separate sections here doesn't add much,
         # so we don't.
-        header = os.path.relpath(src.frozenfile, parentdir)
+        header = relpath_for_posix_display(src.frozenfile, parentdir)
         headerlines.append(f'#include "{header}"')
 
     deflines = []
@@ -503,11 +538,10 @@ def regen_makefile(modules):
     frozenfiles = []
     rules = ['']
     for src in _iter_sources(modules):
-        header = os.path.relpath(src.frozenfile, ROOT_DIR)
-        relfile = header.replace('\\', '/')
-        frozenfiles.append(f'\t\t$(srcdir)/{relfile} \\')
+        header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
+        frozenfiles.append(f'\t\t$(srcdir)/{header} \\')
 
-        pyfile = os.path.relpath(src.pyfile, ROOT_DIR)
+        pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
         # Note that we freeze the module to the target .h file
         # instead of going through an intermediate file like we used to.
         rules.append(f'{header}: Programs/_freeze_module {pyfile}')
@@ -546,9 +580,9 @@ def regen_pcbuild(modules):
         # See bpo-45186 and bpo-45188.
         if src.id not in ESSENTIAL and src.id != 'hello':
             continue
-        pyfile = os.path.relpath(src.pyfile, ROOT_DIR).replace('/', '\\')
-        header = os.path.relpath(src.frozenfile, ROOT_DIR).replace('/', '\\')
-        intfile = header.split('\\')[-1].strip('.h') + '.g.h'
+        pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
+        header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
+        intfile = ntpath.splitext(ntpath.basename(header))[0] + '.g.h'
         projlines.append(f'    <None Include="..\\{pyfile}">')
         projlines.append(f'      <ModName>{src.frozenid}</ModName>')
         projlines.append(f'      <IntFile>$(IntDir){intfile}</IntFile>')
@@ -600,7 +634,7 @@ def _freeze_module(frozenid, pyfile, frozenfile):
     print('#', '  '.join(os.path.relpath(a) for a in argv), flush=True)
     try:
         subprocess.run(argv, check=True)
-    except subprocess.CalledProcessError:
+    except (FileNotFoundError, subprocess.CalledProcessError):
         if not os.path.exists(TOOL):
             sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"')
         raise  # re-raise