Take the first step in resolving the messy pkgutil vs importlib edge cases by basing pkgutil explicitly on importlib, deprecating its internal import emulation and setting __main__.__loader__ correctly so that runpy still works (Affects #15343, #15314, #15357)

1 files changed, 56 insertions, 17 deletions

diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py
index d8bc2637af5..a08cabd2cb3 100644
--- a/Lib/test/test_cmd_line_script.py
+++ b/Lib/test/test_cmd_line_script.py
@@ -1,6 +1,8 @@
 # tests command line execution of scripts
 
 import importlib
+import importlib.machinery
+import zipimport
 import unittest
 import sys
 import os
@@ -11,7 +13,8 @@ import textwrap
 from test import support
 from test.script_helper import (
     make_pkg, make_script, make_zip_pkg, make_zip_script,
-    assert_python_ok, assert_python_failure, temp_dir)
+    assert_python_ok, assert_python_failure, temp_dir,
+    spawn_python, kill_python)
 
 verbose = support.verbose
 
@@ -34,6 +37,8 @@ f()
 assertEqual(result, ['Top level assignment', 'Lower level reference'])
 # Check population of magic variables
 assertEqual(__name__, '__main__')
+_loader = __loader__ if isinstance(__loader__, type) else type(__loader__)
+print('__loader__==%a' % _loader)
 print('__file__==%a' % __file__)
 assertEqual(__cached__, None)
 print('__package__==%r' % __package__)
@@ -85,11 +90,13 @@ def _make_launch_script(script_dir, script_basename, module_name, path=None):
 class CmdLineTest(unittest.TestCase):
     def _check_output(self, script_name, exit_code, data,
                              expected_file, expected_argv0,
-                             expected_path0, expected_package):
+                             expected_path0, expected_package,
+                             expected_loader):
         if verbose > 1:
             print("Output from test script %r:" % script_name)
             print(data)
         self.assertEqual(exit_code, 0)
+        printed_loader = '__loader__==%a' % expected_loader
         printed_file = '__file__==%a' % expected_file
         printed_package = '__package__==%r' % expected_package
         printed_argv0 = 'sys.argv[0]==%a' % expected_argv0
@@ -101,6 +108,7 @@ class CmdLineTest(unittest.TestCase):
             print(printed_package)
             print(printed_argv0)
             print(printed_cwd)
+        self.assertIn(printed_loader.encode('utf-8'), data)
         self.assertIn(printed_file.encode('utf-8'), data)
         self.assertIn(printed_package.encode('utf-8'), data)
         self.assertIn(printed_argv0.encode('utf-8'), data)
@@ -109,14 +117,15 @@ class CmdLineTest(unittest.TestCase):
 
     def _check_script(self, script_name, expected_file,
                             expected_argv0, expected_path0,
-                            expected_package,
+                            expected_package, expected_loader,
                             *cmd_line_switches):
         if not __debug__:
             cmd_line_switches += ('-' + 'O' * sys.flags.optimize,)
         run_args = cmd_line_switches + (script_name,) + tuple(example_args)
         rc, out, err = assert_python_ok(*run_args)
         self._check_output(script_name, rc, out + err, expected_file,
-                           expected_argv0, expected_path0, expected_package)
+                           expected_argv0, expected_path0,
+                           expected_package, expected_loader)
 
     def _check_import_error(self, script_name, expected_msg,
                             *cmd_line_switches):
@@ -128,11 +137,27 @@ class CmdLineTest(unittest.TestCase):
             print('Expected output: %r' % expected_msg)
         self.assertIn(expected_msg.encode('utf-8'), err)
 
+    def test_dash_c_loader(self):
+        rc, out, err = assert_python_ok("-c", "print(__loader__)")
+        expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8")
+        self.assertIn(expected, out)
+
+    def test_stdin_loader(self):
+        p = spawn_python()
+        try:
+            p.stdin.write(b"print(__loader__)\n")
+            p.stdin.flush()
+        finally:
+            out = kill_python(p)
+        expected = repr(importlib.machinery.BuiltinImporter).encode("utf-8")
+        self.assertIn(expected, out)
+
     def test_basic_script(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, 'script')
             self._check_script(script_name, script_name, script_name,
-                               script_dir, None)
+                               script_dir, None,
+                               importlib.machinery.SourceFileLoader)
 
     def test_script_compiled(self):
         with temp_dir() as script_dir:
@@ -141,13 +166,15 @@ class CmdLineTest(unittest.TestCase):
             os.remove(script_name)
             pyc_file = support.make_legacy_pyc(script_name)
             self._check_script(pyc_file, pyc_file,
-                               pyc_file, script_dir, None)
+                               pyc_file, script_dir, None,
+                               importlib.machinery.SourcelessFileLoader)
 
     def test_directory(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             self._check_script(script_dir, script_name, script_dir,
-                               script_dir, '')
+                               script_dir, '',
+                               importlib.machinery.SourceFileLoader)
 
     def test_directory_compiled(self):
         with temp_dir() as script_dir:
@@ -156,7 +183,8 @@ class CmdLineTest(unittest.TestCase):
             os.remove(script_name)
             pyc_file = support.make_legacy_pyc(script_name)
             self._check_script(script_dir, pyc_file, script_dir,
-                               script_dir, '')
+                               script_dir, '',
+                               importlib.machinery.SourcelessFileLoader)
 
     def test_directory_error(self):
         with temp_dir() as script_dir:
@@ -167,14 +195,16 @@ class CmdLineTest(unittest.TestCase):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', script_name)
-            self._check_script(zip_name, run_name, zip_name, zip_name, '')
+            self._check_script(zip_name, run_name, zip_name, zip_name, '',
+                               zipimport.zipimporter)
 
     def test_zipfile_compiled(self):
         with temp_dir() as script_dir:
             script_name = _make_test_script(script_dir, '__main__')
             compiled_name = py_compile.compile(script_name, doraise=True)
             zip_name, run_name = make_zip_script(script_dir, 'test_zip', compiled_name)
-            self._check_script(zip_name, run_name, zip_name, zip_name, '')
+            self._check_script(zip_name, run_name, zip_name, zip_name, '',
+                               zipimport.zipimporter)
 
     def test_zipfile_error(self):
         with temp_dir() as script_dir:
@@ -189,19 +219,24 @@ class CmdLineTest(unittest.TestCase):
             make_pkg(pkg_dir)
             script_name = _make_test_script(pkg_dir, 'script')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script')
-            self._check_script(launch_name, script_name, script_name, script_dir, 'test_pkg')
+            self._check_script(launch_name, script_name, script_name,
+                               script_dir, 'test_pkg',
+                               importlib.machinery.SourceFileLoader)
 
     def test_module_in_package_in_zipfile(self):
         with temp_dir() as script_dir:
             zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.script', zip_name)
-            self._check_script(launch_name, run_name, run_name, zip_name, 'test_pkg')
+            self._check_script(launch_name, run_name, run_name,
+                               zip_name, 'test_pkg', zipimport.zipimporter)
 
     def test_module_in_subpackage_in_zipfile(self):
         with temp_dir() as script_dir:
             zip_name, run_name = _make_test_zip_pkg(script_dir, 'test_zip', 'test_pkg', 'script', depth=2)
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg.test_pkg.script', zip_name)
-            self._check_script(launch_name, run_name, run_name, zip_name, 'test_pkg.test_pkg')
+            self._check_script(launch_name, run_name, run_name,
+                               zip_name, 'test_pkg.test_pkg',
+                               zipimport.zipimporter)
 
     def test_package(self):
         with temp_dir() as script_dir:
@@ -210,7 +245,8 @@ class CmdLineTest(unittest.TestCase):
             script_name = _make_test_script(pkg_dir, '__main__')
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
             self._check_script(launch_name, script_name,
-                               script_name, script_dir, 'test_pkg')
+                               script_name, script_dir, 'test_pkg',
+                               importlib.machinery.SourceFileLoader)
 
     def test_package_compiled(self):
         with temp_dir() as script_dir:
@@ -222,7 +258,8 @@ class CmdLineTest(unittest.TestCase):
             pyc_file = support.make_legacy_pyc(script_name)
             launch_name = _make_launch_script(script_dir, 'launch', 'test_pkg')
             self._check_script(launch_name, pyc_file,
-                               pyc_file, script_dir, 'test_pkg')
+                               pyc_file, script_dir, 'test_pkg',
+                               importlib.machinery.SourcelessFileLoader)
 
     def test_package_error(self):
         with temp_dir() as script_dir:
@@ -259,7 +296,8 @@ class CmdLineTest(unittest.TestCase):
                 expected = "init_argv0==%r" % '-m'
                 self.assertIn(expected.encode('utf-8'), out)
                 self._check_output(script_name, rc, out,
-                                   script_name, script_name, '', 'test_pkg')
+                                   script_name, script_name, '', 'test_pkg',
+                                   importlib.machinery.SourceFileLoader)
 
     def test_issue8202_dash_c_file_ignored(self):
         # Make sure a "-c" file in the current directory
@@ -285,7 +323,8 @@ class CmdLineTest(unittest.TestCase):
                     f.write("data")
                     rc, out, err = assert_python_ok('-m', 'other', *example_args)
                     self._check_output(script_name, rc, out,
-                                      script_name, script_name, '', '')
+                                      script_name, script_name, '', '',
+                                      importlib.machinery.SourceFileLoader)
 
     def test_dash_m_error_code_is_one(self):
         # If a module is invoked with the -m command line flag