Fix broken tests when not in isolated mode

Summary:
Tests run with Lazy Imports broke in D84660225, which switches from using isolated mode. This led us to discover that several tests were breaking with Lazy Imports on when run in sequential mode (without the `-j` flag that indicates multiprocessing to the test runner).
___
Several of the changes made in D52812410 to test files (in order to work around Lazy Imports incompatibilities such as implicit imports) weren't added in D83595369.
This Diff adds the needed changes from there (discarding the ones that aren't needed and aren't preventative), as well as adding some changes here.
____
The categorization is as follows:

In D52812410 but not ported over to here since they didn't affect tests:
- `Lib‎/email‎/‎_header_value_parser.py`
- `Lib/test/test_doctest.py`
- `Lib‎/test‎/‎test___all__.py`
- `Lib/test/test_clinic.py`

In D52812410 and ported over to here since they seemed good from a safety perspective (even though they didn't affect tests):
- `Lib‎/test‎/libregrtest‎/‎utils.py`
- `Lib‎/test‎/test_capi‎/‎test_misc.py`
- `Lib‎/test‎/test_unittest‎/‎test_discovery.py`

In D52812410 and ported over to here since they affected tests:
- `Lib‎/test‎/support‎/‎__init__.py`
- `Lib/test/support/import_helper.py`
- `Lib‎/test‎/test_import‎/data‎/package2‎/‎submodule1.py`
- `Lib/test/test_import/__init__.py`
- `Lib/test/test_heapq.py`
- `Lib/test/test_pkg.py`
- `Lib/test/test_traceback.py`

Added to make new test failures pass:
- `Lib/test/test_asyncio/test_eager_task_factory.py`
- `Lib/test/test__interpreters.py`
- `Lib/test/test_builtin.py`
- `Lib/test/test_monitoring.py`

Reviewed By: itamaro

Differential Revision: D84747057

fbshipit-source-id: 3b143e045713ce1ffdf1739d288ee4f13c9b8dd4

Author: Subbarao Garlapati

Lib/idlelib/idle_test/test_sidebar.py

@@ -15,10 +15,15 @@
 from idlelib.percolator import Percolator
 import idlelib.pyshell
 from idlelib.pyshell import fix_x11_paste, PyShell, PyShellFileList
-from idlelib.run import fix_scaling
 import idlelib.sidebar
 from idlelib.sidebar import get_end_linenumber, get_lineno
 
+# Lazy Imports Compatibility: Explictly load `fix_scaling` so that the test loader is able to find it with `getattr()`.
+try:
+    import idlelib.run
+    fix_scaling = idlelib.run.fix_scaling
+except Exception:
+    pass
 
 class Dummy_editwin:
     def __init__(self, text):

Lib/idlelib/run.py

@@ -27,7 +27,13 @@
 from idlelib import stackviewer  # StackTreeItem
 import __main__
 
-import tkinter  # Use tcl and, if startup fails, messagebox.
+# Lazy Imports Incompatibility: The following modules must be imported eagerly
+try:
+    import tkinter  # Use tcl and, if startup fails, messagebox.
+    from tkinter import simpledialog, messagebox, font, dialog, filedialog, commondialog, ttk
+except Exception:
+    pass
+
 if not hasattr(sys.modules['idlelib.run'], 'firstrun'):
     # Undo modifications of tkinter by idlelib imports; see bpo-25507.
     for mod in ('simpledialog', 'messagebox', 'font',

Lib/test/libregrtest/utils.py

@@ -177,7 +177,9 @@ def setup_threading_excepthook() -> None:
 
 def clear_caches():
     # Clear the warnings registry, so they can be displayed again
-    for mod in sys.modules.values():
+    # Modules with __getattr__ could trigger imports and change sys.modules,
+    # so we put the modules in a list of modules.
+    for mod in list(sys.modules.values()):
         if hasattr(mod, '__warningregistry__'):
             del mod.__warningregistry__
 

Lib/test/support/import_helper.py

@@ -171,6 +171,7 @@ def import_fresh_module(name, fresh=(), blocked=(), *,
         for modname in blocked:
             sys.modules[modname] = None
 
+        previously = _imp._set_lazy_imports(False)
         try:
             with frozen_modules(usefrozen):
                 # Return None when one of the "fresh" modules can not be imported.
@@ -181,6 +182,7 @@ def import_fresh_module(name, fresh=(), blocked=(), *,
                     return None
                 return importlib.import_module(name)
         finally:
+            _imp._set_lazy_imports(*previously)
             _save_and_remove_modules(names)
             sys.modules.update(orig_modules)
 

Lib/test/test__interpreters.py

@@ -976,7 +976,7 @@ def test_invalid_syntax(self):
             modname = 'spam_spam_spam'
             filename = self.add_module(modname, script)
             self.assert_run_failed(SyntaxError, f"""
-                import {modname}
+                import {modname}; {modname}
                 """)
 
     def test_NameError(self):

Lib/test/test_asyncio/test_eager_task_factory.py

@@ -5,7 +5,7 @@
 import unittest
 
 from unittest import mock
-from asyncio import tasks
+from asyncio import staggered, tasks
 from test.test_asyncio import utils as test_utils
 from test.support.script_helper import assert_python_ok
 
@@ -225,7 +225,7 @@ async def blocked():
             await fut
 
         async def run():
-            winner, index, excs = await asyncio.staggered.staggered_race(
+            winner, index, excs = await staggered.staggered_race(
                 [
                     lambda: blocked(),
                     lambda: asyncio.sleep(1, result="sleep1"),
@@ -247,7 +247,7 @@ async def fail():
             raise ValueError("no good")
 
         async def run():
-            winner, index, excs = await asyncio.staggered.staggered_race(
+            winner, index, excs = await staggered.staggered_race(
                 [
                     lambda: fail(),
                     lambda: asyncio.sleep(1, result="sleep1"),

Lib/test/test_builtin.py

@@ -7,6 +7,7 @@
 import decimal
 import fractions
 import gc
+import importlib
 import io
 import locale
 import math
@@ -968,12 +969,17 @@ def test_eval_builtins_mapping(self):
                                eval, code, ns)
 
     def test_exec_builtins_mapping_import(self):
-        code = compile("import foo.bar", "test", "exec")
+        # Force the import to happen even with Lazy Imports enabled
+        code = compile("import foo.bar; foo.bar", "test", "exec")
         ns = {'__builtins__': types.MappingProxyType({})}
         self.assertRaisesRegex(ImportError, "__import__ not found", exec, code, ns)
         ns = {'__builtins__': types.MappingProxyType({'__import__': lambda *args: args})}
+        # Go back to using original code object since bar doesn't exist
+        code = compile("import foo.bar", "test", "exec")
         exec(code, ns)
-        self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0))
+        # Skip this assert since lazy imports doesn't resolve foo.bar
+        if not importlib.is_lazy_imports_enabled():
+            self.assertEqual(ns['foo'], ('foo.bar', ns, ns, None, 0))
 
     def test_eval_builtins_mapping_reduce(self):
         # list_iterator.__reduce__() calls _PyEval_GetBuiltin("iter")

Lib/test/test_capi/test_misc.py

@@ -1434,6 +1434,11 @@ def test_isolated_subinterpreter(self):
             import time
             import _testinternalcapi
             from test.support import threading_helper
+
+            # FIXME: To prevent threading._shutdown() from locking (locked _shutdown_locks)
+            # when using Lazy Imports, we need to eagerly import threading here.
+            # Possibly related to gh-105716.
+            threading
             """)
 
         def create_pipe():

Lib/test/test_heapq.py

@@ -39,7 +39,7 @@ def load_tests(loader, tests, ignore):
     class HeapqMergeDocTestFinder:
         def find(self, *args, **kwargs):
             dtf = doctest.DocTestFinder()
-            return dtf.find(py_heapq.merge)
+            return dtf.find(py_heapq.merge, module=py_heapq)
 
     tests.addTests(doctest.DocTestSuite(py_heapq,
                                         test_finder=HeapqMergeDocTestFinder()))

Lib/test/test_import/__init__.py

@@ -11,6 +11,7 @@
     FrozenImporter,
     SourceFileLoader,
 )
+from importlib import is_lazy_imports_enabled
 import marshal
 import os
 import py_compile
@@ -1494,6 +1495,7 @@ def test_relimport_star(self):
         from .. import relimport
         self.assertHasAttr(relimport, "RelativeImportTests")
 
+    @unittest.skipIf(is_lazy_imports_enabled(), "Test relevant only when running with lazy imports disabled")
     def test_issue3221(self):
         # Note for mergers: the 'absolute' tests from the 2.x branch
         # are missing in Py3k because implicit relative imports are
@@ -1916,6 +1918,7 @@ def test_nonexistent_module(self):
             self.fail("ImportError should have been raised")
         self.assert_traceback(tb, [__file__])
 
+    @unittest.skipIf(is_lazy_imports_enabled(), "Lazy Imports has different behavior in nested modules")
     def test_nonexistent_module_nested(self):
         self.create_module("foo", "import nonexistent_xyzzy")
         try:
@@ -1936,6 +1939,7 @@ def test_exec_failure(self):
             self.fail("ZeroDivisionError should have been raised")
         self.assert_traceback(tb, [__file__, 'foo.py'])
 
+    @unittest.skipIf(is_lazy_imports_enabled(), "Lazy Imports has different behavior in nested modules")
     def test_exec_failure_nested(self):
         self.create_module("foo", "import bar")
         self.create_module("bar", "1/0")
@@ -2100,6 +2104,7 @@ def test_crossreference1(self):
         import test.test_import.data.circular_imports.use
         import test.test_import.data.circular_imports.source
 
+    @unittest.skipIf(is_lazy_imports_enabled(), "Test relevant only when running with lazy imports disabled")
     def test_crossreference2(self):
         with self.assertRaises(AttributeError) as cm:
             import test.test_import.data.circular_imports.source
@@ -2109,6 +2114,7 @@ def test_crossreference2(self):
         self.assertIn('partially initialized module', errmsg)
         self.assertIn('circular import', errmsg)
 
+    @unittest.skipIf(is_lazy_imports_enabled(), "Test relevant only when running with lazy imports disabled")
     def test_circular_from_import(self):
         with self.assertRaises(ImportError) as cm:
             import test.test_import.data.circular_imports.from_cycle1
@@ -2127,6 +2133,7 @@ def test_circular_import(self):
         ):
             import test.test_import.data.circular_imports.import_cycle
 
+    @unittest.skipIf(is_lazy_imports_enabled(), "Test relevant only when running with lazy imports disabled")
     def test_absolute_circular_submodule(self):
         with self.assertRaises(AttributeError) as cm:
             import test.test_import.data.circular_imports.subpkg2.parent