From c0362efb8dde71df1372b674d4b08a144105f12b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20Kr=C3=BCger?= Date: Wed, 24 Jan 2024 18:14:49 +0100 Subject: [PATCH] Move asyncio loop management from plugin to main. (#3729) Remove support for ugly async IO loop handling kludges required by Python 3.7. --- nikola/__main__.py | 18 +++---- nikola/plugins/command/auto/__init__.py | 28 ++--------- tests/integration/test_dev_server.py | 64 +++++++------------------ 3 files changed, 32 insertions(+), 78 deletions(-) diff --git a/nikola/__main__.py b/nikola/__main__.py index cb7ef0b5a9..4ac28e456c 100644 --- a/nikola/__main__.py +++ b/nikola/__main__.py @@ -26,16 +26,10 @@ """The main function of Nikola.""" -import importlib.util -import os -import shutil -import sys -import textwrap -import traceback -import doit.cmd_base +from blinker import signal from collections import defaultdict -from blinker import signal +import doit.cmd_base from doit.cmd_base import TaskLoader2, _wrap from doit.cmd_clean import Clean as DoitClean from doit.cmd_completion import TabCompletion @@ -45,6 +39,13 @@ from doit.loader import generate_tasks from doit.reporter import ExecutedOnlyReporter +import importlib.util +import os +import shutil +import sys +import textwrap +import traceback + from . import __version__ from .nikola import Nikola from .plugin_categories import Command @@ -56,7 +57,6 @@ except ImportError: pass # This is only so raw_input/input does nicer things if it's available - config = {} # DO NOT USE unless you know what you are doing! diff --git a/nikola/plugins/command/auto/__init__.py b/nikola/plugins/command/auto/__init__.py index d272c2365f..9668b784fc 100644 --- a/nikola/plugins/command/auto/__init__.py +++ b/nikola/plugins/command/auto/__init__.py @@ -64,9 +64,6 @@ REBUILDING_REFRESH_DELAY = 0.35 IDLE_REFRESH_DELAY = 0.05 -if sys.platform == 'win32': - asyncio.set_event_loop(asyncio.ProactorEventLoop()) - def base_path_from_siteuri(siteuri: str) -> str: """Extract the path part from a URI such as site['SITE_URL']. @@ -275,27 +272,18 @@ def _execute(self, options, args): self.wd_observer.schedule(ConfigEventHandler(_conf_fn, self.queue_rebuild, loop), _conf_dn, recursive=False) self.wd_observer.start() - win_sleeper = None - # https://bugs.python.org/issue23057 (fixed in Python 3.8) - if sys.platform == 'win32' and sys.version_info < (3, 8): - win_sleeper = asyncio.ensure_future(windows_ctrlc_workaround()) - if not self.has_server: self.logger.info("Watching for changes...") # Run the event loop forever (no server mode). try: # Run rebuild queue loop.run_until_complete(self.run_rebuild_queue()) - loop.run_forever() except KeyboardInterrupt: pass finally: - if win_sleeper: - win_sleeper.cancel() self.wd_observer.stop() self.wd_observer.join() - loop.close() return if options['ipv6'] or '::' in host: @@ -326,16 +314,17 @@ def _execute(self, options, args): pass finally: self.logger.info("Server is shutting down.") - if win_sleeper: - win_sleeper.cancel() if self.dns_sd: self.dns_sd.Reset() rebuild_queue_fut.cancel() reload_queue_fut.cancel() + + # Not sure why this isn't done by the web_runner.cleanup() code: + loop.run_until_complete(self.remove_websockets(None)) + loop.run_until_complete(self.web_runner.cleanup()) self.wd_observer.stop() self.wd_observer.join() - loop.close() async def set_up_server(self, host: str, port: int, base_path: str, out_folder: str) -> None: """Set up aiohttp server and start it.""" @@ -482,7 +471,7 @@ async def websocket_handler(self, request): return ws - async def remove_websockets(self, app) -> None: + async def remove_websockets(self, _app) -> None: """Remove all websockets.""" for ws in self.sockets: await ws.close() @@ -512,13 +501,6 @@ async def send_to_websockets(self, message: dict) -> None: self.sockets.remove(ws) -async def windows_ctrlc_workaround() -> None: - """Work around bpo-23057.""" - # https://bugs.python.org/issue23057 - while True: - await asyncio.sleep(1) - - class IndexHtmlStaticResource(StaticResource): """A StaticResource implementation that serves /index.html in directory roots.""" diff --git a/tests/integration/test_dev_server.py b/tests/integration/test_dev_server.py index 769678f4b5..64f44e7c61 100644 --- a/tests/integration/test_dev_server.py +++ b/tests/integration/test_dev_server.py @@ -5,7 +5,6 @@ import pathlib import requests import socket -import sys from typing import Optional, Tuple, Any, Dict from ..helper import FakeSite @@ -38,6 +37,7 @@ def find_unused_port() -> int: class MyFakeSite(FakeSite): def __init__(self, config: Dict[str, Any], configuration_filename="conf.py"): + super(MyFakeSite, self).__init__() self.configured = True self.debug = True self.THEMES = [] @@ -70,13 +70,13 @@ def test_serves_root_dir( test_was_successful = False test_problem_description = "Async test setup apparently broken" test_inner_error: Optional[BaseException] = None - loop_for_this_test = None + loop = None async def grab_loop_and_run_test() -> None: - nonlocal test_problem_description, loop_for_this_test + nonlocal test_problem_description, loop - loop_for_this_test = asyncio.get_running_loop() - watchdog_handle = loop_for_this_test.call_later(TEST_MAX_DURATION, lambda: loop_for_this_test.stop()) + loop = asyncio.get_running_loop() + watchdog_handle = loop.call_later(TEST_MAX_DURATION, loop.stop) test_problem_description = f"Test did not complete within {TEST_MAX_DURATION} seconds." def run_test() -> None: @@ -113,13 +113,11 @@ def run_test() -> None: LOGGER.info("Test completed successfully.") else: LOGGER.error("Test failed: %s", test_problem_description) - loop_for_this_test.call_soon_threadsafe(lambda: watchdog_handle.cancel()) + loop.call_soon_threadsafe(watchdog_handle.cancel) + # Simulate Ctrl+C: + loop.call_soon_threadsafe(lambda: loop.call_later(0.01, loop.stop)) - # We give the outer grab_loop_and_run_test a chance to complete - # before burning the bridge: - loop_for_this_test.call_soon_threadsafe(lambda: loop_for_this_test.call_later(0.05, lambda: loop_for_this_test.stop())) - - await loop_for_this_test.run_in_executor(None, run_test) + await loop.run_in_executor(None, run_test) # We defeat the nikola site building functionality, so this does not actually get called. # But the code setting up site building wants a command list: @@ -128,40 +126,14 @@ def run_test() -> None: # Defeat the site building functionality, and instead insert the test: command_auto.run_initial_rebuild = grab_loop_and_run_test - try: - # Start the development server - # which under the hood runs our test when trying to build the site: - command_auto.execute(options=options) - - # Verify the test succeeded: - if test_inner_error is not None: - raise test_inner_error - assert test_was_successful, test_problem_description - finally: - # Nikola is written with the assumption that it can - # create the event loop at will without ever cleaning it up. - # As this tests runs several times in succession, - # that assumption becomes a problem. - LOGGER.info("Cleaning up loop.") - # Loop cleanup: - assert loop_for_this_test is not None - assert not loop_for_this_test.is_running() - loop_for_this_test.close() - asyncio.set_event_loop(None) - # We would like to leave it at that, - # but doing so causes the next test to fail. - # - # We did not find asyncio - API to reset the loop - # to "back to square one, as if just freshly started". - # - # The following code does not feel right, it's a kludge, - # but it apparently works for now: - if sys.platform == 'win32': - # For this case, the auto module has special code - # (at module load time! 😟) which we reluctantly reproduce here: - asyncio.set_event_loop(asyncio.ProactorEventLoop()) - else: - asyncio.set_event_loop(asyncio.new_event_loop()) + # Start the development server + # which under the hood runs our test when trying to build the site: + command_auto.execute(options=options) + + # Verify the test succeeded: + if test_inner_error is not None: + raise test_inner_error + assert test_was_successful, test_problem_description @pytest.fixture(scope="module", @@ -184,7 +156,7 @@ def site_and_base_path(request) -> Tuple[MyFakeSite, str]: "SITE_URL": request.param, "OUTPUT_FOLDER": OUTPUT_FOLDER.as_posix(), } - return (MyFakeSite(config), auto.base_path_from_siteuri(request.param)) + return MyFakeSite(config), auto.base_path_from_siteuri(request.param) @pytest.fixture(scope="module")