Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Move asyncio loop management from plugin to main. #3729

Merged
merged 1 commit into from
Jan 24, 2024
Merged

Move asyncio loop management from plugin to main. #3729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions nikola/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,10 @@

"""The main function of Nikola."""

import importlib.util
import os
import shutil
import sys
import textwrap
import traceback
import doit.cmd_base
from blinker import signal
from collections import defaultdict

from blinker import signal
import doit.cmd_base
from doit.cmd_base import TaskLoader2, _wrap
from doit.cmd_clean import Clean as DoitClean
from doit.cmd_completion import TabCompletion
Expand All @@ -45,6 +39,13 @@
from doit.loader import generate_tasks
from doit.reporter import ExecutedOnlyReporter

import importlib.util
import os
import shutil
import sys
import textwrap
import traceback

from . import __version__
from .nikola import Nikola
from .plugin_categories import Command
Expand All @@ -56,7 +57,6 @@
except ImportError:
pass # This is only so raw_input/input does nicer things if it's available


config = {}

# DO NOT USE unless you know what you are doing!
Expand Down
28 changes: 5 additions & 23 deletions nikola/plugins/command/auto/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,9 +64,6 @@
REBUILDING_REFRESH_DELAY = 0.35
IDLE_REFRESH_DELAY = 0.05

if sys.platform == 'win32':
asyncio.set_event_loop(asyncio.ProactorEventLoop())


def base_path_from_siteuri(siteuri: str) -> str:
"""Extract the path part from a URI such as site['SITE_URL'].
Expand Down Expand Up @@ -275,27 +272,18 @@ def _execute(self, options, args):
self.wd_observer.schedule(ConfigEventHandler(_conf_fn, self.queue_rebuild, loop), _conf_dn, recursive=False)
self.wd_observer.start()

win_sleeper = None
# https://bugs.python.org/issue23057 (fixed in Python 3.8)
if sys.platform == 'win32' and sys.version_info < (3, 8):
win_sleeper = asyncio.ensure_future(windows_ctrlc_workaround())

if not self.has_server:
self.logger.info("Watching for changes...")
# Run the event loop forever (no server mode).
try:
# Run rebuild queue
loop.run_until_complete(self.run_rebuild_queue())

loop.run_forever()
except KeyboardInterrupt:
pass
finally:
if win_sleeper:
win_sleeper.cancel()
self.wd_observer.stop()
self.wd_observer.join()
loop.close()
return

if options['ipv6'] or '::' in host:
Expand Down Expand Up @@ -326,16 +314,17 @@ def _execute(self, options, args):
pass
finally:
self.logger.info("Server is shutting down.")
if win_sleeper:
win_sleeper.cancel()
if self.dns_sd:
self.dns_sd.Reset()
rebuild_queue_fut.cancel()
reload_queue_fut.cancel()

# Not sure why this isn't done by the web_runner.cleanup() code:
loop.run_until_complete(self.remove_websockets(None))

loop.run_until_complete(self.web_runner.cleanup())
self.wd_observer.stop()
self.wd_observer.join()
loop.close()

async def set_up_server(self, host: str, port: int, base_path: str, out_folder: str) -> None:
"""Set up aiohttp server and start it."""
Expand Down Expand Up @@ -482,7 +471,7 @@ async def websocket_handler(self, request):

return ws

async def remove_websockets(self, app) -> None:
async def remove_websockets(self, _app) -> None:
"""Remove all websockets."""
for ws in self.sockets:
await ws.close()
Expand Down Expand Up @@ -512,13 +501,6 @@ async def send_to_websockets(self, message: dict) -> None:
self.sockets.remove(ws)


async def windows_ctrlc_workaround() -> None:
"""Work around bpo-23057."""
# https://bugs.python.org/issue23057
while True:
await asyncio.sleep(1)


class IndexHtmlStaticResource(StaticResource):
"""A StaticResource implementation that serves /index.html in directory roots."""

Expand Down
64 changes: 18 additions & 46 deletions tests/integration/test_dev_server.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import pathlib
import requests
import socket
import sys
from typing import Optional, Tuple, Any, Dict

from ..helper import FakeSite
Expand Down Expand Up @@ -38,6 +37,7 @@ def find_unused_port() -> int:

class MyFakeSite(FakeSite):
def __init__(self, config: Dict[str, Any], configuration_filename="conf.py"):
super(MyFakeSite, self).__init__()
self.configured = True
self.debug = True
self.THEMES = []
Expand Down Expand Up @@ -70,13 +70,13 @@ def test_serves_root_dir(
test_was_successful = False
test_problem_description = "Async test setup apparently broken"
test_inner_error: Optional[BaseException] = None
loop_for_this_test = None
loop = None

async def grab_loop_and_run_test() -> None:
nonlocal test_problem_description, loop_for_this_test
nonlocal test_problem_description, loop

loop_for_this_test = asyncio.get_running_loop()
watchdog_handle = loop_for_this_test.call_later(TEST_MAX_DURATION, lambda: loop_for_this_test.stop())
loop = asyncio.get_running_loop()
watchdog_handle = loop.call_later(TEST_MAX_DURATION, loop.stop)
test_problem_description = f"Test did not complete within {TEST_MAX_DURATION} seconds."

def run_test() -> None:
Expand Down Expand Up @@ -113,13 +113,11 @@ def run_test() -> None:
LOGGER.info("Test completed successfully.")
else:
LOGGER.error("Test failed: %s", test_problem_description)
loop_for_this_test.call_soon_threadsafe(lambda: watchdog_handle.cancel())
loop.call_soon_threadsafe(watchdog_handle.cancel)
# Simulate Ctrl+C:
loop.call_soon_threadsafe(lambda: loop.call_later(0.01, loop.stop))

# We give the outer grab_loop_and_run_test a chance to complete
# before burning the bridge:
loop_for_this_test.call_soon_threadsafe(lambda: loop_for_this_test.call_later(0.05, lambda: loop_for_this_test.stop()))

await loop_for_this_test.run_in_executor(None, run_test)
await loop.run_in_executor(None, run_test)

# We defeat the nikola site building functionality, so this does not actually get called.
# But the code setting up site building wants a command list:
Expand All @@ -128,40 +126,14 @@ def run_test() -> None:
# Defeat the site building functionality, and instead insert the test:
command_auto.run_initial_rebuild = grab_loop_and_run_test

try:
# Start the development server
# which under the hood runs our test when trying to build the site:
command_auto.execute(options=options)

# Verify the test succeeded:
if test_inner_error is not None:
raise test_inner_error
assert test_was_successful, test_problem_description
finally:
# Nikola is written with the assumption that it can
# create the event loop at will without ever cleaning it up.
# As this tests runs several times in succession,
# that assumption becomes a problem.
LOGGER.info("Cleaning up loop.")
# Loop cleanup:
assert loop_for_this_test is not None
assert not loop_for_this_test.is_running()
loop_for_this_test.close()
asyncio.set_event_loop(None)
# We would like to leave it at that,
# but doing so causes the next test to fail.
#
# We did not find asyncio - API to reset the loop
# to "back to square one, as if just freshly started".
#
# The following code does not feel right, it's a kludge,
# but it apparently works for now:
if sys.platform == 'win32':
# For this case, the auto module has special code
# (at module load time! 😟) which we reluctantly reproduce here:
asyncio.set_event_loop(asyncio.ProactorEventLoop())
else:
asyncio.set_event_loop(asyncio.new_event_loop())
# Start the development server
# which under the hood runs our test when trying to build the site:
command_auto.execute(options=options)

# Verify the test succeeded:
if test_inner_error is not None:
raise test_inner_error
assert test_was_successful, test_problem_description


@pytest.fixture(scope="module",
Expand All @@ -184,7 +156,7 @@ def site_and_base_path(request) -> Tuple[MyFakeSite, str]:
"SITE_URL": request.param,
"OUTPUT_FOLDER": OUTPUT_FOLDER.as_posix(),
}
return (MyFakeSite(config), auto.base_path_from_siteuri(request.param))
return MyFakeSite(config), auto.base_path_from_siteuri(request.param)


@pytest.fixture(scope="module")
Expand Down