diff --git a/extension/src/background.ts b/extension/src/background.ts index 682809b..7c1c0de 100644 --- a/extension/src/background.ts +++ b/extension/src/background.ts @@ -69,7 +69,7 @@ async function makeCaptureRequest( } catch (err) { console.error(err) // fetch only throws when host is unavailable - throw new Error(`${endpoint} unavailable, check if server is running`) + throw new Error(`${endpoint} unavailable, check if server is running: ${(err as Error).toString()}`) } if (!response.ok) { // http error diff --git a/extension/src/options_page.ts b/extension/src/options_page.ts index f84c7b9..0c86383 100644 --- a/extension/src/options_page.ts +++ b/extension/src/options_page.ts @@ -61,8 +61,18 @@ async function saveOptions() { // TODO could also check for permissions and display message? const endpoint = getEndpoint().value - await ensurePermissions(endpoint) - refreshPermissionValidation(endpoint) + try { + // note: this might fail if we messed up manifest for instance, and didn't add optional hosts + const got = await ensurePermissions(endpoint) + if (!got) { + throw Error("Permissions weren't granted!") + } + } catch (error) { + // todo show notification instead? + alert(`${(error as Error).toString()}`) + throw error + } + refreshPermissionValidation(endpoint) // TODO need to call it after every typed character const opts = { endpoint: endpoint, diff --git a/extension/webpack.config.js b/extension/webpack.config.js index fcc2743..043e390 100644 --- a/extension/webpack.config.js +++ b/extension/webpack.config.js @@ -45,8 +45,8 @@ const commands = { "capture-simple": { "description": "Quick capture: url, title and selection", "suggested_key": { - "default": `Ctrl+${modifier}+C`, - "mac": `Command+${modifier}+C`, + "default": `Ctrl+${modifier}+H`, + "mac": `Command+${modifier}+H`, }, } } @@ -147,20 +147,27 @@ const manifestExtra = { manifestExtra[action_name] = action if (v3) { - manifestExtra['host_permissions'] = host_permissions + if (target === T.FIREFOX) { + // firefox doesn't support optional host permissions + // note that these will still have to be granted by user (unlike in chrome) + manifestExtra['host_permissions'] = [...host_permissions, ...optional_host_permissions] + } else { + manifestExtra['host_permissions'] = host_permissions + manifestExtra['optional_host_permissions'] = optional_host_permissions + } +} else { + manifestExtra.permissions.push(...host_permissions) + manifestExtra.optional_permissions.push(...optional_host_permissions) +} - // FIXME not sure if firefox supports this?? - // https://bugzilla.mozilla.org/show_bug.cgi?id=1766026 - manifestExtra['optional_host_permissions'] = optional_host_permissions +if (v3) { + // this isn't really required in chrome, but without it, webext lint fails for chrome addon const gecko_id = target === T.FIREFOX ? ext_id : "{00000000-0000-0000-0000-000000000000}" manifestExtra['browser_specific_settings'] = { "gecko": { "id": gecko_id, }, } -} else { - manifestExtra.permissions.push(...host_permissions) - manifestExtra.optional_permissions.push(...optional_host_permissions) } diff --git a/server/test_with_browser.py b/server/test_with_browser.py deleted file mode 100755 index 06822f5..0000000 --- a/server/test_with_browser.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin env python3 -from pathlib import Path -from time import sleep -from contextlib import contextmanager -from tempfile import TemporaryDirectory -import re -import json -import os - -import pytest -from selenium import webdriver # type: ignore - - -from test_grasp import grasp_test_server - - -addon_path = (Path(__file__).parent.parent / 'dist').absolute() - - -def skip_if_ci(reason): - return pytest.mark.skipif('CI' in os.environ, reason=reason) - - -@contextmanager -def get_webdriver(): - with TemporaryDirectory() as td: - profile = webdriver.FirefoxProfile(td) - # use firefox from here to test https://www.mozilla.org/en-GB/firefox/developer/ - driver = webdriver.Firefox(profile, firefox_binary='/L/soft/firefox-dev/firefox/firefox') - try: - driver.install_addon(str(addon_path), temporary=True) - yield driver - finally: - # confirm('ready to continue?') - driver.close() - - -def open_options_page(driver): - # necessary to trigger prefs.js initialisation.. - driver.get('http://example.com') - sleep(1) - - moz_profile = Path(driver.capabilities['moz:profile']) - prefs_file = moz_profile / 'prefs.js' - addon_id = None - for line in prefs_file.read_text().splitlines(): - # temporary-addon\":\"53104c22-acd0-4d44-904c-22d11d31559a\"}") - m = re.search(r'temporary-addon.....([0-9a-z-]+)."', line) - if m is None: - continue - addon_id = m.group(1) - assert addon_id is not None - - options = f'moz-extension://{addon_id}/options.html' - driver.get(options) - -# TODO could also check for errors - -def change_port(driver, port: str): - open_options_page(driver) - - ep = driver.find_element_by_id('endpoint_id') - ep.clear() - ep.send_keys(f'http://localhost:{port}') - - se = driver.find_element_by_id('save_id') - se.click() - - driver.switch_to.alert.accept() - - -def trigger_grasp(): - # soo... triggering extension under automation turns out to be notoriously complicated - - # I tried webdriver.ActionChains, they work e.g. with TAB keys, but don't seem to trigger the extension - - # this https://www.blazemeter.com/blog/6-easy-steps-testing-your-chrome-extension-selenium/ is interesting, but isn't applicabe to my extension - # might be a good way of messing with settings though - - # so the only remaining choice seems to be GUI interaction as described here https://stackoverflow.com/a/56452597/706389 - - # TODO force focusing window? - - # err. that didn't work... - # pyautogui.locateOnScreen('/L/soft/browser-extensions/grasp/unicorn.png') - - print("sending hotkey!") - import pyautogui - pyautogui.hotkey('ctrl', 'alt', 'c') - - -def confirm(what: str): - # pylint: disable=import-error - import click # type: ignore - click.confirm(what, abort=True) - - -@skip_if_ci('no GUI to run the browser..') -def test(tmp_path: Path): - capture_file = tmp_path / 'output.org' - port = '17890' - template = '* [[%:link][%:description]]' - - with grasp_test_server(capture_file=capture_file, template=template, port=port), get_webdriver() as driver: - change_port(driver, port=port) - - driver.get('https://en.wikipedia.org/wiki/Automation') - - sleep(2) # just in case.. - # TODO select something? - trigger_grasp() - confirm('ready to continue?') - - sleep(1) # just in case - - # TODO come up with a better test and involve other template parameters - assert capture_file.read_text() == '* [[https://en.wikipedia.org/wiki/Automation][Automation - Wikipedia]]' - - -def main(): - with TemporaryDirectory() as tdir: - test(Path(tdir)) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index ae47575..206205b 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,15 @@ def main() -> None: ], extras_require={ 'testing': ['pytest', 'requests'], - 'linting': ['pytest', 'mypy', 'lxml', 'ruff'], # lxml for mypy coverage report + 'linting': [ + 'pytest', + 'mypy', 'lxml', # lxml for mypy coverage report + 'ruff', + + 'selenium', + 'click', + 'loguru', + ], }, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/addon_helper.py b/tests/addon_helper.py new file mode 100644 index 0000000..82f0e1c --- /dev/null +++ b/tests/addon_helper.py @@ -0,0 +1,119 @@ +from dataclasses import dataclass +from functools import cached_property +import json +from pathlib import Path +import subprocess +import re + + +import psutil +from selenium import webdriver + + +@dataclass +class AddonHelper: + driver: webdriver.Remote + + @cached_property + def addon_id(self) -> str: + return get_addon_id(driver=self.driver) + + @property + def extension_prefix(self) -> str: + protocol = { + 'chrome': 'chrome-extension', + 'firefox': 'moz-extension', + }[self.driver.name] + return f'{protocol}://{self.addon_id}' + + def open_page(self, path: str) -> None: + self.driver.get(self.extension_prefix + '/' + path) + + +# NOTE looks like it used to be posssible in webdriver api? +# at least as of 2011 https://github.com/gavinp/chromium/blob/681563ea0f892a051f4ef3d5e53438e0bb7d2261/chrome/test/webdriver/test/chromedriver.py#L35-L40 +# but here https://github.com/SeleniumHQ/selenium/blob/master/cpp/webdriver-server/command_types.h there are no Extension commands +# also see driver.command_executor._commands +def _get_chrome_addon_id(driver: webdriver.Remote) -> str: + """ + For a temporary addon extension id is autogenerated, so we need to extract it every time + """ + user_data_dir = Path(driver.capabilities['chrome']['userDataDir']) + prefs_file = user_data_dir / 'Default/Preferences' + assert prefs_file.exists(), prefs_file + + # for some idiotic reason, after chrome launches, extension settings aren't immediately available + # this can take up to 30 secons in this loop until they are populated + while True: + prefs = json.loads(prefs_file.read_text()) + extension_settings = prefs.get('extensions', {}).get('settings', None) + if extension_settings is not None: + # there are some other weird builtin extension as well + # this seems like the easiest way to filter them out extracing by extension name or path + [addon_id] = [k for k, v in extension_settings.items() if v['creation_flags'] != 1] + return addon_id + + +def _get_firefox_addon_id(driver: webdriver.Remote) -> str: + moz_profile = Path(driver.capabilities['moz:profile']) + prefs_file = moz_profile / 'prefs.js' + assert prefs_file.exists(), prefs_file + + while True: + for line in prefs_file.read_text().splitlines(): + m = re.fullmatch(r'user_pref\("extensions.webextensions.uuids", "(.*)"\);', line) + if m is None: + continue + # this contains a json with escaped quotes + user_prefs_s = m.group(1).replace('\\', '') + user_prefs = json.loads(user_prefs_s) + addon_ids = [v for k, v in user_prefs.items() if 'mozilla.' not in k] + if len(addon_ids) == 0: + # for some stupid reason it doesn't appear immediately in the file + continue + [addon_id] = addon_ids + return addon_id + + +def get_addon_id(driver: webdriver.Remote) -> str: + extractor = { + 'firefox': _get_firefox_addon_id, + 'chrome': _get_chrome_addon_id, + }[driver.name] + return extractor(driver) + + +def get_window_id(driver: webdriver.Remote) -> str: + if driver.name == 'firefox': + pid = str(driver.capabilities['moz:processID']) + elif driver.name == 'chrome': + # ugh no pid in capabilities... + driver_pid = driver.service.process.pid # type: ignore[attr-defined] + process = psutil.Process(driver_pid) + [chrome_process] = process.children() + cmdline = chrome_process.cmdline() + assert '--enable-automation' in cmdline, cmdline + pid = str(chrome_process.pid) + else: + raise RuntimeError(driver.name) + return get_wid_by_pid(pid) + + +def get_wid_by_pid(pid: str) -> str: + # https://askubuntu.com/a/385037/427470 + wids = subprocess.check_output(['xdotool', 'search', '--pid', pid]).decode('utf8').splitlines() + wids = [w.strip() for w in wids if len(w.strip()) > 0] + + def has_wm_desktop(wid: str) -> bool: + # TODO no idea why is that important. found out experimentally + out = subprocess.check_output(['xprop', '-id', wid, '_NET_WM_DESKTOP']).decode('utf8') + return 'not found' not in out + + [wid] = filter(has_wm_desktop, wids) + return wid + + +def focus_browser_window(driver: webdriver.Remote) -> None: + # FIXME assert not is_headless(driver) # just in case + wid = get_window_id(driver) + subprocess.check_call(['xdotool', 'windowactivate', '--sync', wid]) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1ab6ca1 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,14 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + '--browser', + choices=['firefox', 'chrome'], + default='firefox', # FIXME add --all or something + ) + + +@pytest.fixture +def browser(request) -> str: + return request.config.getoption("--browser") diff --git a/tests/test_end2end.py b/tests/test_end2end.py new file mode 100644 index 0000000..64cff53 --- /dev/null +++ b/tests/test_end2end.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +import re +import socket +import sys +import time +from typing import Iterator + +import click +from loguru import logger +import psutil +import pytest +from selenium import webdriver + +from .addon_helper import AddonHelper, focus_browser_window +from . import paths # type: ignore[attr-defined] + + +@dataclass +class OptionsPage: + # I suppose it's inevitable it's at least somewhat driver aware? since we want it to locate elements etc + helper: AddonHelper + + def open(self) -> None: + # TODO extract from manifest -> options_id -> options.html + # seems like addon just links to the actual manifest on filesystem, so will need to read from that + page_name = 'options.html' + self.helper.open_page(page_name) + + def change_endpoint(self, endpoint: str, *, wait_for_permissions: bool = False) -> None: + driver = self.helper.driver + + current_url = driver.current_url + assert current_url.endswith('options.html'), current_url # just in case + + ep = driver.find_element('id', 'endpoint_id') + while ep.get_attribute('value') == '': + # data is set asynchronously, so need to wait for data to appear + # TODO is there some webdriver wait? + time.sleep(0.001) + ep.clear() + ep.send_keys(endpoint) + + se = driver.find_element('id', 'save_id') + se.click() + + if wait_for_permissions: + # we can't accept this alert via webdriver, it's a native chrome alert, not DOM + confirm('You should see prompt for permissions. Accept them') + + alert = driver.switch_to.alert + assert alert.text == 'Saved!', alert.text # just in case + alert.accept() + + +@dataclass +class Popup: + helper: AddonHelper + + def open(self) -> None: + # I suppose triggeing via hotkey is bound to be cursed? + # maybe replace with selenium_bridge thing... or ydotool? + # although to type into in, I'll need pyautogui anyway... + import pyautogui + + modifier = {'firefox': 'alt', 'chrome': 'shift'}[self.helper.driver.name] + + focus_browser_window(self.helper.driver) + + pyautogui.hotkey('ctrl', modifier, 'y') + time.sleep(2) # TODO not sure if can do better? + + def enter_data(self, *, comment: str, tags: str) -> None: + import pyautogui + + pyautogui.write(comment) + + pyautogui.hotkey('tab') # switch to tags + + # erase default, without interval doesn't remove everything + for _ in range(10): + pyautogui.hotkey('backspace') + # pyautogui.hotkey(['backspace' for i in range(10)], interval=0.05) + pyautogui.write(tags) + + # FIXME ffs. focus in the popup isn't working when it's running via webdriver?? + # tried both regular and dev edition firefox with latest geckodriver + + def submit(self) -> None: + import pyautogui + + pyautogui.hotkey('shift', 'enter') + + +@dataclass +class Addon: + helper: AddonHelper + + def quick_capture(self) -> None: + import pyautogui + + modifier = {'firefox': 'alt', 'chrome': 'shift'}[self.helper.driver.name] + + focus_browser_window(self.helper.driver) + + pyautogui.hotkey('ctrl', modifier, 'h') + + @property + def options_page(self) -> OptionsPage: + return OptionsPage(helper=self.helper) + + @property + def popup(self) -> Popup: + return Popup(helper=self.helper) + + +@dataclass +class Server: + port: str + capture_file: Path + + +@contextmanager +def tmp_popen(*args, **kwargs): + with psutil.Popen(*args, **kwargs) as p: + try: + yield p + finally: + for c in p.children(recursive=True): + c.kill() + p.kill() + p.wait() + + +@pytest.fixture +def server(tmp_path: Path, grasp_port: str) -> Iterator[Server]: + capture_file = tmp_path / 'capture.org' + + server_bin = Path(__file__).absolute().parent.parent / 'server/grasp_server.py' + assert server_bin.exists(), server_bin + + cmdline = [sys.executable, server_bin, '--port', grasp_port, '--path', capture_file] + logger.debug(f'running {cmdline}') + with tmp_popen(cmdline): + # todo wait till it's ready? + yield Server(port=grasp_port, capture_file=capture_file) + + +# TODO move to addon_helper?? not sure +@pytest.fixture +def addon(tmp_path: Path, browser: str) -> Iterator[Addon]: + addon_path = Path('extension/dist').absolute() / browser + assert (addon_path / 'manifest.json').exists() + + driver: webdriver.Remote + # FIXME headless (there is some code in promnesia) + if browser == 'firefox': + ff_options = webdriver.FirefoxOptions() + # NOTE: using dev edition firefox + ff_options.binary_location = paths.firefox + + # TODO not sure what's the difference with install_addon? + # seems like this one is depecated + # https://github.com/SeleniumHQ/selenium/blob/ba27d0f7675a3d8139544e5522b8f0690a2ba4ce/py/selenium/webdriver/firefox/firefox_profile.py#L82 + # ff_profile = webdriver.FirefoxProfile() + # ff_profile.add_extension(str(addon_path)) + # ff_options.profile = ff_profile + + # WTF?? it's autodownloaded by selenium??? + # .local/lib/python3.10/site-packages/selenium/webdriver/common/selenium_manager.py + # service = webdriver.FirefoxService(executable_path=paths.geckodriver) + + # todo use tmp_path for profile path? + driver = webdriver.Firefox(options=ff_options) + elif browser == 'chrome': + # todo name it chromium? + # NOTE: something like this https://storage.googleapis.com/chrome-for-testing-public/123.0.6312.122/linux64/chrome-linux64.zip + cr_options = webdriver.ChromeOptions() + # shit. ok, this seems to work... + cr_options.binary_location = paths.chrome + cr_options.add_argument('--load-extension=' + str(addon_path)) + cr_options.add_argument('--user-data-dir=' + str(tmp_path)) + # NOTE: there is also .add_extension, but it seems to require a packed extension (zip or crx?) + + service = webdriver.ChromeService(executable_path=paths.chromedriver) + driver = webdriver.Chrome(service=service, options=cr_options) + else: + raise RuntimeError(browser) + + with driver: + if browser == 'firefox': + # todo log driver.capabilities['moz:geckodriverVersion'] and 'browserVersion' + # TODO doesn't work with regular Firefox? says addon must be signed + # FIXME crap, it seems that the addon is installed as an xpi from a temporary location? + # so if we modify code we have to rerun the test + assert isinstance(driver, webdriver.Firefox), driver + driver.install_addon(str(addon_path), temporary=True) + elif browser == 'chrome': + pass + else: + raise RuntimeError(browser) + + helper = AddonHelper(driver=driver) + yield Addon(helper=helper) + + +# TODO adapt for multiple params +def myparametrize(param: str, values): + """ + by default pytest isn't showing param names in the test name which is annoying + """ + return pytest.mark.parametrize(param, values, ids=[f'{param}={v}' for v in values]) + + +def confirm(what: str) -> None: + click.confirm(click.style(what, blink=True, fg='yellow'), abort=True) + + +# chrome v3 works +# firefox v2 works +# firefox v3 BROKEN -- needs the user to approve the localhost access FIXME -- request permission at capture time? +def test_capture_no_configuration(addon: Addon) -> None: + """ + This checks that capture works with default hostname/port without opening settings first + """ + # it's kinda tricky -- grasp on my system is already running on default 12212 port + # so if it's already running, not sure what to do since we'll have to somehow detect it and extract data from it + # (probably a bad idea to try to access it from the test either) + + # TODO can we just request Driver object directly? + driver = addon.helper.driver + + driver.get('https://example.com') + + addon.quick_capture() + + confirm('Should show a successful capture notification, and the link should be in your default capture file') + + +# chrome v3 works +# firefox v2 works +# firefox v3 BROKEN (same issue as above) +def test_capture_bad_port(addon: Addon) -> None: + """ + Check that we get error notification instead of silently failing if the endpoint is wrong + """ + driver = addon.helper.driver + + addon.options_page.open() + addon.options_page.change_endpoint(endpoint='http://localhost:12345/capture') + + driver.get('https://example.com') + + addon.quick_capture() + + confirm("Should show a notification with error that the endpoint isn't available") + + +# chrome v3 works +# firefox v2 works +# firefox v3 works +@myparametrize("grasp_port", ["17890"]) +def test_capture_custom_endpoint(addon: Addon, server: Server) -> None: + driver = addon.helper.driver + + addon.options_page.open() + # hack to make chrome think we changed the endpoint + # (it'll be actual host name instead of localhost) + hostname = socket.gethostname() + addon.options_page.change_endpoint( + endpoint=f'http://{hostname}:{server.port}/capture', + wait_for_permissions=True, + ) + + driver.get('https://example.com') + + assert not server.capture_file.exists() # just in case + + addon.quick_capture() + time.sleep(1) # just to give it time to actually capture + + assert 'https://example.com' in server.capture_file.read_text() + + +# chrome v3 works +# firefox v3 BROKEN (doesn't focus in popup, must be geckodriver issue) +# UPD -- sometimes is working?? wtf... +# firefox v2 BROKEN (same as above) +@myparametrize("grasp_port", ["17890"]) +def test_capture_with_extra_data(addon: Addon, server: Server) -> None: + driver = addon.helper.driver + + addon.options_page.open() + addon.options_page.change_endpoint(endpoint=f'http://localhost:{server.port}/capture') + + driver.get('https://example.com') + + ## select some text (fuck my life) + driver.execute_script( + ''' + var h1 = document.querySelector('h1') + var node = h1.firstChild + var range = document.createRange() + range.setStart(node, 5) + range.setEnd(node, 5 + 7) + + var sel = window.getSelection() + sel.removeAllRanges() + sel.addRange(range) + ''' + ) + ## + + assert not server.capture_file.exists() # just in case + + popup = addon.popup + popup.open() + popup.enter_data(comment='some multiline\nnote', tags='tag2 tag1') + popup.submit() + + captured = server.capture_file.read_text() + captured = re.sub(r'\[.*?\]', '[date]', captured) # dates are volatile, can't test against them + + assert captured == '''\ +* [date] Example Domain :tag2:tag1: +https://example.com/ +Selection: +le Doma +Comment: +some multiline +note +''' diff --git a/tox.ini b/tox.ini index 42ccd30..c135467 100644 --- a/tox.ini +++ b/tox.ini @@ -26,7 +26,7 @@ passenv = [testenv:ruff] commands = {envpython} -m pip install --use-pep517 -e .[linting] - {envpython} -m ruff check server/ + {envpython} -m ruff check server/ tests/ [testenv:tests] @@ -34,7 +34,7 @@ commands = {envpython} -m pip install --use-pep517 -e .[testing] # posargs allow test filtering, e.g. tox ... -- -k test_name {envpython} -m pytest \ - 'server' --ignore 'server/test_with_browser.py' \ + 'server' \ {posargs} @@ -47,3 +47,6 @@ commands = --txt-report .coverage.mypy \ --html-report .coverage.mypy \ {posargs} + {envpython} -m mypy --install-types --non-interactive \ + -p tests \ + {posargs}