From 1094d35ccaa8e8c30d50e5bb6b8d5d9ae4ac89a1 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 17 Nov 2022 17:32:57 +0100 Subject: [PATCH 01/14] WIP --- trafficlight/internals/client.py | 5 +++++ trafficlight/tests/fallback_key_test.py | 28 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 trafficlight/tests/fallback_key_test.py diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index a4538fd..f9a5534 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -240,3 +240,8 @@ async def advance_clock(self, duration: int) -> None: await self._perform_action( {"action": "advance_clock", "data": {"milliseconds": duration}} ) + + async def go_offline(self) -> None: + await self._perform_action( + {"action": "go_offline", "data": {}} + ) \ No newline at end of file diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py new file mode 100644 index 0000000..b223ece --- /dev/null +++ b/trafficlight/tests/fallback_key_test.py @@ -0,0 +1,28 @@ +import asyncio + +from trafficlight.client_types import ElementWeb +from trafficlight.homerunner import HomeServer +from trafficlight.internals.client import MatrixClient +from trafficlight.internals.test import Test +from trafficlight.server_types import Synapse + + +class FallbackKeyTest(Test): + def __init__(self) -> None: + super().__init__() + self._client_under_test([ElementWeb()], "alice") + self._client_under_test([ElementWeb()], "bob") + self._server_under_test(Synapse(), ["server"]) + + async def run( + self, alice: MatrixClient, bob: MatrixClient, server: HomeServer + ) -> None: + await asyncio.gather( + alice.register(server), bob.register(second_server) + ) + await alice.create_room("fallback test room") + await alice.invite_user( + bob.localpart + ":" + second_server.server_name + ) + await bob.accept_invite() + await alice.go_offline() From 38c9df40aee71f5569635334ace4a2fc0dacc3ac Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 24 Nov 2022 15:14:44 +0100 Subject: [PATCH 02/14] WIP --- trafficlight/tests/fallback_key_test.py | 32 +++++++++++++++++++------ 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index b223ece..0e183ce 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -1,8 +1,9 @@ import asyncio +from nio import AsyncClient from trafficlight.client_types import ElementWeb from trafficlight.homerunner import HomeServer -from trafficlight.internals.client import MatrixClient +from trafficlight.internals.client import MatrixClient, NetworkProxyClient from trafficlight.internals.test import Test from trafficlight.server_types import Synapse @@ -12,17 +13,34 @@ def __init__(self) -> None: super().__init__() self._client_under_test([ElementWeb()], "alice") self._client_under_test([ElementWeb()], "bob") + self._network_proxy("alice_proxy") self._server_under_test(Synapse(), ["server"]) async def run( - self, alice: MatrixClient, bob: MatrixClient, server: HomeServer + self, alice: MatrixClient, bob: MatrixClient, server: HomeServer, alice_proxy: NetworkProxyClient ) -> None: + await alice_proxy.proxy_to(server) await asyncio.gather( - alice.register(server), bob.register(second_server) + alice.register(alice_proxy), bob.register(server) ) await alice.create_room("fallback test room") - await alice.invite_user( - bob.localpart + ":" + second_server.server_name - ) + await alice.invite_user(f"{bob.localpart}:{server.server_name}") await bob.accept_invite() - await alice.go_offline() + # disable sync for alice, so she can't upload more device keys + await alice_proxy.disable_endpoint("/_matrix/client/r0/sync") + + # for 1...60 + bob_client_n = AsyncClient( + server.cs_api, f"@{bob.localpart}:{server.server_name}" + ) + login_resp = await bob_client_n.login(bob.password) + + # check that we logged in succesfully + if isinstance(login_resp, LoginResponse): + + else: + print(f'homeserver = "{homeserver}"; user = "{user_id}"') + print(f"Failed to log in: {login_resp}") + sys.exit(1) + + await alice_proxy.enable_endpoint("/_matrix/client/r0/sync") From 74563762db8055a72a9fdfacbf4d6676afc809c6 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 24 Nov 2022 16:47:21 +0100 Subject: [PATCH 03/14] WIP2 --- trafficlight/tests/fallback_key_test.py | 37 +++++++++++++++---------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index 0e183ce..9ce1010 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -23,24 +23,31 @@ async def run( await asyncio.gather( alice.register(alice_proxy), bob.register(server) ) - await alice.create_room("fallback test room") + # TODO: does alice need to enable fallback keys? + room_id = await alice.create_room("fallback test room") await alice.invite_user(f"{bob.localpart}:{server.server_name}") await bob.accept_invite() # disable sync for alice, so she can't upload more device keys await alice_proxy.disable_endpoint("/_matrix/client/r0/sync") + for i in range(1, 61): + await login_and_send_message_in_room(server, bob, room_id, f"Hello world {i}!") + await alice_proxy.enable_endpoint("/_matrix/client/r0/sync") + # last message would have exhausted the OTKs, + # so we should have fallen back to the fallback key + await alice.verify_message_in_timeline("Hello world 60!") - # for 1...60 - bob_client_n = AsyncClient( - server.cs_api, f"@{bob.localpart}:{server.server_name}" +async def login_and_send_message_in_room(server: HomeServer, user: MatrixClient, room_id: str, message: str) -> None: + # TODO: will this encrypt (and hence track the room and claim keys)?? + # need to install python-olm and pip install "matrix-nio[e2e] + client = AsyncClient( + server.cs_api, f"@{user.localpart}:{server.server_name}" + ) + try: + await client.login(user.password) + await client.room_send( + room_id, + message_type="m.room.message", + content={"msgtype": "m.text", "body": message}, ) - login_resp = await bob_client_n.login(bob.password) - - # check that we logged in succesfully - if isinstance(login_resp, LoginResponse): - - else: - print(f'homeserver = "{homeserver}"; user = "{user_id}"') - print(f"Failed to log in: {login_resp}") - sys.exit(1) - - await alice_proxy.enable_endpoint("/_matrix/client/r0/sync") + finally: + await client.close() From c232675f194aedfa7d9df26e83e5d22e96a4a9ff Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Thu, 24 Nov 2022 17:35:08 +0100 Subject: [PATCH 04/14] list dependencies --- setup.cfg | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup.cfg b/setup.cfg index fbb3e3e..24b7597 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,9 @@ install_requires = attrs aiohttp quart + nio + python-olm + matrix-nio[e2e] [options.package_data] trafficlight = py.typed From cac449f43095c0a0ff54fc9a905ee99e8d674881 Mon Sep 17 00:00:00 2001 From: Bruno Windels <274386+bwindels@users.noreply.github.com> Date: Mon, 28 Nov 2022 14:34:49 +0100 Subject: [PATCH 05/14] WIP --- trafficlight/tests/fallback_key_test.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index 9ce1010..25210b5 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -1,6 +1,7 @@ import asyncio -from nio import AsyncClient +import logging +from nio import AsyncClient, AsyncClientConfig, SyncResponse from trafficlight.client_types import ElementWeb from trafficlight.homerunner import HomeServer from trafficlight.internals.client import MatrixClient, NetworkProxyClient @@ -39,13 +40,27 @@ async def run( async def login_and_send_message_in_room(server: HomeServer, user: MatrixClient, room_id: str, message: str) -> None: # TODO: will this encrypt (and hence track the room and claim keys)?? # need to install python-olm and pip install "matrix-nio[e2e] + logging.info(f"trying to login as @{user.localpart}:{server.server_name} with password ${user.password} and send a message in #{room_id}...") client = AsyncClient( - server.cs_api, f"@{user.localpart}:{server.server_name}" + server.cs_api, + f"@{user.localpart}:{server.server_name}", + config=AsyncClientConfig(encryption_enabled=True) ) try: await client.login(user.password) + loop = asyncio.get_running_loop() + first_room_future = loop.create_future() + async def sync_cb(response): + rooms = client.rooms.values() + room = rooms[0] + if room: + first_room_future.set_result(room) + client.add_response_callback(sync_cb, SyncResponse) + client.sync_forever(30000) + room = await first_room_future + logging.info("got room with id {room.id} {room.room_id}") await client.room_send( - room_id, + room.room_id, message_type="m.room.message", content={"msgtype": "m.text", "body": message}, ) From 9df7e654a923d386a3d1bdc8ae84ff04e9da5b4c Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Tue, 13 Dec 2022 15:02:13 +0530 Subject: [PATCH 06/14] Incorporate changes from main --- trafficlight/tests/fallback_key_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index 25210b5..b53b7c7 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -2,20 +2,20 @@ import logging from nio import AsyncClient, AsyncClientConfig, SyncResponse -from trafficlight.client_types import ElementWeb +from trafficlight.client_types import ElementWebStable from trafficlight.homerunner import HomeServer from trafficlight.internals.client import MatrixClient, NetworkProxyClient from trafficlight.internals.test import Test -from trafficlight.server_types import Synapse +from trafficlight.server_types import SynapseDevelop class FallbackKeyTest(Test): def __init__(self) -> None: super().__init__() - self._client_under_test([ElementWeb()], "alice") - self._client_under_test([ElementWeb()], "bob") + self._client_under_test([ElementWebStable()], "alice") + self._client_under_test([ElementWebStable()], "bob") self._network_proxy("alice_proxy") - self._server_under_test(Synapse(), ["server"]) + self._server_under_test(SynapseDevelop(), ["server"]) async def run( self, alice: MatrixClient, bob: MatrixClient, server: HomeServer, alice_proxy: NetworkProxyClient From 8c404ee5d61b34e43b9ad4e53551a1b66e0f4650 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Wed, 14 Dec 2022 22:42:17 +0530 Subject: [PATCH 07/14] Make test pass --- trafficlight/internals/client.py | 7 +-- trafficlight/tests/fallback_key_test.py | 74 +++++++++++++++---------- 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index dc5ca90..bbf6982 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -175,9 +175,10 @@ async def verify_crosssign(self) -> None: await self._perform_action({"action": "verify_crosssign_emoji", "data": {}}) async def create_room(self, room_name: str) -> None: - await self._perform_action( + response = await self._perform_action( {"action": "create_room", "data": {"name": room_name}} ) + return response["response"] async def create_dm(self, user_id: str) -> None: await self._perform_action({"action": "create_dm", "data": {"userId": user_id}}) @@ -257,6 +258,4 @@ async def advance_clock(self, duration: int) -> None: ) async def go_offline(self) -> None: - await self._perform_action( - {"action": "go_offline", "data": {}} - ) \ No newline at end of file + await self._perform_action({"action": "go_offline", "data": {}}) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index b53b7c7..ba4c6ff 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -1,14 +1,16 @@ import asyncio - import logging -from nio import AsyncClient, AsyncClientConfig, SyncResponse + +from nio import AsyncClient, AsyncClientConfig, SyncResponse, store + from trafficlight.client_types import ElementWebStable from trafficlight.homerunner import HomeServer from trafficlight.internals.client import MatrixClient, NetworkProxyClient from trafficlight.internals.test import Test from trafficlight.server_types import SynapseDevelop - +# CLIENT_COUNT=2 REQUIRES_PROXY=true CYPRESS_BASE_URL="https://app.element.io" ./trafficlight/scripts-dev/run-localdev-setup.sh && tmux kill-server +# Passing Test class FallbackKeyTest(Test): def __init__(self) -> None: super().__init__() @@ -18,51 +20,65 @@ def __init__(self) -> None: self._server_under_test(SynapseDevelop(), ["server"]) async def run( - self, alice: MatrixClient, bob: MatrixClient, server: HomeServer, alice_proxy: NetworkProxyClient + self, + alice: MatrixClient, + bob: MatrixClient, + server: HomeServer, + alice_proxy: NetworkProxyClient, ) -> None: await alice_proxy.proxy_to(server) - await asyncio.gather( - alice.register(alice_proxy), bob.register(server) - ) - # TODO: does alice need to enable fallback keys? + await asyncio.gather(alice.register(alice_proxy), bob.register(server)) room_id = await alice.create_room("fallback test room") + logging.info(f"Got room-id as {room_id}") await alice.invite_user(f"{bob.localpart}:{server.server_name}") await bob.accept_invite() # disable sync for alice, so she can't upload more device keys await alice_proxy.disable_endpoint("/_matrix/client/r0/sync") - for i in range(1, 61): - await login_and_send_message_in_room(server, bob, room_id, f"Hello world {i}!") + for i in range(60): + await login_and_send_message_in_room( + server, bob, room_id, f"Hello world {i + 1}!" + ) await alice_proxy.enable_endpoint("/_matrix/client/r0/sync") # last message would have exhausted the OTKs, # so we should have fallen back to the fallback key await alice.verify_message_in_timeline("Hello world 60!") -async def login_and_send_message_in_room(server: HomeServer, user: MatrixClient, room_id: str, message: str) -> None: - # TODO: will this encrypt (and hence track the room and claim keys)?? + +async def login_and_send_message_in_room( + server: HomeServer, user: MatrixClient, room_id: str, message: str +) -> None: # need to install python-olm and pip install "matrix-nio[e2e] - logging.info(f"trying to login as @{user.localpart}:{server.server_name} with password ${user.password} and send a message in #{room_id}...") + logging.info( + f"trying to login as @{user.localpart}:{server.server_name} with password ${user.password} and send a message in #{room_id}..." + ) + user_id = f"@{user.localpart}:{server.server_name}" client = AsyncClient( server.cs_api, - f"@{user.localpart}:{server.server_name}", - config=AsyncClientConfig(encryption_enabled=True) + user_id, + config=AsyncClientConfig( + encryption_enabled=True, store=store.SqliteMemoryStore + ), ) - try: - await client.login(user.password) - loop = asyncio.get_running_loop() - first_room_future = loop.create_future() - async def sync_cb(response): - rooms = client.rooms.values() - room = rooms[0] - if room: - first_room_future.set_result(room) - client.add_response_callback(sync_cb, SyncResponse) - client.sync_forever(30000) - room = await first_room_future - logging.info("got room with id {room.id} {room.room_id}") + + # This method will be called after each sync + async def handle_sync(response): + # Send the message after initial sync await client.room_send( - room.room_id, + room_id, message_type="m.room.message", content={"msgtype": "m.text", "body": message}, + ignore_unverified_devices=True, ) + # Stop syncing + task.cancel() + + try: + client.add_response_callback(handle_sync, SyncResponse) + await client.login(user.password) + # sync_forever must be used for encryption to work + task = asyncio.create_task(client.sync_forever(timeout=30000)) + await task + except Exception as e: + logging.exception(str(e)) finally: await client.close() From 5807aa9679f8c2283114d7b4c25adce78bca11d0 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 12:10:22 +0530 Subject: [PATCH 08/14] Fix type annotation --- trafficlight/internals/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index bbf6982..a11b3b5 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -174,7 +174,7 @@ async def accept_crosssign(self) -> None: async def verify_crosssign(self) -> None: await self._perform_action({"action": "verify_crosssign_emoji", "data": {}}) - async def create_room(self, room_name: str) -> None: + async def create_room(self, room_name: str) -> str: response = await self._perform_action( {"action": "create_room", "data": {"name": room_name}} ) From 0c577e33fe115338f49218dfe26c001a9870a93b Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 12:11:47 +0530 Subject: [PATCH 09/14] Add return type to function --- trafficlight/tests/fallback_key_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index ba4c6ff..ce8aca5 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -9,6 +9,7 @@ from trafficlight.internals.test import Test from trafficlight.server_types import SynapseDevelop + # CLIENT_COUNT=2 REQUIRES_PROXY=true CYPRESS_BASE_URL="https://app.element.io" ./trafficlight/scripts-dev/run-localdev-setup.sh && tmux kill-server # Passing Test class FallbackKeyTest(Test): @@ -61,7 +62,7 @@ async def login_and_send_message_in_room( ) # This method will be called after each sync - async def handle_sync(response): + async def handle_sync(response) -> None: # Send the message after initial sync await client.room_send( room_id, From f693de8f6cb584ffda9151dc45c840fce3d3d905 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 12:54:55 +0530 Subject: [PATCH 10/14] Fix lint errors --- trafficlight/internals/client.py | 4 ++-- trafficlight/tests/fallback_key_test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index a11b3b5..347665c 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Union, cast from trafficlight.homerunner import HomeServer @@ -178,7 +178,7 @@ async def create_room(self, room_name: str) -> str: response = await self._perform_action( {"action": "create_room", "data": {"name": room_name}} ) - return response["response"] + return cast(str, response["response"]) async def create_dm(self, user_id: str) -> None: await self._perform_action({"action": "create_dm", "data": {"userId": user_id}}) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index ce8aca5..dbddb93 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -62,7 +62,7 @@ async def login_and_send_message_in_room( ) # This method will be called after each sync - async def handle_sync(response) -> None: + async def handle_sync(_) -> None: # Send the message after initial sync await client.room_send( room_id, From 5ee4e85ab55fa5af8ec5b8eafc408c90e8be3567 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 13:27:32 +0530 Subject: [PATCH 11/14] Use none type --- trafficlight/tests/fallback_key_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index dbddb93..ea1cb05 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -62,7 +62,7 @@ async def login_and_send_message_in_room( ) # This method will be called after each sync - async def handle_sync(_) -> None: + async def handle_sync(_: None) -> None: # Send the message after initial sync await client.room_send( room_id, From 023584922152866d21984cba273f748f84ef108d Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 17:30:45 +0530 Subject: [PATCH 12/14] Remove unused method --- trafficlight/internals/client.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index 347665c..75228ad 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -256,6 +256,3 @@ async def advance_clock(self, duration: int) -> None: await self._perform_action( {"action": "advance_clock", "data": {"milliseconds": duration}} ) - - async def go_offline(self) -> None: - await self._perform_action({"action": "go_offline", "data": {}}) From 009f5eff45f7746f3db8398fb43896b4b341cfaf Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 17:31:57 +0530 Subject: [PATCH 13/14] Let trafficlight catch exceptions --- trafficlight/tests/fallback_key_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/trafficlight/tests/fallback_key_test.py b/trafficlight/tests/fallback_key_test.py index ea1cb05..8ce9d46 100644 --- a/trafficlight/tests/fallback_key_test.py +++ b/trafficlight/tests/fallback_key_test.py @@ -79,7 +79,5 @@ async def handle_sync(_: None) -> None: # sync_forever must be used for encryption to work task = asyncio.create_task(client.sync_forever(timeout=30000)) await task - except Exception as e: - logging.exception(str(e)) finally: await client.close() From fa2d03828c82d6f64f17d1f3ed873c9c2eed7270 Mon Sep 17 00:00:00 2001 From: RMidhunSuresh Date: Thu, 15 Dec 2022 17:46:41 +0530 Subject: [PATCH 14/14] Access from object --- trafficlight/internals/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trafficlight/internals/client.py b/trafficlight/internals/client.py index 75228ad..5d19e8d 100644 --- a/trafficlight/internals/client.py +++ b/trafficlight/internals/client.py @@ -178,7 +178,7 @@ async def create_room(self, room_name: str) -> str: response = await self._perform_action( {"action": "create_room", "data": {"name": room_name}} ) - return cast(str, response["response"]) + return cast(str, response["roomId"]) async def create_dm(self, user_id: str) -> None: await self._perform_action({"action": "create_dm", "data": {"userId": user_id}})