From 67c74afdceaba819d53c84d9d3d4f53c99ca2be0 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 21 Jun 2023 18:08:26 +0200 Subject: [PATCH 01/18] initial version --- .../backup_clouds/onedrive/__init__.py | 0 .../backup_clouds/onedrive/backup_cloud.py | 92 +++++++++++++++++++ .../modules/backup_clouds/onedrive/config.py | 17 ++++ 3 files changed, 109 insertions(+) create mode 100644 packages/modules/backup_clouds/onedrive/__init__.py create mode 100644 packages/modules/backup_clouds/onedrive/backup_cloud.py create mode 100644 packages/modules/backup_clouds/onedrive/config.py diff --git a/packages/modules/backup_clouds/onedrive/__init__.py b/packages/modules/backup_clouds/onedrive/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py new file mode 100644 index 0000000000..4cfeeedebe --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +import logging +import msal +import atexit +import os +import json +from msdrive import OneDrive +from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration +from modules.common.abstract_device import DeviceDescriptor +from modules.common.configurable_backup_cloud import ConfigurableBackupCloud + +log = logging.getLogger(__name__) + +# Define the scope of access +scope = ["https://graph.microsoft.com/Files.ReadWrite"] + +# Define the authority and the token endpoint for MSA/Live accounts +authority = "https://login.microsoftonline.com/consumers/" +clientID = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2" + + +# to-do move into one or more functions +def get_tokens(persistent_tokencache: str) -> dict: + result = None + cache = msal.SerializableTokenCache() + + # to-do: add write to config after update + '''if os.path.exists("my_cache.bin"): # to do: read from config + cache.deserialize(open("my_cache.bin", "r").read()) + atexit.register(lambda: open("my_cache.bin", "w").write(cache.serialize()) # to-do: write to config + if cache.has_state_changed else None + ) +''' + if persistent_tokencache: + cache.deserialize(persistent_tokencache) + + # Create a public client application with msal + app = msal.PublicClientApplication(client_id=clientID, authority=authority, token_cache=cache) + + accounts = app.get_accounts() + if accounts: + chosen = accounts[0] # assume that we only will have a single account in cache + # Now let's try to find a token in cache for this account + result = app.acquire_token_silent(scopes=scope, account=chosen) + + if not result: # We have no token for this account, so the end user shall sign-in + + flow = app.initiate_device_flow(scope) + if "user_code" not in flow: + raise ValueError( + "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) + + print(flow["message"]) # to-do: present to user, open in browser and ask to sign in + + # Ideally you should wait here, in order to save some unnecessary polling + # input("Press Enter after signing in from another device to proceed, CTRL+C to abort.") + + result = app.acquire_token_by_device_flow(flow) # By default it will block + # You can follow this instruction to shorten the block time + # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow + # or you may even turn off the blocking behavior, + # and then keep calling acquire_token_by_device_flow(flow) in your own customized loop + + # Check if the token was obtained successfully + if "access_token" in result: + # Print the access token + print(result["access_token"]) + else: + # Print the error + print(result.get("error"), result.get("error_description")) + + +def upload_backup(config: OneDriveBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: + # upload a single file to onedrive useing credentials from OneDriveBackupCloudConfiguration + # https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online + # to-do: acquire tokens + tokens = get_tokens(config.tokencache) + log.debug("token object retrieved, access_token: %s", tokens.access_token) + onedrive = OneDrive(access_token=tokens.access_token) + log.debug("instantiated OneDrive object") + onedrive.upload_item(item_path=config.backuppath+backup_filename, file_path=backup_filename, + conflict_behavior="replace") + log.debug("uploaded file %s to OneDrive", backup_filename) + + +def create_backup_cloud(config: OneDriveBackupCloud): + def updater(backup_filename: str, backup_file: bytes): + upload_backup(config.configuration, backup_filename, backup_file) + return ConfigurableBackupCloud(config=config, component_updater=updater) + + +device_descriptor = DeviceDescriptor(configuration_factory=OneDriveBackupCloud) diff --git a/packages/modules/backup_clouds/onedrive/config.py b/packages/modules/backup_clouds/onedrive/config.py new file mode 100644 index 0000000000..9e014195c3 --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/config.py @@ -0,0 +1,17 @@ +from typing import Optional + + +class OneDriveBackupCloudConfiguration: + def __init__(self, backuppath: str = "/openWB/Backup/", persistent_tokencache: Optional[str] = None) -> None: + self.backuppath = backuppath + self.persistent_tokencache = persistent_tokencache + + +class OneDriveBackupCloud: + def __init__(self, + name: str = "OneDrive", + type: str = "OneDrive", + configuration: OneDriveBackupCloudConfiguration = None) -> None: + self.name = name + self.type = type + self.configuration = configuration or OneDriveBackupCloudConfiguration() From 5114adcd28cd26b5e08df29dce62c305f5a1a93b Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 21 Jun 2023 18:14:31 +0200 Subject: [PATCH 02/18] update requirements.txt --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 2c403e7fa0..18810cd533 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,5 @@ pkce==1.0.3 # skodaconnect==1.3.4 evdev==1.5.0 #telnetlib3==2.0.2 +msal==1.22.0 +onedrive-sharepoint-python-sdk==0.0.2 From 115517bf1e9c10927ad524cdbdd8f110d3cd1cde Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 21 Jun 2023 21:08:36 +0000 Subject: [PATCH 03/18] draft 2 --- .../backup_clouds/onedrive/backup_cloud.py | 36 +++++++++++++------ .../modules/backup_clouds/onedrive/config.py | 2 +- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index 4cfeeedebe..bfa33ae4b7 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -8,6 +8,8 @@ from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration from modules.common.abstract_device import DeviceDescriptor from modules.common.configurable_backup_cloud import ConfigurableBackupCloud +import pathlib + log = logging.getLogger(__name__) @@ -18,6 +20,7 @@ authority = "https://login.microsoftonline.com/consumers/" clientID = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2" +localbackuppath = '/var/www/html/op' # to-do move into one or more functions def get_tokens(persistent_tokencache: str) -> dict: @@ -25,24 +28,31 @@ def get_tokens(persistent_tokencache: str) -> dict: cache = msal.SerializableTokenCache() # to-do: add write to config after update - '''if os.path.exists("my_cache.bin"): # to do: read from config - cache.deserialize(open("my_cache.bin", "r").read()) + if os.path.exists("/var/www/html/openWB/packages/modules/backup_clouds/onedrive/my_cache.bin"): # to do: read from config + log.debug("reading token cache from file") + cache.deserialize(open("/var/www/html/openWB/packages/modules/backup_clouds/onedrive/my_cache.bin", "r").read()) + else: + log.debug("token cache not found") atexit.register(lambda: open("my_cache.bin", "w").write(cache.serialize()) # to-do: write to config if cache.has_state_changed else None ) -''' - if persistent_tokencache: - cache.deserialize(persistent_tokencache) + + #if persistent_tokencache: + # cache.deserialize(persistent_tokencache) # Create a public client application with msal + log.debug("creating MSAL public client application") app = msal.PublicClientApplication(client_id=clientID, authority=authority, token_cache=cache) + log.debug("getting accounts") accounts = app.get_accounts() if accounts: chosen = accounts[0] # assume that we only will have a single account in cache + log.debug("selected account " + str(chosen["username"])) # Now let's try to find a token in cache for this account result = app.acquire_token_silent(scopes=scope, account=chosen) + log.debug("done acquring tokens") if not result: # We have no token for this account, so the end user shall sign-in flow = app.initiate_device_flow(scope) @@ -50,7 +60,7 @@ def get_tokens(persistent_tokencache: str) -> dict: raise ValueError( "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) - print(flow["message"]) # to-do: present to user, open in browser and ask to sign in + log.debug(flow["message"]) # to-do: present to user, open in browser and ask to sign in # Ideally you should wait here, in order to save some unnecessary polling # input("Press Enter after signing in from another device to proceed, CTRL+C to abort.") @@ -68,17 +78,23 @@ def get_tokens(persistent_tokencache: str) -> dict: else: # Print the error print(result.get("error"), result.get("error_description")) + return result def upload_backup(config: OneDriveBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: # upload a single file to onedrive useing credentials from OneDriveBackupCloudConfiguration # https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online # to-do: acquire tokens - tokens = get_tokens(config.tokencache) - log.debug("token object retrieved, access_token: %s", tokens.access_token) - onedrive = OneDrive(access_token=tokens.access_token) + # tokens = get_tokens(config.tokencache) + tokens = get_tokens("sudo token") + log.debug("token object retrieved, access_token: %s", tokens.__len__) + onedrive = OneDrive(access_token=tokens["access_token"]) log.debug("instantiated OneDrive object") - onedrive.upload_item(item_path=config.backuppath+backup_filename, file_path=backup_filename, + + localbackup = os.path.join(pathlib.Path().resolve(), 'data', 'backup', backup_filename) + remote_filename = backup_filename.replace(':','-') # file won't upload when name contains ':' + + onedrive.upload_item(item_path=(config.backuppath+remote_filename), file_path=localbackup, conflict_behavior="replace") log.debug("uploaded file %s to OneDrive", backup_filename) diff --git a/packages/modules/backup_clouds/onedrive/config.py b/packages/modules/backup_clouds/onedrive/config.py index 9e014195c3..7fc695833c 100644 --- a/packages/modules/backup_clouds/onedrive/config.py +++ b/packages/modules/backup_clouds/onedrive/config.py @@ -10,7 +10,7 @@ def __init__(self, backuppath: str = "/openWB/Backup/", persistent_tokencache: O class OneDriveBackupCloud: def __init__(self, name: str = "OneDrive", - type: str = "OneDrive", + type: str = "onedrive", configuration: OneDriveBackupCloudConfiguration = None) -> None: self.name = name self.type = type From 085024d40e4f3c19dc013d388a4281720735fddd Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Sat, 24 Jun 2023 21:43:18 +0000 Subject: [PATCH 04/18] retrieve scope from config --- .../backup_clouds/onedrive/backup_cloud.py | 44 +++++++++---------- 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index bfa33ae4b7..de6379a84a 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -9,25 +9,17 @@ from modules.common.abstract_device import DeviceDescriptor from modules.common.configurable_backup_cloud import ConfigurableBackupCloud import pathlib +import base64 log = logging.getLogger(__name__) -# Define the scope of access -scope = ["https://graph.microsoft.com/Files.ReadWrite"] -# Define the authority and the token endpoint for MSA/Live accounts -authority = "https://login.microsoftonline.com/consumers/" -clientID = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2" - -localbackuppath = '/var/www/html/op' - -# to-do move into one or more functions -def get_tokens(persistent_tokencache: str) -> dict: +def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: result = None cache = msal.SerializableTokenCache() - # to-do: add write to config after update + ''' # to-do: add write to config after update if os.path.exists("/var/www/html/openWB/packages/modules/backup_clouds/onedrive/my_cache.bin"): # to do: read from config log.debug("reading token cache from file") cache.deserialize(open("/var/www/html/openWB/packages/modules/backup_clouds/onedrive/my_cache.bin", "r").read()) @@ -35,14 +27,14 @@ def get_tokens(persistent_tokencache: str) -> dict: log.debug("token cache not found") atexit.register(lambda: open("my_cache.bin", "w").write(cache.serialize()) # to-do: write to config if cache.has_state_changed else None - ) + )''' - #if persistent_tokencache: - # cache.deserialize(persistent_tokencache) + if config.persistent_tokencache: + cache.deserialize(base64.b64decode(config.persistent_tokencache)) # Create a public client application with msal log.debug("creating MSAL public client application") - app = msal.PublicClientApplication(client_id=clientID, authority=authority, token_cache=cache) + app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache) log.debug("getting accounts") accounts = app.get_accounts() @@ -50,12 +42,14 @@ def get_tokens(persistent_tokencache: str) -> dict: chosen = accounts[0] # assume that we only will have a single account in cache log.debug("selected account " + str(chosen["username"])) # Now let's try to find a token in cache for this account - result = app.acquire_token_silent(scopes=scope, account=chosen) + result = app.acquire_token_silent(scopes=config.scope, account=chosen) log.debug("done acquring tokens") if not result: # We have no token for this account, so the end user shall sign-in + # to-do: stop execution if no authcode is provided, log error + + flow = app.initiate_device_flow(config.scope) - flow = app.initiate_device_flow(scope) if "user_code" not in flow: raise ValueError( "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) @@ -84,19 +78,21 @@ def get_tokens(persistent_tokencache: str) -> dict: def upload_backup(config: OneDriveBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: # upload a single file to onedrive useing credentials from OneDriveBackupCloudConfiguration # https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online - # to-do: acquire tokens - # tokens = get_tokens(config.tokencache) - tokens = get_tokens("sudo token") + tokens = get_tokens(config) # type: ignore log.debug("token object retrieved, access_token: %s", tokens.__len__) + log.debug("instantiate OneDrive connection") onedrive = OneDrive(access_token=tokens["access_token"]) - log.debug("instantiated OneDrive object") localbackup = os.path.join(pathlib.Path().resolve(), 'data', 'backup', backup_filename) - remote_filename = backup_filename.replace(':','-') # file won't upload when name contains ':' - + remote_filename = backup_filename.replace(':', '-') # file won't upload when name contains ':' + + if not config.backuppath.endswith("/"): + log.debug("fixing missing ending slash in backuppath: " + config.backuppath) + config.backuppath = config.backuppath + "/" + + log.debug("uploading file %s to OneDrive", backup_filename) onedrive.upload_item(item_path=(config.backuppath+remote_filename), file_path=localbackup, conflict_behavior="replace") - log.debug("uploaded file %s to OneDrive", backup_filename) def create_backup_cloud(config: OneDriveBackupCloud): From 005199a515dd8bdd1eb72e1b80a39efbfb16a1ab Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Sat, 24 Jun 2023 21:43:49 +0000 Subject: [PATCH 05/18] prepare config --- packages/helpermodules/command.py | 46 +++++++++++++++++++ .../modules/backup_clouds/onedrive/config.py | 15 +++++- 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 72b65d6de5..63e2697155 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -10,6 +10,9 @@ import traceback from pathlib import Path import paho.mqtt.client as mqtt +import pickle +import io +from msal import PublicClientApplication from control.chargepoint import chargepoint from control.chargepoint.chargepoint_template import get_autolock_plan_default, get_chargepoint_template_default @@ -24,6 +27,7 @@ from modules.chargepoints.internal_openwb.chargepoint_module import ChargepointModule from modules.chargepoints.internal_openwb.config import InternalChargepointMode from modules.common.component_type import ComponentType, special_to_general_type_mapping, type_to_topic_mapping +from modules.backup_clouds.onedrive.config import OneDriveBackupCloudConfiguration import dataclass_utils log = logging.getLogger(__name__) @@ -687,6 +691,48 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: f'Restore-Status: {result.returncode}
Meldung: {result.stdout.decode("utf-8")}', MessageType.ERROR) + def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: + """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen + """ + + # to-do: Konfiguration aus OneDrive modul importieren? + # Define the scope of access + scope = {"https://graph.microsoft.com/Files.ReadWrite"} # Replace with your desired scope + + # Define the authority and the token endpoint for MSA/Live accounts + authority = "https://login.microsoftonline.com/consumers/" + clientID = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2" + + # Create a public client application with msal + app = PublicClientApplication(client_id=clientID, authority=authority) + + flow = app.initiate_device_flow(scope) + if "user_code" not in flow: + raise Exception( + "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) + + flow["expires_at"] = 0 # Mark it as expired immediately + pickleBuffer = io.BytesIO() + pickle.dump(flow, pickleBuffer) + + #Pub().pub(f'openWB/set/vehicle/template/ev_template/{new_id}', ev_template_default) + # Pub().pub("openWB/set/command/max_id/ev_template", new_id) + pub_user_message( + payload, connection_id, + 'Authorisierung gestartet, bitte den Link öffen, Code eingeben, ' + 'und Zugang authorisieren. Anschließend Zugangsberechtigung abrufen.', + MessageType.SUCCESS) + + def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: + """ holt die Tokens für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen + """ + # Pub().pub(f'openWB/set/vehicle/template/ev_template/{new_id}', ev_template_default) + # Pub().pub("openWB/set/command/max_id/ev_template", new_id) + pub_user_message( + payload, connection_id, + 'Zugangsberechtigung erfolgreich abgerufen.', + MessageType.SUCCESS) + class ErrorHandlingContext: def __init__(self, payload: dict, connection_id: str): diff --git a/packages/modules/backup_clouds/onedrive/config.py b/packages/modules/backup_clouds/onedrive/config.py index 7fc695833c..c2b8e049f2 100644 --- a/packages/modules/backup_clouds/onedrive/config.py +++ b/packages/modules/backup_clouds/onedrive/config.py @@ -2,9 +2,22 @@ class OneDriveBackupCloudConfiguration: - def __init__(self, backuppath: str = "/openWB/Backup/", persistent_tokencache: Optional[str] = None) -> None: + def __init__(self, backuppath: str = "/openWB/Backup/", + persistent_tokencache: Optional[str] = None, + url: Optional[str] = None, + authcode: Optional[str] = None, + scope: Optional[list] = ["https://graph.microsoft.com/Files.ReadWrite"], + authority: Optional[str] = "https://login.microsoftonline.com/consumers/", + clientID: Optional[str] = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2", + flow: Optional[str] = None) -> None: self.backuppath = backuppath self.persistent_tokencache = persistent_tokencache + self.url = url + self.authcode = authcode + self.scope = scope + self.authority = authority + self.clientID = clientID + self.flow = flow class OneDriveBackupCloud: From 616309d00088bedf00b1653c32337ae6d25b43a5 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Mon, 26 Jun 2023 19:24:08 +0000 Subject: [PATCH 06/18] save persistent token cahe --- .../backup_clouds/onedrive/backup_cloud.py | 52 +++++++------------ 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index de6379a84a..fe394d3742 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 import logging import msal -import atexit import os import json from msdrive import OneDrive +import paho.mqtt.publish as publish from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration from modules.common.abstract_device import DeviceDescriptor from modules.common.configurable_backup_cloud import ConfigurableBackupCloud @@ -15,22 +15,24 @@ log = logging.getLogger(__name__) +def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> None: + # encode cache to base64 and save to config + log.debug("saving updated tokencache to config") + cache_bytes = cache.encode("ascii") + cache_base64_bytes = base64.b64encode(cache_bytes) + cache_base64_string = cache_base64_bytes.decode("ascii") + config.persistent_tokencache = cache_base64_string + publish.single("openWB/system/backup_cloud/config", json.dumps(config), retain=True, hostname="localhost") + + def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: result = None cache = msal.SerializableTokenCache() - ''' # to-do: add write to config after update - if os.path.exists("/var/www/html/openWB/packages/modules/backup_clouds/onedrive/my_cache.bin"): # to do: read from config - log.debug("reading token cache from file") - cache.deserialize(open("/var/www/html/openWB/packages/modules/backup_clouds/onedrive/my_cache.bin", "r").read()) - else: - log.debug("token cache not found") - atexit.register(lambda: open("my_cache.bin", "w").write(cache.serialize()) # to-do: write to config - if cache.has_state_changed else None - )''' - if config.persistent_tokencache: cache.deserialize(base64.b64decode(config.persistent_tokencache)) + else: + raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.") # Create a public client application with msal log.debug("creating MSAL public client application") @@ -43,35 +45,19 @@ def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: log.debug("selected account " + str(chosen["username"])) # Now let's try to find a token in cache for this account result = app.acquire_token_silent(scopes=config.scope, account=chosen) + else: + raise Exception("No matching account found,please re-configure and re-authorize access Cloud backup settings.") log.debug("done acquring tokens") if not result: # We have no token for this account, so the end user shall sign-in - # to-do: stop execution if no authcode is provided, log error - - flow = app.initiate_device_flow(config.scope) - - if "user_code" not in flow: - raise ValueError( - "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) - - log.debug(flow["message"]) # to-do: present to user, open in browser and ask to sign in - - # Ideally you should wait here, in order to save some unnecessary polling - # input("Press Enter after signing in from another device to proceed, CTRL+C to abort.") - - result = app.acquire_token_by_device_flow(flow) # By default it will block - # You can follow this instruction to shorten the block time - # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow - # or you may even turn off the blocking behavior, - # and then keep calling acquire_token_by_device_flow(flow) in your own customized loop + raise Exception("No token found, please re-configure and re-authorize access Cloud backup settings.") - # Check if the token was obtained successfully if "access_token" in result: - # Print the access token - print(result["access_token"]) + log.debug("access token retrieved") + save_tokencache(config=config, cache=cache.serialize()) else: # Print the error - print(result.get("error"), result.get("error_description")) + raise Exception("Error retrieving access token", result.get("error"), result.get("error_description")) return result From 1994012b73d40222029badc289e77382a5ff544c Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Mon, 26 Jun 2023 20:48:25 +0000 Subject: [PATCH 07/18] save token cache --- .../backup_clouds/onedrive/backup_cloud.py | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index fe394d3742..3e8bb1039c 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -15,14 +15,25 @@ log = logging.getLogger(__name__) +def encode_str_base64(string: str) -> str: + string_bytes = string.encode("ascii") + string_base64_bytes = base64.b64encode(string_bytes) + string_base64_string = string_base64_bytes.decode("ascii") + return string_base64_string + + def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> None: # encode cache to base64 and save to config log.debug("saving updated tokencache to config") - cache_bytes = cache.encode("ascii") - cache_base64_bytes = base64.b64encode(cache_bytes) - cache_base64_string = cache_base64_bytes.decode("ascii") - config.persistent_tokencache = cache_base64_string - publish.single("openWB/system/backup_cloud/config", json.dumps(config), retain=True, hostname="localhost") + config.persistent_tokencache = encode_str_base64(cache) + + # construct full configuartion object for cloud backup + backupcloud = OneDriveBackupCloud() + backupcloud.configuration = config + backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__) + log.debug("Config to MQTT:" + str(backupcloud_to_mqtt)) + + publish.single("openWB/system/backup_cloud/config", to_mqtt, retain=True, hostname="localhost") def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: From 605e975da1278ee4054e14359df67fbcd8e5c3a1 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 28 Jun 2023 07:00:07 +0000 Subject: [PATCH 08/18] auth flow, token cache --- packages/helpermodules/command.py | 96 ++++++++++++++----- .../backup_clouds/onedrive/backup_cloud.py | 17 +++- .../modules/backup_clouds/onedrive/config.py | 4 +- 3 files changed, 91 insertions(+), 26 deletions(-) diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 63e2697155..ed79fc789d 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -10,25 +10,26 @@ import traceback from pathlib import Path import paho.mqtt.client as mqtt +import paho.mqtt.publish as publish import pickle -import io from msal import PublicClientApplication from control.chargepoint import chargepoint from control.chargepoint.chargepoint_template import get_autolock_plan_default, get_chargepoint_template_default +import msal from helpermodules import measurement_log from helpermodules.broker import InternalBrokerClient from helpermodules.messaging import MessageType, pub_user_message, pub_error_global from helpermodules.parse_send_debug import parse_send_debug_data -from helpermodules.pub import Pub +from helpermodules.pub import Pub, pub_single from helpermodules.subdata import SubData from helpermodules.utils.topic_parser import decode_payload from control import bat, bridge, chargelog, data, ev, counter, counter_all, pv from modules.chargepoints.internal_openwb.chargepoint_module import ChargepointModule from modules.chargepoints.internal_openwb.config import InternalChargepointMode from modules.common.component_type import ComponentType, special_to_general_type_mapping, type_to_topic_mapping -from modules.backup_clouds.onedrive.config import OneDriveBackupCloudConfiguration import dataclass_utils +from modules.backup_clouds.onedrive.backup_cloud import save_tokencache log = logging.getLogger(__name__) @@ -692,31 +693,39 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: MessageType.ERROR) def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: - """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen - """ - - # to-do: Konfiguration aus OneDrive modul importieren? - # Define the scope of access - scope = {"https://graph.microsoft.com/Files.ReadWrite"} # Replace with your desired scope - - # Define the authority and the token endpoint for MSA/Live accounts - authority = "https://login.microsoftonline.com/consumers/" - clientID = "e529d8d2-3b0f-4ae4-b2ba-2d9a2bba55b2" + """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) für Onedrive Backup""" + + cloudbackupconfig = SubData.system_data["system"].backup_cloud + if cloudbackupconfig is None: + pub_user_message(payload, connection_id, + "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " + "und erneut versuchen.
", MessageType.WARNING) + return + # Create a public client application with msal - app = PublicClientApplication(client_id=clientID, authority=authority) + client_id = cloudbackupconfig.config.configuration.clientID + authority = cloudbackupconfig.config.configuration.authority + app = PublicClientApplication(client_id=client_id, authority=authority) - flow = app.initiate_device_flow(scope) + # create device flow to obtain auth code + flow = app.initiate_device_flow(cloudbackupconfig.config.configuration.scope) if "user_code" not in flow: raise Exception( "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) - flow["expires_at"] = 0 # Mark it as expired immediately - pickleBuffer = io.BytesIO() - pickle.dump(flow, pickleBuffer) + flow["expires_at"] = 0 # Mark it as expired immediately to prevent + pickleString = str(pickle.dumps(flow), encoding='latin1') + + cloudbackupconfig.config.configuration.flow = str(pickleString) + cloudbackupconfig.config.configuration.authcode = flow["user_code"] + cloudbackupconfig.config.configuration.authurl = flow["verification_uri"] + cloudbackupconfig_to_mqtt = json.dumps(cloudbackupconfig.config.__dict__, default=lambda o: o.__dict__) + + publish.single( + "openWB/set/system/backup_cloud/config", cloudbackupconfig_to_mqtt, retain=True, hostname="localhost" + ) - #Pub().pub(f'openWB/set/vehicle/template/ev_template/{new_id}', ev_template_default) - # Pub().pub("openWB/set/command/max_id/ev_template", new_id) pub_user_message( payload, connection_id, 'Authorisierung gestartet, bitte den Link öffen, Code eingeben, ' @@ -726,8 +735,51 @@ def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: """ holt die Tokens für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen """ - # Pub().pub(f'openWB/set/vehicle/template/ev_template/{new_id}', ev_template_default) - # Pub().pub("openWB/set/command/max_id/ev_template", new_id) + cloudbackupconfig = SubData.system_data["system"].backup_cloud + + if cloudbackupconfig is None: + pub_user_message(payload, connection_id, + "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " + "und erneut versuchen.
", MessageType.WARNING) + return + + # Create a public client application with msal + result = None + cache = msal.SerializableTokenCache() + client_id = cloudbackupconfig.config.configuration.clientID + authority = cloudbackupconfig.config.configuration.authority + app = PublicClientApplication(client_id=client_id, authority=authority, token_cache=cache) + + t = cloudbackupconfig.config.configuration.flow + if t is None: + pub_user_message(payload, connection_id, + "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " + "und erneut versuchen.
", MessageType.WARNING) + return + flow = pickle.loads(bytes(t, encoding='latin1')) + + result = app.acquire_token_by_device_flow(flow) + # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow + # https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-device-code + # Check if the token was obtained successfully + if "access_token" in result: + log.debug("retrieved access token") + + # Tokens retrieved, remove auth codes as they are single use only. + cloudbackupconfig.config.flow = None + cloudbackupconfig.config.authcode = None + cloudbackupconfig.config.authurl = None + + # save tokens + save_tokencache(config=cloudbackupconfig.config.configuration, cache=cache.serialize()) + + else: + pub_user_message(payload, connection_id, + "Es konnten keine Tokens abgerufen werden: %s
%s" + % (result.get("error"), result.get("error_description")), MessageType.WARNING + ) + return + pub_user_message( payload, connection_id, 'Zugangsberechtigung erfolgreich abgerufen.', diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index 3e8bb1039c..72dde1540b 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -28,12 +28,24 @@ def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> Non config.persistent_tokencache = encode_str_base64(cache) # construct full configuartion object for cloud backup - backupcloud = OneDriveBackupCloud() + backupcloud = OneDriveBackupCloud() backupcloud.configuration = config backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__) log.debug("Config to MQTT:" + str(backupcloud_to_mqtt)) - publish.single("openWB/system/backup_cloud/config", to_mqtt, retain=True, hostname="localhost") + publish.single("openWB/set/system/backup_cloud/config", backupcloud_to_mqtt, retain=True, hostname="localhost") + + +'''def create_msal_app(config: OneDriveBackupCloudConfiguration) -> msal.PublicClientApplication: + cache = msal.SerializableTokenCache() + if config.persistent_tokencache: + cache.deserialize(base64.b64decode(config.persistent_tokencache)) + else: + raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.") + return msal.PublicClientApplication( + config.clientID, authority=config.authority, + token_cache=msal.SerializableTokenCache()) +''' def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: @@ -48,6 +60,7 @@ def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: # Create a public client application with msal log.debug("creating MSAL public client application") app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache) + # app = create_msal_app(config) log.debug("getting accounts") accounts = app.get_accounts() diff --git a/packages/modules/backup_clouds/onedrive/config.py b/packages/modules/backup_clouds/onedrive/config.py index c2b8e049f2..c221cafe34 100644 --- a/packages/modules/backup_clouds/onedrive/config.py +++ b/packages/modules/backup_clouds/onedrive/config.py @@ -4,7 +4,7 @@ class OneDriveBackupCloudConfiguration: def __init__(self, backuppath: str = "/openWB/Backup/", persistent_tokencache: Optional[str] = None, - url: Optional[str] = None, + authurl: Optional[str] = None, authcode: Optional[str] = None, scope: Optional[list] = ["https://graph.microsoft.com/Files.ReadWrite"], authority: Optional[str] = "https://login.microsoftonline.com/consumers/", @@ -12,7 +12,7 @@ def __init__(self, backuppath: str = "/openWB/Backup/", flow: Optional[str] = None) -> None: self.backuppath = backuppath self.persistent_tokencache = persistent_tokencache - self.url = url + self.authurl = authurl self.authcode = authcode self.scope = scope self.authority = authority From 4d000ba26962b5fa0df70125c62eefdbb6666f36 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 28 Jun 2023 09:08:43 +0000 Subject: [PATCH 09/18] minor flake8 fixes --- packages/helpermodules/command.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index ed79fc789d..9cb45bf2e0 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -21,7 +21,7 @@ from helpermodules.broker import InternalBrokerClient from helpermodules.messaging import MessageType, pub_user_message, pub_error_global from helpermodules.parse_send_debug import parse_send_debug_data -from helpermodules.pub import Pub, pub_single +from helpermodules.pub import Pub from helpermodules.subdata import SubData from helpermodules.utils.topic_parser import decode_payload from control import bat, bridge, chargelog, data, ev, counter, counter_all, pv @@ -694,7 +694,7 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) für Onedrive Backup""" - + cloudbackupconfig = SubData.system_data["system"].backup_cloud if cloudbackupconfig is None: @@ -702,7 +702,7 @@ def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " "und erneut versuchen.
", MessageType.WARNING) return - + # Create a public client application with msal client_id = cloudbackupconfig.config.configuration.clientID authority = cloudbackupconfig.config.configuration.authority From 9f8740f8fdf943d44ba73f55274ed90adc1f7a98 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 28 Jun 2023 09:13:41 +0000 Subject: [PATCH 10/18] clean up comments --- .../modules/backup_clouds/onedrive/backup_cloud.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index 72dde1540b..180c6baf33 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -36,18 +36,6 @@ def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> Non publish.single("openWB/set/system/backup_cloud/config", backupcloud_to_mqtt, retain=True, hostname="localhost") -'''def create_msal_app(config: OneDriveBackupCloudConfiguration) -> msal.PublicClientApplication: - cache = msal.SerializableTokenCache() - if config.persistent_tokencache: - cache.deserialize(base64.b64decode(config.persistent_tokencache)) - else: - raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.") - return msal.PublicClientApplication( - config.clientID, authority=config.authority, - token_cache=msal.SerializableTokenCache()) -''' - - def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: result = None cache = msal.SerializableTokenCache() @@ -60,7 +48,6 @@ def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: # Create a public client application with msal log.debug("creating MSAL public client application") app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache) - # app = create_msal_app(config) log.debug("getting accounts") accounts = app.get_accounts() From 4d70337d5e79914d2b0fdd4f38e274677077ee88 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Wed, 28 Jun 2023 14:14:14 +0000 Subject: [PATCH 11/18] flake8, refactor --- packages/helpermodules/command.py | 12 ++++++------ .../modules/backup_clouds/onedrive/backup_cloud.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 9cb45bf2e0..60d950cc79 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -704,9 +704,10 @@ def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: return # Create a public client application with msal - client_id = cloudbackupconfig.config.configuration.clientID - authority = cloudbackupconfig.config.configuration.authority - app = PublicClientApplication(client_id=client_id, authority=authority) + app = PublicClientApplication( + client_id=cloudbackupconfig.config.configuration.clientID, + authority=cloudbackupconfig.config.configuration.authority + ) # create device flow to obtain auth code flow = app.initiate_device_flow(cloudbackupconfig.config.configuration.scope) @@ -746,9 +747,8 @@ def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: # Create a public client application with msal result = None cache = msal.SerializableTokenCache() - client_id = cloudbackupconfig.config.configuration.clientID - authority = cloudbackupconfig.config.configuration.authority - app = PublicClientApplication(client_id=client_id, authority=authority, token_cache=cache) + app = PublicClientApplication(client_id=cloudbackupconfig.config.configuration.clientID, + authority=cloudbackupconfig.config.configuration.authority, token_cache=cache) t = cloudbackupconfig.config.configuration.flow if t is None: diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index 180c6baf33..87a967425d 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -28,7 +28,7 @@ def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> Non config.persistent_tokencache = encode_str_base64(cache) # construct full configuartion object for cloud backup - backupcloud = OneDriveBackupCloud() + backupcloud = OneDriveBackupCloud() backupcloud.configuration = config backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__) log.debug("Config to MQTT:" + str(backupcloud_to_mqtt)) From 19a983586b0c146bcb977fb8271b943e83ff8149 Mon Sep 17 00:00:00 2001 From: Martin Rinas Date: Wed, 28 Jun 2023 18:25:25 +0200 Subject: [PATCH 12/18] Update publish_docs_to_wiki.yml --- .github/workflows/publish_docs_to_wiki.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docs_to_wiki.yml b/.github/workflows/publish_docs_to_wiki.yml index 7f8562b086..709ac266e1 100644 --- a/.github/workflows/publish_docs_to_wiki.yml +++ b/.github/workflows/publish_docs_to_wiki.yml @@ -10,8 +10,8 @@ on: env: USER_TOKEN: ${{ secrets.WIKI_ACTION_TOKEN }} # This is the repository secret - USER_NAME: LKuemmel # Enter the username of your (bot) account - USER_EMAIL: lena.kuemmel@openwb.de # Enter the e-mail of your (bot) account + USER_NAME: MartinRinas # Enter the username of your (bot) account + USER_EMAIL: martrin@microsoft.com # Enter the e-mail of your (bot) account OWNER: ${{ github.event.repository.owner.name }} # This is the repository owner REPOSITORY_NAME: ${{ github.event.repository.name }} # This is the repository name From 26d65c4b3dba6a51d1523fd00a5418c38e5a067e Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Thu, 29 Jun 2023 10:50:03 +0000 Subject: [PATCH 13/18] move logic into api.py --- packages/helpermodules/command.py | 89 +-------- .../modules/backup_clouds/onedrive/api.py | 170 ++++++++++++++++++ .../backup_clouds/onedrive/backup_cloud.py | 65 +------ 3 files changed, 182 insertions(+), 142 deletions(-) create mode 100644 packages/modules/backup_clouds/onedrive/api.py diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 60d950cc79..9be4894c31 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -10,12 +10,10 @@ import traceback from pathlib import Path import paho.mqtt.client as mqtt -import paho.mqtt.publish as publish -import pickle -from msal import PublicClientApplication + from control.chargepoint import chargepoint from control.chargepoint.chargepoint_template import get_autolock_plan_default, get_chargepoint_template_default -import msal +from modules.backup_clouds.onedrive.api import generateMSALAuthCode, retrieveMSALTokens from helpermodules import measurement_log from helpermodules.broker import InternalBrokerClient @@ -29,7 +27,7 @@ from modules.chargepoints.internal_openwb.config import InternalChargepointMode from modules.common.component_type import ComponentType, special_to_general_type_mapping, type_to_topic_mapping import dataclass_utils -from modules.backup_clouds.onedrive.backup_cloud import save_tokencache + log = logging.getLogger(__name__) @@ -693,45 +691,14 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: MessageType.ERROR) def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: - """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) für Onedrive Backup""" - cloudbackupconfig = SubData.system_data["system"].backup_cloud - if cloudbackupconfig is None: pub_user_message(payload, connection_id, - "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " - "und erneut versuchen.
", MessageType.WARNING) + "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " + "und erneut versuchen.
", MessageType.WARNING) return - - # Create a public client application with msal - app = PublicClientApplication( - client_id=cloudbackupconfig.config.configuration.clientID, - authority=cloudbackupconfig.config.configuration.authority - ) - - # create device flow to obtain auth code - flow = app.initiate_device_flow(cloudbackupconfig.config.configuration.scope) - if "user_code" not in flow: - raise Exception( - "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) - - flow["expires_at"] = 0 # Mark it as expired immediately to prevent - pickleString = str(pickle.dumps(flow), encoding='latin1') - - cloudbackupconfig.config.configuration.flow = str(pickleString) - cloudbackupconfig.config.configuration.authcode = flow["user_code"] - cloudbackupconfig.config.configuration.authurl = flow["verification_uri"] - cloudbackupconfig_to_mqtt = json.dumps(cloudbackupconfig.config.__dict__, default=lambda o: o.__dict__) - - publish.single( - "openWB/set/system/backup_cloud/config", cloudbackupconfig_to_mqtt, retain=True, hostname="localhost" - ) - - pub_user_message( - payload, connection_id, - 'Authorisierung gestartet, bitte den Link öffen, Code eingeben, ' - 'und Zugang authorisieren. Anschließend Zugangsberechtigung abrufen.', - MessageType.SUCCESS) + result = generateMSALAuthCode(cloudbackupconfig.config) + pub_user_message(payload, connection_id, result["message"], result["MessageType"]) def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: """ holt die Tokens für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen @@ -744,46 +711,8 @@ def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: "und erneut versuchen.
", MessageType.WARNING) return - # Create a public client application with msal - result = None - cache = msal.SerializableTokenCache() - app = PublicClientApplication(client_id=cloudbackupconfig.config.configuration.clientID, - authority=cloudbackupconfig.config.configuration.authority, token_cache=cache) - - t = cloudbackupconfig.config.configuration.flow - if t is None: - pub_user_message(payload, connection_id, - "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " - "und erneut versuchen.
", MessageType.WARNING) - return - flow = pickle.loads(bytes(t, encoding='latin1')) - - result = app.acquire_token_by_device_flow(flow) - # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow - # https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-device-code - # Check if the token was obtained successfully - if "access_token" in result: - log.debug("retrieved access token") - - # Tokens retrieved, remove auth codes as they are single use only. - cloudbackupconfig.config.flow = None - cloudbackupconfig.config.authcode = None - cloudbackupconfig.config.authurl = None - - # save tokens - save_tokencache(config=cloudbackupconfig.config.configuration, cache=cache.serialize()) - - else: - pub_user_message(payload, connection_id, - "Es konnten keine Tokens abgerufen werden: %s
%s" - % (result.get("error"), result.get("error_description")), MessageType.WARNING - ) - return - - pub_user_message( - payload, connection_id, - 'Zugangsberechtigung erfolgreich abgerufen.', - MessageType.SUCCESS) + result = retrieveMSALTokens(cloudbackupconfig.config) + pub_user_message(payload, connection_id, result["message"], result["MessageType"]) class ErrorHandlingContext: diff --git a/packages/modules/backup_clouds/onedrive/api.py b/packages/modules/backup_clouds/onedrive/api.py new file mode 100644 index 0000000000..51e85dfdb1 --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/api.py @@ -0,0 +1,170 @@ +import logging +import pickle +import json +import paho.mqtt.publish as publish +import msal +import base64 + +from msal import PublicClientApplication +from helpermodules.messaging import MessageType +from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration + + +log = logging.getLogger(__name__) + + +def encode_str_base64(string: str) -> str: + string_bytes = string.encode("ascii") + string_base64_bytes = base64.b64encode(string_bytes) + string_base64_string = string_base64_bytes.decode("ascii") + return string_base64_string + + +def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> None: + # encode cache to base64 and save to config + log.debug("saving updated tokencache to config") + config.persistent_tokencache = encode_str_base64(cache) + + # construct full configuartion object for cloud backup + backupcloud = OneDriveBackupCloud() + backupcloud.configuration = config + backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__) + log.debug("Config to MQTT:" + str(backupcloud_to_mqtt)) + + publish.single("openWB/set/system/backup_cloud/config", backupcloud_to_mqtt, retain=True, hostname="localhost") + + +def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: + result = None + cache = msal.SerializableTokenCache() + + if config.persistent_tokencache: + cache.deserialize(base64.b64decode(config.persistent_tokencache)) + else: + raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.") + + # Create a public client application with msal + log.debug("creating MSAL public client application") + app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache) + + log.debug("getting accounts") + accounts = app.get_accounts() + if accounts: + chosen = accounts[0] # assume that we only will have a single account in cache + log.debug("selected account " + str(chosen["username"])) + # Now let's try to find a token in cache for this account + result = app.acquire_token_silent(scopes=config.scope, account=chosen) + else: + raise Exception("No matching account found,please re-configure and re-authorize access Cloud backup settings.") + + log.debug("done acquring tokens") + if not result: # We have no token for this account, so the end user shall sign-in + raise Exception("No token found, please re-configure and re-authorize access Cloud backup settings.") + + if "access_token" in result: + log.debug("access token retrieved") + save_tokencache(config=config, cache=cache.serialize()) + else: + # Print the error + raise Exception("Error retrieving access token", result.get("error"), result.get("error_description")) + return result + + +def generateMSALAuthCode(cloudbackup: OneDriveBackupCloud) -> dict: + """ startet den Authentifizierungsprozess für MSAL (Microsoft Authentication Library) für Onedrive Backup + und speichert den AuthCode in der Konfiguration""" + result = dict( + message="", + MessageType=MessageType.SUCCESS + ) + + if cloudbackup is None: + result["message"] = """Es ist keine Backup-Cloud konfiguriert. + Bitte Konfiguration speichern und erneut versuchen.
""" + result["MessageType"] = MessageType.WARNING + return result + + # Create a public client application with msal + app = PublicClientApplication( + client_id=cloudbackup.configuration.clientID, + authority=cloudbackup.configuration.authority + ) + + # create device flow to obtain auth code + flow = app.initiate_device_flow(cloudbackup.configuration.scope) + if "user_code" not in flow: + raise Exception( + "Fail to create device flow. Err: %s" % json.dumps(flow, indent=4)) + + flow["expires_at"] = 0 # Mark it as expired immediately to prevent + pickleString = str(pickle.dumps(flow), encoding='latin1') + + cloudbackup.configuration.flow = str(pickleString) + cloudbackup.configuration.authcode = flow["user_code"] + cloudbackup.configuration.authurl = flow["verification_uri"] + cloudbackupconfig_to_mqtt = json.dumps(cloudbackup.__dict__, default=lambda o: o.__dict__) + + publish.single( + "openWB/set/system/backup_cloud/config", cloudbackupconfig_to_mqtt, retain=True, hostname="localhost" + ) + + result["message"] = """Authorisierung gestartet, bitte den Link öffen, Code eingeben, + und Zugang authorisieren. Anschließend Zugangsberechtigung abrufen.""" + result["MessageType"] = MessageType.SUCCESS + + return result + + +def retrieveMSALTokens(cloudbackup: OneDriveBackupCloud) -> dict: + result = dict( + message="", + MessageType=MessageType.SUCCESS + ) + if cloudbackup is None: + result["message"] = """Es ist keine Backup-Cloud konfiguriert. + Bitte Konfiguration speichern und erneut versuchen.
""" + result["MessageType"] = MessageType.WARNING + return result + + # Create a public client application with msal + tokens = None + cache = msal.SerializableTokenCache() + app = PublicClientApplication(client_id=cloudbackup.configuration.clientID, + authority=cloudbackup.configuration.authority, token_cache=cache) + + f = cloudbackup.configuration.flow + if f is None: + result["message"] = """Es ist wurde kein Auth-Code erstellt. + Bitte zunächst Auth-Code erstellen und den Authorisierungsprozess beenden.
""" + result["MessageType"] = MessageType.WARNING + return result + flow = pickle.loads(bytes(f, encoding='latin1')) + + tokens = app.acquire_token_by_device_flow(flow) + # https://msal-python.readthedocs.io/en/latest/#msal.PublicClientApplication.acquire_token_by_device_flow + # https://docs.microsoft.com/en-us/azure/active-directory/develop/v2-oauth2-device-code + # Check if the token was obtained successfully + if "access_token" in tokens: + log.debug("retrieved access token") + + # Tokens retrieved, remove auth codes as they are single use only. + cloudbackup.configuration.flow = None + cloudbackup.configuration.authcode = None + cloudbackup.configuration.authurl = None + + # save tokens + save_tokencache(config=cloudbackup.configuration, cache=cache.serialize()) + result["message"] = """Zugangsberechtigung erfolgreich abgerufen.""" + result["MessageType"] = MessageType.SUCCESS + return result + + else: + result["message"] = """"Es konnten keine Tokens abgerufen werden: + %s
%s""" % (tokens.get("error"), tokens.get("error_description")) + result["MessageType"] = MessageType.WARNING + '''pub_user_message(payload, connection_id, + "Es konnten keine Tokens abgerufen werden: %s
%s" + % (result.get("error"), result.get("error_description")), MessageType.WARNING + ) + ''' + return result diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index 87a967425d..1cb0c6e909 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -1,77 +1,18 @@ #!/usr/bin/env python3 import logging -import msal import os -import json from msdrive import OneDrive -import paho.mqtt.publish as publish +import pathlib + +from modules.backup_clouds.onedrive.api import get_tokens from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration from modules.common.abstract_device import DeviceDescriptor from modules.common.configurable_backup_cloud import ConfigurableBackupCloud -import pathlib -import base64 log = logging.getLogger(__name__) -def encode_str_base64(string: str) -> str: - string_bytes = string.encode("ascii") - string_base64_bytes = base64.b64encode(string_bytes) - string_base64_string = string_base64_bytes.decode("ascii") - return string_base64_string - - -def save_tokencache(config: OneDriveBackupCloudConfiguration, cache: str) -> None: - # encode cache to base64 and save to config - log.debug("saving updated tokencache to config") - config.persistent_tokencache = encode_str_base64(cache) - - # construct full configuartion object for cloud backup - backupcloud = OneDriveBackupCloud() - backupcloud.configuration = config - backupcloud_to_mqtt = json.dumps(backupcloud.__dict__, default=lambda o: o.__dict__) - log.debug("Config to MQTT:" + str(backupcloud_to_mqtt)) - - publish.single("openWB/set/system/backup_cloud/config", backupcloud_to_mqtt, retain=True, hostname="localhost") - - -def get_tokens(config: OneDriveBackupCloudConfiguration) -> dict: - result = None - cache = msal.SerializableTokenCache() - - if config.persistent_tokencache: - cache.deserialize(base64.b64decode(config.persistent_tokencache)) - else: - raise Exception("No tokencache found, please re-configure and re-authorize access Cloud backup settings.") - - # Create a public client application with msal - log.debug("creating MSAL public client application") - app = msal.PublicClientApplication(client_id=config.clientID, authority=config.authority, token_cache=cache) - - log.debug("getting accounts") - accounts = app.get_accounts() - if accounts: - chosen = accounts[0] # assume that we only will have a single account in cache - log.debug("selected account " + str(chosen["username"])) - # Now let's try to find a token in cache for this account - result = app.acquire_token_silent(scopes=config.scope, account=chosen) - else: - raise Exception("No matching account found,please re-configure and re-authorize access Cloud backup settings.") - - log.debug("done acquring tokens") - if not result: # We have no token for this account, so the end user shall sign-in - raise Exception("No token found, please re-configure and re-authorize access Cloud backup settings.") - - if "access_token" in result: - log.debug("access token retrieved") - save_tokencache(config=config, cache=cache.serialize()) - else: - # Print the error - raise Exception("Error retrieving access token", result.get("error"), result.get("error_description")) - return result - - def upload_backup(config: OneDriveBackupCloudConfiguration, backup_filename: str, backup_file: bytes) -> None: # upload a single file to onedrive useing credentials from OneDriveBackupCloudConfiguration # https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_put_content?view=odsp-graph-online From ca207780758bc401f225eb14c40acd27a6d07928 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Thu, 29 Jun 2023 13:54:43 +0000 Subject: [PATCH 14/18] formatting --- packages/helpermodules/command.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index 9be4894c31..f351b5f5f5 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -691,11 +691,12 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: MessageType.ERROR) def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: + ''' fordert einen Authentifizierungscode für MSAL (Microsoft Authentication Library) an um Onedrive Backup zu ermöglichen''' cloudbackupconfig = SubData.system_data["system"].backup_cloud if cloudbackupconfig is None: pub_user_message(payload, connection_id, - "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " - "und erneut versuchen.
", MessageType.WARNING) + "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " + "und erneut versuchen.
", MessageType.WARNING) return result = generateMSALAuthCode(cloudbackupconfig.config) pub_user_message(payload, connection_id, result["message"], result["MessageType"]) @@ -704,13 +705,11 @@ def retrieveMSALTokens(self, connection_id: str, payload: dict) -> None: """ holt die Tokens für MSAL (Microsoft Authentication Library) um Onedrive Backup zu ermöglichen """ cloudbackupconfig = SubData.system_data["system"].backup_cloud - if cloudbackupconfig is None: pub_user_message(payload, connection_id, "Es ist keine Backup-Cloud konfiguriert. Bitte Konfiguration speichern " "und erneut versuchen.
", MessageType.WARNING) return - result = retrieveMSALTokens(cloudbackupconfig.config) pub_user_message(payload, connection_id, result["message"], result["MessageType"]) From 23a2d1fe8b4268543ab221a96db9044931e3832c Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Thu, 29 Jun 2023 14:00:08 +0000 Subject: [PATCH 15/18] Revert "Update publish_docs_to_wiki.yml" This reverts commit 19a983586b0c146bcb977fb8271b943e83ff8149. --- .github/workflows/publish_docs_to_wiki.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish_docs_to_wiki.yml b/.github/workflows/publish_docs_to_wiki.yml index 709ac266e1..7f8562b086 100644 --- a/.github/workflows/publish_docs_to_wiki.yml +++ b/.github/workflows/publish_docs_to_wiki.yml @@ -10,8 +10,8 @@ on: env: USER_TOKEN: ${{ secrets.WIKI_ACTION_TOKEN }} # This is the repository secret - USER_NAME: MartinRinas # Enter the username of your (bot) account - USER_EMAIL: martrin@microsoft.com # Enter the e-mail of your (bot) account + USER_NAME: LKuemmel # Enter the username of your (bot) account + USER_EMAIL: lena.kuemmel@openwb.de # Enter the e-mail of your (bot) account OWNER: ${{ github.event.repository.owner.name }} # This is the repository owner REPOSITORY_NAME: ${{ github.event.repository.name }} # This is the repository name From d93a824539bc124e5ba448e30e84e2c2662f45d3 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Thu, 29 Jun 2023 14:04:45 +0000 Subject: [PATCH 16/18] flake8 --- packages/helpermodules/command.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/helpermodules/command.py b/packages/helpermodules/command.py index f31240d49b..b45946076b 100644 --- a/packages/helpermodules/command.py +++ b/packages/helpermodules/command.py @@ -693,7 +693,8 @@ def restoreBackup(self, connection_id: str, payload: dict) -> None: MessageType.ERROR) def requestMSALAuthCode(self, connection_id: str, payload: dict) -> None: - ''' fordert einen Authentifizierungscode für MSAL (Microsoft Authentication Library) an um Onedrive Backup zu ermöglichen''' + ''' fordert einen Authentifizierungscode für MSAL (Microsoft Authentication Library) + an um Onedrive Backup zu ermöglichen''' cloudbackupconfig = SubData.system_data["system"].backup_cloud if cloudbackupconfig is None: pub_user_message(payload, connection_id, From f225b953b8cbdb49d2d2bf5b9c25f249b7259d83 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Thu, 29 Jun 2023 15:45:44 +0000 Subject: [PATCH 17/18] add required module to workflow --- .github/workflows/github-actions-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/github-actions-python.yml b/.github/workflows/github-actions-python.yml index 0c2905ba13..c58de68f25 100644 --- a/.github/workflows/github-actions-python.yml +++ b/.github/workflows/github-actions-python.yml @@ -14,7 +14,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce + pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce msal onedrive-sharepoint-python-sdk - name: Flake8 with annotations in packages folder uses: TrueBrain/actions-flake8@v2.1 with: From 4bcbfc126daaf389d2af6498426fe83f69bc3ba0 Mon Sep 17 00:00:00 2001 From: MartinRinas Date: Tue, 1 Aug 2023 07:14:40 +0000 Subject: [PATCH 18/18] remove sharepoint-onedrive SDK --- .github/workflows/github-actions-python.yml | 2 +- .../backup_clouds/onedrive/backup_cloud.py | 2 +- .../onedrive/msdrive/constants.py | 3 + .../backup_clouds/onedrive/msdrive/drive.py | 208 ++++++++++++++++++ .../onedrive/msdrive/exceptions.py | 14 ++ .../onedrive/msdrive/onedrive.py | 30 +++ requirements.txt | 1 - 7 files changed, 257 insertions(+), 3 deletions(-) create mode 100644 packages/modules/backup_clouds/onedrive/msdrive/constants.py create mode 100644 packages/modules/backup_clouds/onedrive/msdrive/drive.py create mode 100644 packages/modules/backup_clouds/onedrive/msdrive/exceptions.py create mode 100644 packages/modules/backup_clouds/onedrive/msdrive/onedrive.py diff --git a/.github/workflows/github-actions-python.yml b/.github/workflows/github-actions-python.yml index bfbba6ef7a..7dc26fc5ee 100644 --- a/.github/workflows/github-actions-python.yml +++ b/.github/workflows/github-actions-python.yml @@ -14,7 +14,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce typing_extensions python-dateutil==2.8.2 msal onedrive-sharepoint-python-sdk + pip install flake8 pytest paho-mqtt requests-mock jq pyjwt==2.6.0 bs4 pkce typing_extensions python-dateutil==2.8.2 msal - name: Flake8 with annotations in packages folder uses: TrueBrain/actions-flake8@v2.1 with: diff --git a/packages/modules/backup_clouds/onedrive/backup_cloud.py b/packages/modules/backup_clouds/onedrive/backup_cloud.py index 1cb0c6e909..8f2a4d5f96 100644 --- a/packages/modules/backup_clouds/onedrive/backup_cloud.py +++ b/packages/modules/backup_clouds/onedrive/backup_cloud.py @@ -1,9 +1,9 @@ #!/usr/bin/env python3 import logging import os -from msdrive import OneDrive import pathlib +from modules.backup_clouds.onedrive.msdrive.onedrive import OneDrive from modules.backup_clouds.onedrive.api import get_tokens from modules.backup_clouds.onedrive.config import OneDriveBackupCloud, OneDriveBackupCloudConfiguration from modules.common.abstract_device import DeviceDescriptor diff --git a/packages/modules/backup_clouds/onedrive/msdrive/constants.py b/packages/modules/backup_clouds/onedrive/msdrive/constants.py new file mode 100644 index 0000000000..16bace4960 --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/msdrive/constants.py @@ -0,0 +1,3 @@ +BASE_GRAPH_URL = "https://graph.microsoft.com/v1.0" +SIMPLE_UPLOAD_MAX_SIZE = 4000000 # 4MB +CHUNK_UPLOAD_MAX_SIZE = 3276800 # ~3MB must be divisible by 327680 bytes diff --git a/packages/modules/backup_clouds/onedrive/msdrive/drive.py b/packages/modules/backup_clouds/onedrive/msdrive/drive.py new file mode 100644 index 0000000000..e2b8addf22 --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/msdrive/drive.py @@ -0,0 +1,208 @@ +import os +from .exceptions import InvalidAccessToken, ItemNotFound, RateLimited, DriveException +from requests import Session +from abc import ABC, abstractmethod +from urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter +from requests.exceptions import HTTPError +from .constants import SIMPLE_UPLOAD_MAX_SIZE, CHUNK_UPLOAD_MAX_SIZE + + +class MSDrive(ABC): + """Abstract class for accessing files stored in OneDrive and SharePoint using the Microsoft Graph API.""" + + def __init__(self, access_token: str) -> None: + """Class constructor that accepts a Microsoft access token for use with the API + + Args: + access_token (str): The access token + """ + self.access_token = access_token + + def get_item_data(self, **kwargs) -> dict: + """Get metadata for a DriveItem. + + Args: + drive_id (str): The drive ID (only for SharePoint) + item_id (str): [EITHER] The item ID + item_path (str): [EITHER] The item path + + Returns: + dict: JSON representation of a DriveItem resource + """ + r = self._session().get(self._get_drive_item_url(**kwargs)) + + return r.json() + + def list_items(self, **kwargs) -> dict: + """List the DriveItems in a specific folder path. + + Args: + drive_id (str): The drive ID (only for SharePoint) + folder_path (str): The folder path (or leave out for root) + + Returns: + dict: JSON representation of a collection of DriveItem resources + """ + r = self._session().get(self._get_drive_children_url(**kwargs)) + + return r.json() + + def download_item(self, **kwargs) -> None: + """Download a DriveItem file to a specific local path. + + Args: + drive_id (str): The drive ID (only for SharePoint) + item_id (str): [EITHER] The item ID + item_path (str): [EITHER] The item path + file_path (str): Local path to save the file to (e.g. /tmp/blah.csv) + """ + if not kwargs.get("file_path"): + raise ValueError("Missing file_path argument") + + data = self.get_item_data(**kwargs) + + with Session().get(data["@microsoft.graph.downloadUrl"], stream=True) as r: + r.raise_for_status() + + with open(kwargs["file_path"], "wb") as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + + def upload_item(self, **kwargs) -> None: + """Upload a local file to an existing or new DriveItem. + + Specify the item_path for a new file. + Specify the item_path or item_id for an existing file. + + Args: + drive_id (str): The drive ID (only for SharePoint) + item_id (str): [EITHER] The item ID + item_path (str): [EITHER] The item path + file_path (str): Local path to upload the file from (e.g. /tmp/blah.csv) + """ + if not kwargs.get("file_path"): + raise ValueError("Missing file_path argument") + + file_size = os.stat(kwargs["file_path"]).st_size + + if file_size <= SIMPLE_UPLOAD_MAX_SIZE: + self._upload_item_small(**kwargs) + else: + self._upload_item_large(**kwargs) + + @abstractmethod + def _get_drive_item_url(self, **kwargs) -> str: + raise NotImplementedError("Must be overridden") + + @abstractmethod + def _get_drive_children_url(self, **kwargs) -> str: + raise NotImplementedError("Must be overridden") + + def _session(self) -> Session: + s = Session() + s.hooks["response"] = [self.raise_error_hook] + s.headers.update({"Authorization": "Bearer " + self.access_token}) + + return s + + def _session_upload(self) -> Session: + retries = Retry( + total=3, backoff_factor=1, status_forcelist=[500, 502, 503, 504] + ) + + adapter = HTTPAdapter(max_retries=retries) + + s = Session() + s.mount("http://", adapter) + s.mount("https://", adapter) + s.hooks["response"] = [self.raise_error_hook] + + return s + + def _upload_item_small(self, **kwargs) -> None: + url = self._get_drive_item_url(**kwargs) + file_data = open(kwargs["file_path"], "rb") + + if kwargs.get("item_id"): + url += "/content" + else: + url += ":/content" + + try: + self._session().put(url, data=file_data) + finally: + file_data.close() + + def _upload_item_large(self, **kwargs) -> None: + upload_url = self._get_upload_url(**kwargs) + file_size = os.stat(kwargs["file_path"]).st_size + + with open(kwargs["file_path"], "rb") as f: + chunk_size = CHUNK_UPLOAD_MAX_SIZE + chunk_number = file_size // chunk_size + chunk_leftover = file_size - chunk_size * chunk_number + chunk_data = f.read(chunk_size) + i = 0 + + while chunk_data: + start_index = i * chunk_size + end_index = start_index + chunk_size + + if i == chunk_number: + end_index = start_index + chunk_leftover + + s = self._session_upload() + + # Setting the header with the appropriate chunk data location in the file + headers = { + "Content-Length": str(chunk_size), + "Content-Range": "bytes {}-{}/{}".format( + start_index, end_index - 1, file_size + ), + } + + s.headers.update(headers) + s.put(upload_url, data=chunk_data) + + i = i + 1 + chunk_data = f.read(chunk_size) + + def _get_upload_url(self, **kwargs) -> str: + url = self._get_drive_item_url(**kwargs) + + if kwargs.get("item_id"): + url += "/createUploadSession" + else: + url += ":/createUploadSession" + + r = self._session().post(url) + + return r.json()["uploadUrl"] + + def raise_error_hook(self, resp, *args, **kwargs) -> None: + try: + resp.raise_for_status() + except HTTPError as err: + self._handle_http_error(err) + + def _handle_http_error(self, err: HTTPError) -> None: + if err.response is None: + raise err + + try: + body = err.response.json() + message = body["error"]["message"] + except Exception: + raise err + + if err.response.status_code == 401: + raise InvalidAccessToken(message) + + if err.response.status_code == 404: + raise ItemNotFound(message) + + if err.response.status_code == 429: + raise RateLimited(message) + + raise DriveException(message) diff --git a/packages/modules/backup_clouds/onedrive/msdrive/exceptions.py b/packages/modules/backup_clouds/onedrive/msdrive/exceptions.py new file mode 100644 index 0000000000..8537d69d3d --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/msdrive/exceptions.py @@ -0,0 +1,14 @@ +class DriveException(Exception): + """There was an ambiguous exception that occurred""" + + +class InvalidAccessToken(DriveException): + """Invalid access token""" + + +class ItemNotFound(DriveException): + """Item not found""" + + +class RateLimited(DriveException): + """Rate limit exceeded""" diff --git a/packages/modules/backup_clouds/onedrive/msdrive/onedrive.py b/packages/modules/backup_clouds/onedrive/msdrive/onedrive.py new file mode 100644 index 0000000000..ff75a980c5 --- /dev/null +++ b/packages/modules/backup_clouds/onedrive/msdrive/onedrive.py @@ -0,0 +1,30 @@ +from .drive import MSDrive +from urllib.parse import quote +from .constants import BASE_GRAPH_URL + + +class OneDrive(MSDrive): + """Class for accessing DriveItems stored in OneDrive. + + A DriveItem resource represents a file, folder, or other item stored in a drive. + + All file system objects in OneDrive are returned as DriveItem resources (see https://bit.ly/3HAAxrh). + + """ + + def _get_drive_item_url(self, **kwargs) -> str: + if kwargs.get("item_id"): + return f"{BASE_GRAPH_URL}/me/drive/items/{kwargs['item_id']}" + + if kwargs.get("item_path"): + path = quote(kwargs["item_path"].lstrip("/")) + return f"{BASE_GRAPH_URL}/me/drive/root:/{path}" + + raise ValueError("Missing argument: item_id or item_path") + + def _get_drive_children_url(self, **kwargs) -> str: + if not kwargs.get("folder_path"): + return f"{BASE_GRAPH_URL}/me/drive/root/children" + else: + path = quote(kwargs["folder_path"].lstrip("/").rstrip("/")) + return f"{BASE_GRAPH_URL}/me/drive/root:/{path}:/children" diff --git a/requirements.txt b/requirements.txt index 90370b8cd7..ccc649f713 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,5 +14,4 @@ pkce==1.0.3 evdev==1.5.0 #telnetlib3==2.0.2 msal==1.22.0 -onedrive-sharepoint-python-sdk==0.0.2 python-dateutil==2.8.2